[Fix] hicache: Fail fast on a null pointer from the Mooncake host allocator (#40565)
Co-authored-by: huangtingwei <141888744+huangtingwei9988@users.noreply.github.com> Co-authored-by: Teng Ma <sima.mt@alibaba-inc.com>
This commit is contained in:
co-authored by
huangtingwei
Teng Ma
parent
8a6770d566
commit
81841d6bdc
@@ -54,6 +54,13 @@ class MooncakeHostTensorAllocator(HostTensorAllocator):
|
||||
size *= d
|
||||
size *= torch.tensor([], dtype=self.dtype).element_size()
|
||||
ptr_int = self.allocator.alloc(size)
|
||||
if not ptr_int:
|
||||
raise RuntimeError(
|
||||
f"MooncakeHostMemAllocator.alloc returned a null pointer for "
|
||||
f"{size / 1024**3:.3f} GB ({size} bytes); the mooncake host memory "
|
||||
f"allocator could not map the segment (check host memory "
|
||||
f"availability and the mooncake allocator logs)."
|
||||
)
|
||||
self.ptr = ptr_int
|
||||
c_type = ctypes.c_byte * size
|
||||
c_array = c_type.from_address(ptr_int)
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Unit tests for MooncakeHostTensorAllocator allocation failures."""
|
||||
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.storage.mooncake_store.mooncake_store import (
|
||||
MooncakeHostTensorAllocator,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestMooncakeHostTensorAllocator(CustomTestCase):
|
||||
def _make_allocator(self, alloc_return):
|
||||
# Bypass __init__: it constructs the real MooncakeHostMemAllocator,
|
||||
# which requires the mooncake package and a mappable host segment.
|
||||
allocator = MooncakeHostTensorAllocator.__new__(MooncakeHostTensorAllocator)
|
||||
allocator.allocator = mock.Mock(alloc=mock.Mock(return_value=alloc_return))
|
||||
allocator.ptr = None
|
||||
return allocator
|
||||
|
||||
def test_null_pointer_from_allocator_raises(self):
|
||||
for alloc_return in (0, None):
|
||||
with self.subTest(alloc_return=alloc_return):
|
||||
allocator = self._make_allocator(alloc_return)
|
||||
|
||||
with self.assertRaises(RuntimeError) as ctx:
|
||||
allocator.allocate((1024, 1024), dtype=torch.uint8)
|
||||
|
||||
message = str(ctx.exception)
|
||||
self.assertIn("1048576 bytes", message)
|
||||
self.assertIn("0.001 GB", message)
|
||||
self.assertIn("mooncake host memory", message)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user