[HiCache] Stop populating host-pool mmaps twice (-13% allocation time) (#36705)

Co-authored-by: DellCurry <30748980+DellCurry@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>

test/registered/unit/mem_cache/test_mmap_allocator.py

@@ -5,14 +5,18 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu")
 import sys
 
 sys.modules["libtpu"] = None
+import ctypes
+import ctypes.util
 import mmap
 import os
 import unittest
+import unittest.mock
 
 import torch
 
 from sglang.srt.mem_cache.pool_host.common import ShmHostTensorAllocator
 from sglang.srt.mem_cache.storage.mmap import alloc_mmap, alloc_shm
+from sglang.srt.mem_cache.storage.mmap.mmap_allocator import _mmap_prefaulted
 
 
 class TestMmapAllocator(unittest.TestCase):
@@ -50,6 +54,45 @@ class TestMmapAllocator(unittest.TestCase):
         mm.close()
         os.close(fd)
 
+    def _assert_resident(self, mm, alloc_bytes):
+        # mincore() reports per-page residency, so the invariant is checked
+        # rather than inferred from the flags that were passed.
+        addr = ctypes.addressof(ctypes.c_char.from_buffer(mm))
+        npages = alloc_bytes // mmap.PAGESIZE
+        vec = (ctypes.c_ubyte * npages)()
+        libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True)
+        if libc.mincore(ctypes.c_void_p(addr), ctypes.c_size_t(alloc_bytes), vec) != 0:
+            self.skipTest("mincore unavailable")
+        self.assertTrue(all(v & 1 for v in vec), "mapping was not fully pre-faulted")
+
+    def test_mmap_prefaulted_leaves_no_lazy_page(self):
+        """Both populate paths must return a mapping with every page resident.
+
+        cudaHostRegister pins these buffers, so a page still unfaulted at
+        registration lets the device read memory that is not backed yet.
+        """
+        alloc_bytes = 64 * mmap.PAGESIZE
+        flags = mmap.MAP_SHARED | mmap.MAP_ANONYMOUS
+
+        with self.subTest(path="madvise"):
+            mm = _mmap_prefaulted(-1, alloc_bytes, flags)
+            try:
+                self._assert_resident(mm, alloc_bytes)
+            finally:
+                mm.close()
+
+        # MAP_POPULATE is unreachable on a 5.14+ kernel, so CI never runs it;
+        # force the branch or it ships untested.
+        with self.subTest(path="map_populate"), unittest.mock.patch(
+            "sglang.srt.mem_cache.storage.mmap.mmap_allocator._has_madv_populate_write",
+            return_value=False,
+        ):
+            mm = _mmap_prefaulted(-1, alloc_bytes, flags)
+            try:
+                self._assert_resident(mm, alloc_bytes)
+            finally:
+                mm.close()
+
     def test_shm_host_tensor_allocator(self):
         allocator = ShmHostTensorAllocator()
         dims = (2, 512)