[ROCm] Eliminate HiCache migration stalls and use exact-size host allocations (#42553)

Signed-off-by: JohnQinAMD <yanyuan.qin@amd.com>
Co-authored-by: JohnQinAMD <yanyuan.qin@amd.com>
This commit is contained in:
Zhang, Jiejing
2026-10-12 14:05:51 -07:00
committed by GitHub
co-authored by JohnQinAMD
parent b60444c621
commit 6966c5d027
3 changed files with 514 additions and 1 deletions
@@ -0,0 +1,278 @@
"""Reproduce ROCm HiCache USERPTR eviction stalls on an otherwise idle NUMA host.
Run each allocator in a separate process on the same GPU and node:
HSA_USERPTR_FOR_PAGED_MEM=0 python benchmark/hicache/bench_host_memory_migration.py --allocator registered --pool-gb 180
HSA_USERPTR_FOR_PAGED_MEM=0 python benchmark/hicache/bench_host_memory_migration.py --allocator gtt --pool-gb 180
The registered arm uses the pre-fix mmap + host-register path; the gtt arm
uses alloc_with_host_register from this checkout. Both keep the HIP environment
identical. This is a manual hardware test, not a CI test or an LLM benchmark.
It needs two allowed memory NUMA nodes, sufficient host/GTT memory, and permission
to call move_pages. A container's seccomp policy must permit NUMA migration;
pass its worker's host PID with --kfd-pid if PID namespaces differ.
Only this process's own allocation is migrated; no system policy is changed.
"""
from __future__ import annotations
import argparse
import ctypes
import errno
import json
import os
import statistics
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import torch
from sglang.srt.mem_cache.pool_host.common import (
HostTensorAllocator,
_cuda_host_register,
_cuda_host_unregister,
alloc_with_host_register,
)
COPY_BYTES = 16 * 1024**2
MIGRATE_BYTES = 2 * 1024**2
def memory_nodes():
allowed = next(
line.split()[1]
for line in Path("/proc/self/status").read_text().splitlines()
if line.startswith("Mems_allowed_list:")
)
nodes = []
for entry in allowed.split(","):
bounds = [int(value) for value in entry.split("-")]
for node in range(bounds[0], bounds[-1] + 1):
info = Path(f"/sys/devices/system/node/node{node}/meminfo").read_text()
if any(
"MemTotal:" in line and int(line.split()[-2]) > 0
for line in info.splitlines()
):
nodes.append(node)
if len(nodes) < 2:
raise RuntimeError("Two allowed NUMA nodes with memory are required")
return nodes
def evicted_ms(pid):
counters = {}
for path in Path(f"/sys/class/kfd/kfd/proc/{pid}").glob("stats_*"):
counters[path.name] = int((path / "evicted_ms").read_text())
if not counters:
raise RuntimeError(
"KFD evicted_ms counters unavailable; --kfd-pid must identify this worker on the host"
)
return counters
class PageMover:
def __init__(self, ptr):
page_size = os.sysconf("SC_PAGE_SIZE")
if ptr % page_size or MIGRATE_BYTES % page_size:
raise RuntimeError("Migration range must be page aligned")
self.count = MIGRATE_BYTES // page_size
self.pages = (ctypes.c_void_p * self.count)(
*(ptr + offset * page_size for offset in range(self.count))
)
self.lib = ctypes.CDLL("libnuma.so.1", use_errno=True)
self.lib.move_pages.argtypes = [
ctypes.c_int,
ctypes.c_ulong,
ctypes.POINTER(ctypes.c_void_p),
ctypes.POINTER(ctypes.c_int),
ctypes.POINTER(ctypes.c_int),
ctypes.c_int,
]
self.lib.move_pages.restype = ctypes.c_long
def call(self, target=None):
status = (ctypes.c_int * self.count)(*([-errno.EIO] * self.count))
nodes = (
None
if target is None
else (ctypes.c_int * self.count)(*([target] * self.count))
)
ctypes.set_errno(0)
# pid=0: own process. MPOL_MF_MOVE=2: move only exclusively owned pages.
rc = self.lib.move_pages(
0, self.count, self.pages, nodes, status, 0 if target is None else 2
)
return {
"rc": rc,
"errno": ctypes.get_errno() if rc < 0 else 0,
"status": list(status),
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--allocator", choices=("registered", "gtt"), required=True)
parser.add_argument("--pool-gb", type=float, default=180)
parser.add_argument("--trials", type=int, default=3)
parser.add_argument("--iterations", type=int, default=1000)
parser.add_argument(
"--kfd-pid",
type=int,
default=os.getpid(),
help="This worker's host PID (for containerized runs)",
)
args = parser.parse_args()
size = int(args.pool_gb * 1e9)
if size < COPY_BYTES or args.trials < 1 or args.iterations < 1:
parser.error("Pool must fit 16 MiB; trials and iterations must be positive")
if os.environ.get("HSA_USERPTR_FOR_PAGED_MEM") != "0":
parser.error("Set HSA_USERPTR_FOR_PAGED_MEM=0 before starting either arm")
if not torch.version.hip or not torch.cuda.is_available():
parser.error("A ROCm GPU is required")
nodes = memory_nodes()
torch.cuda.set_device(0)
evicted_ms(args.kfd_pid)
torch.set_num_threads(1)
allocator = HostTensorAllocator()
if args.allocator == "registered":
host = allocator.allocate((size,), torch.uint8, "cpu")
_cuda_host_register(host)
else:
host = alloc_with_host_register((size,), torch.uint8, "cpu", True, allocator)
host[:COPY_BYTES].fill_(42)
mover = PageMover(host.data_ptr())
source = host[:COPY_BYTES]
dest = torch.empty(COPY_BYTES, dtype=torch.uint8, device="cuda")
a = torch.randn((4096, 4096), dtype=torch.bfloat16, device="cuda")
b = torch.randn_like(a)
c = torch.empty_like(a)
def iteration(trigger=None):
start = time.perf_counter()
for _ in range(4):
torch.mm(a, b, out=c)
if trigger is not None:
trigger.set()
dest.copy_(source, non_blocking=True)
torch.cuda.synchronize()
return (time.perf_counter() - start) * 1000
try:
for _ in range(50):
iteration()
control = [iteration() for _ in range(args.iterations)]
median = statistics.median(control)
print(
json.dumps(
{
"allocator": args.allocator,
"pool_bytes": size,
"gpu": torch.cuda.get_device_name(0),
"torch": torch.__version__,
"kfd_pid": args.kfd_pid,
"numa_nodes": nodes,
"control_median_ms": median,
"control_max_ms": max(control),
}
),
flush=True,
)
results = []
with ThreadPoolExecutor(max_workers=1) as executor:
for trial in range(args.trials):
current = mover.call()
if args.allocator == "registered":
if current["rc"] != 0 or any(
node < 0 for node in current["status"]
):
raise RuntimeError(f"Cannot query USERPTR pages: {current}")
target = next(
node for node in nodes if node != current["status"][0]
)
else:
target = nodes[trial % len(nodes)]
before = evicted_ms(args.kfd_pid)
trigger = threading.Event()
def migrate():
if not trigger.wait(30):
raise RuntimeError("GPU workload did not start")
return mover.call(target)
future = executor.submit(migrate)
samples = [iteration(trigger)]
after_migration = 0
while after_migration < args.iterations:
samples.append(iteration())
if future.done():
after_migration += 1
migration = future.result()
after = evicted_ms(args.kfd_pid)
delta = {key: after[key] - before[key] for key in before}
moved = sum(
old >= 0 and old != target and new == target
for old, new in zip(current["status"], migration["status"])
)
rejected = (
migration["rc"] == -1 and migration["errno"] == errno.EFAULT
) or (
migration["rc"] >= 0
and all(value == -errno.EFAULT for value in migration["status"])
)
result = {
"trial": trial + 1,
"target_node": target,
"move_rc": migration["rc"],
"move_errno": migration["errno"],
"page_status_counts": {
str(value): migration["status"].count(value)
for value in set(migration["status"])
},
"moved_pages": moved,
"gtt_migration_rejected": rejected,
"max_iteration_ms": max(samples),
"excess_over_control_ms": sum(
max(0, value - median) for value in samples
),
"evicted_ms_delta": delta,
}
print(json.dumps(result), flush=True)
if args.allocator == "registered" and moved == 0:
raise RuntimeError(
"No pages migrated; USERPTR reproduction is inconclusive"
)
if args.allocator == "gtt" and (
not rejected or max(delta.values()) != 0
):
raise RuntimeError(
"GTT migration rejection / zero-eviction check failed"
)
if not torch.all(dest == 42).item():
raise RuntimeError("Host-to-device copy returned incorrect data")
results.append(result)
reproduced = any(
max(r["evicted_ms_delta"].values()) > 0 and r["max_iteration_ms"] > 100
for r in results
)
print(
json.dumps(
{
"allocator": args.allocator,
"trials": len(results),
"userptr_stall_reproduced": reproduced,
}
),
flush=True,
)
if args.allocator == "registered" and not reproduced:
raise RuntimeError(
"Pages migrated, but no >100 ms GPU eviction stall was reproduced"
)
finally:
torch.cuda.synchronize()
_cuda_host_unregister(host)
if __name__ == "__main__":
main()
+100 -1
View File
@@ -1,8 +1,11 @@
from __future__ import annotations
import ctypes
import json
import logging
import math
import os
import weakref
from collections import defaultdict
from functools import lru_cache
@@ -233,6 +236,97 @@ def _cuda_host_unregister(buffer: torch.Tensor) -> None:
setattr(buffer, _CUDA_HOST_REGISTERED_RANGES_ATTR, remaining_ranges)
@lru_cache(maxsize=1)
def _hip_runtime() -> ctypes.CDLL:
from sglang.srt.distributed.device_communicators.cuda_wrapper import (
find_loaded_library,
)
# Bind the copy torch loaded; dlopen by soname could load a second HIP runtime.
path = find_loaded_library("libamdhip64")
if path is None:
raise RuntimeError("libamdhip64 is not loaded in this process")
lib = ctypes.CDLL(path)
lib.hipHostMalloc.argtypes = [
ctypes.POINTER(ctypes.c_void_p),
ctypes.c_size_t,
ctypes.c_uint,
]
lib.hipHostMalloc.restype = ctypes.c_int
lib.hipHostFree.argtypes = [ctypes.c_void_p]
lib.hipHostFree.restype = ctypes.c_int
lib.hipGetErrorString.argtypes = [ctypes.c_int]
lib.hipGetErrorString.restype = ctypes.c_char_p
return lib
def _hip_host_malloc(nbytes: int) -> int:
lib = _hip_runtime()
ptr = ctypes.c_void_p()
# hipHostMallocDefault, the flags torch's pinned allocator uses.
rc = lib.hipHostMalloc(ctypes.byref(ptr), nbytes, 0)
if rc != 0:
raise RuntimeError(
f"hipHostMalloc of {nbytes / 1e9:.2f} GB for a HiCache host pool failed "
f"(rc={rc}, {lib.hipGetErrorString(rc).decode()}). With "
"HSA_USERPTR_FOR_PAGED_MEM=0 host pools are GTT memory, which all ranks "
"on a node share up to /sys/class/drm/card*/device/mem_info_gtt_total. "
"Reduce --hicache-size or unset HSA_USERPTR_FOR_PAGED_MEM."
)
return ptr.value
def _hip_host_free(ptr: int) -> None:
rc = _hip_runtime().hipHostFree(ptr)
if rc != 0:
logger.warning("hipHostFree failed (rc=%d) for ptr=%#x", rc, ptr)
@lru_cache(maxsize=1)
def _log_hip_owned_host_pool() -> None:
logger.info(
"HSA_USERPTR_FOR_PAGED_MEM=0: allocating HiCache host pools with "
"hipHostMalloc (GTT) instead of hipHostRegister (USERPTR)."
)
if envs.SGLANG_HUGEPAGE_SIZE.get():
logger.warning(
"SGLANG_HUGEPAGE_SIZE is ignored for hipHostMalloc HiCache host pools."
)
def _uses_hip_owned_host_memory(
pin_memory: bool, allocator: HostTensorAllocator
) -> bool:
# hipHostRegister memory is USERPTR even with HSA_USERPTR_FOR_PAGED_MEM=0; any page
# migration in it stalls every GPU queue while KFD revalidates. GTT never migrates.
return (
_is_hip
and pin_memory
# Storage allocators own their backing (shm fds, registered transfer buffers).
and type(allocator) is HostTensorAllocator
and os.environ.get("HSA_USERPTR_FOR_PAGED_MEM") == "0"
)
def _alloc_hip_host_tensor(dims: tuple, dtype: torch.dtype) -> torch.Tensor:
# Not torch pin_memory: its host allocator rounds each block up to a power of
# two, so a 180 GB pool would pin 256 GiB of GTT.
numel = math.prod(dims)
nbytes = numel * dtype.itemsize
ptr = _hip_host_malloc(nbytes)
try:
array = (ctypes.c_uint8 * nbytes).from_address(ptr)
buffer = torch.frombuffer(array, dtype=dtype, count=numel).reshape(dims)
except BaseException:
_hip_host_free(ptr)
raise
# Freed with the tensor's last view; at interpreter exit the driver reclaims it.
weakref.finalize(array, _hip_host_free, ptr).atexit = False
# Not host-registered, so HostKVCache.destroy() must not cudaHostUnregister it.
setattr(buffer, _CUDA_HOST_REGISTERED_RANGES_ATTR, [])
return buffer
def alloc_with_host_register(
dims: tuple,
dtype: torch.dtype,
@@ -243,8 +337,13 @@ def alloc_with_host_register(
) -> torch.Tensor:
"""
Allocate tensor and register host memory with cudaHostRegister.
CudaHostRegister only applies when pin_memory=True.
CudaHostRegister only applies when pin_memory=True. On ROCm with
HSA_USERPTR_FOR_PAGED_MEM=0, default-allocator pools come from hipHostMalloc.
"""
if _uses_hip_owned_host_memory(pin_memory=pin_memory, allocator=allocator):
assert device == "cpu", f"HiCache host pools are CPU memory; got {device!r}"
_log_hip_owned_host_pool()
return _alloc_hip_host_tensor(dims=dims, dtype=dtype)
buffer = allocator.allocate(dims, dtype=dtype, device=device)
if pin_memory:
_cuda_host_register(buffer, registration_granularity_bytes)
@@ -1,3 +1,6 @@
import ctypes
import gc
import os
import unittest
from types import SimpleNamespace
from unittest import mock
@@ -10,6 +13,7 @@ from sglang.srt.mem_cache.memory_pool_host import (
DeepSeekV4PagedHostPool,
DeepSeekV4StateHostPool,
)
from sglang.srt.mem_cache.pool_host import common as host_common
from sglang.srt.mem_cache.pool_host import mha as mha_pool_host
from sglang.srt.mem_cache.pool_host import mla as mla_pool_host
from sglang.srt.mem_cache.pool_host.common import (
@@ -66,6 +70,138 @@ class _FakeCudart:
class TestHiCacheHostRegister(unittest.TestCase):
def _hip_non_userptr(self):
return (
mock.patch.object(host_common, "_is_hip", True),
mock.patch.dict(os.environ, {"HSA_USERPTR_FOR_PAGED_MEM": "0"}),
)
def test_hip_non_userptr_pool_is_exact_size_and_never_unregistered(self):
"""The pool must not be rounded up to a power of two (torch pin_memory pins
256 GiB for a 180 GB pool) and destroy() must not cudaHostUnregister it."""
dims = (3, 5, 7) # 105 bf16 elements: 210 bytes, not a power of two
backing = (ctypes.c_uint8 * 210)()
allocator = host_common.HostTensorAllocator()
cudart = _FakeCudart()
is_hip, env = self._hip_non_userptr()
with (
is_hip,
env,
mock.patch.object(
host_common, "_hip_host_malloc", return_value=ctypes.addressof(backing)
) as malloc,
mock.patch.object(host_common, "_hip_host_free"),
mock.patch.object(allocator, "allocate") as allocate,
mock.patch.object(torch.cuda, "cudart", return_value=cudart),
):
buffer = host_common.alloc_with_host_register(
dims=dims,
dtype=torch.bfloat16,
device="cpu",
pin_memory=True,
allocator=allocator,
)
malloc.assert_called_once_with(210)
allocate.assert_not_called()
self.assertEqual(buffer.shape, dims)
self.assertEqual(buffer.data_ptr(), ctypes.addressof(backing))
_cuda_host_unregister(buffer)
self.assertEqual(cudart.registrations, [])
self.assertEqual(cudart.unregistrations, [])
def test_hip_host_pool_is_freed_once_after_its_last_view(self):
"""hipHostFree must wait for every view (the pools keep per-layer views)
and run exactly once."""
backing = (ctypes.c_uint8 * 64)()
ptr = ctypes.addressof(backing)
is_hip, env = self._hip_non_userptr()
with (
is_hip,
env,
mock.patch.object(host_common, "_hip_host_malloc", return_value=ptr),
mock.patch.object(host_common, "_hip_host_free") as free,
):
buffer = host_common.alloc_with_host_register(
dims=(4, 16),
dtype=torch.uint8,
device="cpu",
pin_memory=True,
allocator=host_common.HostTensorAllocator(),
)
layer_view = buffer[1]
del buffer
gc.collect()
free.assert_not_called()
del layer_view
gc.collect()
free.assert_called_once_with(ptr)
def test_non_userptr_path_preserves_other_allocators_and_platforms(self):
class StorageAllocator(host_common.HostTensorAllocator):
pass
for hip, userptr, custom, pinned in (
(False, "0", False, True),
(True, "1", False, True),
(True, None, False, True),
(True, "0", True, True),
(True, "0", False, False),
):
with self.subTest(hip=hip, userptr=userptr, custom=custom, pinned=pinned):
allocator = (
StorageAllocator if custom else host_common.HostTensorAllocator
)()
buffer = _FakeBuffer(0x10000000, 4096)
with (
mock.patch.object(host_common, "_is_hip", hip),
mock.patch.dict(os.environ),
mock.patch.object(host_common, "_hip_host_malloc") as malloc,
mock.patch.object(
allocator, "allocate", return_value=buffer
) as allocate,
mock.patch.object(host_common, "_cuda_host_register") as register,
):
os.environ.pop("HSA_USERPTR_FOR_PAGED_MEM", None)
if userptr is not None:
os.environ["HSA_USERPTR_FOR_PAGED_MEM"] = userptr
got = host_common.alloc_with_host_register(
(4096,), torch.uint8, "cpu", pinned, allocator, 512
)
self.assertIs(got, buffer)
malloc.assert_not_called()
allocate.assert_called_once_with(
(4096,), dtype=torch.uint8, device="cpu"
)
if pinned:
register.assert_called_once_with(buffer, 512)
else:
register.assert_not_called()
def test_gtt_exhaustion_names_the_cap_and_does_not_fall_back_to_userptr(self):
"""Running out of GTT must fail startup with the remedy, not silently
fall back to a USERPTR pool."""
allocator = host_common.HostTensorAllocator()
out_of_memory = SimpleNamespace(
hipHostMalloc=lambda ptr, nbytes, flags: 2,
hipGetErrorString=lambda rc: b"out of memory",
)
is_hip, env = self._hip_non_userptr()
with (
is_hip,
env,
mock.patch.object(host_common, "_hip_runtime", return_value=out_of_memory),
mock.patch.object(allocator, "allocate") as allocate,
self.assertRaisesRegex(RuntimeError, "mem_info_gtt_total.*--hicache-size"),
):
host_common.alloc_with_host_register(
dims=(8,),
dtype=torch.uint8,
device="cpu",
pin_memory=True,
allocator=allocator,
)
allocate.assert_not_called()
def test_dsa_page_layouts_with_draft_use_page_registration_granularity(self):
target_buffers = [torch.empty(1, dtype=torch.uint8) for _ in range(3)]
draft_buffer = torch.empty(1, dtype=torch.uint8)