[AMD] Add engram host/device address mapping (#42552)

This commit is contained in:
Thomas Wang
2026-10-05 19:12:12 -07:00
committed by GitHub
parent f2f0be731c
commit 1abd356595
+47 -4
View File
@@ -8,6 +8,7 @@ from __future__ import annotations
import ctypes
import errno
import functools
import glob
import logging
import mmap
@@ -33,6 +34,9 @@ from sglang.kernels.ops.embeddings.engram_hash import (
engram_hash_ids_and_commit,
)
from sglang.srt.distributed import tensor_model_parallel_all_reduce
from sglang.srt.distributed.device_communicators.cuda_wrapper import (
find_loaded_library,
)
from sglang.srt.distributed.parallel_state import inplace_all_reduce
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import (
@@ -554,6 +558,39 @@ def _drop_page_cache_once(reason: str) -> None:
)
@functools.cache
def _hip_runtime() -> ctypes.CDLL:
"""
torch.cuda.cudart() does not expose hipHostGetDevicePointer, so call it
through ctypes to map a registered host address to its device address.
"""
path = find_loaded_library("libamdhip64")
if path is None:
raise RuntimeError("libamdhip64 is not loaded in the current process")
lib = ctypes.CDLL(path)
lib.hipHostGetDevicePointer.restype = ctypes.c_int
lib.hipHostGetDevicePointer.argtypes = [
ctypes.POINTER(ctypes.c_void_p),
ctypes.c_void_p,
ctypes.c_uint,
]
return lib
def _registered_device_ptr(host_ptr: int) -> int:
"""
Address kernels must use for registered host memory.
UVA makes it the host address on CUDA; HIP may map it elsewhere.
"""
if not _is_hip:
return host_ptr
device_ptr = ctypes.c_void_p()
err = _hip_runtime().hipHostGetDevicePointer(ctypes.byref(device_ptr), host_ptr, 0)
if err != 0 or not device_ptr.value:
raise RuntimeError(f"hipHostGetDevicePointer failed: {err}")
return device_ptr.value
class _HostTable:
"""Host-memory backing for one engram table ('shared' or 'per_rank' layout).
@@ -602,6 +639,7 @@ class _HostTable:
err = torch.cuda.cudart().cudaHostRegister(self.bytes.data_ptr(), nbytes, 0)
if int(err) != 0:
raise RuntimeError(f"cudaHostRegister({nbytes} bytes) failed: {err}")
self.device_ptr = _registered_device_ptr(self.bytes.data_ptr())
def _open_shared_fd(self, nbytes: int, name: str) -> int:
owner = None
@@ -724,11 +762,18 @@ class EngramEmbedding(nn.Module):
scale = raw[w_bytes:].view(torch.float8_e8m0fnu).view(n, dim // FP8_BLOCK_SIZE)
self.weight = nn.Parameter(weight, requires_grad=False)
self.scale = nn.Parameter(scale, requires_grad=False)
device_ptr = self.host_table.device_ptr
self._host_table_ptrs = (device_ptr, device_ptr + w_bytes)
@property
def _shared(self) -> bool:
return self.host_table is not None and self.host_table.layout == "shared"
def _table_ptrs(self) -> tuple[int, int]:
if self.host_table is None:
return self.weight.data_ptr(), self.scale.data_ptr()
return self._host_table_ptrs
def _load_rows(self, param: nn.Parameter, loaded_weight: torch.Tensor):
rows = slice(self.row_start, self.row_start + self.rows)
if self._shared:
@@ -755,8 +800,7 @@ class EngramEmbedding(nn.Module):
return self._empty(indices)
out = self._empty(indices)
engram_gather(
self.weight.data_ptr(),
self.scale.data_ptr(),
*self._table_ptrs(),
indices.reshape(-1),
out.view(-1, self.dim),
self.dim,
@@ -815,8 +859,7 @@ class EngramEmbedding(nn.Module):
return values.to(torch.bfloat16).masked_fill(~owned.unsqueeze(-1), 0)
out = self._empty(indices)
engram_gather(
self.weight.data_ptr(),
self.scale.data_ptr(),
*self._table_ptrs(),
indices.reshape(-1),
out.view(-1, self.dim),
self.dim,