NIXL: Use stride desc API (#37827)

Signed-off-by: Ilia Yastrebov <iyastrebov@nvidia.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: ishandhanani <82981111+ishandhanani@users.noreply.github.com>
This commit is contained in:
Ilia Yastrebov
2026-10-01 10:53:00 -07:00
committed by GitHub
co-authored by Claude Fable 5.1 ishandhanani
parent fb92ba1127
commit a0fcebb3bb
2 changed files with 89 additions and 77 deletions
+81 -69
View File
@@ -414,6 +414,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
kv_status_msg_carries_reason = True
# ABORT handler defers the ack until the transfer worker drains.
supports_deferred_decode_kv_release = True
# Newer NIXL prepares compressed (addr, len, dev_id, stride, count) runs
# directly; cleared on the first TypeError to expand per block instead.
_use_strided_descs: bool = True
def __init__(
self,
@@ -734,6 +737,44 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
return False, True
time.sleep(NIXL_ERR_SETTLE_POLL_S)
@staticmethod
def _pack_stride_descs(runs: list[tuple[int, int, int, int, int]]) -> np.ndarray:
"""Pack (addr, len, dev_id, stride, count) runs into an Nx5 uint64 array."""
# uint64: Intel XPU addresses have bit 63 set and overflow int64.
return np.array(runs, dtype=np.uint64).reshape(-1, 5)
@staticmethod
def _expand_stride_descs(stride_descs: np.ndarray) -> np.ndarray:
"""Expand Nx5 runs into one (addr, len, dev_id) descriptor per block."""
parts: list[np.ndarray] = []
for addr, length, dev_id, stride, count in stride_descs.tolist():
out = np.empty((count, 3), dtype=np.uint64)
out[:, 0] = addr + np.arange(count, dtype=np.uint64) * stride
out[:, 1] = length
out[:, 2] = dev_id
parts.append(out)
return np.concatenate(parts) if parts else np.empty((0, 3), dtype=np.uint64)
def _prep_xfer_dlist(self, peer_name: str, stride_descs: np.ndarray, mem_kind: str):
"""Prepare a NIXL dlist handle from Nx5 runs.
Tries the strided API first and falls back to expanding one descriptor
per block when the installed NIXL rejects the Nx5 layout.
"""
if self._use_strided_descs:
try:
return self.agent.prep_xfer_dlist(peer_name, stride_descs, mem_kind)
except TypeError:
logger.warning_once(
"Installed NIXL does not support strided descriptors, "
"falling back to per-block descriptors."
)
self._use_strided_descs = False
descs = self.agent.get_xfer_descs(
self._expand_stride_descs(stride_descs), mem_kind
)
return self.agent.prep_xfer_dlist(peer_name, descs, mem_kind)
def _prep_equal_tp_dlist(
self,
peer_name: str,
@@ -756,12 +797,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
f"data_lens={len(kv_data_lens)}, xfer_lens={len(kv_xfer_lens)}"
)
device_id = _nixl_device_id(mem_kind, gpu_id)
arrays = []
# torch.int exceeds np.int64 range on Intel XPU (addresses have bit 63 set).
# Convert once at entry; all downstream arithmetic stays in uint64.
kv_ptrs_u64 = np.array(kv_ptrs, dtype=np.uint64)
runs: list[tuple[int, int, int, int, int]] = []
for base_ptr, item_len, data_len, xfer_len in zip(
kv_ptrs_u64, kv_item_lens, kv_data_lens, kv_xfer_lens
kv_ptrs, kv_item_lens, kv_data_lens, kv_xfer_lens
):
if xfer_len > item_len:
raise ValueError(
@@ -769,22 +807,10 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
f"xfer_len={xfer_len}, item_len={item_len}, mem_kind={mem_kind}"
)
n = num_slots if num_slots is not None else (data_len // item_len)
addrs = np.arange(n, dtype=np.uint64) * np.uint64(item_len) + base_ptr
arrays.append(
np.column_stack(
[
addrs,
np.full(n, xfer_len, dtype=np.uint64),
np.full(n, device_id, dtype=np.uint64),
]
)
)
# One run per region: n slots of xfer_len bytes, item_len apart.
runs.append((base_ptr, xfer_len, device_id, item_len, n))
prep_handle = self.agent.prep_xfer_dlist(peer_name, np.vstack(arrays), mem_kind)
assert prep_handle is not None, (
f"prep_xfer_dlist returned None for peer '{peer_name}'"
)
return prep_handle
return self._prep_xfer_dlist(peer_name, self._pack_stride_descs(runs), mem_kind)
def _init_equal_tp_prep_handle(
self,
@@ -884,8 +910,11 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
src_kv_item_len = self.kv_args.kv_item_lens[0]
bytes_per_token_to_send = num_heads_to_send * bytes_per_head_slice
bytes_per_token_src = src_kv_item_len // page_size
bytes_per_token_dst = dst_kv_item_len // page_size
# One run per region below relies on tokens (and src head groups) tiling
# a slot exactly; otherwise the runs would address the wrong bytes.
assert page_size * num_groups * bytes_per_token_to_send == src_kv_item_len
assert page_size * bytes_per_token_dst == dst_kv_item_len
src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, layers_pp = (
self.get_mha_kv_ptrs_with_pp(
@@ -897,33 +926,24 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
num_ptr_pairs = len(src_ptrs)
num_slots = self.kv_args.kv_data_lens[0] // src_kv_item_len
slots = np.arange(num_slots, dtype=np.uint64)
tokens = np.arange(page_size, dtype=np.uint64) # reused in dst dlist below
groups = np.arange(num_groups, dtype=np.uint64)
# Src dlist built once and shared.
# Src dlist built once and shared. One run per region: [slot, token, group]
# is one dense progression of head slices, the flat index
# expand_page_indices_for_slice computes.
if self.prep_handle_slice_src is None:
src_ptrs_arr = np.array(src_ptrs, dtype=np.uint64)
addrs = (
src_ptrs_arr[:, None, None, None]
+ slots[None, :, None, None] * np.uint64(src_kv_item_len)
+ tokens[None, None, :, None] * np.uint64(bytes_per_token_src)
+ groups[None, None, None, :] * np.uint64(bytes_per_token_to_send)
).ravel()
src_array = np.column_stack(
[
addrs,
np.full(len(addrs), bytes_per_token_to_send, dtype=np.uint64),
np.full(
len(addrs),
_nixl_device_id(src_mem_kind, self.kv_args.gpu_id),
dtype=np.uint64,
),
]
)
src_handle = self.agent.prep_xfer_dlist("", src_array, src_mem_kind)
assert src_handle is not None, (
f"prep_xfer_dlist returned None for slice src (decode_tp_size={decode_tp_size})"
src_device_id = _nixl_device_id(src_mem_kind, self.kv_args.gpu_id)
src_runs = [
(
ptr,
bytes_per_token_to_send,
src_device_id,
bytes_per_token_to_send,
num_slots * page_size * num_groups,
)
for ptr in src_ptrs
]
src_handle = self._prep_xfer_dlist(
"", self._pack_stride_descs(src_runs), src_mem_kind
)
self.prep_handle_slice_src = (
src_handle,
@@ -938,29 +958,21 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
if decode_kv_args.dst_num_slots is not None
else num_slots
)
dst_slots = np.arange(num_slots_dst, dtype=np.uint64)
# (ptr, slot, token) → ravel.
dst_ptrs_arr = np.array(dst_ptrs, dtype=np.uint64)
addrs = (
dst_ptrs_arr[:, None, None]
+ dst_slots[None, :, None] * np.uint64(dst_kv_item_len)
+ tokens[None, None, :] * np.uint64(bytes_per_token_dst)
+ np.uint64(dst_head_offset)
).ravel()
dst_array = np.column_stack(
[
addrs,
np.full(len(addrs), bytes_per_token_to_send, dtype=np.uint64),
np.full(
len(addrs),
_nixl_device_id(dst_mem_kind, decode_kv_args.gpu_id),
dtype=np.uint64,
),
]
)
dst_handle = self.agent.prep_xfer_dlist(peer_name, dst_array, dst_mem_kind)
assert dst_handle is not None, (
f"prep_xfer_dlist returned None for slice dst for peer '{peer_name}'"
# One run per region over (slot, token): this peer's head slice sits at
# dst_head_offset in every token, bytes_per_token_dst apart.
dst_device_id = _nixl_device_id(dst_mem_kind, decode_kv_args.gpu_id)
dst_runs = [
(
ptr + dst_head_offset,
bytes_per_token_to_send,
dst_device_id,
bytes_per_token_dst,
num_slots_dst * page_size,
)
for ptr in dst_ptrs
]
dst_handle = self._prep_xfer_dlist(
peer_name, self._pack_stride_descs(dst_runs), dst_mem_kind
)
self.prep_handles_slice_dst[peer_name] = (
dst_handle,