From a0fcebb3bb7043e62b6ca8885b9e723f2262e091 Mon Sep 17 00:00:00 2001 From: Ilia Yastrebov Date: Thu, 1 Oct 2026 19:53:00 +0200 Subject: [PATCH] NIXL: Use stride desc API (#37827) Signed-off-by: Ilia Yastrebov Co-authored-by: Claude Fable 5.1 Co-authored-by: ishandhanani <82981111+ishandhanani@users.noreply.github.com> --- python/sglang/srt/disaggregation/nixl/conn.py | 150 ++++++++++-------- .../disaggregation/test_nixl_backend_basic.py | 16 +- 2 files changed, 89 insertions(+), 77 deletions(-) diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index fb099846b7..d2d3fb9e69 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -414,6 +414,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): kv_status_msg_carries_reason = True # ABORT handler defers the ack until the transfer worker drains. supports_deferred_decode_kv_release = True + # Newer NIXL prepares compressed (addr, len, dev_id, stride, count) runs + # directly; cleared on the first TypeError to expand per block instead. + _use_strided_descs: bool = True def __init__( self, @@ -734,6 +737,44 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): return False, True time.sleep(NIXL_ERR_SETTLE_POLL_S) + @staticmethod + def _pack_stride_descs(runs: list[tuple[int, int, int, int, int]]) -> np.ndarray: + """Pack (addr, len, dev_id, stride, count) runs into an Nx5 uint64 array.""" + # uint64: Intel XPU addresses have bit 63 set and overflow int64. + return np.array(runs, dtype=np.uint64).reshape(-1, 5) + + @staticmethod + def _expand_stride_descs(stride_descs: np.ndarray) -> np.ndarray: + """Expand Nx5 runs into one (addr, len, dev_id) descriptor per block.""" + parts: list[np.ndarray] = [] + for addr, length, dev_id, stride, count in stride_descs.tolist(): + out = np.empty((count, 3), dtype=np.uint64) + out[:, 0] = addr + np.arange(count, dtype=np.uint64) * stride + out[:, 1] = length + out[:, 2] = dev_id + parts.append(out) + return np.concatenate(parts) if parts else np.empty((0, 3), dtype=np.uint64) + + def _prep_xfer_dlist(self, peer_name: str, stride_descs: np.ndarray, mem_kind: str): + """Prepare a NIXL dlist handle from Nx5 runs. + + Tries the strided API first and falls back to expanding one descriptor + per block when the installed NIXL rejects the Nx5 layout. + """ + if self._use_strided_descs: + try: + return self.agent.prep_xfer_dlist(peer_name, stride_descs, mem_kind) + except TypeError: + logger.warning_once( + "Installed NIXL does not support strided descriptors, " + "falling back to per-block descriptors." + ) + self._use_strided_descs = False + descs = self.agent.get_xfer_descs( + self._expand_stride_descs(stride_descs), mem_kind + ) + return self.agent.prep_xfer_dlist(peer_name, descs, mem_kind) + def _prep_equal_tp_dlist( self, peer_name: str, @@ -756,12 +797,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): f"data_lens={len(kv_data_lens)}, xfer_lens={len(kv_xfer_lens)}" ) device_id = _nixl_device_id(mem_kind, gpu_id) - arrays = [] - # torch.int exceeds np.int64 range on Intel XPU (addresses have bit 63 set). - # Convert once at entry; all downstream arithmetic stays in uint64. - kv_ptrs_u64 = np.array(kv_ptrs, dtype=np.uint64) + runs: list[tuple[int, int, int, int, int]] = [] for base_ptr, item_len, data_len, xfer_len in zip( - kv_ptrs_u64, kv_item_lens, kv_data_lens, kv_xfer_lens + kv_ptrs, kv_item_lens, kv_data_lens, kv_xfer_lens ): if xfer_len > item_len: raise ValueError( @@ -769,22 +807,10 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): f"xfer_len={xfer_len}, item_len={item_len}, mem_kind={mem_kind}" ) n = num_slots if num_slots is not None else (data_len // item_len) - addrs = np.arange(n, dtype=np.uint64) * np.uint64(item_len) + base_ptr - arrays.append( - np.column_stack( - [ - addrs, - np.full(n, xfer_len, dtype=np.uint64), - np.full(n, device_id, dtype=np.uint64), - ] - ) - ) + # One run per region: n slots of xfer_len bytes, item_len apart. + runs.append((base_ptr, xfer_len, device_id, item_len, n)) - prep_handle = self.agent.prep_xfer_dlist(peer_name, np.vstack(arrays), mem_kind) - assert prep_handle is not None, ( - f"prep_xfer_dlist returned None for peer '{peer_name}'" - ) - return prep_handle + return self._prep_xfer_dlist(peer_name, self._pack_stride_descs(runs), mem_kind) def _init_equal_tp_prep_handle( self, @@ -884,8 +910,11 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): src_kv_item_len = self.kv_args.kv_item_lens[0] bytes_per_token_to_send = num_heads_to_send * bytes_per_head_slice - bytes_per_token_src = src_kv_item_len // page_size bytes_per_token_dst = dst_kv_item_len // page_size + # One run per region below relies on tokens (and src head groups) tiling + # a slot exactly; otherwise the runs would address the wrong bytes. + assert page_size * num_groups * bytes_per_token_to_send == src_kv_item_len + assert page_size * bytes_per_token_dst == dst_kv_item_len src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, layers_pp = ( self.get_mha_kv_ptrs_with_pp( @@ -897,33 +926,24 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): num_ptr_pairs = len(src_ptrs) num_slots = self.kv_args.kv_data_lens[0] // src_kv_item_len - slots = np.arange(num_slots, dtype=np.uint64) - tokens = np.arange(page_size, dtype=np.uint64) # reused in dst dlist below - groups = np.arange(num_groups, dtype=np.uint64) - # Src dlist built once and shared. + # Src dlist built once and shared. One run per region: [slot, token, group] + # is one dense progression of head slices, the flat index + # expand_page_indices_for_slice computes. if self.prep_handle_slice_src is None: - src_ptrs_arr = np.array(src_ptrs, dtype=np.uint64) - addrs = ( - src_ptrs_arr[:, None, None, None] - + slots[None, :, None, None] * np.uint64(src_kv_item_len) - + tokens[None, None, :, None] * np.uint64(bytes_per_token_src) - + groups[None, None, None, :] * np.uint64(bytes_per_token_to_send) - ).ravel() - src_array = np.column_stack( - [ - addrs, - np.full(len(addrs), bytes_per_token_to_send, dtype=np.uint64), - np.full( - len(addrs), - _nixl_device_id(src_mem_kind, self.kv_args.gpu_id), - dtype=np.uint64, - ), - ] - ) - src_handle = self.agent.prep_xfer_dlist("", src_array, src_mem_kind) - assert src_handle is not None, ( - f"prep_xfer_dlist returned None for slice src (decode_tp_size={decode_tp_size})" + src_device_id = _nixl_device_id(src_mem_kind, self.kv_args.gpu_id) + src_runs = [ + ( + ptr, + bytes_per_token_to_send, + src_device_id, + bytes_per_token_to_send, + num_slots * page_size * num_groups, + ) + for ptr in src_ptrs + ] + src_handle = self._prep_xfer_dlist( + "", self._pack_stride_descs(src_runs), src_mem_kind ) self.prep_handle_slice_src = ( src_handle, @@ -938,29 +958,21 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): if decode_kv_args.dst_num_slots is not None else num_slots ) - dst_slots = np.arange(num_slots_dst, dtype=np.uint64) - # (ptr, slot, token) → ravel. - dst_ptrs_arr = np.array(dst_ptrs, dtype=np.uint64) - addrs = ( - dst_ptrs_arr[:, None, None] - + dst_slots[None, :, None] * np.uint64(dst_kv_item_len) - + tokens[None, None, :] * np.uint64(bytes_per_token_dst) - + np.uint64(dst_head_offset) - ).ravel() - dst_array = np.column_stack( - [ - addrs, - np.full(len(addrs), bytes_per_token_to_send, dtype=np.uint64), - np.full( - len(addrs), - _nixl_device_id(dst_mem_kind, decode_kv_args.gpu_id), - dtype=np.uint64, - ), - ] - ) - dst_handle = self.agent.prep_xfer_dlist(peer_name, dst_array, dst_mem_kind) - assert dst_handle is not None, ( - f"prep_xfer_dlist returned None for slice dst for peer '{peer_name}'" + # One run per region over (slot, token): this peer's head slice sits at + # dst_head_offset in every token, bytes_per_token_dst apart. + dst_device_id = _nixl_device_id(dst_mem_kind, decode_kv_args.gpu_id) + dst_runs = [ + ( + ptr + dst_head_offset, + bytes_per_token_to_send, + dst_device_id, + bytes_per_token_dst, + num_slots_dst * page_size, + ) + for ptr in dst_ptrs + ] + dst_handle = self._prep_xfer_dlist( + peer_name, self._pack_stride_descs(dst_runs), dst_mem_kind ) self.prep_handles_slice_dst[peer_name] = ( dst_handle, diff --git a/test/registered/unit/disaggregation/test_nixl_backend_basic.py b/test/registered/unit/disaggregation/test_nixl_backend_basic.py index 2bf28ec9bd..4dabc4b490 100644 --- a/test/registered/unit/disaggregation/test_nixl_backend_basic.py +++ b/test/registered/unit/disaggregation/test_nixl_backend_basic.py @@ -1469,19 +1469,19 @@ class TestNixlHeteroTpReplicatedKV(CustomTestCase): src_handle, num_groups, _num_ptr_pairs, _num_slots = mgr.prep_handle_slice_src self.assertEqual(num_groups, 2) - # Every source descriptor [addr, addr+len) must lie inside a registered - # base region [ptr, ptr+REGION_LEN). Pre-fix, num_groups=4 pushed the - # top group's addresses past the region -> NIXL_ERR_NOT_FOUND. + # Every source run must lie inside a registered base region + # [ptr, ptr+REGION_LEN), gaps between its blocks included. Pre-fix, + # num_groups=4 pushed the top group past the region -> NIXL_ERR_NOT_FOUND. src_call = next(c for c in mgr.agent.calls if c[0] == "") src_array = src_call[1] regions = [(p, p + self.REGION_LEN) for p in self.SRC_PTRS] - for addr, length, _dev in src_array: + for addr, length, _dev, stride, count in src_array: addr = int(addr) - length = int(length) + end = addr + (int(count) - 1) * int(stride) + int(length) self.assertTrue( - any(lo <= addr and addr + length <= hi for lo, hi in regions), - f"descriptor [{addr:#x}, {addr + length:#x}) escapes all " - f"registered source regions {[(hex(lo), hex(hi)) for lo, hi in regions]}", + any(lo <= addr and end <= hi for lo, hi in regions), + f"run [{addr:#x}, {end:#x}) escapes all registered source " + f"regions {[(hex(lo), hex(hi)) for lo, hi in regions]}", ) def test_head_group_idx_maps_replicated_ranks_by_integer_division(self):