[PD][NIXL] Fix prepared dlists for KV entries of different lengths (DeepSeek-V4) (#42005)

This commit is contained in:
Alex
2026-10-02 17:59:31 +08:00
committed by GitHub
parent 41bd213c5f
commit 1db33bc367
2 changed files with 188 additions and 3 deletions
+23 -3
View File
@@ -352,6 +352,22 @@ def expand_page_indices_for_slice(
return (pair_offsets[:, None] + within_pair[None, :]).ravel().astype(np.int32)
def num_kv_slots(kv_data_lens: List[int], kv_item_lens: List[int]) -> int:
"""Slots every registered KV entry can address.
Prepared dlists index entry i's slot p as i * num_slots + p, so every entry
must expose the same count. Entries can have different lengths, so use
the shortest.
"""
return min(
(
data_len // item_len
for data_len, item_len in zip(kv_data_lens, kv_item_lens)
),
default=0,
)
def repeat_indices_over_layers(
indices: npt.NDArray[np.int32], num_layers: int, layer_length: int
) -> npt.NDArray[np.int32]:
@@ -512,8 +528,8 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
if self.disaggregation_mode == DisaggregationMode.PREFILL:
if self.kv_args.kv_item_lens:
self._num_slots_src = (
self.kv_args.kv_data_lens[0] // self.kv_args.kv_item_lens[0]
self._num_slots_src = num_kv_slots(
self.kv_args.kv_data_lens, self.kv_args.kv_item_lens
)
transfer_queue_size = envs.SGLANG_DISAGGREGATION_QUEUE_SIZE.get()
self.transfer_queues: List[FastQueue] = [
@@ -996,6 +1012,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
self.kv_args.kv_item_lens[seg.start : seg.end],
self.kv_args.kv_data_lens[seg.start : seg.end],
self.kv_args.gpu_id,
num_slots=self._num_slots_src,
mem_kind=seg.src_mem_kind,
)
self.prep_handles_segment_src[src_key] = src_handle
@@ -1159,6 +1176,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
"",
self.kv_args.kv_data_ptrs,
self.kv_args.gpu_id,
num_slots=self._num_slots_src,
mem_kind=src_mem_kind,
)
dst_num_slots = (
@@ -3391,7 +3409,9 @@ class NixlKVReceiver(CommonKVReceiver):
staging_total_size_str = b""
if self.kv_mgr.kv_args.kv_item_lens:
dst_kv_item_len = self.kv_mgr.kv_args.kv_item_lens[0]
dst_num_slots = self.kv_mgr.kv_args.kv_data_lens[0] // dst_kv_item_len
dst_num_slots = num_kv_slots(
self.kv_mgr.kv_args.kv_data_lens, self.kv_mgr.kv_args.kv_item_lens
)
else:
dst_kv_item_len = 0
dst_num_slots = 0
@@ -0,0 +1,165 @@
"""NIXL prepared dlists over DeepSeek-V4's unequal-length KV entries.
DeepSeek-V4 registers C4 KV, C4 indexer and C128 KV pools for PD transfer, and
their row counts differ. Every entry must still map page p to its own page p.
"""
import unittest
from types import SimpleNamespace
import numpy as np
import torch
from sglang.srt.disaggregation.nixl.conn import NixlKVManager, num_kv_slots
from sglang.srt.mem_cache.deepseek_v4_memory_pool import (
DeepSeekV4IndexerPool,
DeepSeekV4SingleKVPool,
)
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
PAGE = 256
FULL_PAGES = 40
def dsv4_kv_buffers(layers=3):
"""The PD KV entries of a DeepSeek-V4 pool, in wire order."""
def kv(ratio, size):
return DeepSeekV4SingleKVPool(
size,
PAGE // ratio,
torch.uint8,
448,
64,
layers,
"cpu",
False,
global_page_size=PAGE,
).kv_buffer
c4_size = FULL_PAGES * PAGE // 4
indexer = DeepSeekV4IndexerPool(
c4_size,
PAGE // 4,
torch.uint8,
128,
layers,
"cpu",
False,
use_fp4_indexer=False,
)
return [
*kv(4, c4_size),
*indexer.contiguous_page_row_buffers(),
*kv(128, FULL_PAGES * PAGE // 128),
]
def geometry(bufs):
return (
[b.data_ptr() for b in bufs],
[b.nbytes for b in bufs],
[b[0].nbytes for b in bufs],
)
class FakeAgent:
"""Prepared dlists expand strided runs to per-block rows, as NIXL does.
Transfer indices address the expanded descriptors, so the prepared handle
must be the expanded array.
"""
def __init__(self):
self.posted = []
def prep_xfer_dlist(self, peer_name, descs, mem_kind):
return NixlKVManager._expand_stride_descs(np.asarray(descs))
def make_prepped_xfer(self, op, src, src_indices, dst, dst_indices, notif):
self.posted.append((src[src_indices], dst[dst_indices]))
return len(self.posted)
def transfer(self, handle):
return "PROC"
class NixlDsv4KvSlotsTest(CustomTestCase):
def setUp(self):
self.src, self.dst = dsv4_kv_buffers(), dsv4_kv_buffers()
rows = {b.shape[0] for b in self.src}
self.assertGreater(len(rows), 1, "fixture must have unequal entries")
def _manager(self):
ptrs, lens, items = geometry(self.src)
mgr = object.__new__(NixlKVManager)
mgr.kv_args = SimpleNamespace(
kv_data_ptrs=ptrs,
kv_data_lens=lens,
kv_item_lens=items,
kv_data_mem_kinds=["VRAM"] * len(ptrs),
gpu_id=0,
kv_layer_ids=[],
)
mgr.pp_size = 1
mgr.is_mla_backend = True
mgr.is_hybrid_mla_backend = False
mgr.attn_tp_size = 1
mgr.src_mem_kind = "VRAM"
mgr.prep_handles = {}
mgr.prep_handles_segment_src = {}
mgr._num_slots_src = num_kv_slots(lens, items)
mgr.agent = FakeAgent()
return mgr
def _peer(self):
ptrs, lens, items = geometry(self.dst)
return SimpleNamespace(
agent_name="decode",
gpu_id=1,
dst_kv_ptrs=ptrs,
dst_kv_item_lens=items,
dst_kv_mem_kinds=["VRAM"] * len(ptrs),
requires_dcp_relayout=False,
dst_kv_layer_ids=[],
# What the decode advertises for its own entries.
dst_num_slots=num_kv_slots(lens, items),
dst_homogeneous_mem_kind=None,
kv_xfer_segments=None,
)
def test_every_entry_reads_and_writes_its_own_page(self):
mgr, peer = self._manager(), self._peer()
mgr.decode_kv_args_table = {"decode": peer}
mgr._prepare_payload_xfer(peer)
# The highest page any allocator can hand out fits every entry.
pages = np.array([1, 17, FULL_PAGES], dtype=np.int32)
mgr.send_kvcache("decode", pages, peer.dst_kv_ptrs, pages[::-1].copy(), 1, "n")
((src_rows, dst_rows),) = mgr.agent.posted
n = len(self.src)
for i in range(n):
for k, page in enumerate(pages):
s, d = src_rows[i * len(pages) + k], dst_rows[i * len(pages) + k]
item = self.src[i][0].nbytes
self.assertEqual(
s[0], self.src[i].data_ptr() + int(page) * item, (i, page)
)
self.assertEqual(
d[0], self.dst[i].data_ptr() + int(pages[::-1][k]) * item, (i, page)
)
def test_destination_dlist_stays_inside_every_entry(self):
mgr, peer = self._manager(), self._peer()
mgr.decode_kv_args_table = {"decode": peer}
mgr._prepare_payload_xfer(peer)
rows = mgr.prep_handles["decode"].reshape(len(self.dst), -1, 3)
for buf, entry in zip(self.dst, rows):
end = buf.data_ptr() + buf.nbytes
self.assertTrue((entry[:, 0] + entry[:, 1] <= end).all())
if __name__ == "__main__":
unittest.main()