[PD] Support heterogeneous TP for hybrid SWA KV transfer (#40909)
This commit is contained in:
@@ -347,6 +347,8 @@ Enable the staging buffer when prefill and decode use **different TP sizes** wit
|
||||
|
||||
> **Note:** The staging buffer is designed for non-MLA models (e.g. GQA, MHA). MLA models (e.g. DeepSeek-V2/V3) should not enable this flag.
|
||||
|
||||
> **Note:** You can use different attention TP sizes for hybrid sliding-window models (e.g. `openai/gpt-oss-120b`) with Mooncake or NIXL. For unequal TP, both backends transfer the sliding-window KV by token and KV-head slice. The staging buffer carries only full-attention KV. Use `--pp-size 1` and leave `--enable-unified-memory` disabled for this configuration. MXFP8 KV cache is not supported for unequal-TP SWA transfer.
|
||||
|
||||
### Environment Variables
|
||||
|
||||
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
|
||||
|
||||
@@ -81,6 +81,9 @@ class KVArgs:
|
||||
# scatter transfer can slice each independently head-sharded sub-block; None
|
||||
# per tensor when the single contiguous slice already matches the layout.
|
||||
state_conv_shard_groups: List[List[Optional[List[int]]]]
|
||||
# Physical layout of the SWA sub-pool ("nhd", "hnd" or "vectorized_5d");
|
||||
# None when the SWA state is not held in an MHA sub-pool.
|
||||
swa_kv_cache_layout: Optional[str]
|
||||
ib_device: str
|
||||
gpu_id: int
|
||||
kv_head_num: int
|
||||
|
||||
@@ -1230,6 +1230,26 @@ class CommonKVManager(BaseKVManager):
|
||||
"servers (including prefill CP ranks for MLA)"
|
||||
)
|
||||
|
||||
# Every send path rejects head-sharded MXFP8 block scales across
|
||||
# attention TP sizes, so each request would fail at its first transfer.
|
||||
if (
|
||||
info.attn_tp_size != self.attn_tp_size
|
||||
and not self.is_mla_backend
|
||||
and not self.is_hybrid_mla_backend
|
||||
):
|
||||
block_scale_states = [
|
||||
st
|
||||
for st in getattr(self.kv_args, "state_types", None) or []
|
||||
if st in (StateType.BLOCK_SCALE, StateType.BLOCK_SCALE_SWA)
|
||||
]
|
||||
if block_scale_states:
|
||||
raise RuntimeError(
|
||||
"PD Disaggregation does NOT support different attention TP "
|
||||
"sizes with MXFP8 KV cache: prefill attn_tp_size="
|
||||
f"{info.attn_tp_size}, decode attn_tp_size={self.attn_tp_size}, "
|
||||
f"state types {[st.value for st in block_scale_states]}."
|
||||
)
|
||||
|
||||
if self.dcp_size > 1:
|
||||
if not (self.is_mla_backend or self.is_hybrid_mla_backend):
|
||||
raise RuntimeError(
|
||||
@@ -1541,6 +1561,108 @@ class CommonKVManager(BaseKVManager):
|
||||
layers_current_pp_stage = len(src_k_ptrs)
|
||||
return src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, layers_current_pp_stage
|
||||
|
||||
def _get_mha_head_slice_blocks(
|
||||
self,
|
||||
src_ptrs: List[int],
|
||||
src_item_lens: List[int],
|
||||
dst_ptrs: List[int],
|
||||
dst_item_lens: List[int],
|
||||
src_page_indices: npt.NDArray[np.int32],
|
||||
dst_page_indices: npt.NDArray[np.int32],
|
||||
dst_attn_tp_size: int,
|
||||
dst_tp_rank: int,
|
||||
) -> List[Tuple[npt.NDArray[np.uint64], npt.NDArray[np.uint64], int]]:
|
||||
# SWA state has no layer ids and its pool starts every PP stage at layer
|
||||
# 0, so a prefill stage's entries cannot be paired with the decode layers.
|
||||
if self.pp_size > 1:
|
||||
raise RuntimeError(
|
||||
"PD Disaggregation does NOT support PD different TP sizes for "
|
||||
"non-MLA SWA hybrid models with pipeline parallelism yet."
|
||||
)
|
||||
# The addresses below assume NHD pages [page_size, heads, head_dim]; HND
|
||||
# and vectorized_5d pages have the same size in another element order.
|
||||
if self.kv_args.swa_kv_cache_layout != "nhd":
|
||||
raise RuntimeError(
|
||||
"Head-sliced PD transfer needs the NHD KV cache layout, got "
|
||||
f"{self.kv_args.swa_kv_cache_layout}"
|
||||
)
|
||||
from sglang.srt.disaggregation.common.staging_buffer import (
|
||||
compute_head_slice_params,
|
||||
resolve_total_kv_heads,
|
||||
)
|
||||
|
||||
src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, num_layers = (
|
||||
self.get_mha_kv_ptrs_with_pp(src_ptrs, dst_ptrs)
|
||||
)
|
||||
_, _, dst_k_lens, dst_v_lens, _ = self.get_mha_kv_ptrs_with_pp(
|
||||
src_item_lens, dst_item_lens
|
||||
)
|
||||
entries = list(
|
||||
zip(
|
||||
src_k_ptrs + src_v_ptrs,
|
||||
dst_k_ptrs + dst_v_ptrs,
|
||||
src_item_lens,
|
||||
dst_k_lens + dst_v_lens,
|
||||
)
|
||||
)
|
||||
# A single page-envelope region (unified memory) has no per-layer K/V
|
||||
# split to slice heads from.
|
||||
if len(entries) != len(src_ptrs) or 2 * num_layers != len(src_ptrs):
|
||||
raise RuntimeError(
|
||||
"Head-sliced PD transfer needs matching per-layer K/V buffers: "
|
||||
f"prefill has {len(src_ptrs)} entries, decode has {len(dst_ptrs)}"
|
||||
)
|
||||
if len(src_page_indices) != len(dst_page_indices):
|
||||
raise RuntimeError(
|
||||
"Head-sliced PD transfer page count mismatch: "
|
||||
f"prefill={len(src_page_indices)}, decode={len(dst_page_indices)}"
|
||||
)
|
||||
|
||||
total_kv_heads = resolve_total_kv_heads(self.kv_args, self.attn_tp_size)
|
||||
src_head_start, num_heads, dst_head_start, _ = compute_head_slice_params(
|
||||
src_attn_tp_size=self.attn_tp_size,
|
||||
dst_attn_tp_size=dst_attn_tp_size,
|
||||
src_tp_rank=self.kv_args.engine_rank,
|
||||
dst_tp_rank=dst_tp_rank,
|
||||
total_kv_heads=total_kv_heads,
|
||||
)
|
||||
src_heads = max(1, total_kv_heads // self.attn_tp_size)
|
||||
dst_heads = max(1, total_kv_heads // dst_attn_tp_size)
|
||||
page_size = self.kv_args.page_size
|
||||
tokens = np.arange(page_size, dtype=np.uint64)
|
||||
src_pages = np.asarray(src_page_indices, dtype=np.uint64)[:, None]
|
||||
dst_pages = np.asarray(dst_page_indices, dtype=np.uint64)[:, None]
|
||||
|
||||
blocks = []
|
||||
for src_ptr, dst_ptr, src_item_len, dst_item_len in entries:
|
||||
head_len = src_item_len // (page_size * src_heads)
|
||||
# Both peers must hold whole heads of one size, or this state is
|
||||
# sharded differently from total_kv_head_num.
|
||||
if (
|
||||
head_len * page_size * src_heads != src_item_len
|
||||
or head_len * page_size * dst_heads != dst_item_len
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"KV item lengths (prefill={src_item_len}, decode={dst_item_len}) "
|
||||
f"do not split into {src_heads} and {dst_heads} heads of "
|
||||
f"{total_kv_heads} total with page_size={page_size}"
|
||||
)
|
||||
# Each page item is [page_size, heads, head_dim]: one block per token.
|
||||
src_addrs = (
|
||||
np.uint64(src_ptr)
|
||||
+ src_pages * np.uint64(src_item_len)
|
||||
+ tokens * np.uint64(src_heads * head_len)
|
||||
+ np.uint64(src_head_start * head_len)
|
||||
)
|
||||
dst_addrs = (
|
||||
np.uint64(dst_ptr)
|
||||
+ dst_pages * np.uint64(dst_item_len)
|
||||
+ tokens * np.uint64(dst_heads * head_len)
|
||||
+ np.uint64(dst_head_start * head_len)
|
||||
)
|
||||
blocks.append((src_addrs.ravel(), dst_addrs.ravel(), num_heads * head_len))
|
||||
return blocks
|
||||
|
||||
def get_mla_kv_ptrs_with_pp(
|
||||
self,
|
||||
src_kv_ptrs: List[int],
|
||||
|
||||
@@ -1966,12 +1966,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
and self.attn_tp_size
|
||||
!= target_rank_registration_info.dst_attn_tp_size
|
||||
)
|
||||
if (
|
||||
slice_kv_heads = (
|
||||
has_heterogeneous_attn_tp
|
||||
and not self.is_mla_backend
|
||||
and not self.is_hybrid_mla_backend
|
||||
and not is_qwen4_qsa_state
|
||||
):
|
||||
)
|
||||
if slice_kv_heads and st != StateType.SWA:
|
||||
raise RuntimeError(
|
||||
f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet."
|
||||
)
|
||||
@@ -2038,6 +2039,25 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
src_indices = src_indices[: len(dst_indices_local)]
|
||||
else:
|
||||
dst_indices_local = dst_indices_local[: len(src_indices)]
|
||||
if slice_kv_heads:
|
||||
rc = (
|
||||
self._send_swa_state_slice(
|
||||
req=req,
|
||||
prefill_swa_page_indices=src_indices,
|
||||
src_state_data_ptrs=src_data_ptrs,
|
||||
src_state_item_lens=src_item_lens,
|
||||
dst_state_data_ptrs=dst_data_ptrs,
|
||||
dst_swa_page_indices=dst_indices_local,
|
||||
dst_state_item_lens=dst_item_lens,
|
||||
dst_tp_rank=target_rank_registration_info.dst_tp_rank,
|
||||
dst_attn_tp_size=(
|
||||
target_rank_registration_info.dst_attn_tp_size
|
||||
),
|
||||
executor=executor,
|
||||
)
|
||||
or rc
|
||||
)
|
||||
continue
|
||||
rc = (
|
||||
self._send_kvcache_generic(
|
||||
mooncake_session_id=req.mooncake_session_id,
|
||||
@@ -2292,6 +2312,49 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
|
||||
return self._transfer_data(req.mooncake_session_id, transfer_blocks)
|
||||
|
||||
def _send_swa_state_slice(
|
||||
self,
|
||||
req: TransferInfo,
|
||||
prefill_swa_page_indices: List[int],
|
||||
src_state_data_ptrs: List[int],
|
||||
src_state_item_lens: List[int],
|
||||
dst_state_data_ptrs: List[int],
|
||||
dst_swa_page_indices: List[int],
|
||||
dst_state_item_lens: List[int],
|
||||
dst_tp_rank: int,
|
||||
dst_attn_tp_size: int,
|
||||
executor: concurrent.futures.ThreadPoolExecutor,
|
||||
) -> int:
|
||||
blocks = self._get_mha_head_slice_blocks(
|
||||
src_ptrs=src_state_data_ptrs,
|
||||
src_item_lens=src_state_item_lens,
|
||||
dst_ptrs=dst_state_data_ptrs,
|
||||
dst_item_lens=dst_state_item_lens,
|
||||
src_page_indices=np.asarray(prefill_swa_page_indices),
|
||||
dst_page_indices=np.asarray(dst_swa_page_indices),
|
||||
dst_attn_tp_size=dst_attn_tp_size,
|
||||
dst_tp_rank=dst_tp_rank,
|
||||
)
|
||||
logger.warning_once(
|
||||
"Using SWA state head-slice transfer for different attention TP sizes: "
|
||||
f"prefill={self.attn_tp_size}, decode={dst_attn_tp_size}."
|
||||
)
|
||||
futures = [
|
||||
executor.submit(
|
||||
self._transfer_data,
|
||||
req.mooncake_session_id,
|
||||
list(
|
||||
zip(
|
||||
src_addrs.tolist(),
|
||||
dst_addrs.tolist(),
|
||||
[length] * src_addrs.size,
|
||||
)
|
||||
),
|
||||
)
|
||||
for src_addrs, dst_addrs, length in blocks
|
||||
]
|
||||
return self._await_transfer_futures(futures)
|
||||
|
||||
def _poison_unsettled_send(self, room: int) -> None:
|
||||
"""Keep the drain ack from firing after a submitted transfer failed.
|
||||
|
||||
|
||||
@@ -621,7 +621,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
f"(ptr=0x{ptr:x}, size={size})"
|
||||
)
|
||||
|
||||
def set_kv_buffer_tensors(self, k_buffers: list, v_buffers: list, page_size: int):
|
||||
def set_kv_buffer_tensors(
|
||||
self,
|
||||
k_buffers: list,
|
||||
v_buffers: list,
|
||||
page_size: int,
|
||||
slot_layer_ids: Optional[List[int]] = None,
|
||||
):
|
||||
# NOTE: matches mooncake behavior -- staging buffers are now
|
||||
# created in __init__ (per-worker), independent of the kv
|
||||
# tensors. This setter only stashes the tensor metadata used by
|
||||
@@ -630,6 +636,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
"k_buffers": k_buffers,
|
||||
"v_buffers": v_buffers,
|
||||
"page_size": page_size,
|
||||
"slot_layer_ids": list(slot_layer_ids or []),
|
||||
}
|
||||
|
||||
def register_staging_room_bootstrap(self, room, bootstrap_infos, receiver):
|
||||
@@ -2608,6 +2615,63 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
raise Exception("Failed to post Mamba state slice transfer")
|
||||
return xfer_handle
|
||||
|
||||
def _send_swa_state_slice(
|
||||
self,
|
||||
peer_name: str,
|
||||
prefill_swa_page_indices: List[int],
|
||||
src_state_data_ptrs: List[int],
|
||||
src_state_item_lens: List[int],
|
||||
dst_state_data_ptrs: List[int],
|
||||
dst_swa_page_indices: List[int],
|
||||
dst_state_item_lens: List[int],
|
||||
dst_gpu_id: int,
|
||||
notif: str,
|
||||
decode_tp_size: int,
|
||||
decode_tp_rank: int,
|
||||
):
|
||||
blocks = self._get_mha_head_slice_blocks(
|
||||
src_ptrs=src_state_data_ptrs,
|
||||
src_item_lens=src_state_item_lens,
|
||||
dst_ptrs=dst_state_data_ptrs,
|
||||
dst_item_lens=dst_state_item_lens,
|
||||
src_page_indices=np.asarray(prefill_swa_page_indices),
|
||||
dst_page_indices=np.asarray(dst_swa_page_indices),
|
||||
dst_attn_tp_size=decode_tp_size,
|
||||
dst_tp_rank=decode_tp_rank,
|
||||
)
|
||||
logger.warning_once(
|
||||
"Using SWA state head-slice transfer for different attention TP sizes: "
|
||||
f"prefill={self.attn_tp_size}, decode={decode_tp_size}."
|
||||
)
|
||||
src_reqs, dst_reqs = [], []
|
||||
for src_addrs, dst_addrs, length in blocks:
|
||||
lengths = np.full(src_addrs.size, length, dtype=np.uint64)
|
||||
src_reqs.append(
|
||||
np.column_stack(
|
||||
[src_addrs, lengths, np.full_like(lengths, self.kv_args.gpu_id)]
|
||||
)
|
||||
)
|
||||
dst_reqs.append(
|
||||
np.column_stack([dst_addrs, lengths, np.full_like(lengths, dst_gpu_id)])
|
||||
)
|
||||
if not src_reqs:
|
||||
return None
|
||||
src_descs = self.agent.get_xfer_descs(np.vstack(src_reqs), "VRAM")
|
||||
dst_descs = self.agent.get_xfer_descs(np.vstack(dst_reqs), "VRAM")
|
||||
xfer_handle = self.agent.initialize_xfer(
|
||||
"WRITE",
|
||||
src_descs,
|
||||
dst_descs,
|
||||
peer_name,
|
||||
notif.encode("ascii"),
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("Failed to create SWA state slice transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("Failed to post SWA state slice transfer")
|
||||
return xfer_handle
|
||||
|
||||
def maybe_send_extra(
|
||||
self,
|
||||
peer_name: str,
|
||||
@@ -2743,6 +2807,24 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
state_type=st,
|
||||
force_flat=True,
|
||||
)
|
||||
elif (
|
||||
st == StateType.SWA
|
||||
and not self.is_mla_backend
|
||||
and self.attn_tp_size != decode_tp_size
|
||||
):
|
||||
h = self._send_swa_state_slice(
|
||||
peer_name=peer_name,
|
||||
prefill_swa_page_indices=src_indices,
|
||||
src_state_data_ptrs=src_ptrs,
|
||||
src_state_item_lens=src_lens,
|
||||
dst_state_data_ptrs=dst_ptrs,
|
||||
dst_swa_page_indices=dst_indices,
|
||||
dst_state_item_lens=dst_lens,
|
||||
dst_gpu_id=dst_gpu_id,
|
||||
notif=comp_notif,
|
||||
decode_tp_size=decode_tp_size,
|
||||
decode_tp_rank=decode_tp_rank,
|
||||
)
|
||||
elif st in (
|
||||
StateType.SWA,
|
||||
StateType.BLOCK_SCALE,
|
||||
|
||||
@@ -1392,6 +1392,7 @@ def setup_state_kv_args(
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
DSATokenToKVPool,
|
||||
HybridLinearKVPool,
|
||||
MHATokenToKVPool,
|
||||
MHATokenToKVPoolMXFP8,
|
||||
MiniMaxSparseKVPool,
|
||||
)
|
||||
@@ -1407,6 +1408,7 @@ def setup_state_kv_args(
|
||||
kv_args.state_layer_ids = []
|
||||
kv_args.is_hybrid_mla_backend = False
|
||||
kv_args.state_conv_shard_groups = []
|
||||
kv_args.swa_kv_cache_layout = None
|
||||
# V4's KVCache is organized by compression-ratio buckets rather than by layer.
|
||||
kv_args.mla_compression_ratios = (
|
||||
list(token_to_kv_pool.compression_ratios)
|
||||
@@ -1457,6 +1459,12 @@ def setup_state_kv_args(
|
||||
append_state_component(
|
||||
kv_args, StateType.SWA, data_ptrs, data_lens, item_lens
|
||||
)
|
||||
if isinstance(token_to_kv_pool, SWAKVPool) and isinstance(
|
||||
token_to_kv_pool.swa_kv_pool, MHATokenToKVPool
|
||||
):
|
||||
kv_args.swa_kv_cache_layout = (
|
||||
token_to_kv_pool.swa_kv_pool.kv_cache_layout
|
||||
)
|
||||
# MXFP8 KV: each sub-pool's block scales ride as their own component
|
||||
# so they inherit the index payload of the KV they describe.
|
||||
# Only the concrete SWAKVPool owns a full sub-pool; other
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Prefill TP2 -> decode TP1: two prefill ranks write disjoint KV heads per SWA page."""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
|
||||
from sglang.test.server_fixtures.disaggregation_fixture import (
|
||||
PDDisaggregationServerBase,
|
||||
configure_nixl_pd_backend,
|
||||
)
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST_MXFP4_WITH_MOE,
|
||||
try_cached_model,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=300, stage="extra-b", runner_config="4-gpu-h100")
|
||||
|
||||
SWA_SERVER_ARGS = ["--page-size", "64", "--attention-backend", "triton"]
|
||||
|
||||
|
||||
class TestDisaggregationSWAPrefillLargerTP(PDDisaggregationServerBase, GSM8KMixin):
|
||||
prefill_tp_size = 2
|
||||
decode_tp_size = 1
|
||||
decode_base_gpu_id = 2
|
||||
extra_prefill_args = SWA_SERVER_ARGS
|
||||
extra_decode_args = SWA_SERVER_ARGS
|
||||
# Same eval and floor as the equal-TP case in
|
||||
# test_disaggregation_decode_radix_cache_swa.py.
|
||||
gsm8k_score_threshold = 0.45
|
||||
gsm8k_num_examples = 500
|
||||
gsm8k_num_threads = 100
|
||||
gsm8k_num_shots = 6
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super().setUpClass()
|
||||
cls.model = try_cached_model(DEFAULT_MODEL_NAME_FOR_TEST_MXFP4_WITH_MOE)
|
||||
configure_nixl_pd_backend(cls)
|
||||
cls.launch_all()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,3 +1,4 @@
|
||||
import concurrent.futures
|
||||
import struct
|
||||
import threading
|
||||
import unittest
|
||||
@@ -37,6 +38,7 @@ from sglang.srt.disaggregation.mooncake.conn import (
|
||||
MooncakeKVSender,
|
||||
TransferInfo,
|
||||
)
|
||||
from sglang.srt.disaggregation.nixl.conn import NixlKVManager
|
||||
from sglang.srt.disaggregation.utils import (
|
||||
MetadataBuffers,
|
||||
build_transfer_entry_pairs,
|
||||
@@ -101,6 +103,7 @@ class TestMixedSWAKVBlockScales(CustomTestCase):
|
||||
MHATokenToKVPoolMXFP8 if quantized else MHATokenToKVPool
|
||||
)
|
||||
pool.get_contiguous_buf_infos = Mock(return_value=swa_infos)
|
||||
pool.kv_cache_layout = "nhd"
|
||||
if quantized:
|
||||
pool.get_kv_scale_buf_infos = Mock(return_value=scale_infos)
|
||||
setattr(target, name, pool)
|
||||
@@ -1062,6 +1065,249 @@ class TestQwen4StateWire(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
class _MemcpyNixlAgent:
|
||||
"""NIXL agent stand-in whose WRITE copies bytes within one host array."""
|
||||
|
||||
def __init__(self, memory: np.ndarray):
|
||||
self.memory = memory
|
||||
|
||||
def get_xfer_descs(self, reqs, mem_type):
|
||||
return reqs
|
||||
|
||||
def initialize_xfer(self, op, src_descs, dst_descs, peer_name, notif):
|
||||
return src_descs, dst_descs
|
||||
|
||||
def transfer(self, handle):
|
||||
for (src, length, _), (dst, dst_length, _) in zip(*handle):
|
||||
assert length == dst_length
|
||||
src, dst, length = int(src), int(dst), int(length)
|
||||
self.memory[dst : dst + length] = self.memory[src : src + length]
|
||||
return "DONE"
|
||||
|
||||
|
||||
class TestHeteroTpSwaStateTransfer(CustomTestCase):
|
||||
PAGE_SIZE = 2
|
||||
HEAD_BYTES = 4
|
||||
NUM_ENTRIES = 4 # K then V of two SWA layers
|
||||
NUM_PAGES = 6
|
||||
# The window occupies unrelated physical pages on the two sides.
|
||||
SRC_PAGES = [4, 1, 5]
|
||||
DST_PAGES = [2, 5, 0]
|
||||
# (prefill attn TP, decode attn TP, total KV heads); the last three
|
||||
# replicate KV heads across the attention TP ranks of one side.
|
||||
TOPOLOGIES = [(2, 4, 8), (4, 2, 8), (1, 4, 2), (8, 2, 4), (2, 8, 4)]
|
||||
|
||||
@staticmethod
|
||||
def _rank_heads(rank, tp_size, total_heads):
|
||||
# Model-side KV sharding: ranks beyond the head count replicate a head.
|
||||
if total_heads >= tp_size:
|
||||
per_rank = total_heads // tp_size
|
||||
return list(range(rank * per_rank, (rank + 1) * per_rank))
|
||||
return [rank // (tp_size // total_heads)]
|
||||
|
||||
@staticmethod
|
||||
def _source_ranks(prefill_tp, decode_tp, dst_rank):
|
||||
decode = object.__new__(CommonKVManager)
|
||||
decode.attn_tp_size = decode_tp
|
||||
decode.kv_args = SimpleNamespace(engine_rank=dst_rank)
|
||||
decode.is_mla_backend = decode.is_hybrid_mla_backend = False
|
||||
decode.attn_cp_size, decode.attn_cp_rank = 1, 0
|
||||
decode.pp_size, decode.pp_rank = 1, 0
|
||||
info = SimpleNamespace(attn_tp_size=prefill_tp, attn_cp_size=1, pp_size=1)
|
||||
decode._resolve_rank_mapping(info)
|
||||
return info.target_tp_ranks
|
||||
|
||||
def _send_mooncake(self, memory, prefill, dst_rank, decode_tp, dst_ptrs, item_len):
|
||||
def copy_blocks(session_id, blocks):
|
||||
for src, dst, length in blocks:
|
||||
memory[dst : dst + length] = memory[src : src + length]
|
||||
return 0
|
||||
|
||||
prefill.is_hybrid_mla_backend = False
|
||||
prefill.enable_deferred_decode_kv_release = False
|
||||
prefill._transfer_data = copy_blocks
|
||||
req = SimpleNamespace(
|
||||
mooncake_session_id="decode", dst_state_indices=[self.DST_PAGES]
|
||||
)
|
||||
dst_info = SimpleNamespace(
|
||||
dst_state_data_ptrs=[dst_ptrs],
|
||||
dst_state_item_lens=[[item_len] * len(dst_ptrs)],
|
||||
dst_state_dim_per_tensor=[[]],
|
||||
dst_state_layer_ids=[[]],
|
||||
dst_attn_tp_size=decode_tp,
|
||||
dst_tp_rank=dst_rank,
|
||||
)
|
||||
with concurrent.futures.ThreadPoolExecutor(1) as executor:
|
||||
rc = prefill.maybe_send_extra(
|
||||
req, [np.array(self.SRC_PAGES, dtype=np.int32)], executor, dst_info
|
||||
)
|
||||
self.assertEqual(rc, 0)
|
||||
|
||||
def _send_nixl(self, memory, prefill, dst_rank, decode_tp, dst_ptrs, item_len):
|
||||
prefill.agent = _MemcpyNixlAgent(memory)
|
||||
handles = prefill.maybe_send_extra(
|
||||
"decode",
|
||||
[np.array(self.SRC_PAGES, dtype=np.int32)],
|
||||
[dst_ptrs],
|
||||
[self.DST_PAGES],
|
||||
0,
|
||||
"7_state_0",
|
||||
decode_tp,
|
||||
decode_tp_rank=dst_rank,
|
||||
dst_state_item_lens=[[item_len] * len(dst_ptrs)],
|
||||
)
|
||||
self.assertEqual(len(handles), 1)
|
||||
|
||||
def _prefill_manager(
|
||||
self, *, backend, src_rank, prefill_tp, total_heads, src_ptrs, item_len
|
||||
):
|
||||
prefill = object.__new__(backend)
|
||||
prefill.attn_tp_size = prefill_tp
|
||||
prefill.is_mla_backend = False
|
||||
prefill.pp_size = 1
|
||||
prefill.kv_args = SimpleNamespace(
|
||||
engine_rank=src_rank,
|
||||
gpu_id=0,
|
||||
page_size=self.PAGE_SIZE,
|
||||
total_kv_head_num=total_heads,
|
||||
prefill_start_layer=0,
|
||||
state_types=[StateType.SWA],
|
||||
state_data_ptrs=[src_ptrs],
|
||||
state_item_lens=[[item_len] * len(src_ptrs)],
|
||||
state_dim_per_tensor=[[]],
|
||||
state_layer_ids=[[]],
|
||||
swa_kv_cache_layout="nhd",
|
||||
)
|
||||
return prefill
|
||||
|
||||
def _pool_shape(self, tp_size, total_heads):
|
||||
# [rank, entry, page, token, local head, byte]
|
||||
heads = len(self._rank_heads(rank=0, tp_size=tp_size, total_heads=total_heads))
|
||||
return (
|
||||
tp_size,
|
||||
self.NUM_ENTRIES,
|
||||
self.NUM_PAGES,
|
||||
self.PAGE_SIZE,
|
||||
heads,
|
||||
self.HEAD_BYTES,
|
||||
)
|
||||
|
||||
def _check_transfer(self, backend, send, prefill_tp, decode_tp, total_heads):
|
||||
rng = np.random.default_rng(0)
|
||||
window_shape = (self.NUM_ENTRIES, len(self.SRC_PAGES), self.PAGE_SIZE)
|
||||
window = rng.integers(
|
||||
0, 256, window_shape + (total_heads, self.HEAD_BYTES), dtype=np.uint8
|
||||
)
|
||||
src_shape = self._pool_shape(tp_size=prefill_tp, total_heads=total_heads)
|
||||
dst_shape = self._pool_shape(tp_size=decode_tp, total_heads=total_heads)
|
||||
src_bytes = int(np.prod(src_shape))
|
||||
# Prefill pages outside the window hold bytes a wrong page stride copies.
|
||||
memory = np.concatenate(
|
||||
[
|
||||
rng.integers(0, 256, src_bytes, dtype=np.uint8),
|
||||
np.zeros(int(np.prod(dst_shape)), dtype=np.uint8),
|
||||
]
|
||||
)
|
||||
src_pools = memory[:src_bytes].reshape(src_shape)
|
||||
dst_pools = memory[src_bytes:].reshape(dst_shape)
|
||||
expected = np.zeros_like(dst_pools)
|
||||
for rank in range(prefill_tp):
|
||||
heads = self._rank_heads(
|
||||
rank=rank, tp_size=prefill_tp, total_heads=total_heads
|
||||
)
|
||||
src_pools[rank][:, self.SRC_PAGES] = window[..., heads, :]
|
||||
for rank in range(decode_tp):
|
||||
heads = self._rank_heads(
|
||||
rank=rank, tp_size=decode_tp, total_heads=total_heads
|
||||
)
|
||||
expected[rank][:, self.DST_PAGES] = window[..., heads, :]
|
||||
|
||||
src_item_len = int(np.prod(src_shape[3:]))
|
||||
dst_item_len = int(np.prod(dst_shape[3:]))
|
||||
for dst_rank in range(decode_tp):
|
||||
dst_ptrs = [
|
||||
src_bytes
|
||||
+ (dst_rank * self.NUM_ENTRIES + e) * self.NUM_PAGES * dst_item_len
|
||||
for e in range(self.NUM_ENTRIES)
|
||||
]
|
||||
for src_rank in self._source_ranks(
|
||||
prefill_tp=prefill_tp, decode_tp=decode_tp, dst_rank=dst_rank
|
||||
):
|
||||
src_ptrs = [
|
||||
(src_rank * self.NUM_ENTRIES + e) * self.NUM_PAGES * src_item_len
|
||||
for e in range(self.NUM_ENTRIES)
|
||||
]
|
||||
prefill = self._prefill_manager(
|
||||
backend=backend,
|
||||
src_rank=src_rank,
|
||||
prefill_tp=prefill_tp,
|
||||
total_heads=total_heads,
|
||||
src_ptrs=src_ptrs,
|
||||
item_len=src_item_len,
|
||||
)
|
||||
send(
|
||||
memory=memory,
|
||||
prefill=prefill,
|
||||
dst_rank=dst_rank,
|
||||
decode_tp=decode_tp,
|
||||
dst_ptrs=dst_ptrs,
|
||||
item_len=dst_item_len,
|
||||
)
|
||||
|
||||
np.testing.assert_array_equal(dst_pools, expected)
|
||||
|
||||
def test_decode_ranks_receive_their_heads_of_the_window(self):
|
||||
for backend, send in (
|
||||
(MooncakeKVManager, self._send_mooncake),
|
||||
(NixlKVManager, self._send_nixl),
|
||||
):
|
||||
for prefill_tp, decode_tp, total_heads in self.TOPOLOGIES:
|
||||
with self.subTest(
|
||||
backend=backend.__name__,
|
||||
prefill_tp=prefill_tp,
|
||||
decode_tp=decode_tp,
|
||||
total_heads=total_heads,
|
||||
):
|
||||
self._check_transfer(
|
||||
backend=backend,
|
||||
send=send,
|
||||
prefill_tp=prefill_tp,
|
||||
decode_tp=decode_tp,
|
||||
total_heads=total_heads,
|
||||
)
|
||||
|
||||
def test_unsliceable_layouts_are_rejected(self):
|
||||
prefill = object.__new__(CommonKVManager)
|
||||
prefill.attn_tp_size = 2
|
||||
prefill.kv_args = SimpleNamespace(
|
||||
engine_rank=0, page_size=2, total_kv_head_num=8, prefill_start_layer=0
|
||||
)
|
||||
for pp_size, layout, src_item_lens, dst_item_lens, message in (
|
||||
# SWA state has no layer ids, so a later PP stage would land on the
|
||||
# decode peer's first SWA layers.
|
||||
(2, "nhd", [16, 16], [8, 8], "pipeline parallelism"),
|
||||
(1, "nhd", [64], [32], "per-layer K/V"),
|
||||
# One SWA head per rank on both sides, against 8 total KV heads.
|
||||
(1, "nhd", [8, 8], [8, 8], "do not split"),
|
||||
# Item lengths that are valid NHD, so only the layout can reject.
|
||||
(1, "vectorized_5d", [16, 16], [8, 8], "NHD KV cache layout"),
|
||||
):
|
||||
prefill.pp_size = pp_size
|
||||
prefill.kv_args.swa_kv_cache_layout = layout
|
||||
with self.subTest(message=message):
|
||||
with self.assertRaisesRegex(RuntimeError, message):
|
||||
prefill._get_mha_head_slice_blocks(
|
||||
src_ptrs=[0x1000 * i for i in range(len(src_item_lens))],
|
||||
src_item_lens=src_item_lens,
|
||||
dst_ptrs=[0x9000 * i for i in range(len(dst_item_lens))],
|
||||
dst_item_lens=dst_item_lens,
|
||||
src_page_indices=np.array([1]),
|
||||
dst_page_indices=np.array([2]),
|
||||
dst_attn_tp_size=4,
|
||||
dst_tp_rank=0,
|
||||
)
|
||||
|
||||
|
||||
class TestMooncakeTransferInfoIsDummy(unittest.TestCase):
|
||||
"""Truth table for mooncake's payload-inferred is_dummy, with frames built
|
||||
as KVSender sends them: kv and aux are empty iff dummy, state indices are
|
||||
|
||||
@@ -9,6 +9,8 @@ draft pool exists.
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.srt.disaggregation.mooncake.conn import MooncakeKVManager
|
||||
from sglang.srt.disaggregation.nixl.conn import NixlKVManager
|
||||
from sglang.srt.disaggregation.utils import (
|
||||
build_staging_slot_metadata,
|
||||
build_transfer_entry_pairs,
|
||||
@@ -50,6 +52,15 @@ class TestStagingDraftKvSlots(CustomTestCase):
|
||||
self.assertEqual(slot_ids, [87, 91, 92, 87, 91, 92])
|
||||
self.assertNotEqual(slot_ids, _kv_layer_ids(target, draft))
|
||||
|
||||
def test_every_backend_setter_accepts_slot_layer_ids(self):
|
||||
# Shared staging init passes slot_layer_ids to every backend's setter.
|
||||
for backend in (MooncakeKVManager, NixlKVManager):
|
||||
with self.subTest(backend=backend.__name__):
|
||||
manager = object.__new__(backend)
|
||||
manager.set_kv_buffer_tensors(
|
||||
["tK87"], ["tV87"], 16, slot_layer_ids=[87, 87]
|
||||
)
|
||||
|
||||
def test_without_draft_matches_kv_layer_ids(self):
|
||||
# The two orders coincide with no draft pool, so every deployment that
|
||||
# predates draft KV must keep its exact slot labelling.
|
||||
|
||||
Reference in New Issue
Block a user