[PD] Support heterogeneous TP for hybrid SWA KV transfer (#40909)
This commit is contained in:
@@ -81,6 +81,9 @@ class KVArgs:
|
||||
# scatter transfer can slice each independently head-sharded sub-block; None
|
||||
# per tensor when the single contiguous slice already matches the layout.
|
||||
state_conv_shard_groups: List[List[Optional[List[int]]]]
|
||||
# Physical layout of the SWA sub-pool ("nhd", "hnd" or "vectorized_5d");
|
||||
# None when the SWA state is not held in an MHA sub-pool.
|
||||
swa_kv_cache_layout: Optional[str]
|
||||
ib_device: str
|
||||
gpu_id: int
|
||||
kv_head_num: int
|
||||
|
||||
@@ -1230,6 +1230,26 @@ class CommonKVManager(BaseKVManager):
|
||||
"servers (including prefill CP ranks for MLA)"
|
||||
)
|
||||
|
||||
# Every send path rejects head-sharded MXFP8 block scales across
|
||||
# attention TP sizes, so each request would fail at its first transfer.
|
||||
if (
|
||||
info.attn_tp_size != self.attn_tp_size
|
||||
and not self.is_mla_backend
|
||||
and not self.is_hybrid_mla_backend
|
||||
):
|
||||
block_scale_states = [
|
||||
st
|
||||
for st in getattr(self.kv_args, "state_types", None) or []
|
||||
if st in (StateType.BLOCK_SCALE, StateType.BLOCK_SCALE_SWA)
|
||||
]
|
||||
if block_scale_states:
|
||||
raise RuntimeError(
|
||||
"PD Disaggregation does NOT support different attention TP "
|
||||
"sizes with MXFP8 KV cache: prefill attn_tp_size="
|
||||
f"{info.attn_tp_size}, decode attn_tp_size={self.attn_tp_size}, "
|
||||
f"state types {[st.value for st in block_scale_states]}."
|
||||
)
|
||||
|
||||
if self.dcp_size > 1:
|
||||
if not (self.is_mla_backend or self.is_hybrid_mla_backend):
|
||||
raise RuntimeError(
|
||||
@@ -1541,6 +1561,108 @@ class CommonKVManager(BaseKVManager):
|
||||
layers_current_pp_stage = len(src_k_ptrs)
|
||||
return src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, layers_current_pp_stage
|
||||
|
||||
def _get_mha_head_slice_blocks(
|
||||
self,
|
||||
src_ptrs: List[int],
|
||||
src_item_lens: List[int],
|
||||
dst_ptrs: List[int],
|
||||
dst_item_lens: List[int],
|
||||
src_page_indices: npt.NDArray[np.int32],
|
||||
dst_page_indices: npt.NDArray[np.int32],
|
||||
dst_attn_tp_size: int,
|
||||
dst_tp_rank: int,
|
||||
) -> List[Tuple[npt.NDArray[np.uint64], npt.NDArray[np.uint64], int]]:
|
||||
# SWA state has no layer ids and its pool starts every PP stage at layer
|
||||
# 0, so a prefill stage's entries cannot be paired with the decode layers.
|
||||
if self.pp_size > 1:
|
||||
raise RuntimeError(
|
||||
"PD Disaggregation does NOT support PD different TP sizes for "
|
||||
"non-MLA SWA hybrid models with pipeline parallelism yet."
|
||||
)
|
||||
# The addresses below assume NHD pages [page_size, heads, head_dim]; HND
|
||||
# and vectorized_5d pages have the same size in another element order.
|
||||
if self.kv_args.swa_kv_cache_layout != "nhd":
|
||||
raise RuntimeError(
|
||||
"Head-sliced PD transfer needs the NHD KV cache layout, got "
|
||||
f"{self.kv_args.swa_kv_cache_layout}"
|
||||
)
|
||||
from sglang.srt.disaggregation.common.staging_buffer import (
|
||||
compute_head_slice_params,
|
||||
resolve_total_kv_heads,
|
||||
)
|
||||
|
||||
src_k_ptrs, src_v_ptrs, dst_k_ptrs, dst_v_ptrs, num_layers = (
|
||||
self.get_mha_kv_ptrs_with_pp(src_ptrs, dst_ptrs)
|
||||
)
|
||||
_, _, dst_k_lens, dst_v_lens, _ = self.get_mha_kv_ptrs_with_pp(
|
||||
src_item_lens, dst_item_lens
|
||||
)
|
||||
entries = list(
|
||||
zip(
|
||||
src_k_ptrs + src_v_ptrs,
|
||||
dst_k_ptrs + dst_v_ptrs,
|
||||
src_item_lens,
|
||||
dst_k_lens + dst_v_lens,
|
||||
)
|
||||
)
|
||||
# A single page-envelope region (unified memory) has no per-layer K/V
|
||||
# split to slice heads from.
|
||||
if len(entries) != len(src_ptrs) or 2 * num_layers != len(src_ptrs):
|
||||
raise RuntimeError(
|
||||
"Head-sliced PD transfer needs matching per-layer K/V buffers: "
|
||||
f"prefill has {len(src_ptrs)} entries, decode has {len(dst_ptrs)}"
|
||||
)
|
||||
if len(src_page_indices) != len(dst_page_indices):
|
||||
raise RuntimeError(
|
||||
"Head-sliced PD transfer page count mismatch: "
|
||||
f"prefill={len(src_page_indices)}, decode={len(dst_page_indices)}"
|
||||
)
|
||||
|
||||
total_kv_heads = resolve_total_kv_heads(self.kv_args, self.attn_tp_size)
|
||||
src_head_start, num_heads, dst_head_start, _ = compute_head_slice_params(
|
||||
src_attn_tp_size=self.attn_tp_size,
|
||||
dst_attn_tp_size=dst_attn_tp_size,
|
||||
src_tp_rank=self.kv_args.engine_rank,
|
||||
dst_tp_rank=dst_tp_rank,
|
||||
total_kv_heads=total_kv_heads,
|
||||
)
|
||||
src_heads = max(1, total_kv_heads // self.attn_tp_size)
|
||||
dst_heads = max(1, total_kv_heads // dst_attn_tp_size)
|
||||
page_size = self.kv_args.page_size
|
||||
tokens = np.arange(page_size, dtype=np.uint64)
|
||||
src_pages = np.asarray(src_page_indices, dtype=np.uint64)[:, None]
|
||||
dst_pages = np.asarray(dst_page_indices, dtype=np.uint64)[:, None]
|
||||
|
||||
blocks = []
|
||||
for src_ptr, dst_ptr, src_item_len, dst_item_len in entries:
|
||||
head_len = src_item_len // (page_size * src_heads)
|
||||
# Both peers must hold whole heads of one size, or this state is
|
||||
# sharded differently from total_kv_head_num.
|
||||
if (
|
||||
head_len * page_size * src_heads != src_item_len
|
||||
or head_len * page_size * dst_heads != dst_item_len
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"KV item lengths (prefill={src_item_len}, decode={dst_item_len}) "
|
||||
f"do not split into {src_heads} and {dst_heads} heads of "
|
||||
f"{total_kv_heads} total with page_size={page_size}"
|
||||
)
|
||||
# Each page item is [page_size, heads, head_dim]: one block per token.
|
||||
src_addrs = (
|
||||
np.uint64(src_ptr)
|
||||
+ src_pages * np.uint64(src_item_len)
|
||||
+ tokens * np.uint64(src_heads * head_len)
|
||||
+ np.uint64(src_head_start * head_len)
|
||||
)
|
||||
dst_addrs = (
|
||||
np.uint64(dst_ptr)
|
||||
+ dst_pages * np.uint64(dst_item_len)
|
||||
+ tokens * np.uint64(dst_heads * head_len)
|
||||
+ np.uint64(dst_head_start * head_len)
|
||||
)
|
||||
blocks.append((src_addrs.ravel(), dst_addrs.ravel(), num_heads * head_len))
|
||||
return blocks
|
||||
|
||||
def get_mla_kv_ptrs_with_pp(
|
||||
self,
|
||||
src_kv_ptrs: List[int],
|
||||
|
||||
@@ -1966,12 +1966,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
and self.attn_tp_size
|
||||
!= target_rank_registration_info.dst_attn_tp_size
|
||||
)
|
||||
if (
|
||||
slice_kv_heads = (
|
||||
has_heterogeneous_attn_tp
|
||||
and not self.is_mla_backend
|
||||
and not self.is_hybrid_mla_backend
|
||||
and not is_qwen4_qsa_state
|
||||
):
|
||||
)
|
||||
if slice_kv_heads and st != StateType.SWA:
|
||||
raise RuntimeError(
|
||||
f"PD Disaggregation does NOT support PD different TP sizes for non-MLA {st.upper()} hybrid models yet."
|
||||
)
|
||||
@@ -2038,6 +2039,25 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
src_indices = src_indices[: len(dst_indices_local)]
|
||||
else:
|
||||
dst_indices_local = dst_indices_local[: len(src_indices)]
|
||||
if slice_kv_heads:
|
||||
rc = (
|
||||
self._send_swa_state_slice(
|
||||
req=req,
|
||||
prefill_swa_page_indices=src_indices,
|
||||
src_state_data_ptrs=src_data_ptrs,
|
||||
src_state_item_lens=src_item_lens,
|
||||
dst_state_data_ptrs=dst_data_ptrs,
|
||||
dst_swa_page_indices=dst_indices_local,
|
||||
dst_state_item_lens=dst_item_lens,
|
||||
dst_tp_rank=target_rank_registration_info.dst_tp_rank,
|
||||
dst_attn_tp_size=(
|
||||
target_rank_registration_info.dst_attn_tp_size
|
||||
),
|
||||
executor=executor,
|
||||
)
|
||||
or rc
|
||||
)
|
||||
continue
|
||||
rc = (
|
||||
self._send_kvcache_generic(
|
||||
mooncake_session_id=req.mooncake_session_id,
|
||||
@@ -2292,6 +2312,49 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
|
||||
return self._transfer_data(req.mooncake_session_id, transfer_blocks)
|
||||
|
||||
def _send_swa_state_slice(
|
||||
self,
|
||||
req: TransferInfo,
|
||||
prefill_swa_page_indices: List[int],
|
||||
src_state_data_ptrs: List[int],
|
||||
src_state_item_lens: List[int],
|
||||
dst_state_data_ptrs: List[int],
|
||||
dst_swa_page_indices: List[int],
|
||||
dst_state_item_lens: List[int],
|
||||
dst_tp_rank: int,
|
||||
dst_attn_tp_size: int,
|
||||
executor: concurrent.futures.ThreadPoolExecutor,
|
||||
) -> int:
|
||||
blocks = self._get_mha_head_slice_blocks(
|
||||
src_ptrs=src_state_data_ptrs,
|
||||
src_item_lens=src_state_item_lens,
|
||||
dst_ptrs=dst_state_data_ptrs,
|
||||
dst_item_lens=dst_state_item_lens,
|
||||
src_page_indices=np.asarray(prefill_swa_page_indices),
|
||||
dst_page_indices=np.asarray(dst_swa_page_indices),
|
||||
dst_attn_tp_size=dst_attn_tp_size,
|
||||
dst_tp_rank=dst_tp_rank,
|
||||
)
|
||||
logger.warning_once(
|
||||
"Using SWA state head-slice transfer for different attention TP sizes: "
|
||||
f"prefill={self.attn_tp_size}, decode={dst_attn_tp_size}."
|
||||
)
|
||||
futures = [
|
||||
executor.submit(
|
||||
self._transfer_data,
|
||||
req.mooncake_session_id,
|
||||
list(
|
||||
zip(
|
||||
src_addrs.tolist(),
|
||||
dst_addrs.tolist(),
|
||||
[length] * src_addrs.size,
|
||||
)
|
||||
),
|
||||
)
|
||||
for src_addrs, dst_addrs, length in blocks
|
||||
]
|
||||
return self._await_transfer_futures(futures)
|
||||
|
||||
def _poison_unsettled_send(self, room: int) -> None:
|
||||
"""Keep the drain ack from firing after a submitted transfer failed.
|
||||
|
||||
|
||||
@@ -621,7 +621,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
f"(ptr=0x{ptr:x}, size={size})"
|
||||
)
|
||||
|
||||
def set_kv_buffer_tensors(self, k_buffers: list, v_buffers: list, page_size: int):
|
||||
def set_kv_buffer_tensors(
|
||||
self,
|
||||
k_buffers: list,
|
||||
v_buffers: list,
|
||||
page_size: int,
|
||||
slot_layer_ids: Optional[List[int]] = None,
|
||||
):
|
||||
# NOTE: matches mooncake behavior -- staging buffers are now
|
||||
# created in __init__ (per-worker), independent of the kv
|
||||
# tensors. This setter only stashes the tensor metadata used by
|
||||
@@ -630,6 +636,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
"k_buffers": k_buffers,
|
||||
"v_buffers": v_buffers,
|
||||
"page_size": page_size,
|
||||
"slot_layer_ids": list(slot_layer_ids or []),
|
||||
}
|
||||
|
||||
def register_staging_room_bootstrap(self, room, bootstrap_infos, receiver):
|
||||
@@ -2608,6 +2615,63 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
raise Exception("Failed to post Mamba state slice transfer")
|
||||
return xfer_handle
|
||||
|
||||
def _send_swa_state_slice(
|
||||
self,
|
||||
peer_name: str,
|
||||
prefill_swa_page_indices: List[int],
|
||||
src_state_data_ptrs: List[int],
|
||||
src_state_item_lens: List[int],
|
||||
dst_state_data_ptrs: List[int],
|
||||
dst_swa_page_indices: List[int],
|
||||
dst_state_item_lens: List[int],
|
||||
dst_gpu_id: int,
|
||||
notif: str,
|
||||
decode_tp_size: int,
|
||||
decode_tp_rank: int,
|
||||
):
|
||||
blocks = self._get_mha_head_slice_blocks(
|
||||
src_ptrs=src_state_data_ptrs,
|
||||
src_item_lens=src_state_item_lens,
|
||||
dst_ptrs=dst_state_data_ptrs,
|
||||
dst_item_lens=dst_state_item_lens,
|
||||
src_page_indices=np.asarray(prefill_swa_page_indices),
|
||||
dst_page_indices=np.asarray(dst_swa_page_indices),
|
||||
dst_attn_tp_size=decode_tp_size,
|
||||
dst_tp_rank=decode_tp_rank,
|
||||
)
|
||||
logger.warning_once(
|
||||
"Using SWA state head-slice transfer for different attention TP sizes: "
|
||||
f"prefill={self.attn_tp_size}, decode={decode_tp_size}."
|
||||
)
|
||||
src_reqs, dst_reqs = [], []
|
||||
for src_addrs, dst_addrs, length in blocks:
|
||||
lengths = np.full(src_addrs.size, length, dtype=np.uint64)
|
||||
src_reqs.append(
|
||||
np.column_stack(
|
||||
[src_addrs, lengths, np.full_like(lengths, self.kv_args.gpu_id)]
|
||||
)
|
||||
)
|
||||
dst_reqs.append(
|
||||
np.column_stack([dst_addrs, lengths, np.full_like(lengths, dst_gpu_id)])
|
||||
)
|
||||
if not src_reqs:
|
||||
return None
|
||||
src_descs = self.agent.get_xfer_descs(np.vstack(src_reqs), "VRAM")
|
||||
dst_descs = self.agent.get_xfer_descs(np.vstack(dst_reqs), "VRAM")
|
||||
xfer_handle = self.agent.initialize_xfer(
|
||||
"WRITE",
|
||||
src_descs,
|
||||
dst_descs,
|
||||
peer_name,
|
||||
notif.encode("ascii"),
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise Exception("Failed to create SWA state slice transfer")
|
||||
state = self.agent.transfer(xfer_handle)
|
||||
if state == "ERR":
|
||||
raise Exception("Failed to post SWA state slice transfer")
|
||||
return xfer_handle
|
||||
|
||||
def maybe_send_extra(
|
||||
self,
|
||||
peer_name: str,
|
||||
@@ -2743,6 +2807,24 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
state_type=st,
|
||||
force_flat=True,
|
||||
)
|
||||
elif (
|
||||
st == StateType.SWA
|
||||
and not self.is_mla_backend
|
||||
and self.attn_tp_size != decode_tp_size
|
||||
):
|
||||
h = self._send_swa_state_slice(
|
||||
peer_name=peer_name,
|
||||
prefill_swa_page_indices=src_indices,
|
||||
src_state_data_ptrs=src_ptrs,
|
||||
src_state_item_lens=src_lens,
|
||||
dst_state_data_ptrs=dst_ptrs,
|
||||
dst_swa_page_indices=dst_indices,
|
||||
dst_state_item_lens=dst_lens,
|
||||
dst_gpu_id=dst_gpu_id,
|
||||
notif=comp_notif,
|
||||
decode_tp_size=decode_tp_size,
|
||||
decode_tp_rank=decode_tp_rank,
|
||||
)
|
||||
elif st in (
|
||||
StateType.SWA,
|
||||
StateType.BLOCK_SCALE,
|
||||
|
||||
@@ -1392,6 +1392,7 @@ def setup_state_kv_args(
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
DSATokenToKVPool,
|
||||
HybridLinearKVPool,
|
||||
MHATokenToKVPool,
|
||||
MHATokenToKVPoolMXFP8,
|
||||
MiniMaxSparseKVPool,
|
||||
)
|
||||
@@ -1407,6 +1408,7 @@ def setup_state_kv_args(
|
||||
kv_args.state_layer_ids = []
|
||||
kv_args.is_hybrid_mla_backend = False
|
||||
kv_args.state_conv_shard_groups = []
|
||||
kv_args.swa_kv_cache_layout = None
|
||||
# V4's KVCache is organized by compression-ratio buckets rather than by layer.
|
||||
kv_args.mla_compression_ratios = (
|
||||
list(token_to_kv_pool.compression_ratios)
|
||||
@@ -1457,6 +1459,12 @@ def setup_state_kv_args(
|
||||
append_state_component(
|
||||
kv_args, StateType.SWA, data_ptrs, data_lens, item_lens
|
||||
)
|
||||
if isinstance(token_to_kv_pool, SWAKVPool) and isinstance(
|
||||
token_to_kv_pool.swa_kv_pool, MHATokenToKVPool
|
||||
):
|
||||
kv_args.swa_kv_cache_layout = (
|
||||
token_to_kv_pool.swa_kv_pool.kv_cache_layout
|
||||
)
|
||||
# MXFP8 KV: each sub-pool's block scales ride as their own component
|
||||
# so they inherit the index payload of the KV they describe.
|
||||
# Only the concrete SWAKVPool owns a full sub-pool; other
|
||||
|
||||
Reference in New Issue
Block a user