[PD] Share one head-slice helper across mooncake, mori, and nixl (#39660)
Co-authored-by: BBuf <1182563586@qq.com>
This commit is contained in:
@@ -1359,10 +1359,12 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
each page to ensure correctness for any page_size and head-slicing configuration.
|
||||
This may introduce performance overhead (increased TTFT) for long sequences.
|
||||
"""
|
||||
from sglang.srt.disaggregation.common.staging_buffer import (
|
||||
compute_head_slice_params,
|
||||
)
|
||||
|
||||
# Extract configuration
|
||||
local_tp_rank_in_group = self.kv_args.engine_rank % self.attn_tp_size
|
||||
src_kv_item_len = self.kv_args.kv_item_lens[0]
|
||||
dst_tp_rank_in_group = dst_tp_rank % dst_attn_tp_size
|
||||
page_size = self.kv_args.page_size
|
||||
|
||||
# Use total KV head count (not per-rank) for correct head distribution.
|
||||
@@ -1371,36 +1373,23 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
if total_kv_heads <= 0:
|
||||
total_kv_heads = self.kv_args.kv_head_num * self.attn_tp_size
|
||||
|
||||
src_heads_per_rank = max(1, total_kv_heads // self.attn_tp_size)
|
||||
dst_heads_per_rank = max(1, total_kv_heads // dst_attn_tp_size)
|
||||
bytes_per_head_slice_to_send = (
|
||||
dst_kv_item_len // page_size // dst_heads_per_rank
|
||||
)
|
||||
|
||||
# GQA replication: how many prefill ranks share the same KV head
|
||||
src_replication = max(1, self.attn_tp_size // total_kv_heads)
|
||||
|
||||
# Determine slicing parameters based on TP configuration
|
||||
if self.attn_tp_size > dst_attn_tp_size:
|
||||
# Send KVCache from multiple prefill instances to 1 decode instance
|
||||
src_head_start_offset = 0
|
||||
num_heads_to_send = src_heads_per_rank
|
||||
unique_head_idx = local_tp_rank_in_group // src_replication
|
||||
dst_head_start_offset = (
|
||||
unique_head_idx * src_heads_per_rank
|
||||
) % dst_heads_per_rank
|
||||
else:
|
||||
# Send KVCache from 1 prefill instance to multiple decode instances
|
||||
# GQA replication (total_kv_heads < dst_attn_tp_size): consecutive decode
|
||||
# ranks share one KV head (QKVParallelLinear: tp_rank // num_kv_head_replicas),
|
||||
# so map by integer division NOT modulo or ranks 1..r-1 fetch the wrong head.
|
||||
dst_replication = max(1, dst_attn_tp_size // total_kv_heads)
|
||||
unique_dst_head_idx = dst_tp_rank_in_group // dst_replication
|
||||
src_head_start_offset = (
|
||||
unique_dst_head_idx * dst_heads_per_rank
|
||||
) % src_heads_per_rank
|
||||
num_heads_to_send = dst_heads_per_rank
|
||||
dst_head_start_offset = 0
|
||||
(
|
||||
src_head_start_offset,
|
||||
num_heads_to_send,
|
||||
dst_head_start_offset,
|
||||
_,
|
||||
) = compute_head_slice_params(
|
||||
self.attn_tp_size,
|
||||
dst_attn_tp_size,
|
||||
self.kv_args.engine_rank,
|
||||
dst_tp_rank,
|
||||
total_kv_heads,
|
||||
)
|
||||
|
||||
src_data_ptrs = self.kv_args.kv_data_ptrs
|
||||
src_layer_ids = self.kv_args.kv_layer_ids
|
||||
|
||||
@@ -931,6 +931,10 @@ class MoriKVManager(CommonKVManager):
|
||||
return grouped_plan.materialize(item_len)
|
||||
|
||||
def _build_tp_slice_config(self, peer_info: KVArgsRegisterInfo) -> TPSliceConfig:
|
||||
from sglang.srt.disaggregation.common.staging_buffer import (
|
||||
compute_head_slice_params,
|
||||
)
|
||||
|
||||
page_size = self.kv_args.page_size
|
||||
|
||||
src_item_len = self.kv_args.kv_item_lens[0]
|
||||
@@ -946,27 +950,21 @@ class MoriKVManager(CommonKVManager):
|
||||
if total_kv_heads <= 0:
|
||||
total_kv_heads = self.kv_args.kv_head_num * prefill_tp_size
|
||||
|
||||
src_heads_per_rank = max(1, total_kv_heads // prefill_tp_size)
|
||||
dst_heads_per_rank = max(1, total_kv_heads // decode_tp_size)
|
||||
|
||||
bytes_per_head_slice = bytes_per_token_dst // dst_heads_per_rank
|
||||
if bytes_per_head_slice == 0:
|
||||
raise ValueError("Head slice size evaluates to zero")
|
||||
|
||||
src_replication = max(1, prefill_tp_size // total_kv_heads)
|
||||
|
||||
local_tp_rank = self.kv_args.engine_rank % prefill_tp_size
|
||||
dst_tp_rank = peer_info.decode_tp_rank % decode_tp_size
|
||||
|
||||
if prefill_tp_size > decode_tp_size:
|
||||
src_head_start = 0
|
||||
num_heads_to_send = src_heads_per_rank
|
||||
unique_head_idx = local_tp_rank // src_replication
|
||||
dst_head_start = (unique_head_idx * src_heads_per_rank) % dst_heads_per_rank
|
||||
else:
|
||||
src_head_start = (dst_tp_rank * dst_heads_per_rank) % src_heads_per_rank
|
||||
num_heads_to_send = dst_heads_per_rank
|
||||
dst_head_start = 0
|
||||
src_head_start, num_heads_to_send, dst_head_start, _ = (
|
||||
compute_head_slice_params(
|
||||
prefill_tp_size,
|
||||
decode_tp_size,
|
||||
self.kv_args.engine_rank,
|
||||
peer_info.decode_tp_rank,
|
||||
total_kv_heads,
|
||||
)
|
||||
)
|
||||
|
||||
src_head_slice_offset = src_head_start * bytes_per_head_slice
|
||||
dst_head_slice_offset = dst_head_start * bytes_per_head_slice
|
||||
|
||||
@@ -834,6 +834,10 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
interleave num_groups per token, peers select via head_group_idx.
|
||||
prefill_tp > decode_tp: num_groups=1. Dst dlist is per-peer.
|
||||
"""
|
||||
from sglang.srt.disaggregation.common.staging_buffer import (
|
||||
compute_head_slice_params,
|
||||
)
|
||||
|
||||
decode_tp_size = decode_kv_args.decode_tp_size
|
||||
dst_kv_item_len = decode_kv_args.dst_kv_item_len
|
||||
prefill_tp_size = self.attn_tp_size
|
||||
@@ -844,39 +848,25 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
if total_kv_heads <= 0:
|
||||
total_kv_heads = self.kv_args.kv_head_num * prefill_tp_size
|
||||
|
||||
src_heads_per_rank = max(1, total_kv_heads // prefill_tp_size)
|
||||
dst_heads_per_rank = max(1, total_kv_heads // decode_tp_size)
|
||||
bytes_per_head_slice = dst_kv_item_len // page_size // dst_heads_per_rank
|
||||
|
||||
if prefill_tp_size > decode_tp_size:
|
||||
# Multiple prefill ranks feed one decode rank: each prefill rank sends
|
||||
# all its src heads to a specific head-range in the decode rank.
|
||||
src_replication = max(1, prefill_tp_size // total_kv_heads)
|
||||
local_tp_rank_in_group = self.kv_args.engine_rank % prefill_tp_size
|
||||
_, num_heads_to_send, dst_head_start, _ = compute_head_slice_params(
|
||||
prefill_tp_size,
|
||||
decode_tp_size,
|
||||
self.kv_args.engine_rank,
|
||||
decode_kv_args.decode_tp_rank,
|
||||
total_kv_heads,
|
||||
)
|
||||
num_groups = 1
|
||||
num_heads_to_send = src_heads_per_rank
|
||||
head_group_idx = 0
|
||||
unique_head_idx = local_tp_rank_in_group // src_replication
|
||||
dst_head_start = (unique_head_idx * src_heads_per_rank) % dst_heads_per_rank
|
||||
dst_head_offset = dst_head_start * bytes_per_head_slice
|
||||
else:
|
||||
# One prefill rank feeds multiple decode ranks: interleave num_groups
|
||||
# head-groups in the src dlist so each decode rank picks its slice.
|
||||
#
|
||||
# Under GQA the decode side can have MORE attn-TP ranks than there are
|
||||
# KV heads (decode_tp_size > total_kv_heads). In that case consecutive
|
||||
# decode ranks replicate a shared KV head, so the src dlist must
|
||||
# interleave one group per UNIQUE source head-slice, not one per decode
|
||||
# rank -- otherwise it addresses past the registered KV region and
|
||||
# prep_xfer_dlist raises NIXL_ERR_NOT_FOUND.
|
||||
#
|
||||
# Reuse the shared replicated-KV head map (integer division under
|
||||
# replication, not modulo) that the mooncake backend already relies
|
||||
# on, so the two backends stay in sync.
|
||||
from sglang.srt.disaggregation.common.staging_buffer import (
|
||||
compute_head_slice_params,
|
||||
)
|
||||
|
||||
src_head_start, num_heads_to_send, _, _ = compute_head_slice_params(
|
||||
prefill_tp_size,
|
||||
decode_tp_size,
|
||||
@@ -884,9 +874,9 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
decode_kv_args.decode_tp_rank,
|
||||
total_kv_heads,
|
||||
)
|
||||
# num_groups (distinct head-groups packed in one prefill rank's src
|
||||
# region) and head_group_idx (this peer's group) are NIXL-specific and
|
||||
# not returned by the shared helper, so derive them here.
|
||||
# One group per UNIQUE head-slice, not per decode rank: replicating
|
||||
# ranks share one, and over-counting addresses past the registered
|
||||
# KV region, where prep_xfer_dlist raises NIXL_ERR_NOT_FOUND.
|
||||
dst_replication = max(1, decode_tp_size // total_kv_heads)
|
||||
num_groups = decode_tp_size // prefill_tp_size // dst_replication
|
||||
head_group_idx = src_head_start // dst_heads_per_rank
|
||||
|
||||
Reference in New Issue
Block a user