[Feat] KVShard: PD transfer and e2e test (#38473)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
Shunkangz
2026-10-20 22:48:43 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 Zhangheng
parent 7656071309
commit eb4b167d27
7 changed files with 446 additions and 6 deletions
@@ -37,9 +37,11 @@ from sglang.srt.disaggregation.pp_consensus_store import PPConsensusStore
from sglang.srt.disaggregation.utils import (
DisaggregationMode,
filter_kv_indices_for_cp_rank,
filter_kv_indices_for_shard_rank,
get_dsv41_spec_layout,
)
from sglang.srt.environ import envs
from sglang.srt.mem_cache.page_interleave import get_kv_shard_group
from sglang.srt.runtime_context import (
get_disagg,
get_parallel,
@@ -116,6 +118,8 @@ class PrefillServerInfo:
dsv41_spec_layout: Optional[dict] = None
kv_layout: Optional[str] = None
decode_allocation_policy: str = "early"
# Each prefill shard-group rank owns a subset of logical KV pages.
enable_kv_cache_sharding: bool = False
# PD true-retraction rebootstrap: the prefill's HTTP API port. The decode
# already knows the prefill host (the bootstrap_addr host), so it can POST
@@ -282,6 +286,11 @@ class CommonKVManager(BaseKVManager):
kv_layout: Optional[str] = None
dsv41_spec_layout: Optional[dict] = None
# Logical-page KV sharding (PD prefill only): this rank's slot in the
# shard group. Each rank owns a subset of logical KV pages.
kv_shard_rank: int = 0
kv_shard_size: int = 1
# Used by decode when the prefill reported Failed without a reason frame.
DEFAULT_PREFILL_FAILURE_REASON = (
"Failed to get kvcache from prefill instance, it might be dead"
@@ -340,6 +349,10 @@ class CommonKVManager(BaseKVManager):
self.pp_size = get_parallel().pp_size
self.pp_rank = self.kv_args.pp_rank
self.local_ip = get_local_ip_auto()
if get_parallel().enable_kv_cache_sharding:
kv_shard_group = get_kv_shard_group(is_mla_backend)
self.kv_shard_rank = kv_shard_group.rank_in_group
self.kv_shard_size = kv_shard_group.world_size
cp_sharded_prefill = self.attn_cp_size > 1 and (
self.is_hybrid_mla_backend or get_parallel().enable_dsa_cache_layer_split
)
@@ -356,6 +369,8 @@ class CommonKVManager(BaseKVManager):
envs.SGLANG_DISAGGREGATION_ALL_CP_RANKS_TRANSFER.get()
or cp_sharded_prefill
or hybrid_decode_pulls_all_ranks
# No sharded rank holds a full page, so every CP rank sends.
or self.kv_shard_size > 1
)
# bind zmq socket
@@ -1286,6 +1301,11 @@ class CommonKVManager(BaseKVManager):
"PD decode DCP currently requires prefill attention CP=1, "
f"got {info.attn_cp_size}."
)
if info.enable_kv_cache_sharding:
raise RuntimeError(
"PD decode DCP does not support a prefill with "
"--enable-kv-cache-sharding."
)
self._resolve_rank_mapping(info)
self.prefill_info_table[bootstrap_addr] = info
@@ -1296,7 +1316,23 @@ class CommonKVManager(BaseKVManager):
"""Compute TP/CP/PP rank mapping and store on the PrefillServerInfo object.
Deterministic for a given (bootstrap_addr, decode engine) pair."""
# TP rank mapping
if self.attn_tp_size == info.attn_tp_size:
# Only MLA shards across attn-TP, so non-MLA never reads the sharding flag.
if (
self.is_mla_backend
and info.enable_kv_cache_sharding
and info.attn_cp_size == 1
):
# MLA KV sharded across prefill attn-TP: every decode rank pulls
# from all prefill TP ranks (None marks them all as real senders).
assert self.attn_tp_size == info.attn_tp_size, (
"MLA KV cache sharding requires equal prefill/decode attn TP "
f"sizes, got prefill={info.attn_tp_size} decode={self.attn_tp_size}"
)
target_tp_rank = None
target_tp_ranks = list(range(info.attn_tp_size))
required_dst_info_num = self.attn_tp_size
required_prefill_response_num = info.attn_tp_size
elif self.attn_tp_size == info.attn_tp_size:
target_tp_rank = self.kv_args.engine_rank % self.attn_tp_size
required_dst_info_num = 1
required_prefill_response_num = 1
@@ -1350,6 +1386,7 @@ class CommonKVManager(BaseKVManager):
pull_from_all_cp_ranks = (
self.enable_all_cp_ranks_for_transfer
or info.enable_dsa_cache_layer_split
or info.enable_kv_cache_sharding
)
if not pull_from_all_cp_ranks:
# Only retrieve from prefill CP rank 0 when not using all ranks
@@ -1446,6 +1483,7 @@ class CommonKVManager(BaseKVManager):
"dsv41_spec_layout": self.dsv41_spec_layout,
"load_balance_method": get_parallel().load_balance_method,
"enable_dsa_cache_layer_split": get_parallel().enable_dsa_cache_layer_split,
"enable_kv_cache_sharding": get_parallel().enable_kv_cache_sharding,
# Self-register the HTTP API port so the decode can derive the PD
# retract rebootstrap /generate URL from bootstrap info instead of a
# router-injected pd_rebootstrap_prefill_url.
@@ -2098,7 +2136,17 @@ class CommonKVSender(BaseKVSender):
self.curr_idx += len(kv_indices)
is_last_chunk = self.curr_idx == self.num_kv_indices
if (
if self.kv_mgr.kv_shard_size > 1:
kv_indices, index_slice = filter_kv_indices_for_shard_rank(
kv_indices=kv_indices,
index_slice=index_slice,
shard_rank=self.kv_mgr.kv_shard_rank,
shard_size=self.kv_mgr.kv_shard_size,
)
# Like the scheduler, skip a non-final chunk with nothing to send.
if len(kv_indices) == 0 and not is_last_chunk:
return kv_indices, index_slice, is_last_chunk, True
elif (
self.kv_mgr.enable_all_cp_ranks_for_transfer
and not get_parallel().enable_dsa_cache_layer_split
):
@@ -2664,6 +2712,7 @@ class CommonKVBootstrapServer(BaseKVBootstrapServer):
self.dsv41_spec_layout: Optional[dict] = None
self.follow_bootstrap_room: Optional[bool] = None
self.enable_dsa_cache_layer_split: Optional[bool] = None
self.enable_kv_cache_sharding: Optional[bool] = None
self.prefill_http_port: Optional[int] = None
self.decode_allocation_policy = "early"
self.prefill_port_table: Dict[
@@ -2784,6 +2833,11 @@ class CommonKVBootstrapServer(BaseKVBootstrapServer):
data.get("enable_dsa_cache_layer_split", False)
)
if self.enable_kv_cache_sharding is None:
self.enable_kv_cache_sharding = bool(
data.get("enable_kv_cache_sharding", False)
)
if system_dp_size == 1:
dp_group = attn_dp_rank
else:
@@ -2850,6 +2904,7 @@ class CommonKVBootstrapServer(BaseKVBootstrapServer):
else True
),
enable_dsa_cache_layer_split=bool(self.enable_dsa_cache_layer_split),
enable_kv_cache_sharding=bool(self.enable_kv_cache_sharding),
prefill_http_port=self.prefill_http_port,
decode_allocation_policy=self.decode_allocation_policy,
)
+18 -2
View File
@@ -143,6 +143,15 @@ def _transfer_start_layer(*, pool, hf_text_config) -> int:
)
def max_bootstrap_input_len(*, capacity: int, page_size: int, shard_size: int) -> int:
"""Bootstrap prompt limit; sharded admission never fits a longer prompt."""
if shard_size == 1:
return capacity
# Strictest path, add_one_req_ignore_eos: page-rounded prompt, one page per
# shard, 1 new token and 1 spare token must stay below capacity.
return (capacity - shard_size * page_size - 3) // page_size * page_size
def maybe_release_metadata_buffer(
req: Req, allocator: ReqToMetadataIdxAllocator
) -> None:
@@ -192,8 +201,15 @@ class PrefillBootstrapQueue:
self.queue: List[Req] = []
self.scheduler = scheduler
self.scheduler_stage_metrics = scheduler_stage_metrics
# Each rank owns a subset of logical KV pages, so capacity widens by N.
self.max_total_num_tokens = (
self.scheduler.tp_worker.model_runner.effective_logical_max_total_num_tokens
* self.scheduler.kv_shard_widening
)
self.max_input_len = max_bootstrap_input_len(
capacity=self.max_total_num_tokens,
page_size=self.scheduler.token_to_kv_pool_allocator.page_size,
shard_size=self.scheduler.kv_shard_widening,
)
self.transfer_backend = transfer_backend
self.pp_poll_sync_work_list: List = []
@@ -446,8 +462,8 @@ class PrefillBootstrapQueue:
self.add(req, num_kv_heads)
def _check_if_req_exceed_kv_capacity(self, req: Req) -> bool:
if len(req.origin_input_ids) > self.max_total_num_tokens:
message = f"Request {req.rid} exceeds the maximum number of tokens: {len(req.origin_input_ids)} > {self.max_total_num_tokens}"
if len(req.origin_input_ids) > self.max_input_len:
message = f"Request {req.rid} exceeds the maximum number of tokens: {len(req.origin_input_ids)} > {self.max_input_len}"
logger.error(message)
req.time_stats.trace_ctx.abort(abort_info={"reason": message})
prepare_abort(req, message, status_code=HTTPStatus.BAD_REQUEST)
+14
View File
@@ -764,6 +764,20 @@ def _get_cp_rank_page_bounds(
return local_start, local_start + n_pages
def filter_kv_indices_for_shard_rank(
*,
kv_indices: np.ndarray,
index_slice: slice,
shard_rank: int,
shard_size: int,
) -> Tuple[np.ndarray, np.ndarray]:
"""Logical page l lives on rank l % N as local page l // N; returns the owned
local ids and their send positions, which replace the contiguous index_slice."""
owned = kv_indices % shard_size == shard_rank
positions = np.arange(index_slice.start, index_slice.stop, dtype=np.int64)
return kv_indices[owned] // shard_size, positions[owned]
def filter_kv_indices_for_cp_rank(
kv_mgr: CommonKVManager,
kv_indices: np.ndarray,