[Feat] KVShard: Support kv cache sharding for MTP (#40929)

Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
Shunkangz
2026-10-23 15:20:19 +08:00
committed by GitHub
co-authored by Zhangheng
parent 6e667c4310
commit 79ab8f6d7c
6 changed files with 1168 additions and 377 deletions
@@ -84,6 +84,7 @@ from sglang.srt.mem_cache.memory_pool import (
from sglang.srt.mem_cache.page_interleave import (
PageShardSpec,
get_kv_shard_group,
get_shared_kv_shard_pool,
make_page_shard_spec,
)
from sglang.srt.mem_cache.page_interleave_pool import (
@@ -1763,6 +1764,15 @@ class KVCacheConfigurator:
is_dsv4_model: bool,
req_to_token_pool: ReqToTokenPool,
) -> KVCache:
# MTP drafts sharing the target's sharded allocator need the same shard
# layout with separate KV storage. Handle them before the normal pool
# dispatch; targets and drafts without that allocator fall through.
mtp_pool = self._build_mtp_kv_shard_pool(
sizes=sizes, is_dsa_model=is_dsa_model, is_dsv4_model=is_dsv4_model
)
if mtp_pool is not None:
return mtp_pool
# Page-granularity envelope layout for the MHA-shaped (full / SWA) pools,
# selected by swapping in the PageMajorMHATokenToKVPool subclass. The
# default keeps upstream's per-layer layout. The Mamba state pool is routed
@@ -1889,6 +1899,115 @@ class KVCacheConfigurator:
)
return token_to_kv_pool
def _build_mtp_kv_shard_pool(
self, *, sizes: _PoolSizes, is_dsa_model: bool, is_dsv4_model: bool
) -> Optional[KVCache]:
"""Build a draft pool in its target's logical space, with its own KV.
This is a pool-level opt-in through a shared sharded allocator. Target
pool creation and speculative serving remain separate integration work.
"""
if not self.is_draft_worker:
return None
target_pool = get_shared_kv_shard_pool(self)
if target_pool is None:
return None
if (
self.spec_algorithm != SpeculativeAlgorithm.EAGLE
or (self.model_config.num_nextn_predict_layers or 0) <= 0
or get_spec().enable_multi_layer_eagle
):
raise ValueError(
"Shared KV sharding supports only an EAGLE MTP draft runner."
)
if get_parallel().attn_dcp_size > 1:
raise ValueError("MTP KV sharding is incompatible with DCP.")
if (
is_dsa_model
or is_dsv4_model
or self.is_hybrid_swa
or self.mambaish_config is not None
or self.sliding_window_size is not None
or is_minimax_sparse(self.model_config.hf_config)
or current_platform.is_out_of_tree()
or _is_npu
):
raise ValueError("MTP KV sharding supports only dense MLA and MHA pools.")
if (
self.post_capture_kv_active
or get_memory().enable_page_major_kv_layout
or self.kv_cache_dtype_str == "mxfp8"
or is_float4_e2m1fn_x2(self.kv_cache_dtype)
):
raise ValueError("MTP KV sharding requires the plain per-layer KV layout.")
if (
self.page_size != target_pool.page_size
or self.pool_page_size != target_pool.page_size
):
raise ValueError(
"MTP and target KV pools must use the same physical page size."
)
if sizes.max_total_num_tokens != target_pool.size:
raise ValueError(
"MTP and target sharded KV pools must have the same per-rank capacity."
)
target_is_mla = isinstance(target_pool, PageInterleaveMLATokenToKVPool)
if self.use_mla_backend != target_is_mla or not isinstance(
target_pool,
(PageInterleaveMLATokenToKVPool, PageInterleaveMHATokenToKVPool),
):
raise ValueError(
"MTP and target KV pools must use the same attention geometry."
)
fields = (
("kv_lora_rank", "qk_rope_head_dim")
if self.use_mla_backend
else ("head_dim", "v_head_dim")
)
if (
self.kv_cache_dtype != target_pool.dtype
or any(
getattr(self.model_config, name) != getattr(target_pool, name)
for name in fields
)
or (
not self.use_mla_backend
and self.model_config.get_num_kv_heads(get_parallel().attn_tp_size)
!= target_pool.head_num
)
):
raise ValueError(
"MTP and target KV pools must use the same KV geometry and dtype."
)
kwargs = dict(
size=target_pool.size,
page_size=target_pool.page_size,
dtype=self.kv_cache_dtype,
layer_num=self.layer_info.num_effective_layers,
device=self.device,
enable_memory_saver=get_exec().features.enable_memory_saver,
start_layer=self.layer_info.start_layer,
end_layer=self.layer_info.end_layer,
shard_spec=target_pool.shard_spec,
shard_group=target_pool.shard_group,
)
if self.use_mla_backend:
return PageInterleaveMLATokenToKVPool(
**kwargs,
kv_lora_rank=self.model_config.kv_lora_rank,
qk_rope_head_dim=self.model_config.qk_rope_head_dim,
)
return PageInterleaveMHATokenToKVPool(
**kwargs,
head_num=target_pool.head_num,
head_dim=self.model_config.head_dim,
v_head_dim=self.model_config.v_head_dim,
enable_alt_stream=not get_disagg().enable_pdmux,
)
def _build_dsv4_kv_pool(
self,
*,
+69 -5
View File
@@ -37,12 +37,18 @@ from typing import TYPE_CHECKING, Optional, Tuple
import msgspec
import torch
from sglang.srt.runtime_context import get_parallel, get_schedule
from sglang.srt.mem_cache import page_interleave_pool
from sglang.srt.mem_cache.allocator.page_interleave import (
PageInterleavePoolAllocator,
)
from sglang.srt.runtime_context import get_parallel, get_schedule, get_spec
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.utils.common import ceil_align
if TYPE_CHECKING:
from sglang.srt.distributed.parallel_state import GroupCoordinator
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
from sglang.srt.mem_cache.page_interleave_pool import PageInterleaveKVPoolMixin
logger = logging.getLogger(__name__)
@@ -127,11 +133,42 @@ def get_kv_shard_group(use_mla_backend: bool) -> GroupCoordinator:
return cp_group
def get_shared_kv_shard_pool(
kvc: KVCacheConfigurator,
) -> Optional[PageInterleaveKVPoolMixin]:
"""Find a draft's sharded target through their shared logical allocator."""
if not kvc.is_draft_worker:
return None
allocator = kvc.token_to_kv_pool_allocator
if not isinstance(allocator, PageInterleavePoolAllocator):
return None
pool = allocator.get_kvcache()
if not isinstance(pool, page_interleave_pool.PageInterleaveKVPoolMixin):
raise ValueError("A sharded MTP draft requires the target's sharded KV pool.")
if (
allocator.shard_spec != pool.shard_spec
or allocator.shard_size != pool.shard_spec.shard_size
or allocator.page_size != pool.page_size
or allocator.size != pool.size * allocator.shard_size
):
raise ValueError(
"The shared allocator and target KV pool have different shard layouts."
)
return pool
def get_kv_shard_group_info(
kvc: KVCacheConfigurator,
) -> Tuple[Optional[int], int]:
"""``(shard_rank, shard_size)`` for the KV pool; ``(None, 1)`` disables."""
if kvc.is_draft_worker or not get_parallel().enable_kv_cache_sharding:
if kvc.is_draft_worker:
pool = get_shared_kv_shard_pool(kvc)
if pool is None:
return None, 1
return pool.shard_spec.shard_rank, pool.shard_spec.shard_size
if not get_parallel().enable_kv_cache_sharding:
return None, 1
group = get_kv_shard_group(kvc.use_mla_backend)
if group.world_size <= 1:
@@ -163,7 +200,14 @@ def make_page_shard_spec(kvc: KVCacheConfigurator) -> Optional[PageShardSpec]:
or chunk size. Both scratch slots are charged against HBM before sizing
persistent KV; large model contexts therefore reduce persistent capacity.
Admission still checks the actual batch's padded prefix and chunk spans.
MTP drafts inherit the target's exact bounds: both pools serve the same
scheduled batch and logical locations, even if their model configs differ
in context length.
"""
if kvc.is_draft_worker:
pool = get_shared_kv_shard_pool(kvc)
return pool.shard_spec if pool is not None else None
shard_rank, shard_size = get_kv_shard_group_info(kvc)
if shard_rank is None:
return None
@@ -181,11 +225,31 @@ def make_page_shard_spec(kvc: KVCacheConfigurator) -> Optional[PageShardSpec]:
)
def compute_page_shard_scratch_bytes(kvc: KVCacheConfigurator) -> int:
def compute_page_shard_scratch_bytes(
kvc: KVCacheConfigurator, *, include_mtp: bool = False
) -> int:
"""Fixed HBM cost of the double-buffered assembly scratch, charged against
the KV budget before pool sizing. Use the same spec as the pool builders:
two slots, each holding ONE layer's ``[prefix | chunk | trash page]``."""
two slots, each holding ONE layer's ``[prefix | chunk | trash page]``.
``include_mtp`` additionally budgets a target's single EAGLE MTP runner
at the same KV geometry, as required by the draft pool builder. Its two
scratch slots are reused across draft layers. Target sizing integration
can opt into this estimate without changing the per-pool default.
"""
spec = make_page_shard_spec(kvc)
if spec is None:
return 0
return 2 * spec.scratch_rows * _page_shard_row_bytes(kvc)
scratch_bytes = 2 * spec.scratch_rows * _page_shard_row_bytes(kvc)
if include_mtp:
if (
kvc.is_draft_worker
or kvc.spec_algorithm != SpeculativeAlgorithm.EAGLE
or (kvc.spec_aux_config.eagle_draft_num_layers or 0) <= 0
or get_spec().enable_multi_layer_eagle
):
raise ValueError(
"MTP scratch budgeting requires a target with one EAGLE MTP runner."
)
scratch_bytes *= 2
return scratch_bytes
@@ -61,6 +61,7 @@ from sglang.kernels.ops.attention.dsa.dequant_k_cache import (
)
from sglang.kernels.ops.kvcache import dsa_index_k_cache
from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator
from sglang.srt.mem_cache import page_interleave
from sglang.srt.mem_cache.index_key_cache import IndexKeyCache
from sglang.srt.mem_cache.memory_pool import (
GPU_MEMORY_TYPE_KV_CACHE,
@@ -70,15 +71,12 @@ from sglang.srt.mem_cache.memory_pool import (
RadixAttention,
unwrap_write_loc,
)
from sglang.srt.mem_cache.page_interleave import (
PageInterleavePlacement,
PageShardSpec,
)
from sglang.srt.mem_cache.utils import get_mla_kv_buffer_triton
from sglang.srt.utils import ceil_div, get_bool_env_var
if TYPE_CHECKING:
from sglang.srt.distributed.parallel_state import GroupCoordinator
from sglang.srt.mem_cache.page_interleave import PageShardSpec
logger = logging.getLogger(__name__)
@@ -123,7 +121,8 @@ class PageInterleaveKVPoolMixin:
assert spec.chunk_tokens % spec.page_size == 0
self.shard_spec = spec
self.placement = PageInterleavePlacement(spec)
self.shard_group = shard_group
self.placement = page_interleave.PageInterleavePlacement(spec)
self.shard_rank = spec.shard_rank
self.shard_size = spec.shard_size
self.device_module = torch.get_device_module(self.device)