[Feat] KVShard: Support kv cache sharding for MTP (#40929)
Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
@@ -84,6 +84,7 @@ from sglang.srt.mem_cache.memory_pool import (
|
||||
from sglang.srt.mem_cache.page_interleave import (
|
||||
PageShardSpec,
|
||||
get_kv_shard_group,
|
||||
get_shared_kv_shard_pool,
|
||||
make_page_shard_spec,
|
||||
)
|
||||
from sglang.srt.mem_cache.page_interleave_pool import (
|
||||
@@ -1763,6 +1764,15 @@ class KVCacheConfigurator:
|
||||
is_dsv4_model: bool,
|
||||
req_to_token_pool: ReqToTokenPool,
|
||||
) -> KVCache:
|
||||
# MTP drafts sharing the target's sharded allocator need the same shard
|
||||
# layout with separate KV storage. Handle them before the normal pool
|
||||
# dispatch; targets and drafts without that allocator fall through.
|
||||
mtp_pool = self._build_mtp_kv_shard_pool(
|
||||
sizes=sizes, is_dsa_model=is_dsa_model, is_dsv4_model=is_dsv4_model
|
||||
)
|
||||
if mtp_pool is not None:
|
||||
return mtp_pool
|
||||
|
||||
# Page-granularity envelope layout for the MHA-shaped (full / SWA) pools,
|
||||
# selected by swapping in the PageMajorMHATokenToKVPool subclass. The
|
||||
# default keeps upstream's per-layer layout. The Mamba state pool is routed
|
||||
@@ -1889,6 +1899,115 @@ class KVCacheConfigurator:
|
||||
)
|
||||
return token_to_kv_pool
|
||||
|
||||
def _build_mtp_kv_shard_pool(
|
||||
self, *, sizes: _PoolSizes, is_dsa_model: bool, is_dsv4_model: bool
|
||||
) -> Optional[KVCache]:
|
||||
"""Build a draft pool in its target's logical space, with its own KV.
|
||||
|
||||
This is a pool-level opt-in through a shared sharded allocator. Target
|
||||
pool creation and speculative serving remain separate integration work.
|
||||
"""
|
||||
if not self.is_draft_worker:
|
||||
return None
|
||||
|
||||
target_pool = get_shared_kv_shard_pool(self)
|
||||
if target_pool is None:
|
||||
return None
|
||||
|
||||
if (
|
||||
self.spec_algorithm != SpeculativeAlgorithm.EAGLE
|
||||
or (self.model_config.num_nextn_predict_layers or 0) <= 0
|
||||
or get_spec().enable_multi_layer_eagle
|
||||
):
|
||||
raise ValueError(
|
||||
"Shared KV sharding supports only an EAGLE MTP draft runner."
|
||||
)
|
||||
if get_parallel().attn_dcp_size > 1:
|
||||
raise ValueError("MTP KV sharding is incompatible with DCP.")
|
||||
if (
|
||||
is_dsa_model
|
||||
or is_dsv4_model
|
||||
or self.is_hybrid_swa
|
||||
or self.mambaish_config is not None
|
||||
or self.sliding_window_size is not None
|
||||
or is_minimax_sparse(self.model_config.hf_config)
|
||||
or current_platform.is_out_of_tree()
|
||||
or _is_npu
|
||||
):
|
||||
raise ValueError("MTP KV sharding supports only dense MLA and MHA pools.")
|
||||
if (
|
||||
self.post_capture_kv_active
|
||||
or get_memory().enable_page_major_kv_layout
|
||||
or self.kv_cache_dtype_str == "mxfp8"
|
||||
or is_float4_e2m1fn_x2(self.kv_cache_dtype)
|
||||
):
|
||||
raise ValueError("MTP KV sharding requires the plain per-layer KV layout.")
|
||||
if (
|
||||
self.page_size != target_pool.page_size
|
||||
or self.pool_page_size != target_pool.page_size
|
||||
):
|
||||
raise ValueError(
|
||||
"MTP and target KV pools must use the same physical page size."
|
||||
)
|
||||
if sizes.max_total_num_tokens != target_pool.size:
|
||||
raise ValueError(
|
||||
"MTP and target sharded KV pools must have the same per-rank capacity."
|
||||
)
|
||||
target_is_mla = isinstance(target_pool, PageInterleaveMLATokenToKVPool)
|
||||
if self.use_mla_backend != target_is_mla or not isinstance(
|
||||
target_pool,
|
||||
(PageInterleaveMLATokenToKVPool, PageInterleaveMHATokenToKVPool),
|
||||
):
|
||||
raise ValueError(
|
||||
"MTP and target KV pools must use the same attention geometry."
|
||||
)
|
||||
fields = (
|
||||
("kv_lora_rank", "qk_rope_head_dim")
|
||||
if self.use_mla_backend
|
||||
else ("head_dim", "v_head_dim")
|
||||
)
|
||||
if (
|
||||
self.kv_cache_dtype != target_pool.dtype
|
||||
or any(
|
||||
getattr(self.model_config, name) != getattr(target_pool, name)
|
||||
for name in fields
|
||||
)
|
||||
or (
|
||||
not self.use_mla_backend
|
||||
and self.model_config.get_num_kv_heads(get_parallel().attn_tp_size)
|
||||
!= target_pool.head_num
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"MTP and target KV pools must use the same KV geometry and dtype."
|
||||
)
|
||||
|
||||
kwargs = dict(
|
||||
size=target_pool.size,
|
||||
page_size=target_pool.page_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
layer_num=self.layer_info.num_effective_layers,
|
||||
device=self.device,
|
||||
enable_memory_saver=get_exec().features.enable_memory_saver,
|
||||
start_layer=self.layer_info.start_layer,
|
||||
end_layer=self.layer_info.end_layer,
|
||||
shard_spec=target_pool.shard_spec,
|
||||
shard_group=target_pool.shard_group,
|
||||
)
|
||||
if self.use_mla_backend:
|
||||
return PageInterleaveMLATokenToKVPool(
|
||||
**kwargs,
|
||||
kv_lora_rank=self.model_config.kv_lora_rank,
|
||||
qk_rope_head_dim=self.model_config.qk_rope_head_dim,
|
||||
)
|
||||
return PageInterleaveMHATokenToKVPool(
|
||||
**kwargs,
|
||||
head_num=target_pool.head_num,
|
||||
head_dim=self.model_config.head_dim,
|
||||
v_head_dim=self.model_config.v_head_dim,
|
||||
enable_alt_stream=not get_disagg().enable_pdmux,
|
||||
)
|
||||
|
||||
def _build_dsv4_kv_pool(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -37,12 +37,18 @@ from typing import TYPE_CHECKING, Optional, Tuple
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.runtime_context import get_parallel, get_schedule
|
||||
from sglang.srt.mem_cache import page_interleave_pool
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import (
|
||||
PageInterleavePoolAllocator,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel, get_schedule, get_spec
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.srt.utils.common import ceil_align
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
|
||||
from sglang.srt.mem_cache.page_interleave_pool import PageInterleaveKVPoolMixin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -127,11 +133,42 @@ def get_kv_shard_group(use_mla_backend: bool) -> GroupCoordinator:
|
||||
return cp_group
|
||||
|
||||
|
||||
def get_shared_kv_shard_pool(
|
||||
kvc: KVCacheConfigurator,
|
||||
) -> Optional[PageInterleaveKVPoolMixin]:
|
||||
"""Find a draft's sharded target through their shared logical allocator."""
|
||||
if not kvc.is_draft_worker:
|
||||
return None
|
||||
|
||||
allocator = kvc.token_to_kv_pool_allocator
|
||||
if not isinstance(allocator, PageInterleavePoolAllocator):
|
||||
return None
|
||||
|
||||
pool = allocator.get_kvcache()
|
||||
if not isinstance(pool, page_interleave_pool.PageInterleaveKVPoolMixin):
|
||||
raise ValueError("A sharded MTP draft requires the target's sharded KV pool.")
|
||||
if (
|
||||
allocator.shard_spec != pool.shard_spec
|
||||
or allocator.shard_size != pool.shard_spec.shard_size
|
||||
or allocator.page_size != pool.page_size
|
||||
or allocator.size != pool.size * allocator.shard_size
|
||||
):
|
||||
raise ValueError(
|
||||
"The shared allocator and target KV pool have different shard layouts."
|
||||
)
|
||||
return pool
|
||||
|
||||
|
||||
def get_kv_shard_group_info(
|
||||
kvc: KVCacheConfigurator,
|
||||
) -> Tuple[Optional[int], int]:
|
||||
"""``(shard_rank, shard_size)`` for the KV pool; ``(None, 1)`` disables."""
|
||||
if kvc.is_draft_worker or not get_parallel().enable_kv_cache_sharding:
|
||||
if kvc.is_draft_worker:
|
||||
pool = get_shared_kv_shard_pool(kvc)
|
||||
if pool is None:
|
||||
return None, 1
|
||||
return pool.shard_spec.shard_rank, pool.shard_spec.shard_size
|
||||
if not get_parallel().enable_kv_cache_sharding:
|
||||
return None, 1
|
||||
group = get_kv_shard_group(kvc.use_mla_backend)
|
||||
if group.world_size <= 1:
|
||||
@@ -163,7 +200,14 @@ def make_page_shard_spec(kvc: KVCacheConfigurator) -> Optional[PageShardSpec]:
|
||||
or chunk size. Both scratch slots are charged against HBM before sizing
|
||||
persistent KV; large model contexts therefore reduce persistent capacity.
|
||||
Admission still checks the actual batch's padded prefix and chunk spans.
|
||||
|
||||
MTP drafts inherit the target's exact bounds: both pools serve the same
|
||||
scheduled batch and logical locations, even if their model configs differ
|
||||
in context length.
|
||||
"""
|
||||
if kvc.is_draft_worker:
|
||||
pool = get_shared_kv_shard_pool(kvc)
|
||||
return pool.shard_spec if pool is not None else None
|
||||
shard_rank, shard_size = get_kv_shard_group_info(kvc)
|
||||
if shard_rank is None:
|
||||
return None
|
||||
@@ -181,11 +225,31 @@ def make_page_shard_spec(kvc: KVCacheConfigurator) -> Optional[PageShardSpec]:
|
||||
)
|
||||
|
||||
|
||||
def compute_page_shard_scratch_bytes(kvc: KVCacheConfigurator) -> int:
|
||||
def compute_page_shard_scratch_bytes(
|
||||
kvc: KVCacheConfigurator, *, include_mtp: bool = False
|
||||
) -> int:
|
||||
"""Fixed HBM cost of the double-buffered assembly scratch, charged against
|
||||
the KV budget before pool sizing. Use the same spec as the pool builders:
|
||||
two slots, each holding ONE layer's ``[prefix | chunk | trash page]``."""
|
||||
two slots, each holding ONE layer's ``[prefix | chunk | trash page]``.
|
||||
|
||||
``include_mtp`` additionally budgets a target's single EAGLE MTP runner
|
||||
at the same KV geometry, as required by the draft pool builder. Its two
|
||||
scratch slots are reused across draft layers. Target sizing integration
|
||||
can opt into this estimate without changing the per-pool default.
|
||||
"""
|
||||
spec = make_page_shard_spec(kvc)
|
||||
if spec is None:
|
||||
return 0
|
||||
return 2 * spec.scratch_rows * _page_shard_row_bytes(kvc)
|
||||
scratch_bytes = 2 * spec.scratch_rows * _page_shard_row_bytes(kvc)
|
||||
if include_mtp:
|
||||
if (
|
||||
kvc.is_draft_worker
|
||||
or kvc.spec_algorithm != SpeculativeAlgorithm.EAGLE
|
||||
or (kvc.spec_aux_config.eagle_draft_num_layers or 0) <= 0
|
||||
or get_spec().enable_multi_layer_eagle
|
||||
):
|
||||
raise ValueError(
|
||||
"MTP scratch budgeting requires a target with one EAGLE MTP runner."
|
||||
)
|
||||
scratch_bytes *= 2
|
||||
return scratch_bytes
|
||||
|
||||
@@ -61,6 +61,7 @@ from sglang.kernels.ops.attention.dsa.dequant_k_cache import (
|
||||
)
|
||||
from sglang.kernels.ops.kvcache import dsa_index_k_cache
|
||||
from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from sglang.srt.mem_cache import page_interleave
|
||||
from sglang.srt.mem_cache.index_key_cache import IndexKeyCache
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
GPU_MEMORY_TYPE_KV_CACHE,
|
||||
@@ -70,15 +71,12 @@ from sglang.srt.mem_cache.memory_pool import (
|
||||
RadixAttention,
|
||||
unwrap_write_loc,
|
||||
)
|
||||
from sglang.srt.mem_cache.page_interleave import (
|
||||
PageInterleavePlacement,
|
||||
PageShardSpec,
|
||||
)
|
||||
from sglang.srt.mem_cache.utils import get_mla_kv_buffer_triton
|
||||
from sglang.srt.utils import ceil_div, get_bool_env_var
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
from sglang.srt.mem_cache.page_interleave import PageShardSpec
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -123,7 +121,8 @@ class PageInterleaveKVPoolMixin:
|
||||
assert spec.chunk_tokens % spec.page_size == 0
|
||||
|
||||
self.shard_spec = spec
|
||||
self.placement = PageInterleavePlacement(spec)
|
||||
self.shard_group = shard_group
|
||||
self.placement = page_interleave.PageInterleavePlacement(spec)
|
||||
self.shard_rank = spec.shard_rank
|
||||
self.shard_size = spec.shard_size
|
||||
self.device_module = torch.get_device_module(self.device)
|
||||
|
||||
Reference in New Issue
Block a user