[kv-shard 3/4] Enable Control Plane C (#40102)

This commit is contained in:
Shunkangz
2026-10-10 22:53:58 +08:00
committed by GitHub
parent 406540ab5f
commit de487f8039
17 changed files with 934 additions and 7 deletions
@@ -90,6 +90,7 @@ POSITIONAL_FIELD_ORDER = (
"cp_strategy",
"enable_dsa_cache_layer_split",
"enable_cp_decode_attn_tp",
"enable_kv_cache_sharding",
"enable_dp_attention",
"enable_dp_attention_local_control_broadcast",
"enable_dp_lm_head",
@@ -181,6 +181,14 @@ class Parallel(msgspec.Struct):
bool,
"Enable attention tensor-parallel weight slicing during decode under context parallel (cp_size>1). Slices the replicated attention linears to the local CP partition, eliminating redundant decode GEMMs.",
] = False
enable_kv_cache_sharding: A[
bool,
"Shard at-rest KV cache storage at page granularity across the group "
"that currently replicates it (attention-CP ranks for GQA models "
"under prefill CP; attention-TP ranks for MLA models), multiplying "
"the group's unique KV capacity. Prefill workers of PD "
"disaggregation only.",
] = False
# Deprecated spelling of `attn_dp_size`: `--dp-size N --enable-dp-attention`
# resolves to `attn_dp_size = N`, `dp_size = 1`. TODO: remove after 2026-12-31.
enable_dp_attention: A[bool, Arg(no_cli=True, resolvable=True)] = False
@@ -0,0 +1,282 @@
# SPDX-License-Identifier: Apache-2.0
"""Resolution and validation for logical-page KV cache sharding."""
from __future__ import annotations
import logging
from typing import Any, Optional
from sglang.srt.arg_groups.overrides import (
attention_backends_of,
declare_resolution,
model_config_of,
resolved_view,
resolving_view,
use_mla_backend,
)
from sglang.srt.configs.model_config import is_deepseek_dsa
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
from sglang.srt.runtime_context import attn_dp_enabled_of
from sglang.srt.utils.common import get_device_memory_capacity
logger = logging.getLogger(__name__)
def validate_kv_shard_attention_backend(server_args: Any, cfg: Any = None) -> None:
cfg = cfg or resolving_view(server_args)
prefill_backend, _ = attention_backends_of(resolved_view(server_args))
if prefill_backend == "fa3":
return
if prefill_backend != "trtllm_mla":
raise ValueError(
"--enable-kv-cache-sharding currently requires the fa3 prefill "
"attention backend, or trtllm_mla for plain-TP MLA; "
f"got {prefill_backend!r}."
)
if not use_mla_backend(server_args):
raise ValueError("KV sharding with trtllm_mla requires an MLA model.")
if cfg.attn_cp_size > 1:
raise ValueError(
"KV sharding with trtllm_mla only supports plain-TP MLA "
"(attn_cp_size must be 1)."
)
if cfg.disable_chunked_prefix_cache:
raise ValueError("KV sharding with trtllm_mla requires chunked prefix caching.")
def handle_kv_cache_sharding(server_args: Any, gpu_mem: Optional[float] = None) -> None:
"""Validate KV sharding and declare its derived eager-prefill settings."""
cfg = resolving_view(server_args)
if not cfg.enable_kv_cache_sharding:
return
if gpu_mem is None:
# `handle_gpu_memory_settings` resolves capacity internally now, so the
# pipeline no longer threads it through. Only the chunk-size rounding
# below reads it, and only when mem_fraction_static was left unset.
gpu_mem = get_device_memory_capacity(cfg.device)
if cfg.disaggregation_mode != "prefill":
raise ValueError(
"--enable-kv-cache-sharding is only supported on PD prefill "
"workers: a rank's pool holds only its stripe of every cached "
"page, which is incompatible with local decode."
)
if cfg.speculative_algorithm is not None:
raise ValueError(
"--enable-kv-cache-sharding does not support speculative decoding."
)
if cfg.dllm_algorithm is not None:
raise ValueError(
"--enable-kv-cache-sharding does not support diffusion language "
"models: DLLM admission bypasses the shard-scratch budget and "
"resolves out of PD-prefill mode."
)
if cfg.radix_cache_backend is not None:
raise ValueError(
"--enable-kv-cache-sharding requires the built-in "
"UnifiedRadixCache with a rotation-aware UnifiedTreeCore and does "
"not support "
f"--radix-cache-backend={cfg.radix_cache_backend!r}."
)
tree_core_backend = envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get()
# A disabled radix cache never matches or inserts, so its tree core never
# sees a rotation base; chunked continuations keep the base on the request.
if not cfg.disable_radix_cache and tree_core_backend != "python":
raise ValueError(
"--enable-kv-cache-sharding requires the Python UnifiedTreeCore: "
f"SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND={tree_core_backend!r} is "
"not supported for sharded KV."
)
if cfg.enable_unified_cache_external_linker:
raise ValueError(
"--enable-kv-cache-sharding does not support "
"--enable-unified-cache-external-linker: load-back allocates "
"through the flat allocator API."
)
if cfg.enable_streaming_session:
raise ValueError(
"--enable-kv-cache-sharding does not support streaming sessions: "
"their virtual nodes do not carry KV-shard rotation metadata."
)
if cfg.enable_session_radix_cache:
raise ValueError(
"--enable-kv-cache-sharding does not support session radix cache yet."
)
if cfg.enable_flexkv:
raise ValueError(
"--enable-kv-cache-sharding does not support --enable-flexkv: "
"FlexKV load-back allocates through the stock flat allocator API."
)
if cfg.enable_unified_memory:
raise ValueError(
"--enable-kv-cache-sharding does not support --enable-unified-memory: "
"the unified pool uses a different virtual-id allocator and layout."
)
if cfg.dcp_size > 1:
raise ValueError(
"--enable-kv-cache-sharding and decode context parallelism "
"(--dcp-size) are mutually exclusive users of the widened "
"allocator index space."
)
if cfg.enable_hierarchical_cache:
raise ValueError(
"--enable-kv-cache-sharding does not support "
"--enable-hierarchical-cache yet: HiCache backup/load assumes "
"each rank holds full pages."
)
if cfg.enable_lmcache:
raise ValueError(
"--enable-kv-cache-sharding does not support --enable-lmcache."
)
if cfg.enable_hisparse:
raise ValueError(
"--enable-kv-cache-sharding does not support --enable-hisparse: "
"HiSparse selects a non-sharded allocator and assumes each rank "
"owns complete KV pages."
)
# The dynamic sizer only runs with pipeline parallelism. Its profiling
# and predicted chunks can exceed the fixed shard-scratch capacity.
if cfg.enable_dynamic_chunking and cfg.pp_size > 1:
raise ValueError(
"--enable-kv-cache-sharding does not support dynamic chunk sizing "
"with pipeline parallelism: profiling and predicted chunks can "
"exceed the fixed sharded scratch capacity. Disable "
"--enable-dynamic-chunking."
)
if cfg.enable_two_batch_overlap:
raise ValueError(
"--enable-kv-cache-sharding does not support "
"--enable-two-batch-overlap: the sharded pool's per-batch gather "
"plan is a process-wide singleton, but TBO rebuilds attention "
"metadata multiple times per extend forward and clobbers the plan."
)
model_config = model_config_of(server_args)
if model_config.is_encoder_decoder:
raise ValueError(
"--enable-kv-cache-sharding does not support encoder-decoder "
"models: cross-attention uses a separate encoder page table that "
"is not part of the sharded gather plan."
)
if model_config.attention_chunk_size is not None:
raise ValueError(
"--enable-kv-cache-sharding does not support chunked/local-attention "
"models yet: local-attention metadata is built from the page table "
"before the scratch translation."
)
validate_kv_shard_attention_backend(server_args, cfg)
if cfg.disaggregation_transfer_backend not in ("mooncake", "nixl"):
raise ValueError(
"--enable-kv-cache-sharding currently only supports the mooncake "
"and nixl transfer backends: both route sends through "
"CommonKVSender._prepare_send_indices and pair destinations by "
"fancy index. Got --disaggregation-transfer-backend "
f"{cfg.disaggregation_transfer_backend!r}."
)
if envs.SGLANG_DISAGG_STAGING_BUFFER.get():
raise ValueError(
"--enable-kv-cache-sharding is incompatible with "
"SGLANG_DISAGG_STAGING_BUFFER (the staging path assumes "
"contiguous send slices)."
)
if use_mla_backend(server_args):
if is_deepseek_dsa(model_config.hf_config):
raise ValueError(
"--enable-kv-cache-sharding does not support DSA models yet "
"(indexer buffers are not striped)."
)
if cfg.attn_cp_size <= 1:
if attn_dp_enabled_of(cfg):
raise ValueError(
"--enable-kv-cache-sharding for MLA models without "
"prefill CP shards across attention-TP and requires plain "
"TP attention; set --attn-dp-size=1."
)
if cfg.tp_size <= 1:
raise ValueError(
"--enable-kv-cache-sharding for MLA models without "
"prefill CP shards across attention-TP ranks and needs "
"tp_size > 1."
)
elif not cfg.enable_prefill_cp or cfg.attn_cp_size <= 1:
raise ValueError(
"--enable-kv-cache-sharding for GQA models shards across "
"attention-CP ranks and requires --enable-prefill-cp with "
"attn_cp_size > 1."
)
page_size = cfg.page_size if cfg.page_size is not None else 1
if page_size <= 1:
raise ValueError(
"--enable-kv-cache-sharding requires a real page size "
"(--page-size 64 recommended): the physical page is the P/D "
"transfer descriptor and gather-copy unit; page_size=1 "
"degenerates both into per-token operations."
)
chunk_size = cfg.chunked_prefill_size
if chunk_size is None or chunk_size <= 0:
raise ValueError(
"--enable-kv-cache-sharding requires chunked prefill "
"(--chunked-prefill-size > 0)."
)
declarations = {}
# Chunk boundaries must stay on physical pages. Uneven ownership across
# ranks is supported; only the prefix gather pads to a full-group span.
if chunk_size % page_size != 0:
rounded = (chunk_size + page_size - 1) // page_size * page_size
declarations["chunked_prefill_size"] = rounded
# The raw input, deliberately not the resolving view: the question is
# whether the user pinned mem_fraction_static, and the resolved value
# would answer yes for every run.
raw_mem_fraction = (getattr(server_args, "_raw_input", None) or {}).get(
"mem_fraction_static"
)
activation_delta_mb = 1.5 * (max(rounded, 2048) - max(chunk_size, 2048))
if raw_mem_fraction is None and activation_delta_mb > 0:
if not gpu_mem or cfg.mem_fraction_static is None:
raise ValueError(
"Cannot safely auto-round --chunked-prefill-size for KV "
"sharding without a known device-memory capacity; pass a "
f"multiple of page_size ({page_size})."
)
corrected_mem_fraction = (
float(cfg.mem_fraction_static) - activation_delta_mb / gpu_mem
)
if corrected_mem_fraction <= 0:
raise ValueError(
"Auto-rounding --chunked-prefill-size for KV sharding "
"would consume all automatically reserved GPU memory; "
f"pass a smaller multiple of {page_size}."
)
declarations["mem_fraction_static"] = corrected_mem_fraction
logger.warning(
"Rounding --chunked-prefill-size up from %d to %d "
"(a multiple of page_size = %d) for KV sharding.",
chunk_size,
rounded,
page_size,
)
if cfg.cuda_graph_config.prefill.backend != Backend.DISABLED:
logger.warning(
"Prefill CUDA graph (%s) is incompatible with "
"--enable-kv-cache-sharding; disabling it.",
cfg.cuda_graph_config.prefill.backend,
)
declarations["cuda_graph_config"] = with_phase(
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
)
if declarations:
declare_resolution(
server_args,
"handle_kv_cache_sharding",
**declarations,
)
@@ -1705,6 +1705,12 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool:
False for any config the runtime won't post-capture-size, else it gets an
under-reserved fraction."""
cfg = resolving_view(server_args)
# Logical-page sharding always runs prefill eagerly. This predicate runs
# before the sharding hook disables prefill capture, so gate on the feature
# itself to retain eager activation headroom in the memory heuristic.
if cfg.enable_kv_cache_sharding:
return False
mla_enabled = use_mla_backend(server_args)
if not envs.SGLANG_ENABLE_POST_CAPTURE_KV_SIZING.get():
return False
+10
View File
@@ -6,6 +6,7 @@ from __future__ import annotations
from typing import Any
from sglang.srt.arg_groups.arg_utils import record_fields
from sglang.srt.arg_groups.kv_shard_hook import handle_kv_cache_sharding
from sglang.srt.arg_groups.overrides import (
_page_size_default,
_pipeline_parallel_overlap_disable,
@@ -264,6 +265,10 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(handle_context_parallelism, server_args)
# Validate logical-page KV cache sharding after its page size, attention
# backend, memory budget, and shard topology have all resolved.
run_hook(handle_kv_cache_sharding, server_args)
from sglang.srt.arg_groups.moe_hook import (
handle_a2a_moe,
handle_moe_kernel_config,
@@ -334,6 +339,11 @@ def run_resolution_pipeline(server_args: Any) -> None:
finalize_cuda_graph_prefill_max_context(server_args)
# Capability adjustments can late-force a different attention backend or
# disable chunked prefill. Re-run the idempotent sharding gate over the
# final resolving view.
run_hook(handle_kv_cache_sharding, server_args)
# Validate after all batch-size declarations are visible.
run_hook(validate_deepep_v2_speculative_draft, server_args)
run_hook(validate_deepep_v2_dispatch_token_budget, server_args)
@@ -67,6 +67,7 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
"handle_data_parallelism",
"handle_load_balance_method",
"handle_context_parallelism",
"handle_kv_cache_sharding",
"handle_moe_kernel_config",
"handle_a2a_moe",
"handle_eplb_and_dispatch",
@@ -49,6 +49,7 @@ from sglang.srt.mem_cache.allocator.hisparse import (
DeepSeekV4HiSparseTokenToKVPoolAllocator,
HiSparseTokenToKVPoolAllocator,
)
from sglang.srt.mem_cache.allocator.page_interleave import PageInterleavePoolAllocator
from sglang.srt.mem_cache.allocator.swa import (
PureSWATokenToKVPoolAllocator,
SWATokenToKVPoolAllocator,
@@ -80,6 +81,16 @@ from sglang.srt.mem_cache.memory_pool import (
ReqToTokenPool,
get_minimax_sparse_index_dtype,
)
from sglang.srt.mem_cache.page_interleave import (
PageShardSpec,
get_kv_shard_group,
make_page_shard_spec,
)
from sglang.srt.mem_cache.page_interleave_pool import (
PageInterleaveKVPoolMixin,
PageInterleaveMHATokenToKVPool,
PageInterleaveMLATokenToKVPool,
)
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.platforms import current_platform
@@ -229,6 +240,7 @@ def _pp_local_per_request_bytes(
if TYPE_CHECKING:
from sglang.srt.distributed.parallel_state import GroupCoordinator
from sglang.srt.mem_cache.unified_memory_pool import (
UnifiedKVPool,
UnifiedPoolBundle,
@@ -311,6 +323,10 @@ class KVCacheConfigurator:
mambaish_config: Optional[Any] = field(init=False)
hybrid_gdn_config: Optional[Any] = field(init=False)
hybrid_kda_config: Optional[Any] = field(init=False)
kv_shard_rank: Optional[int] = field(init=False)
kv_shard_size: int = field(init=False)
kv_shard_spec: Optional[PageShardSpec] = field(init=False)
kv_shard_group: Optional[GroupCoordinator] = field(init=False)
is_hybrid_swa_mtp_draft: bool = field(init=False)
draft_swa_full_capacity: bool = field(init=False)
@@ -318,6 +334,7 @@ class KVCacheConfigurator:
self.mambaish_config = mambaish_config(self.model_config)
self.hybrid_gdn_config = hybrid_gdn_config(self.model_config)
self.hybrid_kda_config = hybrid_kda_config(self.model_config)
self._resolve_kv_shard()
self.is_hybrid_swa_mtp_draft = self.layer_info.is_hybrid_swa_mtp_draft
self.draft_swa_full_capacity = self.is_hybrid_swa_mtp_draft and bool(
self.layer_info.swa_attention_layer_ids
@@ -351,6 +368,25 @@ class KVCacheConfigurator:
max_total_num_tokens * get_parallel().attn_dcp_size // self.loc_space_scale
)
def _resolve_kv_shard(self) -> None:
"""Resolve the logical-page KV sharding identity: ``(rank, size)`` of
the shard group, or ``(None, 1)`` when sharding is off. Consumed by the
pool builders and by the widened allocator branch."""
self.kv_shard_spec = make_page_shard_spec(self)
self.kv_shard_rank = None
self.kv_shard_size = 1
self.kv_shard_group = None
if self.kv_shard_spec is None:
return
reject_out_of_tree_path(
current_platform,
subsystem="logical-page KV sharding (--enable-kv-cache-sharding)",
)
self.kv_shard_rank = self.kv_shard_spec.shard_rank
self.kv_shard_size = self.kv_shard_spec.shard_size
self.kv_shard_group = get_kv_shard_group(self.use_mla_backend)
def _build_fp4_quant_method(self, *, num_layers: int):
if not is_float4_e2m1fn_x2(self.kv_cache_dtype):
return None
@@ -689,6 +725,13 @@ class KVCacheConfigurator:
req_to_token_pool=req_to_token_pool,
)
if self.kv_shard_rank is not None:
if not isinstance(token_to_kv_pool, PageInterleaveKVPoolMixin):
raise ValueError(
"--enable-kv-cache-sharding does not support this model "
f"family (pool {type(token_to_kv_pool).__name__})."
)
if draft_virtual_id_space is not None:
assert token_to_kv_pool.size >= draft_virtual_id_space, (
"draft token_to_kv_pool smaller than the shared unified "
@@ -2099,6 +2142,17 @@ class KVCacheConfigurator:
def _build_mla_kv_pool(
self, *, max_total_num_tokens: int, mla_pool_class: type
) -> KVCache:
mla_pool_kwargs = {}
if self.kv_shard_rank is not None:
# The striped pool subclasses the plain MLA pool; a vendor pool
# class would otherwise be silently replaced.
assert mla_pool_class is MLATokenToKVPool, (
"--enable-kv-cache-sharding is incompatible with "
f"the {mla_pool_class.__name__} pool"
)
mla_pool_class = PageInterleaveMLATokenToKVPool
mla_pool_kwargs["shard_spec"] = self.kv_shard_spec
mla_pool_kwargs["shard_group"] = self.kv_shard_group
token_to_kv_pool = mla_pool_class(
max_total_num_tokens,
page_size=self.pool_page_size,
@@ -2110,6 +2164,7 @@ class KVCacheConfigurator:
enable_memory_saver=get_exec().features.enable_memory_saver,
start_layer=self.layer_info.start_layer,
end_layer=self.layer_info.end_layer,
**mla_pool_kwargs,
)
return token_to_kv_pool
@@ -2363,6 +2418,23 @@ class KVCacheConfigurator:
pool_kwargs["quant_method"] = quant_method
else:
pool_kwargs["post_capture_active"] = self.post_capture_kv_active
if self.kv_shard_rank is not None:
# The striped pool subclasses the plain per-layer MHA pool; the
# mxfp8 / NoOp / page-major variants have incompatible buffer
# layouts. Specialized quantization methods need packed-data and
# scale-aware sharded writes/gathers; ordinary CUDA FP8 uses the
# plain pool and has no specialized quant_method.
assert pool_cls is MHATokenToKVPool, (
"--enable-kv-cache-sharding is incompatible with "
f"the {pool_cls.__name__} pool"
)
assert quant_method is None, (
"--enable-kv-cache-sharding does not support specialized KV-cache "
"quantization methods (e.g. FP4); ordinary CUDA FP8 is supported."
)
pool_cls = PageInterleaveMHATokenToKVPool
pool_kwargs["shard_spec"] = self.kv_shard_spec
pool_kwargs["shard_group"] = self.kv_shard_group
token_to_kv_pool = pool_cls(
max_total_num_tokens,
page_size=self.pool_page_size,
@@ -2487,6 +2559,19 @@ class KVCacheConfigurator:
need_sort=need_sort,
host_to_device_ratio=hisparse_cfg.host_to_device_ratio,
)
elif self.kv_shard_size > 1:
# Logical-page sharding: index space widened xN over
# the stock 1x pool; the allocator page stays physical.
token_to_kv_pool_allocator = PageInterleavePoolAllocator(
sizes.max_total_num_tokens,
physical_page_size=self.page_size,
shard_size=self.kv_shard_size,
dtype=self.kv_cache_dtype,
device=self.device,
kvcache=token_to_kv_pool,
need_sort=need_sort,
shard_spec=self.kv_shard_spec,
)
elif (
get_schedule().page_size == 1 and not get_parallel().dcp_enabled
):
@@ -68,9 +68,11 @@ class PageShardSpec(msgspec.Struct, frozen=True):
@property
def logical_page_size(self) -> int:
"""The N*page_size span of one logical page. It bounds the assembly
scratch and rounds chunked_prefill_size; the allocator and the tree
both keep drawing and matching at the physical ``page_size``."""
"""The N*page_size span used to align prefix-gather scratch.
Allocation, prefix matching, and chunk boundaries use the physical
``page_size``.
"""
return self.shard_size * self.page_size
+5 -1
View File
@@ -17,7 +17,11 @@ from typing import TYPE_CHECKING, Any, Callable, Optional
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.runtime_context import get_disagg, get_memory, get_serving
from sglang.srt.runtime_context import (
get_disagg,
get_memory,
get_serving,
)
if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
@@ -42,6 +42,7 @@ from sglang.srt.mem_cache.memory_pool import (
DSATokenToKVPool,
get_minimax_sparse_index_dtype,
)
from sglang.srt.mem_cache.page_interleave import compute_page_shard_scratch_bytes
from sglang.srt.runtime_context import (
get_disagg,
get_exec,
@@ -213,7 +214,10 @@ class MemoryPoolConfigurator:
class DefaultPoolConfigurator(MemoryPoolConfigurator):
"""Standard models (MHA, MLA, DSA, FP4): coeff = cell_size, bias = 0."""
"""Standard models (MHA, MLA, DSA, FP4).
coeff = cell_size; bias = double-buffered KV-shard scratch, otherwise zero.
"""
def __init__(self, kvc: KVCacheConfigurator):
self.kv_cache_dtype_str = kvc.kv_cache_dtype_str
@@ -242,6 +246,10 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
else get_schedule().max_total_tokens or kvc.model_config.context_len
)
# Logical-page KV sharding reserves a fixed double-buffered assembly
# scratch next to the pool; charge it before token sizing.
self._fixed_overhead_bytes = compute_page_shard_scratch_bytes(kvc)
# EAGLE/STANDALONE: assumes the draft shares the target's per-layer KV size,
# which holds for EAGLE/MTP drafts that reuse the target's attention config.
if (
@@ -598,7 +606,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
def calculate_pool_sizes(
self, available_bytes: int, page_size: int
) -> MemoryPoolConfig:
available_bytes = max(available_bytes, 0)
available_bytes = max(available_bytes - self._fixed_overhead_bytes, 0)
max_total_num_tokens = (
available_bytes // self._cell_size
if self._cell_size