[kv-shard 3/4] Enable Control Plane C (#40102)
This commit is contained in:
@@ -90,6 +90,7 @@ POSITIONAL_FIELD_ORDER = (
|
||||
"cp_strategy",
|
||||
"enable_dsa_cache_layer_split",
|
||||
"enable_cp_decode_attn_tp",
|
||||
"enable_kv_cache_sharding",
|
||||
"enable_dp_attention",
|
||||
"enable_dp_attention_local_control_broadcast",
|
||||
"enable_dp_lm_head",
|
||||
|
||||
@@ -181,6 +181,14 @@ class Parallel(msgspec.Struct):
|
||||
bool,
|
||||
"Enable attention tensor-parallel weight slicing during decode under context parallel (cp_size>1). Slices the replicated attention linears to the local CP partition, eliminating redundant decode GEMMs.",
|
||||
] = False
|
||||
enable_kv_cache_sharding: A[
|
||||
bool,
|
||||
"Shard at-rest KV cache storage at page granularity across the group "
|
||||
"that currently replicates it (attention-CP ranks for GQA models "
|
||||
"under prefill CP; attention-TP ranks for MLA models), multiplying "
|
||||
"the group's unique KV capacity. Prefill workers of PD "
|
||||
"disaggregation only.",
|
||||
] = False
|
||||
# Deprecated spelling of `attn_dp_size`: `--dp-size N --enable-dp-attention`
|
||||
# resolves to `attn_dp_size = N`, `dp_size = 1`. TODO: remove after 2026-12-31.
|
||||
enable_dp_attention: A[bool, Arg(no_cli=True, resolvable=True)] = False
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Resolution and validation for logical-page KV cache sharding."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
attention_backends_of,
|
||||
declare_resolution,
|
||||
model_config_of,
|
||||
resolved_view,
|
||||
resolving_view,
|
||||
use_mla_backend,
|
||||
)
|
||||
from sglang.srt.configs.model_config import is_deepseek_dsa
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of
|
||||
from sglang.srt.utils.common import get_device_memory_capacity
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def validate_kv_shard_attention_backend(server_args: Any, cfg: Any = None) -> None:
|
||||
cfg = cfg or resolving_view(server_args)
|
||||
prefill_backend, _ = attention_backends_of(resolved_view(server_args))
|
||||
if prefill_backend == "fa3":
|
||||
return
|
||||
if prefill_backend != "trtllm_mla":
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding currently requires the fa3 prefill "
|
||||
"attention backend, or trtllm_mla for plain-TP MLA; "
|
||||
f"got {prefill_backend!r}."
|
||||
)
|
||||
if not use_mla_backend(server_args):
|
||||
raise ValueError("KV sharding with trtllm_mla requires an MLA model.")
|
||||
if cfg.attn_cp_size > 1:
|
||||
raise ValueError(
|
||||
"KV sharding with trtllm_mla only supports plain-TP MLA "
|
||||
"(attn_cp_size must be 1)."
|
||||
)
|
||||
if cfg.disable_chunked_prefix_cache:
|
||||
raise ValueError("KV sharding with trtllm_mla requires chunked prefix caching.")
|
||||
|
||||
|
||||
def handle_kv_cache_sharding(server_args: Any, gpu_mem: Optional[float] = None) -> None:
|
||||
"""Validate KV sharding and declare its derived eager-prefill settings."""
|
||||
cfg = resolving_view(server_args)
|
||||
if not cfg.enable_kv_cache_sharding:
|
||||
return
|
||||
|
||||
if gpu_mem is None:
|
||||
# `handle_gpu_memory_settings` resolves capacity internally now, so the
|
||||
# pipeline no longer threads it through. Only the chunk-size rounding
|
||||
# below reads it, and only when mem_fraction_static was left unset.
|
||||
gpu_mem = get_device_memory_capacity(cfg.device)
|
||||
|
||||
if cfg.disaggregation_mode != "prefill":
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding is only supported on PD prefill "
|
||||
"workers: a rank's pool holds only its stripe of every cached "
|
||||
"page, which is incompatible with local decode."
|
||||
)
|
||||
if cfg.speculative_algorithm is not None:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support speculative decoding."
|
||||
)
|
||||
if cfg.dllm_algorithm is not None:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support diffusion language "
|
||||
"models: DLLM admission bypasses the shard-scratch budget and "
|
||||
"resolves out of PD-prefill mode."
|
||||
)
|
||||
if cfg.radix_cache_backend is not None:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding requires the built-in "
|
||||
"UnifiedRadixCache with a rotation-aware UnifiedTreeCore and does "
|
||||
"not support "
|
||||
f"--radix-cache-backend={cfg.radix_cache_backend!r}."
|
||||
)
|
||||
tree_core_backend = envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get()
|
||||
# A disabled radix cache never matches or inserts, so its tree core never
|
||||
# sees a rotation base; chunked continuations keep the base on the request.
|
||||
if not cfg.disable_radix_cache and tree_core_backend != "python":
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding requires the Python UnifiedTreeCore: "
|
||||
f"SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND={tree_core_backend!r} is "
|
||||
"not supported for sharded KV."
|
||||
)
|
||||
if cfg.enable_unified_cache_external_linker:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support "
|
||||
"--enable-unified-cache-external-linker: load-back allocates "
|
||||
"through the flat allocator API."
|
||||
)
|
||||
if cfg.enable_streaming_session:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support streaming sessions: "
|
||||
"their virtual nodes do not carry KV-shard rotation metadata."
|
||||
)
|
||||
if cfg.enable_session_radix_cache:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support session radix cache yet."
|
||||
)
|
||||
if cfg.enable_flexkv:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support --enable-flexkv: "
|
||||
"FlexKV load-back allocates through the stock flat allocator API."
|
||||
)
|
||||
if cfg.enable_unified_memory:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support --enable-unified-memory: "
|
||||
"the unified pool uses a different virtual-id allocator and layout."
|
||||
)
|
||||
if cfg.dcp_size > 1:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding and decode context parallelism "
|
||||
"(--dcp-size) are mutually exclusive users of the widened "
|
||||
"allocator index space."
|
||||
)
|
||||
if cfg.enable_hierarchical_cache:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support "
|
||||
"--enable-hierarchical-cache yet: HiCache backup/load assumes "
|
||||
"each rank holds full pages."
|
||||
)
|
||||
if cfg.enable_lmcache:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support --enable-lmcache."
|
||||
)
|
||||
if cfg.enable_hisparse:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support --enable-hisparse: "
|
||||
"HiSparse selects a non-sharded allocator and assumes each rank "
|
||||
"owns complete KV pages."
|
||||
)
|
||||
# The dynamic sizer only runs with pipeline parallelism. Its profiling
|
||||
# and predicted chunks can exceed the fixed shard-scratch capacity.
|
||||
if cfg.enable_dynamic_chunking and cfg.pp_size > 1:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support dynamic chunk sizing "
|
||||
"with pipeline parallelism: profiling and predicted chunks can "
|
||||
"exceed the fixed sharded scratch capacity. Disable "
|
||||
"--enable-dynamic-chunking."
|
||||
)
|
||||
if cfg.enable_two_batch_overlap:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support "
|
||||
"--enable-two-batch-overlap: the sharded pool's per-batch gather "
|
||||
"plan is a process-wide singleton, but TBO rebuilds attention "
|
||||
"metadata multiple times per extend forward and clobbers the plan."
|
||||
)
|
||||
|
||||
model_config = model_config_of(server_args)
|
||||
if model_config.is_encoder_decoder:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support encoder-decoder "
|
||||
"models: cross-attention uses a separate encoder page table that "
|
||||
"is not part of the sharded gather plan."
|
||||
)
|
||||
if model_config.attention_chunk_size is not None:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support chunked/local-attention "
|
||||
"models yet: local-attention metadata is built from the page table "
|
||||
"before the scratch translation."
|
||||
)
|
||||
|
||||
validate_kv_shard_attention_backend(server_args, cfg)
|
||||
if cfg.disaggregation_transfer_backend not in ("mooncake", "nixl"):
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding currently only supports the mooncake "
|
||||
"and nixl transfer backends: both route sends through "
|
||||
"CommonKVSender._prepare_send_indices and pair destinations by "
|
||||
"fancy index. Got --disaggregation-transfer-backend "
|
||||
f"{cfg.disaggregation_transfer_backend!r}."
|
||||
)
|
||||
if envs.SGLANG_DISAGG_STAGING_BUFFER.get():
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding is incompatible with "
|
||||
"SGLANG_DISAGG_STAGING_BUFFER (the staging path assumes "
|
||||
"contiguous send slices)."
|
||||
)
|
||||
|
||||
if use_mla_backend(server_args):
|
||||
if is_deepseek_dsa(model_config.hf_config):
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support DSA models yet "
|
||||
"(indexer buffers are not striped)."
|
||||
)
|
||||
if cfg.attn_cp_size <= 1:
|
||||
if attn_dp_enabled_of(cfg):
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding for MLA models without "
|
||||
"prefill CP shards across attention-TP and requires plain "
|
||||
"TP attention; set --attn-dp-size=1."
|
||||
)
|
||||
if cfg.tp_size <= 1:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding for MLA models without "
|
||||
"prefill CP shards across attention-TP ranks and needs "
|
||||
"tp_size > 1."
|
||||
)
|
||||
elif not cfg.enable_prefill_cp or cfg.attn_cp_size <= 1:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding for GQA models shards across "
|
||||
"attention-CP ranks and requires --enable-prefill-cp with "
|
||||
"attn_cp_size > 1."
|
||||
)
|
||||
|
||||
page_size = cfg.page_size if cfg.page_size is not None else 1
|
||||
if page_size <= 1:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding requires a real page size "
|
||||
"(--page-size 64 recommended): the physical page is the P/D "
|
||||
"transfer descriptor and gather-copy unit; page_size=1 "
|
||||
"degenerates both into per-token operations."
|
||||
)
|
||||
chunk_size = cfg.chunked_prefill_size
|
||||
if chunk_size is None or chunk_size <= 0:
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding requires chunked prefill "
|
||||
"(--chunked-prefill-size > 0)."
|
||||
)
|
||||
|
||||
declarations = {}
|
||||
# Chunk boundaries must stay on physical pages. Uneven ownership across
|
||||
# ranks is supported; only the prefix gather pads to a full-group span.
|
||||
if chunk_size % page_size != 0:
|
||||
rounded = (chunk_size + page_size - 1) // page_size * page_size
|
||||
declarations["chunked_prefill_size"] = rounded
|
||||
|
||||
# The raw input, deliberately not the resolving view: the question is
|
||||
# whether the user pinned mem_fraction_static, and the resolved value
|
||||
# would answer yes for every run.
|
||||
raw_mem_fraction = (getattr(server_args, "_raw_input", None) or {}).get(
|
||||
"mem_fraction_static"
|
||||
)
|
||||
activation_delta_mb = 1.5 * (max(rounded, 2048) - max(chunk_size, 2048))
|
||||
if raw_mem_fraction is None and activation_delta_mb > 0:
|
||||
if not gpu_mem or cfg.mem_fraction_static is None:
|
||||
raise ValueError(
|
||||
"Cannot safely auto-round --chunked-prefill-size for KV "
|
||||
"sharding without a known device-memory capacity; pass a "
|
||||
f"multiple of page_size ({page_size})."
|
||||
)
|
||||
corrected_mem_fraction = (
|
||||
float(cfg.mem_fraction_static) - activation_delta_mb / gpu_mem
|
||||
)
|
||||
if corrected_mem_fraction <= 0:
|
||||
raise ValueError(
|
||||
"Auto-rounding --chunked-prefill-size for KV sharding "
|
||||
"would consume all automatically reserved GPU memory; "
|
||||
f"pass a smaller multiple of {page_size}."
|
||||
)
|
||||
declarations["mem_fraction_static"] = corrected_mem_fraction
|
||||
|
||||
logger.warning(
|
||||
"Rounding --chunked-prefill-size up from %d to %d "
|
||||
"(a multiple of page_size = %d) for KV sharding.",
|
||||
chunk_size,
|
||||
rounded,
|
||||
page_size,
|
||||
)
|
||||
|
||||
if cfg.cuda_graph_config.prefill.backend != Backend.DISABLED:
|
||||
logger.warning(
|
||||
"Prefill CUDA graph (%s) is incompatible with "
|
||||
"--enable-kv-cache-sharding; disabling it.",
|
||||
cfg.cuda_graph_config.prefill.backend,
|
||||
)
|
||||
declarations["cuda_graph_config"] = with_phase(
|
||||
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
|
||||
)
|
||||
|
||||
if declarations:
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"handle_kv_cache_sharding",
|
||||
**declarations,
|
||||
)
|
||||
@@ -1705,6 +1705,12 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool:
|
||||
False for any config the runtime won't post-capture-size, else it gets an
|
||||
under-reserved fraction."""
|
||||
cfg = resolving_view(server_args)
|
||||
# Logical-page sharding always runs prefill eagerly. This predicate runs
|
||||
# before the sharding hook disables prefill capture, so gate on the feature
|
||||
# itself to retain eager activation headroom in the memory heuristic.
|
||||
if cfg.enable_kv_cache_sharding:
|
||||
return False
|
||||
|
||||
mla_enabled = use_mla_backend(server_args)
|
||||
if not envs.SGLANG_ENABLE_POST_CAPTURE_KV_SIZING.get():
|
||||
return False
|
||||
|
||||
@@ -6,6 +6,7 @@ from __future__ import annotations
|
||||
from typing import Any
|
||||
|
||||
from sglang.srt.arg_groups.arg_utils import record_fields
|
||||
from sglang.srt.arg_groups.kv_shard_hook import handle_kv_cache_sharding
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
_page_size_default,
|
||||
_pipeline_parallel_overlap_disable,
|
||||
@@ -264,6 +265,10 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
|
||||
run_hook(handle_context_parallelism, server_args)
|
||||
|
||||
# Validate logical-page KV cache sharding after its page size, attention
|
||||
# backend, memory budget, and shard topology have all resolved.
|
||||
run_hook(handle_kv_cache_sharding, server_args)
|
||||
|
||||
from sglang.srt.arg_groups.moe_hook import (
|
||||
handle_a2a_moe,
|
||||
handle_moe_kernel_config,
|
||||
@@ -334,6 +339,11 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
|
||||
finalize_cuda_graph_prefill_max_context(server_args)
|
||||
|
||||
# Capability adjustments can late-force a different attention backend or
|
||||
# disable chunked prefill. Re-run the idempotent sharding gate over the
|
||||
# final resolving view.
|
||||
run_hook(handle_kv_cache_sharding, server_args)
|
||||
|
||||
# Validate after all batch-size declarations are visible.
|
||||
run_hook(validate_deepep_v2_speculative_draft, server_args)
|
||||
run_hook(validate_deepep_v2_dispatch_token_budget, server_args)
|
||||
|
||||
@@ -67,6 +67,7 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
|
||||
"handle_data_parallelism",
|
||||
"handle_load_balance_method",
|
||||
"handle_context_parallelism",
|
||||
"handle_kv_cache_sharding",
|
||||
"handle_moe_kernel_config",
|
||||
"handle_a2a_moe",
|
||||
"handle_eplb_and_dispatch",
|
||||
|
||||
@@ -49,6 +49,7 @@ from sglang.srt.mem_cache.allocator.hisparse import (
|
||||
DeepSeekV4HiSparseTokenToKVPoolAllocator,
|
||||
HiSparseTokenToKVPoolAllocator,
|
||||
)
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import PageInterleavePoolAllocator
|
||||
from sglang.srt.mem_cache.allocator.swa import (
|
||||
PureSWATokenToKVPoolAllocator,
|
||||
SWATokenToKVPoolAllocator,
|
||||
@@ -80,6 +81,16 @@ from sglang.srt.mem_cache.memory_pool import (
|
||||
ReqToTokenPool,
|
||||
get_minimax_sparse_index_dtype,
|
||||
)
|
||||
from sglang.srt.mem_cache.page_interleave import (
|
||||
PageShardSpec,
|
||||
get_kv_shard_group,
|
||||
make_page_shard_spec,
|
||||
)
|
||||
from sglang.srt.mem_cache.page_interleave_pool import (
|
||||
PageInterleaveKVPoolMixin,
|
||||
PageInterleaveMHATokenToKVPool,
|
||||
PageInterleaveMLATokenToKVPool,
|
||||
)
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.platforms import current_platform
|
||||
@@ -229,6 +240,7 @@ def _pp_local_per_request_bytes(
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
from sglang.srt.mem_cache.unified_memory_pool import (
|
||||
UnifiedKVPool,
|
||||
UnifiedPoolBundle,
|
||||
@@ -311,6 +323,10 @@ class KVCacheConfigurator:
|
||||
mambaish_config: Optional[Any] = field(init=False)
|
||||
hybrid_gdn_config: Optional[Any] = field(init=False)
|
||||
hybrid_kda_config: Optional[Any] = field(init=False)
|
||||
kv_shard_rank: Optional[int] = field(init=False)
|
||||
kv_shard_size: int = field(init=False)
|
||||
kv_shard_spec: Optional[PageShardSpec] = field(init=False)
|
||||
kv_shard_group: Optional[GroupCoordinator] = field(init=False)
|
||||
is_hybrid_swa_mtp_draft: bool = field(init=False)
|
||||
draft_swa_full_capacity: bool = field(init=False)
|
||||
|
||||
@@ -318,6 +334,7 @@ class KVCacheConfigurator:
|
||||
self.mambaish_config = mambaish_config(self.model_config)
|
||||
self.hybrid_gdn_config = hybrid_gdn_config(self.model_config)
|
||||
self.hybrid_kda_config = hybrid_kda_config(self.model_config)
|
||||
self._resolve_kv_shard()
|
||||
self.is_hybrid_swa_mtp_draft = self.layer_info.is_hybrid_swa_mtp_draft
|
||||
self.draft_swa_full_capacity = self.is_hybrid_swa_mtp_draft and bool(
|
||||
self.layer_info.swa_attention_layer_ids
|
||||
@@ -351,6 +368,25 @@ class KVCacheConfigurator:
|
||||
max_total_num_tokens * get_parallel().attn_dcp_size // self.loc_space_scale
|
||||
)
|
||||
|
||||
def _resolve_kv_shard(self) -> None:
|
||||
"""Resolve the logical-page KV sharding identity: ``(rank, size)`` of
|
||||
the shard group, or ``(None, 1)`` when sharding is off. Consumed by the
|
||||
pool builders and by the widened allocator branch."""
|
||||
self.kv_shard_spec = make_page_shard_spec(self)
|
||||
self.kv_shard_rank = None
|
||||
self.kv_shard_size = 1
|
||||
self.kv_shard_group = None
|
||||
if self.kv_shard_spec is None:
|
||||
return
|
||||
reject_out_of_tree_path(
|
||||
current_platform,
|
||||
subsystem="logical-page KV sharding (--enable-kv-cache-sharding)",
|
||||
)
|
||||
|
||||
self.kv_shard_rank = self.kv_shard_spec.shard_rank
|
||||
self.kv_shard_size = self.kv_shard_spec.shard_size
|
||||
self.kv_shard_group = get_kv_shard_group(self.use_mla_backend)
|
||||
|
||||
def _build_fp4_quant_method(self, *, num_layers: int):
|
||||
if not is_float4_e2m1fn_x2(self.kv_cache_dtype):
|
||||
return None
|
||||
@@ -689,6 +725,13 @@ class KVCacheConfigurator:
|
||||
req_to_token_pool=req_to_token_pool,
|
||||
)
|
||||
|
||||
if self.kv_shard_rank is not None:
|
||||
if not isinstance(token_to_kv_pool, PageInterleaveKVPoolMixin):
|
||||
raise ValueError(
|
||||
"--enable-kv-cache-sharding does not support this model "
|
||||
f"family (pool {type(token_to_kv_pool).__name__})."
|
||||
)
|
||||
|
||||
if draft_virtual_id_space is not None:
|
||||
assert token_to_kv_pool.size >= draft_virtual_id_space, (
|
||||
"draft token_to_kv_pool smaller than the shared unified "
|
||||
@@ -2099,6 +2142,17 @@ class KVCacheConfigurator:
|
||||
def _build_mla_kv_pool(
|
||||
self, *, max_total_num_tokens: int, mla_pool_class: type
|
||||
) -> KVCache:
|
||||
mla_pool_kwargs = {}
|
||||
if self.kv_shard_rank is not None:
|
||||
# The striped pool subclasses the plain MLA pool; a vendor pool
|
||||
# class would otherwise be silently replaced.
|
||||
assert mla_pool_class is MLATokenToKVPool, (
|
||||
"--enable-kv-cache-sharding is incompatible with "
|
||||
f"the {mla_pool_class.__name__} pool"
|
||||
)
|
||||
mla_pool_class = PageInterleaveMLATokenToKVPool
|
||||
mla_pool_kwargs["shard_spec"] = self.kv_shard_spec
|
||||
mla_pool_kwargs["shard_group"] = self.kv_shard_group
|
||||
token_to_kv_pool = mla_pool_class(
|
||||
max_total_num_tokens,
|
||||
page_size=self.pool_page_size,
|
||||
@@ -2110,6 +2164,7 @@ class KVCacheConfigurator:
|
||||
enable_memory_saver=get_exec().features.enable_memory_saver,
|
||||
start_layer=self.layer_info.start_layer,
|
||||
end_layer=self.layer_info.end_layer,
|
||||
**mla_pool_kwargs,
|
||||
)
|
||||
return token_to_kv_pool
|
||||
|
||||
@@ -2363,6 +2418,23 @@ class KVCacheConfigurator:
|
||||
pool_kwargs["quant_method"] = quant_method
|
||||
else:
|
||||
pool_kwargs["post_capture_active"] = self.post_capture_kv_active
|
||||
if self.kv_shard_rank is not None:
|
||||
# The striped pool subclasses the plain per-layer MHA pool; the
|
||||
# mxfp8 / NoOp / page-major variants have incompatible buffer
|
||||
# layouts. Specialized quantization methods need packed-data and
|
||||
# scale-aware sharded writes/gathers; ordinary CUDA FP8 uses the
|
||||
# plain pool and has no specialized quant_method.
|
||||
assert pool_cls is MHATokenToKVPool, (
|
||||
"--enable-kv-cache-sharding is incompatible with "
|
||||
f"the {pool_cls.__name__} pool"
|
||||
)
|
||||
assert quant_method is None, (
|
||||
"--enable-kv-cache-sharding does not support specialized KV-cache "
|
||||
"quantization methods (e.g. FP4); ordinary CUDA FP8 is supported."
|
||||
)
|
||||
pool_cls = PageInterleaveMHATokenToKVPool
|
||||
pool_kwargs["shard_spec"] = self.kv_shard_spec
|
||||
pool_kwargs["shard_group"] = self.kv_shard_group
|
||||
token_to_kv_pool = pool_cls(
|
||||
max_total_num_tokens,
|
||||
page_size=self.pool_page_size,
|
||||
@@ -2487,6 +2559,19 @@ class KVCacheConfigurator:
|
||||
need_sort=need_sort,
|
||||
host_to_device_ratio=hisparse_cfg.host_to_device_ratio,
|
||||
)
|
||||
elif self.kv_shard_size > 1:
|
||||
# Logical-page sharding: index space widened xN over
|
||||
# the stock 1x pool; the allocator page stays physical.
|
||||
token_to_kv_pool_allocator = PageInterleavePoolAllocator(
|
||||
sizes.max_total_num_tokens,
|
||||
physical_page_size=self.page_size,
|
||||
shard_size=self.kv_shard_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
device=self.device,
|
||||
kvcache=token_to_kv_pool,
|
||||
need_sort=need_sort,
|
||||
shard_spec=self.kv_shard_spec,
|
||||
)
|
||||
elif (
|
||||
get_schedule().page_size == 1 and not get_parallel().dcp_enabled
|
||||
):
|
||||
|
||||
@@ -68,9 +68,11 @@ class PageShardSpec(msgspec.Struct, frozen=True):
|
||||
|
||||
@property
|
||||
def logical_page_size(self) -> int:
|
||||
"""The N*page_size span of one logical page. It bounds the assembly
|
||||
scratch and rounds chunked_prefill_size; the allocator and the tree
|
||||
both keep drawing and matching at the physical ``page_size``."""
|
||||
"""The N*page_size span used to align prefix-gather scratch.
|
||||
|
||||
Allocation, prefix matching, and chunk boundaries use the physical
|
||||
``page_size``.
|
||||
"""
|
||||
return self.shard_size * self.page_size
|
||||
|
||||
|
||||
|
||||
@@ -17,7 +17,11 @@ from typing import TYPE_CHECKING, Any, Callable, Optional
|
||||
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
from sglang.srt.runtime_context import get_disagg, get_memory, get_serving
|
||||
from sglang.srt.runtime_context import (
|
||||
get_disagg,
|
||||
get_memory,
|
||||
get_serving,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
|
||||
@@ -42,6 +42,7 @@ from sglang.srt.mem_cache.memory_pool import (
|
||||
DSATokenToKVPool,
|
||||
get_minimax_sparse_index_dtype,
|
||||
)
|
||||
from sglang.srt.mem_cache.page_interleave import compute_page_shard_scratch_bytes
|
||||
from sglang.srt.runtime_context import (
|
||||
get_disagg,
|
||||
get_exec,
|
||||
@@ -213,7 +214,10 @@ class MemoryPoolConfigurator:
|
||||
|
||||
|
||||
class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
"""Standard models (MHA, MLA, DSA, FP4): coeff = cell_size, bias = 0."""
|
||||
"""Standard models (MHA, MLA, DSA, FP4).
|
||||
|
||||
coeff = cell_size; bias = double-buffered KV-shard scratch, otherwise zero.
|
||||
"""
|
||||
|
||||
def __init__(self, kvc: KVCacheConfigurator):
|
||||
self.kv_cache_dtype_str = kvc.kv_cache_dtype_str
|
||||
@@ -242,6 +246,10 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
else get_schedule().max_total_tokens or kvc.model_config.context_len
|
||||
)
|
||||
|
||||
# Logical-page KV sharding reserves a fixed double-buffered assembly
|
||||
# scratch next to the pool; charge it before token sizing.
|
||||
self._fixed_overhead_bytes = compute_page_shard_scratch_bytes(kvc)
|
||||
|
||||
# EAGLE/STANDALONE: assumes the draft shares the target's per-layer KV size,
|
||||
# which holds for EAGLE/MTP drafts that reuse the target's attention config.
|
||||
if (
|
||||
@@ -598,7 +606,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
def calculate_pool_sizes(
|
||||
self, available_bytes: int, page_size: int
|
||||
) -> MemoryPoolConfig:
|
||||
available_bytes = max(available_bytes, 0)
|
||||
available_bytes = max(available_bytes - self._fixed_overhead_bytes, 0)
|
||||
max_total_num_tokens = (
|
||||
available_bytes // self._cell_size
|
||||
if self._cell_size
|
||||
|
||||
Reference in New Issue
Block a user