From de487f8039e06853b5f376fd5f068ea9d7c400bb Mon Sep 17 00:00:00 2001
From: Shunkangz <182541032+Shunkangz@users.noreply.github.com>
Date: Sat, 10 Oct 2026 22:53:58 +0800
Subject: [PATCH] [kv-shard 3/4] Enable Control Plane C (#40102)
---
.../advanced_features/server_arguments.mdx | 6 +
python/sglang/srt/arg_groups/field_order.py | 1 +
.../sglang/srt/arg_groups/fields/parallel.py | 8 +
python/sglang/srt/arg_groups/kv_shard_hook.py | 282 ++++++++++++++++
python/sglang/srt/arg_groups/overrides.py | 6 +
python/sglang/srt/arg_groups/pipeline.py | 10 +
.../sglang/srt/arg_groups/resolution_hooks.py | 1 +
.../srt/mem_cache/kv_cache_configurator.py | 85 +++++
.../sglang/srt/mem_cache/page_interleave.py | 8 +-
python/sglang/srt/mem_cache/registry.py | 6 +-
.../srt/model_executor/pool_configurator.py | 12 +-
test/registered/dcp/test_dcp_layout_unit.py | 2 +
.../test_mooncake_efa_allocator.py | 1 +
.../unit/mem_cache/test_registry.py | 27 ++
.../model_executor/test_pool_configurator.py | 74 +++++
.../unit/models/test_deepseek_mla_dispatch.py | 108 ++++++-
.../unit/server_args/test_server_args.py | 304 ++++++++++++++++++
17 files changed, 934 insertions(+), 7 deletions(-)
create mode 100644 python/sglang/srt/arg_groups/kv_shard_hook.py
diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx
index 762f425de5..5cecf8b564 100644
--- a/docs/docs/advanced_features/server_arguments.mdx
+++ b/docs/docs/advanced_features/server_arguments.mdx
@@ -2797,6 +2797,12 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec
None |
zigzag, interleave |
+
+ | `--enable-kv-cache-sharding` |
+ Shard at-rest KV cache storage at page granularity across the group that currently replicates it (attention-CP ranks for GQA models under prefill CP; attention-TP ranks for MLA models), multiplying the group's unique KV capacity. Prefill workers of PD disaggregation only. |
+ `False` |
+ bool flag (set to enable) |
+
| `--enable-fused-qk-norm-rope` |
Enable fused qk normalization and rope rotary embedding. |
diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py
index 1611e714b7..0f2f8b799a 100644
--- a/python/sglang/srt/arg_groups/field_order.py
+++ b/python/sglang/srt/arg_groups/field_order.py
@@ -90,6 +90,7 @@ POSITIONAL_FIELD_ORDER = (
"cp_strategy",
"enable_dsa_cache_layer_split",
"enable_cp_decode_attn_tp",
+ "enable_kv_cache_sharding",
"enable_dp_attention",
"enable_dp_attention_local_control_broadcast",
"enable_dp_lm_head",
diff --git a/python/sglang/srt/arg_groups/fields/parallel.py b/python/sglang/srt/arg_groups/fields/parallel.py
index ed3e6a5ce9..4a7a242ed8 100644
--- a/python/sglang/srt/arg_groups/fields/parallel.py
+++ b/python/sglang/srt/arg_groups/fields/parallel.py
@@ -181,6 +181,14 @@ class Parallel(msgspec.Struct):
bool,
"Enable attention tensor-parallel weight slicing during decode under context parallel (cp_size>1). Slices the replicated attention linears to the local CP partition, eliminating redundant decode GEMMs.",
] = False
+ enable_kv_cache_sharding: A[
+ bool,
+ "Shard at-rest KV cache storage at page granularity across the group "
+ "that currently replicates it (attention-CP ranks for GQA models "
+ "under prefill CP; attention-TP ranks for MLA models), multiplying "
+ "the group's unique KV capacity. Prefill workers of PD "
+ "disaggregation only.",
+ ] = False
# Deprecated spelling of `attn_dp_size`: `--dp-size N --enable-dp-attention`
# resolves to `attn_dp_size = N`, `dp_size = 1`. TODO: remove after 2026-12-31.
enable_dp_attention: A[bool, Arg(no_cli=True, resolvable=True)] = False
diff --git a/python/sglang/srt/arg_groups/kv_shard_hook.py b/python/sglang/srt/arg_groups/kv_shard_hook.py
new file mode 100644
index 0000000000..8203c4652d
--- /dev/null
+++ b/python/sglang/srt/arg_groups/kv_shard_hook.py
@@ -0,0 +1,282 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Resolution and validation for logical-page KV cache sharding."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Optional
+
+from sglang.srt.arg_groups.overrides import (
+ attention_backends_of,
+ declare_resolution,
+ model_config_of,
+ resolved_view,
+ resolving_view,
+ use_mla_backend,
+)
+from sglang.srt.configs.model_config import is_deepseek_dsa
+from sglang.srt.environ import envs
+from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
+from sglang.srt.runtime_context import attn_dp_enabled_of
+from sglang.srt.utils.common import get_device_memory_capacity
+
+logger = logging.getLogger(__name__)
+
+
+def validate_kv_shard_attention_backend(server_args: Any, cfg: Any = None) -> None:
+ cfg = cfg or resolving_view(server_args)
+ prefill_backend, _ = attention_backends_of(resolved_view(server_args))
+ if prefill_backend == "fa3":
+ return
+ if prefill_backend != "trtllm_mla":
+ raise ValueError(
+ "--enable-kv-cache-sharding currently requires the fa3 prefill "
+ "attention backend, or trtllm_mla for plain-TP MLA; "
+ f"got {prefill_backend!r}."
+ )
+ if not use_mla_backend(server_args):
+ raise ValueError("KV sharding with trtllm_mla requires an MLA model.")
+ if cfg.attn_cp_size > 1:
+ raise ValueError(
+ "KV sharding with trtllm_mla only supports plain-TP MLA "
+ "(attn_cp_size must be 1)."
+ )
+ if cfg.disable_chunked_prefix_cache:
+ raise ValueError("KV sharding with trtllm_mla requires chunked prefix caching.")
+
+
+def handle_kv_cache_sharding(server_args: Any, gpu_mem: Optional[float] = None) -> None:
+ """Validate KV sharding and declare its derived eager-prefill settings."""
+ cfg = resolving_view(server_args)
+ if not cfg.enable_kv_cache_sharding:
+ return
+
+ if gpu_mem is None:
+ # `handle_gpu_memory_settings` resolves capacity internally now, so the
+ # pipeline no longer threads it through. Only the chunk-size rounding
+ # below reads it, and only when mem_fraction_static was left unset.
+ gpu_mem = get_device_memory_capacity(cfg.device)
+
+ if cfg.disaggregation_mode != "prefill":
+ raise ValueError(
+ "--enable-kv-cache-sharding is only supported on PD prefill "
+ "workers: a rank's pool holds only its stripe of every cached "
+ "page, which is incompatible with local decode."
+ )
+ if cfg.speculative_algorithm is not None:
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support speculative decoding."
+ )
+ if cfg.dllm_algorithm is not None:
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support diffusion language "
+ "models: DLLM admission bypasses the shard-scratch budget and "
+ "resolves out of PD-prefill mode."
+ )
+ if cfg.radix_cache_backend is not None:
+ raise ValueError(
+ "--enable-kv-cache-sharding requires the built-in "
+ "UnifiedRadixCache with a rotation-aware UnifiedTreeCore and does "
+ "not support "
+ f"--radix-cache-backend={cfg.radix_cache_backend!r}."
+ )
+ tree_core_backend = envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get()
+ # A disabled radix cache never matches or inserts, so its tree core never
+ # sees a rotation base; chunked continuations keep the base on the request.
+ if not cfg.disable_radix_cache and tree_core_backend != "python":
+ raise ValueError(
+ "--enable-kv-cache-sharding requires the Python UnifiedTreeCore: "
+ f"SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND={tree_core_backend!r} is "
+ "not supported for sharded KV."
+ )
+ if cfg.enable_unified_cache_external_linker:
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support "
+ "--enable-unified-cache-external-linker: load-back allocates "
+ "through the flat allocator API."
+ )
+ if cfg.enable_streaming_session:
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support streaming sessions: "
+ "their virtual nodes do not carry KV-shard rotation metadata."
+ )
+ if cfg.enable_session_radix_cache:
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support session radix cache yet."
+ )
+ if cfg.enable_flexkv:
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support --enable-flexkv: "
+ "FlexKV load-back allocates through the stock flat allocator API."
+ )
+ if cfg.enable_unified_memory:
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support --enable-unified-memory: "
+ "the unified pool uses a different virtual-id allocator and layout."
+ )
+ if cfg.dcp_size > 1:
+ raise ValueError(
+ "--enable-kv-cache-sharding and decode context parallelism "
+ "(--dcp-size) are mutually exclusive users of the widened "
+ "allocator index space."
+ )
+ if cfg.enable_hierarchical_cache:
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support "
+ "--enable-hierarchical-cache yet: HiCache backup/load assumes "
+ "each rank holds full pages."
+ )
+ if cfg.enable_lmcache:
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support --enable-lmcache."
+ )
+ if cfg.enable_hisparse:
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support --enable-hisparse: "
+ "HiSparse selects a non-sharded allocator and assumes each rank "
+ "owns complete KV pages."
+ )
+ # The dynamic sizer only runs with pipeline parallelism. Its profiling
+ # and predicted chunks can exceed the fixed shard-scratch capacity.
+ if cfg.enable_dynamic_chunking and cfg.pp_size > 1:
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support dynamic chunk sizing "
+ "with pipeline parallelism: profiling and predicted chunks can "
+ "exceed the fixed sharded scratch capacity. Disable "
+ "--enable-dynamic-chunking."
+ )
+ if cfg.enable_two_batch_overlap:
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support "
+ "--enable-two-batch-overlap: the sharded pool's per-batch gather "
+ "plan is a process-wide singleton, but TBO rebuilds attention "
+ "metadata multiple times per extend forward and clobbers the plan."
+ )
+
+ model_config = model_config_of(server_args)
+ if model_config.is_encoder_decoder:
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support encoder-decoder "
+ "models: cross-attention uses a separate encoder page table that "
+ "is not part of the sharded gather plan."
+ )
+ if model_config.attention_chunk_size is not None:
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support chunked/local-attention "
+ "models yet: local-attention metadata is built from the page table "
+ "before the scratch translation."
+ )
+
+ validate_kv_shard_attention_backend(server_args, cfg)
+ if cfg.disaggregation_transfer_backend not in ("mooncake", "nixl"):
+ raise ValueError(
+ "--enable-kv-cache-sharding currently only supports the mooncake "
+ "and nixl transfer backends: both route sends through "
+ "CommonKVSender._prepare_send_indices and pair destinations by "
+ "fancy index. Got --disaggregation-transfer-backend "
+ f"{cfg.disaggregation_transfer_backend!r}."
+ )
+ if envs.SGLANG_DISAGG_STAGING_BUFFER.get():
+ raise ValueError(
+ "--enable-kv-cache-sharding is incompatible with "
+ "SGLANG_DISAGG_STAGING_BUFFER (the staging path assumes "
+ "contiguous send slices)."
+ )
+
+ if use_mla_backend(server_args):
+ if is_deepseek_dsa(model_config.hf_config):
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support DSA models yet "
+ "(indexer buffers are not striped)."
+ )
+ if cfg.attn_cp_size <= 1:
+ if attn_dp_enabled_of(cfg):
+ raise ValueError(
+ "--enable-kv-cache-sharding for MLA models without "
+ "prefill CP shards across attention-TP and requires plain "
+ "TP attention; set --attn-dp-size=1."
+ )
+ if cfg.tp_size <= 1:
+ raise ValueError(
+ "--enable-kv-cache-sharding for MLA models without "
+ "prefill CP shards across attention-TP ranks and needs "
+ "tp_size > 1."
+ )
+ elif not cfg.enable_prefill_cp or cfg.attn_cp_size <= 1:
+ raise ValueError(
+ "--enable-kv-cache-sharding for GQA models shards across "
+ "attention-CP ranks and requires --enable-prefill-cp with "
+ "attn_cp_size > 1."
+ )
+
+ page_size = cfg.page_size if cfg.page_size is not None else 1
+ if page_size <= 1:
+ raise ValueError(
+ "--enable-kv-cache-sharding requires a real page size "
+ "(--page-size 64 recommended): the physical page is the P/D "
+ "transfer descriptor and gather-copy unit; page_size=1 "
+ "degenerates both into per-token operations."
+ )
+ chunk_size = cfg.chunked_prefill_size
+ if chunk_size is None or chunk_size <= 0:
+ raise ValueError(
+ "--enable-kv-cache-sharding requires chunked prefill "
+ "(--chunked-prefill-size > 0)."
+ )
+
+ declarations = {}
+ # Chunk boundaries must stay on physical pages. Uneven ownership across
+ # ranks is supported; only the prefix gather pads to a full-group span.
+ if chunk_size % page_size != 0:
+ rounded = (chunk_size + page_size - 1) // page_size * page_size
+ declarations["chunked_prefill_size"] = rounded
+
+ # The raw input, deliberately not the resolving view: the question is
+ # whether the user pinned mem_fraction_static, and the resolved value
+ # would answer yes for every run.
+ raw_mem_fraction = (getattr(server_args, "_raw_input", None) or {}).get(
+ "mem_fraction_static"
+ )
+ activation_delta_mb = 1.5 * (max(rounded, 2048) - max(chunk_size, 2048))
+ if raw_mem_fraction is None and activation_delta_mb > 0:
+ if not gpu_mem or cfg.mem_fraction_static is None:
+ raise ValueError(
+ "Cannot safely auto-round --chunked-prefill-size for KV "
+ "sharding without a known device-memory capacity; pass a "
+ f"multiple of page_size ({page_size})."
+ )
+ corrected_mem_fraction = (
+ float(cfg.mem_fraction_static) - activation_delta_mb / gpu_mem
+ )
+ if corrected_mem_fraction <= 0:
+ raise ValueError(
+ "Auto-rounding --chunked-prefill-size for KV sharding "
+ "would consume all automatically reserved GPU memory; "
+ f"pass a smaller multiple of {page_size}."
+ )
+ declarations["mem_fraction_static"] = corrected_mem_fraction
+
+ logger.warning(
+ "Rounding --chunked-prefill-size up from %d to %d "
+ "(a multiple of page_size = %d) for KV sharding.",
+ chunk_size,
+ rounded,
+ page_size,
+ )
+
+ if cfg.cuda_graph_config.prefill.backend != Backend.DISABLED:
+ logger.warning(
+ "Prefill CUDA graph (%s) is incompatible with "
+ "--enable-kv-cache-sharding; disabling it.",
+ cfg.cuda_graph_config.prefill.backend,
+ )
+ declarations["cuda_graph_config"] = with_phase(
+ cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
+ )
+
+ if declarations:
+ declare_resolution(
+ server_args,
+ "handle_kv_cache_sharding",
+ **declarations,
+ )
diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py
index eda212a2a5..ed2aa21a1c 100644
--- a/python/sglang/srt/arg_groups/overrides.py
+++ b/python/sglang/srt/arg_groups/overrides.py
@@ -1705,6 +1705,12 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool:
False for any config the runtime won't post-capture-size, else it gets an
under-reserved fraction."""
cfg = resolving_view(server_args)
+ # Logical-page sharding always runs prefill eagerly. This predicate runs
+ # before the sharding hook disables prefill capture, so gate on the feature
+ # itself to retain eager activation headroom in the memory heuristic.
+ if cfg.enable_kv_cache_sharding:
+ return False
+
mla_enabled = use_mla_backend(server_args)
if not envs.SGLANG_ENABLE_POST_CAPTURE_KV_SIZING.get():
return False
diff --git a/python/sglang/srt/arg_groups/pipeline.py b/python/sglang/srt/arg_groups/pipeline.py
index 4669a6628d..e1f8fe47dd 100644
--- a/python/sglang/srt/arg_groups/pipeline.py
+++ b/python/sglang/srt/arg_groups/pipeline.py
@@ -6,6 +6,7 @@ from __future__ import annotations
from typing import Any
from sglang.srt.arg_groups.arg_utils import record_fields
+from sglang.srt.arg_groups.kv_shard_hook import handle_kv_cache_sharding
from sglang.srt.arg_groups.overrides import (
_page_size_default,
_pipeline_parallel_overlap_disable,
@@ -264,6 +265,10 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(handle_context_parallelism, server_args)
+ # Validate logical-page KV cache sharding after its page size, attention
+ # backend, memory budget, and shard topology have all resolved.
+ run_hook(handle_kv_cache_sharding, server_args)
+
from sglang.srt.arg_groups.moe_hook import (
handle_a2a_moe,
handle_moe_kernel_config,
@@ -334,6 +339,11 @@ def run_resolution_pipeline(server_args: Any) -> None:
finalize_cuda_graph_prefill_max_context(server_args)
+ # Capability adjustments can late-force a different attention backend or
+ # disable chunked prefill. Re-run the idempotent sharding gate over the
+ # final resolving view.
+ run_hook(handle_kv_cache_sharding, server_args)
+
# Validate after all batch-size declarations are visible.
run_hook(validate_deepep_v2_speculative_draft, server_args)
run_hook(validate_deepep_v2_dispatch_token_budget, server_args)
diff --git a/python/sglang/srt/arg_groups/resolution_hooks.py b/python/sglang/srt/arg_groups/resolution_hooks.py
index 00fb1c607d..018db11f85 100644
--- a/python/sglang/srt/arg_groups/resolution_hooks.py
+++ b/python/sglang/srt/arg_groups/resolution_hooks.py
@@ -67,6 +67,7 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
"handle_data_parallelism",
"handle_load_balance_method",
"handle_context_parallelism",
+ "handle_kv_cache_sharding",
"handle_moe_kernel_config",
"handle_a2a_moe",
"handle_eplb_and_dispatch",
diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py
index 4da8bc98e3..1b64296f56 100644
--- a/python/sglang/srt/mem_cache/kv_cache_configurator.py
+++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py
@@ -49,6 +49,7 @@ from sglang.srt.mem_cache.allocator.hisparse import (
DeepSeekV4HiSparseTokenToKVPoolAllocator,
HiSparseTokenToKVPoolAllocator,
)
+from sglang.srt.mem_cache.allocator.page_interleave import PageInterleavePoolAllocator
from sglang.srt.mem_cache.allocator.swa import (
PureSWATokenToKVPoolAllocator,
SWATokenToKVPoolAllocator,
@@ -80,6 +81,16 @@ from sglang.srt.mem_cache.memory_pool import (
ReqToTokenPool,
get_minimax_sparse_index_dtype,
)
+from sglang.srt.mem_cache.page_interleave import (
+ PageShardSpec,
+ get_kv_shard_group,
+ make_page_shard_spec,
+)
+from sglang.srt.mem_cache.page_interleave_pool import (
+ PageInterleaveKVPoolMixin,
+ PageInterleaveMHATokenToKVPool,
+ PageInterleaveMLATokenToKVPool,
+)
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.platforms import current_platform
@@ -229,6 +240,7 @@ def _pp_local_per_request_bytes(
if TYPE_CHECKING:
+ from sglang.srt.distributed.parallel_state import GroupCoordinator
from sglang.srt.mem_cache.unified_memory_pool import (
UnifiedKVPool,
UnifiedPoolBundle,
@@ -311,6 +323,10 @@ class KVCacheConfigurator:
mambaish_config: Optional[Any] = field(init=False)
hybrid_gdn_config: Optional[Any] = field(init=False)
hybrid_kda_config: Optional[Any] = field(init=False)
+ kv_shard_rank: Optional[int] = field(init=False)
+ kv_shard_size: int = field(init=False)
+ kv_shard_spec: Optional[PageShardSpec] = field(init=False)
+ kv_shard_group: Optional[GroupCoordinator] = field(init=False)
is_hybrid_swa_mtp_draft: bool = field(init=False)
draft_swa_full_capacity: bool = field(init=False)
@@ -318,6 +334,7 @@ class KVCacheConfigurator:
self.mambaish_config = mambaish_config(self.model_config)
self.hybrid_gdn_config = hybrid_gdn_config(self.model_config)
self.hybrid_kda_config = hybrid_kda_config(self.model_config)
+ self._resolve_kv_shard()
self.is_hybrid_swa_mtp_draft = self.layer_info.is_hybrid_swa_mtp_draft
self.draft_swa_full_capacity = self.is_hybrid_swa_mtp_draft and bool(
self.layer_info.swa_attention_layer_ids
@@ -351,6 +368,25 @@ class KVCacheConfigurator:
max_total_num_tokens * get_parallel().attn_dcp_size // self.loc_space_scale
)
+ def _resolve_kv_shard(self) -> None:
+ """Resolve the logical-page KV sharding identity: ``(rank, size)`` of
+ the shard group, or ``(None, 1)`` when sharding is off. Consumed by the
+ pool builders and by the widened allocator branch."""
+ self.kv_shard_spec = make_page_shard_spec(self)
+ self.kv_shard_rank = None
+ self.kv_shard_size = 1
+ self.kv_shard_group = None
+ if self.kv_shard_spec is None:
+ return
+ reject_out_of_tree_path(
+ current_platform,
+ subsystem="logical-page KV sharding (--enable-kv-cache-sharding)",
+ )
+
+ self.kv_shard_rank = self.kv_shard_spec.shard_rank
+ self.kv_shard_size = self.kv_shard_spec.shard_size
+ self.kv_shard_group = get_kv_shard_group(self.use_mla_backend)
+
def _build_fp4_quant_method(self, *, num_layers: int):
if not is_float4_e2m1fn_x2(self.kv_cache_dtype):
return None
@@ -689,6 +725,13 @@ class KVCacheConfigurator:
req_to_token_pool=req_to_token_pool,
)
+ if self.kv_shard_rank is not None:
+ if not isinstance(token_to_kv_pool, PageInterleaveKVPoolMixin):
+ raise ValueError(
+ "--enable-kv-cache-sharding does not support this model "
+ f"family (pool {type(token_to_kv_pool).__name__})."
+ )
+
if draft_virtual_id_space is not None:
assert token_to_kv_pool.size >= draft_virtual_id_space, (
"draft token_to_kv_pool smaller than the shared unified "
@@ -2099,6 +2142,17 @@ class KVCacheConfigurator:
def _build_mla_kv_pool(
self, *, max_total_num_tokens: int, mla_pool_class: type
) -> KVCache:
+ mla_pool_kwargs = {}
+ if self.kv_shard_rank is not None:
+ # The striped pool subclasses the plain MLA pool; a vendor pool
+ # class would otherwise be silently replaced.
+ assert mla_pool_class is MLATokenToKVPool, (
+ "--enable-kv-cache-sharding is incompatible with "
+ f"the {mla_pool_class.__name__} pool"
+ )
+ mla_pool_class = PageInterleaveMLATokenToKVPool
+ mla_pool_kwargs["shard_spec"] = self.kv_shard_spec
+ mla_pool_kwargs["shard_group"] = self.kv_shard_group
token_to_kv_pool = mla_pool_class(
max_total_num_tokens,
page_size=self.pool_page_size,
@@ -2110,6 +2164,7 @@ class KVCacheConfigurator:
enable_memory_saver=get_exec().features.enable_memory_saver,
start_layer=self.layer_info.start_layer,
end_layer=self.layer_info.end_layer,
+ **mla_pool_kwargs,
)
return token_to_kv_pool
@@ -2363,6 +2418,23 @@ class KVCacheConfigurator:
pool_kwargs["quant_method"] = quant_method
else:
pool_kwargs["post_capture_active"] = self.post_capture_kv_active
+ if self.kv_shard_rank is not None:
+ # The striped pool subclasses the plain per-layer MHA pool; the
+ # mxfp8 / NoOp / page-major variants have incompatible buffer
+ # layouts. Specialized quantization methods need packed-data and
+ # scale-aware sharded writes/gathers; ordinary CUDA FP8 uses the
+ # plain pool and has no specialized quant_method.
+ assert pool_cls is MHATokenToKVPool, (
+ "--enable-kv-cache-sharding is incompatible with "
+ f"the {pool_cls.__name__} pool"
+ )
+ assert quant_method is None, (
+ "--enable-kv-cache-sharding does not support specialized KV-cache "
+ "quantization methods (e.g. FP4); ordinary CUDA FP8 is supported."
+ )
+ pool_cls = PageInterleaveMHATokenToKVPool
+ pool_kwargs["shard_spec"] = self.kv_shard_spec
+ pool_kwargs["shard_group"] = self.kv_shard_group
token_to_kv_pool = pool_cls(
max_total_num_tokens,
page_size=self.pool_page_size,
@@ -2487,6 +2559,19 @@ class KVCacheConfigurator:
need_sort=need_sort,
host_to_device_ratio=hisparse_cfg.host_to_device_ratio,
)
+ elif self.kv_shard_size > 1:
+ # Logical-page sharding: index space widened xN over
+ # the stock 1x pool; the allocator page stays physical.
+ token_to_kv_pool_allocator = PageInterleavePoolAllocator(
+ sizes.max_total_num_tokens,
+ physical_page_size=self.page_size,
+ shard_size=self.kv_shard_size,
+ dtype=self.kv_cache_dtype,
+ device=self.device,
+ kvcache=token_to_kv_pool,
+ need_sort=need_sort,
+ shard_spec=self.kv_shard_spec,
+ )
elif (
get_schedule().page_size == 1 and not get_parallel().dcp_enabled
):
diff --git a/python/sglang/srt/mem_cache/page_interleave.py b/python/sglang/srt/mem_cache/page_interleave.py
index a9e5c837dd..3764cca215 100644
--- a/python/sglang/srt/mem_cache/page_interleave.py
+++ b/python/sglang/srt/mem_cache/page_interleave.py
@@ -68,9 +68,11 @@ class PageShardSpec(msgspec.Struct, frozen=True):
@property
def logical_page_size(self) -> int:
- """The N*page_size span of one logical page. It bounds the assembly
- scratch and rounds chunked_prefill_size; the allocator and the tree
- both keep drawing and matching at the physical ``page_size``."""
+ """The N*page_size span used to align prefix-gather scratch.
+
+ Allocation, prefix matching, and chunk boundaries use the physical
+ ``page_size``.
+ """
return self.shard_size * self.page_size
diff --git a/python/sglang/srt/mem_cache/registry.py b/python/sglang/srt/mem_cache/registry.py
index 25343d74f5..9f42286bb5 100644
--- a/python/sglang/srt/mem_cache/registry.py
+++ b/python/sglang/srt/mem_cache/registry.py
@@ -17,7 +17,11 @@ from typing import TYPE_CHECKING, Any, Callable, Optional
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
-from sglang.srt.runtime_context import get_disagg, get_memory, get_serving
+from sglang.srt.runtime_context import (
+ get_disagg,
+ get_memory,
+ get_serving,
+)
if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py
index 38da3198b7..7f20cc0953 100644
--- a/python/sglang/srt/model_executor/pool_configurator.py
+++ b/python/sglang/srt/model_executor/pool_configurator.py
@@ -42,6 +42,7 @@ from sglang.srt.mem_cache.memory_pool import (
DSATokenToKVPool,
get_minimax_sparse_index_dtype,
)
+from sglang.srt.mem_cache.page_interleave import compute_page_shard_scratch_bytes
from sglang.srt.runtime_context import (
get_disagg,
get_exec,
@@ -213,7 +214,10 @@ class MemoryPoolConfigurator:
class DefaultPoolConfigurator(MemoryPoolConfigurator):
- """Standard models (MHA, MLA, DSA, FP4): coeff = cell_size, bias = 0."""
+ """Standard models (MHA, MLA, DSA, FP4).
+
+ coeff = cell_size; bias = double-buffered KV-shard scratch, otherwise zero.
+ """
def __init__(self, kvc: KVCacheConfigurator):
self.kv_cache_dtype_str = kvc.kv_cache_dtype_str
@@ -242,6 +246,10 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
else get_schedule().max_total_tokens or kvc.model_config.context_len
)
+ # Logical-page KV sharding reserves a fixed double-buffered assembly
+ # scratch next to the pool; charge it before token sizing.
+ self._fixed_overhead_bytes = compute_page_shard_scratch_bytes(kvc)
+
# EAGLE/STANDALONE: assumes the draft shares the target's per-layer KV size,
# which holds for EAGLE/MTP drafts that reuse the target's attention config.
if (
@@ -598,7 +606,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
def calculate_pool_sizes(
self, available_bytes: int, page_size: int
) -> MemoryPoolConfig:
- available_bytes = max(available_bytes, 0)
+ available_bytes = max(available_bytes - self._fixed_overhead_bytes, 0)
max_total_num_tokens = (
available_bytes // self._cell_size
if self._cell_size
diff --git a/test/registered/dcp/test_dcp_layout_unit.py b/test/registered/dcp/test_dcp_layout_unit.py
index 6adcc4a789..c19b3d656e 100644
--- a/test/registered/dcp/test_dcp_layout_unit.py
+++ b/test/registered/dcp/test_dcp_layout_unit.py
@@ -443,6 +443,8 @@ class TestGetDcpLens(CustomTestCase):
kv_cache_dtype=torch.bfloat16,
device="cpu",
is_draft_worker=False,
+ kv_shard_size=1,
+ kv_shard_spec=None,
)
# The allocator widens from get_parallel(), not from the injected
# server_args stand-in -- drive the cause, not the effect.
diff --git a/test/registered/unit/disaggregation/test_mooncake_efa_allocator.py b/test/registered/unit/disaggregation/test_mooncake_efa_allocator.py
index 2ed4793bad..4d2b17e604 100644
--- a/test/registered/unit/disaggregation/test_mooncake_efa_allocator.py
+++ b/test/registered/unit/disaggregation/test_mooncake_efa_allocator.py
@@ -75,6 +75,7 @@ class TestMooncakeEfaAllocator(unittest.TestCase):
def test_efa_disables_post_capture_kv_sizing(self):
cfg = SimpleNamespace(
+ enable_kv_cache_sharding=False,
enable_unified_memory=False,
device="cuda",
dcp_size=1,
diff --git a/test/registered/unit/mem_cache/test_registry.py b/test/registered/unit/mem_cache/test_registry.py
index 3d42211dcf..cbc1f6df36 100644
--- a/test/registered/unit/mem_cache/test_registry.py
+++ b/test/registered/unit/mem_cache/test_registry.py
@@ -44,6 +44,7 @@ def _make_ctx(
enable_hierarchical_cache=False,
disable_radix_cache=False,
full_tokens_per_layer=None,
+ enable_kv_cache_sharding=False,
):
# The factory reads the published bags for the cache-backend leaves, so the
# fixture publishes them; the instance stays for the whole-object contract
@@ -55,6 +56,7 @@ def _make_ctx(
enable_lmcache=enable_lmcache,
enable_flexkv=False,
enable_unified_cache_external_linker=False,
+ enable_kv_cache_sharding=enable_kv_cache_sharding,
)
return TreeCacheBuildContext(
server_args=server_args,
@@ -155,6 +157,31 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase):
create_unified.assert_called_once_with(ctx)
self.assertIs(result, create_unified.return_value)
+ def test_kv_sharding_with_disable_radix_routes_to_unified(self):
+ ctx = _make_ctx(self, disable_radix_cache=True, enable_kv_cache_sharding=True)
+ with patch(
+ "sglang.srt.mem_cache.registry.create_unified_radix_cache"
+ ) as create_unified:
+ result = default_radix_cache_factory(ctx)
+ create_unified.assert_called_once_with(ctx)
+ self.assertIs(result, create_unified.return_value)
+
+ def test_kv_sharding_uses_unified_radix_cache(self):
+ ctx = _make_ctx(self, enable_kv_cache_sharding=True)
+ fake_components = MagicMock()
+ fake_radix = MagicMock()
+ with patch.dict(
+ "sys.modules",
+ {
+ "sglang.srt.mem_cache.unified_cache.components": fake_components,
+ "sglang.srt.mem_cache.unified_radix_cache": fake_radix,
+ },
+ ):
+ result = default_radix_cache_factory(ctx)
+
+ fake_radix.UnifiedRadixCache.assert_called_once_with(ctx.params)
+ self.assertIs(result, fake_radix.UnifiedRadixCache.return_value)
+
def test_hybrid_swa_with_disable_radix_routes_to_unified(self):
ctx = _make_ctx(
self,
diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py
index 7d20eb7be6..bf5700b96a 100644
--- a/test/registered/unit/model_executor/test_pool_configurator.py
+++ b/test/registered/unit/model_executor/test_pool_configurator.py
@@ -6,7 +6,12 @@ from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from sglang.srt.configs.model_config import AttentionArch
+from sglang.srt.mem_cache.page_interleave import (
+ compute_page_shard_scratch_bytes,
+ make_page_shard_spec,
+)
from sglang.srt.runtime_context import (
+ get_context,
get_memory,
get_parallel,
get_schedule,
@@ -249,6 +254,75 @@ class TestDefaultConfigurator(CustomTestCase):
_, _, config = self._run(available, page_size=128)
self.assertEqual(config.max_total_num_tokens % 128, 0)
+ def test_eight_context_scratch_is_reserved_before_kv_pool_sizing(self):
+ page_size = 16
+ mr = _make_model_runner(
+ self,
+ use_mla_backend=True,
+ num_layers=3,
+ page_size=page_size,
+ chunked_prefill_size=1024,
+ max_running_requests=1,
+ )
+ mr.model_config.context_len = 8193
+ with (
+ mock_cpu_env(),
+ patch(
+ "sglang.srt.mem_cache.page_interleave.get_kv_shard_group_info",
+ return_value=(0, 4),
+ ),
+ get_context().override_server_args(
+ prefill_max_requests=1,
+ chunked_prefill_size=1024,
+ max_running_requests=1,
+ ),
+ ):
+ spec = make_page_shard_spec(mr)
+ # Each context rounds to 8256 tokens, a 64-token gather multiple,
+ # even with request limits of one. Only the prefix is multiplied.
+ self.assertEqual(spec.max_prefix_tokens, 8 * 8256)
+ self.assertEqual(spec.chunk_tokens, 1024)
+ row_bytes = (512 + 64) * KV_SIZE
+ scratch_bytes = 2 * (8 * 8256 + 1024 + page_size) * row_bytes
+ self.assertEqual(compute_page_shard_scratch_bytes(mr), scratch_bytes)
+
+ # Scratch has two one-layer slots; persistent KV spans all three
+ # layers. Charge the former before dividing by the latter's cost.
+ cell_bytes = row_bytes * mr.num_effective_layers
+ page_bytes = page_size * cell_bytes
+ from sglang.srt.model_executor.pool_configurator import (
+ create_memory_pool_configurator,
+ )
+
+ configurator = create_memory_pool_configurator(mr)
+ for desired_tokens in (page_size, 128):
+ for slack_bytes in (0, page_bytes - 1):
+ with self.subTest(
+ desired_tokens=desired_tokens, slack_bytes=slack_bytes
+ ):
+ budget = (
+ scratch_bytes + desired_tokens * cell_bytes + slack_bytes
+ )
+ config = configurator.calculate_pool_sizes(budget, page_size)
+ self.assertEqual(config.max_total_num_tokens, desired_tokens)
+ self.assertEqual(
+ budget
+ - scratch_bytes
+ - config.max_total_num_tokens * cell_bytes,
+ slack_bytes,
+ )
+
+ # The fixed eight-context scratch is never silently reduced to
+ # leave room for persistent KV, including a sub-page remainder.
+ for budget in (
+ scratch_bytes - 1,
+ scratch_bytes,
+ scratch_bytes + page_bytes - 1,
+ ):
+ with self.subTest(insufficient_budget=budget):
+ with self.assertRaisesRegex(RuntimeError, "Not enough memory"):
+ configurator.calculate_pool_sizes(budget, page_size)
+
def test_constraint_respected(self):
"""calculate_pool_sizes_from_max_tokens respects the limit."""
mr, cfg, config = self._run(10_000_000)
diff --git a/test/registered/unit/models/test_deepseek_mla_dispatch.py b/test/registered/unit/models/test_deepseek_mla_dispatch.py
index dbd1448574..91173aa933 100644
--- a/test/registered/unit/models/test_deepseek_mla_dispatch.py
+++ b/test/registered/unit/models/test_deepseek_mla_dispatch.py
@@ -1,4 +1,4 @@
-"""Hermetic unit tests for DeepSeek MLA attention-method dispatch on ROCm.
+"""Hermetic unit tests for DeepSeek MLA attention-method dispatch.
`_dispatch_mla_subtype` picks the forward method for MLA attention. On ROCm the
fused-decode-MLA + fused-RoPE fast path (`MLA_FUSED_ROPE_ROCM`) is only correct
@@ -142,5 +142,111 @@ class TestCPMLADispatch(CustomTestCase):
)
+class TestKVShardedMLADispatch(CustomTestCase):
+ def setUp(self):
+ super().setUp()
+ self.attn = SimpleNamespace(
+ chunked_prefix_cache_threshold=0,
+ disable_chunked_prefix_cache=False,
+ flashinfer_mla_disable_ragged=False,
+ )
+ self.exec_config = SimpleNamespace(
+ deterministic=SimpleNamespace(enable_deterministic_inference=False)
+ )
+ for patcher in (
+ mock.patch.object(abh, "_is_hip", False),
+ mock.patch.object(
+ abh,
+ "get_parallel",
+ return_value=SimpleNamespace(enable_kv_cache_sharding=True),
+ ),
+ mock.patch.object(abh, "get_exec", return_value=self.exec_config),
+ mock.patch.object(abh, "is_in_tc_piecewise_cuda_graph", return_value=False),
+ mock.patch.object(abh, "is_in_breakable_cuda_graph", return_value=False),
+ mock.patch.object(cp_base, "_STRATEGY", None),
+ ):
+ patcher.start()
+ self.addCleanup(patcher.stop)
+
+ def _batch(self, prefix=64, capacity=8192):
+ num_tokens = 3952
+ return SimpleNamespace(
+ forward_mode=ForwardMode.EXTEND,
+ input_ids=range(num_tokens),
+ attn_cp_metadata=None,
+ extend_prefix_lens_cpu=[prefix],
+ extend_seq_lens_cpu=[num_tokens],
+ seq_lens_cpu=[prefix + num_tokens],
+ get_max_chunk_capacity=lambda: capacity,
+ )
+
+ def test_fa3_disabled_chunked_prefix_cache_uses_absorbed_mla(self):
+ # A cached prefix or the second prompt chunk must not enter FA3's
+ # chunked-prefix path, which asserts prefix caching is enabled.
+ self.attn.disable_chunked_prefix_cache = True
+ self.assertEqual(
+ abh.handle_attention_fa3(self.attn, self._batch()),
+ AttnForwardMethod.MLA,
+ )
+
+ def test_fa3_deterministic_inference_uses_absorbed_mla(self):
+ self.exec_config.deterministic.enable_deterministic_inference = True
+ for prefix in (0, 64):
+ with self.subTest(prefix=prefix):
+ self.assertEqual(
+ abh.handle_attention_fa3(self.attn, self._batch(prefix=prefix)),
+ AttnForwardMethod.MLA,
+ )
+
+ def test_fa3_preserves_capacity_based_mha_dispatch(self):
+ # Both readers can use sharded scratch. Retain one-shot attention
+ # when the complete sequence fits, and chunked MHA otherwise.
+ for prefix in (0, 64):
+ for capacity, expected in (
+ (8192, AttnForwardMethod.MHA_ONE_SHOT),
+ (0, AttnForwardMethod.MHA_CHUNKED_KV),
+ ):
+ with self.subTest(prefix=prefix, capacity=capacity):
+ self.assertEqual(
+ abh.handle_attention_fa3(
+ self.attn, self._batch(prefix=prefix, capacity=capacity)
+ ),
+ expected,
+ )
+
+ def test_fa3_short_prefix_uses_absorbed_mla(self):
+ self.attn.chunked_prefix_cache_threshold = 4096
+ self.assertEqual(
+ abh.handle_attention_fa3(self.attn, self._batch()),
+ AttnForwardMethod.MLA,
+ )
+
+ def test_fa3_active_cp_uses_absorbed_mla(self):
+ # CP must gather rank-local latent activations before the pool write.
+ with mock.patch.object(cp_base, "_STRATEGY", ZigzagCPStrategy(cp_size=4)):
+ for prefix in (0, 64):
+ for capacity in (0, 8192):
+ with self.subTest(prefix=prefix, capacity=capacity):
+ self.assertEqual(
+ abh.handle_attention_fa3(
+ self.attn,
+ self._batch(prefix=prefix, capacity=capacity),
+ ),
+ AttnForwardMethod.MLA,
+ )
+
+ def test_trtllm_mla_keeps_chunked_mha_for_prefill(self):
+ # TRTLLM has a separate dispatcher and requires prefix caching for
+ # sharding; changing FA3 dispatch must not change its supported path.
+ for prefix in (0, 64):
+ with self.subTest(prefix=prefix):
+ self.assertEqual(
+ abh.handle_attention_trtllm_mla(
+ self.attn, self._batch(prefix=prefix)
+ ),
+ AttnForwardMethod.MHA_CHUNKED_KV,
+ )
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py
index ee79af3019..a4e0301388 100644
--- a/test/registered/unit/server_args/test_server_args.py
+++ b/test/registered/unit/server_args/test_server_args.py
@@ -42,6 +42,10 @@ from sglang.srt.arg_groups.kv_cache_hook import (
handle_nvfp4_prefill_kv_dequant_dtype,
validate_prefill_only_disable_kv_cache_args,
)
+from sglang.srt.arg_groups.kv_shard_hook import (
+ handle_kv_cache_sharding,
+ validate_kv_shard_attention_backend,
+)
from sglang.srt.arg_groups.mamba_hook import handle_mamba_backend
from sglang.srt.arg_groups.memory_hook import handle_gpu_memory_settings
from sglang.srt.arg_groups.model_path_hook import handle_load_format
@@ -51,7 +55,9 @@ from sglang.srt.arg_groups.moe_hook import (
validate_deepep_v2_speculative_draft,
)
from sglang.srt.arg_groups.overrides import (
+ declare_resolution,
max_speculative_num_draft_tokens,
+ post_capture_kv_sizing_planned,
resolution_result,
)
from sglang.srt.arg_groups.parallel_hook import (
@@ -75,6 +81,7 @@ from sglang.srt.arg_groups.validation_hook import (
check_pipeline_parallel_compat,
check_two_batch_overlap,
)
+from sglang.srt.configs.model_config import AttentionArch
from sglang.srt.entrypoints.sidecar import (
SGLANG_GRPC_ENDPOINT_ENV,
Sidecar,
@@ -106,6 +113,7 @@ from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import (
DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN,
CustomTestCase,
+ enter_scope,
)
register_cpu_ci(est_time=14, suite="base-a-test-cpu")
@@ -3719,6 +3727,302 @@ class TestTwoBatchOverlapBackend(CustomTestCase):
check_two_batch_overlap(args)
+class TestKvCacheShardingCompatibility(CustomTestCase):
+ def setUp(self):
+ super().setUp()
+ # Sharding currently requires Python, independent of the default backend.
+ enter_scope(
+ self, envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override("python")
+ )
+
+ def _args(self, **overrides):
+ model_config = overrides.pop("model_config", None)
+ args = ServerArgs(
+ model_path="dummy",
+ enable_kv_cache_sharding=True,
+ disaggregation_mode="prefill",
+ )
+ for key, value in overrides.items():
+ setattr(args, key, value)
+ args._model_config = model_config or SimpleNamespace(
+ is_encoder_decoder=False,
+ attention_chunk_size=None,
+ attention_arch=AttentionArch.MHA,
+ hf_config=SimpleNamespace(architectures=["LlamaForCausalLM"]),
+ )
+ return args
+
+ def _rounding_args(self, *, raw_mem_fraction, resolved_mem_fraction):
+ args = self._args(
+ tp_size=2,
+ page_size=64,
+ chunked_prefill_size=2050,
+ attention_backend="fa3",
+ disaggregation_transfer_backend="mooncake",
+ mem_fraction_static=resolved_mem_fraction,
+ cuda_graph_config=CudaGraphConfig(
+ prefill=PhaseConfig(backend=Backend.DISABLED)
+ ),
+ )
+ args._model_config = SimpleNamespace(
+ is_encoder_decoder=False,
+ attention_chunk_size=None,
+ attention_arch=AttentionArch.MLA,
+ hf_config=SimpleNamespace(architectures=["LlamaForCausalLM"]),
+ )
+ args._raw_input = {"mem_fraction_static": raw_mem_fraction}
+ args._resolved_overrides = [
+ (
+ "_handle_gpu_memory_settings",
+ {"mem_fraction_static": resolved_mem_fraction},
+ )
+ ]
+ return args
+
+ def test_encoder_decoder_is_rejected_before_backend_setup(self):
+ args = self._args(model_config=SimpleNamespace(is_encoder_decoder=True))
+
+ with self.assertRaisesRegex(ValueError, "does not support encoder-decoder"):
+ handle_kv_cache_sharding(args)
+
+ def test_trtllm_mla_accepts_plain_tp_with_chunked_prefix_cache(self):
+ args = self._rounding_args(raw_mem_fraction=0.8, resolved_mem_fraction=0.8)
+ args.attention_backend = "trtllm_mla"
+
+ with (
+ patch.object(envs.SGLANG_DISAGG_STAGING_BUFFER, "get", return_value=False),
+ ):
+ handle_kv_cache_sharding(args, 80 * 1024)
+
+ self.assertEqual(resolution_result(args, "chunked_prefill_size"), 2112)
+
+ def test_mla_without_cp_rejects_attention_dp(self):
+ for backend in ("fa3", "trtllm_mla"):
+ with self.subTest(backend=backend):
+ args = self._rounding_args(
+ raw_mem_fraction=0.8, resolved_mem_fraction=0.8
+ )
+ args.attention_backend = backend
+ args.attn_dp_size = 2
+
+ with (
+ envs.SGLANG_DISAGG_STAGING_BUFFER.override(False),
+ self.assertRaisesRegex(ValueError, "set --attn-dp-size=1"),
+ ):
+ handle_kv_cache_sharding(args, 80 * 1024)
+
+ def test_mla_without_cp_rejects_normalized_legacy_attention_dp(self):
+ for backend in ("fa3", "trtllm_mla"):
+ with self.subTest(backend=backend):
+ args = self._rounding_args(
+ raw_mem_fraction=0.8, resolved_mem_fraction=0.8
+ )
+ args.attention_backend = backend
+ args.dp_size = 2
+ args.enable_dp_attention = True
+ parallel_hook.handle_deprecated_dp_attention(args)
+ self.assertFalse(resolution_result(args, "enable_dp_attention"))
+ self.assertEqual(resolution_result(args, "attn_dp_size"), 2)
+
+ with (
+ envs.SGLANG_DISAGG_STAGING_BUFFER.override(False),
+ self.assertRaisesRegex(ValueError, "set --attn-dp-size=1"),
+ ):
+ handle_kv_cache_sharding(args, 80 * 1024)
+
+ def test_mla_without_cp_allows_data_parallel_replicas(self):
+ args = self._rounding_args(raw_mem_fraction=0.8, resolved_mem_fraction=0.8)
+ args.dp_size = 2
+
+ with envs.SGLANG_DISAGG_STAGING_BUFFER.override(False):
+ handle_kv_cache_sharding(args, 80 * 1024)
+
+ self.assertEqual(resolution_result(args, "chunked_prefill_size"), 2112)
+
+ def test_trtllm_mla_rejects_unsupported_shard_topologies(self):
+ cases = (
+ (False, 1, False, "requires an MLA model"),
+ (True, 2, False, "only supports plain-TP MLA"),
+ (True, 1, True, "requires chunked prefix caching"),
+ )
+ for is_mla, attn_cp_size, disable_chunked_prefix_cache, error in cases:
+ with self.subTest(error=error):
+ args = self._args()
+ args.attention_backend = "trtllm_mla"
+ args._model_config.attention_arch = (
+ AttentionArch.MLA if is_mla else AttentionArch.MHA
+ )
+ view = SimpleNamespace(
+ attn_cp_size=attn_cp_size,
+ disable_chunked_prefix_cache=disable_chunked_prefix_cache,
+ )
+
+ with self.assertRaisesRegex(ValueError, error):
+ validate_kv_shard_attention_backend(args, view)
+
+ def test_hisparse_allocator_is_rejected(self):
+ args = self._args(enable_hisparse=True)
+
+ with self.assertRaisesRegex(ValueError, "does not support --enable-hisparse"):
+ handle_kv_cache_sharding(args)
+
+ def test_incompatible_cache_tree_modes_are_rejected(self):
+ cases = (
+ ({"radix_cache_backend": "custom"}, "built-in UnifiedRadixCache"),
+ (
+ {"enable_unified_cache_external_linker": True},
+ "does not support --enable-unified-cache-external-linker",
+ ),
+ (
+ {"enable_streaming_session": True},
+ "does not support streaming sessions",
+ ),
+ (
+ {"enable_session_radix_cache": True},
+ "does not support session radix cache yet",
+ ),
+ )
+ for overrides, error in cases:
+ with self.subTest(overrides=overrides):
+ with self.assertRaisesRegex(ValueError, error):
+ handle_kv_cache_sharding(self._args(**overrides))
+
+ def test_rust_unified_tree_core_is_rejected(self):
+ with envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override("rust"):
+ with self.assertRaisesRegex(ValueError, "requires the Python"):
+ handle_kv_cache_sharding(self._args())
+
+ def test_rust_tree_core_is_allowed_with_radix_disabled(self):
+ args = self._rounding_args(raw_mem_fraction=0.8, resolved_mem_fraction=0.8)
+ args.disable_radix_cache = True
+ with (
+ envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override("rust"),
+ envs.SGLANG_DISAGG_STAGING_BUFFER.override(False),
+ ):
+ handle_kv_cache_sharding(args, 80 * 1024)
+
+ self.assertEqual(resolution_result(args, "chunked_prefill_size"), 2112)
+
+ def test_dynamic_chunking_is_rejected_only_when_pipeline_parallelism_uses_it(self):
+ for pp_size in (1, 2):
+ with self.subTest(pp_size=pp_size):
+ args = self._rounding_args(
+ raw_mem_fraction=0.8, resolved_mem_fraction=0.8
+ )
+ args.enable_dynamic_chunking = True
+ args.pp_size = pp_size
+ with envs.SGLANG_DISAGG_STAGING_BUFFER.override(False):
+ if pp_size > 1:
+ with self.assertRaisesRegex(
+ ValueError, "dynamic chunk sizing with pipeline parallelism"
+ ):
+ handle_kv_cache_sharding(args, 80 * 1024)
+ else:
+ handle_kv_cache_sharding(args, 80 * 1024)
+ self.assertEqual(
+ resolution_result(args, "chunked_prefill_size"), 2112
+ )
+
+ def test_post_capture_kv_sizing_keeps_eager_prefill_headroom(self):
+ prefill_graph = SimpleNamespace(backend=Backend.BREAKABLE, bs=[4096])
+ args = ServerArgs(
+ model_path="dummy",
+ enable_kv_cache_sharding=True,
+ device="cuda",
+ dcp_size=1,
+ kv_cache_dtype="auto",
+ prefill_only_disable_kv_cache=False,
+ enable_memory_saver=False,
+ disaggregation_mode="prefill",
+ cuda_graph_config=CudaGraphConfig(prefill=prefill_graph),
+ chunked_prefill_size=4096,
+ )
+ with (
+ patch(
+ "sglang.srt.arg_groups.overrides.use_mla_backend",
+ return_value=False,
+ ),
+ patch(
+ "sglang.srt.arg_groups.overrides.max_prefill_buffer_tokens",
+ return_value=4096,
+ ),
+ patch(
+ "sglang.srt.arg_groups.overrides.model_config_of",
+ return_value=SimpleNamespace(
+ hf_config=SimpleNamespace(architectures=["LlamaForCausalLM"])
+ ),
+ ),
+ patch.object(
+ envs.SGLANG_ENABLE_POST_CAPTURE_KV_SIZING,
+ "get",
+ return_value=True,
+ ),
+ patch.object(
+ envs.SGLANG_MOONCAKE_CUSTOM_MEM_POOL,
+ "get",
+ return_value=None,
+ ),
+ ):
+ self.assertFalse(post_capture_kv_sizing_planned(args))
+ args.enable_kv_cache_sharding = False
+ self.assertTrue(post_capture_kv_sizing_planned(args))
+
+ def test_chunk_rounding_expands_automatic_activation_headroom(self):
+ args = self._rounding_args(raw_mem_fraction=None, resolved_mem_fraction=0.9)
+ gpu_mem = 80 * 1024
+
+ with (
+ patch.object(envs.SGLANG_DISAGG_STAGING_BUFFER, "get", return_value=False),
+ ):
+ handle_kv_cache_sharding(args, gpu_mem)
+
+ rounded = 2112 # ceil(2050 / page_size) * 64; 33 pages across two shards
+ expected_fraction = 0.9 - 1.5 * (rounded - 2050) / gpu_mem
+ self.assertEqual(resolution_result(args, "chunked_prefill_size"), rounded)
+ self.assertAlmostEqual(
+ resolution_result(args, "mem_fraction_static"), expected_fraction
+ )
+
+ def test_page_aligned_chunk_need_not_span_the_whole_shard_group(self):
+ args = self._rounding_args(raw_mem_fraction=None, resolved_mem_fraction=0.9)
+ args.chunked_prefill_size = 2112 # 33 pages, not divisible by TP size 2.
+ with envs.SGLANG_DISAGG_STAGING_BUFFER.override(False):
+ handle_kv_cache_sharding(args, 80 * 1024)
+
+ self.assertEqual(resolution_result(args, "chunked_prefill_size"), 2112)
+ self.assertEqual(resolution_result(args, "mem_fraction_static"), 0.9)
+
+ def test_chunk_rounding_preserves_explicit_mem_fraction(self):
+ args = self._rounding_args(raw_mem_fraction=0.8, resolved_mem_fraction=0.8)
+
+ with (
+ patch.object(envs.SGLANG_DISAGG_STAGING_BUFFER, "get", return_value=False),
+ ):
+ handle_kv_cache_sharding(args, 80 * 1024)
+
+ self.assertEqual(resolution_result(args, "chunked_prefill_size"), 2112)
+ self.assertEqual(resolution_result(args, "mem_fraction_static"), 0.8)
+
+ def test_final_pass_rejects_late_model_capability_overrides(self):
+ cases = (
+ ({"attention_backend": "triton"}, "requires the fa3"),
+ ({"chunked_prefill_size": -1}, "requires chunked prefill"),
+ )
+ with (
+ patch.object(envs.SGLANG_DISAGG_STAGING_BUFFER, "get", return_value=False),
+ ):
+ for declarations, error in cases:
+ with self.subTest(declarations=declarations):
+ args = self._rounding_args(
+ raw_mem_fraction=0.8, resolved_mem_fraction=0.8
+ )
+ handle_kv_cache_sharding(args, 80 * 1024)
+ declare_resolution(args, "late_model_capability", **declarations)
+
+ with self.assertRaisesRegex(ValueError, error):
+ handle_kv_cache_sharding(args, 80 * 1024)
+
+
class TestDcpKvEventContract(CustomTestCase):
"""DCP widens the radix-tree page to page_size * dcp_size, which the
advertised KV-event block size must reflect."""