diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx index 762f425de5..5cecf8b564 100644 --- a/docs/docs/advanced_features/server_arguments.mdx +++ b/docs/docs/advanced_features/server_arguments.mdx @@ -2797,6 +2797,12 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec None zigzag, interleave + + `--enable-kv-cache-sharding` + Shard at-rest KV cache storage at page granularity across the group that currently replicates it (attention-CP ranks for GQA models under prefill CP; attention-TP ranks for MLA models), multiplying the group's unique KV capacity. Prefill workers of PD disaggregation only. + `False` + bool flag (set to enable) + `--enable-fused-qk-norm-rope` Enable fused qk normalization and rope rotary embedding. diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py index 1611e714b7..0f2f8b799a 100644 --- a/python/sglang/srt/arg_groups/field_order.py +++ b/python/sglang/srt/arg_groups/field_order.py @@ -90,6 +90,7 @@ POSITIONAL_FIELD_ORDER = ( "cp_strategy", "enable_dsa_cache_layer_split", "enable_cp_decode_attn_tp", + "enable_kv_cache_sharding", "enable_dp_attention", "enable_dp_attention_local_control_broadcast", "enable_dp_lm_head", diff --git a/python/sglang/srt/arg_groups/fields/parallel.py b/python/sglang/srt/arg_groups/fields/parallel.py index ed3e6a5ce9..4a7a242ed8 100644 --- a/python/sglang/srt/arg_groups/fields/parallel.py +++ b/python/sglang/srt/arg_groups/fields/parallel.py @@ -181,6 +181,14 @@ class Parallel(msgspec.Struct): bool, "Enable attention tensor-parallel weight slicing during decode under context parallel (cp_size>1). Slices the replicated attention linears to the local CP partition, eliminating redundant decode GEMMs.", ] = False + enable_kv_cache_sharding: A[ + bool, + "Shard at-rest KV cache storage at page granularity across the group " + "that currently replicates it (attention-CP ranks for GQA models " + "under prefill CP; attention-TP ranks for MLA models), multiplying " + "the group's unique KV capacity. Prefill workers of PD " + "disaggregation only.", + ] = False # Deprecated spelling of `attn_dp_size`: `--dp-size N --enable-dp-attention` # resolves to `attn_dp_size = N`, `dp_size = 1`. TODO: remove after 2026-12-31. enable_dp_attention: A[bool, Arg(no_cli=True, resolvable=True)] = False diff --git a/python/sglang/srt/arg_groups/kv_shard_hook.py b/python/sglang/srt/arg_groups/kv_shard_hook.py new file mode 100644 index 0000000000..8203c4652d --- /dev/null +++ b/python/sglang/srt/arg_groups/kv_shard_hook.py @@ -0,0 +1,282 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Resolution and validation for logical-page KV cache sharding.""" + +from __future__ import annotations + +import logging +from typing import Any, Optional + +from sglang.srt.arg_groups.overrides import ( + attention_backends_of, + declare_resolution, + model_config_of, + resolved_view, + resolving_view, + use_mla_backend, +) +from sglang.srt.configs.model_config import is_deepseek_dsa +from sglang.srt.environ import envs +from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase +from sglang.srt.runtime_context import attn_dp_enabled_of +from sglang.srt.utils.common import get_device_memory_capacity + +logger = logging.getLogger(__name__) + + +def validate_kv_shard_attention_backend(server_args: Any, cfg: Any = None) -> None: + cfg = cfg or resolving_view(server_args) + prefill_backend, _ = attention_backends_of(resolved_view(server_args)) + if prefill_backend == "fa3": + return + if prefill_backend != "trtllm_mla": + raise ValueError( + "--enable-kv-cache-sharding currently requires the fa3 prefill " + "attention backend, or trtllm_mla for plain-TP MLA; " + f"got {prefill_backend!r}." + ) + if not use_mla_backend(server_args): + raise ValueError("KV sharding with trtllm_mla requires an MLA model.") + if cfg.attn_cp_size > 1: + raise ValueError( + "KV sharding with trtllm_mla only supports plain-TP MLA " + "(attn_cp_size must be 1)." + ) + if cfg.disable_chunked_prefix_cache: + raise ValueError("KV sharding with trtllm_mla requires chunked prefix caching.") + + +def handle_kv_cache_sharding(server_args: Any, gpu_mem: Optional[float] = None) -> None: + """Validate KV sharding and declare its derived eager-prefill settings.""" + cfg = resolving_view(server_args) + if not cfg.enable_kv_cache_sharding: + return + + if gpu_mem is None: + # `handle_gpu_memory_settings` resolves capacity internally now, so the + # pipeline no longer threads it through. Only the chunk-size rounding + # below reads it, and only when mem_fraction_static was left unset. + gpu_mem = get_device_memory_capacity(cfg.device) + + if cfg.disaggregation_mode != "prefill": + raise ValueError( + "--enable-kv-cache-sharding is only supported on PD prefill " + "workers: a rank's pool holds only its stripe of every cached " + "page, which is incompatible with local decode." + ) + if cfg.speculative_algorithm is not None: + raise ValueError( + "--enable-kv-cache-sharding does not support speculative decoding." + ) + if cfg.dllm_algorithm is not None: + raise ValueError( + "--enable-kv-cache-sharding does not support diffusion language " + "models: DLLM admission bypasses the shard-scratch budget and " + "resolves out of PD-prefill mode." + ) + if cfg.radix_cache_backend is not None: + raise ValueError( + "--enable-kv-cache-sharding requires the built-in " + "UnifiedRadixCache with a rotation-aware UnifiedTreeCore and does " + "not support " + f"--radix-cache-backend={cfg.radix_cache_backend!r}." + ) + tree_core_backend = envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get() + # A disabled radix cache never matches or inserts, so its tree core never + # sees a rotation base; chunked continuations keep the base on the request. + if not cfg.disable_radix_cache and tree_core_backend != "python": + raise ValueError( + "--enable-kv-cache-sharding requires the Python UnifiedTreeCore: " + f"SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND={tree_core_backend!r} is " + "not supported for sharded KV." + ) + if cfg.enable_unified_cache_external_linker: + raise ValueError( + "--enable-kv-cache-sharding does not support " + "--enable-unified-cache-external-linker: load-back allocates " + "through the flat allocator API." + ) + if cfg.enable_streaming_session: + raise ValueError( + "--enable-kv-cache-sharding does not support streaming sessions: " + "their virtual nodes do not carry KV-shard rotation metadata." + ) + if cfg.enable_session_radix_cache: + raise ValueError( + "--enable-kv-cache-sharding does not support session radix cache yet." + ) + if cfg.enable_flexkv: + raise ValueError( + "--enable-kv-cache-sharding does not support --enable-flexkv: " + "FlexKV load-back allocates through the stock flat allocator API." + ) + if cfg.enable_unified_memory: + raise ValueError( + "--enable-kv-cache-sharding does not support --enable-unified-memory: " + "the unified pool uses a different virtual-id allocator and layout." + ) + if cfg.dcp_size > 1: + raise ValueError( + "--enable-kv-cache-sharding and decode context parallelism " + "(--dcp-size) are mutually exclusive users of the widened " + "allocator index space." + ) + if cfg.enable_hierarchical_cache: + raise ValueError( + "--enable-kv-cache-sharding does not support " + "--enable-hierarchical-cache yet: HiCache backup/load assumes " + "each rank holds full pages." + ) + if cfg.enable_lmcache: + raise ValueError( + "--enable-kv-cache-sharding does not support --enable-lmcache." + ) + if cfg.enable_hisparse: + raise ValueError( + "--enable-kv-cache-sharding does not support --enable-hisparse: " + "HiSparse selects a non-sharded allocator and assumes each rank " + "owns complete KV pages." + ) + # The dynamic sizer only runs with pipeline parallelism. Its profiling + # and predicted chunks can exceed the fixed shard-scratch capacity. + if cfg.enable_dynamic_chunking and cfg.pp_size > 1: + raise ValueError( + "--enable-kv-cache-sharding does not support dynamic chunk sizing " + "with pipeline parallelism: profiling and predicted chunks can " + "exceed the fixed sharded scratch capacity. Disable " + "--enable-dynamic-chunking." + ) + if cfg.enable_two_batch_overlap: + raise ValueError( + "--enable-kv-cache-sharding does not support " + "--enable-two-batch-overlap: the sharded pool's per-batch gather " + "plan is a process-wide singleton, but TBO rebuilds attention " + "metadata multiple times per extend forward and clobbers the plan." + ) + + model_config = model_config_of(server_args) + if model_config.is_encoder_decoder: + raise ValueError( + "--enable-kv-cache-sharding does not support encoder-decoder " + "models: cross-attention uses a separate encoder page table that " + "is not part of the sharded gather plan." + ) + if model_config.attention_chunk_size is not None: + raise ValueError( + "--enable-kv-cache-sharding does not support chunked/local-attention " + "models yet: local-attention metadata is built from the page table " + "before the scratch translation." + ) + + validate_kv_shard_attention_backend(server_args, cfg) + if cfg.disaggregation_transfer_backend not in ("mooncake", "nixl"): + raise ValueError( + "--enable-kv-cache-sharding currently only supports the mooncake " + "and nixl transfer backends: both route sends through " + "CommonKVSender._prepare_send_indices and pair destinations by " + "fancy index. Got --disaggregation-transfer-backend " + f"{cfg.disaggregation_transfer_backend!r}." + ) + if envs.SGLANG_DISAGG_STAGING_BUFFER.get(): + raise ValueError( + "--enable-kv-cache-sharding is incompatible with " + "SGLANG_DISAGG_STAGING_BUFFER (the staging path assumes " + "contiguous send slices)." + ) + + if use_mla_backend(server_args): + if is_deepseek_dsa(model_config.hf_config): + raise ValueError( + "--enable-kv-cache-sharding does not support DSA models yet " + "(indexer buffers are not striped)." + ) + if cfg.attn_cp_size <= 1: + if attn_dp_enabled_of(cfg): + raise ValueError( + "--enable-kv-cache-sharding for MLA models without " + "prefill CP shards across attention-TP and requires plain " + "TP attention; set --attn-dp-size=1." + ) + if cfg.tp_size <= 1: + raise ValueError( + "--enable-kv-cache-sharding for MLA models without " + "prefill CP shards across attention-TP ranks and needs " + "tp_size > 1." + ) + elif not cfg.enable_prefill_cp or cfg.attn_cp_size <= 1: + raise ValueError( + "--enable-kv-cache-sharding for GQA models shards across " + "attention-CP ranks and requires --enable-prefill-cp with " + "attn_cp_size > 1." + ) + + page_size = cfg.page_size if cfg.page_size is not None else 1 + if page_size <= 1: + raise ValueError( + "--enable-kv-cache-sharding requires a real page size " + "(--page-size 64 recommended): the physical page is the P/D " + "transfer descriptor and gather-copy unit; page_size=1 " + "degenerates both into per-token operations." + ) + chunk_size = cfg.chunked_prefill_size + if chunk_size is None or chunk_size <= 0: + raise ValueError( + "--enable-kv-cache-sharding requires chunked prefill " + "(--chunked-prefill-size > 0)." + ) + + declarations = {} + # Chunk boundaries must stay on physical pages. Uneven ownership across + # ranks is supported; only the prefix gather pads to a full-group span. + if chunk_size % page_size != 0: + rounded = (chunk_size + page_size - 1) // page_size * page_size + declarations["chunked_prefill_size"] = rounded + + # The raw input, deliberately not the resolving view: the question is + # whether the user pinned mem_fraction_static, and the resolved value + # would answer yes for every run. + raw_mem_fraction = (getattr(server_args, "_raw_input", None) or {}).get( + "mem_fraction_static" + ) + activation_delta_mb = 1.5 * (max(rounded, 2048) - max(chunk_size, 2048)) + if raw_mem_fraction is None and activation_delta_mb > 0: + if not gpu_mem or cfg.mem_fraction_static is None: + raise ValueError( + "Cannot safely auto-round --chunked-prefill-size for KV " + "sharding without a known device-memory capacity; pass a " + f"multiple of page_size ({page_size})." + ) + corrected_mem_fraction = ( + float(cfg.mem_fraction_static) - activation_delta_mb / gpu_mem + ) + if corrected_mem_fraction <= 0: + raise ValueError( + "Auto-rounding --chunked-prefill-size for KV sharding " + "would consume all automatically reserved GPU memory; " + f"pass a smaller multiple of {page_size}." + ) + declarations["mem_fraction_static"] = corrected_mem_fraction + + logger.warning( + "Rounding --chunked-prefill-size up from %d to %d " + "(a multiple of page_size = %d) for KV sharding.", + chunk_size, + rounded, + page_size, + ) + + if cfg.cuda_graph_config.prefill.backend != Backend.DISABLED: + logger.warning( + "Prefill CUDA graph (%s) is incompatible with " + "--enable-kv-cache-sharding; disabling it.", + cfg.cuda_graph_config.prefill.backend, + ) + declarations["cuda_graph_config"] = with_phase( + cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED + ) + + if declarations: + declare_resolution( + server_args, + "handle_kv_cache_sharding", + **declarations, + ) diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index eda212a2a5..ed2aa21a1c 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -1705,6 +1705,12 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool: False for any config the runtime won't post-capture-size, else it gets an under-reserved fraction.""" cfg = resolving_view(server_args) + # Logical-page sharding always runs prefill eagerly. This predicate runs + # before the sharding hook disables prefill capture, so gate on the feature + # itself to retain eager activation headroom in the memory heuristic. + if cfg.enable_kv_cache_sharding: + return False + mla_enabled = use_mla_backend(server_args) if not envs.SGLANG_ENABLE_POST_CAPTURE_KV_SIZING.get(): return False diff --git a/python/sglang/srt/arg_groups/pipeline.py b/python/sglang/srt/arg_groups/pipeline.py index 4669a6628d..e1f8fe47dd 100644 --- a/python/sglang/srt/arg_groups/pipeline.py +++ b/python/sglang/srt/arg_groups/pipeline.py @@ -6,6 +6,7 @@ from __future__ import annotations from typing import Any from sglang.srt.arg_groups.arg_utils import record_fields +from sglang.srt.arg_groups.kv_shard_hook import handle_kv_cache_sharding from sglang.srt.arg_groups.overrides import ( _page_size_default, _pipeline_parallel_overlap_disable, @@ -264,6 +265,10 @@ def run_resolution_pipeline(server_args: Any) -> None: run_hook(handle_context_parallelism, server_args) + # Validate logical-page KV cache sharding after its page size, attention + # backend, memory budget, and shard topology have all resolved. + run_hook(handle_kv_cache_sharding, server_args) + from sglang.srt.arg_groups.moe_hook import ( handle_a2a_moe, handle_moe_kernel_config, @@ -334,6 +339,11 @@ def run_resolution_pipeline(server_args: Any) -> None: finalize_cuda_graph_prefill_max_context(server_args) + # Capability adjustments can late-force a different attention backend or + # disable chunked prefill. Re-run the idempotent sharding gate over the + # final resolving view. + run_hook(handle_kv_cache_sharding, server_args) + # Validate after all batch-size declarations are visible. run_hook(validate_deepep_v2_speculative_draft, server_args) run_hook(validate_deepep_v2_dispatch_token_budget, server_args) diff --git a/python/sglang/srt/arg_groups/resolution_hooks.py b/python/sglang/srt/arg_groups/resolution_hooks.py index 00fb1c607d..018db11f85 100644 --- a/python/sglang/srt/arg_groups/resolution_hooks.py +++ b/python/sglang/srt/arg_groups/resolution_hooks.py @@ -67,6 +67,7 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset( "handle_data_parallelism", "handle_load_balance_method", "handle_context_parallelism", + "handle_kv_cache_sharding", "handle_moe_kernel_config", "handle_a2a_moe", "handle_eplb_and_dispatch", diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 4da8bc98e3..1b64296f56 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -49,6 +49,7 @@ from sglang.srt.mem_cache.allocator.hisparse import ( DeepSeekV4HiSparseTokenToKVPoolAllocator, HiSparseTokenToKVPoolAllocator, ) +from sglang.srt.mem_cache.allocator.page_interleave import PageInterleavePoolAllocator from sglang.srt.mem_cache.allocator.swa import ( PureSWATokenToKVPoolAllocator, SWATokenToKVPoolAllocator, @@ -80,6 +81,16 @@ from sglang.srt.mem_cache.memory_pool import ( ReqToTokenPool, get_minimax_sparse_index_dtype, ) +from sglang.srt.mem_cache.page_interleave import ( + PageShardSpec, + get_kv_shard_group, + make_page_shard_spec, +) +from sglang.srt.mem_cache.page_interleave_pool import ( + PageInterleaveKVPoolMixin, + PageInterleaveMHATokenToKVPool, + PageInterleaveMLATokenToKVPool, +) from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.platforms import current_platform @@ -229,6 +240,7 @@ def _pp_local_per_request_bytes( if TYPE_CHECKING: + from sglang.srt.distributed.parallel_state import GroupCoordinator from sglang.srt.mem_cache.unified_memory_pool import ( UnifiedKVPool, UnifiedPoolBundle, @@ -311,6 +323,10 @@ class KVCacheConfigurator: mambaish_config: Optional[Any] = field(init=False) hybrid_gdn_config: Optional[Any] = field(init=False) hybrid_kda_config: Optional[Any] = field(init=False) + kv_shard_rank: Optional[int] = field(init=False) + kv_shard_size: int = field(init=False) + kv_shard_spec: Optional[PageShardSpec] = field(init=False) + kv_shard_group: Optional[GroupCoordinator] = field(init=False) is_hybrid_swa_mtp_draft: bool = field(init=False) draft_swa_full_capacity: bool = field(init=False) @@ -318,6 +334,7 @@ class KVCacheConfigurator: self.mambaish_config = mambaish_config(self.model_config) self.hybrid_gdn_config = hybrid_gdn_config(self.model_config) self.hybrid_kda_config = hybrid_kda_config(self.model_config) + self._resolve_kv_shard() self.is_hybrid_swa_mtp_draft = self.layer_info.is_hybrid_swa_mtp_draft self.draft_swa_full_capacity = self.is_hybrid_swa_mtp_draft and bool( self.layer_info.swa_attention_layer_ids @@ -351,6 +368,25 @@ class KVCacheConfigurator: max_total_num_tokens * get_parallel().attn_dcp_size // self.loc_space_scale ) + def _resolve_kv_shard(self) -> None: + """Resolve the logical-page KV sharding identity: ``(rank, size)`` of + the shard group, or ``(None, 1)`` when sharding is off. Consumed by the + pool builders and by the widened allocator branch.""" + self.kv_shard_spec = make_page_shard_spec(self) + self.kv_shard_rank = None + self.kv_shard_size = 1 + self.kv_shard_group = None + if self.kv_shard_spec is None: + return + reject_out_of_tree_path( + current_platform, + subsystem="logical-page KV sharding (--enable-kv-cache-sharding)", + ) + + self.kv_shard_rank = self.kv_shard_spec.shard_rank + self.kv_shard_size = self.kv_shard_spec.shard_size + self.kv_shard_group = get_kv_shard_group(self.use_mla_backend) + def _build_fp4_quant_method(self, *, num_layers: int): if not is_float4_e2m1fn_x2(self.kv_cache_dtype): return None @@ -689,6 +725,13 @@ class KVCacheConfigurator: req_to_token_pool=req_to_token_pool, ) + if self.kv_shard_rank is not None: + if not isinstance(token_to_kv_pool, PageInterleaveKVPoolMixin): + raise ValueError( + "--enable-kv-cache-sharding does not support this model " + f"family (pool {type(token_to_kv_pool).__name__})." + ) + if draft_virtual_id_space is not None: assert token_to_kv_pool.size >= draft_virtual_id_space, ( "draft token_to_kv_pool smaller than the shared unified " @@ -2099,6 +2142,17 @@ class KVCacheConfigurator: def _build_mla_kv_pool( self, *, max_total_num_tokens: int, mla_pool_class: type ) -> KVCache: + mla_pool_kwargs = {} + if self.kv_shard_rank is not None: + # The striped pool subclasses the plain MLA pool; a vendor pool + # class would otherwise be silently replaced. + assert mla_pool_class is MLATokenToKVPool, ( + "--enable-kv-cache-sharding is incompatible with " + f"the {mla_pool_class.__name__} pool" + ) + mla_pool_class = PageInterleaveMLATokenToKVPool + mla_pool_kwargs["shard_spec"] = self.kv_shard_spec + mla_pool_kwargs["shard_group"] = self.kv_shard_group token_to_kv_pool = mla_pool_class( max_total_num_tokens, page_size=self.pool_page_size, @@ -2110,6 +2164,7 @@ class KVCacheConfigurator: enable_memory_saver=get_exec().features.enable_memory_saver, start_layer=self.layer_info.start_layer, end_layer=self.layer_info.end_layer, + **mla_pool_kwargs, ) return token_to_kv_pool @@ -2363,6 +2418,23 @@ class KVCacheConfigurator: pool_kwargs["quant_method"] = quant_method else: pool_kwargs["post_capture_active"] = self.post_capture_kv_active + if self.kv_shard_rank is not None: + # The striped pool subclasses the plain per-layer MHA pool; the + # mxfp8 / NoOp / page-major variants have incompatible buffer + # layouts. Specialized quantization methods need packed-data and + # scale-aware sharded writes/gathers; ordinary CUDA FP8 uses the + # plain pool and has no specialized quant_method. + assert pool_cls is MHATokenToKVPool, ( + "--enable-kv-cache-sharding is incompatible with " + f"the {pool_cls.__name__} pool" + ) + assert quant_method is None, ( + "--enable-kv-cache-sharding does not support specialized KV-cache " + "quantization methods (e.g. FP4); ordinary CUDA FP8 is supported." + ) + pool_cls = PageInterleaveMHATokenToKVPool + pool_kwargs["shard_spec"] = self.kv_shard_spec + pool_kwargs["shard_group"] = self.kv_shard_group token_to_kv_pool = pool_cls( max_total_num_tokens, page_size=self.pool_page_size, @@ -2487,6 +2559,19 @@ class KVCacheConfigurator: need_sort=need_sort, host_to_device_ratio=hisparse_cfg.host_to_device_ratio, ) + elif self.kv_shard_size > 1: + # Logical-page sharding: index space widened xN over + # the stock 1x pool; the allocator page stays physical. + token_to_kv_pool_allocator = PageInterleavePoolAllocator( + sizes.max_total_num_tokens, + physical_page_size=self.page_size, + shard_size=self.kv_shard_size, + dtype=self.kv_cache_dtype, + device=self.device, + kvcache=token_to_kv_pool, + need_sort=need_sort, + shard_spec=self.kv_shard_spec, + ) elif ( get_schedule().page_size == 1 and not get_parallel().dcp_enabled ): diff --git a/python/sglang/srt/mem_cache/page_interleave.py b/python/sglang/srt/mem_cache/page_interleave.py index a9e5c837dd..3764cca215 100644 --- a/python/sglang/srt/mem_cache/page_interleave.py +++ b/python/sglang/srt/mem_cache/page_interleave.py @@ -68,9 +68,11 @@ class PageShardSpec(msgspec.Struct, frozen=True): @property def logical_page_size(self) -> int: - """The N*page_size span of one logical page. It bounds the assembly - scratch and rounds chunked_prefill_size; the allocator and the tree - both keep drawing and matching at the physical ``page_size``.""" + """The N*page_size span used to align prefix-gather scratch. + + Allocation, prefix matching, and chunk boundaries use the physical + ``page_size``. + """ return self.shard_size * self.page_size diff --git a/python/sglang/srt/mem_cache/registry.py b/python/sglang/srt/mem_cache/registry.py index 25343d74f5..9f42286bb5 100644 --- a/python/sglang/srt/mem_cache/registry.py +++ b/python/sglang/srt/mem_cache/registry.py @@ -17,7 +17,11 @@ from typing import TYPE_CHECKING, Any, Callable, Optional from sglang.srt.hardware_backend.mlx.runtime import use_mlx from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache from sglang.srt.mem_cache.cache_init_params import CacheInitParams -from sglang.srt.runtime_context import get_disagg, get_memory, get_serving +from sglang.srt.runtime_context import ( + get_disagg, + get_memory, + get_serving, +) if TYPE_CHECKING: from sglang.srt.configs.model_config import ModelConfig diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index 38da3198b7..7f20cc0953 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -42,6 +42,7 @@ from sglang.srt.mem_cache.memory_pool import ( DSATokenToKVPool, get_minimax_sparse_index_dtype, ) +from sglang.srt.mem_cache.page_interleave import compute_page_shard_scratch_bytes from sglang.srt.runtime_context import ( get_disagg, get_exec, @@ -213,7 +214,10 @@ class MemoryPoolConfigurator: class DefaultPoolConfigurator(MemoryPoolConfigurator): - """Standard models (MHA, MLA, DSA, FP4): coeff = cell_size, bias = 0.""" + """Standard models (MHA, MLA, DSA, FP4). + + coeff = cell_size; bias = double-buffered KV-shard scratch, otherwise zero. + """ def __init__(self, kvc: KVCacheConfigurator): self.kv_cache_dtype_str = kvc.kv_cache_dtype_str @@ -242,6 +246,10 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): else get_schedule().max_total_tokens or kvc.model_config.context_len ) + # Logical-page KV sharding reserves a fixed double-buffered assembly + # scratch next to the pool; charge it before token sizing. + self._fixed_overhead_bytes = compute_page_shard_scratch_bytes(kvc) + # EAGLE/STANDALONE: assumes the draft shares the target's per-layer KV size, # which holds for EAGLE/MTP drafts that reuse the target's attention config. if ( @@ -598,7 +606,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): def calculate_pool_sizes( self, available_bytes: int, page_size: int ) -> MemoryPoolConfig: - available_bytes = max(available_bytes, 0) + available_bytes = max(available_bytes - self._fixed_overhead_bytes, 0) max_total_num_tokens = ( available_bytes // self._cell_size if self._cell_size diff --git a/test/registered/dcp/test_dcp_layout_unit.py b/test/registered/dcp/test_dcp_layout_unit.py index 6adcc4a789..c19b3d656e 100644 --- a/test/registered/dcp/test_dcp_layout_unit.py +++ b/test/registered/dcp/test_dcp_layout_unit.py @@ -443,6 +443,8 @@ class TestGetDcpLens(CustomTestCase): kv_cache_dtype=torch.bfloat16, device="cpu", is_draft_worker=False, + kv_shard_size=1, + kv_shard_spec=None, ) # The allocator widens from get_parallel(), not from the injected # server_args stand-in -- drive the cause, not the effect. diff --git a/test/registered/unit/disaggregation/test_mooncake_efa_allocator.py b/test/registered/unit/disaggregation/test_mooncake_efa_allocator.py index 2ed4793bad..4d2b17e604 100644 --- a/test/registered/unit/disaggregation/test_mooncake_efa_allocator.py +++ b/test/registered/unit/disaggregation/test_mooncake_efa_allocator.py @@ -75,6 +75,7 @@ class TestMooncakeEfaAllocator(unittest.TestCase): def test_efa_disables_post_capture_kv_sizing(self): cfg = SimpleNamespace( + enable_kv_cache_sharding=False, enable_unified_memory=False, device="cuda", dcp_size=1, diff --git a/test/registered/unit/mem_cache/test_registry.py b/test/registered/unit/mem_cache/test_registry.py index 3d42211dcf..cbc1f6df36 100644 --- a/test/registered/unit/mem_cache/test_registry.py +++ b/test/registered/unit/mem_cache/test_registry.py @@ -44,6 +44,7 @@ def _make_ctx( enable_hierarchical_cache=False, disable_radix_cache=False, full_tokens_per_layer=None, + enable_kv_cache_sharding=False, ): # The factory reads the published bags for the cache-backend leaves, so the # fixture publishes them; the instance stays for the whole-object contract @@ -55,6 +56,7 @@ def _make_ctx( enable_lmcache=enable_lmcache, enable_flexkv=False, enable_unified_cache_external_linker=False, + enable_kv_cache_sharding=enable_kv_cache_sharding, ) return TreeCacheBuildContext( server_args=server_args, @@ -155,6 +157,31 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase): create_unified.assert_called_once_with(ctx) self.assertIs(result, create_unified.return_value) + def test_kv_sharding_with_disable_radix_routes_to_unified(self): + ctx = _make_ctx(self, disable_radix_cache=True, enable_kv_cache_sharding=True) + with patch( + "sglang.srt.mem_cache.registry.create_unified_radix_cache" + ) as create_unified: + result = default_radix_cache_factory(ctx) + create_unified.assert_called_once_with(ctx) + self.assertIs(result, create_unified.return_value) + + def test_kv_sharding_uses_unified_radix_cache(self): + ctx = _make_ctx(self, enable_kv_cache_sharding=True) + fake_components = MagicMock() + fake_radix = MagicMock() + with patch.dict( + "sys.modules", + { + "sglang.srt.mem_cache.unified_cache.components": fake_components, + "sglang.srt.mem_cache.unified_radix_cache": fake_radix, + }, + ): + result = default_radix_cache_factory(ctx) + + fake_radix.UnifiedRadixCache.assert_called_once_with(ctx.params) + self.assertIs(result, fake_radix.UnifiedRadixCache.return_value) + def test_hybrid_swa_with_disable_radix_routes_to_unified(self): ctx = _make_ctx( self, diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index 7d20eb7be6..bf5700b96a 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -6,7 +6,12 @@ from types import SimpleNamespace from unittest.mock import MagicMock, patch from sglang.srt.configs.model_config import AttentionArch +from sglang.srt.mem_cache.page_interleave import ( + compute_page_shard_scratch_bytes, + make_page_shard_spec, +) from sglang.srt.runtime_context import ( + get_context, get_memory, get_parallel, get_schedule, @@ -249,6 +254,75 @@ class TestDefaultConfigurator(CustomTestCase): _, _, config = self._run(available, page_size=128) self.assertEqual(config.max_total_num_tokens % 128, 0) + def test_eight_context_scratch_is_reserved_before_kv_pool_sizing(self): + page_size = 16 + mr = _make_model_runner( + self, + use_mla_backend=True, + num_layers=3, + page_size=page_size, + chunked_prefill_size=1024, + max_running_requests=1, + ) + mr.model_config.context_len = 8193 + with ( + mock_cpu_env(), + patch( + "sglang.srt.mem_cache.page_interleave.get_kv_shard_group_info", + return_value=(0, 4), + ), + get_context().override_server_args( + prefill_max_requests=1, + chunked_prefill_size=1024, + max_running_requests=1, + ), + ): + spec = make_page_shard_spec(mr) + # Each context rounds to 8256 tokens, a 64-token gather multiple, + # even with request limits of one. Only the prefix is multiplied. + self.assertEqual(spec.max_prefix_tokens, 8 * 8256) + self.assertEqual(spec.chunk_tokens, 1024) + row_bytes = (512 + 64) * KV_SIZE + scratch_bytes = 2 * (8 * 8256 + 1024 + page_size) * row_bytes + self.assertEqual(compute_page_shard_scratch_bytes(mr), scratch_bytes) + + # Scratch has two one-layer slots; persistent KV spans all three + # layers. Charge the former before dividing by the latter's cost. + cell_bytes = row_bytes * mr.num_effective_layers + page_bytes = page_size * cell_bytes + from sglang.srt.model_executor.pool_configurator import ( + create_memory_pool_configurator, + ) + + configurator = create_memory_pool_configurator(mr) + for desired_tokens in (page_size, 128): + for slack_bytes in (0, page_bytes - 1): + with self.subTest( + desired_tokens=desired_tokens, slack_bytes=slack_bytes + ): + budget = ( + scratch_bytes + desired_tokens * cell_bytes + slack_bytes + ) + config = configurator.calculate_pool_sizes(budget, page_size) + self.assertEqual(config.max_total_num_tokens, desired_tokens) + self.assertEqual( + budget + - scratch_bytes + - config.max_total_num_tokens * cell_bytes, + slack_bytes, + ) + + # The fixed eight-context scratch is never silently reduced to + # leave room for persistent KV, including a sub-page remainder. + for budget in ( + scratch_bytes - 1, + scratch_bytes, + scratch_bytes + page_bytes - 1, + ): + with self.subTest(insufficient_budget=budget): + with self.assertRaisesRegex(RuntimeError, "Not enough memory"): + configurator.calculate_pool_sizes(budget, page_size) + def test_constraint_respected(self): """calculate_pool_sizes_from_max_tokens respects the limit.""" mr, cfg, config = self._run(10_000_000) diff --git a/test/registered/unit/models/test_deepseek_mla_dispatch.py b/test/registered/unit/models/test_deepseek_mla_dispatch.py index dbd1448574..91173aa933 100644 --- a/test/registered/unit/models/test_deepseek_mla_dispatch.py +++ b/test/registered/unit/models/test_deepseek_mla_dispatch.py @@ -1,4 +1,4 @@ -"""Hermetic unit tests for DeepSeek MLA attention-method dispatch on ROCm. +"""Hermetic unit tests for DeepSeek MLA attention-method dispatch. `_dispatch_mla_subtype` picks the forward method for MLA attention. On ROCm the fused-decode-MLA + fused-RoPE fast path (`MLA_FUSED_ROPE_ROCM`) is only correct @@ -142,5 +142,111 @@ class TestCPMLADispatch(CustomTestCase): ) +class TestKVShardedMLADispatch(CustomTestCase): + def setUp(self): + super().setUp() + self.attn = SimpleNamespace( + chunked_prefix_cache_threshold=0, + disable_chunked_prefix_cache=False, + flashinfer_mla_disable_ragged=False, + ) + self.exec_config = SimpleNamespace( + deterministic=SimpleNamespace(enable_deterministic_inference=False) + ) + for patcher in ( + mock.patch.object(abh, "_is_hip", False), + mock.patch.object( + abh, + "get_parallel", + return_value=SimpleNamespace(enable_kv_cache_sharding=True), + ), + mock.patch.object(abh, "get_exec", return_value=self.exec_config), + mock.patch.object(abh, "is_in_tc_piecewise_cuda_graph", return_value=False), + mock.patch.object(abh, "is_in_breakable_cuda_graph", return_value=False), + mock.patch.object(cp_base, "_STRATEGY", None), + ): + patcher.start() + self.addCleanup(patcher.stop) + + def _batch(self, prefix=64, capacity=8192): + num_tokens = 3952 + return SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + input_ids=range(num_tokens), + attn_cp_metadata=None, + extend_prefix_lens_cpu=[prefix], + extend_seq_lens_cpu=[num_tokens], + seq_lens_cpu=[prefix + num_tokens], + get_max_chunk_capacity=lambda: capacity, + ) + + def test_fa3_disabled_chunked_prefix_cache_uses_absorbed_mla(self): + # A cached prefix or the second prompt chunk must not enter FA3's + # chunked-prefix path, which asserts prefix caching is enabled. + self.attn.disable_chunked_prefix_cache = True + self.assertEqual( + abh.handle_attention_fa3(self.attn, self._batch()), + AttnForwardMethod.MLA, + ) + + def test_fa3_deterministic_inference_uses_absorbed_mla(self): + self.exec_config.deterministic.enable_deterministic_inference = True + for prefix in (0, 64): + with self.subTest(prefix=prefix): + self.assertEqual( + abh.handle_attention_fa3(self.attn, self._batch(prefix=prefix)), + AttnForwardMethod.MLA, + ) + + def test_fa3_preserves_capacity_based_mha_dispatch(self): + # Both readers can use sharded scratch. Retain one-shot attention + # when the complete sequence fits, and chunked MHA otherwise. + for prefix in (0, 64): + for capacity, expected in ( + (8192, AttnForwardMethod.MHA_ONE_SHOT), + (0, AttnForwardMethod.MHA_CHUNKED_KV), + ): + with self.subTest(prefix=prefix, capacity=capacity): + self.assertEqual( + abh.handle_attention_fa3( + self.attn, self._batch(prefix=prefix, capacity=capacity) + ), + expected, + ) + + def test_fa3_short_prefix_uses_absorbed_mla(self): + self.attn.chunked_prefix_cache_threshold = 4096 + self.assertEqual( + abh.handle_attention_fa3(self.attn, self._batch()), + AttnForwardMethod.MLA, + ) + + def test_fa3_active_cp_uses_absorbed_mla(self): + # CP must gather rank-local latent activations before the pool write. + with mock.patch.object(cp_base, "_STRATEGY", ZigzagCPStrategy(cp_size=4)): + for prefix in (0, 64): + for capacity in (0, 8192): + with self.subTest(prefix=prefix, capacity=capacity): + self.assertEqual( + abh.handle_attention_fa3( + self.attn, + self._batch(prefix=prefix, capacity=capacity), + ), + AttnForwardMethod.MLA, + ) + + def test_trtllm_mla_keeps_chunked_mha_for_prefill(self): + # TRTLLM has a separate dispatcher and requires prefix caching for + # sharding; changing FA3 dispatch must not change its supported path. + for prefix in (0, 64): + with self.subTest(prefix=prefix): + self.assertEqual( + abh.handle_attention_trtllm_mla( + self.attn, self._batch(prefix=prefix) + ), + AttnForwardMethod.MHA_CHUNKED_KV, + ) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index ee79af3019..a4e0301388 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -42,6 +42,10 @@ from sglang.srt.arg_groups.kv_cache_hook import ( handle_nvfp4_prefill_kv_dequant_dtype, validate_prefill_only_disable_kv_cache_args, ) +from sglang.srt.arg_groups.kv_shard_hook import ( + handle_kv_cache_sharding, + validate_kv_shard_attention_backend, +) from sglang.srt.arg_groups.mamba_hook import handle_mamba_backend from sglang.srt.arg_groups.memory_hook import handle_gpu_memory_settings from sglang.srt.arg_groups.model_path_hook import handle_load_format @@ -51,7 +55,9 @@ from sglang.srt.arg_groups.moe_hook import ( validate_deepep_v2_speculative_draft, ) from sglang.srt.arg_groups.overrides import ( + declare_resolution, max_speculative_num_draft_tokens, + post_capture_kv_sizing_planned, resolution_result, ) from sglang.srt.arg_groups.parallel_hook import ( @@ -75,6 +81,7 @@ from sglang.srt.arg_groups.validation_hook import ( check_pipeline_parallel_compat, check_two_batch_overlap, ) +from sglang.srt.configs.model_config import AttentionArch from sglang.srt.entrypoints.sidecar import ( SGLANG_GRPC_ENDPOINT_ENV, Sidecar, @@ -106,6 +113,7 @@ from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import ( DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN, CustomTestCase, + enter_scope, ) register_cpu_ci(est_time=14, suite="base-a-test-cpu") @@ -3719,6 +3727,302 @@ class TestTwoBatchOverlapBackend(CustomTestCase): check_two_batch_overlap(args) +class TestKvCacheShardingCompatibility(CustomTestCase): + def setUp(self): + super().setUp() + # Sharding currently requires Python, independent of the default backend. + enter_scope( + self, envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override("python") + ) + + def _args(self, **overrides): + model_config = overrides.pop("model_config", None) + args = ServerArgs( + model_path="dummy", + enable_kv_cache_sharding=True, + disaggregation_mode="prefill", + ) + for key, value in overrides.items(): + setattr(args, key, value) + args._model_config = model_config or SimpleNamespace( + is_encoder_decoder=False, + attention_chunk_size=None, + attention_arch=AttentionArch.MHA, + hf_config=SimpleNamespace(architectures=["LlamaForCausalLM"]), + ) + return args + + def _rounding_args(self, *, raw_mem_fraction, resolved_mem_fraction): + args = self._args( + tp_size=2, + page_size=64, + chunked_prefill_size=2050, + attention_backend="fa3", + disaggregation_transfer_backend="mooncake", + mem_fraction_static=resolved_mem_fraction, + cuda_graph_config=CudaGraphConfig( + prefill=PhaseConfig(backend=Backend.DISABLED) + ), + ) + args._model_config = SimpleNamespace( + is_encoder_decoder=False, + attention_chunk_size=None, + attention_arch=AttentionArch.MLA, + hf_config=SimpleNamespace(architectures=["LlamaForCausalLM"]), + ) + args._raw_input = {"mem_fraction_static": raw_mem_fraction} + args._resolved_overrides = [ + ( + "_handle_gpu_memory_settings", + {"mem_fraction_static": resolved_mem_fraction}, + ) + ] + return args + + def test_encoder_decoder_is_rejected_before_backend_setup(self): + args = self._args(model_config=SimpleNamespace(is_encoder_decoder=True)) + + with self.assertRaisesRegex(ValueError, "does not support encoder-decoder"): + handle_kv_cache_sharding(args) + + def test_trtllm_mla_accepts_plain_tp_with_chunked_prefix_cache(self): + args = self._rounding_args(raw_mem_fraction=0.8, resolved_mem_fraction=0.8) + args.attention_backend = "trtllm_mla" + + with ( + patch.object(envs.SGLANG_DISAGG_STAGING_BUFFER, "get", return_value=False), + ): + handle_kv_cache_sharding(args, 80 * 1024) + + self.assertEqual(resolution_result(args, "chunked_prefill_size"), 2112) + + def test_mla_without_cp_rejects_attention_dp(self): + for backend in ("fa3", "trtllm_mla"): + with self.subTest(backend=backend): + args = self._rounding_args( + raw_mem_fraction=0.8, resolved_mem_fraction=0.8 + ) + args.attention_backend = backend + args.attn_dp_size = 2 + + with ( + envs.SGLANG_DISAGG_STAGING_BUFFER.override(False), + self.assertRaisesRegex(ValueError, "set --attn-dp-size=1"), + ): + handle_kv_cache_sharding(args, 80 * 1024) + + def test_mla_without_cp_rejects_normalized_legacy_attention_dp(self): + for backend in ("fa3", "trtllm_mla"): + with self.subTest(backend=backend): + args = self._rounding_args( + raw_mem_fraction=0.8, resolved_mem_fraction=0.8 + ) + args.attention_backend = backend + args.dp_size = 2 + args.enable_dp_attention = True + parallel_hook.handle_deprecated_dp_attention(args) + self.assertFalse(resolution_result(args, "enable_dp_attention")) + self.assertEqual(resolution_result(args, "attn_dp_size"), 2) + + with ( + envs.SGLANG_DISAGG_STAGING_BUFFER.override(False), + self.assertRaisesRegex(ValueError, "set --attn-dp-size=1"), + ): + handle_kv_cache_sharding(args, 80 * 1024) + + def test_mla_without_cp_allows_data_parallel_replicas(self): + args = self._rounding_args(raw_mem_fraction=0.8, resolved_mem_fraction=0.8) + args.dp_size = 2 + + with envs.SGLANG_DISAGG_STAGING_BUFFER.override(False): + handle_kv_cache_sharding(args, 80 * 1024) + + self.assertEqual(resolution_result(args, "chunked_prefill_size"), 2112) + + def test_trtllm_mla_rejects_unsupported_shard_topologies(self): + cases = ( + (False, 1, False, "requires an MLA model"), + (True, 2, False, "only supports plain-TP MLA"), + (True, 1, True, "requires chunked prefix caching"), + ) + for is_mla, attn_cp_size, disable_chunked_prefix_cache, error in cases: + with self.subTest(error=error): + args = self._args() + args.attention_backend = "trtllm_mla" + args._model_config.attention_arch = ( + AttentionArch.MLA if is_mla else AttentionArch.MHA + ) + view = SimpleNamespace( + attn_cp_size=attn_cp_size, + disable_chunked_prefix_cache=disable_chunked_prefix_cache, + ) + + with self.assertRaisesRegex(ValueError, error): + validate_kv_shard_attention_backend(args, view) + + def test_hisparse_allocator_is_rejected(self): + args = self._args(enable_hisparse=True) + + with self.assertRaisesRegex(ValueError, "does not support --enable-hisparse"): + handle_kv_cache_sharding(args) + + def test_incompatible_cache_tree_modes_are_rejected(self): + cases = ( + ({"radix_cache_backend": "custom"}, "built-in UnifiedRadixCache"), + ( + {"enable_unified_cache_external_linker": True}, + "does not support --enable-unified-cache-external-linker", + ), + ( + {"enable_streaming_session": True}, + "does not support streaming sessions", + ), + ( + {"enable_session_radix_cache": True}, + "does not support session radix cache yet", + ), + ) + for overrides, error in cases: + with self.subTest(overrides=overrides): + with self.assertRaisesRegex(ValueError, error): + handle_kv_cache_sharding(self._args(**overrides)) + + def test_rust_unified_tree_core_is_rejected(self): + with envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override("rust"): + with self.assertRaisesRegex(ValueError, "requires the Python"): + handle_kv_cache_sharding(self._args()) + + def test_rust_tree_core_is_allowed_with_radix_disabled(self): + args = self._rounding_args(raw_mem_fraction=0.8, resolved_mem_fraction=0.8) + args.disable_radix_cache = True + with ( + envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override("rust"), + envs.SGLANG_DISAGG_STAGING_BUFFER.override(False), + ): + handle_kv_cache_sharding(args, 80 * 1024) + + self.assertEqual(resolution_result(args, "chunked_prefill_size"), 2112) + + def test_dynamic_chunking_is_rejected_only_when_pipeline_parallelism_uses_it(self): + for pp_size in (1, 2): + with self.subTest(pp_size=pp_size): + args = self._rounding_args( + raw_mem_fraction=0.8, resolved_mem_fraction=0.8 + ) + args.enable_dynamic_chunking = True + args.pp_size = pp_size + with envs.SGLANG_DISAGG_STAGING_BUFFER.override(False): + if pp_size > 1: + with self.assertRaisesRegex( + ValueError, "dynamic chunk sizing with pipeline parallelism" + ): + handle_kv_cache_sharding(args, 80 * 1024) + else: + handle_kv_cache_sharding(args, 80 * 1024) + self.assertEqual( + resolution_result(args, "chunked_prefill_size"), 2112 + ) + + def test_post_capture_kv_sizing_keeps_eager_prefill_headroom(self): + prefill_graph = SimpleNamespace(backend=Backend.BREAKABLE, bs=[4096]) + args = ServerArgs( + model_path="dummy", + enable_kv_cache_sharding=True, + device="cuda", + dcp_size=1, + kv_cache_dtype="auto", + prefill_only_disable_kv_cache=False, + enable_memory_saver=False, + disaggregation_mode="prefill", + cuda_graph_config=CudaGraphConfig(prefill=prefill_graph), + chunked_prefill_size=4096, + ) + with ( + patch( + "sglang.srt.arg_groups.overrides.use_mla_backend", + return_value=False, + ), + patch( + "sglang.srt.arg_groups.overrides.max_prefill_buffer_tokens", + return_value=4096, + ), + patch( + "sglang.srt.arg_groups.overrides.model_config_of", + return_value=SimpleNamespace( + hf_config=SimpleNamespace(architectures=["LlamaForCausalLM"]) + ), + ), + patch.object( + envs.SGLANG_ENABLE_POST_CAPTURE_KV_SIZING, + "get", + return_value=True, + ), + patch.object( + envs.SGLANG_MOONCAKE_CUSTOM_MEM_POOL, + "get", + return_value=None, + ), + ): + self.assertFalse(post_capture_kv_sizing_planned(args)) + args.enable_kv_cache_sharding = False + self.assertTrue(post_capture_kv_sizing_planned(args)) + + def test_chunk_rounding_expands_automatic_activation_headroom(self): + args = self._rounding_args(raw_mem_fraction=None, resolved_mem_fraction=0.9) + gpu_mem = 80 * 1024 + + with ( + patch.object(envs.SGLANG_DISAGG_STAGING_BUFFER, "get", return_value=False), + ): + handle_kv_cache_sharding(args, gpu_mem) + + rounded = 2112 # ceil(2050 / page_size) * 64; 33 pages across two shards + expected_fraction = 0.9 - 1.5 * (rounded - 2050) / gpu_mem + self.assertEqual(resolution_result(args, "chunked_prefill_size"), rounded) + self.assertAlmostEqual( + resolution_result(args, "mem_fraction_static"), expected_fraction + ) + + def test_page_aligned_chunk_need_not_span_the_whole_shard_group(self): + args = self._rounding_args(raw_mem_fraction=None, resolved_mem_fraction=0.9) + args.chunked_prefill_size = 2112 # 33 pages, not divisible by TP size 2. + with envs.SGLANG_DISAGG_STAGING_BUFFER.override(False): + handle_kv_cache_sharding(args, 80 * 1024) + + self.assertEqual(resolution_result(args, "chunked_prefill_size"), 2112) + self.assertEqual(resolution_result(args, "mem_fraction_static"), 0.9) + + def test_chunk_rounding_preserves_explicit_mem_fraction(self): + args = self._rounding_args(raw_mem_fraction=0.8, resolved_mem_fraction=0.8) + + with ( + patch.object(envs.SGLANG_DISAGG_STAGING_BUFFER, "get", return_value=False), + ): + handle_kv_cache_sharding(args, 80 * 1024) + + self.assertEqual(resolution_result(args, "chunked_prefill_size"), 2112) + self.assertEqual(resolution_result(args, "mem_fraction_static"), 0.8) + + def test_final_pass_rejects_late_model_capability_overrides(self): + cases = ( + ({"attention_backend": "triton"}, "requires the fa3"), + ({"chunked_prefill_size": -1}, "requires chunked prefill"), + ) + with ( + patch.object(envs.SGLANG_DISAGG_STAGING_BUFFER, "get", return_value=False), + ): + for declarations, error in cases: + with self.subTest(declarations=declarations): + args = self._rounding_args( + raw_mem_fraction=0.8, resolved_mem_fraction=0.8 + ) + handle_kv_cache_sharding(args, 80 * 1024) + declare_resolution(args, "late_model_capability", **declarations) + + with self.assertRaisesRegex(ValueError, error): + handle_kv_cache_sharding(args, 80 * 1024) + + class TestDcpKvEventContract(CustomTestCase): """DCP widens the radix-tree page to page_size * dcp_size, which the advertised KV-event block size must reflect."""