Files
sglang/python/sglang/srt/model_executor/pool_configurator.py
T

1621 lines
69 KiB
Python

"""Memory pool configurators for profiling and sizing KV cache pools.
Each model architecture has its own configurator that computes pool sizes
from available GPU memory using a unified coeff+bias model:
available_bytes = max_tokens * coeff + bias
"""
from __future__ import annotations
import logging
from bisect import bisect_right
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional
import torch
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.configs.hybrid_arch import mambaish_config
from sglang.srt.configs.model_config import (
AttentionArch,
dsa_layer_skips_topk,
get_dsa_index_head_dim,
get_dsa_index_kpool,
get_minimax_sparse_attention_config,
get_minimax_sparse_disable_value_layer_ids,
get_minimax_sparse_layer_ids,
is_deepseek_dsa,
is_deepseek_v4,
is_minimax_sparse,
)
from sglang.srt.environ import envs
from sglang.srt.mem_cache.allocation_sizing import get_alloc_len_per_decode
from sglang.srt.mem_cache.deepseek_v4_memory_pool import (
collect_sources_by_ratio,
get_compress_state_write_pad,
get_dsv4_indexer_bytes_per_token,
get_swa_ring_size,
resolve_compress_state_ring_size,
)
from sglang.srt.mem_cache.memory_pool import (
DSATokenToKVPool,
get_minimax_sparse_index_dtype,
)
from sglang.srt.runtime_context import (
get_disagg,
get_exec,
get_memory,
get_model,
get_parallel,
get_schedule,
get_spec,
max_speculative_num_draft_tokens,
)
from sglang.srt.utils.common import (
ceil_align,
ceil_div,
is_float4_e2m1fn_x2,
is_hip,
is_npu,
spec_decode_alloc_len_per_request,
)
_is_hip = is_hip()
_is_npu = is_npu()
@dataclass
class MemoryPoolConfig:
"""Resolved memory pool config, shared between target and draft workers."""
max_total_num_tokens: int
max_running_requests: Optional[int] = None
full_max_total_num_tokens: Optional[int] = None
swa_max_total_num_tokens: Optional[int] = None
unified_memory_pool_bytes: Optional[int] = None
# DSV4 compressed-attention pool sizes (target only; draft workers leave at 0).
c4_max_total_num_tokens: int = 0
c128_max_total_num_tokens: int = 0
c4_state_pool_size: int = 0
c128_state_pool_size: int = 0
mem_fraction_static: Optional[float] = None
# Unified pool only: the profiled byte budget the buffer is sized from,
# instead of re-summing token counts; None when a user token cap is the budget.
unified_total_bytes: Optional[int] = None
def __post_init__(self):
if self.max_total_num_tokens <= 0:
msg = "Not enough memory. Please try to increase --mem-fraction-static."
if self.mem_fraction_static is not None:
msg += f" Current value: mem_fraction_static={self.mem_fraction_static}"
raise RuntimeError(msg)
if TYPE_CHECKING:
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
logger = logging.getLogger(__name__)
def _dflash_draft_cell_size(kvc: KVCacheConfigurator) -> int:
"""Bytes/token the DFLASH draft KV pool adds, 0 if none; replicated across DCP
ranks because the draft pool spans the widened virtual location space."""
if kvc.is_draft_worker or not kvc.spec_algorithm.is_dflash_family():
return 0
cell_size = kvc.spec_aux_config.dflash_draft_cell_size_per_token
if cell_size is None or int(cell_size) <= 0:
return 0
return int(cell_size) * get_parallel().attn_dcp_size
def _get_dsa_cache_layer_ids(kvc: KVCacheConfigurator, num_layers: int) -> list[int]:
"""Global layer ids represented by the local DSA pool's dense layer slots."""
if kvc.mambaish_config and not kvc.is_draft_worker:
layer_ids = [
layer_id
for layer_id in kvc.mambaish_config.full_attention_layer_ids
if kvc.layer_info.start_layer <= layer_id < kvc.layer_info.end_layer
]
else:
layer_ids = list(range(kvc.layer_info.start_layer, kvc.layer_info.end_layer))
# Draft pools and a few platform-specific pools may expose a synthetic layer
# count. They do not use indexShare, so only the length matters for sizing.
if len(layer_ids) != num_layers:
return list(range(num_layers))
return layer_ids
def _get_dsv4_compress_state_dtype_sizes() -> tuple[int, int]:
dtype_name = envs.SGLANG_DSV4_COMPRESS_STATE_DTYPE.get().strip().lower()
if dtype_name in ("float32", "fp32"):
return 4, 4
if dtype_name in ("bfloat16", "bf16"):
return 2, 2
raise ValueError(
"Unsupported SGLANG_DSV4_COMPRESS_STATE_DTYPE="
f"{dtype_name!r}. Expected one of: float32, fp32, bfloat16, bf16."
)
def check_dsv4_unified_fp8_pd_supported(
*, unified_fp8: bool, disaggregation_mode: str, pp_size: int, enable_hisparse: bool
) -> None:
"""PP and HiSparse still index kv_data as one region per layer; fp8 PD adds rope groups."""
if not unified_fp8 or disaggregation_mode == "null":
return
if pp_size > 1:
raise ValueError(
"SGLANG_DSV4_UNIFIED_KV_FP8=1 does not support PD disaggregation "
f"with pp_size={pp_size}: the PP re-slicing of the per-stage KV "
"regions has no coverage for the extra rope regions. Run PD with "
"pp_size=1 or unset the fp8 switch."
)
if enable_hisparse:
# HiSparse appends its device tail to kv_data and locates it as
# dst_kv_ptrs[c4_layer_num:] (mooncake/conn.py, decode.py), i.e. the
# slice that fp8 fills with the C4 rope regions.
raise ValueError(
"SGLANG_DSV4_UNIFIED_KV_FP8=1 does not support PD disaggregation "
"with --enable-hisparse: the host/device split locates its device "
"regions by layer count, which the fp8 rope regions shift."
)
class MemoryPoolConfigurator:
"""Base class for memory pool configurators.
Subclasses compute pool sizes for their architecture via coeff+bias model.
Both entry points return MemoryPoolConfig (with max_running_requests=None,
to be filled by the consumer).
"""
def calculate_pool_sizes(
self, available_bytes: int, page_size: int
) -> MemoryPoolConfig:
"""Profiling path: compute pool sizes from available bytes."""
raise NotImplementedError
def calculate_pool_sizes_from_max_tokens(
self, max_total_num_tokens: int, page_size: int
) -> MemoryPoolConfig:
"""Constraint path: recalculate pool sizes from a constrained max_tokens."""
raise NotImplementedError
def finalize_with_max_running_requests(
self, config: MemoryPoolConfig
) -> MemoryPoolConfig:
return config
@staticmethod
def validate_swa_pool_size(
swa_tokens: int, sliding_window_size: Optional[int], page_size: int
) -> None:
"""Reject an SWA pool too small to ever admit a request.
Prefill charges min(extend + decode, window) + page_size of SWA headroom
per request, so a pool at or below that floor rejects every request no
matter how far it drains: the scheduler spins in the waiting queue and
the server hangs at warmup instead of failing here.
"""
if sliding_window_size is None:
return
if sliding_window_size + page_size >= swa_tokens:
raise ValueError(
f"SWA pool ({swa_tokens} tokens) cannot hold even one request: "
f"the prefill admission floor is sliding_window_size "
f"({sliding_window_size}) + page_size ({page_size}). "
f"Increase --swa-full-tokens-ratio or the total KV budget."
)
class DefaultPoolConfigurator(MemoryPoolConfigurator):
"""Standard models (MHA, MLA, DSA, FP4): coeff = cell_size, bias = 0."""
def __init__(self, kvc: KVCacheConfigurator):
self.kv_cache_dtype_str = kvc.kv_cache_dtype_str
dcp_size = get_parallel().attn_dcp_size
# Determine effective number of layers for KV cache
if mambaish := mambaish_config(kvc.model_config):
effective_layer_ids = [
i
for i in mambaish.full_attention_layer_ids
if kvc.layer_info.start_layer <= i < kvc.layer_info.end_layer
]
num_layers = len(effective_layer_ids)
else:
num_layers = kvc.layer_info.num_effective_layers
self._cell_size = self._compute_cell_size(kvc, num_layers)
has_kv_on_another_pp_stage = (
self._cell_size == 0
and mambaish is not None
and bool(mambaish.full_attention_layer_ids)
and kvc.pp_size > 1
)
self._zero_kv_max_tokens = (
torch.iinfo(torch.int64).max
if has_kv_on_another_pp_stage
else get_schedule().max_total_tokens or kvc.model_config.context_len
)
# EAGLE/STANDALONE: assumes the draft shares the target's per-layer KV size,
# which holds for EAGLE/MTP drafts that reuse the target's attention config.
if (
kvc.spec_algorithm.is_eagle() or kvc.spec_algorithm.is_standalone()
) and not kvc.is_draft_worker:
eagle_draft_num_layers = kvc.spec_aux_config.eagle_draft_num_layers
fused_full_entry = kvc.fused_entry_bytes("full")
if fused_full_entry is not None:
self._cell_size = int(fused_full_entry)
elif (
eagle_draft_num_layers is not None
and int(eagle_draft_num_layers) > 0
and int(num_layers) > 0
):
draft_num_layers = int(eagle_draft_num_layers)
if is_deepseek_dsa(kvc.model_config.hf_config):
target_indexer_size = self._compute_dsa_indexer_cell_size(
kvc=kvc,
num_layers=num_layers,
)
if _is_npu and dcp_size > 1:
target_indexer_size *= dcp_size
target_kv_size = self._cell_size - target_indexer_size
from sglang.srt.layers.cp.utils import (
get_glm_dsa_layer_split_effective_num_layers,
)
target_kv_num_layers = get_glm_dsa_layer_split_effective_num_layers(
kvc, num_layers
)
draft_kv_size = int(
target_kv_size * draft_num_layers / target_kv_num_layers
)
draft_indexer_size = self._compute_dsa_indexer_cell_size(
kvc=kvc,
num_layers=draft_num_layers,
allocate_all_layers=True,
)
# The draft pool is replicated and consumes the widened
# allocator-global slot space on NPU DCP.
if _is_npu and dcp_size > 1:
draft_kv_size *= dcp_size
draft_indexer_size *= dcp_size
self._cell_size += draft_kv_size + draft_indexer_size
else:
self._cell_size = int(
self._cell_size * (1 + draft_num_layers / int(num_layers))
)
# DFLASH/DSPARK: the draft's per-token KV cost can differ from the target's
# (e.g. MLA target, per-head K/V draft), so size from the draft config.
if kvc.spec_algorithm.is_dflash_family() and not kvc.is_draft_worker:
from sglang.srt.speculative.dflash_utils import (
scale_kv_cell_size_per_token_for_dflash,
)
draft_num_layers = kvc.spec_aux_config.dflash_draft_num_layers
fused_full_entry = kvc.fused_entry_bytes("full")
if fused_full_entry is not None:
self._cell_size = int(fused_full_entry)
elif (
draft_num_layers is not None
and int(draft_num_layers) > 0
and int(num_layers) > 0
):
self._cell_size = scale_kv_cell_size_per_token_for_dflash(
target_cell_size_per_token=self._cell_size,
target_num_layers=int(num_layers),
draft_num_layers=int(draft_num_layers)
* get_parallel().attn_dcp_size,
draft_cell_size_per_token=_dflash_draft_cell_size(kvc) or None,
)
def _compute_cell_size(self, kvc: KVCacheConfigurator, num_layers: int) -> int:
"""Compute per-token KV cache cost in bytes."""
# args to config cell size
model_config = kvc.model_config
kv_cache_dtype = kvc.kv_cache_dtype
from sglang.srt.layers.cp.utils import (
get_glm_dsa_layer_split_effective_num_layers,
)
effective_num_layers = (
num_layers
if kvc.server_args.enable_hisparse
else get_glm_dsa_layer_split_effective_num_layers(kvc, num_layers)
)
kv_size = torch._utils._element_size(kv_cache_dtype)
tp_size = get_parallel().attn_tp_size
dcp_size = get_parallel().attn_dcp_size
if kvc.use_mla_backend:
if envs.SGLANG_NPU_ENABLE_SPARSE_KV_OFFLOAD.get():
# NPU sparse KV offload uses an index-only device pool.
from sglang.srt.hardware_backend.npu.sparsity_driven_kv_offload.config import (
get_sparsity_driven_kv_offload_cell_size,
)
offload_cell_size = get_sparsity_driven_kv_offload_cell_size(
model_config=model_config,
use_mla_backend=kvc.use_mla_backend,
num_layers=num_layers,
element_size=kv_size,
)
if offload_cell_size is not None:
return offload_cell_size
from sglang.srt.mem_cache.kv_cache_configurator import (
calculate_mla_kv_cache_dim,
)
cell_size = (
calculate_mla_kv_cache_dim(
model_config=model_config,
kv_cache_dtype=kv_cache_dtype,
)
* effective_num_layers
* kv_size
)
if is_float4_e2m1fn_x2(kv_cache_dtype):
# kv_scale_buffer
scale_block_size = 16
cell_size = (cell_size // 2) + (
(
(model_config.kv_lora_rank + model_config.qk_rope_head_dim)
// scale_block_size
)
* effective_num_layers
* kv_size
)
# Add indexer KV cache overhead for DSA models (DeepSeek V3.2)
if is_deepseek_dsa(model_config.hf_config):
indexer_cell_size = self._compute_dsa_indexer_cell_size(
kvc=kvc,
num_layers=num_layers,
)
if _is_npu and not kvc.is_draft_worker and dcp_size > 1:
indexer_cell_size *= dcp_size
cell_size += indexer_cell_size
elif is_minimax_sparse(model_config.hf_config):
from sglang.srt.server_args import m3_fp8_attn_gemm_enabled
# Mirrors MiniMaxSparseKVPool: main pool (K+V all layers) + indexer pool
# (sparse-only, single-head; kv layers store K+V, k-only layers store K).
sparse_cfg = get_minimax_sparse_attention_config(model_config.hf_config)
dense_layer_ids, sparse_layer_ids = get_minimax_sparse_layer_ids(sparse_cfg)
indexer_k_only_layer_ids = set(
get_minimax_sparse_disable_value_layer_ids(sparse_cfg)
)
local_dense_layer_ids = [
l
for l in dense_layer_ids
if kvc.layer_info.start_layer <= l < kvc.layer_info.end_layer
]
local_sparse_layer_ids = [
l
for l in sparse_layer_ids
if kvc.layer_info.start_layer <= l < kvc.layer_info.end_layer
]
num_dense = len(local_dense_layer_ids)
num_sparse = len(local_sparse_layer_ids)
num_indexer_k_only = sum(
1 for l in local_sparse_layer_ids if l in indexer_k_only_layer_ids
)
num_indexer_kv = num_sparse - num_indexer_k_only
kv_heads = model_config.get_num_kv_heads(get_parallel().attn_tp_size)
head_dim = model_config.head_dim
indexer_head_dim = sparse_cfg["sparse_index_dim"]
indexer_dtype_size = torch._utils._element_size(
get_minimax_sparse_index_dtype(
fp8_attn_gemm=m3_fp8_attn_gemm_enabled(
resolving_view(kvc.server_args)
),
kv_cache_dtype=kvc.kv_cache_dtype,
model_dtype=kvc.model_dtype,
)
)
full_pool_ratio = 1
if get_memory().enable_hisparse:
from sglang.srt.mem_cache.sparsity import parse_hisparse_config
full_pool_ratio = parse_hisparse_config().host_to_device_ratio
main_pool_bytes = (
(num_dense * full_pool_ratio + num_sparse)
* 2
* kv_heads
* head_dim
* kv_size
)
indexer_bytes = (
(num_indexer_kv * 2 + num_indexer_k_only)
* indexer_head_dim
* indexer_dtype_size
)
# FP4 scale buffer adjustment doesn't apply to MiniMax sparse:
# cell_size is already a sum over heterogeneous sub-pools.
return main_pool_bytes + indexer_bytes * full_pool_ratio
else:
n = model_config.get_num_kv_heads(tp_size, dcp_size)
cell_size = (
n
* (model_config.head_dim + model_config.v_head_dim)
* effective_num_layers
* kv_size
)
if is_float4_e2m1fn_x2(kv_cache_dtype):
from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import (
get_kv_cache_quant_method,
resolve_kv_cache_quant,
)
quant_name = resolve_kv_cache_quant(kvc.kv_cache_dtype_str)
if quant_name is None:
raise ValueError(
"FP4 storage dtype requires an explicit KV recipe name."
)
quant_method = get_kv_cache_quant_method(
quant_name,
num_layers=effective_num_layers,
device=kvc.device,
page_size=kvc.page_size,
)
quant_method.configure_attention_backends_from_server_args(
kvc.server_args
)
cell_size = quant_method.compute_cell_size(
n,
model_config.head_dim,
effective_num_layers,
kv_size,
)
elif self.kv_cache_dtype_str == "mxfp8":
scale_block_size = 32
cell_size += (
n * (model_config.head_dim + model_config.v_head_dim) * num_layers
) // scale_block_size
cell_size += self._compute_qsa_cell_size(
hf_config=model_config.hf_config, num_layers=num_layers
)
return cell_size
@staticmethod
def _compute_qsa_cell_size(*, hf_config, num_layers: int) -> int:
from sglang.srt.layers.attention.qsa.config import (
parse_qsa_profile,
)
from sglang.srt.mem_cache.qsa_kv_pool import (
QSATokenToKVPool,
resolve_qsa_indexer_dtype,
)
if num_layers == 0:
return 0
qsa_profile = parse_qsa_profile(hf_config)
if qsa_profile is None:
return 0
return QSATokenToKVPool.qsa_bytes_per_token(
kv_heads=qsa_profile.kv_heads,
head_dim=qsa_profile.head_dim,
compress_ratio=qsa_profile.compress_ratio,
num_layers=num_layers,
compressed_dtype=resolve_qsa_indexer_dtype(get_model().qsa_indexer_dtype),
)
def _compute_dsa_indexer_cell_size(
self,
*,
kvc: KVCacheConfigurator,
num_layers: int,
allocate_all_layers: bool = False,
) -> int:
index_head_dim = get_dsa_index_head_dim(kvc.model_config.hf_config)
indexer_size_per_token = ceil_div(
index_head_dim + index_head_dim // DSATokenToKVPool.quant_block_size * 4,
get_dsa_index_kpool(kvc.model_config.hf_config),
)
element_size = torch._utils._element_size(
DSATokenToKVPool.index_k_with_scale_buffer_dtype
)
if _is_npu:
from sglang.srt.hardware_backend.npu.utils import is_npu_arch35
dtype = kvc.kv_cache_dtype
# GPU sizing above assumes FP8 indexers; NPU also needs BF16 sizing.
if dtype != torch.float8_e4m3fn:
indexer_size_per_token = index_head_dim
element_size = torch._utils._element_size(dtype)
if not is_npu_arch35():
allocate_all_layers = True
memory_config = get_memory()
indexer_ratio = 1
if memory_config.enable_hisparse:
from sglang.srt.mem_cache.sparsity import parse_hisparse_config
indexer_ratio = parse_hisparse_config().host_to_device_ratio
from sglang.srt.mem_cache.kv_cache_configurator import (
_should_elide_dsa_index_k,
)
if (
allocate_all_layers
or kvc.server_args.enable_hisparse
or not _should_elide_dsa_index_k(is_draft_worker=kvc.is_draft_worker)
):
num_indexer_layers = num_layers
else:
from sglang.srt.layers.cp.utils import (
get_glm_dsa_cp_layer_shard_info,
get_layer_shard_range,
)
_, shard_size = get_glm_dsa_cp_layer_shard_info(kvc)
if shard_size > 1:
# GLM-5.3 hybrid-layer support is intentionally limited to the
# non-LayerSplit pool below.
active_indexer_layers = [
layer_id
for layer_id in range(
kvc.layer_info.start_layer, kvc.layer_info.end_layer
)
if not dsa_layer_skips_topk(kvc.model_config.hf_config, layer_id)
]
active_set = set(active_indexer_layers)
max_owned = 0
for rank in range(shard_size):
start, end = get_layer_shard_range(rank, shard_size, num_layers)
max_owned = max(
max_owned,
sum(
kvc.layer_info.start_layer + i in active_set
for i in range(start, end)
),
)
num_indexer_layers = max_owned + 1
else:
num_indexer_layers = sum(
not dsa_layer_skips_topk(kvc.model_config.hf_config, layer_id)
for layer_id in _get_dsa_cache_layer_ids(kvc, num_layers)
)
return int(
indexer_size_per_token * num_indexer_layers * element_size * indexer_ratio
)
def calculate_pool_sizes(
self, available_bytes: int, page_size: int
) -> MemoryPoolConfig:
available_bytes = max(available_bytes, 0)
max_total_num_tokens = (
available_bytes // self._cell_size
if self._cell_size
else self._zero_kv_max_tokens
)
max_total_num_tokens = max_total_num_tokens // page_size * page_size
return MemoryPoolConfig(max_total_num_tokens=max_total_num_tokens)
def calculate_pool_sizes_from_max_tokens(
self, max_total_num_tokens: int, page_size: int
) -> MemoryPoolConfig:
max_total_num_tokens = max_total_num_tokens // page_size * page_size
return MemoryPoolConfig(max_total_num_tokens=max_total_num_tokens)
class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
"""Splits memory between the full and SWA pools of MHA/MLA sliding-window models."""
def __init__(self, kvc: KVCacheConfigurator):
self.kv_cache_dtype_str = kvc.kv_cache_dtype_str
model_config = kvc.model_config
kv_cache_dtype = kvc.kv_cache_dtype
kv_size = torch._utils._element_size(kv_cache_dtype)
tp_size = get_parallel().attn_tp_size
self._full_layers_num = len(kvc.layer_info.full_attention_layer_ids)
self._swa_layers_num = len(kvc.layer_info.swa_attention_layer_ids)
assert self._swa_layers_num > 0, (
"Hybrid SWA model must have at least one SWA layer"
)
self._swa_full_tokens_ratio = get_schedule().swa_full_tokens_ratio
self._sliding_window_size = kvc.sliding_window_size
self._page_size = kvc.page_size
self._enable_unified_memory = get_memory().enable_unified_memory
if model_config.attention_arch == AttentionArch.MLA:
# MLA pool sizing uses latent dimensions rather than MHA heads.
from sglang.srt.mem_cache.kv_cache_configurator import (
calculate_mla_kv_cache_dim,
)
self._full_per_token = (
calculate_mla_kv_cache_dim(
model_config=model_config,
kv_cache_dtype=kv_cache_dtype,
)
* kv_size
)
if is_deepseek_dsa(model_config.hf_config):
index_head_dim = get_dsa_index_head_dim(model_config.hf_config)
index_elements = ceil_div(
index_head_dim
+ index_head_dim // DSATokenToKVPool.quant_block_size * 4,
get_dsa_index_kpool(model_config.hf_config),
)
self._full_per_token += index_elements * torch._utils._element_size(
DSATokenToKVPool.index_k_with_scale_buffer_dtype
)
self._swa_per_token = (
model_config.swa_kv_lora_rank + model_config.swa_qk_rope_head_dim
) * kv_size
else:
# Full layer per-token memory (bytes)
self._full_per_token = (
model_config.get_num_kv_heads(tp_size)
* (model_config.head_dim + model_config.v_head_dim)
* kv_size
)
# SWA layer per-token memory (bytes)
self._swa_per_token = (
model_config.get_swa_num_kv_heads(tp_size)
* (model_config.swa_head_dim + model_config.swa_v_head_dim)
* kv_size
)
if self.kv_cache_dtype_str == "mxfp8":
scale_block_size = 32
self._full_per_token += (
model_config.get_num_kv_heads(tp_size)
* (model_config.head_dim + model_config.v_head_dim)
) // scale_block_size
self._swa_per_token += (
model_config.get_swa_num_kv_heads(tp_size)
* (model_config.swa_head_dim + model_config.swa_v_head_dim)
) // scale_block_size
# Draft KV tensors use full, SWA, or full-capacity SWA geometry.
self._draft_full_layers_num = 0
self._draft_swa_layers_num = 0
self._draft_swa_full_layers_num = 0
if (
kvc.spec_algorithm.is_eagle() or kvc.spec_algorithm.is_standalone()
) and not kvc.is_draft_worker:
draft_layers = kvc.spec_aux_config.eagle_draft_num_layers
if draft_layers is not None and int(draft_layers) > 0:
draft_layers = int(draft_layers)
mtp_local_layer_ids = getattr(
getattr(model_config, "hf_text_config", None),
"mtp_local_layer_ids",
None,
)
if mtp_local_layer_ids is not None:
local_layer_ids = set(mtp_local_layer_ids)
self._draft_swa_full_layers_num = sum(
layer_id in local_layer_ids for layer_id in range(draft_layers)
)
else:
draft_swa_layers = kvc.spec_aux_config.eagle_draft_swa_num_layers
if draft_swa_layers is not None:
self._draft_swa_layers_num = min(
max(int(draft_swa_layers), 0), draft_layers
)
self._draft_full_layers_num = (
draft_layers
- self._draft_swa_layers_num
- self._draft_swa_full_layers_num
)
self._draft_cell_size = _dflash_draft_cell_size(kvc)
self._fused_full_entry = kvc.fused_entry_bytes("full")
if self._fused_full_entry is not None:
self._draft_full_layers_num = 0
self._draft_swa_layers_num = 0
self._draft_swa_full_layers_num = 0
self._draft_cell_size = 0
self._recompute_cell_size()
def _full_cell_bytes(self) -> int:
"""Bytes per full-side token: the fused entry when the draft is placed
there, else the target's rows plus the private draft's full rows."""
if self._fused_full_entry is not None:
return self._fused_full_entry
return self._full_per_token * (
self._full_layers_num + self._draft_full_layers_num
)
def _swa_cell_bytes(self) -> int:
return self._swa_per_token * (self._swa_layers_num + self._draft_swa_layers_num)
def _recompute_cell_size(self) -> None:
# Bytes per token of max_total_num_tokens: full_tokens when hybrid, else
# swa_tokens with no ratio, since all-SWA has no full pool to relate to.
if self._full_layers_num == 0:
self._cell_size = (
self._swa_per_token * self._swa_layers_num
+ self._full_per_token * self._draft_full_layers_num
+ self._swa_per_token * self._draft_swa_layers_num
+ self._swa_per_token * self._draft_swa_full_layers_num
+ self._draft_cell_size
)
else:
self._cell_size = (
self._full_cell_bytes()
+ self._swa_per_token * self._draft_swa_full_layers_num
+ self._swa_full_tokens_ratio * self._swa_cell_bytes()
+ self._draft_cell_size
)
def _draft_pool_bytes_per_token(self) -> int:
return int(
self._full_per_token * self._draft_full_layers_num
+ self._swa_per_token
* (self._draft_swa_layers_num + self._draft_swa_full_layers_num)
+ self._draft_cell_size
)
def _unified_full_bytes_per_token(self) -> int:
"""Bytes one full-side token takes in the unified pool: the fused
entry (host + draft + pad) when the draft lives in it, else the
target's own rows. A private draft pool is priced separately."""
if self._fused_full_entry is not None:
return self._fused_full_entry
return self._full_per_token * self._full_layers_num
def _unified_pool_bytes(self, full_tokens: int, swa_tokens: int) -> int:
return (
full_tokens * self._unified_full_bytes_per_token()
+ swa_tokens * self._swa_per_token * self._swa_layers_num
)
def _max_unified_full_tokens(
self,
available_bytes: int,
page_size: int,
fixed_swa_tokens: Optional[int] = None,
) -> int:
"""Find the largest page-aligned full capacity whose allocations fit."""
draft_bytes_per_token = self._draft_pool_bytes_per_token()
target_full_bytes_per_token = self._unified_full_bytes_per_token()
assert target_full_bytes_per_token > 0
def allocation_bytes(full_pages: int) -> int:
full_tokens = full_pages * page_size
swa_tokens = (
fixed_swa_tokens
if fixed_swa_tokens is not None
else int(full_tokens * self._swa_full_tokens_ratio)
// page_size
* page_size
)
target_bytes = self._unified_pool_bytes(full_tokens, swa_tokens)
virtual_span = max(target_bytes // target_full_bytes_per_token - 1, 0)
draft_tokens = ceil_align(virtual_span, page_size) + page_size
return target_bytes + draft_tokens * draft_bytes_per_token
max_pages = available_bytes // target_full_bytes_per_token // page_size
full_pages = (
bisect_right(range(max_pages + 1), available_bytes, key=allocation_bytes)
- 1
)
return max(full_pages, 0) * page_size
def _solve_pool_sizes(
self, max_total_num_tokens: int, page_size: int
) -> MemoryPoolConfig:
def align_page_size(x: int) -> int:
return (x // page_size) * page_size
if self._full_layers_num == 0:
# All-SWA: no full pool, max_total = actual SWA pool size.
# Ratio is not applied -- see _recompute_cell_size.
swa_tokens = align_page_size(max_total_num_tokens)
logger.info(
f"Use sliding window memory pool (all SWA). "
f"swa_layer_tokens={swa_tokens}"
)
return MemoryPoolConfig(
max_total_num_tokens=swa_tokens,
full_max_total_num_tokens=0,
swa_max_total_num_tokens=swa_tokens,
)
# Hybrid: full_tokens = max_total_num_tokens, swa_tokens = full_tokens * ratio
full_tokens = align_page_size(max_total_num_tokens)
swa_tokens = align_page_size(int(full_tokens * self._swa_full_tokens_ratio))
if not self._enable_unified_memory:
self.validate_swa_pool_size(
swa_tokens, self._sliding_window_size, self._page_size
)
logger.info(
f"Use sliding window memory pool. "
f"full_layer_tokens={full_tokens}, swa_layer_tokens={swa_tokens}"
)
return self._make_pool_config(full_tokens, swa_tokens)
def _make_pool_config(self, full_tokens: int, swa_tokens: int) -> MemoryPoolConfig:
return MemoryPoolConfig(
max_total_num_tokens=full_tokens,
full_max_total_num_tokens=full_tokens,
swa_max_total_num_tokens=swa_tokens,
unified_memory_pool_bytes=(
self._unified_pool_bytes(full_tokens, swa_tokens)
if self._enable_unified_memory
else None
),
)
def calculate_pool_sizes(
self, available_bytes: int, page_size: int
) -> MemoryPoolConfig:
if self._enable_unified_memory and self._full_layers_num > 0:
max_total_num_tokens = self._max_unified_full_tokens(
available_bytes, page_size
)
else:
max_total_num_tokens = int(available_bytes // self._cell_size)
return self._solve_pool_sizes(max_total_num_tokens, page_size)
def calculate_pool_sizes_from_max_tokens(
self, max_total_num_tokens: int, page_size: int
) -> MemoryPoolConfig:
return self._solve_pool_sizes(max_total_num_tokens, page_size)
def compute_swa_request_cap(
*,
page_size: int,
window: int,
attn_dp_size: int,
max_running_requests: int | None = None,
) -> int:
"""Worst-case SWA slots the scheduler holds live at max_running_requests."""
draft_tokens = get_spec().speculative_num_draft_tokens or 1
eviction_interval = max(1, envs.SGLANG_SWA_EVICTION_INTERVAL.get())
# __________[padding][eviction_interval][window]
# Padding to make sure eviction point is page-aligned.
trailing_tokens = window + eviction_interval * draft_tokens + page_size
if get_spec().speculative_algorithm is None:
decode_alloc = page_size
elif get_schedule().disable_overlap_schedule:
# spec-v1: new_tokens_required_next_decode per request.
decode_alloc = spec_decode_alloc_len_per_request(
page_size=page_size,
speculative_num_steps=get_spec().speculative_num_steps,
speculative_eagle_topk=get_spec().speculative_eagle_topk,
speculative_num_draft_tokens=get_spec().speculative_num_draft_tokens,
)
else:
# spec-v2: the overlap allocator keeps 2 * alloc_len outstanding
# (eagle_utils.eagle_prepare_for_decode: kv_committed_len + 2 * alloc_len).
decode_alloc = 2 * get_alloc_len_per_decode()
per_request = trailing_tokens + decode_alloc
if max_running_requests is None:
max_running_requests = get_schedule().max_running_requests
num_reqs = max_running_requests // attn_dp_size
if get_disagg().disaggregation_mode == "decode":
return (
per_request * num_reqs
+ (window + page_size) * get_disagg().disaggregation_decode_extra_slots
)
else:
chunks_in_flight = 1 if get_schedule().disable_overlap_schedule else 2
return (
per_request * num_reqs
+ chunks_in_flight * get_schedule().chunked_prefill_size
+ page_size
)
class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator):
"""Hybrid SWA with the SWA pool sized from the explicit max_running_requests
worst case instead of swa_full_tokens_ratio; the rest goes to the full pool."""
def __init__(self, kvc: KVCacheConfigurator):
self.kv_cache_dtype_str = kvc.kv_cache_dtype_str
super().__init__(kvc)
assert self._full_layers_num > 0
self._swa_cap = compute_swa_request_cap(
page_size=kvc.page_size,
window=kvc.sliding_window_size,
attn_dp_size=kvc.attn_dp_size,
)
@staticmethod
def is_applicable(kvc: KVCacheConfigurator) -> bool:
"""True when SWAChunkCache can be sized from explicit max requests."""
if get_schedule().max_running_requests is None:
return False
if not get_memory().disable_radix_cache:
return False
# The cap holds at most one prefill chunk per batch in flight; an
# unchunked prefill writes the whole prompt into the SWA pool.
chunked_prefill_size = get_schedule().chunked_prefill_size
if get_disagg().disaggregation_mode != "decode" and (
chunked_prefill_size is None or chunked_prefill_size <= 0
):
return False
if kvc.sliding_window_size is None:
return False
return len(kvc.layer_info.full_attention_layer_ids) > 0
def calculate_pool_sizes(
self, available_bytes: int, page_size: int
) -> MemoryPoolConfig:
# SWA pool sized tightly from the cap; the rest of the budget goes to full.
swa_tokens = ceil_align(self._swa_cap, page_size)
fixed_swa_bytes = swa_tokens * self._swa_cell_bytes()
if self._enable_unified_memory and self._draft_pool_bytes_per_token() > 0:
full_tokens = self._max_unified_full_tokens(
available_bytes, page_size, fixed_swa_tokens=swa_tokens
)
else:
full_cell_size = (
self._full_cell_bytes()
+ self._swa_per_token * self._draft_swa_full_layers_num
)
full_tokens = (
int((available_bytes - fixed_swa_bytes) // full_cell_size) // page_size
) * page_size
if full_tokens <= 0:
raise RuntimeError(
f"SWA pool cap ({swa_tokens} tokens, "
f"{fixed_swa_bytes / (1 << 30):.2f} GiB) leaves no room for the full "
f"KV pool within the available {available_bytes / (1 << 30):.2f} GiB. "
f"Reduce --max-running-requests, lower SGLANG_SWA_EVICTION_INTERVAL, "
f"or increase --mem-fraction-static."
)
return self._make_pool_config(full_tokens, swa_tokens)
def calculate_pool_sizes_from_max_tokens(
self, max_total_num_tokens: int, page_size: int
) -> MemoryPoolConfig:
# Constrained max_total goes to the full pool; SWA stays at its cap.
swa_tokens = ceil_align(self._swa_cap, page_size)
full_tokens = (max_total_num_tokens // page_size) * page_size
return self._make_pool_config(
full_tokens, min(swa_tokens, max_total_num_tokens)
)
# Used when --swa-full-tokens-ratio is at its default and cap mode is unusable.
DSV4_DEFAULT_SWA_FULL_TOKENS_RATIO = 0.1
def _operator_swa_full_tokens_ratio() -> Optional[float]:
"""The operator's --swa-full-tokens-ratio, or None when it was not given."""
schedule = get_schedule()
if not schedule._swa_full_tokens_ratio_explicitly_set:
return None
return schedule.swa_full_tokens_ratio
@dataclass
class _DSV4PoolSizes:
full_max_total_num_tokens: int
swa_max_total_num_tokens: int
c4_max_total_num_tokens: int
c128_max_total_num_tokens: int
c4_state_pool_size: int
c128_state_pool_size: int
class DSV4PoolConfigurator(MemoryPoolConfigurator):
"""DSV4 compressed attention: coeff is bytes_per_full_token, inflated by (T+D)/T
for a draft worker; bias is the request-scoped pools that do not scale with it."""
# object.__new__ stubs (SWA floor tests) skip __init__
_dspark_draft_on_bf16 = False
def __init__(self, kvc: KVCacheConfigurator):
self.kv_cache_dtype_str = kvc.kv_cache_dtype_str
cfg = kvc.model_config
self.qk_nope_head_dim = cfg.qk_nope_head_dim
self.qk_rope_head_dim = cfg.qk_rope_head_dim
self.indexer_head_dim = cfg.index_head_dim
self.attn_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
is_unified_kv_fp8,
is_unified_kv_triton,
)
from sglang.srt.mem_cache.deepseek_v4_memory_pool import (
dsv4_unified_row_bytes,
)
# Resolve the unified-kv gate before any sizing so the two cannot drift.
self._unified = is_unified_kv_triton()
self._unified_fp8 = is_unified_kv_fp8()
# DSpark draft still allocates a bf16 ring; target fp8 * (T+1)/T would
# under-count that ring (640 vs 1024). MTP keeps the old inflation.
self._dspark_draft_on_bf16 = bool(
self._unified_fp8 and kvc.spec_algorithm.is_dspark()
)
# Row width across both unified pools: 1024 B bf16, 640 B fp8.
self._unified_row_bytes = dsv4_unified_row_bytes(
self.qk_nope_head_dim, self.qk_rope_head_dim, self._unified_fp8
)
if self._unified:
# Unified_kv stores the whole latent: one bf16 row, or fp8 nope + bf16 rope.
self.kv_bytes = self._unified_row_bytes
elif get_exec().kernel.dsv4_attn_backend == "trtllm":
self.kv_bytes = self.attn_head_dim
else:
# One FlashMLA-layout latent slot, in bytes.
self.kv_bytes = self.qk_nope_head_dim + self.qk_rope_head_dim * 2 + 8
# HIP takes the FP4-accurate byte count here. The NVIDIA FP4 path
# keeps the FP8 estimate.
self.indexer_bytes_per_token = get_dsv4_indexer_bytes_per_token(
self.indexer_head_dim,
_is_hip and get_exec().kernel.enable_deepseek_v4_fp4_indexer,
)
self.context_len = kvc.model_config.context_len
# PP-local slice; matches DeepSeekV4TokenToKVPool's stage_ratios.
stage = range(kvc.layer_info.start_layer, kvc.layer_info.end_layer)
self.stage_compress_ratios = cfg.compress_ratios[stage.start : stage.stop]
if kvc.pp_size > 1:
logger.info(
f"DSV4 pool PP slice: rank={kvc.pp_group.rank_in_group} "
f"layers=[{stage.start},{stage.stop}) "
f"local={len(self.stage_compress_ratios)}/{len(cfg.compress_ratios)}"
)
# Layers of this stage that own compressed storage, per ratio. Same rule
# as the pool, so a ratio budgeted here is one it allocates.
self.stage_owner_layers = collect_sources_by_ratio(
cfg.compress_ratios, cfg.hf_config.kv_source_layer_ids, stage
)
self.operator_swa_ratio = _operator_swa_full_tokens_ratio()
self.swa_ratio = (
self.operator_swa_ratio
if self.operator_swa_ratio is not None
else DSV4_DEFAULT_SWA_FULL_TOKENS_RATIO
)
self.sliding_window_size = kvc.sliding_window_size
self.page_size = kvc.page_size
self.is_speculative = get_spec().speculative_algorithm is not None
self.online_c128_mtp_max_draft_tokens = max_speculative_num_draft_tokens() or 0
self.attn_dp_size = kvc.attn_dp_size
self.requested_max_running_requests_per_worker = (
get_schedule().max_running_requests // kvc.attn_dp_size
if get_schedule().max_running_requests is not None
else None
)
self.disaggregation_mode = get_disagg().disaggregation_mode
self.disaggregation_decode_extra_slots = (
get_disagg().disaggregation_decode_extra_slots or 0
)
if get_memory().enable_hisparse:
from sglang.srt.mem_cache.sparsity import parse_hisparse_config
self.c4_shrink_factor = parse_hisparse_config().host_to_device_ratio
else:
self.c4_shrink_factor = 1
assert self.c4_shrink_factor >= 1
if self.c4_shrink_factor > 1:
logger.info(f"HiSparse c4 host-to-device ratio = {self.c4_shrink_factor}")
# Ratio 1 keeps no compress state, so it has no ring.
self.ring_sizes = {
ratio: resolve_compress_state_ring_size(ratio)
for ratio in self.stage_owner_layers
if ratio != 1
}
self.num_layers_total = len(self.stage_compress_ratios)
# The low-ratio indexer pools are built with force_fp4=True
# (deepseek_v4_memory_pool), so they are fp4 whatever dtype c4 uses.
self.low_ratio_index_bytes = get_dsv4_indexer_bytes_per_token(
self.indexer_head_dim, use_fp4_indexer=True
)
# kvc.sliding_window_size is None on a runner without SWA layers.
self._swa_ring_size = get_swa_ring_size(cfg.window_size, self.is_speculative)
self._spec_infl = 1.0
# The unified pool ignores --kv-cache-dtype; V4 resolves "auto" to fp8_e4m3
# (overrides.py _deepseek_v4_kv_cache_dtype), so only bfloat16 is explicit.
if self._unified_fp8 and self.kv_cache_dtype_str == "bfloat16":
logger.warning(
"--kv-cache-dtype=bfloat16 is ignored on the unified_kv path; "
"SGLANG_DSV4_UNIFIED_KV_FP8=1 stores the latent as fp8. Unset the "
"env switch to get a bf16 unified pool."
)
check_dsv4_unified_fp8_pd_supported(
unified_fp8=self._unified_fp8,
disaggregation_mode=self.disaggregation_mode,
pp_size=kvc.pp_size,
enable_hisparse=get_memory().enable_hisparse,
)
if self.is_speculative:
# Ring is sized once here, so it must serve the largest adaptive tier.
self._assert_ring_serves_draft_tokens(
max_speculative_num_draft_tokens() or 0
)
self.encoder_replay = get_exec().features.enable_encoder_swa_bounded_replay
self.paged_draft_layers = 0
if self.encoder_replay and kvc.spec_algorithm.is_dspark():
self.paged_draft_layers = int(
kvc.spec_aux_config.dflash_draft_num_layers or 0
)
assert self.paged_draft_layers > 0, "DSpark draft layer count is required"
self.request_window_bytes = 0
if self.encoder_replay:
slots = self.requested_max_running_requests_per_worker + 1
capacity = ceil_align(
self.sliding_window_size + self.online_c128_mtp_max_draft_tokens,
self.page_size,
)
layers = self.num_layers_total
scratch = (
max(
get_schedule().chunked_prefill_size,
slots * max(128, self.online_c128_mtp_max_draft_tokens),
)
+ slots * 128
+ self.page_size
)
self.request_window_bytes = (
(slots * capacity + self.page_size) * layers * (self.kv_bytes + 16)
+ 4 * scratch * (self.kv_bytes + 16)
+ slots * 3 * 16 * self.attn_head_dim * 8
)
if not self.paged_draft_layers:
self.swa_ratio = 0
self.swa_prefix_tails = self._resolve_swa_prefix_tails()
self.swa_cap_tokens = (
0
if self.encoder_replay and not self.paged_draft_layers
else self._resolve_swa_cap_tokens()
)
self.bytes_per_swa_token = self._get_bytes_per_swa_token()
self.bytes_per_full_token = self._get_bytes_per_full_token()
if self.is_speculative and not self.encoder_replay:
# Reserve the draft worker by inflating per-token bytes by
# (target+draft)/target, as scale_kv_cell_size_per_token_for_dflash does.
draft_layers = 1
target_layers = self.num_layers_total
self._spec_infl = (target_layers + draft_layers) / target_layers
self.bytes_per_full_token *= self._spec_infl
self.bytes_per_swa_token *= self._spec_infl
# Online c128 keeps one in-progress (max, sum, kv) state per index and
# assumes forward-only; MTP would need rollback across draft and verify.
if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
allow_experimental_online_c128_mtp = (
envs.SGLANG_EXPERIMENTAL_ONLINE_C128_MTP.get()
and kvc.spec_algorithm.is_eagle()
)
assert kvc.spec_algorithm.is_none() or allow_experimental_online_c128_mtp, (
"SGLANG_OPT_USE_ONLINE_COMPRESS does not support speculative decode "
"(MTP) yet, except the experimental EAGLE topk=1 path gated by "
"SGLANG_EXPERIMENTAL_ONLINE_C128_MTP=1"
)
if allow_experimental_online_c128_mtp:
assert self.online_c128_mtp_max_draft_tokens > 0, (
"SGLANG_EXPERIMENTAL_ONLINE_C128_MTP requires "
"speculative_num_draft_tokens to be set."
)
logger.warning(
"DSV4 compressed attention: experimental online c128 + MTP enabled "
f"(EAGLE topk=1 only, "
f"draft_banks={self.online_c128_mtp_max_draft_tokens}). "
"Validate correctness carefully."
)
else:
logger.info(
"DSV4 compressed attention: online c128 enabled (ring_size=1)"
)
def num_layers(self, ratio: int) -> int:
"""Layers of this stage owning ratio's compressed storage."""
return len(self.stage_owner_layers.get(ratio, ()))
def _assert_ring_serves_draft_tokens(self, num_draft_tokens: int) -> None:
"""A verify batch writes its whole optimistic tail into the ring, so ring
capacity bounds the draft count."""
for compress_ratio in (4, 128):
if self.num_layers(compress_ratio) == 0:
continue
if compress_ratio == 128 and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
# Online c128 keeps per-draft state instead of a ring; sized separately.
continue
ring_size = self.ring_sizes[compress_ratio]
max_draft_tokens = get_compress_state_write_pad(compress_ratio, ring_size)
assert num_draft_tokens <= max_draft_tokens, (
f"speculative_num_draft_tokens={num_draft_tokens} exceeds what the c{compress_ratio} "
f"compress state ring can keep resident (ring_size={ring_size} serves at most "
f"{max_draft_tokens} draft tokens). Lower the draft count, or grow the ring in "
f"get_compress_state_ring_size()."
)
def _resolve_swa_prefix_tails(self) -> int:
"""Cached prefix tails cap mode keeps addressable: a prefix is reusable only
while its last sliding_window tokens still hold SWA slots."""
prefix_tails = get_schedule().swa_prefix_tails
if prefix_tails is not None:
return prefix_tails
if get_memory().disable_radix_cache:
# Nothing is kept for reuse, so the request cap alone bounds the pool.
return 0
max_running_requests = self.requested_max_running_requests_per_worker
return 4 * max_running_requests if max_running_requests is not None else 0
def _resolve_swa_cap_tokens(self) -> Optional[int]:
"""SWA slots to reserve in cap mode, None to keep ratio sizing. Cap mode
budgets from the request cap plus radix headroom, not full_tokens."""
if self.operator_swa_ratio is not None:
return None
if self._unified:
# Ring mode: SWA is a fixed per-request ring, with no paged pool to size.
return None
max_running_requests = self.requested_max_running_requests_per_worker
if max_running_requests is None or self.sliding_window_size is None:
return None
chunked_prefill_size = get_schedule().chunked_prefill_size
if self.disaggregation_mode != "decode" and (
chunked_prefill_size is None or chunked_prefill_size <= 0
):
return None
cap = compute_swa_request_cap(
page_size=self.page_size,
window=self.sliding_window_size,
attn_dp_size=self.attn_dp_size,
)
headroom = self.swa_prefix_tails * (self.sliding_window_size + self.page_size)
return ceil_align(cap + headroom, self.page_size)
def _get_paged_kv_bytes_per_token(self, compress_ratio: int = 0) -> float:
# Unified rings, the NPU pool and the trtllm uniform-FP8 pool do not go
# through DeepSeekV4SingleKVPool.create_buffer, so they carry no page pad.
if self._unified or _is_npu or get_exec().kernel.dsv4_attn_backend == "trtllm":
return self.kv_bytes
from sglang.srt.mem_cache.deepseek_v4_memory_pool import (
resolve_compressed_kv_layout,
select_dsv4_kv_layout,
)
layout, compressed_option = select_dsv4_kv_layout()
page_size = self.page_size
if compress_ratio:
layout = resolve_compressed_kv_layout(
layout, compress_ratio, compressed_option
)
page_size = self.page_size // compress_ratio
return layout.page_bytes(page_size) / page_size
def _get_bytes_per_swa_token(self) -> float:
"""Bytes one SWA slot costs across the stage. c4_state_pool_size = swa_tokens
// page_size * ring, so c4 compress state is priced per SWA slot too."""
if self.encoder_replay:
# Target SWA lives in the request window; only the draft owns paged SWA
# bytes, and its layers carry no compressed state.
return self.kv_bytes * self.paged_draft_layers
c4_state_dtype_size, _ = _get_dsv4_compress_state_dtype_sizes()
c4_state_bytes = 2 * 2 * self.attn_head_dim * c4_state_dtype_size
c4_indexer_state_bytes = 2 * 2 * self.indexer_head_dim * c4_state_dtype_size
c4_state_ratio = self.ring_sizes.get(4, 0) / self.page_size
return self._get_paged_kv_bytes_per_token() * self.num_layers_total + (
c4_state_ratio
* (c4_state_bytes + c4_indexer_state_bytes)
* self.num_layers(4)
)
def _compressed_bytes_per_full_token(self, ratio: int) -> float:
"""Compressed KV (+ indexer) bytes one full token adds per layer of `ratio`;
compress state is priced per SWA slot (c4) or as fixed bytes (c128, ratio 2)."""
if ratio in (1, 2):
return (self.kv_bytes + self.low_ratio_index_bytes) / ratio
if ratio == 4:
c4_frac = 1 / (4 * self.c4_shrink_factor)
return (
c4_frac * self._get_paged_kv_bytes_per_token(4)
+ 1 / 4 * self.indexer_bytes_per_token
)
assert ratio == 128, f"unsupported compression ratio: {ratio}"
return 1 / 128 * self._get_paged_kv_bytes_per_token(128)
def _get_bytes_per_full_token(self) -> float:
# Cap mode and ring mode both move the SWA pool and the c4 state that
# follows it out of the coefficient and into fixed bytes.
swa_ratio = (
0 if self._unified or self.swa_cap_tokens is not None else self.swa_ratio
)
return swa_ratio * self.bytes_per_swa_token + sum(
self._compressed_bytes_per_full_token(ratio) * len(layers)
for ratio, layers in self.stage_owner_layers.items()
)
def _get_swa_fixed_bytes(self) -> float:
"""Bias bytes of the encoder-replay request window plus the cap-mode pool."""
paged_bytes = (
0
if self.swa_cap_tokens is None
else self.swa_cap_tokens * self.bytes_per_swa_token
)
return self.request_window_bytes + paged_bytes
def _get_swa_tokens(self, full_token: int, page_size: int) -> int:
# swa_cap_tokens was already page-aligned at resolve time.
if self.swa_cap_tokens is None:
return int(full_token * self.swa_ratio) // page_size * page_size
return self.swa_cap_tokens
def _compute_dsv4_sizes(self, full_token: int, page_size: int) -> _DSV4PoolSizes:
full_token = full_token // page_size * page_size
swa_tokens = self._get_swa_tokens(full_token, page_size)
if self.swa_cap_tokens is None:
# Only ratio sizing can under-size a request: cap mode sizes from the
# request floor, and encoder replay deliberately runs swa_tokens == 0.
if not self._unified:
self.validate_swa_pool_size(
swa_tokens, self.sliding_window_size, page_size
)
source = "explicit" if self.operator_swa_ratio is not None else "default"
mode = (
"ring (paged swa_tokens vestigial)"
if self._unified
else f"ratio ({source})"
)
logger.info(
f"DSV4 SWA sizing: mode={mode}, swa_tokens={swa_tokens}, "
f"swa_full_tokens_ratio={self.swa_ratio}"
)
else:
logger.info(
f"DSV4 SWA sizing: mode=cap, swa_tokens={swa_tokens}, "
f"request_cap+headroom={self.swa_cap_tokens}, "
f"prefix_tails={self.swa_prefix_tails}"
)
return _DSV4PoolSizes(
full_max_total_num_tokens=full_token,
swa_max_total_num_tokens=swa_tokens,
c4_max_total_num_tokens=full_token // (4 * self.c4_shrink_factor),
c128_max_total_num_tokens=full_token // 128,
# Unified_kv: request-scoped, finalized once concurrency is known.
# Otherwise the ring is addressed per SWA page (swa_loc // page_size).
c4_state_pool_size=(
0
if self._unified
else swa_tokens // page_size * self.ring_sizes.get(4, 0)
),
c128_state_pool_size=0,
)
def _get_num_req_slots(self, max_running_requests: int) -> int:
if self.disaggregation_mode == "decode":
return max_running_requests + self.disaggregation_decode_extra_slots + 1
return max_running_requests + 1
def _get_c128_state_fixed_bytes(self, max_running_requests: int) -> int:
num_layers = self.num_layers(128)
if num_layers == 0:
return 0
_, c128_state_dtype_size = _get_dsv4_compress_state_dtype_sizes()
num_req_slots = self._get_num_req_slots(max_running_requests)
ring_size = self.ring_sizes[128]
if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
state_rows = num_req_slots + ring_size + 1
state_rows *= 1 + self.online_c128_mtp_max_draft_tokens
state_last_dim = 3 * self.attn_head_dim
else:
state_pool_size = num_req_slots * ring_size
state_rows = state_pool_size + ring_size + 1
state_rows = ceil_div(state_rows, 128) * 128
state_last_dim = 2 * self.attn_head_dim
return state_rows * state_last_dim * c128_state_dtype_size * num_layers
def _get_c2_state_fixed_bytes(self, max_running_requests: int) -> int:
"""Ratio-2 pending-pair state, one fp32 (kv, score) ring per request slot
on each ratio-2 kv_source layer; mirrors _make_pair_state_pool."""
num_layers = self.num_layers(2)
if num_layers == 0:
return 0
num_req_slots = self._get_num_req_slots(max_running_requests)
ring_size = self.ring_sizes[2]
# CompressStatePool allocates `size + ring_size + 1` rows, padded to the ratio.
state_rows = num_req_slots * ring_size + ring_size + 1
state_rows = ceil_div(state_rows, 2) * 2
state_last_dim = 2 * self.attn_head_dim
return state_rows * state_last_dim * torch.float32.itemsize * num_layers
def _unified_c4_state_pool_size(self, max_running_requests: int) -> int:
# Unified C4 state loc is req_pool_idx * c4_ring_size + pos % c4_ring_size.
num_req_slots = self._get_num_req_slots(max_running_requests)
return num_req_slots * self.ring_sizes[4]
def _fixed_c4_state_bytes(self, max_running_requests: int) -> int:
num_layers = self.num_layers(4)
if not self._unified or num_layers == 0:
return 0
c4_state_dtype_size, _ = _get_dsv4_compress_state_dtype_sizes()
# Mirror CompressStatePool.__init__: it allocates `size + ring_size + 1`
# rows, padded to the compress ratio.
state_rows = self._unified_c4_state_pool_size(max_running_requests)
state_rows = ceil_div(state_rows + self.ring_sizes[4] + 1, 4) * 4
# overlap c4: last_dim = 2 * (1 + overlap) * head_dim = 4 * head_dim.
core_bytes = 4 * self.attn_head_dim * c4_state_dtype_size
indexer_bytes = 4 * self.indexer_head_dim * c4_state_dtype_size
return state_rows * (core_bytes + indexer_bytes) * num_layers
def _resolve_max_running_requests_per_worker(self, available_bytes: int) -> int:
# Approximates ModelRunner._resolve_max_num_reqs. Over-estimating is safe:
# a larger fixed bias yields a smaller full_token.
if self.requested_max_running_requests_per_worker is not None:
return self.requested_max_running_requests_per_worker
full_token = int(available_bytes / self.bytes_per_full_token)
estimated = int(full_token / self.context_len * 512)
estimated = max(min(estimated, 4096), 2048)
return min(estimated, full_token // 2)
def _fixed_swa_bytes(self, max_running_requests: int) -> int:
"""Unified_kv SWA ring bytes, sized by req slots; 0 off unified, where SWA is
priced per token. DSpark+fp8 adds a bf16 draft ring instead of _spec_infl."""
if not self._unified:
return 0
num_req_slots = self._get_num_req_slots(max_running_requests)
target_ring = (
num_req_slots
* self._swa_ring_size
* self._unified_row_bytes
* self.num_layers_total
)
if self._dspark_draft_on_bf16:
from sglang.srt.mem_cache.deepseek_v4_memory_pool import (
dsv4_unified_row_bytes,
)
draft_row = dsv4_unified_row_bytes(
self.qk_nope_head_dim, self.qk_rope_head_dim, fp8=False
)
# 1 layer is what shipped DSpark drafts allocate; a multi-layer draft
# under-counts ~9 MB/layer, covered by the (T+1)/T inflation.
draft_ring = num_req_slots * self._swa_ring_size * draft_row
return int(target_ring + draft_ring)
return int(target_ring * self._spec_infl)
def _to_config(self, sizes: _DSV4PoolSizes) -> MemoryPoolConfig:
full = sizes.full_max_total_num_tokens
swa = sizes.swa_max_total_num_tokens
logger.info(
f"DSV4 pool sizes: full={full}, swa={swa}, "
f"c4={sizes.c4_max_total_num_tokens}, "
f"c128={sizes.c128_max_total_num_tokens}, "
f"c4_state={sizes.c4_state_pool_size}, "
f"c128_state={sizes.c128_state_pool_size}"
)
return MemoryPoolConfig(
max_total_num_tokens=full,
full_max_total_num_tokens=full,
swa_max_total_num_tokens=swa,
c4_max_total_num_tokens=sizes.c4_max_total_num_tokens,
c128_max_total_num_tokens=sizes.c128_max_total_num_tokens,
c4_state_pool_size=sizes.c4_state_pool_size,
c128_state_pool_size=sizes.c128_state_pool_size,
)
def finalize_with_max_running_requests(
self, config: MemoryPoolConfig
) -> MemoryPoolConfig:
assert config.max_running_requests is not None
num_req_slots = self._get_num_req_slots(config.max_running_requests)
if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
config.c128_state_pool_size = num_req_slots
else:
config.c128_state_pool_size = num_req_slots * self.ring_sizes.get(128, 0)
# Ring mode: C4 state is request-scoped, so size it from the known concurrency.
if self._unified and self.num_layers(4) > 0:
config.c4_state_pool_size = self._unified_c4_state_pool_size(
config.max_running_requests
)
return config
def calculate_pool_sizes(
self, available_bytes: int, page_size: int
) -> MemoryPoolConfig:
assert page_size % 128 == 0, (
"page_size must be multiple of 128 for compressed attention"
)
max_running_requests_per_worker = self._resolve_max_running_requests_per_worker(
available_bytes
)
c128_state_fixed_bytes = self._get_c128_state_fixed_bytes(
max_running_requests_per_worker
)
c2_state_fixed_bytes = self._get_c2_state_fixed_bytes(
max_running_requests_per_worker
)
swa_ring_fixed_bytes = self._fixed_swa_bytes(max_running_requests_per_worker)
c4_state_fixed_bytes = self._fixed_c4_state_bytes(
max_running_requests_per_worker
)
swa_fixed_bytes = self._get_swa_fixed_bytes()
fixed_bytes = (
c128_state_fixed_bytes
+ c2_state_fixed_bytes
+ swa_fixed_bytes
+ swa_ring_fixed_bytes
+ c4_state_fixed_bytes
)
available_bytes_for_tokens = max(available_bytes - fixed_bytes, 0)
full_token = int(available_bytes_for_tokens / self.bytes_per_full_token)
if full_token <= 0 and self.swa_cap_tokens is not None:
raise RuntimeError(
f"The DSV4 SWA pool cap ({self.swa_cap_tokens} tokens, "
f"{swa_fixed_bytes / (1 << 30):.2f} GB) leaves no room for the full "
f"KV pool within the available {available_bytes / (1 << 30):.2f} GB. "
f"Reduce --max-running-requests, lower --swa-prefix-tails "
f"or SGLANG_SWA_EVICTION_INTERVAL, or increase --mem-fraction-static."
)
sizes = self._compute_dsv4_sizes(full_token, page_size)
logger.info(
f"DSV4 memory calculation: unified={self._unified}, "
f"unified_fp8={self._unified_fp8}, "
f"dspark_draft_bf16={self._dspark_draft_on_bf16}, "
f"bytes_per_full_token={self.bytes_per_full_token:.2f}, "
f"available_bytes={available_bytes / (1 << 30):.2f} GB, "
f"c128_state_fixed={c128_state_fixed_bytes / (1 << 30):.2f} GB, "
f"c2_state_fixed={c2_state_fixed_bytes / (1 << 30):.2f} GB, "
f"swa_fixed={swa_fixed_bytes / (1 << 30):.2f} GB, "
f"swa_ring_fixed={swa_ring_fixed_bytes / (1 << 30):.2f} GB, "
f"c4_state_fixed={c4_state_fixed_bytes / (1 << 30):.2f} GB, "
f"full_token={sizes.full_max_total_num_tokens}"
)
return self._to_config(sizes)
def calculate_pool_sizes_from_max_tokens(
self, max_total_num_tokens: int, page_size: int
) -> MemoryPoolConfig:
# Token count, not a byte budget: the fixed pools are not re-subtracted, so
# the input must not exceed what calculate_pool_sizes derived for it.
assert page_size % 128 == 0, (
"page_size must be multiple of 128 for compressed attention"
)
sizes = self._compute_dsv4_sizes(max_total_num_tokens, page_size)
return self._to_config(sizes)
def create_memory_pool_configurator(
kvc: KVCacheConfigurator,
) -> MemoryPoolConfigurator:
if is_deepseek_v4(kvc.model_config.hf_config) and kvc.is_hybrid_swa:
return DSV4PoolConfigurator(kvc)
if kvc.is_hybrid_swa:
if SWAChunkCapPoolConfigurator.is_applicable(kvc):
return SWAChunkCapPoolConfigurator(kvc)
return HybridSWAPoolConfigurator(kvc)
# Future: MambaPoolConfigurator
return DefaultPoolConfigurator(kvc)