Files
sglang/python/sglang/srt/arg_groups/kv_cache_hook.py
T

951 lines
43 KiB
Python

# SPDX-License-Identifier: Apache-2.0
"""Server-argument resolution for KV-cache dtype and pool compatibility."""
from __future__ import annotations
import logging
from typing import Any, Optional
from sglang.srt.arg_groups.overrides import (
attention_backends_of,
declare_resolution,
model_config_of,
resolution_result,
resolved_view,
resolving_view,
use_mla_backend,
)
from sglang.srt.configs.hybrid_arch import mambaish_config
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.runtime_context import attn_dp_enabled_of, get_platform
logger = logging.getLogger(__name__)
_NVFP4_PREFILL_BACKEND = {
"fp8_e4m3": "flashinfer",
"nvfp4": "trtllm_mha",
}
_NVFP4_PREFILL_DEQUANT_DTYPE = {
backend: dtype for dtype, backend in _NVFP4_PREFILL_BACKEND.items()
}
def handle_nvfp4_prefill_kv_dequant_dtype(server_args: Any) -> None:
"""Resolve the public prefill dequantization dtype to attention backends."""
cfg = resolving_view(server_args)
requested_dtype = cfg.prefill_kv_cache_dequant_dtype
if cfg.kv_cache_dtype != "nvfp4":
if requested_dtype != "auto":
raise ValueError(
"--prefill-kv-cache-dequant-dtype applies only with "
"--kv-cache-dtype=nvfp4."
)
return
if requested_dtype == "auto":
explicit_backend = cfg.prefill_attention_backend or cfg.attention_backend
if explicit_backend in _NVFP4_PREFILL_DEQUANT_DTYPE:
requested_dtype = _NVFP4_PREFILL_DEQUANT_DTYPE[explicit_backend]
else:
if explicit_backend is not None:
raise ValueError(
"NVFP4 prefill supports an FP8 E4M3 workspace or native "
f"NVFP4, but backend {explicit_backend!r} provides neither."
)
requested_dtype = "nvfp4" if get_platform().is_sm100 else "fp8_e4m3"
if requested_dtype == "nvfp4" and not get_platform().is_sm100:
raise ValueError(
"Native NVFP4 prefill currently requires SM100; use "
"--prefill-kv-cache-dequant-dtype=fp8_e4m3 on this platform."
)
target_prefill_backend = _NVFP4_PREFILL_BACKEND[requested_dtype]
explicit_prefill_backend = cfg.prefill_attention_backend
if (
explicit_prefill_backend is not None
and explicit_prefill_backend != target_prefill_backend
):
raise ValueError(
f"--prefill-kv-cache-dequant-dtype={requested_dtype} requires prefill "
f"backend {target_prefill_backend!r}, but "
f"--prefill-attention-backend={explicit_prefill_backend!r} was set. "
"Remove the backend option and select the KV dtype only."
)
explicit_decode_backend = cfg.decode_attention_backend
if explicit_decode_backend not in (None, "trtllm_mha"):
raise ValueError(
"NVFP4 decode requires --decode-attention-backend=trtllm_mha; got "
f"{explicit_decode_backend!r}. Remove the backend option; NVFP4 "
"selects the supported decode implementation automatically."
)
updates = {
"prefill_attention_backend": target_prefill_backend,
"decode_attention_backend": "trtllm_mha",
}
if cfg.prefill_kv_cache_dequant_dtype == "auto":
updates["prefill_kv_cache_dequant_dtype"] = requested_dtype
declare_resolution(server_args, "_handle_nvfp4_prefill_kv_dequant_dtype", **updates)
logger.info(
"NVFP4 prefill dequant dtype: %s; prefill input: %s; decode input: nvfp4.",
requested_dtype,
requested_dtype,
)
def handle_mxfp8_kv_cache_compatibility(server_args: Any) -> None:
"""MXFP8 KV cache uses operands available only on SM100+ (Blackwell)."""
cfg = resolving_view(server_args)
if cfg.kv_cache_dtype != "mxfp8":
return
if not get_platform().is_blackwell:
raise ValueError(
"--kv-cache-dtype mxfp8 requires an SM100+ (Blackwell) GPU for the "
"block-scaled operands used by the FA4 MXFP8 attention path."
)
def handle_kv4_compatibility(server_args: Any) -> None:
"""Check FP4 KV cache compatibility with the attention backend"""
cfg = resolving_view(server_args)
if cfg.kv_cache_dtype not in ("nvfp4", "fp4_mx_block16"):
return
uses_mla = use_mla_backend(server_args)
prefill_backend, decode_backend = attention_backends_of(resolved_view(server_args))
attention_backend = resolved_view(server_args).attention_backend
if get_platform().is_cuda:
if cfg.kv_cache_dtype == "nvfp4" and not (
get_platform().is_sm100 or get_platform().is_sm120
):
raise RuntimeError(
"--kv-cache-dtype=nvfp4 requires Blackwell SM100 or SM120. "
"Use --kv-cache-dtype=fp4_mx_block16 for the block-size-16 FP4 recipe."
)
if cfg.enable_unified_memory:
raise ValueError(
"FP4 KV cache does not yet support --enable-unified-memory: "
"the unified MHA pool does not allocate FP4 block scales or "
"the prefill dequant workspace."
)
# SM100 trtllm_mha owns physical, kernel-native NVFP4 scales. The
# transfer and host-tier pools do not preserve that layout yet. Keep
# these combinations fail-fast while allowing target verification to
# reuse the same monolithic cache and GenMHA kernels.
uses_sm100_trtllm_nvfp4 = (
cfg.kv_cache_dtype == "nvfp4"
and get_platform().is_sm100
and "trtllm_mha" in (prefill_backend, decode_backend)
)
uses_mixed_nvfp4 = (
cfg.kv_cache_dtype == "nvfp4"
and prefill_backend == "flashinfer"
and decode_backend == "trtllm_mha"
)
uses_sm100_mixed_nvfp4 = uses_sm100_trtllm_nvfp4 and uses_mixed_nvfp4
speculative_algorithm = (
cfg.speculative_algorithm.upper()
if cfg.speculative_algorithm is not None
else None
)
supported_native_spec_algorithms = {
"EAGLE",
"EAGLE3",
"NEXTN",
"NGRAM",
}
if uses_sm100_trtllm_nvfp4 and speculative_algorithm in (
"DFLASH",
"DSPARK",
):
# These workers commit only a prefix of a dense candidate block
# through set_kv_buffer_prefix_valid(). That specialized writer
# does not produce GenMHA's physical K/V scale layout yet.
raise ValueError(
"SM100 native NVFP4 speculative decoding does not yet support "
f"{speculative_algorithm}; use EAGLE/NEXTN or NGRAM."
)
if (
uses_sm100_trtllm_nvfp4
and speculative_algorithm is not None
and speculative_algorithm not in supported_native_spec_algorithms
):
# Do not silently treat STANDALONE, FROZEN_KV_MTP, or a custom
# plugin algorithm as EAGLE. Their draft/cache-commit contracts may
# differ, and none currently has native-layout coverage here.
raise ValueError(
"SM100 native NVFP4 speculative decoding supports EAGLE, "
"EAGLE3, NEXTN, and breadth-1 NGRAM; got "
f"{speculative_algorithm}."
)
if (
uses_sm100_trtllm_nvfp4
and speculative_algorithm == "NGRAM"
and cfg.speculative_ngram_max_bfs_breadth != 1
):
# TRT-LLM MHA's target-verify metadata supports a linear chain
# only. NGRAM defaults to a breadth-10 tree, so reject that default
# explicitly rather than reaching the later generic paged-backend
# assertion with a misleading compatibility message.
raise ValueError(
"SM100 native NVFP4 NGRAM speculative decoding requires "
"--speculative-ngram-max-bfs-breadth=1 because trtllm_mha "
"supports linear target verification only; got "
f"{cfg.speculative_ngram_max_bfs_breadth}."
)
uses_draft_model = speculative_algorithm not in (None, "NGRAM")
if uses_sm100_trtllm_nvfp4 and uses_draft_model:
# A draft worker owns another physical KV pool. It cannot inherit a
# target-only hybrid pair: draft-extend would then select the
# prefill child even though the draft pool and its block scales use
# the native GenMHA layout. Give every draft phase one layout and
# one backend, including prefill-graph capture and multi-step
# decode. This also covers explicit hybrid target configurations,
# for which the model-default hook intentionally does not choose a
# draft backend.
draft_backend = cfg.speculative_draft_attention_backend
if draft_backend is None:
logger.warning(
"SM100 native NVFP4 speculative decoding uses trtllm_mha "
"for the draft worker."
)
declare_resolution(
server_args,
"_handle_kv4_compatibility",
speculative_draft_attention_backend="trtllm_mha",
)
elif draft_backend != "trtllm_mha":
raise ValueError(
"SM100 native NVFP4 speculative decoding requires "
"--speculative-draft-attention-backend=trtllm_mha so the "
"draft worker consumes its physical NVFP4 KV layout; got "
f"{draft_backend!r}."
)
if (
uses_sm100_mixed_nvfp4
and cfg.speculative_algorithm is not None
and cfg.speculative_attention_mode == "prefill"
):
# FlashInfer prefill reads a transient FP8 dequant workspace. Its
# host-built page layout cannot be refreshed inside a target-verify
# CUDA graph, whereas the decode child consumes the physical NVFP4
# cache directly for both eager and graph execution.
logger.warning(
"SM100 mixed NVFP4 speculative decoding routes target verify "
"to trtllm_mha; overriding --speculative-attention-mode=prefill "
"to decode."
)
declare_resolution(
server_args,
"_handle_kv4_compatibility",
speculative_attention_mode="decode",
)
if uses_sm100_trtllm_nvfp4 and cfg.disaggregation_mode != "null":
raise ValueError(
"SM100 native NVFP4 with trtllm_mha does not yet support PD "
"disaggregation because its physical block-scale layout is not "
"implemented by the KV transfer path."
)
if uses_sm100_trtllm_nvfp4 and (
cfg.enable_hierarchical_cache or cfg.enable_lmcache
):
raise ValueError(
"SM100 native NVFP4 with trtllm_mha does not yet support "
"hierarchical KV cache or LMCache because their host pools do "
"not preserve the physical block-scale layout."
)
if prefill_backend != decode_backend and prefill_backend != "fa4":
# NVFP4 with FP8 prefill is a supported mixed-storage recipe.
if not uses_mixed_nvfp4:
logger.warning(
f"Attention: Using KV4 with PREFILL = {prefill_backend} "
f"and DECODE = {decode_backend}. "
"Compatibility issues are unlikely, but may occur in rare "
"edge cases."
)
else:
if prefill_backend == "fa4":
if uses_mla: # FA4 + MLA
KV4_FA4_MLA_BACKEND_CHOICES = [
"flashinfer",
"trtllm_mla",
]
assert decode_backend in KV4_FA4_MLA_BACKEND_CHOICES, (
f"KV4 FA4 MLA expects decode_attention_backend to be one of "
f"{KV4_FA4_MLA_BACKEND_CHOICES}, but got {decode_backend}"
)
else: # FA4 + MHA
KV4_FA4_MHA_BACKEND_CHOICES = [
"triton",
"torch_native",
"flex_attention",
]
assert decode_backend in KV4_FA4_MHA_BACKEND_CHOICES, (
f"KV4 FA4 MHA expects decode_attention_backend to be one of "
f"{KV4_FA4_MHA_BACKEND_CHOICES}, but got {decode_backend}"
)
else:
if uses_mla: # !FA4 + MLA
KV4_ATTENTION_MLA_BACKEND_CHOICES = [
"flashinfer",
"trtllm_mla",
]
assert attention_backend in KV4_ATTENTION_MLA_BACKEND_CHOICES, (
f"KV4 MLA expects attention_backend to be one of "
f"{KV4_ATTENTION_MLA_BACKEND_CHOICES}, but got {attention_backend}"
)
else: # !FA4 + MHA
KV4_ATTENTION_MHA_BACKEND_CHOICES = [
"triton",
"torch_native",
"flex_attention",
"trtllm_mha",
]
assert attention_backend in KV4_ATTENTION_MHA_BACKEND_CHOICES, (
f"KV4 MHA expects attention_backend to be one of "
f"{KV4_ATTENTION_MHA_BACKEND_CHOICES}, but got {attention_backend}"
)
else:
raise RuntimeError("KV4 is not tested on non-CUDA platforms.")
def handle_prefill_only_disable_kv_cache(server_args: Any) -> None:
"""Validate --prefill-only-disable-kv-cache backend constraint.
Must run after _handle_attention_backend_compatibility() (which fills
the default attention_backend if unset) and _handle_multi_item_scoring()
(which may further mutate it). The assertion below guards against
accidental call-site reordering: if the resolved attention_backend is
still None, backends haven't settled yet and the resolved (prefill,
decode) pair would be a stale (None, None).
"""
cfg = resolving_view(server_args)
if not cfg.prefill_only_disable_kv_cache:
return
assert resolved_view(server_args).attention_backend is not None, (
"_handle_prefill_only_disable_kv_cache must run after "
"_handle_attention_backend_compatibility() so the prefill backend is resolved."
)
prefill_backend, _ = attention_backends_of(resolved_view(server_args))
if prefill_backend not in ("fa3", "fa4"):
raise ValueError(
"--prefill-only-disable-kv-cache currently requires the FA prefill backend "
f"(fa3/fa4), but got prefill backend {prefill_backend!r}. Other prefill-only "
"workloads and backends may be supported in a future change."
)
def handle_cache_compatibility(server_args: Any) -> None:
cfg = resolving_view(server_args)
if (
cfg.disaggregation_decode_retraction_backup == "host_pool"
and cfg.disaggregation_mode != "decode"
):
raise ValueError(
"--disaggregation-decode-retraction-backup=host_pool is only "
"supported on a PD decode server."
)
if cfg.disaggregation_decode_retraction_backup == "host_pool" and cfg.dcp_size > 1:
raise ValueError(
"--disaggregation-decode-retraction-backup=host_pool does not "
"support --dcp-size > 1."
)
if (
cfg.disaggregation_decode_retraction_backup == "host_pool"
and cfg.enable_priority_scheduling
and not cfg.disable_priority_preemption
):
raise ValueError(
"--disaggregation-decode-retraction-backup=host_pool requires "
"--disable-priority-preemption when priority scheduling is enabled."
)
if cfg.radix_eviction_policy == "tlru":
tlru_config = cfg.radix_eviction_policy_config or {}
threshold = tlru_config.get("threshold", 0)
next_prompt_estimate = tlru_config.get("next_prompt_estimate", 0)
if threshold < 0 or next_prompt_estimate < 0:
raise ValueError(
"--radix-eviction-policy tlru requires non-negative 'threshold' and "
"'next_prompt_estimate' in --radix-eviction-policy-config, got "
f"{threshold} and {next_prompt_estimate}."
)
if threshold <= next_prompt_estimate:
raise ValueError(
"--radix-eviction-policy tlru needs 'threshold' greater than "
f"'next_prompt_estimate' in --radix-eviction-policy-config, got "
f"{threshold} <= {next_prompt_estimate}; otherwise no tokens are "
"ever TEL-safe and T-LRU is exactly LRU."
)
if cfg.enable_hierarchical_cache and cfg.disable_radix_cache:
raise ValueError(
"The arguments enable-hierarchical-cache and disable-radix-cache are mutually exclusive "
"and cannot be used at the same time. Please use only one of them."
)
if cfg.disaggregation_decode_enable_offload_kvcache:
if cfg.disaggregation_mode != "decode":
raise ValueError(
"The argument disaggregation-decode-enable-offload-kvcache is only supported for decode side."
)
if cfg.hicache_storage_backend is None:
raise ValueError(
"The argument disaggregation-decode-enable-offload-kvcache is only supported when hicache-storage-backend is provided."
)
if cfg.disaggregation_decode_retraction_backup == "host_pool":
raise ValueError(
"The arguments disaggregation-decode-enable-offload-kvcache and "
"disaggregation-decode-retraction-backup=host_pool are mutually exclusive: "
"both build a decode host pool."
)
if cfg._swa_full_tokens_ratio_explicitly_set is None:
declare_resolution(
server_args,
"_handle_cache_compatibility",
_swa_full_tokens_ratio_explicitly_set=cfg.swa_full_tokens_ratio is not None,
)
# Validate the effective ratio: model branches may declare a reset
# (e.g. Step3p forces 1.0 under hierarchical cache) that supersedes
# the user input before it ever takes effect.
# `resolution_result`, not a view: a view answers `None` while nobody has
# claimed the field, and the value to range-check is the effective one.
if not (0 < resolution_result(server_args, "swa_full_tokens_ratio") <= 1.0):
raise ValueError("--swa-full-tokens-ratio should be in range (0, 1.0].")
prefix_tails = resolved_view(server_args).swa_prefix_tails
if prefix_tails is not None and prefix_tails < 0:
raise ValueError("--swa-prefix-tails should be a non-negative integer.")
if cfg.enable_lmcache:
if cfg.enable_hierarchical_cache:
raise ValueError(
"--enable-lmcache and --enable-hierarchical-cache are "
"mutually exclusive"
)
if cfg.enable_unified_cache_external_linker:
raise ValueError(
"--enable-lmcache and --enable-unified-cache-external-linker "
"are mutually exclusive"
)
if cfg.disable_radix_cache:
raise ValueError("--enable-lmcache requires radix cache to be enabled")
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
if SpeculativeAlgorithm.from_string(cfg.speculative_algorithm).is_speculative():
raise NotImplementedError(
"LMCacheUnifiedRadixCache does not yet support speculative decoding"
)
if attn_dp_enabled_of(cfg):
raise NotImplementedError(
"LMCacheUnifiedRadixCache does not yet support DP attention"
)
if cfg.dcp_size > 1:
raise NotImplementedError(
"--enable-lmcache with --dcp-size > 1 is not supported: "
"LMCache has no DCP-aware index translation"
)
if cfg.enable_streaming_session:
raise NotImplementedError(
"LMCacheUnifiedRadixCache does not yet support streaming sessions"
)
if cfg.hicache_host_memory_mode == "buffer_only":
raise ValueError(
"--hicache-host-memory-mode=buffer_only is a HiCache-only mode"
)
if cfg.disaggregation_mode != "null":
raise NotImplementedError(
"LMCacheUnifiedRadixCache currently supports colocated "
"prefill/decode scheduling only"
)
# Backends whose speculative verify id rails are translation-audited for
# the unified pool.
_SPEC_VERIFY_AUDITED_BACKENDS = frozenset(
{
"triton",
"trtllm_mla",
"cutedsl_mla",
"tokenspeed_mla",
"flashmla",
"flashinfer",
"fa3",
}
)
# The MHA backends that read through the KV-index translator on every path a
# draft forward takes. A fused draft region holds dense K/V rows, so a fused
# draft runs only on these.
TRANSLATED_MHA_RAILS = frozenset({"triton", "flashinfer", "fa3"})
def _assert_spec_verify_backends(
server_args: Any, *, algorithm: str, allowed: Optional[frozenset] = None
) -> None:
"""Refuse target backends whose verify id rails are not translation-audited.
Only the target's prefill/decode pair is checked here: fusion declines for
a draft off the translated rails, and a private-pool draft indexes its own
pool by virtual id."""
if allowed is None:
allowed = _SPEC_VERIFY_AUDITED_BACKENDS
dcp_note = ""
if resolving_view(server_args).dcp_size > 1:
# flashinfer's spec verify gathers its CSR args with no DCP read
# translation (`translate_dcp_read_ids`); the MLA verify family builds
# its DCP block table itself.
allowed = allowed - {"flashinfer"}
dcp_note = " (flashinfer is excluded under --dcp-size > 1)"
backends = set(attention_backends_of(resolved_view(server_args)))
backends.discard(None)
assert backends <= allowed, (
f"--enable-unified-memory + {algorithm} requires spec-verify-audited "
f"attention backends {sorted(allowed)}{dcp_note} for both prefill "
f"and decode; got {sorted(backends)}. Other backends do "
"not translate speculative verify indices to the unified "
"pool's physical ids yet."
)
def handle_unified_memory_pool(server_args: Any) -> None:
cfg = resolving_view(server_args)
if not cfg.enable_unified_memory:
return
if cfg.disaggregation_mode != "null":
# Constraints of the whole-envelope transfer; see the unified MHA and
# MLA pool get_contiguous_buf_infos implementations.
supported_backends = server_args._unified_memory_pd_transfer_backends()
assert cfg.disaggregation_transfer_backend in supported_backends, (
"--enable-unified-memory with PD disaggregation supports only these "
f"transfer backends: {', '.join(sorted(supported_backends))}; got "
f"{cfg.disaggregation_transfer_backend!r}."
)
assert cfg.pp_size == 1, (
"--enable-unified-memory with PD disaggregation does not support "
"pipeline parallelism (whole-envelope transfer has no per-layer "
"entries to subset)."
)
assert not (
cfg.disaggregation_transfer_backend == "mooncake"
and cfg.speculative_algorithm is not None
), (
"--enable-unified-memory with PD disaggregation does not support "
"speculative decoding with the Mooncake transfer backend."
)
assert not envs.SGLANG_DISABLE_LAZY_COMPACTION.get(), (
"--enable-unified-memory with PD disaggregation requires lazy "
"compaction; unset SGLANG_DISABLE_LAZY_COMPACTION."
)
assert not cfg.enable_hisparse, (
"--enable-unified-memory with PD disaggregation is not compatible "
"with --enable-hisparse: the decode-side HiSparse prealloc path "
"ships host/C4 rows straight from the allocator, bypassing the "
"virtual->physical translation the unified pool needs."
)
if cfg.disaggregation_decode_enable_offload_kvcache:
assert cfg.hicache_storage_backend == "file", (
"--enable-unified-memory with decode KV offload currently "
"supports --hicache-storage-backend=file only."
)
model_config = model_config_of(server_args)
assert not use_mla_backend(server_args), (
"--enable-unified-memory decode KV offload does not support "
"MLA models yet."
)
assert mambaish_config(model_config) is None, (
"--enable-unified-memory decode KV offload does not support "
"hybrid-Mamba models."
)
assert not model_config.is_hybrid_swa, (
"--enable-unified-memory decode KV offload does not support "
"hybrid-SWA H2D/D2H transfers yet."
)
if cfg.disaggregation_decode_retraction_backup == "host_pool":
model_config = model_config_of(server_args)
assert mambaish_config(model_config) is None, (
"--enable-unified-memory host-pool decode retraction does not "
"support hybrid-Mamba models."
)
assert cfg.speculative_algorithm in (
None,
"DSPARK",
"EAGLE",
"EAGLE3",
"DFLASH",
), (
"--enable-unified-memory only supports --speculative-algorithm "
"DSPARK, DFLASH, EAGLE and EAGLE3; other speculative algorithms are "
"not yet audited for the unified pool's virtual-to-physical loc "
"translation. Got "
f"--speculative-algorithm={cfg.speculative_algorithm!r}."
)
assert cfg.speculative_eagle_topk in (None, 1), (
"--enable-unified-memory supports a linear draft chain only "
"(--speculative-eagle-topk in {None, 1}); tree verify relocates "
"accepted tokens one at a time inside the target pool, which the "
"unified pool's page-granular move_kv_cache cannot express. Got "
f"--speculative-eagle-topk={cfg.speculative_eagle_topk!r}."
)
if cfg.speculative_algorithm in ("EAGLE", "EAGLE3"):
_mc = model_config_of(server_args)
assert _mc.is_hybrid_swa or mambaish_config(_mc) is not None, (
"--enable-unified-memory + EAGLE/EAGLE3 requires a unified "
"target (hybrid-SWA or a mamba hybrid): the draft's KV lives "
"fused inside the full-attention page envelope."
)
# The target verifies on its own pages: the MLA family on an MLA host.
eagle_allowed = (
_SPEC_VERIFY_AUDITED_BACKENDS
if use_mla_backend(server_args)
else TRANSLATED_MHA_RAILS
)
eagle_backends = set(attention_backends_of(resolved_view(server_args)))
assert (
None not in eagle_backends
and eagle_backends
and eagle_backends <= eagle_allowed
), (
"--enable-unified-memory + EAGLE/EAGLE3 requires the target on the "
f"spec-verify-audited attention backends {sorted(eagle_allowed)} "
f"(got {sorted(eagle_backends, key=str)})."
)
# An unset draft backend inherits the target's pair.
draft_backend = cfg.speculative_draft_attention_backend
draft_backends = {draft_backend} if draft_backend else eagle_backends
assert draft_backends <= TRANSLATED_MHA_RAILS, (
"--enable-unified-memory + EAGLE/EAGLE3 runs the draft on "
f"{sorted(draft_backends)}, but a fused draft is MHA-shaped and "
"reads its KV through the translated MHA rails "
f"{sorted(TRANSLATED_MHA_RAILS)}. "
"Set --speculative-draft-attention-backend to one of them."
)
if cfg.speculative_algorithm == "DSPARK":
_assert_spec_verify_backends(server_args, algorithm="DSPARK")
if cfg.speculative_algorithm == "DFLASH":
_assert_spec_verify_backends(
server_args,
algorithm="DFLASH",
allowed=TRANSLATED_MHA_RAILS,
)
assert not cfg.enable_two_batch_overlap, (
"--enable-unified-memory does not support --enable-two-batch-overlap: "
"the iteration's read tables are indexed by the whole batch's rows, and "
"a TBO child's readers do not offset into them."
)
assert not cfg.enable_lmcache, (
"--enable-unified-memory is not yet compatible with --enable-lmcache: "
"the LMCache offload path indexes the device buffers with the ids it "
"is handed, and under the unified pool those are VIRTUAL."
)
assert not cfg.enable_unified_cache_external_linker, (
"--enable-unified-memory does not support "
"--enable-unified-cache-external-linker: direct L3 transfers do not "
"preserve unified page-envelope indices and compaction lifetimes. "
"Use --enable-hierarchical-cache for supported L2/L3 transfers."
)
assert not (
cfg.speculative_algorithm in ("EAGLE", "EAGLE3")
and cfg.disaggregation_decode_retraction_backup == "host_pool"
), (
"--enable-unified-memory + EAGLE/EAGLE3 does not support "
"--disaggregation-decode-retraction-backup=host_pool: the backup builds "
"the draft's host pool off the draft's own device pool, which a fused "
"EAGLE draft does not have."
)
assert not (
cfg.speculative_algorithm in ("EAGLE", "EAGLE3")
and cfg.enable_hierarchical_cache
), (
"--enable-unified-memory + EAGLE/EAGLE3 does not support "
"--enable-hierarchical-cache: HiCache keeps the draft on a private "
"pool, and would pack an MTP head's KV into the target's "
"page-envelope host pool, which refuses per-layer draft loads."
)
if cfg.dcp_size > 1:
_validate_unified_memory_dcp(server_args)
# Prefill cuda-graph capture IS wired for the unified pool: the captured
# batch reads `out_cache_loc` out of the registry slot, which
# `populate_from_forward_batch` refills from the already-rebound (kernel-
# facing) loc before every replay, and the read tables are refilled
# out-of-graph from the live v2p.
#
# The FULL backend is the one exception, and not for a unified reason: its
# metadata path (`_init_full_cg_prefill_metadata`) exists only on the
# fa3/fa4 family. Any other backend lands in the decode-shaped
# `_apply_cuda_graph_metadata`, which has no EXTEND branch at all. Inkling
# declares FULL as a MODEL default, indistinguishable here from a flag the
# user typed, so warn and fall back rather than refuse to boot.
_cg_cfg = cfg.cuda_graph_config
if _cg_cfg is not None and _cg_cfg.prefill.backend == Backend.FULL:
full_cg_backends = {"fa3", "fa4"}
backends = set(attention_backends_of(resolved_view(server_args)))
backends.discard(None)
if not backends <= full_cg_backends:
_cg_cfg.prefill.backend = Backend.DISABLED
logger.warning(
"--enable-unified-memory: disabling the FULL prefill "
"cuda-graph backend. It builds its block table in "
"_init_full_cg_prefill_metadata, which only %s implement; the "
"resolved attention backends are %s. Decode capture and the "
"other prefill backends are unaffected.",
sorted(full_cg_backends),
sorted(backends),
)
# trtllm_mha refills its graph page table before replay from
# `cache_seqlens_int32`, which only the in-graph metadata kernel writes, so
# it uses the previous replay's lengths. Refuse until it uses the batch's.
_, decode_backend = attention_backends_of(resolved_view(server_args))
if decode_backend == "trtllm_mha":
assert _cg_cfg is None or _cg_cfg.decode.backend == Backend.DISABLED, (
"--enable-unified-memory does not yet support decode cuda graphs "
"with the trtllm_mha attention backend: its replay refills the page "
"table from the previous replay's sequence lengths. Pass "
"--disable-cuda-graph, or pick another decode attention backend "
"(fa3 / fa4 / flashinfer / triton)."
)
def _validate_unified_memory_dcp(server_args: Any) -> None:
"""Gate --enable-unified-memory + --dcp-size > 1 to the audited path.
Under DCP the unified allocator hands out a WIDENED virtual id space
(dcp_size logical ids per stored row) and every read index reaches
`translate_kv_loc*` already collapsed by a DCP index kernel. Only the
pieces below have been converted to that two-stage contract.
"""
assert use_mla_backend(server_args), (
"--enable-unified-memory with decode context parallelism "
"(--dcp-size > 1) supports MLA models only (e.g. kimi-linear): the "
"MHA unified pool has no DCP-aware masked write path "
"(UnifiedMHATokenToKVPool.set_kv_buffer asserts dcp_kv_mask is None)."
)
assert not model_config_of(server_args).is_hybrid_swa, (
"--enable-unified-memory with decode context parallelism "
"(--dcp-size > 1) does not support hybrid sliding-window models: "
"UnifiedSWATokenToKVPoolAllocator does not widen its virtual id "
"space, and the full->swa mapping is not DCP-sharded."
)
cfg = resolving_view(server_args)
assert cfg.disaggregation_mode == "null", (
"--enable-unified-memory with decode context parallelism "
"(--dcp-size > 1) does not support PD disaggregation: the transfer "
"ships whole page envelopes, which under DCP hold only this rank's "
"shard of each widened page. Rejected here rather than at the first KV "
"transfer, where translate_kv_indices_for_transfer would abort a "
"server that had already booted."
)
# The trtllm_mla family builds its DCP block table through the pool's v2p
# gather (create_mla_kv_page_table_for_dcp), so it speaks the same
# two-stage contract as flashinfer.
dcp_allowed = {"flashinfer", "trtllm_mla", "cutedsl_mla", "tokenspeed_mla"}
backends = set(attention_backends_of(resolved_view(server_args)))
backends.discard(None)
assert backends <= dcp_allowed, (
"--enable-unified-memory with decode context parallelism "
f"(--dcp-size > 1) requires {sorted(dcp_allowed)} for the "
f"full-attention layers; got {sorted(backends)}. The other paged MLA "
"backends build their block table from raw (widened) req_to_token "
"page ids and do not translate them through the unified pool's "
"virtual->physical page table."
)
def handle_page_major_kv_layout(server_args: Any):
# --enable-unified-memory implies --enable-page-major-kv-layout, so the
# unified pool goes through this one gate (declared before the guard).
cfg = resolving_view(server_args)
if cfg.enable_unified_memory:
declare_resolution(
server_args,
"_handle_page_major_kv_layout",
enable_page_major_kv_layout=True,
)
if not cfg.enable_page_major_kv_layout:
return
assert cfg.enable_unified_memory, (
"--enable-page-major-kv-layout without --enable-unified-memory is "
"temporarily unsupported: the strided MHA K/V views were removed "
"and the static-pool page-major layout awaits its per-layer-view "
"reimplementation. Run with --enable-unified-memory, or drop "
"--enable-page-major-kv-layout."
)
from sglang.srt.mem_cache.unified_memory_pool import (
unified_memory_supported_for_model,
)
model_config = model_config_of(server_args)
assert unified_memory_supported_for_model(
model_config, use_mla_backend=use_mla_backend(server_args)
), (
"--enable-unified-memory does not yet admit asymmetric K/V rows "
"(head_dim != v_head_dim); this model has "
f"head_dim={model_config.head_dim}, "
f"v_head_dim={model_config.v_head_dim}, "
f"swa_head_dim={model_config.swa_head_dim}, "
f"swa_v_head_dim={model_config.swa_v_head_dim}. The token-major "
"views can hold them, but the backends' write and read paths are "
"not audited for it; run this model without --enable-unified-memory."
)
# Allow-list. Every backend below reads through the translator, so what
# gates one is only whether its kernels address the per-layer views by
# their strides (the slot stride is the whole entry, not one row):
# * MLA models: the full paged MLA family, incl. flashmla (ps=64
# snap).
# * MHA/SWA models: fa3 / fa4 / flashinfer / trtllm_mha alongside
# Triton. fa4 is the fa3 class.
# Names are the RESOLVED ids from attention_backends_of.
if use_mla_backend(server_args):
allowed_full = {
"triton",
"fa3",
"trtllm_mla",
"flashinfer",
"cutedsl_mla",
"tokenspeed_mla",
"flashmla",
}
else:
allowed_full = {
"triton",
"fa3",
"fa4",
"flashinfer",
"trtllm_mha",
}
backends = set(attention_backends_of(resolved_view(server_args)))
backends.discard(None)
assert backends <= allowed_full, (
"--enable-page-major-kv-layout: the resolved attention backends "
f"{sorted(backends)} are not in the allowed set "
f"{sorted(allowed_full)} for this configuration (unified memory "
"allows the stride-aware per-layer-view families). Pass a "
"compatible --attention-backend."
)
# The Mamba/KDA state is stored in envelope-strided views; only
# stride-audited kernels may read it (Stage 4 audit, per slot):
# - decode: triton; flashinfer (recurrent_kda compiles the state slot
# stride as a free int64); helion (specializes KDA state strides 0-3
# and rejects a non-unit innermost stride); cutedsl (KDA fused sigmoid-
# gating update is stride-safe) on KDA-hybrid models only.
# - prefill: triton; flashkda (the wrapper gathers/scatters a contiguous
# per-slot copy); helion; cutedsl (kernel_h compiles h0/ht with dynamic
# int64 strides), with the same KDA-only caveat.
# - mamba (mamba2/short-conv state): triton only.
# use_mla_backend() distinguishes the KDA-hybrid family (K3/KimiLinear
# are MLA-hybrid) from GDN models (GQA-hybrid) for the KDA-only caveat.
decode_allowed = {"triton", "flashinfer"}
prefill_allowed = {"triton", "flashkda"}
if use_mla_backend(server_args):
decode_allowed.update({"cutedsl", "helion"})
prefill_allowed.update({"cutedsl", "helion"})
resolved_linear_decode = cfg.linear_attn_decode_backend or cfg.linear_attn_backend
resolved_linear_prefill = cfg.linear_attn_prefill_backend or cfg.linear_attn_backend
assert resolved_linear_decode in decode_allowed | {None}, (
"--enable-page-major-kv-layout: linear-attention DECODE backend must "
f"be one of {sorted(decode_allowed)} for the strided conv/SSM state; "
f"got {resolved_linear_decode!r}."
)
assert resolved_linear_prefill in prefill_allowed | {None}, (
"--enable-page-major-kv-layout: linear-attention PREFILL backend must "
f"be one of {sorted(prefill_allowed)} for the strided conv/SSM state; "
f"got {resolved_linear_prefill!r}."
)
assert cfg.mamba_backend in (None, "triton"), (
"--enable-page-major-kv-layout requires the Triton Mamba kernels for "
f"the strided conv/SSM state; got {cfg.mamba_backend!r}. Pass "
"--mamba-backend triton."
)
def validate_prefill_only_disable_kv_cache_args(server_args: Any):
"""Validate --prefill-only-disable-kv-cache flag/precondition constraints.
Backend resolution is checked separately by
_handle_prefill_only_disable_kv_cache after backends settle.
"""
cfg = resolving_view(server_args)
if not cfg.prefill_only_disable_kv_cache:
return
# This flag is intentionally scoped to embedding mode for now. Other
# prefill-only paths (for example scoring and MIS) can benefit from
# the same idea later, but some of them still stage K/V through the
# paged cache today.
if not cfg.is_embedding:
raise ValueError(
"--prefill-only-disable-kv-cache currently requires --is-embedding. "
"Other prefill-only workloads may be supported in a future change once "
"their attention paths stop reading or writing the paged KV cache."
)
if cfg.kv_cache_dtype in ("nvfp4", "fp4_mx_block16"):
raise ValueError(
"--prefill-only-disable-kv-cache does not currently support "
"--kv-cache-dtype=nvfp4 or --kv-cache-dtype=fp4_mx_block16 because "
"the FP4 pool uses a separate allocation path."
)
if cfg.kv_cache_dtype == "mxfp8":
raise ValueError(
"--prefill-only-disable-kv-cache does not currently support "
"--kv-cache-dtype=mxfp8 because the MXFP8 pool stores separate "
"scale-factor buffers."
)
# Structural preconditions for the FA backend's fa_skip_kv_cache path,
# which is the only embedding path that doesn't read or write the pool:
# - chunked_prefill_size == -1 keeps a request in a single forward,
# so K/V never has to be reused across prefill chunks.
# - disable_radix_cache stops the prefix cache from indexing pool
# slots that no longer hold real data.
if cfg.chunked_prefill_size != -1:
raise ValueError(
"--prefill-only-disable-kv-cache requires --chunked-prefill-size=-1 so the FA "
"backend takes the fa_skip_kv_cache path; otherwise the pool would be touched "
"between prefill chunks."
)
if not cfg.disable_radix_cache:
raise ValueError(
"--prefill-only-disable-kv-cache requires --disable-radix-cache because the "
"radix cache indexes KV pool slots that no longer hold real data."
)
# Context-parallel prefill writes K/V to the pool via set_kv_buffer.
# NoOpMHATokenToKVPool intentionally raises on writes, so the engine would
# boot fine but fail on the first request.
if resolved_view(server_args).attn_cp_size > 1:
raise ValueError(
"--prefill-only-disable-kv-cache is incompatible with --attn-cp-size > 1: "
"the context-parallel attention path writes K/V to the pool via set_kv_buffer, "
"which the no-op pool intentionally rejects."
)
if cfg.enable_prefill_cp:
raise ValueError(
"--prefill-only-disable-kv-cache is incompatible with "
"--enable-prefill-cp: the prefill-CP path stages K/V through "
"the paged cache, which the no-op pool does not support."
)
# HiSparse selects a different pool class (HiSparseDSATokenToKVPool /
# HiSparseTokenToKVPoolAllocator) that is not the no-op pool.
if cfg.enable_hisparse:
raise ValueError(
"--prefill-only-disable-kv-cache is incompatible with --enable-hisparse: "
"HiSparse uses a dedicated pool family that is not the no-op MHA pool."
)