[DSV4] Size compressed pools from one per-ratio table in DSV4PoolConfigurator (#41049)

This commit is contained in:
Liangsheng Yin
2026-09-24 15:19:02 -07:00
committed by GitHub
parent a1eb691ab5
commit 0c578d97fe
6 changed files with 279 additions and 486 deletions
@@ -713,9 +713,8 @@ class DSV4AttnMetadata:
assert dst_val is not None, f"{field_name=} {src_val=} {dst_val=}"
dst_val.copy_(src_val)
# These fields are safe to replace because captured kernels only need
# the current per-replay objects, or the field is produced inside the
# captured graph before the attention graph break consumes it.
# Safe to replace: captured kernels read only the per-replay objects, or
# the field is produced in-graph before the attention graph break reads it.
for field_name in reference_assign_fields:
setattr(self, field_name, getattr(other, field_name))
@@ -723,9 +722,8 @@ class DSV4AttnMetadata:
self, num_tokens: Optional[int] = None, low_ratio_buffers=None
) -> None:
assert self.page_table.dim() == 2
# CP pads causal metadata for per-rank partitioning, while cache-write
# locations remain one-per-logical-token. num_tokens tracks that unpadded
# length; legacy paths use the metadata length.
# CP pads causal metadata per rank, but cache-write locations stay one per
# logical token; num_tokens is that unpadded length.
if num_tokens is None:
num_tokens = self.seq_lens_casual.shape[0]
assert self.raw_out_loc.shape[0] == num_tokens, (
@@ -1148,7 +1146,6 @@ class DeepseekV4AttnBackend(
self.softmax_scale: float = head_dim**-0.5
self.head_dim_v: int = model_runner.model_config.v_head_dim
self.cuda_int32_kwargs = {"device": self.device, "dtype": torch.int32}
self.swa_page_size = 128
assert model_runner.page_size is not None
assert model_runner.req_to_token_pool is not None
self.page_size = model_runner.page_size
@@ -1156,6 +1153,8 @@ class DeepseekV4AttnBackend(
self.req_to_token_pool = model_runner.req_to_token_pool
self.token_to_kv_pool: DeepSeekV4TokenToKVPool = model_runner.token_to_kv_pool
# The C4 state ring is addressed per SWA page, so a page holds whole windows.
assert self.token_to_kv_pool.swa_page_size % SWA_WINDOW == 0
self.hisparse_coordinator = model_runner.hisparse_coordinator
self.req_to_token = model_runner.req_to_token_pool.req_to_token
# Nothing is built for a compress ratio outside the pool's set.
@@ -1207,11 +1206,8 @@ class DeepseekV4AttnBackend(
self.speculative_num_steps = speculative_num_steps
self.speculative_num_draft_tokens: int = get_spec().speculative_num_draft_tokens
if self.speculative_num_draft_tokens is not None:
# Persistent target-verify metadata buffers. Allocated here (not
# lazily) so they are ordinary tensors: the first touch of a lazy
# buffer would inherit the caller's context, and a creation inside
# an inference_mode forward would forbid the in-place updates the
# graph-capture path performs outside inference mode.
# Allocated eagerly: a lazy buffer first created inside inference_mode
# would forbid the in-place updates graph capture performs outside it.
num_reqs = self.req_to_token.shape[0]
self.extend_seq_lens_buffer = torch.full(
(num_reqs,),
@@ -1263,9 +1259,8 @@ class DeepseekV4AttnBackend(
"DSV4 ragged verify does not support online c128 MTP; "
"set SGLANG_RAGGED_VERIFY_MODE off or disable online compress."
)
# Layout invariants (verify_lens >= 1, total == sum) are enforced in
# RaggedVerifyLayout.__post_init__; don't re-check the device tensor
# here -- that would D2H-sync the host-free verify prep path.
# Invariants are checked in RaggedVerifyLayout.__post_init__; a device-side
# check here would D2H-sync the host-free verify prep path.
layout = layout.padded_to_bucket(padded_bs=bs)
return layout
@@ -1450,9 +1445,8 @@ class DeepseekV4AttnBackend(
else:
def create(compress_ratio: Literal[4, 128]):
# Online c128 uses a different planner that cannot be created in
# prefill cuda-graph mode. Keep c4 graph-friendly while matching
# c128's existing online path.
# Online c128's planner cannot be created in prefill cuda-graph
# mode, so it keeps the online path.
use_graph_plan = use_prefill_cuda_graph and not (
compress_ratio == 128 and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get()
)
@@ -2069,7 +2063,7 @@ class DeepseekV4AttnBackend(
)
# Upgrade Raw->Full so compress + core_attn + indexer materialization is
# recorded inside the cuda graph; already Full when PREP_IN_CUDA_GRAPH=0.
# recorded inside the cuda graph.
if isinstance(self.forward_metadata, DSV4RawVerifyMetadata):
self.forward_metadata = self.make_forward_metadata_from_raw_verify(
raw_metadata=self.forward_metadata,
@@ -2480,13 +2474,12 @@ class DeepseekV4AttnBackend(
logical_forward_mode = _get_logical_forward_mode(forward_batch)
req_pool_indices = forward_batch.req_pool_indices
seq_lens = forward_batch.seq_lens.to(torch.int32)
# Regular prefill batches already carry scheduler-maintained CPU lengths.
# Keep using those when present; needs_cpu_seq_lens only controls whether
# speculative overlap must publish a new GPU-to-CPU mirror each step.
# Prefill batches carry scheduler-maintained CPU lengths; needs_cpu_seq_lens
# only controls whether spec overlap publishes a GPU-to-CPU mirror each step.
seq_lens_cpu = forward_batch.seq_lens_cpu
assert self.req_to_token_pool.req_to_token is self.req_to_token
assert self.swa_page_size % SWA_WINDOW == 0 and self.page_size % 128 == 0
assert self.page_size % 128 == 0
if max_seq_len_override is None:
max_seq_len_override = forward_batch.max_seq_len_override
if max_seq_len_override is not None:
@@ -2635,9 +2628,8 @@ class DeepseekV4AttnBackend(
*,
static_forward_batch: Optional[ForwardBatch] = None,
) -> None:
# Build graph-compatible metadata against the padded static batch. The
# batch still carries live seq/extend lens, so the online c128 prefill
# plan remains batch-specific without constructing a second metadata set.
# The padded static batch still carries live seq/extend lens, so the
# online c128 prefill plan stays batch-specific.
metadata_batch = (
static_forward_batch if static_forward_batch is not None else forward_batch
)
@@ -2727,8 +2719,8 @@ class DeepseekV4AttnBackend(
if 2 in core.low_ratios:
core.c2_flashmla_metadata = _create_flashmla_metadata()
# PREP_IN_CUDA_GRAPH=True: warmup upgraded raw->full on the host;
# restore raw so capture re-runs the upgrade inside the graph.
# Warmup upgraded raw->full on the host; restore raw so capture
# re-runs the upgrade inside the graph.
current_raw = getattr(self, "_current_capture_raw", None)
if current_raw is not None:
self.forward_metadata = current_raw
@@ -3870,10 +3862,8 @@ class DeepseekV4AttnBackend(
flash_mla_with_kvcache_sm120,
)
# The pad to 64 heads only serves the decode kernel's h_q
# specialization; the prefill kernel takes arbitrary h_q, so
# drop it instead of attending on garbage heads (4x the work
# at attn-TP 4).
# The 64-head pad only serves the decode kernel's h_q
# specialization; the prefill kernel takes any h_q, so drop it.
real_heads = layer.tp_q_head_num
if q.shape[0] > SM120_DECODE_MAX_TOKENS:
if q.shape[-2] > real_heads:
@@ -3940,14 +3930,8 @@ class DeepseekV4AttnBackend(
core_attn_metadata: DSV4AttnMetadata,
attn_sink: torch.Tensor,
) -> torch.Tensor:
"""Unified prefill via flash_mla_sparse_fwd. Replaces the
flash_mla_with_kvcache call on the extend path. Per request,
positionally gathers the SWA window (always) and the compressed
cache (c4/c128) into a flat bf16 workspace, then lets
flash_mla_sparse_fwd consume the workspace via per-query rebased
indices. Chunk-invariant scaffolding lives in
``self.forward_metadata.sparse_prefill_cache``.
"""
"""Gathers each request's SWA window and c4/c128 cache into a flat bf16
workspace that flash_mla_sparse_fwd reads through per-query rebased indices."""
if _is_xpu:
from sgl_kernel import flash_mla_sparse_fwd
else:
@@ -4077,14 +4061,8 @@ class DeepseekV4AttnBackend(
core_attn_metadata: DSV4AttnMetadata,
attn_sink: torch.Tensor,
) -> torch.Tensor:
"""Experimental DeepSeek-V4 sparse prefill path using Q8KV8 kernels.
This mirrors ``_forward_prefill_sparse``'s cache/index construction, but
writes the gathered KV workspace as FP8 and calls the SM90 Q8KV8 sparse
prefill kernel. The path is selected by ``--dsv4-prefill-backend
flashmla_sparse_q8``; ``SGLANG_DSV4_Q8KV8_PREFILL`` remains as a debug
override for focused runtime validation.
"""
"""_forward_prefill_sparse with an FP8 workspace and the SM90 Q8KV8 kernel;
SGLANG_DSV4_Q8KV8_PREFILL is a debug override on top of the CLI selection."""
from sglang.kernels.ops.attention.sparse_mla_q8kv8_prefill_sm90 import (
sparse_mla_q8kv8_prefill_fwd,
@@ -4270,8 +4248,6 @@ class DeepseekV4AttnBackend(
num_groups: Optional[int] = None,
dspark_swa_buffers: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
) -> DSV4AttnMetadata:
assert self.swa_page_size == SWA_WINDOW
small_metadata = (
not is_prefill
and seq_lens_casual.is_cuda
@@ -90,10 +90,6 @@ def _create_dummy_paged_compress_data(compress_ratio: int):
@dataclass
class UnifiedKvMetadata:
"""
unified-kv per-forward metadata
"""
# SWA ring write target (req_slot*ring + pos%ring)
swa_loc: Optional[torch.Tensor] = None
@@ -119,10 +115,8 @@ class UnifiedKvMetadata:
pf_cu_q: Optional[torch.Tensor] = None
pf_final_pos: Optional[torch.Tensor] = None
# Per-token req-slot map used by the SWA ring store, precomputed once per
# step so the forward store does not recompute a repeat_interleave per layer.
# Read by the target-verify store (num_draft*bs tokens); for plain decode it
# equals req_pool_indices and is unused (the decode store reads that live).
# Per-token req-slot map for the target-verify SWA ring store (num_draft*bs
# tokens); unused by plain decode, whose store reads req_pool_indices live.
verify_store_state_slot: Optional[torch.Tensor] = None
# SWA-page-offset compressed-store locations (= c*_out_loc + unified_swa_pages),
@@ -441,9 +435,8 @@ class DSV4Metadata:
c4_compress_metadata: Optional[FusedCompressMetadata] = None
c128_compress_metadata: Optional[FusedCompressMetadata] = None
# FP4 indexer buffers that captured kernels bind by address. Deliberately
# absent from copy_: the addresses must stay pinned across replays, and the
# workspace builders refresh their contents instead.
# Captured kernels bind these FP4 indexer buffers by address; absent from copy_
# so addresses stay pinned across replays (the builders refresh contents).
fp4_decode_workspace: Optional[FP4DecodeWorkspace] = field(default=None, repr=False)
fp4_prefill_workspace: Optional[FP4PrefillWorkspace] = field(
default=None, repr=False
@@ -563,12 +556,8 @@ class _GraphBucket(enum.Enum):
class DeepseekV4HipRadixBackend(
AttentionBackend, C4IndexerBackendMixin, CompressorBackendMixin
):
# DSV4 TBO runs ONLY in eager prefill (prefill cuda-graph is disabled);
# decode/target-verify graphs are non-TBO (primary backend only). So the TBO
# child backends must not be driven through cuda-graph capture/replay — doing
# so rebuilds this backend's compressor/indexer metadata per replay step on
# both children and leaks ROCm HSA resources (HSA_STATUS_ERROR_OUT_OF_RESOURCES).
# TboAttnBackend reads this to skip children in the *_graph paths only.
# DSV4 TBO runs only in eager prefill; driving TBO children through graph
# capture/replay leaks ROCm HSA resources (HSA_STATUS_ERROR_OUT_OF_RESOURCES).
tbo_supports_cuda_graph = False
supports_ragged_verify_graph: bool = True
use_captured_forward_metadata_for_breakable_cuda_graph: bool = True
@@ -592,7 +581,6 @@ class DeepseekV4HipRadixBackend(
self.softmax_scale: float = head_dim**-0.5
self.head_dim_v: int = model_runner.model_config.v_head_dim
self.cuda_int32_kwargs = {"device": self.device, "dtype": torch.int32}
self.swa_page_size = 128
assert model_runner.page_size is not None
assert model_runner.req_to_token_pool is not None
self.page_size = model_runner.page_size
@@ -600,6 +588,8 @@ class DeepseekV4HipRadixBackend(
self.req_to_token_pool = model_runner.req_to_token_pool
self.token_to_kv_pool: DeepSeekV4TokenToKVPool = model_runner.token_to_kv_pool
# The C4 state ring is addressed per SWA page, so a page holds whole windows.
assert self.token_to_kv_pool.swa_page_size % SWA_WINDOW == 0
self.hisparse_coordinator = model_runner.hisparse_coordinator
self.req_to_token = model_runner.req_to_token_pool.req_to_token
self.MAX_SEQ_LEN_FOR_CAPTURE = self.req_to_token.shape[1]
@@ -620,16 +610,14 @@ class DeepseekV4HipRadixBackend(
self.is_dspark = model_runner.spec_algorithm.is_dspark()
self.is_dspark_draft = self.is_draft_worker and self.is_dspark
# Draft layers are all COMPRESS_RATIO_NEXTN_LAYER (0), so the draft pool
# has neither compressed kv nor an indexer pool. Settled here rather than
# per forward: it cannot change after construction.
# has neither compressed kv nor an indexer pool.
self.need_compress = not self.is_draft_worker
self.target_verify_num_draft_tokens = self.speculative_num_draft_tokens
if self.is_dspark_draft:
assert self.speculative_num_draft_tokens is not None
assert self.speculative_num_draft_tokens > 1
# DSpark draft workers verify gamma rows. The server arg keeps the
# CUDA-side convention gamma + 1, so use an explicit effective value
# instead of mutating speculative_num_draft_tokens in place.
# DSpark draft workers verify gamma rows; the server arg keeps the
# CUDA-side convention gamma + 1.
self.target_verify_num_draft_tokens = self.speculative_num_draft_tokens - 1
# Past MAX_FUSED_ROWS the fp4 schedule falls back to AITER's preamble,
# which frees the scratch its kernels read -- not capture-safe.
@@ -701,8 +689,7 @@ class DeepseekV4HipRadixBackend(
)
# extend_start_loc and the CPU mirrors below only feed the torch
# fallback; the triton kernel cumsums extend_seq_lens on device, so
# every caller can share it instead of dropping to a per-request loop.
# fallback; the triton kernel cumsums extend_seq_lens on device.
_expanded = ExpandPrefillCausally.execute(
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
@@ -742,10 +729,8 @@ class DeepseekV4HipRadixBackend(
exact_num_tokens=exact_num_tokens or host_proves_exact_num_tokens,
)
if attach_decode_streams:
# Target-verify runs through the unified_kv DECODE kernel, so build
# per-token decode streams here. req_pool_indices_repeated is the
# per-token (num_draft*bs -> bs) req-slot map produced by the prefill
# expansion above.
# Verify runs the unified_kv DECODE kernel; req_pool_indices_repeated
# is the per-token (num_draft*bs -> bs) req-slot map.
self._attach_unified_kv_decode_streams(
core_attn_metadata,
req_pool_indices_repeated,
@@ -811,12 +796,9 @@ class DeepseekV4HipRadixBackend(
seq_lens_cpu: Optional[List[int]] = None,
ragged_layout=None,
) -> Union[DSV4Metadata, DSV4RawVerifyMetadata]:
# DSPARK verifies a uniform num_draft block, exactly what
# make_forward_metadata_from_raw_verify expands, so the build can be
# deferred into the graph. Graph path only: the upgrade sizes its page
# table by MAX_SEQ_LEN_FOR_CAPTURE, far wider than the live max_seq_len
# an eager caller passes. EAGLE and ragged layouts stay eager -- no raw
# expansion, and EAGLE's fixed-tier plan trips planner invariants.
# Only DSPARK's uniform draft block defers into the graph; EAGLE's fixed-tier
# plan trips planner invariants. Graph path only: the upgrade sizes page_table
# by MAX_SEQ_LEN_FOR_CAPTURE, wider than an eager caller's max_seq_len.
if (
use_prefill_cuda_graph
and self.is_dspark
@@ -869,10 +851,8 @@ class DeepseekV4HipRadixBackend(
)
extend_seq_lens = verify_lens_dev
seq_lens = seq_lens + verify_lens_dev.to(seq_lens.dtype)
# Total verify tokens to expand. For the graph path the padded layout
# sets total_verify_tokens == graph_num_tokens (tier); the eager path
# resolves a device-assembled layout whose total_verify_tokens is None,
# so fall back to sum(verify_lens) (== real total; padded == tier).
# Graph path: total_verify_tokens is the padded tier; an eager
# device-assembled layout leaves it None, so use sum(verify_lens).
num_tokens = ragged_layout.total_verify_tokens
if num_tokens is None:
num_tokens = int(verify_lens_dev.sum().item())
@@ -1120,14 +1100,12 @@ class DeepseekV4HipRadixBackend(
)
if upgraded_verify:
# The out-graph refresh saw raw metadata and skipped. Without this
# the logits kernel builds its own schedule -- the variant that
# frees the scratch it reads, which every replay would re-read.
# The out-graph refresh saw raw metadata and skipped; otherwise the
# logits kernel builds a schedule that frees scratch replays re-read.
self._refresh_fp4_prefill_workspace(forward_batch)
# Decode's schedule builder is capture-safe because the workspace pins
# the scratch it reads, so it can stay next to the metadata it consumes.
# Prefill/target-verify cannot; see _refresh_fp4_prefill_workspace.
# Decode's schedule builder is capture-safe (the workspace pins its scratch);
# prefill/target-verify's is not, see _refresh_fp4_prefill_workspace.
if self._fp4_workspaces_enabled(metadata) and (
forward_batch.forward_mode.is_decode()
):
@@ -1144,9 +1122,8 @@ class DeepseekV4HipRadixBackend(
def _fp4_workspaces_enabled(self, metadata) -> bool:
return (
self.enable_deepseek_v4_fp4_indexer
# Draft-step backends drive the NextN layer, which is built with
# compress_ratio_override=0 and so owns no C4 indexer. Their
# workspaces would be built, scheduled, and never read.
# Draft-step backends drive the NextN layer, built with
# compress_ratio_override=0, so it owns no C4 indexer.
and self.speculative_num_steps == 0
and isinstance(metadata, DSV4Metadata)
and metadata.indexer_metadata is not None
@@ -1154,13 +1131,8 @@ class DeepseekV4HipRadixBackend(
)
def _refresh_fp4_prefill_workspace(self, forward_batch: ForwardBatch) -> None:
"""Rebuild the FP4 prefill schedule outside CUDA-graph capture.
AITER's prefill scheduler frees the scratch that its schedule kernel
reads, so recording the build into a graph would leave every replay
reading recycled graph-pool memory. Only the pinned buffers it fills
(cta_info / logits / guarded page table) may be read from the graph.
"""
"""Must run outside graph capture: AITER's scheduler frees the scratch its
kernel reads, so only the pinned buffers it fills may be read from a graph."""
metadata = self.forward_metadata
if not self._fp4_workspaces_enabled(metadata):
return
@@ -1337,7 +1309,7 @@ class DeepseekV4HipRadixBackend(
seq_lens_cpu = forward_batch.seq_lens_cpu
assert self.req_to_token_pool.req_to_token is self.req_to_token
assert self.swa_page_size % SWA_WINDOW == 0 and self.page_size % 128 == 0
assert self.page_size % 128 == 0
assert seq_lens_cpu is not None
max_seq_len = (
max_seq_len_override
@@ -1499,7 +1471,7 @@ class DeepseekV4HipRadixBackend(
core.c4_flashmla_metadata = _create_flashmla_metadata()
core.c128_flashmla_metadata = _create_flashmla_metadata()
# PREP_IN_CUDA_GRAPH=True: warmup upgraded raw->full on the host;
# Warmup upgraded raw->full on the host;
# restore raw so capture re-runs the upgrade inside the graph.
current_raw = getattr(self, "_current_capture_raw", None)
if current_raw is not None:
@@ -1532,8 +1504,7 @@ class DeepseekV4HipRadixBackend(
hca_page_indices = core.c128_page_indices
c4_page_indices = core.c4_sparse_page_indices
# A draft pool carries no c4/c128 metadata, but its verify store still
# reads the swa half below, so build these with an empty compressed tail
# rather than skipping the call.
# reads the swa half below, so build these with an empty compressed tail.
if hca_len is None:
hca_len = torch.zeros_like(swa_len)
if csa_len is None:
@@ -1596,9 +1567,8 @@ class DeepseekV4HipRadixBackend(
req_slot * pool.unified_swa_ring_size
+ core.positions_casual.to(torch.int64) % pool.unified_swa_ring_size
).to(torch.int32)
# Per-token req-slot map for the SWA ring store, read directly by the
# forward store (target-verify) instead of recomputing a repeat_interleave
# per layer. Harmless for plain decode (its store reads req_pool_indices).
# Per-token req-slot map for the target-verify SWA ring store; plain
# decode's store reads req_pool_indices instead.
core.unified.verify_store_state_slot = state_slot
def _attach_unified_kv_prefill_meta(
@@ -1673,14 +1643,8 @@ class DeepseekV4HipRadixBackend(
q_rope: Optional[torch.Tensor] = None,
k_rope: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""unified_kv paged-attention path over the unified_kv pool.
``q_rope`` is what tells the two layouts apart: present means ``q`` is a
packed fp8 row and the pool is the two-pool fp8 one, so decode goes to
the asm reader; absent means both are plain bf16 and it goes to Triton.
Prefill needs ``k_rope`` alongside it, because there the current chunk is
a KV source of its own and not just something to store.
"""
"""q_rope present: packed fp8 q over the two-pool fp8 layout (asm decode;
prefill also needs k_rope). Absent: plain bf16 q and pool (Triton)."""
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime
pool = self.token_to_kv_pool
@@ -1706,18 +1670,14 @@ class DeepseekV4HipRadixBackend(
is_decode = forward_batch.forward_mode.is_decode_or_idle() or verify_as_decode
if is_decode:
if verify_as_decode:
# Per-token (num_draft*bs -> bs) req-slot map, precomputed once
# per step in _attach_unified_kv_decode_streams. Writing every
# draft token's K into the ring is safe: spec_extra room prevents
# clobbering the window history same-step tokens still read.
# Writing every draft token's K into the ring is safe: spec_extra
# room prevents clobbering the history same-step tokens still read.
state_slot = core_attn_metadata.unified.verify_store_state_slot[:T]
else:
state_slot = forward_batch.req_pool_indices[:T]
if save_kv_cache:
# Only verify reaches this under fp8 -- plain decode's rows are
# written by the fused kernel itself, which leaves kv None. The
# pair arrives already packed, so this is the same scatter with
# a second pool hanging off it.
# Only verify reaches this under fp8; plain decode's rows are
# written by the fused kernel itself, which leaves kv None.
runtime.store_swa_into_unified(
kv=kv,
state_slot=state_slot,
@@ -1752,10 +1712,8 @@ class DeepseekV4HipRadixBackend(
else:
raise ValueError(f"bad compress_ratio {compress_ratio}")
if q_rope is not None:
# softmax_scale is not passed on: the asm kernel hardcodes
# 1/sqrt(512), which is what self.softmax_scale already is for
# V4's head_dim=512. The other readers here take it explicitly,
# so a head_dim change would leave only this one mis-scaled.
# The asm kernel hardcodes softmax scale 1/sqrt(512) (V4 head_dim)
# and takes no softmax_scale, unlike the other readers here.
assert self.softmax_scale == 512**-0.5, (
"the v4 nm asm kernel hardcodes 1/sqrt(512), this backend is "
f"at {self.softmax_scale}"
@@ -1812,20 +1770,12 @@ class DeepseekV4HipRadixBackend(
cu_q = core_attn_metadata.unified.pf_cu_q
final_pos = core_attn_metadata.unified.pf_final_pos
# DSA CP (round-robin/interleave): unified_pf_* are built over the GLOBAL
# token layout, but under CP each rank owns only 1/cp_size of the queries
# (q/positions are local) while kv was all-gathered to the full sequence.
# Slice the per-query fields to this rank's tokens so their length matches
# the local query count T; values stay global so each local query still
# attends over the full all-gathered KV.
# DSA CP: unified_pf_* cover the GLOBAL token layout while q is local and
# kv all-gathered; slice per-query fields to this rank, values stay global.
from sglang.srt.layers.attention.dsa.utils import (
is_dsa_prefill_cp_round_robin_split,
)
# NOTE (AMD/HIP only): this whole DSA-CP prefill handling lives in the
# HIP backend (DeepseekV4HipRadixBackend, selected only when is_hip()).
# The NVIDIA path uses DeepseekV4AttnBackend and never reaches here, so
# these CP changes do not affect B200/H200 execution.
_cp_size = get_parallel().attn_cp_size
_cp_active = (
_cp_size > 1
@@ -1842,15 +1792,11 @@ class DeepseekV4HipRadixBackend(
chunk_start = chunk_start[_sl].contiguous()
cu_q = cu_q[_sl].contiguous()
final_pos = final_pos[_sl].contiguous()
# positions for the local queries are this rank's round-robin global
# positions {r, r+cp, r+2cp, ...}; forward_batch.positions is the full
# (padded) global layout, so slice it the same way instead of taking
# the first T entries (which would be the wrong, sequential 0..T-1).
# Local queries sit at this rank's round-robin global positions
# {r, r+cp, r+2cp, ...}, not the first T entries of the global layout.
positions = forward_batch.positions.to(torch.int64)[_sl].contiguous()
# The SWA ring must hold the FULL window on EVERY rank (decode and
# later chunks read this rank's ring). kv was all-gathered to the full
# sequence, so write the full kv with full global positions/state_slot
# instead of only this rank's 1/cp_size tokens.
# Every rank's SWA ring must hold the full window (decode and later
# chunks read it), so the store uses the full all-gathered kv.
positions_full = forward_batch.positions.to(torch.int64)[
: state_slot_full.shape[0]
].contiguous()
@@ -1877,10 +1823,8 @@ class DeepseekV4HipRadixBackend(
"fp8 prefill needs the extend rope half beside the packed nope; "
"q_rope came through but k_rope did not"
)
# No empty-segment mask on the result, unlike decode: this kernel
# returns zeros for a token with neither region where the asm decode
# reader leaves the row NaN. Chunk 0 tokens have an empty prefix and
# a non-empty extend, which both readers handle.
# No empty-segment mask, unlike decode: this kernel returns zeros for
# a token with neither region where the asm decode reader leaves NaN.
o = runtime.prefill_fp8_2buff(
q=q,
q_rope=q_rope,
@@ -1912,9 +1856,6 @@ class DeepseekV4HipRadixBackend(
# write this chunk's SWA K into the ring for future chunks / decode
# only the final-window tokens per request
if save_kv_cache:
# Under CP, write the FULL all-gathered window so every rank's ring is
# complete (decode / later chunks read the local ring). Without CP this
# is just the local kv + local metadata as before.
_ring_state_slot = state_slot_full if _cp_active else state_slot
_ring_final_pos = final_pos_full if _cp_active else final_pos
_ring_positions = positions_full if _cp_active else positions
@@ -2148,8 +2089,6 @@ class DeepseekV4HipRadixBackend(
need_compress: bool = True,
is_prefill: bool = False,
) -> DSV4AttnMetadata:
assert self.swa_page_size == SWA_WINDOW
seq_lens_casual = seq_lens_casual.to(torch.int32)
swa_page_indices = self.get_swa_page_indices(
@@ -65,6 +65,42 @@ def get_compress_state_ring_size(compress_ratio: int, num_draft_tokens: int = 0)
return 8 if compress_ratio == 4 else 128
def resolve_compress_state_ring_size(compress_ratio: int) -> int:
"""Ring size under the published spec config. The pool allocates by this
and the configurator budgets by it, so both must call the same function."""
spec = get_spec()
num_draft_tokens = (
0
if spec.speculative_algorithm is None
else spec.speculative_num_draft_tokens or 0
)
return get_compress_state_ring_size(compress_ratio, num_draft_tokens)
def collect_sources_by_ratio(
compression_ratios: Sequence[int], kv_source_layers: Sequence[int], stage: range
) -> dict[int, List[int]]:
"""Layers owning compressed storage: all of ratios 4/128, kv_sources of 1/2."""
for idx in stage:
ratio = compression_ratios[idx]
if ratio not in (0, 1, 2, 4, 128):
raise ValueError(f"Unsupported compression ratio: {ratio}")
sources_by_ratio: dict[int, List[int]] = {}
for ratio in (4, 128, 1, 2):
if ratio in (1, 2):
layers = [
l
for l in kv_source_layers
if l in stage and compression_ratios[l] == ratio
]
else:
layers = [l for l in stage if compression_ratios[l] == ratio]
if layers:
sources_by_ratio[ratio] = layers
return sources_by_ratio
def get_compress_state_write_pad(compress_ratio: int, ring_size: int) -> int:
"""Largest draft-token count this ring can serve; mirrors `mtp_pad` in
`c_plan.cuh`, where the bound is derived."""
@@ -79,11 +115,8 @@ def get_swa_ring_size(sliding_window: int, is_speculative: bool = False) -> int:
def resolve_unified_kv_fp8(unified_fp8: Optional[bool] = None) -> bool:
"""Per-pool fp8 layout. None follows SGLANG_DSV4_UNIFIED_KV_FP8.
A caller may pass False so this pool keeps the bf16 ring while the env
stays on (target fused-Q still keys off the global switch).
"""
"""Per-pool fp8 layout; None follows SGLANG_DSV4_UNIFIED_KV_FP8. False keeps this
pool bf16 while target fused-Q still keys off the global switch."""
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
is_unified_kv_fp8,
)
@@ -309,11 +342,8 @@ class DeepSeekV4SingleKVPool(KVCache):
class DeepSeekV4UniformFP8KVPool(DeepSeekV4SingleKVPool):
"""Uniform 512-dim FP8 (e4m3) variant of the DSv4 single-KV pool.
Each token is 448 NoPE + 64 RoPE contiguous e4m3 values without in-cache
scales or per-page padding. The backend supplies the dequant scale.
"""
"""Each token is 448 NoPE + 64 RoPE contiguous e4m3 values, without in-cache
scales or per-page padding; the backend supplies the dequant scale."""
def get_bytes_per_token(self) -> int:
return self.qk_nope_head_dim + self.qk_rope_head_dim
@@ -355,10 +385,8 @@ class DeepSeekV4UniformFP8KVPool(DeepSeekV4SingleKVPool):
cache_k: torch.Tensor,
freqs_cis: Optional[torch.Tensor] = None,
) -> None:
"""Store normed/roped rows as e4m3 with the backend's fixed unit scale.
uint8 views work around index_put not supporting FP8 dtypes.
"""
"""Store normed/roped rows as e4m3 with the backend's fixed unit scale;
uint8 views work around index_put not supporting FP8 dtypes."""
assert freqs_cis is None, "the uniform-FP8 pool takes finished (rotated) rows"
assert cache_k.dim() == 2 and cache_k.shape[1] == self.kv_cache_total_dim
@@ -572,11 +600,8 @@ class DeepSeekV4IndexerPool(KVCache):
return self.index_k_with_scale_buffer[layer_id]
def contiguous_page_row_buffers(self) -> List[torch.Tensor]:
"""Every indexer buffer as 2D page rows, for PD and HiCache transfer.
FP8 keeps key and scale fused in one buffer per layer; the FP4 layout
stores payload and scale separately, so it yields two buffers per layer.
"""
"""Every indexer buffer as 2D page rows for PD and HiCache: one fused key+scale
buffer per layer under FP8, separate payload and scale buffers under FP4."""
if self.index_k_with_scale_buffer is not None:
return self.index_k_with_scale_buffer
return [
@@ -744,9 +769,7 @@ class DeepSeekV4UnifiedKVPool:
unified_kv[L]: ``[swa_pages + padded_compress_rows, head_dim]`` bf16
Layout (fp8, ``SGLANG_DSV4_UNIFIED_KV_FP8``) -- two parallel pools with the
same row count, so a row index means the same thing in both. Named after the
accessors, which under fp8 each return one half -- ``get_unified_kv`` the
nope, ``get_unified_kv_rope`` the rope:
same row count, so a row index addresses both:
unified_kv[L] (nope): ``[rows, 512]`` fp8, see DSV4_FP8_NOPE_ROW_BYTES
unified_kv_rope[L] (rope): ``[rows, qk_rope_head_dim]`` bf16, never quantized
@@ -835,9 +858,7 @@ class DeepSeekV4UnifiedKVPool:
def get_buf_infos(self) -> Tuple[List[int], List[int], List[int]]:
if self.fp8:
# same single-pool assumption as the outer get_contiguous_buf_infos:
# one pointer and one row size per layer describes the nope pool only,
# so whoever picks this up next would move half a row and not notice.
# One pointer and row size per layer would describe only the nope pool.
# TODO(danli103): report both pools once a consumer needs them.
raise NotImplementedError(
"get_buf_infos describes one pool per layer; the fp8 rope pool "
@@ -959,9 +980,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
c128_state_pool_size = max(
c128_state_pool_size, self.num_req_slots * c128_ring_size
)
# Only the ratios the model has anywhere get a pool config: the backend, PD
# state transfer and HiCache read the registries as "the ratios this model
# has", and a PP stage missing one keeps its empty pool so the PD wire aligns.
# Consumers read the registries as "the ratios this model has"; a PP stage
# missing one keeps its empty pool so the PD wire aligns.
model_ratios = set(compression_ratios)
self.compressed_pool_configs = {
ratio: config
@@ -1007,10 +1027,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
self.swa_size = swa_size
self.swa_page_size = swa_page_size
# The allocator and compress state keep 256-token logical pages, while
# FlashInfer's SM120 DSV4 kernel consumes 64-token physical pages. Storing
# SWA KV in that layout removes the per-layer 256 -> 64 page split; the
# allocator's flat token indices stay valid.
# The allocator keeps 256-token logical pages; FlashInfer's SM120 DSV4 kernel
# reads 64-token physical pages, and flat token indices are valid in both.
swa_kv_page_size = (
64
if get_platform().is_sm120 and envs.SGLANG_OPT_SM120_DIRECT_SWA_KV.get()
@@ -1127,7 +1145,11 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
)
self.full_size = full_size
self.kv_source_layers = list(kv_source_layers)
self.sources_by_ratio = self._collect_sources_by_ratio()
self.sources_by_ratio = collect_sources_by_ratio(
self.compression_ratios,
self.kv_source_layers,
range(self._stage_start, self._stage_end),
)
self._init_compressed_pools(
stage_ratios=stage_ratios,
page_size=page_size,
@@ -1163,13 +1185,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
self.full_to_swa_index_mapping = full_to_swa_index_mapping
def get_ring_size(self, compress_ratio: int) -> int:
spec = get_spec()
num_draft_tokens = (
0
if spec.speculative_algorithm is None
else spec.speculative_num_draft_tokens or 0
)
return get_compress_state_ring_size(compress_ratio, num_draft_tokens)
return resolve_compress_state_ring_size(compress_ratio)
def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor):
assert self.full_to_swa_index_mapping is not None
@@ -1181,10 +1197,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
item_lens: List[int] = []
if self._unified_kv_fp8:
# The page-block transfer below prices one row as buf[0].nbytes and
# ships a single pointer per layer. Under fp8 that covers the nope
# pool only -- the parallel bf16 rope pool would be dropped and the
# remote side would decode rows against stale rope. Refuse instead.
# One pointer and one buf[0]-sized row per layer covers only the fp8 nope
# pool; the remote side would decode rows against stale rope.
# TODO(danli103): ship the rope pool as a second per-layer entry.
raise NotImplementedError(
"PD disaggregation is not supported with "
@@ -1224,9 +1238,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
indexer_pool = self.index_pools.get(ratio)
if indexer_pool is None:
continue
# The transfer addresses every buffer by FULL page id; ratio-1/2 index
# pools page at DSV41_INDEX_PAGE_SIZE, so one item is the run of index
# pages holding a FULL page's page_size // ratio slots.
# The transfer addresses buffers by FULL page id; one ratio-1/2 item is the
# run of DSV41_INDEX_PAGE_SIZE pages holding page_size // ratio slots.
index_pages_per_full_page = 1
if ratio in (1, 2):
slots_per_full_page = self.page_size // ratio
@@ -1255,9 +1268,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
if not self._unified_kv:
return data_ptrs, data_lens, item_lens
if self._unified_kv_fp8:
# Other half of the PD path -- get_contiguous_buf_infos ships the
# compressed region, this one the ring. Same single-pool assumption,
# same silently dropped rope, same fix -- land them together.
# Ring half of the fp8 gap in get_contiguous_buf_infos; fix both together.
raise NotImplementedError(
"PD disaggregation is not supported with "
"SGLANG_DSV4_UNIFIED_KV_FP8=1 (the SWA_RING component assumes a "
@@ -1304,9 +1315,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
return views, item_bytes
def unified_region_buffers(self, ratio: int) -> Tuple[List[torch.Tensor], int]:
"""
Main compressed region of one stage: bf16 latents, or fp8 nope.
"""
"""Main compressed region of one stage: bf16 latents, or fp8 nope."""
assert self._unified_kv, "unified_region_buffers requires unified_kv layout"
assert ratio in (4, 128), f"unsupported compression ratio: {ratio}"
return self._unified_page_views(self.unified_kv_pool.kv_buffer, ratio)
@@ -1314,14 +1323,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
def unified_rope_region_buffers(
self, ratio: int
) -> Optional[Tuple[List[torch.Tensor], int]]:
"""
The bf16 rope half of an fp8 two-pool row, or None when there isn't one.
A row index addresses both pools, so this mirrors exactly the rows
``unified_region_buffers`` does and only the row width differs. It needs
its own host pool: offloading the nope half alone leaves whatever rope the
row held before, which is wrong output rather than a crash.
"""
"""The bf16 rope half of an fp8 two-pool row, or None. It needs its own host
pool: offloading only the nope half leaves stale rope, i.e. wrong output."""
if not self._unified_kv_fp8:
return None
assert self._unified_kv, "unified_rope_region_buffers requires unified_kv"
@@ -1567,7 +1570,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
enable_memory_saver=enable_memory_saver,
ratio=ratio,
online=(ratio == 128 and ONLINE_C128),
request_scoped=ratio in (2, 128),
# Ratio 2 never reaches this factory (_make_pair_state_pool).
request_scoped=ratio == 128,
swa_page_size=self.swa_page_size,
online_mtp_max_draft_tokens=(
self.online_mtp_max_draft_tokens if ratio == 128 else 0
@@ -1624,28 +1628,6 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
enable_memory_saver=enable_memory_saver,
)
def _collect_sources_by_ratio(self) -> dict[int, List[int]]:
"""Layers owning compressed storage: all of ratios 4/128, kv_sources of 1/2."""
stage = range(self._stage_start, self._stage_end)
for idx in stage:
ratio = self.compression_ratios[idx]
if ratio not in (0, 1, 2, 4, 128):
raise ValueError(f"Unsupported compression ratio: {ratio}")
sources_by_ratio: dict[int, List[int]] = {}
for ratio in (4, 128, 1, 2):
if ratio in (1, 2):
layers = [
l
for l in self.kv_source_layers
if l in stage and self.compression_ratios[l] == ratio
]
else:
layers = [l for l in stage if self.compression_ratios[l] == ratio]
if layers:
sources_by_ratio[ratio] = layers
return sources_by_ratio
def source_layer_of(self, layer_id: int) -> int:
"""The layer owning this layer's compressed storage: itself for ratios 4/128,
the nearest preceding kv_source layer for ratios 1/2."""
@@ -2008,10 +1990,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
positions: torch.Tensor,
) -> None:
if self.uniform_fp8:
# Uniform-FP8 (trtllm-gen) layout: norm + RoPE with the existing
# Triton kernel (in-place on kv; safe -- kv is not read again),
# then a plain e4m3 cast + scatter in the pool setter (per-tensor
# scale 1.0). Fusing the store is deferred to the perf phase.
# Uniform-FP8 (trtllm-gen): in-place norm + RoPE (kv is not read again),
# then an e4m3 cast + scatter with per-tensor scale 1.0.
from sglang.kernels.ops.attention.deepseek_v4_rope import (
fused_norm_rope_inplace_triton,
)
@@ -2049,14 +2029,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
freqs_cis: torch.Tensor,
positions: torch.Tensor,
) -> None:
"""unified_kv counterpart of set_swa_key_buffer_radix_fused_norm_rope.
Under unified_kv the (fp8, paged) swa_kv_pool is None -- SWA K lives in
the shared bf16 unified_kv ring instead. Norm+RoPE the draft KV in place
(the same freqs_cis path the main model uses via _compute_kv_bf16) and
scatter it into ``unified_kv[swa_loc]``. Rows with swa_loc < 0
(uncommitted verify tokens) are skipped by the scatter.
"""
"""unified_kv counterpart of set_swa_key_buffer_radix_fused_norm_rope; rows
with swa_loc < 0 (uncommitted verify tokens) are skipped by the scatter."""
from sglang.kernels.ops.attention.dsv4 import fused_norm_rope_inplace
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime
@@ -4,11 +4,6 @@ Each model architecture has its own configurator that computes pool sizes
from available GPU memory using a unified coeff+bias model:
available_bytes = max_tokens * coeff + bias
max_tokens = (available_bytes - bias) / coeff
Two entry points, same core computation:
- calculate_pool_sizes(available_bytes, page_size): profiling path
- calculate_pool_sizes_from_max_tokens(max_tokens, page_size): constraint path
"""
from __future__ import annotations
@@ -36,10 +31,11 @@ from sglang.srt.configs.model_config import (
from sglang.srt.environ import envs
from sglang.srt.mem_cache.allocation_sizing import get_alloc_len_per_decode
from sglang.srt.mem_cache.deepseek_v4_memory_pool import (
get_compress_state_ring_size,
collect_sources_by_ratio,
get_compress_state_write_pad,
get_dsv4_indexer_bytes_per_token,
get_swa_ring_size,
resolve_compress_state_ring_size,
)
from sglang.srt.mem_cache.memory_pool import (
DSATokenToKVPool,
@@ -85,11 +81,8 @@ class MemoryPoolConfig:
mem_fraction_static: Optional[float] = None
# Unified pool only: the PROFILED byte budget for the token-granular
# sub-pools. Set, the factories size the buffer from it directly instead of
# re-summing ratio-derived token counts, which keeps the re-sum's floor
# losses out of the buffer; the token counts stay boot labels / conserve
# caps. None on the token-capped path -- a user token cap IS the budget.
# Unified pool only: the profiled byte budget the buffer is sized from,
# instead of re-summing token counts; None when a user token cap is the budget.
unified_total_bytes: Optional[int] = None
def __post_init__(self):
@@ -107,14 +100,8 @@ logger = logging.getLogger(__name__)
def _dflash_draft_cell_size(kvc: KVCacheConfigurator) -> int:
"""Bytes/token the DFLASH draft KV pool adds to the target's budget, 0 if none.
Unlike an EAGLE draft, which reuses the target's attention config and is
therefore priced by layer count, a DFLASH draft has its own geometry and is
a flat additive term. Under DCP, the target pool is sharded while the draft
pool spans the allocator's widened virtual location space, so the draft
term is replicated across DCP ranks.
"""
"""Bytes/token the DFLASH draft KV pool adds, 0 if none; replicated across DCP
ranks because the draft pool spans the widened virtual location space."""
if kvc.is_draft_worker or not kvc.spec_algorithm.is_dflash_family():
return 0
cell_size = kvc.spec_aux_config.dflash_draft_cell_size_per_token
@@ -200,11 +187,7 @@ class MemoryPoolConfigurator:
class DefaultPoolConfigurator(MemoryPoolConfigurator):
"""Configurator for standard models: MHA, MLA, DSA, FP4.
coeff = cell_size (bytes per token across all layers)
bias = 0
"""
"""Standard models (MHA, MLA, DSA, FP4): coeff = cell_size, bias = 0."""
def __init__(self, kvc: KVCacheConfigurator):
self.kv_cache_dtype_str = kvc.kv_cache_dtype_str
@@ -232,10 +215,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
else get_schedule().max_total_tokens or kvc.model_config.context_len
)
# EAGLE/STANDALONE: scale cell_size to account for draft model KV cache.
# Assumes draft and target share the same per-layer KV size (head_dim,
# num_kv_heads, dtype), which holds for EAGLE/MTP draft models that
# reuse the target architecture's attention config.
# EAGLE/STANDALONE: assumes the draft shares the target's per-layer KV size,
# which holds for EAGLE/MTP drafts that reuse the target's attention config.
if (
kvc.spec_algorithm.is_eagle() or kvc.spec_algorithm.is_standalone()
) and not kvc.is_draft_worker:
@@ -273,11 +254,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
self._cell_size * (1 + draft_num_layers / int(num_layers))
)
# DFLASH/DSPARK: reserve the draft runner's *actual* per-token KV cost.
# The draft allocates its own KV pool at the target's
# max_total_num_tokens, whose per-token footprint can differ from the
# target's (e.g. an MLA-latent target paired with a full per-head K/V
# draft), so size from the draft config rather than the layer ratio.
# DFLASH/DSPARK: the draft's per-token KV cost can differ from the target's
# (e.g. MLA target, per-head K/V draft), so size from the draft config.
if kvc.spec_algorithm.is_dflash_family() and not kvc.is_draft_worker:
from sglang.srt.speculative.dflash_utils import (
scale_kv_cell_size_per_token_for_dflash,
@@ -298,7 +276,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
)
def _compute_cell_size(self, kvc: KVCacheConfigurator, num_layers: int) -> int:
"""Compute per-token KV cache cost in bytes. Subclasses can override."""
"""Compute per-token KV cache cost in bytes."""
# args to config cell size
model_config = kvc.model_config
kv_cache_dtype = kvc.kv_cache_dtype
@@ -540,9 +518,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
_, shard_size = get_glm_dsa_cp_layer_shard_info(kvc)
if shard_size > 1:
# Preserve the existing LayerSplit sizing semantics. GLM-5.3
# hybrid-layer support is intentionally limited to the normal
# (non-LayerSplit) pool below.
# GLM-5.3 hybrid-layer support is intentionally limited to the
# non-LayerSplit pool below.
active_indexer_layers = [
layer_id
for layer_id in range(
@@ -592,11 +569,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
"""Configurator for MHA or MLA models with sliding-window layers.
Splits available memory between full attention and SWA pools.
Does NOT inherit DefaultPoolConfigurator — different coeff model.
"""
"""Splits memory between the full and SWA pools of MHA/MLA sliding-window models."""
def __init__(self, kvc: KVCacheConfigurator):
self.kv_cache_dtype_str = kvc.kv_cache_dtype_str
@@ -704,15 +677,8 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
self._recompute_cell_size()
def _recompute_cell_size(self) -> None:
# Bytes per token of max_total_num_tokens.
#
# Hybrid (full_layers > 0): max_total = full_tokens, so cell_size accounts
# for both pools: F*nf + r*S*ns (where swa_tokens = full_tokens * r).
#
# All-SWA (full_layers == 0): max_total = swa_tokens directly. The ratio
# is meaningless here -- there is no full pool to relate to, and every
# token beyond the sliding window can be evicted. So cell_size = S*ns,
# with no ratio factor applied.
# Bytes per token of max_total_num_tokens: full_tokens when hybrid, else
# swa_tokens with no ratio, since all-SWA has no full pool to relate to.
if self._full_layers_num == 0:
self._cell_size = (
self._swa_per_token * self._swa_layers_num
@@ -781,14 +747,12 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
def _solve_pool_sizes(
self, max_total_num_tokens: int, page_size: int
) -> MemoryPoolConfig:
"""Core computation: split max_total_num_tokens into full/swa pool sizes."""
def align_page_size(x: int) -> int:
return (x // page_size) * page_size
if self._full_layers_num == 0:
# All-SWA: no full pool, max_total = actual SWA pool size.
# Ratio is not applied -- see __init__ comment.
# Ratio is not applied -- see _recompute_cell_size.
swa_tokens = align_page_size(max_total_num_tokens)
logger.info(
f"Use sliding window memory pool (all SWA). "
@@ -885,13 +849,8 @@ def compute_swa_request_cap(*, page_size: int, window: int, attn_dp_size: int) -
class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator):
"""Hybrid SWA configurator with the SWA pool sized from a fixed token cap.
When max_running_requests is explicit, the SWA pool's worst-case
footprint is bounded per request. The SWA pool is sized tightly from that
cap and the freed memory is redirected to the full pool, instead of sizing
both pools by swa_full_tokens_ratio.
"""
"""Hybrid SWA with the SWA pool sized from the explicit max_running_requests
worst case instead of swa_full_tokens_ratio; the rest goes to the full pool."""
def __init__(self, kvc: KVCacheConfigurator):
self.kv_cache_dtype_str = kvc.kv_cache_dtype_str
@@ -984,13 +943,8 @@ class _DSV4PoolSizes:
class DSV4PoolConfigurator(MemoryPoolConfigurator):
"""Configurator for DSV4 compressed-attention models.
Splits available memory across full / swa / c4 / c128 + c4_state / c128_state
pools. coeff is bytes_per_full_token (inflated by (T+D)/T when speculative
decode reserves a draft worker, mirroring dflash's cell_size scaling). bias
is the request-scoped fixed pools that do not scale with full_token.
"""
"""DSV4 compressed attention: coeff is bytes_per_full_token, inflated by (T+D)/T
for a draft worker; bias is the request-scoped pools that do not scale with it."""
# object.__new__ stubs (SWA floor tests) skip __init__
_dspark_draft_on_bf16 = False
@@ -1038,17 +992,19 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
)
self.context_len = kvc.model_config.context_len
# PP-local slice; matches DeepSeekV4TokenToKVPool's stage_ratios.
self.compression_ratios = cfg.compress_ratios[
kvc.layer_info.start_layer : kvc.layer_info.end_layer
]
stage = range(kvc.layer_info.start_layer, kvc.layer_info.end_layer)
self.stage_compress_ratios = cfg.compress_ratios[stage.start : stage.stop]
if kvc.pp_size > 1:
logger.info(
f"DSV4 pool PP slice: rank={kvc.pp_group.rank_in_group} "
f"layers=[{kvc.layer_info.start_layer},{kvc.layer_info.end_layer}) "
f"local={len(self.compression_ratios)}/{len(cfg.compress_ratios)}"
f"layers=[{stage.start},{stage.stop}) "
f"local={len(self.stage_compress_ratios)}/{len(cfg.compress_ratios)}"
)
self.swa_window_size = cfg.window_size
self.swa_page_size = get_schedule().page_size
# Layers of this stage that own compressed storage, per ratio. Same rule
# as the pool, so a ratio budgeted here is one it allocates.
self.stage_owner_layers = collect_sources_by_ratio(
cfg.compress_ratios, cfg.hf_config.kv_source_layer_ids, stage
)
self.operator_swa_ratio = _operator_swa_full_tokens_ratio()
self.swa_ratio = (
self.operator_swa_ratio
@@ -1079,49 +1035,26 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
if self.c4_shrink_factor > 1:
logger.info(f"HiSparse c4 host-to-device ratio = {self.c4_shrink_factor}")
# Same argument as the pool's get_ring_size, so both sides agree.
num_draft_tokens = (
(get_spec().speculative_num_draft_tokens or 0) if self.is_speculative else 0
)
self.c4_ring_size = get_compress_state_ring_size(4, num_draft_tokens)
self.c128_ring_size = get_compress_state_ring_size(128, num_draft_tokens)
self.c2_ring_size = get_compress_state_ring_size(2, num_draft_tokens)
# Ratio 1 keeps no compress state, so it has no ring.
self.ring_sizes = {
ratio: resolve_compress_state_ring_size(ratio)
for ratio in self.stage_owner_layers
if ratio != 1
}
self.num_layers_total = len(self.compression_ratios)
self.num_layers_ca4 = sum(1 for r in self.compression_ratios if r == 4)
self.num_layers_ca128 = sum(1 for r in self.compression_ratios if r == 128)
# Only a ratio-2 kv_source layer compresses, so only it keeps pair state.
self.num_layers_c2_source = sum(
1
for l in cfg.hf_config.kv_source_layer_ids
if kvc.layer_info.start_layer <= l < kvc.layer_info.end_layer
and cfg.compress_ratios[l] == 2
)
self.num_layers_total = len(self.stage_compress_ratios)
# The low-ratio indexer pools are built with force_fp4=True
# (deepseek_v4_memory_pool), so they are fp4 whatever dtype c4 uses.
low_ratio_index_bytes = get_dsv4_indexer_bytes_per_token(
self.low_ratio_index_bytes = get_dsv4_indexer_bytes_per_token(
self.indexer_head_dim, use_fp4_indexer=True
)
self.low_ratio_bytes_per_full_token = sum(
(self.kv_bytes + low_ratio_index_bytes) / cfg.compress_ratios[l]
for l in cfg.hf_config.kv_source_layer_ids
if kvc.layer_info.start_layer <= l < kvc.layer_info.end_layer
and cfg.compress_ratios[l] in (1, 2)
)
from sglang.srt.mem_cache.deepseek_v4_memory_pool import (
dsv4_unified_row_bytes,
)
# swa_window_size is the model's sliding window (cfg.window_size).
self._swa_ring_size = get_swa_ring_size(
self.swa_window_size, self.is_speculative
)
# kvc.sliding_window_size is None on a runner without SWA layers.
self._swa_ring_size = get_swa_ring_size(cfg.window_size, self.is_speculative)
self._spec_infl = 1.0
# The unified pool takes no dtype, so --kv-cache-dtype never reaches it.
# V4 defaults "auto" to fp8_e4m3 (overrides.py
# _deepseek_v4_kv_cache_dtype), so only a bfloat16 here tells us the user
# set it explicitly; warning on the fp8 side would fire on every run.
# The unified pool ignores --kv-cache-dtype; V4 resolves "auto" to fp8_e4m3
# (overrides.py _deepseek_v4_kv_cache_dtype), so only bfloat16 is explicit.
if self._unified_fp8 and self.kv_cache_dtype_str == "bfloat16":
logger.warning(
"--kv-cache-dtype=bfloat16 is ignored on the unified_kv path; "
@@ -1129,9 +1062,8 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
"env switch to get a bf16 unified pool."
)
# get_contiguous_buf_infos ships one pointer per layer and prices a row as
# buf[0].nbytes, which under fp8 covers the nope pool only. Fail at startup
# rather than at the first transfer.
# get_contiguous_buf_infos prices a row as buf[0].nbytes, which under fp8
# covers the nope pool only; fail at startup rather than at the first transfer.
# TODO(danli103): drop this once the transfer ships the rope pool.
if self._unified_fp8 and self.disaggregation_mode != "null":
raise ValueError(
@@ -1185,20 +1117,16 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
self.bytes_per_swa_token = self._get_bytes_per_swa_token()
self.bytes_per_full_token = self._get_bytes_per_full_token()
if self.is_speculative and not self.encoder_replay:
# Reserve memory for the speculative draft worker by inflating
# per-token bytes by (target+draft)/target. Equivalent to dflash's
# scale_kv_cell_size_per_token_for_dflash but applied to
# bytes_per_full_token: tokens = avail / (bpft * (T+D)/T).
# Reserve the draft worker by inflating per-token bytes by
# (target+draft)/target, as scale_kv_cell_size_per_token_for_dflash does.
draft_layers = 1
target_layers = self.num_layers_total
self._spec_infl = (target_layers + draft_layers) / target_layers
self.bytes_per_full_token *= self._spec_infl
self.bytes_per_swa_token *= self._spec_infl
# Online c128 keeps a single in-progress (max, sum, kv) state per index
# and assumes a strict forward-only schedule. Speculative decode (MTP)
# would need rollback / replay across draft and verify, which the
# online path doesn't support yet.
# Online c128 keeps one in-progress (max, sum, kv) state per index and
# assumes forward-only; MTP would need rollback across draft and verify.
if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
allow_experimental_online_c128_mtp = (
envs.SGLANG_EXPERIMENTAL_ONLINE_C128_MTP.get()
@@ -1225,18 +1153,20 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
"DSV4 compressed attention: online c128 enabled (ring_size=1)"
)
def num_layers(self, ratio: int) -> int:
"""Layers of this stage owning ratio's compressed storage."""
return len(self.stage_owner_layers.get(ratio, ()))
def _assert_ring_serves_draft_tokens(self, num_draft_tokens: int) -> None:
"""A verify batch writes its whole optimistic tail into the ring, so ring
capacity bounds the draft count."""
for compress_ratio, ring_size, num_layers in (
(4, self.c4_ring_size, self.num_layers_ca4),
(128, self.c128_ring_size, self.num_layers_ca128),
):
if num_layers == 0:
for compress_ratio in (4, 128):
if self.num_layers(compress_ratio) == 0:
continue
if compress_ratio == 128 and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
# Online c128 keeps per-draft state instead of a ring; sized separately.
continue
ring_size = self.ring_sizes[compress_ratio]
max_draft_tokens = get_compress_state_write_pad(compress_ratio, ring_size)
assert num_draft_tokens <= max_draft_tokens, (
f"speculative_num_draft_tokens={num_draft_tokens} exceeds what the c{compress_ratio} "
@@ -1284,7 +1214,7 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
def _get_bytes_per_swa_token(self) -> float:
"""Bytes one SWA slot costs across the stage. c4_state_pool_size = swa_tokens
/ swa_page_size * ring, so c4 compress state is priced per SWA slot too."""
// page_size * ring, so c4 compress state is priced per SWA slot too."""
if self.encoder_replay:
# Target SWA lives in the request window; only the draft owns paged SWA
# bytes, and its layers carry no compressed state.
@@ -1293,46 +1223,35 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
c4_state_bytes = 2 * 2 * self.attn_head_dim * c4_state_dtype_size
c4_indexer_state_bytes = 2 * 2 * self.indexer_head_dim * c4_state_dtype_size
c4_state_ratio = self.c4_ring_size / self.swa_page_size
return (
self.kv_bytes * self.num_layers_total
+ c4_state_ratio
* (c4_state_bytes + c4_indexer_state_bytes)
* self.num_layers_ca4
)
c4_state_ratio = self.ring_sizes.get(4, 0) / self.page_size
return self.kv_bytes * self.num_layers_total + c4_state_ratio * (
c4_state_bytes + c4_indexer_state_bytes
) * self.num_layers(4)
def _compressed_bytes_per_full_token(self, ratio: int) -> float:
"""Compressed KV (+ indexer) bytes one full token adds per layer of `ratio`;
compress state is priced per SWA slot (c4) or as fixed bytes (c128, ratio 2)."""
if ratio in (1, 2):
return (self.kv_bytes + self.low_ratio_index_bytes) / ratio
if ratio == 4:
c4_frac = 1 / (4 * self.c4_shrink_factor)
return c4_frac * self.kv_bytes + 1 / 4 * self.indexer_bytes_per_token
assert ratio == 128, f"unsupported compression ratio: {ratio}"
return 1 / 128 * self.kv_bytes
def _get_bytes_per_full_token(self) -> float:
_, c128_state_dtype_size = _get_dsv4_compress_state_dtype_sizes()
# Online c128 stores (max, sum, kv) per slot (3*head_dim) instead of
# raw (kv, score) (2*head_dim). Combined with ring_size=1 this still
# nets a large reduction (~3/256x) but the per-slot bytes go up.
c128_online = envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get()
c128_state_bytes = (
(3 if c128_online else 2 * 1) * self.attn_head_dim * c128_state_dtype_size
)
# C128 state is request-scoped and is finalized after
# max_running_requests is known, so it should not scale with
# full-token capacity here.
c128_state_ratio = 0
# Cap mode and ring mode both move the SWA pool and the c4 state that
# follows it out of the coefficient and into fixed bytes.
swa_ratio = (
0 if self._unified or self.swa_cap_tokens is not None else self.swa_ratio
)
c4_frac = 1 / (4 * self.c4_shrink_factor)
return (
swa_ratio * self.bytes_per_swa_token
+ self.low_ratio_bytes_per_full_token
+ c4_frac * self.kv_bytes * self.num_layers_ca4
+ 1 / 128 * self.kv_bytes * self.num_layers_ca128
+ 1 / 4 * self.indexer_bytes_per_token * self.num_layers_ca4
+ c128_state_ratio * c128_state_bytes * self.num_layers_ca128
return swa_ratio * self.bytes_per_swa_token + sum(
self._compressed_bytes_per_full_token(ratio) * len(layers)
for ratio, layers in self.stage_owner_layers.items()
)
def _get_swa_fixed_bytes(self) -> float:
"""Bias bytes the SWA pool takes in cap mode; 0 when sizing by ratio."""
"""Bias bytes of the encoder-replay request window plus the cap-mode pool."""
paged_bytes = (
0
if self.swa_cap_tokens is None
@@ -1378,10 +1297,11 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
c4_max_total_num_tokens=full_token // (4 * self.c4_shrink_factor),
c128_max_total_num_tokens=full_token // 128,
# Unified_kv: request-scoped, finalized once concurrency is known.
# Otherwise the ring is addressed per SWA page (swa_loc // page_size).
c4_state_pool_size=(
0
if self._unified
else swa_tokens // self.swa_page_size * self.c4_ring_size
else swa_tokens // page_size * self.ring_sizes.get(4, 0)
),
c128_state_pool_size=0,
)
@@ -1392,63 +1312,60 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
return max_running_requests + 1
def _get_c128_state_fixed_bytes(self, max_running_requests: int) -> int:
if self.num_layers_ca128 == 0:
num_layers = self.num_layers(128)
if num_layers == 0:
return 0
_, c128_state_dtype_size = _get_dsv4_compress_state_dtype_sizes()
num_req_slots = self._get_num_req_slots(max_running_requests)
ring_size = self.ring_sizes[128]
if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
state_rows = num_req_slots + self.c128_ring_size + 1
state_rows = num_req_slots + ring_size + 1
state_rows *= 1 + self.online_c128_mtp_max_draft_tokens
state_last_dim = 3 * self.attn_head_dim
else:
state_pool_size = num_req_slots * self.c128_ring_size
state_rows = state_pool_size + self.c128_ring_size + 1
state_pool_size = num_req_slots * ring_size
state_rows = state_pool_size + ring_size + 1
state_rows = ceil_div(state_rows, 128) * 128
state_last_dim = 2 * self.attn_head_dim
return (
state_rows * state_last_dim * c128_state_dtype_size * self.num_layers_ca128
)
return state_rows * state_last_dim * c128_state_dtype_size * num_layers
def _get_c2_state_fixed_bytes(self, max_running_requests: int) -> int:
"""Ratio-2 pending-pair state, one fp32 (kv, score) ring per request slot
on each ratio-2 kv_source layer; mirrors _make_pair_state_pool."""
if self.num_layers_c2_source == 0:
num_layers = self.num_layers(2)
if num_layers == 0:
return 0
num_req_slots = self._get_num_req_slots(max_running_requests)
ring_size = self.c2_ring_size
ring_size = self.ring_sizes[2]
# CompressStatePool allocates `size + ring_size + 1` rows, padded to the ratio.
state_rows = num_req_slots * ring_size + ring_size + 1
state_rows = ceil_div(state_rows, 2) * 2
state_last_dim = 2 * self.attn_head_dim
return (
state_rows
* state_last_dim
* torch.float32.itemsize
* self.num_layers_c2_source
)
return state_rows * state_last_dim * torch.float32.itemsize * num_layers
def _unified_c4_state_pool_size(self, max_running_requests: int) -> int:
# Unified C4 state loc is req_pool_idx * c4_ring_size + pos % c4_ring_size.
num_req_slots = self._get_num_req_slots(max_running_requests)
return num_req_slots * self.c4_ring_size
return num_req_slots * self.ring_sizes[4]
def _fixed_c4_state_bytes(self, max_running_requests: int) -> int:
if not self._unified or self.num_layers_ca4 == 0:
num_layers = self.num_layers(4)
if not self._unified or num_layers == 0:
return 0
c4_state_dtype_size, _ = _get_dsv4_compress_state_dtype_sizes()
# Mirror CompressStatePool.__init__: it allocates `size + ring_size + 1`
# rows, padded to the compress ratio.
state_rows = self._unified_c4_state_pool_size(max_running_requests)
state_rows = ceil_div(state_rows + self.c4_ring_size + 1, 4) * 4
state_rows = ceil_div(state_rows + self.ring_sizes[4] + 1, 4) * 4
# overlap c4: last_dim = 2 * (1 + overlap) * head_dim = 4 * head_dim.
core_bytes = 4 * self.attn_head_dim * c4_state_dtype_size
indexer_bytes = 4 * self.indexer_head_dim * c4_state_dtype_size
return state_rows * (core_bytes + indexer_bytes) * self.num_layers_ca4
return state_rows * (core_bytes + indexer_bytes) * num_layers
def _resolve_max_running_requests_per_worker(self, available_bytes: int) -> int:
# Approximates ModelRunner._resolve_max_num_reqs. Over-estimating is safe:
@@ -1462,10 +1379,8 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
return min(estimated, full_token // 2)
def _fixed_swa_bytes(self, max_running_requests: int) -> int:
"""Unified_kv SWA is a fixed per-request ring, sized by concurrency
(num_req_slots) rather than by full_token. MTP inflates the target ring
by _spec_infl; DSpark+fp8 adds a bf16 draft ring instead (640 vs 1024).
Returns 0 on the non-unified path (SWA already counted per-token)."""
"""Unified_kv SWA ring bytes, sized by req slots; 0 off unified, where SWA is
priced per token. DSpark+fp8 adds a bf16 draft ring instead of _spec_infl."""
if not self._unified:
return 0
num_req_slots = self._get_num_req_slots(max_running_requests)
@@ -1483,9 +1398,8 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
draft_row = dsv4_unified_row_bytes(
self.qk_nope_head_dim, self.qk_rope_head_dim, fp8=False
)
# 1 layer is what the shipped DSpark drafts allocate. A multi-stage
# draft would under-count by ~9 MB/layer (128-wide window, ~65 req
# slots), which the (T+1)/T on bytes_per_full_token already covers.
# 1 layer is what shipped DSpark drafts allocate; a multi-layer draft
# under-counts ~9 MB/layer, covered by the (T+1)/T inflation.
draft_ring = num_req_slots * self._swa_ring_size * draft_row
return int(target_ring + draft_ring)
return int(target_ring * self._spec_infl)
@@ -1518,9 +1432,9 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
config.c128_state_pool_size = num_req_slots
else:
config.c128_state_pool_size = num_req_slots * self.c128_ring_size
config.c128_state_pool_size = num_req_slots * self.ring_sizes.get(128, 0)
# Ring mode: C4 state is request-scoped, so size it from the known concurrency.
if self._unified and self.num_layers_ca4 > 0:
if self._unified and self.num_layers(4) > 0:
config.c4_state_pool_size = self._unified_c4_state_pool_size(
config.max_running_requests
)
@@ -1597,7 +1511,6 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
def create_memory_pool_configurator(
kvc: KVCacheConfigurator,
) -> MemoryPoolConfigurator:
"""Factory: select the right configurator for the model architecture."""
if is_deepseek_v4(kvc.model_config.hf_config) and kvc.is_hybrid_swa:
return DSV4PoolConfigurator(kvc)
if kvc.is_hybrid_swa: