[DSV4] Size compressed pools from one per-ratio table in DSV4PoolConfigurator (#41049)
This commit is contained in:
@@ -713,9 +713,8 @@ class DSV4AttnMetadata:
|
||||
assert dst_val is not None, f"{field_name=} {src_val=} {dst_val=}"
|
||||
dst_val.copy_(src_val)
|
||||
|
||||
# These fields are safe to replace because captured kernels only need
|
||||
# the current per-replay objects, or the field is produced inside the
|
||||
# captured graph before the attention graph break consumes it.
|
||||
# Safe to replace: captured kernels read only the per-replay objects, or
|
||||
# the field is produced in-graph before the attention graph break reads it.
|
||||
for field_name in reference_assign_fields:
|
||||
setattr(self, field_name, getattr(other, field_name))
|
||||
|
||||
@@ -723,9 +722,8 @@ class DSV4AttnMetadata:
|
||||
self, num_tokens: Optional[int] = None, low_ratio_buffers=None
|
||||
) -> None:
|
||||
assert self.page_table.dim() == 2
|
||||
# CP pads causal metadata for per-rank partitioning, while cache-write
|
||||
# locations remain one-per-logical-token. num_tokens tracks that unpadded
|
||||
# length; legacy paths use the metadata length.
|
||||
# CP pads causal metadata per rank, but cache-write locations stay one per
|
||||
# logical token; num_tokens is that unpadded length.
|
||||
if num_tokens is None:
|
||||
num_tokens = self.seq_lens_casual.shape[0]
|
||||
assert self.raw_out_loc.shape[0] == num_tokens, (
|
||||
@@ -1148,7 +1146,6 @@ class DeepseekV4AttnBackend(
|
||||
self.softmax_scale: float = head_dim**-0.5
|
||||
self.head_dim_v: int = model_runner.model_config.v_head_dim
|
||||
self.cuda_int32_kwargs = {"device": self.device, "dtype": torch.int32}
|
||||
self.swa_page_size = 128
|
||||
assert model_runner.page_size is not None
|
||||
assert model_runner.req_to_token_pool is not None
|
||||
self.page_size = model_runner.page_size
|
||||
@@ -1156,6 +1153,8 @@ class DeepseekV4AttnBackend(
|
||||
|
||||
self.req_to_token_pool = model_runner.req_to_token_pool
|
||||
self.token_to_kv_pool: DeepSeekV4TokenToKVPool = model_runner.token_to_kv_pool
|
||||
# The C4 state ring is addressed per SWA page, so a page holds whole windows.
|
||||
assert self.token_to_kv_pool.swa_page_size % SWA_WINDOW == 0
|
||||
self.hisparse_coordinator = model_runner.hisparse_coordinator
|
||||
self.req_to_token = model_runner.req_to_token_pool.req_to_token
|
||||
# Nothing is built for a compress ratio outside the pool's set.
|
||||
@@ -1207,11 +1206,8 @@ class DeepseekV4AttnBackend(
|
||||
self.speculative_num_steps = speculative_num_steps
|
||||
self.speculative_num_draft_tokens: int = get_spec().speculative_num_draft_tokens
|
||||
if self.speculative_num_draft_tokens is not None:
|
||||
# Persistent target-verify metadata buffers. Allocated here (not
|
||||
# lazily) so they are ordinary tensors: the first touch of a lazy
|
||||
# buffer would inherit the caller's context, and a creation inside
|
||||
# an inference_mode forward would forbid the in-place updates the
|
||||
# graph-capture path performs outside inference mode.
|
||||
# Allocated eagerly: a lazy buffer first created inside inference_mode
|
||||
# would forbid the in-place updates graph capture performs outside it.
|
||||
num_reqs = self.req_to_token.shape[0]
|
||||
self.extend_seq_lens_buffer = torch.full(
|
||||
(num_reqs,),
|
||||
@@ -1263,9 +1259,8 @@ class DeepseekV4AttnBackend(
|
||||
"DSV4 ragged verify does not support online c128 MTP; "
|
||||
"set SGLANG_RAGGED_VERIFY_MODE off or disable online compress."
|
||||
)
|
||||
# Layout invariants (verify_lens >= 1, total == sum) are enforced in
|
||||
# RaggedVerifyLayout.__post_init__; don't re-check the device tensor
|
||||
# here -- that would D2H-sync the host-free verify prep path.
|
||||
# Invariants are checked in RaggedVerifyLayout.__post_init__; a device-side
|
||||
# check here would D2H-sync the host-free verify prep path.
|
||||
layout = layout.padded_to_bucket(padded_bs=bs)
|
||||
return layout
|
||||
|
||||
@@ -1450,9 +1445,8 @@ class DeepseekV4AttnBackend(
|
||||
else:
|
||||
|
||||
def create(compress_ratio: Literal[4, 128]):
|
||||
# Online c128 uses a different planner that cannot be created in
|
||||
# prefill cuda-graph mode. Keep c4 graph-friendly while matching
|
||||
# c128's existing online path.
|
||||
# Online c128's planner cannot be created in prefill cuda-graph
|
||||
# mode, so it keeps the online path.
|
||||
use_graph_plan = use_prefill_cuda_graph and not (
|
||||
compress_ratio == 128 and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get()
|
||||
)
|
||||
@@ -2069,7 +2063,7 @@ class DeepseekV4AttnBackend(
|
||||
)
|
||||
|
||||
# Upgrade Raw->Full so compress + core_attn + indexer materialization is
|
||||
# recorded inside the cuda graph; already Full when PREP_IN_CUDA_GRAPH=0.
|
||||
# recorded inside the cuda graph.
|
||||
if isinstance(self.forward_metadata, DSV4RawVerifyMetadata):
|
||||
self.forward_metadata = self.make_forward_metadata_from_raw_verify(
|
||||
raw_metadata=self.forward_metadata,
|
||||
@@ -2480,13 +2474,12 @@ class DeepseekV4AttnBackend(
|
||||
logical_forward_mode = _get_logical_forward_mode(forward_batch)
|
||||
req_pool_indices = forward_batch.req_pool_indices
|
||||
seq_lens = forward_batch.seq_lens.to(torch.int32)
|
||||
# Regular prefill batches already carry scheduler-maintained CPU lengths.
|
||||
# Keep using those when present; needs_cpu_seq_lens only controls whether
|
||||
# speculative overlap must publish a new GPU-to-CPU mirror each step.
|
||||
# Prefill batches carry scheduler-maintained CPU lengths; needs_cpu_seq_lens
|
||||
# only controls whether spec overlap publishes a GPU-to-CPU mirror each step.
|
||||
seq_lens_cpu = forward_batch.seq_lens_cpu
|
||||
assert self.req_to_token_pool.req_to_token is self.req_to_token
|
||||
|
||||
assert self.swa_page_size % SWA_WINDOW == 0 and self.page_size % 128 == 0
|
||||
assert self.page_size % 128 == 0
|
||||
if max_seq_len_override is None:
|
||||
max_seq_len_override = forward_batch.max_seq_len_override
|
||||
if max_seq_len_override is not None:
|
||||
@@ -2635,9 +2628,8 @@ class DeepseekV4AttnBackend(
|
||||
*,
|
||||
static_forward_batch: Optional[ForwardBatch] = None,
|
||||
) -> None:
|
||||
# Build graph-compatible metadata against the padded static batch. The
|
||||
# batch still carries live seq/extend lens, so the online c128 prefill
|
||||
# plan remains batch-specific without constructing a second metadata set.
|
||||
# The padded static batch still carries live seq/extend lens, so the
|
||||
# online c128 prefill plan stays batch-specific.
|
||||
metadata_batch = (
|
||||
static_forward_batch if static_forward_batch is not None else forward_batch
|
||||
)
|
||||
@@ -2727,8 +2719,8 @@ class DeepseekV4AttnBackend(
|
||||
if 2 in core.low_ratios:
|
||||
core.c2_flashmla_metadata = _create_flashmla_metadata()
|
||||
|
||||
# PREP_IN_CUDA_GRAPH=True: warmup upgraded raw->full on the host;
|
||||
# restore raw so capture re-runs the upgrade inside the graph.
|
||||
# Warmup upgraded raw->full on the host; restore raw so capture
|
||||
# re-runs the upgrade inside the graph.
|
||||
current_raw = getattr(self, "_current_capture_raw", None)
|
||||
if current_raw is not None:
|
||||
self.forward_metadata = current_raw
|
||||
@@ -3870,10 +3862,8 @@ class DeepseekV4AttnBackend(
|
||||
flash_mla_with_kvcache_sm120,
|
||||
)
|
||||
|
||||
# The pad to 64 heads only serves the decode kernel's h_q
|
||||
# specialization; the prefill kernel takes arbitrary h_q, so
|
||||
# drop it instead of attending on garbage heads (4x the work
|
||||
# at attn-TP 4).
|
||||
# The 64-head pad only serves the decode kernel's h_q
|
||||
# specialization; the prefill kernel takes any h_q, so drop it.
|
||||
real_heads = layer.tp_q_head_num
|
||||
if q.shape[0] > SM120_DECODE_MAX_TOKENS:
|
||||
if q.shape[-2] > real_heads:
|
||||
@@ -3940,14 +3930,8 @@ class DeepseekV4AttnBackend(
|
||||
core_attn_metadata: DSV4AttnMetadata,
|
||||
attn_sink: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Unified prefill via flash_mla_sparse_fwd. Replaces the
|
||||
flash_mla_with_kvcache call on the extend path. Per request,
|
||||
positionally gathers the SWA window (always) and the compressed
|
||||
cache (c4/c128) into a flat bf16 workspace, then lets
|
||||
flash_mla_sparse_fwd consume the workspace via per-query rebased
|
||||
indices. Chunk-invariant scaffolding lives in
|
||||
``self.forward_metadata.sparse_prefill_cache``.
|
||||
"""
|
||||
"""Gathers each request's SWA window and c4/c128 cache into a flat bf16
|
||||
workspace that flash_mla_sparse_fwd reads through per-query rebased indices."""
|
||||
if _is_xpu:
|
||||
from sgl_kernel import flash_mla_sparse_fwd
|
||||
else:
|
||||
@@ -4077,14 +4061,8 @@ class DeepseekV4AttnBackend(
|
||||
core_attn_metadata: DSV4AttnMetadata,
|
||||
attn_sink: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Experimental DeepSeek-V4 sparse prefill path using Q8KV8 kernels.
|
||||
|
||||
This mirrors ``_forward_prefill_sparse``'s cache/index construction, but
|
||||
writes the gathered KV workspace as FP8 and calls the SM90 Q8KV8 sparse
|
||||
prefill kernel. The path is selected by ``--dsv4-prefill-backend
|
||||
flashmla_sparse_q8``; ``SGLANG_DSV4_Q8KV8_PREFILL`` remains as a debug
|
||||
override for focused runtime validation.
|
||||
"""
|
||||
"""_forward_prefill_sparse with an FP8 workspace and the SM90 Q8KV8 kernel;
|
||||
SGLANG_DSV4_Q8KV8_PREFILL is a debug override on top of the CLI selection."""
|
||||
|
||||
from sglang.kernels.ops.attention.sparse_mla_q8kv8_prefill_sm90 import (
|
||||
sparse_mla_q8kv8_prefill_fwd,
|
||||
@@ -4270,8 +4248,6 @@ class DeepseekV4AttnBackend(
|
||||
num_groups: Optional[int] = None,
|
||||
dspark_swa_buffers: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
|
||||
) -> DSV4AttnMetadata:
|
||||
assert self.swa_page_size == SWA_WINDOW
|
||||
|
||||
small_metadata = (
|
||||
not is_prefill
|
||||
and seq_lens_casual.is_cuda
|
||||
|
||||
@@ -90,10 +90,6 @@ def _create_dummy_paged_compress_data(compress_ratio: int):
|
||||
|
||||
@dataclass
|
||||
class UnifiedKvMetadata:
|
||||
"""
|
||||
unified-kv per-forward metadata
|
||||
"""
|
||||
|
||||
# SWA ring write target (req_slot*ring + pos%ring)
|
||||
swa_loc: Optional[torch.Tensor] = None
|
||||
|
||||
@@ -119,10 +115,8 @@ class UnifiedKvMetadata:
|
||||
pf_cu_q: Optional[torch.Tensor] = None
|
||||
pf_final_pos: Optional[torch.Tensor] = None
|
||||
|
||||
# Per-token req-slot map used by the SWA ring store, precomputed once per
|
||||
# step so the forward store does not recompute a repeat_interleave per layer.
|
||||
# Read by the target-verify store (num_draft*bs tokens); for plain decode it
|
||||
# equals req_pool_indices and is unused (the decode store reads that live).
|
||||
# Per-token req-slot map for the target-verify SWA ring store (num_draft*bs
|
||||
# tokens); unused by plain decode, whose store reads req_pool_indices live.
|
||||
verify_store_state_slot: Optional[torch.Tensor] = None
|
||||
|
||||
# SWA-page-offset compressed-store locations (= c*_out_loc + unified_swa_pages),
|
||||
@@ -441,9 +435,8 @@ class DSV4Metadata:
|
||||
|
||||
c4_compress_metadata: Optional[FusedCompressMetadata] = None
|
||||
c128_compress_metadata: Optional[FusedCompressMetadata] = None
|
||||
# FP4 indexer buffers that captured kernels bind by address. Deliberately
|
||||
# absent from copy_: the addresses must stay pinned across replays, and the
|
||||
# workspace builders refresh their contents instead.
|
||||
# Captured kernels bind these FP4 indexer buffers by address; absent from copy_
|
||||
# so addresses stay pinned across replays (the builders refresh contents).
|
||||
fp4_decode_workspace: Optional[FP4DecodeWorkspace] = field(default=None, repr=False)
|
||||
fp4_prefill_workspace: Optional[FP4PrefillWorkspace] = field(
|
||||
default=None, repr=False
|
||||
@@ -563,12 +556,8 @@ class _GraphBucket(enum.Enum):
|
||||
class DeepseekV4HipRadixBackend(
|
||||
AttentionBackend, C4IndexerBackendMixin, CompressorBackendMixin
|
||||
):
|
||||
# DSV4 TBO runs ONLY in eager prefill (prefill cuda-graph is disabled);
|
||||
# decode/target-verify graphs are non-TBO (primary backend only). So the TBO
|
||||
# child backends must not be driven through cuda-graph capture/replay — doing
|
||||
# so rebuilds this backend's compressor/indexer metadata per replay step on
|
||||
# both children and leaks ROCm HSA resources (HSA_STATUS_ERROR_OUT_OF_RESOURCES).
|
||||
# TboAttnBackend reads this to skip children in the *_graph paths only.
|
||||
# DSV4 TBO runs only in eager prefill; driving TBO children through graph
|
||||
# capture/replay leaks ROCm HSA resources (HSA_STATUS_ERROR_OUT_OF_RESOURCES).
|
||||
tbo_supports_cuda_graph = False
|
||||
supports_ragged_verify_graph: bool = True
|
||||
use_captured_forward_metadata_for_breakable_cuda_graph: bool = True
|
||||
@@ -592,7 +581,6 @@ class DeepseekV4HipRadixBackend(
|
||||
self.softmax_scale: float = head_dim**-0.5
|
||||
self.head_dim_v: int = model_runner.model_config.v_head_dim
|
||||
self.cuda_int32_kwargs = {"device": self.device, "dtype": torch.int32}
|
||||
self.swa_page_size = 128
|
||||
assert model_runner.page_size is not None
|
||||
assert model_runner.req_to_token_pool is not None
|
||||
self.page_size = model_runner.page_size
|
||||
@@ -600,6 +588,8 @@ class DeepseekV4HipRadixBackend(
|
||||
|
||||
self.req_to_token_pool = model_runner.req_to_token_pool
|
||||
self.token_to_kv_pool: DeepSeekV4TokenToKVPool = model_runner.token_to_kv_pool
|
||||
# The C4 state ring is addressed per SWA page, so a page holds whole windows.
|
||||
assert self.token_to_kv_pool.swa_page_size % SWA_WINDOW == 0
|
||||
self.hisparse_coordinator = model_runner.hisparse_coordinator
|
||||
self.req_to_token = model_runner.req_to_token_pool.req_to_token
|
||||
self.MAX_SEQ_LEN_FOR_CAPTURE = self.req_to_token.shape[1]
|
||||
@@ -620,16 +610,14 @@ class DeepseekV4HipRadixBackend(
|
||||
self.is_dspark = model_runner.spec_algorithm.is_dspark()
|
||||
self.is_dspark_draft = self.is_draft_worker and self.is_dspark
|
||||
# Draft layers are all COMPRESS_RATIO_NEXTN_LAYER (0), so the draft pool
|
||||
# has neither compressed kv nor an indexer pool. Settled here rather than
|
||||
# per forward: it cannot change after construction.
|
||||
# has neither compressed kv nor an indexer pool.
|
||||
self.need_compress = not self.is_draft_worker
|
||||
self.target_verify_num_draft_tokens = self.speculative_num_draft_tokens
|
||||
if self.is_dspark_draft:
|
||||
assert self.speculative_num_draft_tokens is not None
|
||||
assert self.speculative_num_draft_tokens > 1
|
||||
# DSpark draft workers verify gamma rows. The server arg keeps the
|
||||
# CUDA-side convention gamma + 1, so use an explicit effective value
|
||||
# instead of mutating speculative_num_draft_tokens in place.
|
||||
# DSpark draft workers verify gamma rows; the server arg keeps the
|
||||
# CUDA-side convention gamma + 1.
|
||||
self.target_verify_num_draft_tokens = self.speculative_num_draft_tokens - 1
|
||||
# Past MAX_FUSED_ROWS the fp4 schedule falls back to AITER's preamble,
|
||||
# which frees the scratch its kernels read -- not capture-safe.
|
||||
@@ -701,8 +689,7 @@ class DeepseekV4HipRadixBackend(
|
||||
)
|
||||
|
||||
# extend_start_loc and the CPU mirrors below only feed the torch
|
||||
# fallback; the triton kernel cumsums extend_seq_lens on device, so
|
||||
# every caller can share it instead of dropping to a per-request loop.
|
||||
# fallback; the triton kernel cumsums extend_seq_lens on device.
|
||||
_expanded = ExpandPrefillCausally.execute(
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=seq_lens,
|
||||
@@ -742,10 +729,8 @@ class DeepseekV4HipRadixBackend(
|
||||
exact_num_tokens=exact_num_tokens or host_proves_exact_num_tokens,
|
||||
)
|
||||
if attach_decode_streams:
|
||||
# Target-verify runs through the unified_kv DECODE kernel, so build
|
||||
# per-token decode streams here. req_pool_indices_repeated is the
|
||||
# per-token (num_draft*bs -> bs) req-slot map produced by the prefill
|
||||
# expansion above.
|
||||
# Verify runs the unified_kv DECODE kernel; req_pool_indices_repeated
|
||||
# is the per-token (num_draft*bs -> bs) req-slot map.
|
||||
self._attach_unified_kv_decode_streams(
|
||||
core_attn_metadata,
|
||||
req_pool_indices_repeated,
|
||||
@@ -811,12 +796,9 @@ class DeepseekV4HipRadixBackend(
|
||||
seq_lens_cpu: Optional[List[int]] = None,
|
||||
ragged_layout=None,
|
||||
) -> Union[DSV4Metadata, DSV4RawVerifyMetadata]:
|
||||
# DSPARK verifies a uniform num_draft block, exactly what
|
||||
# make_forward_metadata_from_raw_verify expands, so the build can be
|
||||
# deferred into the graph. Graph path only: the upgrade sizes its page
|
||||
# table by MAX_SEQ_LEN_FOR_CAPTURE, far wider than the live max_seq_len
|
||||
# an eager caller passes. EAGLE and ragged layouts stay eager -- no raw
|
||||
# expansion, and EAGLE's fixed-tier plan trips planner invariants.
|
||||
# Only DSPARK's uniform draft block defers into the graph; EAGLE's fixed-tier
|
||||
# plan trips planner invariants. Graph path only: the upgrade sizes page_table
|
||||
# by MAX_SEQ_LEN_FOR_CAPTURE, wider than an eager caller's max_seq_len.
|
||||
if (
|
||||
use_prefill_cuda_graph
|
||||
and self.is_dspark
|
||||
@@ -869,10 +851,8 @@ class DeepseekV4HipRadixBackend(
|
||||
)
|
||||
extend_seq_lens = verify_lens_dev
|
||||
seq_lens = seq_lens + verify_lens_dev.to(seq_lens.dtype)
|
||||
# Total verify tokens to expand. For the graph path the padded layout
|
||||
# sets total_verify_tokens == graph_num_tokens (tier); the eager path
|
||||
# resolves a device-assembled layout whose total_verify_tokens is None,
|
||||
# so fall back to sum(verify_lens) (== real total; padded == tier).
|
||||
# Graph path: total_verify_tokens is the padded tier; an eager
|
||||
# device-assembled layout leaves it None, so use sum(verify_lens).
|
||||
num_tokens = ragged_layout.total_verify_tokens
|
||||
if num_tokens is None:
|
||||
num_tokens = int(verify_lens_dev.sum().item())
|
||||
@@ -1120,14 +1100,12 @@ class DeepseekV4HipRadixBackend(
|
||||
)
|
||||
|
||||
if upgraded_verify:
|
||||
# The out-graph refresh saw raw metadata and skipped. Without this
|
||||
# the logits kernel builds its own schedule -- the variant that
|
||||
# frees the scratch it reads, which every replay would re-read.
|
||||
# The out-graph refresh saw raw metadata and skipped; otherwise the
|
||||
# logits kernel builds a schedule that frees scratch replays re-read.
|
||||
self._refresh_fp4_prefill_workspace(forward_batch)
|
||||
|
||||
# Decode's schedule builder is capture-safe because the workspace pins
|
||||
# the scratch it reads, so it can stay next to the metadata it consumes.
|
||||
# Prefill/target-verify cannot; see _refresh_fp4_prefill_workspace.
|
||||
# Decode's schedule builder is capture-safe (the workspace pins its scratch);
|
||||
# prefill/target-verify's is not, see _refresh_fp4_prefill_workspace.
|
||||
if self._fp4_workspaces_enabled(metadata) and (
|
||||
forward_batch.forward_mode.is_decode()
|
||||
):
|
||||
@@ -1144,9 +1122,8 @@ class DeepseekV4HipRadixBackend(
|
||||
def _fp4_workspaces_enabled(self, metadata) -> bool:
|
||||
return (
|
||||
self.enable_deepseek_v4_fp4_indexer
|
||||
# Draft-step backends drive the NextN layer, which is built with
|
||||
# compress_ratio_override=0 and so owns no C4 indexer. Their
|
||||
# workspaces would be built, scheduled, and never read.
|
||||
# Draft-step backends drive the NextN layer, built with
|
||||
# compress_ratio_override=0, so it owns no C4 indexer.
|
||||
and self.speculative_num_steps == 0
|
||||
and isinstance(metadata, DSV4Metadata)
|
||||
and metadata.indexer_metadata is not None
|
||||
@@ -1154,13 +1131,8 @@ class DeepseekV4HipRadixBackend(
|
||||
)
|
||||
|
||||
def _refresh_fp4_prefill_workspace(self, forward_batch: ForwardBatch) -> None:
|
||||
"""Rebuild the FP4 prefill schedule outside CUDA-graph capture.
|
||||
|
||||
AITER's prefill scheduler frees the scratch that its schedule kernel
|
||||
reads, so recording the build into a graph would leave every replay
|
||||
reading recycled graph-pool memory. Only the pinned buffers it fills
|
||||
(cta_info / logits / guarded page table) may be read from the graph.
|
||||
"""
|
||||
"""Must run outside graph capture: AITER's scheduler frees the scratch its
|
||||
kernel reads, so only the pinned buffers it fills may be read from a graph."""
|
||||
metadata = self.forward_metadata
|
||||
if not self._fp4_workspaces_enabled(metadata):
|
||||
return
|
||||
@@ -1337,7 +1309,7 @@ class DeepseekV4HipRadixBackend(
|
||||
seq_lens_cpu = forward_batch.seq_lens_cpu
|
||||
assert self.req_to_token_pool.req_to_token is self.req_to_token
|
||||
|
||||
assert self.swa_page_size % SWA_WINDOW == 0 and self.page_size % 128 == 0
|
||||
assert self.page_size % 128 == 0
|
||||
assert seq_lens_cpu is not None
|
||||
max_seq_len = (
|
||||
max_seq_len_override
|
||||
@@ -1499,7 +1471,7 @@ class DeepseekV4HipRadixBackend(
|
||||
core.c4_flashmla_metadata = _create_flashmla_metadata()
|
||||
core.c128_flashmla_metadata = _create_flashmla_metadata()
|
||||
|
||||
# PREP_IN_CUDA_GRAPH=True: warmup upgraded raw->full on the host;
|
||||
# Warmup upgraded raw->full on the host;
|
||||
# restore raw so capture re-runs the upgrade inside the graph.
|
||||
current_raw = getattr(self, "_current_capture_raw", None)
|
||||
if current_raw is not None:
|
||||
@@ -1532,8 +1504,7 @@ class DeepseekV4HipRadixBackend(
|
||||
hca_page_indices = core.c128_page_indices
|
||||
c4_page_indices = core.c4_sparse_page_indices
|
||||
# A draft pool carries no c4/c128 metadata, but its verify store still
|
||||
# reads the swa half below, so build these with an empty compressed tail
|
||||
# rather than skipping the call.
|
||||
# reads the swa half below, so build these with an empty compressed tail.
|
||||
if hca_len is None:
|
||||
hca_len = torch.zeros_like(swa_len)
|
||||
if csa_len is None:
|
||||
@@ -1596,9 +1567,8 @@ class DeepseekV4HipRadixBackend(
|
||||
req_slot * pool.unified_swa_ring_size
|
||||
+ core.positions_casual.to(torch.int64) % pool.unified_swa_ring_size
|
||||
).to(torch.int32)
|
||||
# Per-token req-slot map for the SWA ring store, read directly by the
|
||||
# forward store (target-verify) instead of recomputing a repeat_interleave
|
||||
# per layer. Harmless for plain decode (its store reads req_pool_indices).
|
||||
# Per-token req-slot map for the target-verify SWA ring store; plain
|
||||
# decode's store reads req_pool_indices instead.
|
||||
core.unified.verify_store_state_slot = state_slot
|
||||
|
||||
def _attach_unified_kv_prefill_meta(
|
||||
@@ -1673,14 +1643,8 @@ class DeepseekV4HipRadixBackend(
|
||||
q_rope: Optional[torch.Tensor] = None,
|
||||
k_rope: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
"""unified_kv paged-attention path over the unified_kv pool.
|
||||
|
||||
``q_rope`` is what tells the two layouts apart: present means ``q`` is a
|
||||
packed fp8 row and the pool is the two-pool fp8 one, so decode goes to
|
||||
the asm reader; absent means both are plain bf16 and it goes to Triton.
|
||||
Prefill needs ``k_rope`` alongside it, because there the current chunk is
|
||||
a KV source of its own and not just something to store.
|
||||
"""
|
||||
"""q_rope present: packed fp8 q over the two-pool fp8 layout (asm decode;
|
||||
prefill also needs k_rope). Absent: plain bf16 q and pool (Triton)."""
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime
|
||||
|
||||
pool = self.token_to_kv_pool
|
||||
@@ -1706,18 +1670,14 @@ class DeepseekV4HipRadixBackend(
|
||||
is_decode = forward_batch.forward_mode.is_decode_or_idle() or verify_as_decode
|
||||
if is_decode:
|
||||
if verify_as_decode:
|
||||
# Per-token (num_draft*bs -> bs) req-slot map, precomputed once
|
||||
# per step in _attach_unified_kv_decode_streams. Writing every
|
||||
# draft token's K into the ring is safe: spec_extra room prevents
|
||||
# clobbering the window history same-step tokens still read.
|
||||
# Writing every draft token's K into the ring is safe: spec_extra
|
||||
# room prevents clobbering the history same-step tokens still read.
|
||||
state_slot = core_attn_metadata.unified.verify_store_state_slot[:T]
|
||||
else:
|
||||
state_slot = forward_batch.req_pool_indices[:T]
|
||||
if save_kv_cache:
|
||||
# Only verify reaches this under fp8 -- plain decode's rows are
|
||||
# written by the fused kernel itself, which leaves kv None. The
|
||||
# pair arrives already packed, so this is the same scatter with
|
||||
# a second pool hanging off it.
|
||||
# Only verify reaches this under fp8; plain decode's rows are
|
||||
# written by the fused kernel itself, which leaves kv None.
|
||||
runtime.store_swa_into_unified(
|
||||
kv=kv,
|
||||
state_slot=state_slot,
|
||||
@@ -1752,10 +1712,8 @@ class DeepseekV4HipRadixBackend(
|
||||
else:
|
||||
raise ValueError(f"bad compress_ratio {compress_ratio}")
|
||||
if q_rope is not None:
|
||||
# softmax_scale is not passed on: the asm kernel hardcodes
|
||||
# 1/sqrt(512), which is what self.softmax_scale already is for
|
||||
# V4's head_dim=512. The other readers here take it explicitly,
|
||||
# so a head_dim change would leave only this one mis-scaled.
|
||||
# The asm kernel hardcodes softmax scale 1/sqrt(512) (V4 head_dim)
|
||||
# and takes no softmax_scale, unlike the other readers here.
|
||||
assert self.softmax_scale == 512**-0.5, (
|
||||
"the v4 nm asm kernel hardcodes 1/sqrt(512), this backend is "
|
||||
f"at {self.softmax_scale}"
|
||||
@@ -1812,20 +1770,12 @@ class DeepseekV4HipRadixBackend(
|
||||
cu_q = core_attn_metadata.unified.pf_cu_q
|
||||
final_pos = core_attn_metadata.unified.pf_final_pos
|
||||
|
||||
# DSA CP (round-robin/interleave): unified_pf_* are built over the GLOBAL
|
||||
# token layout, but under CP each rank owns only 1/cp_size of the queries
|
||||
# (q/positions are local) while kv was all-gathered to the full sequence.
|
||||
# Slice the per-query fields to this rank's tokens so their length matches
|
||||
# the local query count T; values stay global so each local query still
|
||||
# attends over the full all-gathered KV.
|
||||
# DSA CP: unified_pf_* cover the GLOBAL token layout while q is local and
|
||||
# kv all-gathered; slice per-query fields to this rank, values stay global.
|
||||
from sglang.srt.layers.attention.dsa.utils import (
|
||||
is_dsa_prefill_cp_round_robin_split,
|
||||
)
|
||||
|
||||
# NOTE (AMD/HIP only): this whole DSA-CP prefill handling lives in the
|
||||
# HIP backend (DeepseekV4HipRadixBackend, selected only when is_hip()).
|
||||
# The NVIDIA path uses DeepseekV4AttnBackend and never reaches here, so
|
||||
# these CP changes do not affect B200/H200 execution.
|
||||
_cp_size = get_parallel().attn_cp_size
|
||||
_cp_active = (
|
||||
_cp_size > 1
|
||||
@@ -1842,15 +1792,11 @@ class DeepseekV4HipRadixBackend(
|
||||
chunk_start = chunk_start[_sl].contiguous()
|
||||
cu_q = cu_q[_sl].contiguous()
|
||||
final_pos = final_pos[_sl].contiguous()
|
||||
# positions for the local queries are this rank's round-robin global
|
||||
# positions {r, r+cp, r+2cp, ...}; forward_batch.positions is the full
|
||||
# (padded) global layout, so slice it the same way instead of taking
|
||||
# the first T entries (which would be the wrong, sequential 0..T-1).
|
||||
# Local queries sit at this rank's round-robin global positions
|
||||
# {r, r+cp, r+2cp, ...}, not the first T entries of the global layout.
|
||||
positions = forward_batch.positions.to(torch.int64)[_sl].contiguous()
|
||||
# The SWA ring must hold the FULL window on EVERY rank (decode and
|
||||
# later chunks read this rank's ring). kv was all-gathered to the full
|
||||
# sequence, so write the full kv with full global positions/state_slot
|
||||
# instead of only this rank's 1/cp_size tokens.
|
||||
# Every rank's SWA ring must hold the full window (decode and later
|
||||
# chunks read it), so the store uses the full all-gathered kv.
|
||||
positions_full = forward_batch.positions.to(torch.int64)[
|
||||
: state_slot_full.shape[0]
|
||||
].contiguous()
|
||||
@@ -1877,10 +1823,8 @@ class DeepseekV4HipRadixBackend(
|
||||
"fp8 prefill needs the extend rope half beside the packed nope; "
|
||||
"q_rope came through but k_rope did not"
|
||||
)
|
||||
# No empty-segment mask on the result, unlike decode: this kernel
|
||||
# returns zeros for a token with neither region where the asm decode
|
||||
# reader leaves the row NaN. Chunk 0 tokens have an empty prefix and
|
||||
# a non-empty extend, which both readers handle.
|
||||
# No empty-segment mask, unlike decode: this kernel returns zeros for
|
||||
# a token with neither region where the asm decode reader leaves NaN.
|
||||
o = runtime.prefill_fp8_2buff(
|
||||
q=q,
|
||||
q_rope=q_rope,
|
||||
@@ -1912,9 +1856,6 @@ class DeepseekV4HipRadixBackend(
|
||||
# write this chunk's SWA K into the ring for future chunks / decode
|
||||
# only the final-window tokens per request
|
||||
if save_kv_cache:
|
||||
# Under CP, write the FULL all-gathered window so every rank's ring is
|
||||
# complete (decode / later chunks read the local ring). Without CP this
|
||||
# is just the local kv + local metadata as before.
|
||||
_ring_state_slot = state_slot_full if _cp_active else state_slot
|
||||
_ring_final_pos = final_pos_full if _cp_active else final_pos
|
||||
_ring_positions = positions_full if _cp_active else positions
|
||||
@@ -2148,8 +2089,6 @@ class DeepseekV4HipRadixBackend(
|
||||
need_compress: bool = True,
|
||||
is_prefill: bool = False,
|
||||
) -> DSV4AttnMetadata:
|
||||
assert self.swa_page_size == SWA_WINDOW
|
||||
|
||||
seq_lens_casual = seq_lens_casual.to(torch.int32)
|
||||
|
||||
swa_page_indices = self.get_swa_page_indices(
|
||||
|
||||
@@ -65,6 +65,42 @@ def get_compress_state_ring_size(compress_ratio: int, num_draft_tokens: int = 0)
|
||||
return 8 if compress_ratio == 4 else 128
|
||||
|
||||
|
||||
def resolve_compress_state_ring_size(compress_ratio: int) -> int:
|
||||
"""Ring size under the published spec config. The pool allocates by this
|
||||
and the configurator budgets by it, so both must call the same function."""
|
||||
spec = get_spec()
|
||||
num_draft_tokens = (
|
||||
0
|
||||
if spec.speculative_algorithm is None
|
||||
else spec.speculative_num_draft_tokens or 0
|
||||
)
|
||||
return get_compress_state_ring_size(compress_ratio, num_draft_tokens)
|
||||
|
||||
|
||||
def collect_sources_by_ratio(
|
||||
compression_ratios: Sequence[int], kv_source_layers: Sequence[int], stage: range
|
||||
) -> dict[int, List[int]]:
|
||||
"""Layers owning compressed storage: all of ratios 4/128, kv_sources of 1/2."""
|
||||
for idx in stage:
|
||||
ratio = compression_ratios[idx]
|
||||
if ratio not in (0, 1, 2, 4, 128):
|
||||
raise ValueError(f"Unsupported compression ratio: {ratio}")
|
||||
|
||||
sources_by_ratio: dict[int, List[int]] = {}
|
||||
for ratio in (4, 128, 1, 2):
|
||||
if ratio in (1, 2):
|
||||
layers = [
|
||||
l
|
||||
for l in kv_source_layers
|
||||
if l in stage and compression_ratios[l] == ratio
|
||||
]
|
||||
else:
|
||||
layers = [l for l in stage if compression_ratios[l] == ratio]
|
||||
if layers:
|
||||
sources_by_ratio[ratio] = layers
|
||||
return sources_by_ratio
|
||||
|
||||
|
||||
def get_compress_state_write_pad(compress_ratio: int, ring_size: int) -> int:
|
||||
"""Largest draft-token count this ring can serve; mirrors `mtp_pad` in
|
||||
`c_plan.cuh`, where the bound is derived."""
|
||||
@@ -79,11 +115,8 @@ def get_swa_ring_size(sliding_window: int, is_speculative: bool = False) -> int:
|
||||
|
||||
|
||||
def resolve_unified_kv_fp8(unified_fp8: Optional[bool] = None) -> bool:
|
||||
"""Per-pool fp8 layout. None follows SGLANG_DSV4_UNIFIED_KV_FP8.
|
||||
|
||||
A caller may pass False so this pool keeps the bf16 ring while the env
|
||||
stays on (target fused-Q still keys off the global switch).
|
||||
"""
|
||||
"""Per-pool fp8 layout; None follows SGLANG_DSV4_UNIFIED_KV_FP8. False keeps this
|
||||
pool bf16 while target fused-Q still keys off the global switch."""
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
|
||||
is_unified_kv_fp8,
|
||||
)
|
||||
@@ -309,11 +342,8 @@ class DeepSeekV4SingleKVPool(KVCache):
|
||||
|
||||
|
||||
class DeepSeekV4UniformFP8KVPool(DeepSeekV4SingleKVPool):
|
||||
"""Uniform 512-dim FP8 (e4m3) variant of the DSv4 single-KV pool.
|
||||
|
||||
Each token is 448 NoPE + 64 RoPE contiguous e4m3 values without in-cache
|
||||
scales or per-page padding. The backend supplies the dequant scale.
|
||||
"""
|
||||
"""Each token is 448 NoPE + 64 RoPE contiguous e4m3 values, without in-cache
|
||||
scales or per-page padding; the backend supplies the dequant scale."""
|
||||
|
||||
def get_bytes_per_token(self) -> int:
|
||||
return self.qk_nope_head_dim + self.qk_rope_head_dim
|
||||
@@ -355,10 +385,8 @@ class DeepSeekV4UniformFP8KVPool(DeepSeekV4SingleKVPool):
|
||||
cache_k: torch.Tensor,
|
||||
freqs_cis: Optional[torch.Tensor] = None,
|
||||
) -> None:
|
||||
"""Store normed/roped rows as e4m3 with the backend's fixed unit scale.
|
||||
|
||||
uint8 views work around index_put not supporting FP8 dtypes.
|
||||
"""
|
||||
"""Store normed/roped rows as e4m3 with the backend's fixed unit scale;
|
||||
uint8 views work around index_put not supporting FP8 dtypes."""
|
||||
|
||||
assert freqs_cis is None, "the uniform-FP8 pool takes finished (rotated) rows"
|
||||
assert cache_k.dim() == 2 and cache_k.shape[1] == self.kv_cache_total_dim
|
||||
@@ -572,11 +600,8 @@ class DeepSeekV4IndexerPool(KVCache):
|
||||
return self.index_k_with_scale_buffer[layer_id]
|
||||
|
||||
def contiguous_page_row_buffers(self) -> List[torch.Tensor]:
|
||||
"""Every indexer buffer as 2D page rows, for PD and HiCache transfer.
|
||||
|
||||
FP8 keeps key and scale fused in one buffer per layer; the FP4 layout
|
||||
stores payload and scale separately, so it yields two buffers per layer.
|
||||
"""
|
||||
"""Every indexer buffer as 2D page rows for PD and HiCache: one fused key+scale
|
||||
buffer per layer under FP8, separate payload and scale buffers under FP4."""
|
||||
if self.index_k_with_scale_buffer is not None:
|
||||
return self.index_k_with_scale_buffer
|
||||
return [
|
||||
@@ -744,9 +769,7 @@ class DeepSeekV4UnifiedKVPool:
|
||||
unified_kv[L]: ``[swa_pages + padded_compress_rows, head_dim]`` bf16
|
||||
|
||||
Layout (fp8, ``SGLANG_DSV4_UNIFIED_KV_FP8``) -- two parallel pools with the
|
||||
same row count, so a row index means the same thing in both. Named after the
|
||||
accessors, which under fp8 each return one half -- ``get_unified_kv`` the
|
||||
nope, ``get_unified_kv_rope`` the rope:
|
||||
same row count, so a row index addresses both:
|
||||
unified_kv[L] (nope): ``[rows, 512]`` fp8, see DSV4_FP8_NOPE_ROW_BYTES
|
||||
unified_kv_rope[L] (rope): ``[rows, qk_rope_head_dim]`` bf16, never quantized
|
||||
|
||||
@@ -835,9 +858,7 @@ class DeepSeekV4UnifiedKVPool:
|
||||
|
||||
def get_buf_infos(self) -> Tuple[List[int], List[int], List[int]]:
|
||||
if self.fp8:
|
||||
# same single-pool assumption as the outer get_contiguous_buf_infos:
|
||||
# one pointer and one row size per layer describes the nope pool only,
|
||||
# so whoever picks this up next would move half a row and not notice.
|
||||
# One pointer and row size per layer would describe only the nope pool.
|
||||
# TODO(danli103): report both pools once a consumer needs them.
|
||||
raise NotImplementedError(
|
||||
"get_buf_infos describes one pool per layer; the fp8 rope pool "
|
||||
@@ -959,9 +980,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
c128_state_pool_size = max(
|
||||
c128_state_pool_size, self.num_req_slots * c128_ring_size
|
||||
)
|
||||
# Only the ratios the model has anywhere get a pool config: the backend, PD
|
||||
# state transfer and HiCache read the registries as "the ratios this model
|
||||
# has", and a PP stage missing one keeps its empty pool so the PD wire aligns.
|
||||
# Consumers read the registries as "the ratios this model has"; a PP stage
|
||||
# missing one keeps its empty pool so the PD wire aligns.
|
||||
model_ratios = set(compression_ratios)
|
||||
self.compressed_pool_configs = {
|
||||
ratio: config
|
||||
@@ -1007,10 +1027,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
|
||||
self.swa_size = swa_size
|
||||
self.swa_page_size = swa_page_size
|
||||
# The allocator and compress state keep 256-token logical pages, while
|
||||
# FlashInfer's SM120 DSV4 kernel consumes 64-token physical pages. Storing
|
||||
# SWA KV in that layout removes the per-layer 256 -> 64 page split; the
|
||||
# allocator's flat token indices stay valid.
|
||||
# The allocator keeps 256-token logical pages; FlashInfer's SM120 DSV4 kernel
|
||||
# reads 64-token physical pages, and flat token indices are valid in both.
|
||||
swa_kv_page_size = (
|
||||
64
|
||||
if get_platform().is_sm120 and envs.SGLANG_OPT_SM120_DIRECT_SWA_KV.get()
|
||||
@@ -1127,7 +1145,11 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
)
|
||||
self.full_size = full_size
|
||||
self.kv_source_layers = list(kv_source_layers)
|
||||
self.sources_by_ratio = self._collect_sources_by_ratio()
|
||||
self.sources_by_ratio = collect_sources_by_ratio(
|
||||
self.compression_ratios,
|
||||
self.kv_source_layers,
|
||||
range(self._stage_start, self._stage_end),
|
||||
)
|
||||
self._init_compressed_pools(
|
||||
stage_ratios=stage_ratios,
|
||||
page_size=page_size,
|
||||
@@ -1163,13 +1185,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
self.full_to_swa_index_mapping = full_to_swa_index_mapping
|
||||
|
||||
def get_ring_size(self, compress_ratio: int) -> int:
|
||||
spec = get_spec()
|
||||
num_draft_tokens = (
|
||||
0
|
||||
if spec.speculative_algorithm is None
|
||||
else spec.speculative_num_draft_tokens or 0
|
||||
)
|
||||
return get_compress_state_ring_size(compress_ratio, num_draft_tokens)
|
||||
return resolve_compress_state_ring_size(compress_ratio)
|
||||
|
||||
def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor):
|
||||
assert self.full_to_swa_index_mapping is not None
|
||||
@@ -1181,10 +1197,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
item_lens: List[int] = []
|
||||
|
||||
if self._unified_kv_fp8:
|
||||
# The page-block transfer below prices one row as buf[0].nbytes and
|
||||
# ships a single pointer per layer. Under fp8 that covers the nope
|
||||
# pool only -- the parallel bf16 rope pool would be dropped and the
|
||||
# remote side would decode rows against stale rope. Refuse instead.
|
||||
# One pointer and one buf[0]-sized row per layer covers only the fp8 nope
|
||||
# pool; the remote side would decode rows against stale rope.
|
||||
# TODO(danli103): ship the rope pool as a second per-layer entry.
|
||||
raise NotImplementedError(
|
||||
"PD disaggregation is not supported with "
|
||||
@@ -1224,9 +1238,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
indexer_pool = self.index_pools.get(ratio)
|
||||
if indexer_pool is None:
|
||||
continue
|
||||
# The transfer addresses every buffer by FULL page id; ratio-1/2 index
|
||||
# pools page at DSV41_INDEX_PAGE_SIZE, so one item is the run of index
|
||||
# pages holding a FULL page's page_size // ratio slots.
|
||||
# The transfer addresses buffers by FULL page id; one ratio-1/2 item is the
|
||||
# run of DSV41_INDEX_PAGE_SIZE pages holding page_size // ratio slots.
|
||||
index_pages_per_full_page = 1
|
||||
if ratio in (1, 2):
|
||||
slots_per_full_page = self.page_size // ratio
|
||||
@@ -1255,9 +1268,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
if not self._unified_kv:
|
||||
return data_ptrs, data_lens, item_lens
|
||||
if self._unified_kv_fp8:
|
||||
# Other half of the PD path -- get_contiguous_buf_infos ships the
|
||||
# compressed region, this one the ring. Same single-pool assumption,
|
||||
# same silently dropped rope, same fix -- land them together.
|
||||
# Ring half of the fp8 gap in get_contiguous_buf_infos; fix both together.
|
||||
raise NotImplementedError(
|
||||
"PD disaggregation is not supported with "
|
||||
"SGLANG_DSV4_UNIFIED_KV_FP8=1 (the SWA_RING component assumes a "
|
||||
@@ -1304,9 +1315,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
return views, item_bytes
|
||||
|
||||
def unified_region_buffers(self, ratio: int) -> Tuple[List[torch.Tensor], int]:
|
||||
"""
|
||||
Main compressed region of one stage: bf16 latents, or fp8 nope.
|
||||
"""
|
||||
"""Main compressed region of one stage: bf16 latents, or fp8 nope."""
|
||||
assert self._unified_kv, "unified_region_buffers requires unified_kv layout"
|
||||
assert ratio in (4, 128), f"unsupported compression ratio: {ratio}"
|
||||
return self._unified_page_views(self.unified_kv_pool.kv_buffer, ratio)
|
||||
@@ -1314,14 +1323,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
def unified_rope_region_buffers(
|
||||
self, ratio: int
|
||||
) -> Optional[Tuple[List[torch.Tensor], int]]:
|
||||
"""
|
||||
The bf16 rope half of an fp8 two-pool row, or None when there isn't one.
|
||||
|
||||
A row index addresses both pools, so this mirrors exactly the rows
|
||||
``unified_region_buffers`` does and only the row width differs. It needs
|
||||
its own host pool: offloading the nope half alone leaves whatever rope the
|
||||
row held before, which is wrong output rather than a crash.
|
||||
"""
|
||||
"""The bf16 rope half of an fp8 two-pool row, or None. It needs its own host
|
||||
pool: offloading only the nope half leaves stale rope, i.e. wrong output."""
|
||||
if not self._unified_kv_fp8:
|
||||
return None
|
||||
assert self._unified_kv, "unified_rope_region_buffers requires unified_kv"
|
||||
@@ -1567,7 +1570,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
enable_memory_saver=enable_memory_saver,
|
||||
ratio=ratio,
|
||||
online=(ratio == 128 and ONLINE_C128),
|
||||
request_scoped=ratio in (2, 128),
|
||||
# Ratio 2 never reaches this factory (_make_pair_state_pool).
|
||||
request_scoped=ratio == 128,
|
||||
swa_page_size=self.swa_page_size,
|
||||
online_mtp_max_draft_tokens=(
|
||||
self.online_mtp_max_draft_tokens if ratio == 128 else 0
|
||||
@@ -1624,28 +1628,6 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
enable_memory_saver=enable_memory_saver,
|
||||
)
|
||||
|
||||
def _collect_sources_by_ratio(self) -> dict[int, List[int]]:
|
||||
"""Layers owning compressed storage: all of ratios 4/128, kv_sources of 1/2."""
|
||||
stage = range(self._stage_start, self._stage_end)
|
||||
for idx in stage:
|
||||
ratio = self.compression_ratios[idx]
|
||||
if ratio not in (0, 1, 2, 4, 128):
|
||||
raise ValueError(f"Unsupported compression ratio: {ratio}")
|
||||
|
||||
sources_by_ratio: dict[int, List[int]] = {}
|
||||
for ratio in (4, 128, 1, 2):
|
||||
if ratio in (1, 2):
|
||||
layers = [
|
||||
l
|
||||
for l in self.kv_source_layers
|
||||
if l in stage and self.compression_ratios[l] == ratio
|
||||
]
|
||||
else:
|
||||
layers = [l for l in stage if self.compression_ratios[l] == ratio]
|
||||
if layers:
|
||||
sources_by_ratio[ratio] = layers
|
||||
return sources_by_ratio
|
||||
|
||||
def source_layer_of(self, layer_id: int) -> int:
|
||||
"""The layer owning this layer's compressed storage: itself for ratios 4/128,
|
||||
the nearest preceding kv_source layer for ratios 1/2."""
|
||||
@@ -2008,10 +1990,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
if self.uniform_fp8:
|
||||
# Uniform-FP8 (trtllm-gen) layout: norm + RoPE with the existing
|
||||
# Triton kernel (in-place on kv; safe -- kv is not read again),
|
||||
# then a plain e4m3 cast + scatter in the pool setter (per-tensor
|
||||
# scale 1.0). Fusing the store is deferred to the perf phase.
|
||||
# Uniform-FP8 (trtllm-gen): in-place norm + RoPE (kv is not read again),
|
||||
# then an e4m3 cast + scatter with per-tensor scale 1.0.
|
||||
from sglang.kernels.ops.attention.deepseek_v4_rope import (
|
||||
fused_norm_rope_inplace_triton,
|
||||
)
|
||||
@@ -2049,14 +2029,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
freqs_cis: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
) -> None:
|
||||
"""unified_kv counterpart of set_swa_key_buffer_radix_fused_norm_rope.
|
||||
|
||||
Under unified_kv the (fp8, paged) swa_kv_pool is None -- SWA K lives in
|
||||
the shared bf16 unified_kv ring instead. Norm+RoPE the draft KV in place
|
||||
(the same freqs_cis path the main model uses via _compute_kv_bf16) and
|
||||
scatter it into ``unified_kv[swa_loc]``. Rows with swa_loc < 0
|
||||
(uncommitted verify tokens) are skipped by the scatter.
|
||||
"""
|
||||
"""unified_kv counterpart of set_swa_key_buffer_radix_fused_norm_rope; rows
|
||||
with swa_loc < 0 (uncommitted verify tokens) are skipped by the scatter."""
|
||||
from sglang.kernels.ops.attention.dsv4 import fused_norm_rope_inplace
|
||||
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime
|
||||
|
||||
|
||||
@@ -4,11 +4,6 @@ Each model architecture has its own configurator that computes pool sizes
|
||||
from available GPU memory using a unified coeff+bias model:
|
||||
|
||||
available_bytes = max_tokens * coeff + bias
|
||||
max_tokens = (available_bytes - bias) / coeff
|
||||
|
||||
Two entry points, same core computation:
|
||||
- calculate_pool_sizes(available_bytes, page_size): profiling path
|
||||
- calculate_pool_sizes_from_max_tokens(max_tokens, page_size): constraint path
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -36,10 +31,11 @@ from sglang.srt.configs.model_config import (
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.allocation_sizing import get_alloc_len_per_decode
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import (
|
||||
get_compress_state_ring_size,
|
||||
collect_sources_by_ratio,
|
||||
get_compress_state_write_pad,
|
||||
get_dsv4_indexer_bytes_per_token,
|
||||
get_swa_ring_size,
|
||||
resolve_compress_state_ring_size,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
DSATokenToKVPool,
|
||||
@@ -85,11 +81,8 @@ class MemoryPoolConfig:
|
||||
|
||||
mem_fraction_static: Optional[float] = None
|
||||
|
||||
# Unified pool only: the PROFILED byte budget for the token-granular
|
||||
# sub-pools. Set, the factories size the buffer from it directly instead of
|
||||
# re-summing ratio-derived token counts, which keeps the re-sum's floor
|
||||
# losses out of the buffer; the token counts stay boot labels / conserve
|
||||
# caps. None on the token-capped path -- a user token cap IS the budget.
|
||||
# Unified pool only: the profiled byte budget the buffer is sized from,
|
||||
# instead of re-summing token counts; None when a user token cap is the budget.
|
||||
unified_total_bytes: Optional[int] = None
|
||||
|
||||
def __post_init__(self):
|
||||
@@ -107,14 +100,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _dflash_draft_cell_size(kvc: KVCacheConfigurator) -> int:
|
||||
"""Bytes/token the DFLASH draft KV pool adds to the target's budget, 0 if none.
|
||||
|
||||
Unlike an EAGLE draft, which reuses the target's attention config and is
|
||||
therefore priced by layer count, a DFLASH draft has its own geometry and is
|
||||
a flat additive term. Under DCP, the target pool is sharded while the draft
|
||||
pool spans the allocator's widened virtual location space, so the draft
|
||||
term is replicated across DCP ranks.
|
||||
"""
|
||||
"""Bytes/token the DFLASH draft KV pool adds, 0 if none; replicated across DCP
|
||||
ranks because the draft pool spans the widened virtual location space."""
|
||||
if kvc.is_draft_worker or not kvc.spec_algorithm.is_dflash_family():
|
||||
return 0
|
||||
cell_size = kvc.spec_aux_config.dflash_draft_cell_size_per_token
|
||||
@@ -200,11 +187,7 @@ class MemoryPoolConfigurator:
|
||||
|
||||
|
||||
class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
"""Configurator for standard models: MHA, MLA, DSA, FP4.
|
||||
|
||||
coeff = cell_size (bytes per token across all layers)
|
||||
bias = 0
|
||||
"""
|
||||
"""Standard models (MHA, MLA, DSA, FP4): coeff = cell_size, bias = 0."""
|
||||
|
||||
def __init__(self, kvc: KVCacheConfigurator):
|
||||
self.kv_cache_dtype_str = kvc.kv_cache_dtype_str
|
||||
@@ -232,10 +215,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
else get_schedule().max_total_tokens or kvc.model_config.context_len
|
||||
)
|
||||
|
||||
# EAGLE/STANDALONE: scale cell_size to account for draft model KV cache.
|
||||
# Assumes draft and target share the same per-layer KV size (head_dim,
|
||||
# num_kv_heads, dtype), which holds for EAGLE/MTP draft models that
|
||||
# reuse the target architecture's attention config.
|
||||
# EAGLE/STANDALONE: assumes the draft shares the target's per-layer KV size,
|
||||
# which holds for EAGLE/MTP drafts that reuse the target's attention config.
|
||||
if (
|
||||
kvc.spec_algorithm.is_eagle() or kvc.spec_algorithm.is_standalone()
|
||||
) and not kvc.is_draft_worker:
|
||||
@@ -273,11 +254,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
self._cell_size * (1 + draft_num_layers / int(num_layers))
|
||||
)
|
||||
|
||||
# DFLASH/DSPARK: reserve the draft runner's *actual* per-token KV cost.
|
||||
# The draft allocates its own KV pool at the target's
|
||||
# max_total_num_tokens, whose per-token footprint can differ from the
|
||||
# target's (e.g. an MLA-latent target paired with a full per-head K/V
|
||||
# draft), so size from the draft config rather than the layer ratio.
|
||||
# DFLASH/DSPARK: the draft's per-token KV cost can differ from the target's
|
||||
# (e.g. MLA target, per-head K/V draft), so size from the draft config.
|
||||
if kvc.spec_algorithm.is_dflash_family() and not kvc.is_draft_worker:
|
||||
from sglang.srt.speculative.dflash_utils import (
|
||||
scale_kv_cell_size_per_token_for_dflash,
|
||||
@@ -298,7 +276,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
)
|
||||
|
||||
def _compute_cell_size(self, kvc: KVCacheConfigurator, num_layers: int) -> int:
|
||||
"""Compute per-token KV cache cost in bytes. Subclasses can override."""
|
||||
"""Compute per-token KV cache cost in bytes."""
|
||||
# args to config cell size
|
||||
model_config = kvc.model_config
|
||||
kv_cache_dtype = kvc.kv_cache_dtype
|
||||
@@ -540,9 +518,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
|
||||
_, shard_size = get_glm_dsa_cp_layer_shard_info(kvc)
|
||||
if shard_size > 1:
|
||||
# Preserve the existing LayerSplit sizing semantics. GLM-5.3
|
||||
# hybrid-layer support is intentionally limited to the normal
|
||||
# (non-LayerSplit) pool below.
|
||||
# GLM-5.3 hybrid-layer support is intentionally limited to the
|
||||
# non-LayerSplit pool below.
|
||||
active_indexer_layers = [
|
||||
layer_id
|
||||
for layer_id in range(
|
||||
@@ -592,11 +569,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
|
||||
|
||||
class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
|
||||
"""Configurator for MHA or MLA models with sliding-window layers.
|
||||
|
||||
Splits available memory between full attention and SWA pools.
|
||||
Does NOT inherit DefaultPoolConfigurator — different coeff model.
|
||||
"""
|
||||
"""Splits memory between the full and SWA pools of MHA/MLA sliding-window models."""
|
||||
|
||||
def __init__(self, kvc: KVCacheConfigurator):
|
||||
self.kv_cache_dtype_str = kvc.kv_cache_dtype_str
|
||||
@@ -704,15 +677,8 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
|
||||
self._recompute_cell_size()
|
||||
|
||||
def _recompute_cell_size(self) -> None:
|
||||
# Bytes per token of max_total_num_tokens.
|
||||
#
|
||||
# Hybrid (full_layers > 0): max_total = full_tokens, so cell_size accounts
|
||||
# for both pools: F*nf + r*S*ns (where swa_tokens = full_tokens * r).
|
||||
#
|
||||
# All-SWA (full_layers == 0): max_total = swa_tokens directly. The ratio
|
||||
# is meaningless here -- there is no full pool to relate to, and every
|
||||
# token beyond the sliding window can be evicted. So cell_size = S*ns,
|
||||
# with no ratio factor applied.
|
||||
# Bytes per token of max_total_num_tokens: full_tokens when hybrid, else
|
||||
# swa_tokens with no ratio, since all-SWA has no full pool to relate to.
|
||||
if self._full_layers_num == 0:
|
||||
self._cell_size = (
|
||||
self._swa_per_token * self._swa_layers_num
|
||||
@@ -781,14 +747,12 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
|
||||
def _solve_pool_sizes(
|
||||
self, max_total_num_tokens: int, page_size: int
|
||||
) -> MemoryPoolConfig:
|
||||
"""Core computation: split max_total_num_tokens into full/swa pool sizes."""
|
||||
|
||||
def align_page_size(x: int) -> int:
|
||||
return (x // page_size) * page_size
|
||||
|
||||
if self._full_layers_num == 0:
|
||||
# All-SWA: no full pool, max_total = actual SWA pool size.
|
||||
# Ratio is not applied -- see __init__ comment.
|
||||
# Ratio is not applied -- see _recompute_cell_size.
|
||||
swa_tokens = align_page_size(max_total_num_tokens)
|
||||
logger.info(
|
||||
f"Use sliding window memory pool (all SWA). "
|
||||
@@ -885,13 +849,8 @@ def compute_swa_request_cap(*, page_size: int, window: int, attn_dp_size: int) -
|
||||
|
||||
|
||||
class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator):
|
||||
"""Hybrid SWA configurator with the SWA pool sized from a fixed token cap.
|
||||
|
||||
When max_running_requests is explicit, the SWA pool's worst-case
|
||||
footprint is bounded per request. The SWA pool is sized tightly from that
|
||||
cap and the freed memory is redirected to the full pool, instead of sizing
|
||||
both pools by swa_full_tokens_ratio.
|
||||
"""
|
||||
"""Hybrid SWA with the SWA pool sized from the explicit max_running_requests
|
||||
worst case instead of swa_full_tokens_ratio; the rest goes to the full pool."""
|
||||
|
||||
def __init__(self, kvc: KVCacheConfigurator):
|
||||
self.kv_cache_dtype_str = kvc.kv_cache_dtype_str
|
||||
@@ -984,13 +943,8 @@ class _DSV4PoolSizes:
|
||||
|
||||
|
||||
class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
"""Configurator for DSV4 compressed-attention models.
|
||||
|
||||
Splits available memory across full / swa / c4 / c128 + c4_state / c128_state
|
||||
pools. coeff is bytes_per_full_token (inflated by (T+D)/T when speculative
|
||||
decode reserves a draft worker, mirroring dflash's cell_size scaling). bias
|
||||
is the request-scoped fixed pools that do not scale with full_token.
|
||||
"""
|
||||
"""DSV4 compressed attention: coeff is bytes_per_full_token, inflated by (T+D)/T
|
||||
for a draft worker; bias is the request-scoped pools that do not scale with it."""
|
||||
|
||||
# object.__new__ stubs (SWA floor tests) skip __init__
|
||||
_dspark_draft_on_bf16 = False
|
||||
@@ -1038,17 +992,19 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
)
|
||||
self.context_len = kvc.model_config.context_len
|
||||
# PP-local slice; matches DeepSeekV4TokenToKVPool's stage_ratios.
|
||||
self.compression_ratios = cfg.compress_ratios[
|
||||
kvc.layer_info.start_layer : kvc.layer_info.end_layer
|
||||
]
|
||||
stage = range(kvc.layer_info.start_layer, kvc.layer_info.end_layer)
|
||||
self.stage_compress_ratios = cfg.compress_ratios[stage.start : stage.stop]
|
||||
if kvc.pp_size > 1:
|
||||
logger.info(
|
||||
f"DSV4 pool PP slice: rank={kvc.pp_group.rank_in_group} "
|
||||
f"layers=[{kvc.layer_info.start_layer},{kvc.layer_info.end_layer}) "
|
||||
f"local={len(self.compression_ratios)}/{len(cfg.compress_ratios)}"
|
||||
f"layers=[{stage.start},{stage.stop}) "
|
||||
f"local={len(self.stage_compress_ratios)}/{len(cfg.compress_ratios)}"
|
||||
)
|
||||
self.swa_window_size = cfg.window_size
|
||||
self.swa_page_size = get_schedule().page_size
|
||||
# Layers of this stage that own compressed storage, per ratio. Same rule
|
||||
# as the pool, so a ratio budgeted here is one it allocates.
|
||||
self.stage_owner_layers = collect_sources_by_ratio(
|
||||
cfg.compress_ratios, cfg.hf_config.kv_source_layer_ids, stage
|
||||
)
|
||||
self.operator_swa_ratio = _operator_swa_full_tokens_ratio()
|
||||
self.swa_ratio = (
|
||||
self.operator_swa_ratio
|
||||
@@ -1079,49 +1035,26 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
if self.c4_shrink_factor > 1:
|
||||
logger.info(f"HiSparse c4 host-to-device ratio = {self.c4_shrink_factor}")
|
||||
|
||||
# Same argument as the pool's get_ring_size, so both sides agree.
|
||||
num_draft_tokens = (
|
||||
(get_spec().speculative_num_draft_tokens or 0) if self.is_speculative else 0
|
||||
)
|
||||
self.c4_ring_size = get_compress_state_ring_size(4, num_draft_tokens)
|
||||
self.c128_ring_size = get_compress_state_ring_size(128, num_draft_tokens)
|
||||
self.c2_ring_size = get_compress_state_ring_size(2, num_draft_tokens)
|
||||
# Ratio 1 keeps no compress state, so it has no ring.
|
||||
self.ring_sizes = {
|
||||
ratio: resolve_compress_state_ring_size(ratio)
|
||||
for ratio in self.stage_owner_layers
|
||||
if ratio != 1
|
||||
}
|
||||
|
||||
self.num_layers_total = len(self.compression_ratios)
|
||||
self.num_layers_ca4 = sum(1 for r in self.compression_ratios if r == 4)
|
||||
self.num_layers_ca128 = sum(1 for r in self.compression_ratios if r == 128)
|
||||
# Only a ratio-2 kv_source layer compresses, so only it keeps pair state.
|
||||
self.num_layers_c2_source = sum(
|
||||
1
|
||||
for l in cfg.hf_config.kv_source_layer_ids
|
||||
if kvc.layer_info.start_layer <= l < kvc.layer_info.end_layer
|
||||
and cfg.compress_ratios[l] == 2
|
||||
)
|
||||
self.num_layers_total = len(self.stage_compress_ratios)
|
||||
# The low-ratio indexer pools are built with force_fp4=True
|
||||
# (deepseek_v4_memory_pool), so they are fp4 whatever dtype c4 uses.
|
||||
low_ratio_index_bytes = get_dsv4_indexer_bytes_per_token(
|
||||
self.low_ratio_index_bytes = get_dsv4_indexer_bytes_per_token(
|
||||
self.indexer_head_dim, use_fp4_indexer=True
|
||||
)
|
||||
self.low_ratio_bytes_per_full_token = sum(
|
||||
(self.kv_bytes + low_ratio_index_bytes) / cfg.compress_ratios[l]
|
||||
for l in cfg.hf_config.kv_source_layer_ids
|
||||
if kvc.layer_info.start_layer <= l < kvc.layer_info.end_layer
|
||||
and cfg.compress_ratios[l] in (1, 2)
|
||||
)
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import (
|
||||
dsv4_unified_row_bytes,
|
||||
)
|
||||
|
||||
# swa_window_size is the model's sliding window (cfg.window_size).
|
||||
self._swa_ring_size = get_swa_ring_size(
|
||||
self.swa_window_size, self.is_speculative
|
||||
)
|
||||
# kvc.sliding_window_size is None on a runner without SWA layers.
|
||||
self._swa_ring_size = get_swa_ring_size(cfg.window_size, self.is_speculative)
|
||||
self._spec_infl = 1.0
|
||||
|
||||
# The unified pool takes no dtype, so --kv-cache-dtype never reaches it.
|
||||
# V4 defaults "auto" to fp8_e4m3 (overrides.py
|
||||
# _deepseek_v4_kv_cache_dtype), so only a bfloat16 here tells us the user
|
||||
# set it explicitly; warning on the fp8 side would fire on every run.
|
||||
# The unified pool ignores --kv-cache-dtype; V4 resolves "auto" to fp8_e4m3
|
||||
# (overrides.py _deepseek_v4_kv_cache_dtype), so only bfloat16 is explicit.
|
||||
if self._unified_fp8 and self.kv_cache_dtype_str == "bfloat16":
|
||||
logger.warning(
|
||||
"--kv-cache-dtype=bfloat16 is ignored on the unified_kv path; "
|
||||
@@ -1129,9 +1062,8 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
"env switch to get a bf16 unified pool."
|
||||
)
|
||||
|
||||
# get_contiguous_buf_infos ships one pointer per layer and prices a row as
|
||||
# buf[0].nbytes, which under fp8 covers the nope pool only. Fail at startup
|
||||
# rather than at the first transfer.
|
||||
# get_contiguous_buf_infos prices a row as buf[0].nbytes, which under fp8
|
||||
# covers the nope pool only; fail at startup rather than at the first transfer.
|
||||
# TODO(danli103): drop this once the transfer ships the rope pool.
|
||||
if self._unified_fp8 and self.disaggregation_mode != "null":
|
||||
raise ValueError(
|
||||
@@ -1185,20 +1117,16 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
self.bytes_per_swa_token = self._get_bytes_per_swa_token()
|
||||
self.bytes_per_full_token = self._get_bytes_per_full_token()
|
||||
if self.is_speculative and not self.encoder_replay:
|
||||
# Reserve memory for the speculative draft worker by inflating
|
||||
# per-token bytes by (target+draft)/target. Equivalent to dflash's
|
||||
# scale_kv_cell_size_per_token_for_dflash but applied to
|
||||
# bytes_per_full_token: tokens = avail / (bpft * (T+D)/T).
|
||||
# Reserve the draft worker by inflating per-token bytes by
|
||||
# (target+draft)/target, as scale_kv_cell_size_per_token_for_dflash does.
|
||||
draft_layers = 1
|
||||
target_layers = self.num_layers_total
|
||||
self._spec_infl = (target_layers + draft_layers) / target_layers
|
||||
self.bytes_per_full_token *= self._spec_infl
|
||||
self.bytes_per_swa_token *= self._spec_infl
|
||||
|
||||
# Online c128 keeps a single in-progress (max, sum, kv) state per index
|
||||
# and assumes a strict forward-only schedule. Speculative decode (MTP)
|
||||
# would need rollback / replay across draft and verify, which the
|
||||
# online path doesn't support yet.
|
||||
# Online c128 keeps one in-progress (max, sum, kv) state per index and
|
||||
# assumes forward-only; MTP would need rollback across draft and verify.
|
||||
if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
|
||||
allow_experimental_online_c128_mtp = (
|
||||
envs.SGLANG_EXPERIMENTAL_ONLINE_C128_MTP.get()
|
||||
@@ -1225,18 +1153,20 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
"DSV4 compressed attention: online c128 enabled (ring_size=1)"
|
||||
)
|
||||
|
||||
def num_layers(self, ratio: int) -> int:
|
||||
"""Layers of this stage owning ratio's compressed storage."""
|
||||
return len(self.stage_owner_layers.get(ratio, ()))
|
||||
|
||||
def _assert_ring_serves_draft_tokens(self, num_draft_tokens: int) -> None:
|
||||
"""A verify batch writes its whole optimistic tail into the ring, so ring
|
||||
capacity bounds the draft count."""
|
||||
for compress_ratio, ring_size, num_layers in (
|
||||
(4, self.c4_ring_size, self.num_layers_ca4),
|
||||
(128, self.c128_ring_size, self.num_layers_ca128),
|
||||
):
|
||||
if num_layers == 0:
|
||||
for compress_ratio in (4, 128):
|
||||
if self.num_layers(compress_ratio) == 0:
|
||||
continue
|
||||
if compress_ratio == 128 and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
|
||||
# Online c128 keeps per-draft state instead of a ring; sized separately.
|
||||
continue
|
||||
ring_size = self.ring_sizes[compress_ratio]
|
||||
max_draft_tokens = get_compress_state_write_pad(compress_ratio, ring_size)
|
||||
assert num_draft_tokens <= max_draft_tokens, (
|
||||
f"speculative_num_draft_tokens={num_draft_tokens} exceeds what the c{compress_ratio} "
|
||||
@@ -1284,7 +1214,7 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
|
||||
def _get_bytes_per_swa_token(self) -> float:
|
||||
"""Bytes one SWA slot costs across the stage. c4_state_pool_size = swa_tokens
|
||||
/ swa_page_size * ring, so c4 compress state is priced per SWA slot too."""
|
||||
// page_size * ring, so c4 compress state is priced per SWA slot too."""
|
||||
if self.encoder_replay:
|
||||
# Target SWA lives in the request window; only the draft owns paged SWA
|
||||
# bytes, and its layers carry no compressed state.
|
||||
@@ -1293,46 +1223,35 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
c4_state_bytes = 2 * 2 * self.attn_head_dim * c4_state_dtype_size
|
||||
c4_indexer_state_bytes = 2 * 2 * self.indexer_head_dim * c4_state_dtype_size
|
||||
|
||||
c4_state_ratio = self.c4_ring_size / self.swa_page_size
|
||||
return (
|
||||
self.kv_bytes * self.num_layers_total
|
||||
+ c4_state_ratio
|
||||
* (c4_state_bytes + c4_indexer_state_bytes)
|
||||
* self.num_layers_ca4
|
||||
)
|
||||
c4_state_ratio = self.ring_sizes.get(4, 0) / self.page_size
|
||||
return self.kv_bytes * self.num_layers_total + c4_state_ratio * (
|
||||
c4_state_bytes + c4_indexer_state_bytes
|
||||
) * self.num_layers(4)
|
||||
|
||||
def _compressed_bytes_per_full_token(self, ratio: int) -> float:
|
||||
"""Compressed KV (+ indexer) bytes one full token adds per layer of `ratio`;
|
||||
compress state is priced per SWA slot (c4) or as fixed bytes (c128, ratio 2)."""
|
||||
if ratio in (1, 2):
|
||||
return (self.kv_bytes + self.low_ratio_index_bytes) / ratio
|
||||
if ratio == 4:
|
||||
c4_frac = 1 / (4 * self.c4_shrink_factor)
|
||||
return c4_frac * self.kv_bytes + 1 / 4 * self.indexer_bytes_per_token
|
||||
assert ratio == 128, f"unsupported compression ratio: {ratio}"
|
||||
return 1 / 128 * self.kv_bytes
|
||||
|
||||
def _get_bytes_per_full_token(self) -> float:
|
||||
_, c128_state_dtype_size = _get_dsv4_compress_state_dtype_sizes()
|
||||
# Online c128 stores (max, sum, kv) per slot (3*head_dim) instead of
|
||||
# raw (kv, score) (2*head_dim). Combined with ring_size=1 this still
|
||||
# nets a large reduction (~3/256x) but the per-slot bytes go up.
|
||||
c128_online = envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get()
|
||||
c128_state_bytes = (
|
||||
(3 if c128_online else 2 * 1) * self.attn_head_dim * c128_state_dtype_size
|
||||
)
|
||||
|
||||
# C128 state is request-scoped and is finalized after
|
||||
# max_running_requests is known, so it should not scale with
|
||||
# full-token capacity here.
|
||||
c128_state_ratio = 0
|
||||
# Cap mode and ring mode both move the SWA pool and the c4 state that
|
||||
# follows it out of the coefficient and into fixed bytes.
|
||||
swa_ratio = (
|
||||
0 if self._unified or self.swa_cap_tokens is not None else self.swa_ratio
|
||||
)
|
||||
|
||||
c4_frac = 1 / (4 * self.c4_shrink_factor)
|
||||
return (
|
||||
swa_ratio * self.bytes_per_swa_token
|
||||
+ self.low_ratio_bytes_per_full_token
|
||||
+ c4_frac * self.kv_bytes * self.num_layers_ca4
|
||||
+ 1 / 128 * self.kv_bytes * self.num_layers_ca128
|
||||
+ 1 / 4 * self.indexer_bytes_per_token * self.num_layers_ca4
|
||||
+ c128_state_ratio * c128_state_bytes * self.num_layers_ca128
|
||||
return swa_ratio * self.bytes_per_swa_token + sum(
|
||||
self._compressed_bytes_per_full_token(ratio) * len(layers)
|
||||
for ratio, layers in self.stage_owner_layers.items()
|
||||
)
|
||||
|
||||
def _get_swa_fixed_bytes(self) -> float:
|
||||
"""Bias bytes the SWA pool takes in cap mode; 0 when sizing by ratio."""
|
||||
"""Bias bytes of the encoder-replay request window plus the cap-mode pool."""
|
||||
paged_bytes = (
|
||||
0
|
||||
if self.swa_cap_tokens is None
|
||||
@@ -1378,10 +1297,11 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
c4_max_total_num_tokens=full_token // (4 * self.c4_shrink_factor),
|
||||
c128_max_total_num_tokens=full_token // 128,
|
||||
# Unified_kv: request-scoped, finalized once concurrency is known.
|
||||
# Otherwise the ring is addressed per SWA page (swa_loc // page_size).
|
||||
c4_state_pool_size=(
|
||||
0
|
||||
if self._unified
|
||||
else swa_tokens // self.swa_page_size * self.c4_ring_size
|
||||
else swa_tokens // page_size * self.ring_sizes.get(4, 0)
|
||||
),
|
||||
c128_state_pool_size=0,
|
||||
)
|
||||
@@ -1392,63 +1312,60 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
return max_running_requests + 1
|
||||
|
||||
def _get_c128_state_fixed_bytes(self, max_running_requests: int) -> int:
|
||||
if self.num_layers_ca128 == 0:
|
||||
num_layers = self.num_layers(128)
|
||||
if num_layers == 0:
|
||||
return 0
|
||||
|
||||
_, c128_state_dtype_size = _get_dsv4_compress_state_dtype_sizes()
|
||||
num_req_slots = self._get_num_req_slots(max_running_requests)
|
||||
ring_size = self.ring_sizes[128]
|
||||
|
||||
if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
|
||||
state_rows = num_req_slots + self.c128_ring_size + 1
|
||||
state_rows = num_req_slots + ring_size + 1
|
||||
state_rows *= 1 + self.online_c128_mtp_max_draft_tokens
|
||||
state_last_dim = 3 * self.attn_head_dim
|
||||
else:
|
||||
state_pool_size = num_req_slots * self.c128_ring_size
|
||||
state_rows = state_pool_size + self.c128_ring_size + 1
|
||||
state_pool_size = num_req_slots * ring_size
|
||||
state_rows = state_pool_size + ring_size + 1
|
||||
state_rows = ceil_div(state_rows, 128) * 128
|
||||
state_last_dim = 2 * self.attn_head_dim
|
||||
|
||||
return (
|
||||
state_rows * state_last_dim * c128_state_dtype_size * self.num_layers_ca128
|
||||
)
|
||||
return state_rows * state_last_dim * c128_state_dtype_size * num_layers
|
||||
|
||||
def _get_c2_state_fixed_bytes(self, max_running_requests: int) -> int:
|
||||
"""Ratio-2 pending-pair state, one fp32 (kv, score) ring per request slot
|
||||
on each ratio-2 kv_source layer; mirrors _make_pair_state_pool."""
|
||||
if self.num_layers_c2_source == 0:
|
||||
num_layers = self.num_layers(2)
|
||||
if num_layers == 0:
|
||||
return 0
|
||||
|
||||
num_req_slots = self._get_num_req_slots(max_running_requests)
|
||||
ring_size = self.c2_ring_size
|
||||
ring_size = self.ring_sizes[2]
|
||||
# CompressStatePool allocates `size + ring_size + 1` rows, padded to the ratio.
|
||||
state_rows = num_req_slots * ring_size + ring_size + 1
|
||||
state_rows = ceil_div(state_rows, 2) * 2
|
||||
state_last_dim = 2 * self.attn_head_dim
|
||||
return (
|
||||
state_rows
|
||||
* state_last_dim
|
||||
* torch.float32.itemsize
|
||||
* self.num_layers_c2_source
|
||||
)
|
||||
return state_rows * state_last_dim * torch.float32.itemsize * num_layers
|
||||
|
||||
def _unified_c4_state_pool_size(self, max_running_requests: int) -> int:
|
||||
# Unified C4 state loc is req_pool_idx * c4_ring_size + pos % c4_ring_size.
|
||||
num_req_slots = self._get_num_req_slots(max_running_requests)
|
||||
return num_req_slots * self.c4_ring_size
|
||||
return num_req_slots * self.ring_sizes[4]
|
||||
|
||||
def _fixed_c4_state_bytes(self, max_running_requests: int) -> int:
|
||||
if not self._unified or self.num_layers_ca4 == 0:
|
||||
num_layers = self.num_layers(4)
|
||||
if not self._unified or num_layers == 0:
|
||||
return 0
|
||||
|
||||
c4_state_dtype_size, _ = _get_dsv4_compress_state_dtype_sizes()
|
||||
# Mirror CompressStatePool.__init__: it allocates `size + ring_size + 1`
|
||||
# rows, padded to the compress ratio.
|
||||
state_rows = self._unified_c4_state_pool_size(max_running_requests)
|
||||
state_rows = ceil_div(state_rows + self.c4_ring_size + 1, 4) * 4
|
||||
state_rows = ceil_div(state_rows + self.ring_sizes[4] + 1, 4) * 4
|
||||
# overlap c4: last_dim = 2 * (1 + overlap) * head_dim = 4 * head_dim.
|
||||
core_bytes = 4 * self.attn_head_dim * c4_state_dtype_size
|
||||
indexer_bytes = 4 * self.indexer_head_dim * c4_state_dtype_size
|
||||
return state_rows * (core_bytes + indexer_bytes) * self.num_layers_ca4
|
||||
return state_rows * (core_bytes + indexer_bytes) * num_layers
|
||||
|
||||
def _resolve_max_running_requests_per_worker(self, available_bytes: int) -> int:
|
||||
# Approximates ModelRunner._resolve_max_num_reqs. Over-estimating is safe:
|
||||
@@ -1462,10 +1379,8 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
return min(estimated, full_token // 2)
|
||||
|
||||
def _fixed_swa_bytes(self, max_running_requests: int) -> int:
|
||||
"""Unified_kv SWA is a fixed per-request ring, sized by concurrency
|
||||
(num_req_slots) rather than by full_token. MTP inflates the target ring
|
||||
by _spec_infl; DSpark+fp8 adds a bf16 draft ring instead (640 vs 1024).
|
||||
Returns 0 on the non-unified path (SWA already counted per-token)."""
|
||||
"""Unified_kv SWA ring bytes, sized by req slots; 0 off unified, where SWA is
|
||||
priced per token. DSpark+fp8 adds a bf16 draft ring instead of _spec_infl."""
|
||||
if not self._unified:
|
||||
return 0
|
||||
num_req_slots = self._get_num_req_slots(max_running_requests)
|
||||
@@ -1483,9 +1398,8 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
draft_row = dsv4_unified_row_bytes(
|
||||
self.qk_nope_head_dim, self.qk_rope_head_dim, fp8=False
|
||||
)
|
||||
# 1 layer is what the shipped DSpark drafts allocate. A multi-stage
|
||||
# draft would under-count by ~9 MB/layer (128-wide window, ~65 req
|
||||
# slots), which the (T+1)/T on bytes_per_full_token already covers.
|
||||
# 1 layer is what shipped DSpark drafts allocate; a multi-layer draft
|
||||
# under-counts ~9 MB/layer, covered by the (T+1)/T inflation.
|
||||
draft_ring = num_req_slots * self._swa_ring_size * draft_row
|
||||
return int(target_ring + draft_ring)
|
||||
return int(target_ring * self._spec_infl)
|
||||
@@ -1518,9 +1432,9 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get():
|
||||
config.c128_state_pool_size = num_req_slots
|
||||
else:
|
||||
config.c128_state_pool_size = num_req_slots * self.c128_ring_size
|
||||
config.c128_state_pool_size = num_req_slots * self.ring_sizes.get(128, 0)
|
||||
# Ring mode: C4 state is request-scoped, so size it from the known concurrency.
|
||||
if self._unified and self.num_layers_ca4 > 0:
|
||||
if self._unified and self.num_layers(4) > 0:
|
||||
config.c4_state_pool_size = self._unified_c4_state_pool_size(
|
||||
config.max_running_requests
|
||||
)
|
||||
@@ -1597,7 +1511,6 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
def create_memory_pool_configurator(
|
||||
kvc: KVCacheConfigurator,
|
||||
) -> MemoryPoolConfigurator:
|
||||
"""Factory: select the right configurator for the model architecture."""
|
||||
if is_deepseek_v4(kvc.model_config.hf_config) and kvc.is_hybrid_swa:
|
||||
return DSV4PoolConfigurator(kvc)
|
||||
if kvc.is_hybrid_swa:
|
||||
|
||||
Reference in New Issue
Block a user