From 0c578d97fe7ee19bdbbfa83fc69d767035a35c44 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 24 Sep 2026 15:19:02 -0700 Subject: [PATCH] [DSV4] Size compressed pools from one per-ratio table in DSV4PoolConfigurator (#41049) --- .../layers/attention/deepseek_v4_backend.py | 76 ++--- .../deepseek_v4_backend_hip_radix.py | 157 +++------- .../srt/mem_cache/deepseek_v4_memory_pool.py | 166 +++++----- .../srt/model_executor/pool_configurator.py | 287 ++++++------------ .../mem_cache/test_dsv4_c4_state_lifecycle.py | 2 +- .../model_executor/test_pool_configurator.py | 77 +++-- 6 files changed, 279 insertions(+), 486 deletions(-) diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 9cd3961c5f..467110ffa1 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -713,9 +713,8 @@ class DSV4AttnMetadata: assert dst_val is not None, f"{field_name=} {src_val=} {dst_val=}" dst_val.copy_(src_val) - # These fields are safe to replace because captured kernels only need - # the current per-replay objects, or the field is produced inside the - # captured graph before the attention graph break consumes it. + # Safe to replace: captured kernels read only the per-replay objects, or + # the field is produced in-graph before the attention graph break reads it. for field_name in reference_assign_fields: setattr(self, field_name, getattr(other, field_name)) @@ -723,9 +722,8 @@ class DSV4AttnMetadata: self, num_tokens: Optional[int] = None, low_ratio_buffers=None ) -> None: assert self.page_table.dim() == 2 - # CP pads causal metadata for per-rank partitioning, while cache-write - # locations remain one-per-logical-token. num_tokens tracks that unpadded - # length; legacy paths use the metadata length. + # CP pads causal metadata per rank, but cache-write locations stay one per + # logical token; num_tokens is that unpadded length. if num_tokens is None: num_tokens = self.seq_lens_casual.shape[0] assert self.raw_out_loc.shape[0] == num_tokens, ( @@ -1148,7 +1146,6 @@ class DeepseekV4AttnBackend( self.softmax_scale: float = head_dim**-0.5 self.head_dim_v: int = model_runner.model_config.v_head_dim self.cuda_int32_kwargs = {"device": self.device, "dtype": torch.int32} - self.swa_page_size = 128 assert model_runner.page_size is not None assert model_runner.req_to_token_pool is not None self.page_size = model_runner.page_size @@ -1156,6 +1153,8 @@ class DeepseekV4AttnBackend( self.req_to_token_pool = model_runner.req_to_token_pool self.token_to_kv_pool: DeepSeekV4TokenToKVPool = model_runner.token_to_kv_pool + # The C4 state ring is addressed per SWA page, so a page holds whole windows. + assert self.token_to_kv_pool.swa_page_size % SWA_WINDOW == 0 self.hisparse_coordinator = model_runner.hisparse_coordinator self.req_to_token = model_runner.req_to_token_pool.req_to_token # Nothing is built for a compress ratio outside the pool's set. @@ -1207,11 +1206,8 @@ class DeepseekV4AttnBackend( self.speculative_num_steps = speculative_num_steps self.speculative_num_draft_tokens: int = get_spec().speculative_num_draft_tokens if self.speculative_num_draft_tokens is not None: - # Persistent target-verify metadata buffers. Allocated here (not - # lazily) so they are ordinary tensors: the first touch of a lazy - # buffer would inherit the caller's context, and a creation inside - # an inference_mode forward would forbid the in-place updates the - # graph-capture path performs outside inference mode. + # Allocated eagerly: a lazy buffer first created inside inference_mode + # would forbid the in-place updates graph capture performs outside it. num_reqs = self.req_to_token.shape[0] self.extend_seq_lens_buffer = torch.full( (num_reqs,), @@ -1263,9 +1259,8 @@ class DeepseekV4AttnBackend( "DSV4 ragged verify does not support online c128 MTP; " "set SGLANG_RAGGED_VERIFY_MODE off or disable online compress." ) - # Layout invariants (verify_lens >= 1, total == sum) are enforced in - # RaggedVerifyLayout.__post_init__; don't re-check the device tensor - # here -- that would D2H-sync the host-free verify prep path. + # Invariants are checked in RaggedVerifyLayout.__post_init__; a device-side + # check here would D2H-sync the host-free verify prep path. layout = layout.padded_to_bucket(padded_bs=bs) return layout @@ -1450,9 +1445,8 @@ class DeepseekV4AttnBackend( else: def create(compress_ratio: Literal[4, 128]): - # Online c128 uses a different planner that cannot be created in - # prefill cuda-graph mode. Keep c4 graph-friendly while matching - # c128's existing online path. + # Online c128's planner cannot be created in prefill cuda-graph + # mode, so it keeps the online path. use_graph_plan = use_prefill_cuda_graph and not ( compress_ratio == 128 and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get() ) @@ -2069,7 +2063,7 @@ class DeepseekV4AttnBackend( ) # Upgrade Raw->Full so compress + core_attn + indexer materialization is - # recorded inside the cuda graph; already Full when PREP_IN_CUDA_GRAPH=0. + # recorded inside the cuda graph. if isinstance(self.forward_metadata, DSV4RawVerifyMetadata): self.forward_metadata = self.make_forward_metadata_from_raw_verify( raw_metadata=self.forward_metadata, @@ -2480,13 +2474,12 @@ class DeepseekV4AttnBackend( logical_forward_mode = _get_logical_forward_mode(forward_batch) req_pool_indices = forward_batch.req_pool_indices seq_lens = forward_batch.seq_lens.to(torch.int32) - # Regular prefill batches already carry scheduler-maintained CPU lengths. - # Keep using those when present; needs_cpu_seq_lens only controls whether - # speculative overlap must publish a new GPU-to-CPU mirror each step. + # Prefill batches carry scheduler-maintained CPU lengths; needs_cpu_seq_lens + # only controls whether spec overlap publishes a GPU-to-CPU mirror each step. seq_lens_cpu = forward_batch.seq_lens_cpu assert self.req_to_token_pool.req_to_token is self.req_to_token - assert self.swa_page_size % SWA_WINDOW == 0 and self.page_size % 128 == 0 + assert self.page_size % 128 == 0 if max_seq_len_override is None: max_seq_len_override = forward_batch.max_seq_len_override if max_seq_len_override is not None: @@ -2635,9 +2628,8 @@ class DeepseekV4AttnBackend( *, static_forward_batch: Optional[ForwardBatch] = None, ) -> None: - # Build graph-compatible metadata against the padded static batch. The - # batch still carries live seq/extend lens, so the online c128 prefill - # plan remains batch-specific without constructing a second metadata set. + # The padded static batch still carries live seq/extend lens, so the + # online c128 prefill plan stays batch-specific. metadata_batch = ( static_forward_batch if static_forward_batch is not None else forward_batch ) @@ -2727,8 +2719,8 @@ class DeepseekV4AttnBackend( if 2 in core.low_ratios: core.c2_flashmla_metadata = _create_flashmla_metadata() - # PREP_IN_CUDA_GRAPH=True: warmup upgraded raw->full on the host; - # restore raw so capture re-runs the upgrade inside the graph. + # Warmup upgraded raw->full on the host; restore raw so capture + # re-runs the upgrade inside the graph. current_raw = getattr(self, "_current_capture_raw", None) if current_raw is not None: self.forward_metadata = current_raw @@ -3870,10 +3862,8 @@ class DeepseekV4AttnBackend( flash_mla_with_kvcache_sm120, ) - # The pad to 64 heads only serves the decode kernel's h_q - # specialization; the prefill kernel takes arbitrary h_q, so - # drop it instead of attending on garbage heads (4x the work - # at attn-TP 4). + # The 64-head pad only serves the decode kernel's h_q + # specialization; the prefill kernel takes any h_q, so drop it. real_heads = layer.tp_q_head_num if q.shape[0] > SM120_DECODE_MAX_TOKENS: if q.shape[-2] > real_heads: @@ -3940,14 +3930,8 @@ class DeepseekV4AttnBackend( core_attn_metadata: DSV4AttnMetadata, attn_sink: torch.Tensor, ) -> torch.Tensor: - """Unified prefill via flash_mla_sparse_fwd. Replaces the - flash_mla_with_kvcache call on the extend path. Per request, - positionally gathers the SWA window (always) and the compressed - cache (c4/c128) into a flat bf16 workspace, then lets - flash_mla_sparse_fwd consume the workspace via per-query rebased - indices. Chunk-invariant scaffolding lives in - ``self.forward_metadata.sparse_prefill_cache``. - """ + """Gathers each request's SWA window and c4/c128 cache into a flat bf16 + workspace that flash_mla_sparse_fwd reads through per-query rebased indices.""" if _is_xpu: from sgl_kernel import flash_mla_sparse_fwd else: @@ -4077,14 +4061,8 @@ class DeepseekV4AttnBackend( core_attn_metadata: DSV4AttnMetadata, attn_sink: torch.Tensor, ) -> torch.Tensor: - """Experimental DeepSeek-V4 sparse prefill path using Q8KV8 kernels. - - This mirrors ``_forward_prefill_sparse``'s cache/index construction, but - writes the gathered KV workspace as FP8 and calls the SM90 Q8KV8 sparse - prefill kernel. The path is selected by ``--dsv4-prefill-backend - flashmla_sparse_q8``; ``SGLANG_DSV4_Q8KV8_PREFILL`` remains as a debug - override for focused runtime validation. - """ + """_forward_prefill_sparse with an FP8 workspace and the SM90 Q8KV8 kernel; + SGLANG_DSV4_Q8KV8_PREFILL is a debug override on top of the CLI selection.""" from sglang.kernels.ops.attention.sparse_mla_q8kv8_prefill_sm90 import ( sparse_mla_q8kv8_prefill_fwd, @@ -4270,8 +4248,6 @@ class DeepseekV4AttnBackend( num_groups: Optional[int] = None, dspark_swa_buffers: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, ) -> DSV4AttnMetadata: - assert self.swa_page_size == SWA_WINDOW - small_metadata = ( not is_prefill and seq_lens_casual.is_cuda diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index b9b05beea3..def42da7d1 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -90,10 +90,6 @@ def _create_dummy_paged_compress_data(compress_ratio: int): @dataclass class UnifiedKvMetadata: - """ - unified-kv per-forward metadata - """ - # SWA ring write target (req_slot*ring + pos%ring) swa_loc: Optional[torch.Tensor] = None @@ -119,10 +115,8 @@ class UnifiedKvMetadata: pf_cu_q: Optional[torch.Tensor] = None pf_final_pos: Optional[torch.Tensor] = None - # Per-token req-slot map used by the SWA ring store, precomputed once per - # step so the forward store does not recompute a repeat_interleave per layer. - # Read by the target-verify store (num_draft*bs tokens); for plain decode it - # equals req_pool_indices and is unused (the decode store reads that live). + # Per-token req-slot map for the target-verify SWA ring store (num_draft*bs + # tokens); unused by plain decode, whose store reads req_pool_indices live. verify_store_state_slot: Optional[torch.Tensor] = None # SWA-page-offset compressed-store locations (= c*_out_loc + unified_swa_pages), @@ -441,9 +435,8 @@ class DSV4Metadata: c4_compress_metadata: Optional[FusedCompressMetadata] = None c128_compress_metadata: Optional[FusedCompressMetadata] = None - # FP4 indexer buffers that captured kernels bind by address. Deliberately - # absent from copy_: the addresses must stay pinned across replays, and the - # workspace builders refresh their contents instead. + # Captured kernels bind these FP4 indexer buffers by address; absent from copy_ + # so addresses stay pinned across replays (the builders refresh contents). fp4_decode_workspace: Optional[FP4DecodeWorkspace] = field(default=None, repr=False) fp4_prefill_workspace: Optional[FP4PrefillWorkspace] = field( default=None, repr=False @@ -563,12 +556,8 @@ class _GraphBucket(enum.Enum): class DeepseekV4HipRadixBackend( AttentionBackend, C4IndexerBackendMixin, CompressorBackendMixin ): - # DSV4 TBO runs ONLY in eager prefill (prefill cuda-graph is disabled); - # decode/target-verify graphs are non-TBO (primary backend only). So the TBO - # child backends must not be driven through cuda-graph capture/replay — doing - # so rebuilds this backend's compressor/indexer metadata per replay step on - # both children and leaks ROCm HSA resources (HSA_STATUS_ERROR_OUT_OF_RESOURCES). - # TboAttnBackend reads this to skip children in the *_graph paths only. + # DSV4 TBO runs only in eager prefill; driving TBO children through graph + # capture/replay leaks ROCm HSA resources (HSA_STATUS_ERROR_OUT_OF_RESOURCES). tbo_supports_cuda_graph = False supports_ragged_verify_graph: bool = True use_captured_forward_metadata_for_breakable_cuda_graph: bool = True @@ -592,7 +581,6 @@ class DeepseekV4HipRadixBackend( self.softmax_scale: float = head_dim**-0.5 self.head_dim_v: int = model_runner.model_config.v_head_dim self.cuda_int32_kwargs = {"device": self.device, "dtype": torch.int32} - self.swa_page_size = 128 assert model_runner.page_size is not None assert model_runner.req_to_token_pool is not None self.page_size = model_runner.page_size @@ -600,6 +588,8 @@ class DeepseekV4HipRadixBackend( self.req_to_token_pool = model_runner.req_to_token_pool self.token_to_kv_pool: DeepSeekV4TokenToKVPool = model_runner.token_to_kv_pool + # The C4 state ring is addressed per SWA page, so a page holds whole windows. + assert self.token_to_kv_pool.swa_page_size % SWA_WINDOW == 0 self.hisparse_coordinator = model_runner.hisparse_coordinator self.req_to_token = model_runner.req_to_token_pool.req_to_token self.MAX_SEQ_LEN_FOR_CAPTURE = self.req_to_token.shape[1] @@ -620,16 +610,14 @@ class DeepseekV4HipRadixBackend( self.is_dspark = model_runner.spec_algorithm.is_dspark() self.is_dspark_draft = self.is_draft_worker and self.is_dspark # Draft layers are all COMPRESS_RATIO_NEXTN_LAYER (0), so the draft pool - # has neither compressed kv nor an indexer pool. Settled here rather than - # per forward: it cannot change after construction. + # has neither compressed kv nor an indexer pool. self.need_compress = not self.is_draft_worker self.target_verify_num_draft_tokens = self.speculative_num_draft_tokens if self.is_dspark_draft: assert self.speculative_num_draft_tokens is not None assert self.speculative_num_draft_tokens > 1 - # DSpark draft workers verify gamma rows. The server arg keeps the - # CUDA-side convention gamma + 1, so use an explicit effective value - # instead of mutating speculative_num_draft_tokens in place. + # DSpark draft workers verify gamma rows; the server arg keeps the + # CUDA-side convention gamma + 1. self.target_verify_num_draft_tokens = self.speculative_num_draft_tokens - 1 # Past MAX_FUSED_ROWS the fp4 schedule falls back to AITER's preamble, # which frees the scratch its kernels read -- not capture-safe. @@ -701,8 +689,7 @@ class DeepseekV4HipRadixBackend( ) # extend_start_loc and the CPU mirrors below only feed the torch - # fallback; the triton kernel cumsums extend_seq_lens on device, so - # every caller can share it instead of dropping to a per-request loop. + # fallback; the triton kernel cumsums extend_seq_lens on device. _expanded = ExpandPrefillCausally.execute( req_pool_indices=req_pool_indices, seq_lens=seq_lens, @@ -742,10 +729,8 @@ class DeepseekV4HipRadixBackend( exact_num_tokens=exact_num_tokens or host_proves_exact_num_tokens, ) if attach_decode_streams: - # Target-verify runs through the unified_kv DECODE kernel, so build - # per-token decode streams here. req_pool_indices_repeated is the - # per-token (num_draft*bs -> bs) req-slot map produced by the prefill - # expansion above. + # Verify runs the unified_kv DECODE kernel; req_pool_indices_repeated + # is the per-token (num_draft*bs -> bs) req-slot map. self._attach_unified_kv_decode_streams( core_attn_metadata, req_pool_indices_repeated, @@ -811,12 +796,9 @@ class DeepseekV4HipRadixBackend( seq_lens_cpu: Optional[List[int]] = None, ragged_layout=None, ) -> Union[DSV4Metadata, DSV4RawVerifyMetadata]: - # DSPARK verifies a uniform num_draft block, exactly what - # make_forward_metadata_from_raw_verify expands, so the build can be - # deferred into the graph. Graph path only: the upgrade sizes its page - # table by MAX_SEQ_LEN_FOR_CAPTURE, far wider than the live max_seq_len - # an eager caller passes. EAGLE and ragged layouts stay eager -- no raw - # expansion, and EAGLE's fixed-tier plan trips planner invariants. + # Only DSPARK's uniform draft block defers into the graph; EAGLE's fixed-tier + # plan trips planner invariants. Graph path only: the upgrade sizes page_table + # by MAX_SEQ_LEN_FOR_CAPTURE, wider than an eager caller's max_seq_len. if ( use_prefill_cuda_graph and self.is_dspark @@ -869,10 +851,8 @@ class DeepseekV4HipRadixBackend( ) extend_seq_lens = verify_lens_dev seq_lens = seq_lens + verify_lens_dev.to(seq_lens.dtype) - # Total verify tokens to expand. For the graph path the padded layout - # sets total_verify_tokens == graph_num_tokens (tier); the eager path - # resolves a device-assembled layout whose total_verify_tokens is None, - # so fall back to sum(verify_lens) (== real total; padded == tier). + # Graph path: total_verify_tokens is the padded tier; an eager + # device-assembled layout leaves it None, so use sum(verify_lens). num_tokens = ragged_layout.total_verify_tokens if num_tokens is None: num_tokens = int(verify_lens_dev.sum().item()) @@ -1120,14 +1100,12 @@ class DeepseekV4HipRadixBackend( ) if upgraded_verify: - # The out-graph refresh saw raw metadata and skipped. Without this - # the logits kernel builds its own schedule -- the variant that - # frees the scratch it reads, which every replay would re-read. + # The out-graph refresh saw raw metadata and skipped; otherwise the + # logits kernel builds a schedule that frees scratch replays re-read. self._refresh_fp4_prefill_workspace(forward_batch) - # Decode's schedule builder is capture-safe because the workspace pins - # the scratch it reads, so it can stay next to the metadata it consumes. - # Prefill/target-verify cannot; see _refresh_fp4_prefill_workspace. + # Decode's schedule builder is capture-safe (the workspace pins its scratch); + # prefill/target-verify's is not, see _refresh_fp4_prefill_workspace. if self._fp4_workspaces_enabled(metadata) and ( forward_batch.forward_mode.is_decode() ): @@ -1144,9 +1122,8 @@ class DeepseekV4HipRadixBackend( def _fp4_workspaces_enabled(self, metadata) -> bool: return ( self.enable_deepseek_v4_fp4_indexer - # Draft-step backends drive the NextN layer, which is built with - # compress_ratio_override=0 and so owns no C4 indexer. Their - # workspaces would be built, scheduled, and never read. + # Draft-step backends drive the NextN layer, built with + # compress_ratio_override=0, so it owns no C4 indexer. and self.speculative_num_steps == 0 and isinstance(metadata, DSV4Metadata) and metadata.indexer_metadata is not None @@ -1154,13 +1131,8 @@ class DeepseekV4HipRadixBackend( ) def _refresh_fp4_prefill_workspace(self, forward_batch: ForwardBatch) -> None: - """Rebuild the FP4 prefill schedule outside CUDA-graph capture. - - AITER's prefill scheduler frees the scratch that its schedule kernel - reads, so recording the build into a graph would leave every replay - reading recycled graph-pool memory. Only the pinned buffers it fills - (cta_info / logits / guarded page table) may be read from the graph. - """ + """Must run outside graph capture: AITER's scheduler frees the scratch its + kernel reads, so only the pinned buffers it fills may be read from a graph.""" metadata = self.forward_metadata if not self._fp4_workspaces_enabled(metadata): return @@ -1337,7 +1309,7 @@ class DeepseekV4HipRadixBackend( seq_lens_cpu = forward_batch.seq_lens_cpu assert self.req_to_token_pool.req_to_token is self.req_to_token - assert self.swa_page_size % SWA_WINDOW == 0 and self.page_size % 128 == 0 + assert self.page_size % 128 == 0 assert seq_lens_cpu is not None max_seq_len = ( max_seq_len_override @@ -1499,7 +1471,7 @@ class DeepseekV4HipRadixBackend( core.c4_flashmla_metadata = _create_flashmla_metadata() core.c128_flashmla_metadata = _create_flashmla_metadata() - # PREP_IN_CUDA_GRAPH=True: warmup upgraded raw->full on the host; + # Warmup upgraded raw->full on the host; # restore raw so capture re-runs the upgrade inside the graph. current_raw = getattr(self, "_current_capture_raw", None) if current_raw is not None: @@ -1532,8 +1504,7 @@ class DeepseekV4HipRadixBackend( hca_page_indices = core.c128_page_indices c4_page_indices = core.c4_sparse_page_indices # A draft pool carries no c4/c128 metadata, but its verify store still - # reads the swa half below, so build these with an empty compressed tail - # rather than skipping the call. + # reads the swa half below, so build these with an empty compressed tail. if hca_len is None: hca_len = torch.zeros_like(swa_len) if csa_len is None: @@ -1596,9 +1567,8 @@ class DeepseekV4HipRadixBackend( req_slot * pool.unified_swa_ring_size + core.positions_casual.to(torch.int64) % pool.unified_swa_ring_size ).to(torch.int32) - # Per-token req-slot map for the SWA ring store, read directly by the - # forward store (target-verify) instead of recomputing a repeat_interleave - # per layer. Harmless for plain decode (its store reads req_pool_indices). + # Per-token req-slot map for the target-verify SWA ring store; plain + # decode's store reads req_pool_indices instead. core.unified.verify_store_state_slot = state_slot def _attach_unified_kv_prefill_meta( @@ -1673,14 +1643,8 @@ class DeepseekV4HipRadixBackend( q_rope: Optional[torch.Tensor] = None, k_rope: Optional[torch.Tensor] = None, ) -> torch.Tensor: - """unified_kv paged-attention path over the unified_kv pool. - - ``q_rope`` is what tells the two layouts apart: present means ``q`` is a - packed fp8 row and the pool is the two-pool fp8 one, so decode goes to - the asm reader; absent means both are plain bf16 and it goes to Triton. - Prefill needs ``k_rope`` alongside it, because there the current chunk is - a KV source of its own and not just something to store. - """ + """q_rope present: packed fp8 q over the two-pool fp8 layout (asm decode; + prefill also needs k_rope). Absent: plain bf16 q and pool (Triton).""" from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime pool = self.token_to_kv_pool @@ -1706,18 +1670,14 @@ class DeepseekV4HipRadixBackend( is_decode = forward_batch.forward_mode.is_decode_or_idle() or verify_as_decode if is_decode: if verify_as_decode: - # Per-token (num_draft*bs -> bs) req-slot map, precomputed once - # per step in _attach_unified_kv_decode_streams. Writing every - # draft token's K into the ring is safe: spec_extra room prevents - # clobbering the window history same-step tokens still read. + # Writing every draft token's K into the ring is safe: spec_extra + # room prevents clobbering the history same-step tokens still read. state_slot = core_attn_metadata.unified.verify_store_state_slot[:T] else: state_slot = forward_batch.req_pool_indices[:T] if save_kv_cache: - # Only verify reaches this under fp8 -- plain decode's rows are - # written by the fused kernel itself, which leaves kv None. The - # pair arrives already packed, so this is the same scatter with - # a second pool hanging off it. + # Only verify reaches this under fp8; plain decode's rows are + # written by the fused kernel itself, which leaves kv None. runtime.store_swa_into_unified( kv=kv, state_slot=state_slot, @@ -1752,10 +1712,8 @@ class DeepseekV4HipRadixBackend( else: raise ValueError(f"bad compress_ratio {compress_ratio}") if q_rope is not None: - # softmax_scale is not passed on: the asm kernel hardcodes - # 1/sqrt(512), which is what self.softmax_scale already is for - # V4's head_dim=512. The other readers here take it explicitly, - # so a head_dim change would leave only this one mis-scaled. + # The asm kernel hardcodes softmax scale 1/sqrt(512) (V4 head_dim) + # and takes no softmax_scale, unlike the other readers here. assert self.softmax_scale == 512**-0.5, ( "the v4 nm asm kernel hardcodes 1/sqrt(512), this backend is " f"at {self.softmax_scale}" @@ -1812,20 +1770,12 @@ class DeepseekV4HipRadixBackend( cu_q = core_attn_metadata.unified.pf_cu_q final_pos = core_attn_metadata.unified.pf_final_pos - # DSA CP (round-robin/interleave): unified_pf_* are built over the GLOBAL - # token layout, but under CP each rank owns only 1/cp_size of the queries - # (q/positions are local) while kv was all-gathered to the full sequence. - # Slice the per-query fields to this rank's tokens so their length matches - # the local query count T; values stay global so each local query still - # attends over the full all-gathered KV. + # DSA CP: unified_pf_* cover the GLOBAL token layout while q is local and + # kv all-gathered; slice per-query fields to this rank, values stay global. from sglang.srt.layers.attention.dsa.utils import ( is_dsa_prefill_cp_round_robin_split, ) - # NOTE (AMD/HIP only): this whole DSA-CP prefill handling lives in the - # HIP backend (DeepseekV4HipRadixBackend, selected only when is_hip()). - # The NVIDIA path uses DeepseekV4AttnBackend and never reaches here, so - # these CP changes do not affect B200/H200 execution. _cp_size = get_parallel().attn_cp_size _cp_active = ( _cp_size > 1 @@ -1842,15 +1792,11 @@ class DeepseekV4HipRadixBackend( chunk_start = chunk_start[_sl].contiguous() cu_q = cu_q[_sl].contiguous() final_pos = final_pos[_sl].contiguous() - # positions for the local queries are this rank's round-robin global - # positions {r, r+cp, r+2cp, ...}; forward_batch.positions is the full - # (padded) global layout, so slice it the same way instead of taking - # the first T entries (which would be the wrong, sequential 0..T-1). + # Local queries sit at this rank's round-robin global positions + # {r, r+cp, r+2cp, ...}, not the first T entries of the global layout. positions = forward_batch.positions.to(torch.int64)[_sl].contiguous() - # The SWA ring must hold the FULL window on EVERY rank (decode and - # later chunks read this rank's ring). kv was all-gathered to the full - # sequence, so write the full kv with full global positions/state_slot - # instead of only this rank's 1/cp_size tokens. + # Every rank's SWA ring must hold the full window (decode and later + # chunks read it), so the store uses the full all-gathered kv. positions_full = forward_batch.positions.to(torch.int64)[ : state_slot_full.shape[0] ].contiguous() @@ -1877,10 +1823,8 @@ class DeepseekV4HipRadixBackend( "fp8 prefill needs the extend rope half beside the packed nope; " "q_rope came through but k_rope did not" ) - # No empty-segment mask on the result, unlike decode: this kernel - # returns zeros for a token with neither region where the asm decode - # reader leaves the row NaN. Chunk 0 tokens have an empty prefix and - # a non-empty extend, which both readers handle. + # No empty-segment mask, unlike decode: this kernel returns zeros for + # a token with neither region where the asm decode reader leaves NaN. o = runtime.prefill_fp8_2buff( q=q, q_rope=q_rope, @@ -1912,9 +1856,6 @@ class DeepseekV4HipRadixBackend( # write this chunk's SWA K into the ring for future chunks / decode # only the final-window tokens per request if save_kv_cache: - # Under CP, write the FULL all-gathered window so every rank's ring is - # complete (decode / later chunks read the local ring). Without CP this - # is just the local kv + local metadata as before. _ring_state_slot = state_slot_full if _cp_active else state_slot _ring_final_pos = final_pos_full if _cp_active else final_pos _ring_positions = positions_full if _cp_active else positions @@ -2148,8 +2089,6 @@ class DeepseekV4HipRadixBackend( need_compress: bool = True, is_prefill: bool = False, ) -> DSV4AttnMetadata: - assert self.swa_page_size == SWA_WINDOW - seq_lens_casual = seq_lens_casual.to(torch.int32) swa_page_indices = self.get_swa_page_indices( diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index b1b80e437e..f7c5555dd4 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -65,6 +65,42 @@ def get_compress_state_ring_size(compress_ratio: int, num_draft_tokens: int = 0) return 8 if compress_ratio == 4 else 128 +def resolve_compress_state_ring_size(compress_ratio: int) -> int: + """Ring size under the published spec config. The pool allocates by this + and the configurator budgets by it, so both must call the same function.""" + spec = get_spec() + num_draft_tokens = ( + 0 + if spec.speculative_algorithm is None + else spec.speculative_num_draft_tokens or 0 + ) + return get_compress_state_ring_size(compress_ratio, num_draft_tokens) + + +def collect_sources_by_ratio( + compression_ratios: Sequence[int], kv_source_layers: Sequence[int], stage: range +) -> dict[int, List[int]]: + """Layers owning compressed storage: all of ratios 4/128, kv_sources of 1/2.""" + for idx in stage: + ratio = compression_ratios[idx] + if ratio not in (0, 1, 2, 4, 128): + raise ValueError(f"Unsupported compression ratio: {ratio}") + + sources_by_ratio: dict[int, List[int]] = {} + for ratio in (4, 128, 1, 2): + if ratio in (1, 2): + layers = [ + l + for l in kv_source_layers + if l in stage and compression_ratios[l] == ratio + ] + else: + layers = [l for l in stage if compression_ratios[l] == ratio] + if layers: + sources_by_ratio[ratio] = layers + return sources_by_ratio + + def get_compress_state_write_pad(compress_ratio: int, ring_size: int) -> int: """Largest draft-token count this ring can serve; mirrors `mtp_pad` in `c_plan.cuh`, where the bound is derived.""" @@ -79,11 +115,8 @@ def get_swa_ring_size(sliding_window: int, is_speculative: bool = False) -> int: def resolve_unified_kv_fp8(unified_fp8: Optional[bool] = None) -> bool: - """Per-pool fp8 layout. None follows SGLANG_DSV4_UNIFIED_KV_FP8. - - A caller may pass False so this pool keeps the bf16 ring while the env - stays on (target fused-Q still keys off the global switch). - """ + """Per-pool fp8 layout; None follows SGLANG_DSV4_UNIFIED_KV_FP8. False keeps this + pool bf16 while target fused-Q still keys off the global switch.""" from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( is_unified_kv_fp8, ) @@ -309,11 +342,8 @@ class DeepSeekV4SingleKVPool(KVCache): class DeepSeekV4UniformFP8KVPool(DeepSeekV4SingleKVPool): - """Uniform 512-dim FP8 (e4m3) variant of the DSv4 single-KV pool. - - Each token is 448 NoPE + 64 RoPE contiguous e4m3 values without in-cache - scales or per-page padding. The backend supplies the dequant scale. - """ + """Each token is 448 NoPE + 64 RoPE contiguous e4m3 values, without in-cache + scales or per-page padding; the backend supplies the dequant scale.""" def get_bytes_per_token(self) -> int: return self.qk_nope_head_dim + self.qk_rope_head_dim @@ -355,10 +385,8 @@ class DeepSeekV4UniformFP8KVPool(DeepSeekV4SingleKVPool): cache_k: torch.Tensor, freqs_cis: Optional[torch.Tensor] = None, ) -> None: - """Store normed/roped rows as e4m3 with the backend's fixed unit scale. - - uint8 views work around index_put not supporting FP8 dtypes. - """ + """Store normed/roped rows as e4m3 with the backend's fixed unit scale; + uint8 views work around index_put not supporting FP8 dtypes.""" assert freqs_cis is None, "the uniform-FP8 pool takes finished (rotated) rows" assert cache_k.dim() == 2 and cache_k.shape[1] == self.kv_cache_total_dim @@ -572,11 +600,8 @@ class DeepSeekV4IndexerPool(KVCache): return self.index_k_with_scale_buffer[layer_id] def contiguous_page_row_buffers(self) -> List[torch.Tensor]: - """Every indexer buffer as 2D page rows, for PD and HiCache transfer. - - FP8 keeps key and scale fused in one buffer per layer; the FP4 layout - stores payload and scale separately, so it yields two buffers per layer. - """ + """Every indexer buffer as 2D page rows for PD and HiCache: one fused key+scale + buffer per layer under FP8, separate payload and scale buffers under FP4.""" if self.index_k_with_scale_buffer is not None: return self.index_k_with_scale_buffer return [ @@ -744,9 +769,7 @@ class DeepSeekV4UnifiedKVPool: unified_kv[L]: ``[swa_pages + padded_compress_rows, head_dim]`` bf16 Layout (fp8, ``SGLANG_DSV4_UNIFIED_KV_FP8``) -- two parallel pools with the - same row count, so a row index means the same thing in both. Named after the - accessors, which under fp8 each return one half -- ``get_unified_kv`` the - nope, ``get_unified_kv_rope`` the rope: + same row count, so a row index addresses both: unified_kv[L] (nope): ``[rows, 512]`` fp8, see DSV4_FP8_NOPE_ROW_BYTES unified_kv_rope[L] (rope): ``[rows, qk_rope_head_dim]`` bf16, never quantized @@ -835,9 +858,7 @@ class DeepSeekV4UnifiedKVPool: def get_buf_infos(self) -> Tuple[List[int], List[int], List[int]]: if self.fp8: - # same single-pool assumption as the outer get_contiguous_buf_infos: - # one pointer and one row size per layer describes the nope pool only, - # so whoever picks this up next would move half a row and not notice. + # One pointer and row size per layer would describe only the nope pool. # TODO(danli103): report both pools once a consumer needs them. raise NotImplementedError( "get_buf_infos describes one pool per layer; the fp8 rope pool " @@ -959,9 +980,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): c128_state_pool_size = max( c128_state_pool_size, self.num_req_slots * c128_ring_size ) - # Only the ratios the model has anywhere get a pool config: the backend, PD - # state transfer and HiCache read the registries as "the ratios this model - # has", and a PP stage missing one keeps its empty pool so the PD wire aligns. + # Consumers read the registries as "the ratios this model has"; a PP stage + # missing one keeps its empty pool so the PD wire aligns. model_ratios = set(compression_ratios) self.compressed_pool_configs = { ratio: config @@ -1007,10 +1027,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): self.swa_size = swa_size self.swa_page_size = swa_page_size - # The allocator and compress state keep 256-token logical pages, while - # FlashInfer's SM120 DSV4 kernel consumes 64-token physical pages. Storing - # SWA KV in that layout removes the per-layer 256 -> 64 page split; the - # allocator's flat token indices stay valid. + # The allocator keeps 256-token logical pages; FlashInfer's SM120 DSV4 kernel + # reads 64-token physical pages, and flat token indices are valid in both. swa_kv_page_size = ( 64 if get_platform().is_sm120 and envs.SGLANG_OPT_SM120_DIRECT_SWA_KV.get() @@ -1127,7 +1145,11 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): ) self.full_size = full_size self.kv_source_layers = list(kv_source_layers) - self.sources_by_ratio = self._collect_sources_by_ratio() + self.sources_by_ratio = collect_sources_by_ratio( + self.compression_ratios, + self.kv_source_layers, + range(self._stage_start, self._stage_end), + ) self._init_compressed_pools( stage_ratios=stage_ratios, page_size=page_size, @@ -1163,13 +1185,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): self.full_to_swa_index_mapping = full_to_swa_index_mapping def get_ring_size(self, compress_ratio: int) -> int: - spec = get_spec() - num_draft_tokens = ( - 0 - if spec.speculative_algorithm is None - else spec.speculative_num_draft_tokens or 0 - ) - return get_compress_state_ring_size(compress_ratio, num_draft_tokens) + return resolve_compress_state_ring_size(compress_ratio) def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor): assert self.full_to_swa_index_mapping is not None @@ -1181,10 +1197,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): item_lens: List[int] = [] if self._unified_kv_fp8: - # The page-block transfer below prices one row as buf[0].nbytes and - # ships a single pointer per layer. Under fp8 that covers the nope - # pool only -- the parallel bf16 rope pool would be dropped and the - # remote side would decode rows against stale rope. Refuse instead. + # One pointer and one buf[0]-sized row per layer covers only the fp8 nope + # pool; the remote side would decode rows against stale rope. # TODO(danli103): ship the rope pool as a second per-layer entry. raise NotImplementedError( "PD disaggregation is not supported with " @@ -1224,9 +1238,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): indexer_pool = self.index_pools.get(ratio) if indexer_pool is None: continue - # The transfer addresses every buffer by FULL page id; ratio-1/2 index - # pools page at DSV41_INDEX_PAGE_SIZE, so one item is the run of index - # pages holding a FULL page's page_size // ratio slots. + # The transfer addresses buffers by FULL page id; one ratio-1/2 item is the + # run of DSV41_INDEX_PAGE_SIZE pages holding page_size // ratio slots. index_pages_per_full_page = 1 if ratio in (1, 2): slots_per_full_page = self.page_size // ratio @@ -1255,9 +1268,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): if not self._unified_kv: return data_ptrs, data_lens, item_lens if self._unified_kv_fp8: - # Other half of the PD path -- get_contiguous_buf_infos ships the - # compressed region, this one the ring. Same single-pool assumption, - # same silently dropped rope, same fix -- land them together. + # Ring half of the fp8 gap in get_contiguous_buf_infos; fix both together. raise NotImplementedError( "PD disaggregation is not supported with " "SGLANG_DSV4_UNIFIED_KV_FP8=1 (the SWA_RING component assumes a " @@ -1304,9 +1315,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): return views, item_bytes def unified_region_buffers(self, ratio: int) -> Tuple[List[torch.Tensor], int]: - """ - Main compressed region of one stage: bf16 latents, or fp8 nope. - """ + """Main compressed region of one stage: bf16 latents, or fp8 nope.""" assert self._unified_kv, "unified_region_buffers requires unified_kv layout" assert ratio in (4, 128), f"unsupported compression ratio: {ratio}" return self._unified_page_views(self.unified_kv_pool.kv_buffer, ratio) @@ -1314,14 +1323,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): def unified_rope_region_buffers( self, ratio: int ) -> Optional[Tuple[List[torch.Tensor], int]]: - """ - The bf16 rope half of an fp8 two-pool row, or None when there isn't one. - - A row index addresses both pools, so this mirrors exactly the rows - ``unified_region_buffers`` does and only the row width differs. It needs - its own host pool: offloading the nope half alone leaves whatever rope the - row held before, which is wrong output rather than a crash. - """ + """The bf16 rope half of an fp8 two-pool row, or None. It needs its own host + pool: offloading only the nope half leaves stale rope, i.e. wrong output.""" if not self._unified_kv_fp8: return None assert self._unified_kv, "unified_rope_region_buffers requires unified_kv" @@ -1567,7 +1570,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): enable_memory_saver=enable_memory_saver, ratio=ratio, online=(ratio == 128 and ONLINE_C128), - request_scoped=ratio in (2, 128), + # Ratio 2 never reaches this factory (_make_pair_state_pool). + request_scoped=ratio == 128, swa_page_size=self.swa_page_size, online_mtp_max_draft_tokens=( self.online_mtp_max_draft_tokens if ratio == 128 else 0 @@ -1624,28 +1628,6 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): enable_memory_saver=enable_memory_saver, ) - def _collect_sources_by_ratio(self) -> dict[int, List[int]]: - """Layers owning compressed storage: all of ratios 4/128, kv_sources of 1/2.""" - stage = range(self._stage_start, self._stage_end) - for idx in stage: - ratio = self.compression_ratios[idx] - if ratio not in (0, 1, 2, 4, 128): - raise ValueError(f"Unsupported compression ratio: {ratio}") - - sources_by_ratio: dict[int, List[int]] = {} - for ratio in (4, 128, 1, 2): - if ratio in (1, 2): - layers = [ - l - for l in self.kv_source_layers - if l in stage and self.compression_ratios[l] == ratio - ] - else: - layers = [l for l in stage if self.compression_ratios[l] == ratio] - if layers: - sources_by_ratio[ratio] = layers - return sources_by_ratio - def source_layer_of(self, layer_id: int) -> int: """The layer owning this layer's compressed storage: itself for ratios 4/128, the nearest preceding kv_source layer for ratios 1/2.""" @@ -2008,10 +1990,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): positions: torch.Tensor, ) -> None: if self.uniform_fp8: - # Uniform-FP8 (trtllm-gen) layout: norm + RoPE with the existing - # Triton kernel (in-place on kv; safe -- kv is not read again), - # then a plain e4m3 cast + scatter in the pool setter (per-tensor - # scale 1.0). Fusing the store is deferred to the perf phase. + # Uniform-FP8 (trtllm-gen): in-place norm + RoPE (kv is not read again), + # then an e4m3 cast + scatter with per-tensor scale 1.0. from sglang.kernels.ops.attention.deepseek_v4_rope import ( fused_norm_rope_inplace_triton, ) @@ -2049,14 +2029,8 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): freqs_cis: torch.Tensor, positions: torch.Tensor, ) -> None: - """unified_kv counterpart of set_swa_key_buffer_radix_fused_norm_rope. - - Under unified_kv the (fp8, paged) swa_kv_pool is None -- SWA K lives in - the shared bf16 unified_kv ring instead. Norm+RoPE the draft KV in place - (the same freqs_cis path the main model uses via _compute_kv_bf16) and - scatter it into ``unified_kv[swa_loc]``. Rows with swa_loc < 0 - (uncommitted verify tokens) are skipped by the scatter. - """ + """unified_kv counterpart of set_swa_key_buffer_radix_fused_norm_rope; rows + with swa_loc < 0 (uncommitted verify tokens) are skipped by the scatter.""" from sglang.kernels.ops.attention.dsv4 import fused_norm_rope_inplace from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index d3e22858cb..333b9cb5d4 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -4,11 +4,6 @@ Each model architecture has its own configurator that computes pool sizes from available GPU memory using a unified coeff+bias model: available_bytes = max_tokens * coeff + bias - max_tokens = (available_bytes - bias) / coeff - -Two entry points, same core computation: -- calculate_pool_sizes(available_bytes, page_size): profiling path -- calculate_pool_sizes_from_max_tokens(max_tokens, page_size): constraint path """ from __future__ import annotations @@ -36,10 +31,11 @@ from sglang.srt.configs.model_config import ( from sglang.srt.environ import envs from sglang.srt.mem_cache.allocation_sizing import get_alloc_len_per_decode from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( - get_compress_state_ring_size, + collect_sources_by_ratio, get_compress_state_write_pad, get_dsv4_indexer_bytes_per_token, get_swa_ring_size, + resolve_compress_state_ring_size, ) from sglang.srt.mem_cache.memory_pool import ( DSATokenToKVPool, @@ -85,11 +81,8 @@ class MemoryPoolConfig: mem_fraction_static: Optional[float] = None - # Unified pool only: the PROFILED byte budget for the token-granular - # sub-pools. Set, the factories size the buffer from it directly instead of - # re-summing ratio-derived token counts, which keeps the re-sum's floor - # losses out of the buffer; the token counts stay boot labels / conserve - # caps. None on the token-capped path -- a user token cap IS the budget. + # Unified pool only: the profiled byte budget the buffer is sized from, + # instead of re-summing token counts; None when a user token cap is the budget. unified_total_bytes: Optional[int] = None def __post_init__(self): @@ -107,14 +100,8 @@ logger = logging.getLogger(__name__) def _dflash_draft_cell_size(kvc: KVCacheConfigurator) -> int: - """Bytes/token the DFLASH draft KV pool adds to the target's budget, 0 if none. - - Unlike an EAGLE draft, which reuses the target's attention config and is - therefore priced by layer count, a DFLASH draft has its own geometry and is - a flat additive term. Under DCP, the target pool is sharded while the draft - pool spans the allocator's widened virtual location space, so the draft - term is replicated across DCP ranks. - """ + """Bytes/token the DFLASH draft KV pool adds, 0 if none; replicated across DCP + ranks because the draft pool spans the widened virtual location space.""" if kvc.is_draft_worker or not kvc.spec_algorithm.is_dflash_family(): return 0 cell_size = kvc.spec_aux_config.dflash_draft_cell_size_per_token @@ -200,11 +187,7 @@ class MemoryPoolConfigurator: class DefaultPoolConfigurator(MemoryPoolConfigurator): - """Configurator for standard models: MHA, MLA, DSA, FP4. - - coeff = cell_size (bytes per token across all layers) - bias = 0 - """ + """Standard models (MHA, MLA, DSA, FP4): coeff = cell_size, bias = 0.""" def __init__(self, kvc: KVCacheConfigurator): self.kv_cache_dtype_str = kvc.kv_cache_dtype_str @@ -232,10 +215,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): else get_schedule().max_total_tokens or kvc.model_config.context_len ) - # EAGLE/STANDALONE: scale cell_size to account for draft model KV cache. - # Assumes draft and target share the same per-layer KV size (head_dim, - # num_kv_heads, dtype), which holds for EAGLE/MTP draft models that - # reuse the target architecture's attention config. + # EAGLE/STANDALONE: assumes the draft shares the target's per-layer KV size, + # which holds for EAGLE/MTP drafts that reuse the target's attention config. if ( kvc.spec_algorithm.is_eagle() or kvc.spec_algorithm.is_standalone() ) and not kvc.is_draft_worker: @@ -273,11 +254,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): self._cell_size * (1 + draft_num_layers / int(num_layers)) ) - # DFLASH/DSPARK: reserve the draft runner's *actual* per-token KV cost. - # The draft allocates its own KV pool at the target's - # max_total_num_tokens, whose per-token footprint can differ from the - # target's (e.g. an MLA-latent target paired with a full per-head K/V - # draft), so size from the draft config rather than the layer ratio. + # DFLASH/DSPARK: the draft's per-token KV cost can differ from the target's + # (e.g. MLA target, per-head K/V draft), so size from the draft config. if kvc.spec_algorithm.is_dflash_family() and not kvc.is_draft_worker: from sglang.srt.speculative.dflash_utils import ( scale_kv_cell_size_per_token_for_dflash, @@ -298,7 +276,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): ) def _compute_cell_size(self, kvc: KVCacheConfigurator, num_layers: int) -> int: - """Compute per-token KV cache cost in bytes. Subclasses can override.""" + """Compute per-token KV cache cost in bytes.""" # args to config cell size model_config = kvc.model_config kv_cache_dtype = kvc.kv_cache_dtype @@ -540,9 +518,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): _, shard_size = get_glm_dsa_cp_layer_shard_info(kvc) if shard_size > 1: - # Preserve the existing LayerSplit sizing semantics. GLM-5.3 - # hybrid-layer support is intentionally limited to the normal - # (non-LayerSplit) pool below. + # GLM-5.3 hybrid-layer support is intentionally limited to the + # non-LayerSplit pool below. active_indexer_layers = [ layer_id for layer_id in range( @@ -592,11 +569,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): class HybridSWAPoolConfigurator(MemoryPoolConfigurator): - """Configurator for MHA or MLA models with sliding-window layers. - - Splits available memory between full attention and SWA pools. - Does NOT inherit DefaultPoolConfigurator — different coeff model. - """ + """Splits memory between the full and SWA pools of MHA/MLA sliding-window models.""" def __init__(self, kvc: KVCacheConfigurator): self.kv_cache_dtype_str = kvc.kv_cache_dtype_str @@ -704,15 +677,8 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator): self._recompute_cell_size() def _recompute_cell_size(self) -> None: - # Bytes per token of max_total_num_tokens. - # - # Hybrid (full_layers > 0): max_total = full_tokens, so cell_size accounts - # for both pools: F*nf + r*S*ns (where swa_tokens = full_tokens * r). - # - # All-SWA (full_layers == 0): max_total = swa_tokens directly. The ratio - # is meaningless here -- there is no full pool to relate to, and every - # token beyond the sliding window can be evicted. So cell_size = S*ns, - # with no ratio factor applied. + # Bytes per token of max_total_num_tokens: full_tokens when hybrid, else + # swa_tokens with no ratio, since all-SWA has no full pool to relate to. if self._full_layers_num == 0: self._cell_size = ( self._swa_per_token * self._swa_layers_num @@ -781,14 +747,12 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator): def _solve_pool_sizes( self, max_total_num_tokens: int, page_size: int ) -> MemoryPoolConfig: - """Core computation: split max_total_num_tokens into full/swa pool sizes.""" - def align_page_size(x: int) -> int: return (x // page_size) * page_size if self._full_layers_num == 0: # All-SWA: no full pool, max_total = actual SWA pool size. - # Ratio is not applied -- see __init__ comment. + # Ratio is not applied -- see _recompute_cell_size. swa_tokens = align_page_size(max_total_num_tokens) logger.info( f"Use sliding window memory pool (all SWA). " @@ -885,13 +849,8 @@ def compute_swa_request_cap(*, page_size: int, window: int, attn_dp_size: int) - class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator): - """Hybrid SWA configurator with the SWA pool sized from a fixed token cap. - - When max_running_requests is explicit, the SWA pool's worst-case - footprint is bounded per request. The SWA pool is sized tightly from that - cap and the freed memory is redirected to the full pool, instead of sizing - both pools by swa_full_tokens_ratio. - """ + """Hybrid SWA with the SWA pool sized from the explicit max_running_requests + worst case instead of swa_full_tokens_ratio; the rest goes to the full pool.""" def __init__(self, kvc: KVCacheConfigurator): self.kv_cache_dtype_str = kvc.kv_cache_dtype_str @@ -984,13 +943,8 @@ class _DSV4PoolSizes: class DSV4PoolConfigurator(MemoryPoolConfigurator): - """Configurator for DSV4 compressed-attention models. - - Splits available memory across full / swa / c4 / c128 + c4_state / c128_state - pools. coeff is bytes_per_full_token (inflated by (T+D)/T when speculative - decode reserves a draft worker, mirroring dflash's cell_size scaling). bias - is the request-scoped fixed pools that do not scale with full_token. - """ + """DSV4 compressed attention: coeff is bytes_per_full_token, inflated by (T+D)/T + for a draft worker; bias is the request-scoped pools that do not scale with it.""" # object.__new__ stubs (SWA floor tests) skip __init__ _dspark_draft_on_bf16 = False @@ -1038,17 +992,19 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): ) self.context_len = kvc.model_config.context_len # PP-local slice; matches DeepSeekV4TokenToKVPool's stage_ratios. - self.compression_ratios = cfg.compress_ratios[ - kvc.layer_info.start_layer : kvc.layer_info.end_layer - ] + stage = range(kvc.layer_info.start_layer, kvc.layer_info.end_layer) + self.stage_compress_ratios = cfg.compress_ratios[stage.start : stage.stop] if kvc.pp_size > 1: logger.info( f"DSV4 pool PP slice: rank={kvc.pp_group.rank_in_group} " - f"layers=[{kvc.layer_info.start_layer},{kvc.layer_info.end_layer}) " - f"local={len(self.compression_ratios)}/{len(cfg.compress_ratios)}" + f"layers=[{stage.start},{stage.stop}) " + f"local={len(self.stage_compress_ratios)}/{len(cfg.compress_ratios)}" ) - self.swa_window_size = cfg.window_size - self.swa_page_size = get_schedule().page_size + # Layers of this stage that own compressed storage, per ratio. Same rule + # as the pool, so a ratio budgeted here is one it allocates. + self.stage_owner_layers = collect_sources_by_ratio( + cfg.compress_ratios, cfg.hf_config.kv_source_layer_ids, stage + ) self.operator_swa_ratio = _operator_swa_full_tokens_ratio() self.swa_ratio = ( self.operator_swa_ratio @@ -1079,49 +1035,26 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): if self.c4_shrink_factor > 1: logger.info(f"HiSparse c4 host-to-device ratio = {self.c4_shrink_factor}") - # Same argument as the pool's get_ring_size, so both sides agree. - num_draft_tokens = ( - (get_spec().speculative_num_draft_tokens or 0) if self.is_speculative else 0 - ) - self.c4_ring_size = get_compress_state_ring_size(4, num_draft_tokens) - self.c128_ring_size = get_compress_state_ring_size(128, num_draft_tokens) - self.c2_ring_size = get_compress_state_ring_size(2, num_draft_tokens) + # Ratio 1 keeps no compress state, so it has no ring. + self.ring_sizes = { + ratio: resolve_compress_state_ring_size(ratio) + for ratio in self.stage_owner_layers + if ratio != 1 + } - self.num_layers_total = len(self.compression_ratios) - self.num_layers_ca4 = sum(1 for r in self.compression_ratios if r == 4) - self.num_layers_ca128 = sum(1 for r in self.compression_ratios if r == 128) - # Only a ratio-2 kv_source layer compresses, so only it keeps pair state. - self.num_layers_c2_source = sum( - 1 - for l in cfg.hf_config.kv_source_layer_ids - if kvc.layer_info.start_layer <= l < kvc.layer_info.end_layer - and cfg.compress_ratios[l] == 2 - ) + self.num_layers_total = len(self.stage_compress_ratios) # The low-ratio indexer pools are built with force_fp4=True # (deepseek_v4_memory_pool), so they are fp4 whatever dtype c4 uses. - low_ratio_index_bytes = get_dsv4_indexer_bytes_per_token( + self.low_ratio_index_bytes = get_dsv4_indexer_bytes_per_token( self.indexer_head_dim, use_fp4_indexer=True ) - self.low_ratio_bytes_per_full_token = sum( - (self.kv_bytes + low_ratio_index_bytes) / cfg.compress_ratios[l] - for l in cfg.hf_config.kv_source_layer_ids - if kvc.layer_info.start_layer <= l < kvc.layer_info.end_layer - and cfg.compress_ratios[l] in (1, 2) - ) - from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( - dsv4_unified_row_bytes, - ) - # swa_window_size is the model's sliding window (cfg.window_size). - self._swa_ring_size = get_swa_ring_size( - self.swa_window_size, self.is_speculative - ) + # kvc.sliding_window_size is None on a runner without SWA layers. + self._swa_ring_size = get_swa_ring_size(cfg.window_size, self.is_speculative) self._spec_infl = 1.0 - # The unified pool takes no dtype, so --kv-cache-dtype never reaches it. - # V4 defaults "auto" to fp8_e4m3 (overrides.py - # _deepseek_v4_kv_cache_dtype), so only a bfloat16 here tells us the user - # set it explicitly; warning on the fp8 side would fire on every run. + # The unified pool ignores --kv-cache-dtype; V4 resolves "auto" to fp8_e4m3 + # (overrides.py _deepseek_v4_kv_cache_dtype), so only bfloat16 is explicit. if self._unified_fp8 and self.kv_cache_dtype_str == "bfloat16": logger.warning( "--kv-cache-dtype=bfloat16 is ignored on the unified_kv path; " @@ -1129,9 +1062,8 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): "env switch to get a bf16 unified pool." ) - # get_contiguous_buf_infos ships one pointer per layer and prices a row as - # buf[0].nbytes, which under fp8 covers the nope pool only. Fail at startup - # rather than at the first transfer. + # get_contiguous_buf_infos prices a row as buf[0].nbytes, which under fp8 + # covers the nope pool only; fail at startup rather than at the first transfer. # TODO(danli103): drop this once the transfer ships the rope pool. if self._unified_fp8 and self.disaggregation_mode != "null": raise ValueError( @@ -1185,20 +1117,16 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): self.bytes_per_swa_token = self._get_bytes_per_swa_token() self.bytes_per_full_token = self._get_bytes_per_full_token() if self.is_speculative and not self.encoder_replay: - # Reserve memory for the speculative draft worker by inflating - # per-token bytes by (target+draft)/target. Equivalent to dflash's - # scale_kv_cell_size_per_token_for_dflash but applied to - # bytes_per_full_token: tokens = avail / (bpft * (T+D)/T). + # Reserve the draft worker by inflating per-token bytes by + # (target+draft)/target, as scale_kv_cell_size_per_token_for_dflash does. draft_layers = 1 target_layers = self.num_layers_total self._spec_infl = (target_layers + draft_layers) / target_layers self.bytes_per_full_token *= self._spec_infl self.bytes_per_swa_token *= self._spec_infl - # Online c128 keeps a single in-progress (max, sum, kv) state per index - # and assumes a strict forward-only schedule. Speculative decode (MTP) - # would need rollback / replay across draft and verify, which the - # online path doesn't support yet. + # Online c128 keeps one in-progress (max, sum, kv) state per index and + # assumes forward-only; MTP would need rollback across draft and verify. if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get(): allow_experimental_online_c128_mtp = ( envs.SGLANG_EXPERIMENTAL_ONLINE_C128_MTP.get() @@ -1225,18 +1153,20 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): "DSV4 compressed attention: online c128 enabled (ring_size=1)" ) + def num_layers(self, ratio: int) -> int: + """Layers of this stage owning ratio's compressed storage.""" + return len(self.stage_owner_layers.get(ratio, ())) + def _assert_ring_serves_draft_tokens(self, num_draft_tokens: int) -> None: """A verify batch writes its whole optimistic tail into the ring, so ring capacity bounds the draft count.""" - for compress_ratio, ring_size, num_layers in ( - (4, self.c4_ring_size, self.num_layers_ca4), - (128, self.c128_ring_size, self.num_layers_ca128), - ): - if num_layers == 0: + for compress_ratio in (4, 128): + if self.num_layers(compress_ratio) == 0: continue if compress_ratio == 128 and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get(): # Online c128 keeps per-draft state instead of a ring; sized separately. continue + ring_size = self.ring_sizes[compress_ratio] max_draft_tokens = get_compress_state_write_pad(compress_ratio, ring_size) assert num_draft_tokens <= max_draft_tokens, ( f"speculative_num_draft_tokens={num_draft_tokens} exceeds what the c{compress_ratio} " @@ -1284,7 +1214,7 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): def _get_bytes_per_swa_token(self) -> float: """Bytes one SWA slot costs across the stage. c4_state_pool_size = swa_tokens - / swa_page_size * ring, so c4 compress state is priced per SWA slot too.""" + // page_size * ring, so c4 compress state is priced per SWA slot too.""" if self.encoder_replay: # Target SWA lives in the request window; only the draft owns paged SWA # bytes, and its layers carry no compressed state. @@ -1293,46 +1223,35 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): c4_state_bytes = 2 * 2 * self.attn_head_dim * c4_state_dtype_size c4_indexer_state_bytes = 2 * 2 * self.indexer_head_dim * c4_state_dtype_size - c4_state_ratio = self.c4_ring_size / self.swa_page_size - return ( - self.kv_bytes * self.num_layers_total - + c4_state_ratio - * (c4_state_bytes + c4_indexer_state_bytes) - * self.num_layers_ca4 - ) + c4_state_ratio = self.ring_sizes.get(4, 0) / self.page_size + return self.kv_bytes * self.num_layers_total + c4_state_ratio * ( + c4_state_bytes + c4_indexer_state_bytes + ) * self.num_layers(4) + + def _compressed_bytes_per_full_token(self, ratio: int) -> float: + """Compressed KV (+ indexer) bytes one full token adds per layer of `ratio`; + compress state is priced per SWA slot (c4) or as fixed bytes (c128, ratio 2).""" + if ratio in (1, 2): + return (self.kv_bytes + self.low_ratio_index_bytes) / ratio + if ratio == 4: + c4_frac = 1 / (4 * self.c4_shrink_factor) + return c4_frac * self.kv_bytes + 1 / 4 * self.indexer_bytes_per_token + assert ratio == 128, f"unsupported compression ratio: {ratio}" + return 1 / 128 * self.kv_bytes def _get_bytes_per_full_token(self) -> float: - _, c128_state_dtype_size = _get_dsv4_compress_state_dtype_sizes() - # Online c128 stores (max, sum, kv) per slot (3*head_dim) instead of - # raw (kv, score) (2*head_dim). Combined with ring_size=1 this still - # nets a large reduction (~3/256x) but the per-slot bytes go up. - c128_online = envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get() - c128_state_bytes = ( - (3 if c128_online else 2 * 1) * self.attn_head_dim * c128_state_dtype_size - ) - - # C128 state is request-scoped and is finalized after - # max_running_requests is known, so it should not scale with - # full-token capacity here. - c128_state_ratio = 0 # Cap mode and ring mode both move the SWA pool and the c4 state that # follows it out of the coefficient and into fixed bytes. swa_ratio = ( 0 if self._unified or self.swa_cap_tokens is not None else self.swa_ratio ) - - c4_frac = 1 / (4 * self.c4_shrink_factor) - return ( - swa_ratio * self.bytes_per_swa_token - + self.low_ratio_bytes_per_full_token - + c4_frac * self.kv_bytes * self.num_layers_ca4 - + 1 / 128 * self.kv_bytes * self.num_layers_ca128 - + 1 / 4 * self.indexer_bytes_per_token * self.num_layers_ca4 - + c128_state_ratio * c128_state_bytes * self.num_layers_ca128 + return swa_ratio * self.bytes_per_swa_token + sum( + self._compressed_bytes_per_full_token(ratio) * len(layers) + for ratio, layers in self.stage_owner_layers.items() ) def _get_swa_fixed_bytes(self) -> float: - """Bias bytes the SWA pool takes in cap mode; 0 when sizing by ratio.""" + """Bias bytes of the encoder-replay request window plus the cap-mode pool.""" paged_bytes = ( 0 if self.swa_cap_tokens is None @@ -1378,10 +1297,11 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): c4_max_total_num_tokens=full_token // (4 * self.c4_shrink_factor), c128_max_total_num_tokens=full_token // 128, # Unified_kv: request-scoped, finalized once concurrency is known. + # Otherwise the ring is addressed per SWA page (swa_loc // page_size). c4_state_pool_size=( 0 if self._unified - else swa_tokens // self.swa_page_size * self.c4_ring_size + else swa_tokens // page_size * self.ring_sizes.get(4, 0) ), c128_state_pool_size=0, ) @@ -1392,63 +1312,60 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): return max_running_requests + 1 def _get_c128_state_fixed_bytes(self, max_running_requests: int) -> int: - if self.num_layers_ca128 == 0: + num_layers = self.num_layers(128) + if num_layers == 0: return 0 _, c128_state_dtype_size = _get_dsv4_compress_state_dtype_sizes() num_req_slots = self._get_num_req_slots(max_running_requests) + ring_size = self.ring_sizes[128] if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get(): - state_rows = num_req_slots + self.c128_ring_size + 1 + state_rows = num_req_slots + ring_size + 1 state_rows *= 1 + self.online_c128_mtp_max_draft_tokens state_last_dim = 3 * self.attn_head_dim else: - state_pool_size = num_req_slots * self.c128_ring_size - state_rows = state_pool_size + self.c128_ring_size + 1 + state_pool_size = num_req_slots * ring_size + state_rows = state_pool_size + ring_size + 1 state_rows = ceil_div(state_rows, 128) * 128 state_last_dim = 2 * self.attn_head_dim - return ( - state_rows * state_last_dim * c128_state_dtype_size * self.num_layers_ca128 - ) + return state_rows * state_last_dim * c128_state_dtype_size * num_layers def _get_c2_state_fixed_bytes(self, max_running_requests: int) -> int: """Ratio-2 pending-pair state, one fp32 (kv, score) ring per request slot on each ratio-2 kv_source layer; mirrors _make_pair_state_pool.""" - if self.num_layers_c2_source == 0: + num_layers = self.num_layers(2) + if num_layers == 0: return 0 num_req_slots = self._get_num_req_slots(max_running_requests) - ring_size = self.c2_ring_size + ring_size = self.ring_sizes[2] # CompressStatePool allocates `size + ring_size + 1` rows, padded to the ratio. state_rows = num_req_slots * ring_size + ring_size + 1 state_rows = ceil_div(state_rows, 2) * 2 state_last_dim = 2 * self.attn_head_dim - return ( - state_rows - * state_last_dim - * torch.float32.itemsize - * self.num_layers_c2_source - ) + return state_rows * state_last_dim * torch.float32.itemsize * num_layers def _unified_c4_state_pool_size(self, max_running_requests: int) -> int: # Unified C4 state loc is req_pool_idx * c4_ring_size + pos % c4_ring_size. num_req_slots = self._get_num_req_slots(max_running_requests) - return num_req_slots * self.c4_ring_size + return num_req_slots * self.ring_sizes[4] def _fixed_c4_state_bytes(self, max_running_requests: int) -> int: - if not self._unified or self.num_layers_ca4 == 0: + num_layers = self.num_layers(4) + if not self._unified or num_layers == 0: return 0 c4_state_dtype_size, _ = _get_dsv4_compress_state_dtype_sizes() # Mirror CompressStatePool.__init__: it allocates `size + ring_size + 1` # rows, padded to the compress ratio. state_rows = self._unified_c4_state_pool_size(max_running_requests) - state_rows = ceil_div(state_rows + self.c4_ring_size + 1, 4) * 4 + state_rows = ceil_div(state_rows + self.ring_sizes[4] + 1, 4) * 4 # overlap c4: last_dim = 2 * (1 + overlap) * head_dim = 4 * head_dim. core_bytes = 4 * self.attn_head_dim * c4_state_dtype_size indexer_bytes = 4 * self.indexer_head_dim * c4_state_dtype_size - return state_rows * (core_bytes + indexer_bytes) * self.num_layers_ca4 + return state_rows * (core_bytes + indexer_bytes) * num_layers def _resolve_max_running_requests_per_worker(self, available_bytes: int) -> int: # Approximates ModelRunner._resolve_max_num_reqs. Over-estimating is safe: @@ -1462,10 +1379,8 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): return min(estimated, full_token // 2) def _fixed_swa_bytes(self, max_running_requests: int) -> int: - """Unified_kv SWA is a fixed per-request ring, sized by concurrency - (num_req_slots) rather than by full_token. MTP inflates the target ring - by _spec_infl; DSpark+fp8 adds a bf16 draft ring instead (640 vs 1024). - Returns 0 on the non-unified path (SWA already counted per-token).""" + """Unified_kv SWA ring bytes, sized by req slots; 0 off unified, where SWA is + priced per token. DSpark+fp8 adds a bf16 draft ring instead of _spec_infl.""" if not self._unified: return 0 num_req_slots = self._get_num_req_slots(max_running_requests) @@ -1483,9 +1398,8 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): draft_row = dsv4_unified_row_bytes( self.qk_nope_head_dim, self.qk_rope_head_dim, fp8=False ) - # 1 layer is what the shipped DSpark drafts allocate. A multi-stage - # draft would under-count by ~9 MB/layer (128-wide window, ~65 req - # slots), which the (T+1)/T on bytes_per_full_token already covers. + # 1 layer is what shipped DSpark drafts allocate; a multi-layer draft + # under-counts ~9 MB/layer, covered by the (T+1)/T inflation. draft_ring = num_req_slots * self._swa_ring_size * draft_row return int(target_ring + draft_ring) return int(target_ring * self._spec_infl) @@ -1518,9 +1432,9 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): if envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get(): config.c128_state_pool_size = num_req_slots else: - config.c128_state_pool_size = num_req_slots * self.c128_ring_size + config.c128_state_pool_size = num_req_slots * self.ring_sizes.get(128, 0) # Ring mode: C4 state is request-scoped, so size it from the known concurrency. - if self._unified and self.num_layers_ca4 > 0: + if self._unified and self.num_layers(4) > 0: config.c4_state_pool_size = self._unified_c4_state_pool_size( config.max_running_requests ) @@ -1597,7 +1511,6 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): def create_memory_pool_configurator( kvc: KVCacheConfigurator, ) -> MemoryPoolConfigurator: - """Factory: select the right configurator for the model architecture.""" if is_deepseek_v4(kvc.model_config.hf_config) and kvc.is_hybrid_swa: return DSV4PoolConfigurator(kvc) if kvc.is_hybrid_swa: diff --git a/test/registered/unit/mem_cache/test_dsv4_c4_state_lifecycle.py b/test/registered/unit/mem_cache/test_dsv4_c4_state_lifecycle.py index c0b2b7a1a8..5081b38051 100644 --- a/test/registered/unit/mem_cache/test_dsv4_c4_state_lifecycle.py +++ b/test/registered/unit/mem_cache/test_dsv4_c4_state_lifecycle.py @@ -67,7 +67,7 @@ class TestUnifiedC4StateLifecycle(unittest.TestCase): configurator = object.__new__(DSV4PoolConfigurator) configurator.disaggregation_mode = "decode" configurator.disaggregation_decode_extra_slots = 3 - configurator.c4_ring_size = 16 + configurator.ring_sizes = {4: 16} self.assertEqual(configurator._unified_c4_state_pool_size(10), 14 * 16) diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index f388850d78..2800da18a1 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -1,9 +1,4 @@ -"""Unit tests for pool_configurator.py -- CPU only, no GPU required. - -Tests the end-to-end computation: available_bytes -> MemoryPoolConfig, -verifying tokens are correct, constraints are respected, and memory -invariants hold (tokens * per_token_cost <= available_bytes). -""" +"""CPU-only tests for pool_configurator.py: available_bytes -> MemoryPoolConfig.""" import contextlib import unittest @@ -25,12 +20,8 @@ register_cpu_ci(est_time=11, suite="base-a-test-cpu") @contextlib.contextmanager def mock_cpu_env(kv_size=2, tp_size=1, swa_eviction_interval=4): - """Mock GPU-dependent functions for CPU-only testing. - - swa_eviction_interval pins SGLANG_SWA_EVICTION_INTERVAL (decode batches between - SWA evictions) to a small value so the chunk-cap formula stays hand-computable; - only SWAChunkCapPoolConfigurator reads it. - """ + """Mock GPU-dependent state; swa_eviction_interval is kept small so the + chunk-cap formula stays hand-computable.""" from sglang.srt.environ import envs with ( @@ -51,11 +42,8 @@ def mock_cpu_env(kv_size=2, tp_size=1, swa_eviction_interval=4): def _publish_config(testcase, **fields): - """Publish the configuration a runner double describes. - - Installed per call and restored on the calling case's cleanup, so a failed - case cannot leave a partial publish for a later file in a monolithic run. - """ + """Restored on the case's cleanup, so a failed case cannot leak a partial + publish into a later file of a monolithic run.""" from sglang.srt.runtime_context import get_context override = get_context().override_server_args(**fields) @@ -141,9 +129,8 @@ def _make_model_runner( mr.model_config = mc mr.kv_cache_dtype = "fake_bf16" - # The configurator reads the published bags, so the fixture publishes the - # configuration it describes. The instance stays for the whole-object - # hand-offs the configurator still does. + # The configurator reads the published bags; mr.server_args still backs its + # whole-object hand-offs. _publish_config( testcase, max_total_tokens=None, @@ -679,14 +666,11 @@ class TestHybridSWAConfigurator(CustomTestCase): cfg = create_memory_pool_configurator(mr) config = cfg.calculate_pool_sizes(available, page_size=1) - # disagg decode drops the prefill term: per req = 4 + 1 + 4 + 1 = 10 (as above). + # disagg decode drops the prefill term: per req = 4 + 1 + 4 + 1 = 10. self.assertEqual(config.swa_max_total_num_tokens, 100) self.assertLessEqual(_actual_memory_used(mr, config), available) def test_chunk_cache_cap_prefill_holds_window_plus_chunk(self): - # Non-decode (prefill) engine: each request keeps its decode footprint, while - # in-flight chunked-prefill tokens are a global batch budget -- two chunks - # under overlap. available = 1_000_000 mr = _make_model_runner( self, @@ -1091,7 +1075,7 @@ class TestDflashDraftKvBudget(CustomTestCase): ) def test_hybrid_swa_budget_shrinks_by_draft_pool(self): - """HybridSWA carried no draft term, so the draft pool fell outside the budget.""" + """The HybridSWA budget must charge the DFLASH draft pool.""" available = 10_000_000 def _tokens(draft_kv_per_token): @@ -1157,14 +1141,13 @@ class TestSWAPoolFloor(CustomTestCase): def _dsv4_sizes(self, max_tokens, page_size, unified=False): """Exercise the DSV4 size arithmetic without a full V4 model fixture: - _compute_dsv4_sizes reads only these six attributes.""" + _compute_dsv4_sizes reads only these attributes.""" from sglang.srt.model_executor.pool_configurator import DSV4PoolConfigurator cfg = object.__new__(DSV4PoolConfigurator) cfg.swa_ratio = 0.1 cfg.sliding_window_size = 128 - cfg.swa_page_size = page_size - cfg.c4_ring_size = 8 + cfg.ring_sizes = {4: 8} cfg.c4_shrink_factor = 1 cfg._unified = unified cfg.operator_swa_ratio = None @@ -1173,9 +1156,8 @@ class TestSWAPoolFloor(CustomTestCase): return cfg._compute_dsv4_sizes(max_tokens, page_size) def test_dsv4_rejects_single_page_pool(self): - # DeepSeek-V4-Flash defaults: page_size=256, swa_full_tokens_ratio=0.1. - # int(4096 * 0.1) page-aligns down to 256 -- exactly one page, below the - # 128 + 256 floor. + # DeepSeek-V4-Flash defaults (page 256, ratio 0.1): int(4096 * 0.1) aligns + # down to one page, below the 128 + 256 floor. with self.assertRaisesRegex(ValueError, "cannot hold even one request"): self._dsv4_sizes(max_tokens=4096, page_size=256) @@ -1188,8 +1170,7 @@ class TestSWAPoolFloor(CustomTestCase): self.assertEqual(sizes.c4_state_pool_size, 3072 // 256 * 8) def test_dsv4_token_cap_never_grows_total_footprint(self): - """Regression: the token-cap path subtracts no fixed-pool bias, so - capping the budget-derived token count must still shrink the total.""" + """Capping the budget-derived token count must not grow the total footprint.""" cfg = self._dsv4_configurator_for_budget() page_size = 128 budget = 256 * (1 << 30) @@ -1216,12 +1197,9 @@ class TestSWAPoolFloor(CustomTestCase): cfg.attn_head_dim = 192 cfg.indexer_head_dim = 128 cfg.num_layers_total = 61 - cfg.num_layers_ca4 = 61 - cfg.num_layers_ca128 = 61 - cfg.c4_ring_size = 8 - cfg.c128_ring_size = 128 - cfg.num_layers_c2_source = 0 - cfg.c2_ring_size = 2 + # Every layer owns both a c4 and a c128 store: an upper bound, not a model. + cfg.stage_owner_layers = {4: list(range(61)), 128: list(range(61))} + cfg.ring_sizes = {4: 8, 128: 128} cfg._swa_ring_size = 128 cfg._spec_infl = 1.0 cfg.context_len = 65536 @@ -1229,7 +1207,7 @@ class TestSWAPoolFloor(CustomTestCase): cfg.requested_max_running_requests_per_worker = None cfg.swa_ratio = 0.1 cfg.sliding_window_size = 4096 - cfg.swa_page_size = 128 + cfg.page_size = 128 cfg.c4_shrink_factor = 1 cfg.online_c128_mtp_max_draft_tokens = 0 cfg.disaggregation_mode = None @@ -1266,17 +1244,30 @@ class TestSWAPoolFloor(CustomTestCase): cfg = object.__new__(DSV4PoolConfigurator) cfg.attn_head_dim = 512 - cfg.num_layers_c2_source = 2 - cfg.c2_ring_size = 2 + cfg.stage_owner_layers = {2: [2, 8], 1: [20]} + cfg.ring_sizes = {2: 2} cfg.disaggregation_mode = None cfg.disaggregation_decode_extra_slots = 0 slots = 10 + 1 # size + ring + 1 rows, padded to the ratio (2); 2 * head_dim fp32 per row. rows = (slots * 2 + 2 + 1 + 1) // 2 * 2 self.assertEqual(cfg._get_c2_state_fixed_bytes(10), rows * 2 * 512 * 4 * 2) - cfg.num_layers_c2_source = 0 + cfg.stage_owner_layers = {1: [20]} self.assertEqual(cfg._get_c2_state_fixed_bytes(10), 0) + def test_dsv4_owner_layers_follow_the_pool_rule(self): + """The configurator budgets the layers the pool allocates: every ratio-4 + / 128 layer, only the kv_source layers of ratios 1 / 2, all PP-local.""" + from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( + collect_sources_by_ratio, + ) + + ratios = [0, 0, 2, 2, 1, 1, 4, 128, 4, 2] + owners = collect_sources_by_ratio(ratios, [2, 4, 9], range(0, 8)) + self.assertEqual(owners, {4: [6], 128: [7], 1: [4], 2: [2]}) + owners = collect_sources_by_ratio(ratios, [2, 4, 9], range(8, 10)) + self.assertEqual(owners, {4: [8], 2: [9]}) + def test_dsv4_paged_dspark_budget_reserves_window_and_draft_layers(self): from sglang.srt.model_executor.pool_configurator import DSV4PoolConfigurator