[DSV4] Budget the ratio-2 pair state pool in DSV4PoolConfigurator (#41048)

This commit is contained in:
Liangsheng Yin
2026-09-23 23:53:46 -07:00
committed by GitHub
parent a164c6d9f1
commit cae4927fbf
4 changed files with 70 additions and 12 deletions
@@ -43,10 +43,10 @@ def get_dsv4_indexer_bytes_per_token(index_head_dim: int, use_fp4_indexer: bool)
return index_head_dim + index_head_dim // 128 * 4
def get_compress_state_ring_size(
compress_ratio: int, is_speculative: bool = False, num_draft_tokens: int = 0
) -> int:
def get_compress_state_ring_size(compress_ratio: int, num_draft_tokens: int = 0) -> int:
"""Rows per request ring; num_draft_tokens == 0 means no speculative decoding."""
assert compress_ratio in [2, 4, 128], f"Unsupported {compress_ratio = }"
is_speculative = num_draft_tokens > 0
if compress_ratio == 2:
# Two positions are one pair, addressed by position % ring_size; a
# speculative ring must be wider than the draft window: pow2 >= 2 + drafts.
@@ -1164,11 +1164,12 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
def get_ring_size(self, compress_ratio: int) -> int:
spec = get_spec()
return get_compress_state_ring_size(
compress_ratio,
spec.speculative_algorithm is not None,
spec.speculative_num_draft_tokens or 0,
num_draft_tokens = (
0
if spec.speculative_algorithm is None
else spec.speculative_num_draft_tokens or 0
)
return get_compress_state_ring_size(compress_ratio, num_draft_tokens)
def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor):
assert self.full_to_swa_index_mapping is not None
@@ -1077,12 +1077,24 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
if self.c4_shrink_factor > 1:
logger.info(f"HiSparse c4 host-to-device ratio = {self.c4_shrink_factor}")
self.c4_ring_size = get_compress_state_ring_size(4, self.is_speculative)
self.c128_ring_size = get_compress_state_ring_size(128, self.is_speculative)
# Same argument as the pool's get_ring_size, so both sides agree.
num_draft_tokens = (
(get_spec().speculative_num_draft_tokens or 0) if self.is_speculative else 0
)
self.c4_ring_size = get_compress_state_ring_size(4, num_draft_tokens)
self.c128_ring_size = get_compress_state_ring_size(128, num_draft_tokens)
self.c2_ring_size = get_compress_state_ring_size(2, num_draft_tokens)
self.num_layers_total = len(self.compression_ratios)
self.num_layers_ca4 = sum(1 for r in self.compression_ratios if r == 4)
self.num_layers_ca128 = sum(1 for r in self.compression_ratios if r == 128)
# Only a ratio-2 kv_source layer compresses, so only it keeps pair state.
self.num_layers_c2_source = sum(
1
for l in cfg.hf_config.kv_source_layer_ids
if kvc.layer_info.start_layer <= l < kvc.layer_info.end_layer
and cfg.compress_ratios[l] == 2
)
# The low-ratio indexer pools are built with force_fp4=True
# (deepseek_v4_memory_pool), so they are fp4 whatever dtype c4 uses.
low_ratio_index_bytes = get_dsv4_indexer_bytes_per_token(
@@ -1398,6 +1410,25 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
state_rows * state_last_dim * c128_state_dtype_size * self.num_layers_ca128
)
def _get_c2_state_fixed_bytes(self, max_running_requests: int) -> int:
"""Ratio-2 pending-pair state, one fp32 (kv, score) ring per request slot
on each ratio-2 kv_source layer; mirrors _make_pair_state_pool."""
if self.num_layers_c2_source == 0:
return 0
num_req_slots = self._get_num_req_slots(max_running_requests)
ring_size = self.c2_ring_size
# CompressStatePool allocates `size + ring_size + 1` rows, padded to the ratio.
state_rows = num_req_slots * ring_size + ring_size + 1
state_rows = ceil_div(state_rows, 2) * 2
state_last_dim = 2 * self.attn_head_dim
return (
state_rows
* state_last_dim
* torch.float32.itemsize
* self.num_layers_c2_source
)
def _unified_c4_state_pool_size(self, max_running_requests: int) -> int:
# Unified C4 state loc is req_pool_idx * c4_ring_size + pos % c4_ring_size.
num_req_slots = self._get_num_req_slots(max_running_requests)
@@ -1506,6 +1537,9 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
c128_state_fixed_bytes = self._get_c128_state_fixed_bytes(
max_running_requests_per_worker
)
c2_state_fixed_bytes = self._get_c2_state_fixed_bytes(
max_running_requests_per_worker
)
swa_ring_fixed_bytes = self._fixed_swa_bytes(max_running_requests_per_worker)
c4_state_fixed_bytes = self._fixed_c4_state_bytes(
max_running_requests_per_worker
@@ -1514,6 +1548,7 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
swa_fixed_bytes = self._get_swa_fixed_bytes()
fixed_bytes = (
c128_state_fixed_bytes
+ c2_state_fixed_bytes
+ swa_fixed_bytes
+ swa_ring_fixed_bytes
+ c4_state_fixed_bytes
@@ -1537,6 +1572,7 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
f"bytes_per_full_token={self.bytes_per_full_token:.2f}, "
f"available_bytes={available_bytes / (1 << 30):.2f} GB, "
f"c128_state_fixed={c128_state_fixed_bytes / (1 << 30):.2f} GB, "
f"c2_state_fixed={c2_state_fixed_bytes / (1 << 30):.2f} GB, "
f"swa_fixed={swa_fixed_bytes / (1 << 30):.2f} GB, "
f"swa_ring_fixed={swa_ring_fixed_bytes / (1 << 30):.2f} GB, "
f"c4_state_fixed={c4_state_fixed_bytes / (1 << 30):.2f} GB, "