[DSV4] Budget the ratio-2 pair state pool in DSV4PoolConfigurator (#41048)
This commit is contained in:
@@ -43,10 +43,10 @@ def get_dsv4_indexer_bytes_per_token(index_head_dim: int, use_fp4_indexer: bool)
|
||||
return index_head_dim + index_head_dim // 128 * 4
|
||||
|
||||
|
||||
def get_compress_state_ring_size(
|
||||
compress_ratio: int, is_speculative: bool = False, num_draft_tokens: int = 0
|
||||
) -> int:
|
||||
def get_compress_state_ring_size(compress_ratio: int, num_draft_tokens: int = 0) -> int:
|
||||
"""Rows per request ring; num_draft_tokens == 0 means no speculative decoding."""
|
||||
assert compress_ratio in [2, 4, 128], f"Unsupported {compress_ratio = }"
|
||||
is_speculative = num_draft_tokens > 0
|
||||
if compress_ratio == 2:
|
||||
# Two positions are one pair, addressed by position % ring_size; a
|
||||
# speculative ring must be wider than the draft window: pow2 >= 2 + drafts.
|
||||
@@ -1164,11 +1164,12 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool):
|
||||
|
||||
def get_ring_size(self, compress_ratio: int) -> int:
|
||||
spec = get_spec()
|
||||
return get_compress_state_ring_size(
|
||||
compress_ratio,
|
||||
spec.speculative_algorithm is not None,
|
||||
spec.speculative_num_draft_tokens or 0,
|
||||
num_draft_tokens = (
|
||||
0
|
||||
if spec.speculative_algorithm is None
|
||||
else spec.speculative_num_draft_tokens or 0
|
||||
)
|
||||
return get_compress_state_ring_size(compress_ratio, num_draft_tokens)
|
||||
|
||||
def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor):
|
||||
assert self.full_to_swa_index_mapping is not None
|
||||
|
||||
@@ -1077,12 +1077,24 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
if self.c4_shrink_factor > 1:
|
||||
logger.info(f"HiSparse c4 host-to-device ratio = {self.c4_shrink_factor}")
|
||||
|
||||
self.c4_ring_size = get_compress_state_ring_size(4, self.is_speculative)
|
||||
self.c128_ring_size = get_compress_state_ring_size(128, self.is_speculative)
|
||||
# Same argument as the pool's get_ring_size, so both sides agree.
|
||||
num_draft_tokens = (
|
||||
(get_spec().speculative_num_draft_tokens or 0) if self.is_speculative else 0
|
||||
)
|
||||
self.c4_ring_size = get_compress_state_ring_size(4, num_draft_tokens)
|
||||
self.c128_ring_size = get_compress_state_ring_size(128, num_draft_tokens)
|
||||
self.c2_ring_size = get_compress_state_ring_size(2, num_draft_tokens)
|
||||
|
||||
self.num_layers_total = len(self.compression_ratios)
|
||||
self.num_layers_ca4 = sum(1 for r in self.compression_ratios if r == 4)
|
||||
self.num_layers_ca128 = sum(1 for r in self.compression_ratios if r == 128)
|
||||
# Only a ratio-2 kv_source layer compresses, so only it keeps pair state.
|
||||
self.num_layers_c2_source = sum(
|
||||
1
|
||||
for l in cfg.hf_config.kv_source_layer_ids
|
||||
if kvc.layer_info.start_layer <= l < kvc.layer_info.end_layer
|
||||
and cfg.compress_ratios[l] == 2
|
||||
)
|
||||
# The low-ratio indexer pools are built with force_fp4=True
|
||||
# (deepseek_v4_memory_pool), so they are fp4 whatever dtype c4 uses.
|
||||
low_ratio_index_bytes = get_dsv4_indexer_bytes_per_token(
|
||||
@@ -1398,6 +1410,25 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
state_rows * state_last_dim * c128_state_dtype_size * self.num_layers_ca128
|
||||
)
|
||||
|
||||
def _get_c2_state_fixed_bytes(self, max_running_requests: int) -> int:
|
||||
"""Ratio-2 pending-pair state, one fp32 (kv, score) ring per request slot
|
||||
on each ratio-2 kv_source layer; mirrors _make_pair_state_pool."""
|
||||
if self.num_layers_c2_source == 0:
|
||||
return 0
|
||||
|
||||
num_req_slots = self._get_num_req_slots(max_running_requests)
|
||||
ring_size = self.c2_ring_size
|
||||
# CompressStatePool allocates `size + ring_size + 1` rows, padded to the ratio.
|
||||
state_rows = num_req_slots * ring_size + ring_size + 1
|
||||
state_rows = ceil_div(state_rows, 2) * 2
|
||||
state_last_dim = 2 * self.attn_head_dim
|
||||
return (
|
||||
state_rows
|
||||
* state_last_dim
|
||||
* torch.float32.itemsize
|
||||
* self.num_layers_c2_source
|
||||
)
|
||||
|
||||
def _unified_c4_state_pool_size(self, max_running_requests: int) -> int:
|
||||
# Unified C4 state loc is req_pool_idx * c4_ring_size + pos % c4_ring_size.
|
||||
num_req_slots = self._get_num_req_slots(max_running_requests)
|
||||
@@ -1506,6 +1537,9 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
c128_state_fixed_bytes = self._get_c128_state_fixed_bytes(
|
||||
max_running_requests_per_worker
|
||||
)
|
||||
c2_state_fixed_bytes = self._get_c2_state_fixed_bytes(
|
||||
max_running_requests_per_worker
|
||||
)
|
||||
swa_ring_fixed_bytes = self._fixed_swa_bytes(max_running_requests_per_worker)
|
||||
c4_state_fixed_bytes = self._fixed_c4_state_bytes(
|
||||
max_running_requests_per_worker
|
||||
@@ -1514,6 +1548,7 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
swa_fixed_bytes = self._get_swa_fixed_bytes()
|
||||
fixed_bytes = (
|
||||
c128_state_fixed_bytes
|
||||
+ c2_state_fixed_bytes
|
||||
+ swa_fixed_bytes
|
||||
+ swa_ring_fixed_bytes
|
||||
+ c4_state_fixed_bytes
|
||||
@@ -1537,6 +1572,7 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator):
|
||||
f"bytes_per_full_token={self.bytes_per_full_token:.2f}, "
|
||||
f"available_bytes={available_bytes / (1 << 30):.2f} GB, "
|
||||
f"c128_state_fixed={c128_state_fixed_bytes / (1 << 30):.2f} GB, "
|
||||
f"c2_state_fixed={c2_state_fixed_bytes / (1 << 30):.2f} GB, "
|
||||
f"swa_fixed={swa_fixed_bytes / (1 << 30):.2f} GB, "
|
||||
f"swa_ring_fixed={swa_ring_fixed_bytes / (1 << 30):.2f} GB, "
|
||||
f"c4_state_fixed={c4_state_fixed_bytes / (1 << 30):.2f} GB, "
|
||||
|
||||
Reference in New Issue
Block a user