diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index d733331098..b1b80e437e 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -43,10 +43,10 @@ def get_dsv4_indexer_bytes_per_token(index_head_dim: int, use_fp4_indexer: bool) return index_head_dim + index_head_dim // 128 * 4 -def get_compress_state_ring_size( - compress_ratio: int, is_speculative: bool = False, num_draft_tokens: int = 0 -) -> int: +def get_compress_state_ring_size(compress_ratio: int, num_draft_tokens: int = 0) -> int: + """Rows per request ring; num_draft_tokens == 0 means no speculative decoding.""" assert compress_ratio in [2, 4, 128], f"Unsupported {compress_ratio = }" + is_speculative = num_draft_tokens > 0 if compress_ratio == 2: # Two positions are one pair, addressed by position % ring_size; a # speculative ring must be wider than the draft window: pow2 >= 2 + drafts. @@ -1164,11 +1164,12 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): def get_ring_size(self, compress_ratio: int) -> int: spec = get_spec() - return get_compress_state_ring_size( - compress_ratio, - spec.speculative_algorithm is not None, - spec.speculative_num_draft_tokens or 0, + num_draft_tokens = ( + 0 + if spec.speculative_algorithm is None + else spec.speculative_num_draft_tokens or 0 ) + return get_compress_state_ring_size(compress_ratio, num_draft_tokens) def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor): assert self.full_to_swa_index_mapping is not None diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index a379e913ed..fce8701364 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -1077,12 +1077,24 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): if self.c4_shrink_factor > 1: logger.info(f"HiSparse c4 host-to-device ratio = {self.c4_shrink_factor}") - self.c4_ring_size = get_compress_state_ring_size(4, self.is_speculative) - self.c128_ring_size = get_compress_state_ring_size(128, self.is_speculative) + # Same argument as the pool's get_ring_size, so both sides agree. + num_draft_tokens = ( + (get_spec().speculative_num_draft_tokens or 0) if self.is_speculative else 0 + ) + self.c4_ring_size = get_compress_state_ring_size(4, num_draft_tokens) + self.c128_ring_size = get_compress_state_ring_size(128, num_draft_tokens) + self.c2_ring_size = get_compress_state_ring_size(2, num_draft_tokens) self.num_layers_total = len(self.compression_ratios) self.num_layers_ca4 = sum(1 for r in self.compression_ratios if r == 4) self.num_layers_ca128 = sum(1 for r in self.compression_ratios if r == 128) + # Only a ratio-2 kv_source layer compresses, so only it keeps pair state. + self.num_layers_c2_source = sum( + 1 + for l in cfg.hf_config.kv_source_layer_ids + if kvc.layer_info.start_layer <= l < kvc.layer_info.end_layer + and cfg.compress_ratios[l] == 2 + ) # The low-ratio indexer pools are built with force_fp4=True # (deepseek_v4_memory_pool), so they are fp4 whatever dtype c4 uses. low_ratio_index_bytes = get_dsv4_indexer_bytes_per_token( @@ -1398,6 +1410,25 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): state_rows * state_last_dim * c128_state_dtype_size * self.num_layers_ca128 ) + def _get_c2_state_fixed_bytes(self, max_running_requests: int) -> int: + """Ratio-2 pending-pair state, one fp32 (kv, score) ring per request slot + on each ratio-2 kv_source layer; mirrors _make_pair_state_pool.""" + if self.num_layers_c2_source == 0: + return 0 + + num_req_slots = self._get_num_req_slots(max_running_requests) + ring_size = self.c2_ring_size + # CompressStatePool allocates `size + ring_size + 1` rows, padded to the ratio. + state_rows = num_req_slots * ring_size + ring_size + 1 + state_rows = ceil_div(state_rows, 2) * 2 + state_last_dim = 2 * self.attn_head_dim + return ( + state_rows + * state_last_dim + * torch.float32.itemsize + * self.num_layers_c2_source + ) + def _unified_c4_state_pool_size(self, max_running_requests: int) -> int: # Unified C4 state loc is req_pool_idx * c4_ring_size + pos % c4_ring_size. num_req_slots = self._get_num_req_slots(max_running_requests) @@ -1506,6 +1537,9 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): c128_state_fixed_bytes = self._get_c128_state_fixed_bytes( max_running_requests_per_worker ) + c2_state_fixed_bytes = self._get_c2_state_fixed_bytes( + max_running_requests_per_worker + ) swa_ring_fixed_bytes = self._fixed_swa_bytes(max_running_requests_per_worker) c4_state_fixed_bytes = self._fixed_c4_state_bytes( max_running_requests_per_worker @@ -1514,6 +1548,7 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): swa_fixed_bytes = self._get_swa_fixed_bytes() fixed_bytes = ( c128_state_fixed_bytes + + c2_state_fixed_bytes + swa_fixed_bytes + swa_ring_fixed_bytes + c4_state_fixed_bytes @@ -1537,6 +1572,7 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): f"bytes_per_full_token={self.bytes_per_full_token:.2f}, " f"available_bytes={available_bytes / (1 << 30):.2f} GB, " f"c128_state_fixed={c128_state_fixed_bytes / (1 << 30):.2f} GB, " + f"c2_state_fixed={c2_state_fixed_bytes / (1 << 30):.2f} GB, " f"swa_fixed={swa_fixed_bytes / (1 << 30):.2f} GB, " f"swa_ring_fixed={swa_ring_fixed_bytes / (1 << 30):.2f} GB, " f"c4_state_fixed={c4_state_fixed_bytes / (1 << 30):.2f} GB, " diff --git a/test/registered/unit/mem_cache/test_dsv4_compress_write_pad.py b/test/registered/unit/mem_cache/test_dsv4_compress_write_pad.py index 93c2c5c0a7..b5a98aecbf 100644 --- a/test/registered/unit/mem_cache/test_dsv4_compress_write_pad.py +++ b/test/registered/unit/mem_cache/test_dsv4_compress_write_pad.py @@ -20,7 +20,7 @@ class TestCompressStateWritePad(CustomTestCase): def test_pad_is_zero_without_speculation(self): """A non-speculative ring is exactly one window wide: nothing rolls back.""" for compress_ratio in (4, 128): - ring_size = get_compress_state_ring_size(compress_ratio, False) + ring_size = get_compress_state_ring_size(compress_ratio) with self.subTest(cr=compress_ratio, ring=ring_size): self.assertEqual( get_compress_state_write_pad(compress_ratio, ring_size), 0 @@ -29,7 +29,7 @@ class TestCompressStateWritePad(CustomTestCase): def test_pad_matches_speculative_ring_capacity(self): """`ring_size - window_size + 2`, with window = 2*cr for the overlapping c4.""" for compress_ratio, expected in ((4, 10), (128, 130)): - ring_size = get_compress_state_ring_size(compress_ratio, True) + ring_size = get_compress_state_ring_size(compress_ratio, num_draft_tokens=4) with self.subTest(cr=compress_ratio, ring=ring_size): self.assertEqual( get_compress_state_write_pad(compress_ratio, ring_size), expected diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index 7d525e408c..a7c47c4609 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -1220,6 +1220,8 @@ class TestSWAPoolFloor(CustomTestCase): cfg.num_layers_ca128 = 61 cfg.c4_ring_size = 8 cfg.c128_ring_size = 128 + cfg.num_layers_c2_source = 0 + cfg.c2_ring_size = 2 cfg._swa_ring_size = 128 cfg._spec_infl = 1.0 cfg.context_len = 65536 @@ -1244,7 +1246,7 @@ class TestSWAPoolFloor(CustomTestCase): cfg._unified_row_bytes = cfg.attn_head_dim * 2 return cfg - # Token pool plus the three request-scoped fixed pools, sized from the + # Token pool plus the request-scoped fixed pools, sized from the # concurrency resolve_max_num_reqs derives from this token count. def _dsv4_total_bytes(self, cfg, tokens): estimated = max(min(int(tokens / cfg.context_len * 512), 4096), 2048) @@ -1254,8 +1256,27 @@ class TestSWAPoolFloor(CustomTestCase): + cfg._fixed_swa_bytes(max_running_requests) + cfg._fixed_c4_state_bytes(max_running_requests) + cfg._get_c128_state_fixed_bytes(max_running_requests) + + cfg._get_c2_state_fixed_bytes(max_running_requests) ) + def test_dsv4_c2_state_fixed_bytes_mirror_pool(self): + """Ratio-2 kv_source layers keep an fp32 (kv, score) ring per request + slot; the budget must charge the rows CompressStatePool allocates.""" + from sglang.srt.model_executor.pool_configurator import DSV4PoolConfigurator + + cfg = object.__new__(DSV4PoolConfigurator) + cfg.attn_head_dim = 512 + cfg.num_layers_c2_source = 2 + cfg.c2_ring_size = 2 + cfg.disaggregation_mode = None + cfg.disaggregation_decode_extra_slots = 0 + slots = 10 + 1 + # size + ring + 1 rows, padded to the ratio (2); 2 * head_dim fp32 per row. + rows = (slots * 2 + 2 + 1 + 1) // 2 * 2 + self.assertEqual(cfg._get_c2_state_fixed_bytes(10), rows * 2 * 512 * 4 * 2) + cfg.num_layers_c2_source = 0 + self.assertEqual(cfg._get_c2_state_fixed_bytes(10), 0) + def test_dsv4_paged_dspark_budget_reserves_window_and_draft_layers(self): from sglang.srt.model_executor.pool_configurator import DSV4PoolConfigurator