From 9894dc789c3009e6bdd4d8a61338bf9fd2d09194 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Sun, 11 Oct 2026 17:37:21 +0800 Subject: [PATCH] G9 P0 fixes: v2 pool_transfers for storage hit query, disagg drain hook, query gate - unified_radix_cache.query_storage_hit_length: build PREFETCH-phase pool_transfers (comp prepare_prefetch + build_hicache_transfers + sidecars) so DeepSeek V4's logical-anchor KV goes through batch_exists_v2 side-pool existence instead of v1 batch_exists which always misses (no physical KV objects are stored for V4). - scheduler.handle_query_cache_hit: allow L3 query when hicache storage backend is attached (not only decode-hicache mode; SWA models forbid decode radix+hicache so P nodes could never query L3); construct RadixKey from array('q') to match tree node key types. - disaggregation/prefill: record_prefill_drain on final chunk in the disagg-prefill result path (it bypasses batch_result_processor, so drain_rate_tokens_per_s stayed 0 on P nodes). - deploy: 3p1d l3test compose against the real mooncake store stack. (cherry picked from commit e7dc175e9494979a51f72a348ad53d797df1359e) --- python/sglang/srt/disaggregation/prefill.py | 4 +++ python/sglang/srt/managers/scheduler.py | 18 ++++++++-- .../srt/mem_cache/unified_radix_cache.py | 35 +++++++++++++++++++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index b72dc2a296..39b6a212b3 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -890,6 +890,10 @@ class SchedulerDisaggregationPrefillMixin: continue self.cache_unfinished_disagg_prefill(req) + # G9: the disagg-prefill path bypasses batch_result_processor's + # prefill loop, so record the drain event here — final chunk + # done, request heading to the inflight (KV transfer) queue. + self.record_prefill_drain(req.seqlen) self.disagg_prefill_inflight_queue.append(req) if self.spec_algorithm.is_eagle() and draft_input is not None: req.output_topk_p = draft_input.topk_p[i] diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index ed4dac6582..068f3ae561 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -5269,11 +5269,16 @@ class Scheduler( from sglang.srt.mem_cache.radix_cache import RadixKey total = len(recv_req.input_ids) + # 树节点键是 array("q"),RadixKey.match_at 断言两侧类型一致—— + # 查询侧必须同样用 array,否则树非空时匹配直接炸(2026-10-11 实测)。 + from array import array + + query_ids = array("q", recv_req.input_ids) try: match_result = self.tree_cache.match_prefix( MatchPrefixParams( key=RadixKey( - token_ids=recv_req.input_ids, + token_ids=query_ids, extra_key=recv_req.extra_key, cache_salt=recv_req.cache_salt, ) @@ -5283,8 +5288,15 @@ class Scheduler( l2_hit = getattr(match_result, "host_hit_length", 0) or 0 l3_hit = 0 last_host_node = getattr(match_result, "last_host_node", None) + # L3 查询条件:本调度器挂了 hicache 存储后端即可(P 节点 HiCache + # write_through/write_back 场景),不必是 decode-hicache 模式。 + # SWA 模型的 decode radix 与 hicache 互斥(kv_cache_builder 硬限制), + # 若只认 enable_decode_hicache,P 节点永远查不到 L3。 + can_query_l3 = getattr(self, "enable_decode_hicache", False) or getattr( + self, "enable_hicache_storage", False + ) if ( - getattr(self, "enable_decode_hicache", False) + can_query_l3 and last_host_node is not None and hasattr(self.tree_cache, "query_storage_hit_length") and ( @@ -5293,7 +5305,7 @@ class Scheduler( ) ): matched_len = l1_hit + l2_hit - suffix_tokens = recv_req.input_ids[matched_len:] + suffix_tokens = query_ids[matched_len:] last_hash = self.tree_cache.get_last_hash_value(last_host_node) prefix_keys = ( self.tree_cache.get_prefix_hash_values(last_host_node) diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 421a215722..e32ca62b9e 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -1972,11 +1972,46 @@ class UnifiedRadixCache(BasePrefixCache): if len(prefetch_key) < self.prefetch_threshold: return 0 + # DeepSeek V4 keeps no physical KV objects in storage (logical anchor), + # so a plain batch_exists on KV keys always misses; the usable prefix is + # decided by the side pools. Mirror prefetch_from_storage's transfer + # construction so the query goes through batch_exists_v2. Both hooks are + # pure sizing/descriptor builders: prepare_prefetch only computes + # staging_tokens, and PREFETCH-phase transfers carry placeholder keys + # whose length sizes the TRAILING_PAGES window for batch_exists_v2. + comp_xfers: dict[ComponentType, list[PoolTransfer]] = {} + for ct in self.tree_components: + if ct == BASE_COMPONENT_TYPE: + continue + prep = self.components[ct].prepare_prefetch( + last_host_node_id, prefetch_tokens=len(prefetch_key) + ) + if prep.staging_tokens == 0: + continue + transfers = self.tree_core.build_hicache_transfers( + ct, + last_host_node_id, + CacheTransferPhase.PREFETCH, + token_ids=prefetch_key.token_ids, + prefetch_tokens=len(prefetch_key), + staging_tokens=prep.staging_tokens, + last_hash=last_hash, + ) + if transfers: + comp_xfers[ct] = transfers + kv_xfer = PoolTransfer(name=PoolName.KV, host_indices=None) + sidecar_xfers = self._build_sidecar_transfers( + CacheTransferPhase.PREFETCH, kv_xfer, comp_xfers + ) + aux_xfers = [x for xfers in comp_xfers.values() for x in xfers] + aux_xfers.extend(sidecar_xfers) + operation = PrefetchOperation( CacheRequestHandle("__storage_hit_query__", 0), prefetch_key, last_hash, prefix_keys, + pool_transfers=aux_xfers or None, ) _, storage_hit_count = self.cache_controller._storage_hit_query(operation) storage_hit_count_tensor = torch.tensor(storage_hit_count, dtype=torch.int)