G9 P0 fixes: v2 pool_transfers for storage hit query, disagg drain hook, query gate
- unified_radix_cache.query_storage_hit_length: build PREFETCH-phase
pool_transfers (comp prepare_prefetch + build_hicache_transfers +
sidecars) so DeepSeek V4's logical-anchor KV goes through
batch_exists_v2 side-pool existence instead of v1 batch_exists which
always misses (no physical KV objects are stored for V4).
- scheduler.handle_query_cache_hit: allow L3 query when hicache storage
backend is attached (not only decode-hicache mode; SWA models forbid
decode radix+hicache so P nodes could never query L3); construct
RadixKey from array('q') to match tree node key types.
- disaggregation/prefill: record_prefill_drain on final chunk in the
disagg-prefill result path (it bypasses batch_result_processor, so
drain_rate_tokens_per_s stayed 0 on P nodes).
- deploy: 3p1d l3test compose against the real mooncake store stack.
(cherry picked from commit e7dc175e94)
This commit is contained in:
@@ -890,6 +890,10 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
continue
|
||||
|
||||
self.cache_unfinished_disagg_prefill(req)
|
||||
# G9: the disagg-prefill path bypasses batch_result_processor's
|
||||
# prefill loop, so record the drain event here — final chunk
|
||||
# done, request heading to the inflight (KV transfer) queue.
|
||||
self.record_prefill_drain(req.seqlen)
|
||||
self.disagg_prefill_inflight_queue.append(req)
|
||||
if self.spec_algorithm.is_eagle() and draft_input is not None:
|
||||
req.output_topk_p = draft_input.topk_p[i]
|
||||
|
||||
@@ -5269,11 +5269,16 @@ class Scheduler(
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
|
||||
total = len(recv_req.input_ids)
|
||||
# 树节点键是 array("q"),RadixKey.match_at 断言两侧类型一致——
|
||||
# 查询侧必须同样用 array,否则树非空时匹配直接炸(2026-10-11 实测)。
|
||||
from array import array
|
||||
|
||||
query_ids = array("q", recv_req.input_ids)
|
||||
try:
|
||||
match_result = self.tree_cache.match_prefix(
|
||||
MatchPrefixParams(
|
||||
key=RadixKey(
|
||||
token_ids=recv_req.input_ids,
|
||||
token_ids=query_ids,
|
||||
extra_key=recv_req.extra_key,
|
||||
cache_salt=recv_req.cache_salt,
|
||||
)
|
||||
@@ -5283,8 +5288,15 @@ class Scheduler(
|
||||
l2_hit = getattr(match_result, "host_hit_length", 0) or 0
|
||||
l3_hit = 0
|
||||
last_host_node = getattr(match_result, "last_host_node", None)
|
||||
# L3 查询条件:本调度器挂了 hicache 存储后端即可(P 节点 HiCache
|
||||
# write_through/write_back 场景),不必是 decode-hicache 模式。
|
||||
# SWA 模型的 decode radix 与 hicache 互斥(kv_cache_builder 硬限制),
|
||||
# 若只认 enable_decode_hicache,P 节点永远查不到 L3。
|
||||
can_query_l3 = getattr(self, "enable_decode_hicache", False) or getattr(
|
||||
self, "enable_hicache_storage", False
|
||||
)
|
||||
if (
|
||||
getattr(self, "enable_decode_hicache", False)
|
||||
can_query_l3
|
||||
and last_host_node is not None
|
||||
and hasattr(self.tree_cache, "query_storage_hit_length")
|
||||
and (
|
||||
@@ -5293,7 +5305,7 @@ class Scheduler(
|
||||
)
|
||||
):
|
||||
matched_len = l1_hit + l2_hit
|
||||
suffix_tokens = recv_req.input_ids[matched_len:]
|
||||
suffix_tokens = query_ids[matched_len:]
|
||||
last_hash = self.tree_cache.get_last_hash_value(last_host_node)
|
||||
prefix_keys = (
|
||||
self.tree_cache.get_prefix_hash_values(last_host_node)
|
||||
|
||||
@@ -1972,11 +1972,46 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
if len(prefetch_key) < self.prefetch_threshold:
|
||||
return 0
|
||||
|
||||
# DeepSeek V4 keeps no physical KV objects in storage (logical anchor),
|
||||
# so a plain batch_exists on KV keys always misses; the usable prefix is
|
||||
# decided by the side pools. Mirror prefetch_from_storage's transfer
|
||||
# construction so the query goes through batch_exists_v2. Both hooks are
|
||||
# pure sizing/descriptor builders: prepare_prefetch only computes
|
||||
# staging_tokens, and PREFETCH-phase transfers carry placeholder keys
|
||||
# whose length sizes the TRAILING_PAGES window for batch_exists_v2.
|
||||
comp_xfers: dict[ComponentType, list[PoolTransfer]] = {}
|
||||
for ct in self.tree_components:
|
||||
if ct == BASE_COMPONENT_TYPE:
|
||||
continue
|
||||
prep = self.components[ct].prepare_prefetch(
|
||||
last_host_node_id, prefetch_tokens=len(prefetch_key)
|
||||
)
|
||||
if prep.staging_tokens == 0:
|
||||
continue
|
||||
transfers = self.tree_core.build_hicache_transfers(
|
||||
ct,
|
||||
last_host_node_id,
|
||||
CacheTransferPhase.PREFETCH,
|
||||
token_ids=prefetch_key.token_ids,
|
||||
prefetch_tokens=len(prefetch_key),
|
||||
staging_tokens=prep.staging_tokens,
|
||||
last_hash=last_hash,
|
||||
)
|
||||
if transfers:
|
||||
comp_xfers[ct] = transfers
|
||||
kv_xfer = PoolTransfer(name=PoolName.KV, host_indices=None)
|
||||
sidecar_xfers = self._build_sidecar_transfers(
|
||||
CacheTransferPhase.PREFETCH, kv_xfer, comp_xfers
|
||||
)
|
||||
aux_xfers = [x for xfers in comp_xfers.values() for x in xfers]
|
||||
aux_xfers.extend(sidecar_xfers)
|
||||
|
||||
operation = PrefetchOperation(
|
||||
CacheRequestHandle("__storage_hit_query__", 0),
|
||||
prefetch_key,
|
||||
last_hash,
|
||||
prefix_keys,
|
||||
pool_transfers=aux_xfers or None,
|
||||
)
|
||||
_, storage_hit_count = self.cache_controller._storage_hit_query(operation)
|
||||
storage_hit_count_tensor = torch.tensor(storage_hit_count, dtype=torch.int)
|
||||
|
||||
Reference in New Issue
Block a user