From 5f2ffa70b8e3d8410bcf1dfec2ce694a64a974e1 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Sat, 10 Oct 2026 15:29:01 +0800 Subject: [PATCH] =?UTF-8?q?hicache:=20SGLANG=5FHICACHE=5FTIMING=20?= =?UTF-8?q?=E6=89=93=E7=82=B9=20L3=20=E9=A2=84=E5=8F=96=E5=85=A8=E9=93=BE?= =?UTF-8?q?=E8=B7=AF=E5=88=86=E9=98=B6=E6=AE=B5=E8=80=97=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit env 开关(默认关),开后在以下阶段输出 HC_TIMING 日志行: query_wait/query(含 batch_exists 逐批)/hit_count all_reduce/ xfer_wait/xfer(含 batch_get 逐批)/ack_reduce/ack_drain 排队/ graft(insert_host)/prefetch_e2e/loadback。 目的:定位 L3 命中单请求 ~3s(≈重算速度)的时间分布。 真实 mooncake store 与 dummy store 实测吞吐一致,排除后端实现因素。 --- python/sglang/srt/environ.py | 3 + .../sglang/srt/managers/cache_controller.py | 61 +++++++++++++++++++ .../hybrid_cache/hybrid_cache_controller.py | 3 + .../srt/mem_cache/unified_radix_cache.py | 32 +++++++++- 4 files changed, 98 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index b497b0c3ce..746bce4da1 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -758,6 +758,9 @@ class Envs: SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None) SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE = EnvInt(None) SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM = EnvBool(False) + # Per-stage timing logs for the L3 storage prefetch path (query / transfer / + # ack reduce / drain / graft / load-back). Emits "HC_TIMING ..." lines. + SGLANG_HICACHE_TIMING = EnvBool(False) SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None) # File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables). SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None) diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index c652c1b099..d9db7f7337 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -32,6 +32,17 @@ from sglang.srt.mem_cache.hicache_storage import ( count_pool_hits, ) +from sglang.srt.environ import envs as _envs + +_HC_TIMING = None + + +def _hc_timing_enabled() -> bool: + global _HC_TIMING + if _HC_TIMING is None: + _HC_TIMING = _envs.SGLANG_HICACHE_TIMING.get() + return _HC_TIMING + if TYPE_CHECKING: from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.pool_host import HostKVCache @@ -1100,6 +1111,7 @@ class HiCacheController: # Get one batch token, and update the completed_tokens if succeed extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys) + _hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0 hit_pages = self._page_transfer_kv_batch( operation, batch_hashes, @@ -1107,6 +1119,14 @@ class HiCacheController: extra_info, kv_derived_transfers, ) + if _hc_timing_enabled(): + logger.info( + "HC_TIMING stage=xfer_batch req=%s keys=%d ms=%.1f hit=%d", + operation.request_id, + len(batch_hashes), + (time.monotonic() - _hc_tb) * 1e3, + hit_pages, + ) # Check termination if hit_pages != len(batch_hashes): all_success = False @@ -1169,7 +1189,16 @@ class HiCacheController: operation = self.prefetch_buffer.get(block=True, timeout=1) if operation is None: continue + _hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0 self._page_transfer(operation) + if _hc_timing_enabled(): + logger.info( + "HC_TIMING stage=xfer req=%s wait_ms=%.1f xfer_ms=%.1f", + operation.request_id, + (_hc_t0 - getattr(operation, "query_done_ts", operation.start_time)) + * 1e3, + (time.monotonic() - _hc_t0) * 1e3, + ) self.prefetch_sync_queue.put( PrefetchAck( @@ -1216,7 +1245,16 @@ class HiCacheController: for start in range(0, len(page_hashes), STORAGE_BATCH_SIZE): batch_hashes = page_hashes[start : start + STORAGE_BATCH_SIZE] extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys) + _hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0 hit_page_num = self.storage_backend.batch_exists(batch_hashes, extra_info) + if _hc_timing_enabled(): + logger.info( + "HC_TIMING stage=query_batch req=%s keys=%d ms=%.1f hit=%d", + operation.request_id, + len(batch_hashes), + (time.monotonic() - _hc_tb) * 1e3, + hit_page_num, + ) hash_value.extend(batch_hashes[:hit_page_num]) storage_query_count += hit_page_num * self.page_size if hit_page_num < len(batch_hashes): @@ -1235,10 +1273,12 @@ class HiCacheController: operation = self.prefetch_queue.get(block=True, timeout=1) if operation is None: continue + _hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0 if operation.is_terminated(): hash_value, storage_hit_count = [], 0 else: hash_value, storage_hit_count = self._storage_hit_query(operation) + _hc_t1 = time.monotonic() if _hc_timing_enabled() else 0.0 storage_hit_count_tensor = torch.tensor( storage_hit_count, dtype=torch.int ) @@ -1248,6 +1288,18 @@ class HiCacheController: self.prefetch_hits_sync_groups, ) storage_hit_count = storage_hit_count_tensor.item() + if _hc_timing_enabled(): + _hc_t2 = time.monotonic() + operation.query_done_ts = _hc_t2 + logger.info( + "HC_TIMING stage=query req=%s wait_ms=%.1f query_ms=%.1f " + "hit_reduce_ms=%.1f hit_tokens=%d", + operation.request_id, + (_hc_t0 - operation.start_time) * 1e3, + (_hc_t1 - _hc_t0) * 1e3, + (_hc_t2 - _hc_t1) * 1e3, + storage_hit_count, + ) # Record the TP-synced hit count; the scheduler thread decides # at drain time whether to revoke (below threshold) or allocate. @@ -1337,6 +1389,8 @@ class HiCacheController: if ack is None: continue self._reduce_prefetch_ack(ack) + if _hc_timing_enabled() and ack.completed_req: + ack.operation.final_ack_enqueue_ts = time.monotonic() self.ack_prefetch_queue.put(ack) except Empty: continue @@ -1345,6 +1399,7 @@ class HiCacheController: """Synchronize all ranks to agree on a PrefetchAck.""" if ack.completed_tokens is not None: # Determine the minimal successful prefix of tokens. + _hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0 completed_tokens_tensor = torch.tensor( ack.completed_tokens, dtype=torch.int ) @@ -1354,3 +1409,9 @@ class HiCacheController: self.prefetch_completion_sync_groups, ) ack.completed_tokens = completed_tokens_tensor.item() + if _hc_timing_enabled(): + logger.info( + "HC_TIMING stage=ack_reduce req=%s ms=%.1f", + ack.operation.request_id, + (time.monotonic() - _hc_t0) * 1e3, + ) diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index f592b69739..b58bb5c413 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -15,6 +15,7 @@ import torch from sglang.srt.managers.cache_controller import ( CacheOperation, + _hc_timing_enabled, ) from sglang.srt.managers.cache_controller import ( HiCacheController as BaseHiCacheController, @@ -1449,6 +1450,8 @@ class HybridCacheController(BaseHiCacheController): continue self._reduce_prefetch_ack(ack) if not getattr(ack.operation, "is_pp_broadcast", False): + if _hc_timing_enabled() and ack.completed_req: + ack.operation.final_ack_enqueue_ts = time.monotonic() self.ack_prefetch_queue.put(ack) continue diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 9fb5d76bfe..930d7854f4 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -12,7 +12,7 @@ import torch from sglang.srt.distributed.communication_tags import P2PTag from sglang.srt.environ import envs -from sglang.srt.managers.cache_controller import CacheOperation +from sglang.srt.managers.cache_controller import CacheOperation, _hc_timing_enabled from sglang.srt.mem_cache.allocator.page_interleave import ( page_interleave_shard_size, ) @@ -1689,6 +1689,7 @@ class UnifiedRadixCache(BasePrefixCache): for comp in self._components_tuple } success = False + _hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0 try: success = self._load_back_transfers( node_id=node_id, @@ -1700,6 +1701,14 @@ class UnifiedRadixCache(BasePrefixCache): ) return success finally: + if _hc_timing_enabled(): + logger.info( + "HC_TIMING stage=loadback req=%s node=%d ms=%.1f success=%s", + getattr(req, "rid", None) if req is not None else "-", + node_id, + (time.monotonic() - _hc_t0) * 1e3, + success, + ) for comp in self._components_tuple: comp.finalize_load_back(req, preps[comp.component_type], success) @@ -2277,12 +2286,15 @@ class UnifiedRadixCache(BasePrefixCache): ) fetched_key = prefetch_key[:completed_tokens] + _hc_tg = time.monotonic() if _hc_timing_enabled() else 0.0 insert_result = self.tree_core.insert_host( last_host_node_id, fetched_key, host_indices[:completed_tokens], hash_value[: completed_tokens // self.page_size], ) + if _hc_timing_enabled(): + operation.graft_ms = (time.monotonic() - _hc_tg) * 1e3 # Apply the host-insert walk's actions before the transfer commit. self._apply_cache_actions(insert_result.cache_actions) @@ -2338,6 +2350,16 @@ class UnifiedRadixCache(BasePrefixCache): loaded_from_storage, self.cache_controller.prefetch_tokens_occupied, ) + if _hc_timing_enabled(): + logger.info( + "HC_TIMING stage=prefetch_e2e req=%s e2e_ms=%.1f graft_ms=%.1f " + "completed=%d loaded=%d", + request.rid, + (time.monotonic() - operation.start_time) * 1e3, + getattr(operation, "graft_ms", -1.0), + completed_tokens, + loaded_from_storage, + ) return def _check_hybrid_prefetch_result( @@ -3060,6 +3082,14 @@ class UnifiedRadixCache(BasePrefixCache): operation.pool_transfers_done = True if ack.completed_req: if is_current: + if _hc_timing_enabled(): + _hc_enq = getattr(operation, "final_ack_enqueue_ts", None) + if _hc_enq is not None: + logger.info( + "HC_TIMING stage=ack_drain req=%s wait_ms=%.1f", + operation.request_id, + (time.monotonic() - _hc_enq) * 1e3, + ) # check_prefetch_progress() is not called for this rid yet. # Let us insert the prefetch result into the radix tree. self._handle_prefetch_result(operation)