hicache: SGLANG_HICACHE_TIMING 打点 L3 预取全链路分阶段耗时
build-sglang-image / build (push) Failing after 1s

env 开关(默认关),开后在以下阶段输出 HC_TIMING 日志行:
query_wait/query(含 batch_exists 逐批)/hit_count all_reduce/
xfer_wait/xfer(含 batch_get 逐批)/ack_reduce/ack_drain 排队/
graft(insert_host)/prefetch_e2e/loadback。
目的:定位 L3 命中单请求 ~3s(≈重算速度)的时间分布。
真实 mooncake store 与 dummy store 实测吞吐一致,排除后端实现因素。
This commit is contained in:
2026-10-10 15:29:01 +08:00
parent f4ef648b16
commit 5f2ffa70b8
4 changed files with 98 additions and 1 deletions
+3
View File
@@ -758,6 +758,9 @@ class Envs:
SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None)
SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE = EnvInt(None)
SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM = EnvBool(False)
# Per-stage timing logs for the L3 storage prefetch path (query / transfer /
# ack reduce / drain / graft / load-back). Emits "HC_TIMING ..." lines.
SGLANG_HICACHE_TIMING = EnvBool(False)
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
# File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables).
SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)
@@ -32,6 +32,17 @@ from sglang.srt.mem_cache.hicache_storage import (
count_pool_hits,
)
from sglang.srt.environ import envs as _envs
_HC_TIMING = None
def _hc_timing_enabled() -> bool:
global _HC_TIMING
if _HC_TIMING is None:
_HC_TIMING = _envs.SGLANG_HICACHE_TIMING.get()
return _HC_TIMING
if TYPE_CHECKING:
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.pool_host import HostKVCache
@@ -1100,6 +1111,7 @@ class HiCacheController:
# Get one batch token, and update the completed_tokens if succeed
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
_hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0
hit_pages = self._page_transfer_kv_batch(
operation,
batch_hashes,
@@ -1107,6 +1119,14 @@ class HiCacheController:
extra_info,
kv_derived_transfers,
)
if _hc_timing_enabled():
logger.info(
"HC_TIMING stage=xfer_batch req=%s keys=%d ms=%.1f hit=%d",
operation.request_id,
len(batch_hashes),
(time.monotonic() - _hc_tb) * 1e3,
hit_pages,
)
# Check termination
if hit_pages != len(batch_hashes):
all_success = False
@@ -1169,7 +1189,16 @@ class HiCacheController:
operation = self.prefetch_buffer.get(block=True, timeout=1)
if operation is None:
continue
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
self._page_transfer(operation)
if _hc_timing_enabled():
logger.info(
"HC_TIMING stage=xfer req=%s wait_ms=%.1f xfer_ms=%.1f",
operation.request_id,
(_hc_t0 - getattr(operation, "query_done_ts", operation.start_time))
* 1e3,
(time.monotonic() - _hc_t0) * 1e3,
)
self.prefetch_sync_queue.put(
PrefetchAck(
@@ -1216,7 +1245,16 @@ class HiCacheController:
for start in range(0, len(page_hashes), STORAGE_BATCH_SIZE):
batch_hashes = page_hashes[start : start + STORAGE_BATCH_SIZE]
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
_hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0
hit_page_num = self.storage_backend.batch_exists(batch_hashes, extra_info)
if _hc_timing_enabled():
logger.info(
"HC_TIMING stage=query_batch req=%s keys=%d ms=%.1f hit=%d",
operation.request_id,
len(batch_hashes),
(time.monotonic() - _hc_tb) * 1e3,
hit_page_num,
)
hash_value.extend(batch_hashes[:hit_page_num])
storage_query_count += hit_page_num * self.page_size
if hit_page_num < len(batch_hashes):
@@ -1235,10 +1273,12 @@ class HiCacheController:
operation = self.prefetch_queue.get(block=True, timeout=1)
if operation is None:
continue
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
if operation.is_terminated():
hash_value, storage_hit_count = [], 0
else:
hash_value, storage_hit_count = self._storage_hit_query(operation)
_hc_t1 = time.monotonic() if _hc_timing_enabled() else 0.0
storage_hit_count_tensor = torch.tensor(
storage_hit_count, dtype=torch.int
)
@@ -1248,6 +1288,18 @@ class HiCacheController:
self.prefetch_hits_sync_groups,
)
storage_hit_count = storage_hit_count_tensor.item()
if _hc_timing_enabled():
_hc_t2 = time.monotonic()
operation.query_done_ts = _hc_t2
logger.info(
"HC_TIMING stage=query req=%s wait_ms=%.1f query_ms=%.1f "
"hit_reduce_ms=%.1f hit_tokens=%d",
operation.request_id,
(_hc_t0 - operation.start_time) * 1e3,
(_hc_t1 - _hc_t0) * 1e3,
(_hc_t2 - _hc_t1) * 1e3,
storage_hit_count,
)
# Record the TP-synced hit count; the scheduler thread decides
# at drain time whether to revoke (below threshold) or allocate.
@@ -1337,6 +1389,8 @@ class HiCacheController:
if ack is None:
continue
self._reduce_prefetch_ack(ack)
if _hc_timing_enabled() and ack.completed_req:
ack.operation.final_ack_enqueue_ts = time.monotonic()
self.ack_prefetch_queue.put(ack)
except Empty:
continue
@@ -1345,6 +1399,7 @@ class HiCacheController:
"""Synchronize all ranks to agree on a PrefetchAck."""
if ack.completed_tokens is not None:
# Determine the minimal successful prefix of tokens.
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
completed_tokens_tensor = torch.tensor(
ack.completed_tokens, dtype=torch.int
)
@@ -1354,3 +1409,9 @@ class HiCacheController:
self.prefetch_completion_sync_groups,
)
ack.completed_tokens = completed_tokens_tensor.item()
if _hc_timing_enabled():
logger.info(
"HC_TIMING stage=ack_reduce req=%s ms=%.1f",
ack.operation.request_id,
(time.monotonic() - _hc_t0) * 1e3,
)
@@ -15,6 +15,7 @@ import torch
from sglang.srt.managers.cache_controller import (
CacheOperation,
_hc_timing_enabled,
)
from sglang.srt.managers.cache_controller import (
HiCacheController as BaseHiCacheController,
@@ -1449,6 +1450,8 @@ class HybridCacheController(BaseHiCacheController):
continue
self._reduce_prefetch_ack(ack)
if not getattr(ack.operation, "is_pp_broadcast", False):
if _hc_timing_enabled() and ack.completed_req:
ack.operation.final_ack_enqueue_ts = time.monotonic()
self.ack_prefetch_queue.put(ack)
continue
@@ -12,7 +12,7 @@ import torch
from sglang.srt.distributed.communication_tags import P2PTag
from sglang.srt.environ import envs
from sglang.srt.managers.cache_controller import CacheOperation
from sglang.srt.managers.cache_controller import CacheOperation, _hc_timing_enabled
from sglang.srt.mem_cache.allocator.page_interleave import (
page_interleave_shard_size,
)
@@ -1689,6 +1689,7 @@ class UnifiedRadixCache(BasePrefixCache):
for comp in self._components_tuple
}
success = False
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
try:
success = self._load_back_transfers(
node_id=node_id,
@@ -1700,6 +1701,14 @@ class UnifiedRadixCache(BasePrefixCache):
)
return success
finally:
if _hc_timing_enabled():
logger.info(
"HC_TIMING stage=loadback req=%s node=%d ms=%.1f success=%s",
getattr(req, "rid", None) if req is not None else "-",
node_id,
(time.monotonic() - _hc_t0) * 1e3,
success,
)
for comp in self._components_tuple:
comp.finalize_load_back(req, preps[comp.component_type], success)
@@ -2277,12 +2286,15 @@ class UnifiedRadixCache(BasePrefixCache):
)
fetched_key = prefetch_key[:completed_tokens]
_hc_tg = time.monotonic() if _hc_timing_enabled() else 0.0
insert_result = self.tree_core.insert_host(
last_host_node_id,
fetched_key,
host_indices[:completed_tokens],
hash_value[: completed_tokens // self.page_size],
)
if _hc_timing_enabled():
operation.graft_ms = (time.monotonic() - _hc_tg) * 1e3
# Apply the host-insert walk's actions before the transfer commit.
self._apply_cache_actions(insert_result.cache_actions)
@@ -2338,6 +2350,16 @@ class UnifiedRadixCache(BasePrefixCache):
loaded_from_storage,
self.cache_controller.prefetch_tokens_occupied,
)
if _hc_timing_enabled():
logger.info(
"HC_TIMING stage=prefetch_e2e req=%s e2e_ms=%.1f graft_ms=%.1f "
"completed=%d loaded=%d",
request.rid,
(time.monotonic() - operation.start_time) * 1e3,
getattr(operation, "graft_ms", -1.0),
completed_tokens,
loaded_from_storage,
)
return
def _check_hybrid_prefetch_result(
@@ -3060,6 +3082,14 @@ class UnifiedRadixCache(BasePrefixCache):
operation.pool_transfers_done = True
if ack.completed_req:
if is_current:
if _hc_timing_enabled():
_hc_enq = getattr(operation, "final_ack_enqueue_ts", None)
if _hc_enq is not None:
logger.info(
"HC_TIMING stage=ack_drain req=%s wait_ms=%.1f",
operation.request_id,
(time.monotonic() - _hc_enq) * 1e3,
)
# check_prefetch_progress() is not called for this rid yet.
# Let us insert the prefetch result into the radix tree.
self._handle_prefetch_result(operation)