env 开关(默认关),开后在以下阶段输出 HC_TIMING 日志行: query_wait/query(含 batch_exists 逐批)/hit_count all_reduce/ xfer_wait/xfer(含 batch_get 逐批)/ack_reduce/ack_drain 排队/ graft(insert_host)/prefetch_e2e/loadback。 目的:定位 L3 命中单请求 ~3s(≈重算速度)的时间分布。 真实 mooncake store 与 dummy store 实测吞吐一致,排除后端实现因素。
This commit is contained in:
@@ -758,6 +758,9 @@ class Envs:
|
||||
SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None)
|
||||
SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE = EnvInt(None)
|
||||
SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM = EnvBool(False)
|
||||
# Per-stage timing logs for the L3 storage prefetch path (query / transfer /
|
||||
# ack reduce / drain / graft / load-back). Emits "HC_TIMING ..." lines.
|
||||
SGLANG_HICACHE_TIMING = EnvBool(False)
|
||||
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
|
||||
# File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables).
|
||||
SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)
|
||||
|
||||
@@ -32,6 +32,17 @@ from sglang.srt.mem_cache.hicache_storage import (
|
||||
count_pool_hits,
|
||||
)
|
||||
|
||||
from sglang.srt.environ import envs as _envs
|
||||
|
||||
_HC_TIMING = None
|
||||
|
||||
|
||||
def _hc_timing_enabled() -> bool:
|
||||
global _HC_TIMING
|
||||
if _HC_TIMING is None:
|
||||
_HC_TIMING = _envs.SGLANG_HICACHE_TIMING.get()
|
||||
return _HC_TIMING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.pool_host import HostKVCache
|
||||
@@ -1100,6 +1111,7 @@ class HiCacheController:
|
||||
# Get one batch token, and update the completed_tokens if succeed
|
||||
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
|
||||
|
||||
_hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
hit_pages = self._page_transfer_kv_batch(
|
||||
operation,
|
||||
batch_hashes,
|
||||
@@ -1107,6 +1119,14 @@ class HiCacheController:
|
||||
extra_info,
|
||||
kv_derived_transfers,
|
||||
)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=xfer_batch req=%s keys=%d ms=%.1f hit=%d",
|
||||
operation.request_id,
|
||||
len(batch_hashes),
|
||||
(time.monotonic() - _hc_tb) * 1e3,
|
||||
hit_pages,
|
||||
)
|
||||
# Check termination
|
||||
if hit_pages != len(batch_hashes):
|
||||
all_success = False
|
||||
@@ -1169,7 +1189,16 @@ class HiCacheController:
|
||||
operation = self.prefetch_buffer.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
self._page_transfer(operation)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=xfer req=%s wait_ms=%.1f xfer_ms=%.1f",
|
||||
operation.request_id,
|
||||
(_hc_t0 - getattr(operation, "query_done_ts", operation.start_time))
|
||||
* 1e3,
|
||||
(time.monotonic() - _hc_t0) * 1e3,
|
||||
)
|
||||
|
||||
self.prefetch_sync_queue.put(
|
||||
PrefetchAck(
|
||||
@@ -1216,7 +1245,16 @@ class HiCacheController:
|
||||
for start in range(0, len(page_hashes), STORAGE_BATCH_SIZE):
|
||||
batch_hashes = page_hashes[start : start + STORAGE_BATCH_SIZE]
|
||||
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
|
||||
_hc_tb = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
hit_page_num = self.storage_backend.batch_exists(batch_hashes, extra_info)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=query_batch req=%s keys=%d ms=%.1f hit=%d",
|
||||
operation.request_id,
|
||||
len(batch_hashes),
|
||||
(time.monotonic() - _hc_tb) * 1e3,
|
||||
hit_page_num,
|
||||
)
|
||||
hash_value.extend(batch_hashes[:hit_page_num])
|
||||
storage_query_count += hit_page_num * self.page_size
|
||||
if hit_page_num < len(batch_hashes):
|
||||
@@ -1235,10 +1273,12 @@ class HiCacheController:
|
||||
operation = self.prefetch_queue.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
if operation.is_terminated():
|
||||
hash_value, storage_hit_count = [], 0
|
||||
else:
|
||||
hash_value, storage_hit_count = self._storage_hit_query(operation)
|
||||
_hc_t1 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
storage_hit_count_tensor = torch.tensor(
|
||||
storage_hit_count, dtype=torch.int
|
||||
)
|
||||
@@ -1248,6 +1288,18 @@ class HiCacheController:
|
||||
self.prefetch_hits_sync_groups,
|
||||
)
|
||||
storage_hit_count = storage_hit_count_tensor.item()
|
||||
if _hc_timing_enabled():
|
||||
_hc_t2 = time.monotonic()
|
||||
operation.query_done_ts = _hc_t2
|
||||
logger.info(
|
||||
"HC_TIMING stage=query req=%s wait_ms=%.1f query_ms=%.1f "
|
||||
"hit_reduce_ms=%.1f hit_tokens=%d",
|
||||
operation.request_id,
|
||||
(_hc_t0 - operation.start_time) * 1e3,
|
||||
(_hc_t1 - _hc_t0) * 1e3,
|
||||
(_hc_t2 - _hc_t1) * 1e3,
|
||||
storage_hit_count,
|
||||
)
|
||||
|
||||
# Record the TP-synced hit count; the scheduler thread decides
|
||||
# at drain time whether to revoke (below threshold) or allocate.
|
||||
@@ -1337,6 +1389,8 @@ class HiCacheController:
|
||||
if ack is None:
|
||||
continue
|
||||
self._reduce_prefetch_ack(ack)
|
||||
if _hc_timing_enabled() and ack.completed_req:
|
||||
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
||||
self.ack_prefetch_queue.put(ack)
|
||||
except Empty:
|
||||
continue
|
||||
@@ -1345,6 +1399,7 @@ class HiCacheController:
|
||||
"""Synchronize all ranks to agree on a PrefetchAck."""
|
||||
if ack.completed_tokens is not None:
|
||||
# Determine the minimal successful prefix of tokens.
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
completed_tokens_tensor = torch.tensor(
|
||||
ack.completed_tokens, dtype=torch.int
|
||||
)
|
||||
@@ -1354,3 +1409,9 @@ class HiCacheController:
|
||||
self.prefetch_completion_sync_groups,
|
||||
)
|
||||
ack.completed_tokens = completed_tokens_tensor.item()
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=ack_reduce req=%s ms=%.1f",
|
||||
ack.operation.request_id,
|
||||
(time.monotonic() - _hc_t0) * 1e3,
|
||||
)
|
||||
|
||||
@@ -15,6 +15,7 @@ import torch
|
||||
|
||||
from sglang.srt.managers.cache_controller import (
|
||||
CacheOperation,
|
||||
_hc_timing_enabled,
|
||||
)
|
||||
from sglang.srt.managers.cache_controller import (
|
||||
HiCacheController as BaseHiCacheController,
|
||||
@@ -1449,6 +1450,8 @@ class HybridCacheController(BaseHiCacheController):
|
||||
continue
|
||||
self._reduce_prefetch_ack(ack)
|
||||
if not getattr(ack.operation, "is_pp_broadcast", False):
|
||||
if _hc_timing_enabled() and ack.completed_req:
|
||||
ack.operation.final_ack_enqueue_ts = time.monotonic()
|
||||
self.ack_prefetch_queue.put(ack)
|
||||
continue
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ import torch
|
||||
|
||||
from sglang.srt.distributed.communication_tags import P2PTag
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.cache_controller import CacheOperation
|
||||
from sglang.srt.managers.cache_controller import CacheOperation, _hc_timing_enabled
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import (
|
||||
page_interleave_shard_size,
|
||||
)
|
||||
@@ -1689,6 +1689,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
for comp in self._components_tuple
|
||||
}
|
||||
success = False
|
||||
_hc_t0 = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
try:
|
||||
success = self._load_back_transfers(
|
||||
node_id=node_id,
|
||||
@@ -1700,6 +1701,14 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
)
|
||||
return success
|
||||
finally:
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=loadback req=%s node=%d ms=%.1f success=%s",
|
||||
getattr(req, "rid", None) if req is not None else "-",
|
||||
node_id,
|
||||
(time.monotonic() - _hc_t0) * 1e3,
|
||||
success,
|
||||
)
|
||||
for comp in self._components_tuple:
|
||||
comp.finalize_load_back(req, preps[comp.component_type], success)
|
||||
|
||||
@@ -2277,12 +2286,15 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
)
|
||||
|
||||
fetched_key = prefetch_key[:completed_tokens]
|
||||
_hc_tg = time.monotonic() if _hc_timing_enabled() else 0.0
|
||||
insert_result = self.tree_core.insert_host(
|
||||
last_host_node_id,
|
||||
fetched_key,
|
||||
host_indices[:completed_tokens],
|
||||
hash_value[: completed_tokens // self.page_size],
|
||||
)
|
||||
if _hc_timing_enabled():
|
||||
operation.graft_ms = (time.monotonic() - _hc_tg) * 1e3
|
||||
|
||||
# Apply the host-insert walk's actions before the transfer commit.
|
||||
self._apply_cache_actions(insert_result.cache_actions)
|
||||
@@ -2338,6 +2350,16 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
loaded_from_storage,
|
||||
self.cache_controller.prefetch_tokens_occupied,
|
||||
)
|
||||
if _hc_timing_enabled():
|
||||
logger.info(
|
||||
"HC_TIMING stage=prefetch_e2e req=%s e2e_ms=%.1f graft_ms=%.1f "
|
||||
"completed=%d loaded=%d",
|
||||
request.rid,
|
||||
(time.monotonic() - operation.start_time) * 1e3,
|
||||
getattr(operation, "graft_ms", -1.0),
|
||||
completed_tokens,
|
||||
loaded_from_storage,
|
||||
)
|
||||
return
|
||||
|
||||
def _check_hybrid_prefetch_result(
|
||||
@@ -3060,6 +3082,14 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
operation.pool_transfers_done = True
|
||||
if ack.completed_req:
|
||||
if is_current:
|
||||
if _hc_timing_enabled():
|
||||
_hc_enq = getattr(operation, "final_ack_enqueue_ts", None)
|
||||
if _hc_enq is not None:
|
||||
logger.info(
|
||||
"HC_TIMING stage=ack_drain req=%s wait_ms=%.1f",
|
||||
operation.request_id,
|
||||
(time.monotonic() - _hc_enq) * 1e3,
|
||||
)
|
||||
# check_prefetch_progress() is not called for this rid yet.
|
||||
# Let us insert the prefetch result into the radix tree.
|
||||
self._handle_prefetch_result(operation)
|
||||
|
||||
Reference in New Issue
Block a user