Observability enhancement for HiCache (#32388)

This commit is contained in:
Zhiqiang Xie
2026-08-05 14:13:06 -07:00
committed by GitHub
parent 55b1c09e73
commit 106bcc1293
12 changed files with 420 additions and 49 deletions
+29 -3
View File
@@ -163,6 +163,11 @@ class HiCacheAck(NamedTuple):
node_ids: List[int]
num_tokens: int = 0
timing_enabled: bool = False
# Tokens transferred per host pool (PoolName value -> count).
num_tokens_by_pool: Optional[dict[str, int]] = None
# Total bytes moved by the op across all pools, including draft piggyback
# and sidecar transfers that the per-pool token counts exclude.
num_bytes: int = 0
class StorageOperation:
@@ -714,11 +719,12 @@ class HiCacheController:
self.write_queue.clear()
start_event = device_module.Event()
finish_event = device_module.Event()
ack_start_event, ack_finish_event, timing_enabled = make_timing_event_pair()
start_event.record()
with device_module.stream(self.write_stream):
start_event.wait(self.write_stream)
ack_start_event.record()
self.mem_pool_host.backup_from_device_all_layer(
self.mem_pool_device, host_indices, device_indices, self.io_backend
)
@@ -729,7 +735,7 @@ class HiCacheController:
device_indices,
self.io_backend,
)
finish_event.record()
ack_finish_event.record()
# NOTE: We must save the host indices and device indices here,
# this is because we need to guarantee that these tensors are
# still alive when the write stream is executing.
@@ -738,7 +744,25 @@ class HiCacheController:
if device_indices.is_cuda:
device_indices.record_stream(self.write_stream)
self.ack_write_queue.append(HiCacheAck(start_event, finish_event, op.node_ids))
self.ack_write_queue.append(
HiCacheAck(
start_event=ack_start_event,
finish_event=ack_finish_event,
node_ids=op.node_ids,
num_tokens=len(op.device_indices),
timing_enabled=timing_enabled,
num_tokens_by_pool={PoolName.KV.value: len(op.device_indices)},
num_bytes=self._transfer_num_bytes(op),
)
)
def _transfer_num_bytes(self, op: CacheOperation) -> int:
"""Total bytes moved by a merged transfer op (draft piggyback included)."""
num_tokens = len(op.device_indices)
num_bytes = num_tokens * self.mem_pool_host.size_per_token
if self.has_draft:
num_bytes += num_tokens * self.mem_pool_host_draft.size_per_token
return num_bytes
def load(
self,
@@ -830,6 +854,8 @@ class HiCacheController:
node_ids=op.node_ids,
num_tokens=len(op.device_indices),
timing_enabled=timing_enabled,
num_tokens_by_pool={PoolName.KV.value: len(op.device_indices)},
num_bytes=self._transfer_num_bytes(op),
)
)
return producer_id