[Fix] hicache: Order write-back host copies after the forward that produced the KV (#44800)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-17 01:31:40 -07:00
committed by GitHub
co-authored by metamergebot xiezhq-hermann
parent 0d13efb2b8
commit f75232cb42
11 changed files with 225 additions and 383 deletions
@@ -373,6 +373,7 @@ class HiCacheController:
# Set by the scheduler to the forward stream; gates load-back H2D
# behind in-flight forwards (see start_loading).
self.load_fence_stream = None
self.write_fence_stream = None
self.ack_write_queue: List[HiCacheAck] = []
self.l2_transfer_engine = L2TransferEngine(io_backend)
@@ -831,7 +832,7 @@ class HiCacheController:
self.start_writing()
return host_indices
def start_writing(self) -> None:
def start_writing(self, *, wait_for_forward: bool = False) -> None:
if len(self.write_queue) == 0:
return
@@ -839,6 +840,12 @@ class HiCacheController:
host_indices, device_indices, pool_transfers = self._move_write_operation(op)
self.write_queue.clear()
if wait_for_forward and self.write_fence_stream is not None:
# An evicted SWA page can belong to an unfinished checkpoint.
self.l2_transfer_engine.device_to_host_stream.wait_stream(
self.write_fence_stream
)
completion = self.l2_transfer_engine.submit_device_to_host(
self._l2_transfers(host_indices, device_indices, pool_transfers)
)
+3
View File
@@ -609,6 +609,9 @@ class Scheduler(
cache_controller.load_fence_stream = (
self.tp_worker.model_runner.forward_stream
)
cache_controller.write_fence_stream = (
self.tp_worker.model_runner.forward_stream
)
if self.enable_unified_memory:
# Keep device rows stable until host transfers are acknowledged.
# Queue reads and relocation both run on the scheduler thread.
@@ -1398,18 +1398,16 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
state.phase = _InsertPhase.TAIL
def _needs_incremental_component_backup(self, node: UnifiedTreeNode) -> bool:
components = self.components
if self.is_write_back:
swa = self.components_by_type.get(ComponentType.SWA)
components = () if swa is None else (swa,)
return any(
component.needs_incremental_backup(node)
for component in components
for component in self.components
if component.component_type != BASE_COMPONENT_TYPE
)
def _should_backup_after_insert(self, state: _InsertWalkState) -> bool:
"""Check whether the insert target needs a Host backup."""
if self.is_write_back:
return False
if state.is_new_leaf:
leaf_end = state.total_prefix_length + len(state.target_node.key)
return (
@@ -3385,7 +3385,7 @@ class UnifiedRadixCache(BasePrefixCache):
if write_back:
# Blocking: submit what is still queued, then wait for every ack.
cc.start_writing()
cc.start_writing(wait_for_forward=True)
while self.ongoing_write_through:
for ack in cc.ack_write_queue:
ack.finish_event.synchronize()