[Fix] hicache: Order write-back host copies after the forward that produced the KV (#44800)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
xiezhq-hermann
parent
0d13efb2b8
commit
f75232cb42
@@ -373,6 +373,7 @@ class HiCacheController:
|
||||
# Set by the scheduler to the forward stream; gates load-back H2D
|
||||
# behind in-flight forwards (see start_loading).
|
||||
self.load_fence_stream = None
|
||||
self.write_fence_stream = None
|
||||
self.ack_write_queue: List[HiCacheAck] = []
|
||||
|
||||
self.l2_transfer_engine = L2TransferEngine(io_backend)
|
||||
@@ -831,7 +832,7 @@ class HiCacheController:
|
||||
self.start_writing()
|
||||
return host_indices
|
||||
|
||||
def start_writing(self) -> None:
|
||||
def start_writing(self, *, wait_for_forward: bool = False) -> None:
|
||||
if len(self.write_queue) == 0:
|
||||
return
|
||||
|
||||
@@ -839,6 +840,12 @@ class HiCacheController:
|
||||
host_indices, device_indices, pool_transfers = self._move_write_operation(op)
|
||||
self.write_queue.clear()
|
||||
|
||||
if wait_for_forward and self.write_fence_stream is not None:
|
||||
# An evicted SWA page can belong to an unfinished checkpoint.
|
||||
self.l2_transfer_engine.device_to_host_stream.wait_stream(
|
||||
self.write_fence_stream
|
||||
)
|
||||
|
||||
completion = self.l2_transfer_engine.submit_device_to_host(
|
||||
self._l2_transfers(host_indices, device_indices, pool_transfers)
|
||||
)
|
||||
|
||||
@@ -609,6 +609,9 @@ class Scheduler(
|
||||
cache_controller.load_fence_stream = (
|
||||
self.tp_worker.model_runner.forward_stream
|
||||
)
|
||||
cache_controller.write_fence_stream = (
|
||||
self.tp_worker.model_runner.forward_stream
|
||||
)
|
||||
if self.enable_unified_memory:
|
||||
# Keep device rows stable until host transfers are acknowledged.
|
||||
# Queue reads and relocation both run on the scheduler thread.
|
||||
|
||||
@@ -1398,18 +1398,16 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
state.phase = _InsertPhase.TAIL
|
||||
|
||||
def _needs_incremental_component_backup(self, node: UnifiedTreeNode) -> bool:
|
||||
components = self.components
|
||||
if self.is_write_back:
|
||||
swa = self.components_by_type.get(ComponentType.SWA)
|
||||
components = () if swa is None else (swa,)
|
||||
return any(
|
||||
component.needs_incremental_backup(node)
|
||||
for component in components
|
||||
for component in self.components
|
||||
if component.component_type != BASE_COMPONENT_TYPE
|
||||
)
|
||||
|
||||
def _should_backup_after_insert(self, state: _InsertWalkState) -> bool:
|
||||
"""Check whether the insert target needs a Host backup."""
|
||||
if self.is_write_back:
|
||||
return False
|
||||
if state.is_new_leaf:
|
||||
leaf_end = state.total_prefix_length + len(state.target_node.key)
|
||||
return (
|
||||
|
||||
@@ -3385,7 +3385,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
if write_back:
|
||||
# Blocking: submit what is still queued, then wait for every ack.
|
||||
cc.start_writing()
|
||||
cc.start_writing(wait_for_forward=True)
|
||||
while self.ongoing_write_through:
|
||||
for ack in cc.ack_write_queue:
|
||||
ack.finish_event.synchronize()
|
||||
|
||||
Reference in New Issue
Block a user