[Fix] HiCache: stale host KV from write-through copies racing the in-flight forward (#42701)

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Zhiqiang Xie <xiezhq@stanford.edu>
This commit is contained in:
Jooman
2026-10-22 14:46:51 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Zhiqiang Xie
parent a904e4d31a
commit 3b0003dac1
3 changed files with 39 additions and 6 deletions
@@ -370,8 +370,8 @@ class HiCacheController:
self.load_queue: List[CacheOperation] = []
self.write_queue: List[CacheOperation] = []
self.ack_load_queue: List[HiCacheAck] = []
# Set by the scheduler to the forward stream; gates load-back H2D
# behind in-flight forwards (see start_loading).
# Set by the scheduler to the forward stream; gate load-back H2D and
# write D2H behind in-flight forwards (see start_loading, start_writing).
self.load_fence_stream = None
self.write_fence_stream = None
self.ack_write_queue: List[HiCacheAck] = []
@@ -838,7 +838,7 @@ class HiCacheController:
self.start_writing()
return host_indices
def start_writing(self, *, wait_for_forward: bool = False) -> None:
def start_writing(self) -> None:
if len(self.write_queue) == 0:
return
@@ -846,8 +846,11 @@ class HiCacheController:
host_indices, device_indices, pool_transfers = self._move_write_operation(op)
self.write_queue.clear()
if wait_for_forward and self.write_fence_stream is not None:
# An evicted SWA page can belong to an unfinished checkpoint.
if self.write_fence_stream is not None:
# The copy may read KV a forward is still writing: under overlap
# scheduling a finished request is cached while the next forward
# writes its last token, and an evicted SWA page can belong to an
# unfinished checkpoint. Later forwards still overlap with the copy.
self.l2_transfer_engine.device_to_host_stream.wait_stream(
self.write_fence_stream
)
@@ -3460,7 +3460,7 @@ class UnifiedRadixCache(BasePrefixCache):
if write_back:
# Blocking: submit what is still queued, then wait for every ack.
cc.start_writing(wait_for_forward=True)
cc.start_writing()
while self.ongoing_write_through:
for ack in cc.ack_write_queue:
ack.finish_event.synchronize()