From 3b0003dac15f32b27cf247c5da555edf459a2e3f Mon Sep 17 00:00:00 2001 From: Jooman <102151423+jumanzii@users.noreply.github.com> Date: Fri, 23 Oct 2026 06:46:51 +0900 Subject: [PATCH] [Fix] HiCache: stale host KV from write-through copies racing the in-flight forward (#42701) Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Zhiqiang Xie --- .../sglang/srt/managers/cache_controller.py | 13 ++++---- .../srt/mem_cache/unified_radix_cache.py | 2 +- ...test_hicache_staged_write_back_dispatch.py | 30 +++++++++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index 29cc484ac1..7b01b33797 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -370,8 +370,8 @@ class HiCacheController: self.load_queue: List[CacheOperation] = [] self.write_queue: List[CacheOperation] = [] self.ack_load_queue: List[HiCacheAck] = [] - # Set by the scheduler to the forward stream; gates load-back H2D - # behind in-flight forwards (see start_loading). + # Set by the scheduler to the forward stream; gate load-back H2D and + # write D2H behind in-flight forwards (see start_loading, start_writing). self.load_fence_stream = None self.write_fence_stream = None self.ack_write_queue: List[HiCacheAck] = [] @@ -838,7 +838,7 @@ class HiCacheController: self.start_writing() return host_indices - def start_writing(self, *, wait_for_forward: bool = False) -> None: + def start_writing(self) -> None: if len(self.write_queue) == 0: return @@ -846,8 +846,11 @@ class HiCacheController: host_indices, device_indices, pool_transfers = self._move_write_operation(op) self.write_queue.clear() - if wait_for_forward and self.write_fence_stream is not None: - # An evicted SWA page can belong to an unfinished checkpoint. + if self.write_fence_stream is not None: + # The copy may read KV a forward is still writing: under overlap + # scheduling a finished request is cached while the next forward + # writes its last token, and an evicted SWA page can belong to an + # unfinished checkpoint. Later forwards still overlap with the copy. self.l2_transfer_engine.device_to_host_stream.wait_stream( self.write_fence_stream ) diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 0f5eafa338..2c5b195dda 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -3460,7 +3460,7 @@ class UnifiedRadixCache(BasePrefixCache): if write_back: # Blocking: submit what is still queued, then wait for every ack. - cc.start_writing(wait_for_forward=True) + cc.start_writing() while self.ongoing_write_through: for ack in cc.ack_write_queue: ack.finish_event.synchronize() diff --git a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py index cffaf022ff..ec45e84c4c 100644 --- a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py +++ b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py @@ -1191,6 +1191,7 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): controller.mem_pool_device = None controller.mem_pool_device_allocator = mock.Mock() controller.ack_write_queue = [] + controller.write_fence_stream = None controller.move_hybrid_indices = mock.Mock( side_effect=AssertionError( "write-back JIT kernel write should not move indices" @@ -1226,6 +1227,7 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): controller.mem_pool_device = None controller.mem_pool_device_allocator = mock.Mock() controller.ack_write_queue = [] + controller.write_fence_stream = None controller.move_hybrid_indices = mock.Mock( return_value=(op.host_indices, op.device_indices, op.pool_transfers) ) @@ -1284,6 +1286,7 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): controller.mem_pool_device = None controller.mem_pool_device_allocator = mock.Mock() controller.ack_write_queue = [] + controller.write_fence_stream = None with mock.patch.object(transfer_module, "device_module", _FakeDeviceModule): controller.l2_transfer_engine = L2TransferEngine("kernel") return controller @@ -1314,6 +1317,31 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): self.assertEqual(controller.ack_write_queue[0].node_ids, [1, 2]) self.assertEqual(controller.ack_write_queue[0].num_tokens, 8) + def test_write_copy_waits_for_queued_forwards(self): + # Under overlap scheduling a finished request is cached while the next + # forward still writes its last token's KV; the D2H copy must start + # after that forward, or the host keeps a half-written token. + events = [] + + class RecordingStream: + def __init__(self, name): + self.name = name + + def wait_stream(self, other): + events.append(f"{self.name} waits {other.name}") + + controller = self._chain_write_controller(events) + controller.l2_transfer_engine.device_to_host_stream = RecordingStream("d2h") + controller.write_fence_stream = RecordingStream("forward") + + with mock.patch.object(transfer_module, "device_module", _FakeDeviceModule): + controller.write(_indices(4, 8), node_id=1) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0], "d2h waits forward") + # Then the write's one copy, of the node's device slots. + self.assertEqual(events[1][1].tolist(), [4, 5, 6, 7]) + def test_hybrid_write_flushes_by_default(self): captured = [] controller = self._chain_write_controller(captured) @@ -1352,6 +1380,7 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): controller.mem_pool_device_allocator = mock.Mock() controller.device = "cuda" controller.ack_write_queue = [] + controller.write_fence_stream = None controller.move_indices = mock.Mock( side_effect=AssertionError( "write-back JIT kernel write should not move indices" @@ -1389,6 +1418,7 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): controller.mem_pool_device_allocator = mock.Mock() controller.device = "cuda" controller.ack_write_queue = [] + controller.write_fence_stream = None controller.move_indices = mock.Mock( return_value=(op.host_indices, op.device_indices) )