diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 47eaefd6f0..e9442b330b 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -645,6 +645,14 @@ class Scheduler( ) else: self.decode_offload_manager = None + # Prefill read-done lets the schedule stream run ahead of the forward that + # writes a stashed chunk. The cache controller's fence streams above order + # its copies behind that forward; the decode offload controller has none. + self.tp_worker.model_runner.kv_offload_fenced_on_forward_stream = ( + self.enable_hierarchical_cache + and self.tree_cache.cache_controller is not None + and self.decode_offload_manager is None + ) # Init running status self.init_running_status() diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 4ecea9e358..609b19ead0 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -468,6 +468,8 @@ class ModelRunner: # Read-done mailbox: the scheduler's WAR barrier reads it from the runner # its worker names, and treats None as the coarse whole-forward fence. self.shared_read_done_event: Optional[torch.cuda.Event] = None + # Set by the scheduler when every KV offload copy waits for forward_stream. + self.kv_offload_fenced_on_forward_stream = False # CPU offload set_offloader(create_offloader()) diff --git a/python/sglang/srt/model_executor/runner_utils/shared_read_event.py b/python/sglang/srt/model_executor/runner_utils/shared_read_event.py index cc77839c14..cc5f563cd4 100644 --- a/python/sglang/srt/model_executor/runner_utils/shared_read_event.py +++ b/python/sglang/srt/model_executor/runner_utils/shared_read_event.py @@ -31,12 +31,12 @@ def maybe_publish_prefill_shared_read_done( if not model_runner.spec_algorithm.supports_prefill_shared_read_done(): return # Host and storage offload copy a stashed chunk's KV ordered only behind the - # schedule stream, so they need the whole-forward barrier. + # schedule stream, so they need the whole-forward barrier unless every such + # copy waits for the forward stream itself. memory = get_memory() - if ( - memory.enable_hierarchical_cache - or memory.hicache_storage_backend is not None - or memory.enable_lmcache + if memory.enable_lmcache or ( + (memory.enable_hierarchical_cache or memory.hicache_storage_backend is not None) + and not model_runner.kv_offload_fenced_on_forward_stream ): return logger.info_once( diff --git a/test/registered/unit/mem_cache/test_hicache_write_back_ordering.py b/test/registered/unit/mem_cache/test_hicache_write_back_ordering.py index a2e2d7956e..929e02a323 100644 --- a/test/registered/unit/mem_cache/test_hicache_write_back_ordering.py +++ b/test/registered/unit/mem_cache/test_hicache_write_back_ordering.py @@ -1,4 +1,4 @@ -"""Write-back eviction must wait for the checkpoint's KV producer.""" +"""HiCache backups must wait for the checkpoint's KV producer.""" import unittest from array import array @@ -109,6 +109,72 @@ class TestHiCacheWriteBackOrdering(UnifiedRadixCacheFixtures, unittest.TestCase) finally: forward.synchronize() + def test_write_through_stashed_chunk_waits_for_its_producer(self): + # Prefill read-done lets the scheduler stash a chunk while its forward + # still writes the KV; the write-through backup must copy the final KV. + self.cfg = CacheConfig( + components=(ComponentType.FULL, ComponentType.SWA), + page_size=1, + sliding_window_size=4, + num_layers=2, + full_attention_layer_ids=(0,), + kv_size=64, + max_context_len=64, + head_num=1, + head_dim=64, + ) + cache, allocator, req_pool = build_fixture(self.cfg) + self._init_hicache(cache, write_policy="write_through") + cache.write_through_threshold = 1 + controller = cache.cache_controller + forward = torch.cuda.Stream() + controller.write_fence_stream = forward + req = self._request(cache, allocator, req_pool, 0, length=12) + kv = allocator.get_kvcache() + buffers = [ + buf + for pool in (kv.full_kv_pool, kv.swa_kv_pool) + for buf in pool.k_buffer + pool.v_buffer + ] + for buf in buffers: + buf.fill_(-1) + read_done = torch.cuda.Event() + torch.cuda.synchronize() + engine = controller.l2_transfer_engine + submit = engine.submit_device_to_host + snapshots = [] + + def record_copy(transfers): + completion = submit(transfers) + completion.finish_event.synchronize() + for transfer in transfers: + if transfer.host_indices.numel(): + snapshots.append( + transfer.host_pool.get_data_page( + int(transfer.host_indices[0]) + ).clone() + ) + return completion + + with torch.cuda.stream(forward): + read_done.record() + torch.cuda._sleep(600_000_000) + for buf in buffers: + buf.fill_(42) + torch.cuda.current_stream().wait_event(read_done) + try: + with patch.object(engine, "submit_device_to_host", side_effect=record_copy): + cache.checkpoint(req, up_to=12) + cache.flush_pending_backups() + self.assertTrue(snapshots) + for snapshot in snapshots: + self.assertTrue( + torch.equal(snapshot, torch.full_like(snapshot, 42)), + f"expected 42, copied {snapshot.unique().tolist()}", + ) + finally: + forward.synchronize() + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/model_executor/runner/test_prefill_shared_read_done.py b/test/registered/unit/model_executor/runner/test_prefill_shared_read_done.py index 9a857d1399..59af7221a4 100644 --- a/test/registered/unit/model_executor/runner/test_prefill_shared_read_done.py +++ b/test/registered/unit/model_executor/runner/test_prefill_shared_read_done.py @@ -35,7 +35,11 @@ class _Event: def _model_runner( - *, spec_algorithm=SpeculativeAlgorithm.NONE, compliant=True, declared=None + *, + spec_algorithm=SpeculativeAlgorithm.NONE, + compliant=True, + declared=None, + offload_fenced=False, ): if declared is None: declared = SharedReadEnds.PRE_REPLAY if compliant else SharedReadEnds.UNKNOWN @@ -45,6 +49,7 @@ def _model_runner( spec_algorithm=spec_algorithm, attn_backend=attn_backend, shared_read_done_event=None, + kv_offload_fenced_on_forward_stream=offload_fenced, ) return runner @@ -92,6 +97,28 @@ def test_kv_offload_keeps_the_coarse_barrier(memory): assert runner.shared_read_done_event is None +@pytest.mark.parametrize( + "memory", + [ + {"enable_hierarchical_cache": True}, + {"enable_hierarchical_cache": True, "hicache_storage_backend": "mooncake"}, + ], +) +def test_fenced_kv_offload_publishes(memory): + runner = _model_runner(offload_fenced=True) + with get_memory().override(**memory): + _prepare_and_publish(runner, _batch()) + published = runner.shared_read_done_event + assert isinstance(published, _Event) and published.recorded + + +def test_lmcache_keeps_the_coarse_barrier_when_hicache_is_fenced(): + runner = _model_runner(offload_fenced=True) + with get_memory().override(enable_hierarchical_cache=True, enable_lmcache=True): + _prepare_and_publish(runner, _batch()) + assert runner.shared_read_done_event is None + + def test_dflash_target_prefill_publishes(): runner = _model_runner(spec_algorithm=SpeculativeAlgorithm.DFLASH) with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True):