diff --git a/python/sglang/srt/mem_cache/registry.py b/python/sglang/srt/mem_cache/registry.py index 9f42286bb5..792271a6af 100644 --- a/python/sglang/srt/mem_cache/registry.py +++ b/python/sglang/srt/mem_cache/registry.py @@ -217,7 +217,12 @@ def create_unified_radix_cache( ) cache.init_cache_linker( - linker_cls(server_args, params, components=set(cache.components)) + linker_cls( + server_args, + params, + components=set(cache.components), + forward_stream=ctx.tp_worker.model_runner.forward_stream, + ) ) counter = cache.linker.layer_done_counter kvcache = params.token_to_kv_pool_allocator.get_kvcache() diff --git a/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_direct_linker.py b/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_direct_linker.py index f95b00a025..12495aa59a 100644 --- a/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_direct_linker.py +++ b/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_direct_linker.py @@ -55,7 +55,9 @@ class MooncakeDirectLinker(UnifiedCacheLinker): *, components, storage=None, + forward_stream=None, ): + self.forward_stream = forward_stream self.page_size = params.page_size kvcache = params.token_to_kv_pool_allocator.get_kvcache() self.pool_group = resolve_hybrid_device_pool_group( @@ -384,9 +386,8 @@ class MooncakeDirectLinker(UnifiedCacheLinker): return True kv = next(transfer for transfer in transfers if transfer.name == PoolName.KV) tokens = len(kv.keys) * self.page_size - ready_event = device_module.Event() - ready_event.record() - self.offload_queue.put((expanded, tokens, ready_event)) + ready_events = self.record_offload_ready() + self.offload_queue.put((expanded, tokens, ready_events)) return True def offload_thread_func(self) -> None: @@ -395,8 +396,9 @@ class MooncakeDirectLinker(UnifiedCacheLinker): try: if task is None: return - expanded, tokens, ready_event = task - ready_event.synchronize() + expanded, tokens, ready_events = task + for event in ready_events: + event.synchronize() results = self.storage.batch_set_v2(expanded) success = all(all(pool_results) for pool_results in results.values()) if success: diff --git a/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py b/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py index d7286f4ec5..5ad7037b3a 100644 --- a/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py +++ b/python/sglang/srt/mem_cache/storage/umbp/umbp_direct_linker.py @@ -91,8 +91,8 @@ class _PoolRangePlan: entries_per_page: int -# One queued offload: the pools it resolved to, and the event guarding its KV. -_OffloadTask = tuple[list[PoolTransfer], object] +# One queued offload: the pools it resolved to, and the events gating the read. +_OffloadTask = tuple[list[PoolTransfer], list[object]] def _offload_task_pages(expanded: list[PoolTransfer]) -> int: @@ -152,7 +152,9 @@ class UMBPDirectLinker(UnifiedCacheLinker): *, components, _storage=None, + forward_stream=None, ): + self.forward_stream = forward_stream self.page_size = params.page_size # Group layers to amortize per-object RPC overhead; 8 is the measured default. self.layer_group = max(1, int(os.getenv("UMBP_LAYER_GROUP", "8"))) @@ -856,9 +858,8 @@ class UMBPDirectLinker(UnifiedCacheLinker): if not expanded: return False self._freeze_gc_once() - ready_event = device_module.Event() - ready_event.record() - self._offload_queue.put((expanded, ready_event)) + ready_events = self.record_offload_ready() + self._offload_queue.put((expanded, ready_events)) return True def _take_offload_batch(self) -> tuple[list[_OffloadTask], bool]: @@ -912,8 +913,9 @@ class UMBPDirectLinker(UnifiedCacheLinker): self._offload_results.put(success) def _run_offload(self, tasks: list[_OffloadTask]) -> bool: - for _, ready_event in tasks: - ready_event.synchronize() + for _, ready_events in tasks: + for event in ready_events: + event.synchronize() next_slot = 0 def materialize_indices(indices: torch.Tensor) -> torch.Tensor: diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py b/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py index 64f341e178..f10eda9d8e 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_cache_linker.py @@ -46,6 +46,7 @@ from sglang.srt.mem_cache.unified_cache.components import ( TreeComponent, ) from sglang.srt.mem_cache.utils import get_storage_hash_str +from sglang.srt.utils import get_device_module if TYPE_CHECKING: from sglang.srt.managers.schedule_batch import Req @@ -53,6 +54,7 @@ if TYPE_CHECKING: from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache logger = logging.getLogger(__name__) +device_module = get_device_module() _EXTERNAL_LINKER_SUPPORTED_COMPONENTS = frozenset( { @@ -133,6 +135,25 @@ class UnifiedCacheLinker(ABC): layer_done_counter: object pools: dict[PoolName, object] + # The stream the forward writes the device pools on; None when the current + # stream is the pages' only writer. + forward_stream: Optional[torch.cuda.Stream] = None + + def record_offload_ready(self) -> list[torch.cuda.Event]: + """Record the events an offload waits on before it reads device pages. + + The current stream has computed which pages to offload, and the forward + stream may still be writing them: prefill read-done lets the scheduler's + stream run ahead of the forward that fills a stashed chunk. + """ + indices_ready = device_module.Event() + indices_ready.record() + events = [indices_ready] + if self.forward_stream is not None: + kv_written = device_module.Event() + kv_written.record(self.forward_stream) + events.append(kv_written) + return events @abstractmethod def lookup(self, rid: str, transfers: list[PoolTransfer]) -> list[int]: diff --git a/test/registered/unit/mem_cache/test_registry.py b/test/registered/unit/mem_cache/test_registry.py index 57a7cb2263..be99329679 100644 --- a/test/registered/unit/mem_cache/test_registry.py +++ b/test/registered/unit/mem_cache/test_registry.py @@ -339,6 +339,7 @@ class TestCreateTreeCacheRouting(_RegistryIsolationMixin, CustomTestCase): ctx.server_args, ctx.params, components={"full"}, + forward_stream=ctx.tp_worker.model_runner.forward_stream, ) cache.init_cache_linker.assert_called_once_with(linker) ctx.params.token_to_kv_pool_allocator.get_kvcache.return_value.register_layer_transfer_counter.assert_called_once_with( diff --git a/test/registered/unit/mem_cache/test_umbp_direct_linker_rank_keys.py b/test/registered/unit/mem_cache/test_umbp_direct_linker_rank_keys.py index fc5c87646f..b8b5343b66 100644 --- a/test/registered/unit/mem_cache/test_umbp_direct_linker_rank_keys.py +++ b/test/registered/unit/mem_cache/test_umbp_direct_linker_rank_keys.py @@ -7,7 +7,7 @@ from unittest import mock import torch -import sglang.srt.mem_cache.storage.umbp.umbp_direct_linker as umbp_direct_linker +import sglang.srt.mem_cache.unified_cache.unified_cache_linker as unified_cache_linker from sglang.srt.mem_cache.hicache_storage import PoolName from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import DevicePoolEntry from sglang.srt.mem_cache.storage.umbp.umbp_direct_linker import ( @@ -120,7 +120,7 @@ class TestEveryRankWrites(CustomTestCase): fake_device = SimpleNamespace( Event=lambda: SimpleNamespace(record=lambda: None) ) - with mock.patch.object(umbp_direct_linker, "device_module", fake_device): + with mock.patch.object(unified_cache_linker, "device_module", fake_device): self.assertTrue(linker.offload(["t"])) self.assertEqual(linker._offload_queue.qsize(), 1) # The result comes from the offload thread once the write lands, so diff --git a/test/registered/unit/mem_cache/test_unified_cache_linker.py b/test/registered/unit/mem_cache/test_unified_cache_linker.py index ea39200f21..77fce944ef 100644 --- a/test/registered/unit/mem_cache/test_unified_cache_linker.py +++ b/test/registered/unit/mem_cache/test_unified_cache_linker.py @@ -1,3 +1,4 @@ +import queue import sys import threading import unittest @@ -5,7 +6,7 @@ from array import array from collections import defaultdict from dataclasses import replace from types import SimpleNamespace -from unittest.mock import MagicMock +from unittest.mock import MagicMock, call, patch import pytest import torch @@ -24,6 +25,9 @@ from sglang.srt.mem_cache.hicache_storage import ( PoolTransfer, ) from sglang.srt.mem_cache.radix_cache import RadixKey +from sglang.srt.mem_cache.storage.mooncake_store.mooncake_direct_linker import ( + MooncakeDirectLinker, +) from sglang.srt.mem_cache.unified_cache.cache_action import ( UpdatePendingBackupOnNodeSplit, ) @@ -1571,5 +1575,53 @@ def test_linker_load_preserves_swa_boundaries( swa.update_external_linker_load.assert_not_called() +class TestOffloadReadyEvents(CustomTestCase): + def _record(self, forward_stream): + device = MagicMock(name="device_module") + device.Event.side_effect = lambda: MagicMock(name="event") + linker = _FakeLinker() + linker.forward_stream = forward_stream + with patch( + "sglang.srt.mem_cache.unified_cache.unified_cache_linker.device_module", + device, + ): + return linker.record_offload_ready() + + def test_offload_waits_on_the_forward_writing_the_pages(self): + forward_stream = MagicMock(name="forward_stream") + indices_ready, kv_written = self._record(forward_stream) + indices_ready.record.assert_called_once_with() + kv_written.record.assert_called_once_with(forward_stream) + # The offload thread waits; the forward stream's own order is untouched. + self.assertEqual(forward_stream.mock_calls, []) + + def test_offload_without_a_forward_stream_waits_on_the_current_one(self): + (indices_ready,) = self._record(None) + indices_ready.record.assert_called_once_with() + + def test_mooncake_offload_thread_waits_on_every_event(self): + linker = MooncakeDirectLinker.__new__(MooncakeDirectLinker) + order = MagicMock() + linker.storage = order.storage + order.storage.batch_set_v2.return_value = {} + linker.stats = {"offload": 1} + linker.offload_results = queue.Queue() + linker.offload_queue = queue.Queue() + linker.offload_queue.put( + (["transfer"], 1, [order.indices_ready, order.kv_written]) + ) + linker.offload_queue.put(None) + linker.offload_thread_func() + self.assertEqual( + order.mock_calls, + [ + call.indices_ready.synchronize(), + call.kv_written.synchronize(), + call.storage.batch_set_v2(["transfer"]), + ], + ) + self.assertTrue(linker.offload_results.get_nowait()) + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-v"]))