[Fix] radix-cache: Order linker offloads behind the forward that writes the pages (#45559)

Co-authored-by: lixiufei-leo <294029388+lixiufei-leo@users.noreply.github.com>
This commit is contained in:
Liangsheng Yin
2026-10-21 13:53:37 -07:00
committed by GitHub
co-authored by lixiufei-leo
parent 028de2c5d4
commit 2c7a9e3b50
7 changed files with 99 additions and 16 deletions
+6 -1
View File
@@ -217,7 +217,12 @@ def create_unified_radix_cache(
)
cache.init_cache_linker(
linker_cls(server_args, params, components=set(cache.components))
linker_cls(
server_args,
params,
components=set(cache.components),
forward_stream=ctx.tp_worker.model_runner.forward_stream,
)
)
counter = cache.linker.layer_done_counter
kvcache = params.token_to_kv_pool_allocator.get_kvcache()
@@ -55,7 +55,9 @@ class MooncakeDirectLinker(UnifiedCacheLinker):
*,
components,
storage=None,
forward_stream=None,
):
self.forward_stream = forward_stream
self.page_size = params.page_size
kvcache = params.token_to_kv_pool_allocator.get_kvcache()
self.pool_group = resolve_hybrid_device_pool_group(
@@ -384,9 +386,8 @@ class MooncakeDirectLinker(UnifiedCacheLinker):
return True
kv = next(transfer for transfer in transfers if transfer.name == PoolName.KV)
tokens = len(kv.keys) * self.page_size
ready_event = device_module.Event()
ready_event.record()
self.offload_queue.put((expanded, tokens, ready_event))
ready_events = self.record_offload_ready()
self.offload_queue.put((expanded, tokens, ready_events))
return True
def offload_thread_func(self) -> None:
@@ -395,8 +396,9 @@ class MooncakeDirectLinker(UnifiedCacheLinker):
try:
if task is None:
return
expanded, tokens, ready_event = task
ready_event.synchronize()
expanded, tokens, ready_events = task
for event in ready_events:
event.synchronize()
results = self.storage.batch_set_v2(expanded)
success = all(all(pool_results) for pool_results in results.values())
if success:
@@ -91,8 +91,8 @@ class _PoolRangePlan:
entries_per_page: int
# One queued offload: the pools it resolved to, and the event guarding its KV.
_OffloadTask = tuple[list[PoolTransfer], object]
# One queued offload: the pools it resolved to, and the events gating the read.
_OffloadTask = tuple[list[PoolTransfer], list[object]]
def _offload_task_pages(expanded: list[PoolTransfer]) -> int:
@@ -152,7 +152,9 @@ class UMBPDirectLinker(UnifiedCacheLinker):
*,
components,
_storage=None,
forward_stream=None,
):
self.forward_stream = forward_stream
self.page_size = params.page_size
# Group layers to amortize per-object RPC overhead; 8 is the measured default.
self.layer_group = max(1, int(os.getenv("UMBP_LAYER_GROUP", "8")))
@@ -856,9 +858,8 @@ class UMBPDirectLinker(UnifiedCacheLinker):
if not expanded:
return False
self._freeze_gc_once()
ready_event = device_module.Event()
ready_event.record()
self._offload_queue.put((expanded, ready_event))
ready_events = self.record_offload_ready()
self._offload_queue.put((expanded, ready_events))
return True
def _take_offload_batch(self) -> tuple[list[_OffloadTask], bool]:
@@ -912,8 +913,9 @@ class UMBPDirectLinker(UnifiedCacheLinker):
self._offload_results.put(success)
def _run_offload(self, tasks: list[_OffloadTask]) -> bool:
for _, ready_event in tasks:
ready_event.synchronize()
for _, ready_events in tasks:
for event in ready_events:
event.synchronize()
next_slot = 0
def materialize_indices(indices: torch.Tensor) -> torch.Tensor:
@@ -46,6 +46,7 @@ from sglang.srt.mem_cache.unified_cache.components import (
TreeComponent,
)
from sglang.srt.mem_cache.utils import get_storage_hash_str
from sglang.srt.utils import get_device_module
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
@@ -53,6 +54,7 @@ if TYPE_CHECKING:
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
logger = logging.getLogger(__name__)
device_module = get_device_module()
_EXTERNAL_LINKER_SUPPORTED_COMPONENTS = frozenset(
{
@@ -133,6 +135,25 @@ class UnifiedCacheLinker(ABC):
layer_done_counter: object
pools: dict[PoolName, object]
# The stream the forward writes the device pools on; None when the current
# stream is the pages' only writer.
forward_stream: Optional[torch.cuda.Stream] = None
def record_offload_ready(self) -> list[torch.cuda.Event]:
"""Record the events an offload waits on before it reads device pages.
The current stream has computed which pages to offload, and the forward
stream may still be writing them: prefill read-done lets the scheduler's
stream run ahead of the forward that fills a stashed chunk.
"""
indices_ready = device_module.Event()
indices_ready.record()
events = [indices_ready]
if self.forward_stream is not None:
kv_written = device_module.Event()
kv_written.record(self.forward_stream)
events.append(kv_written)
return events
@abstractmethod
def lookup(self, rid: str, transfers: list[PoolTransfer]) -> list[int]: