[Fix] radix-cache: Order linker offloads behind the forward that writes the pages (#45559)
Co-authored-by: lixiufei-leo <294029388+lixiufei-leo@users.noreply.github.com>
This commit is contained in:
co-authored by
lixiufei-leo
parent
028de2c5d4
commit
2c7a9e3b50
@@ -217,7 +217,12 @@ def create_unified_radix_cache(
|
||||
)
|
||||
|
||||
cache.init_cache_linker(
|
||||
linker_cls(server_args, params, components=set(cache.components))
|
||||
linker_cls(
|
||||
server_args,
|
||||
params,
|
||||
components=set(cache.components),
|
||||
forward_stream=ctx.tp_worker.model_runner.forward_stream,
|
||||
)
|
||||
)
|
||||
counter = cache.linker.layer_done_counter
|
||||
kvcache = params.token_to_kv_pool_allocator.get_kvcache()
|
||||
|
||||
@@ -55,7 +55,9 @@ class MooncakeDirectLinker(UnifiedCacheLinker):
|
||||
*,
|
||||
components,
|
||||
storage=None,
|
||||
forward_stream=None,
|
||||
):
|
||||
self.forward_stream = forward_stream
|
||||
self.page_size = params.page_size
|
||||
kvcache = params.token_to_kv_pool_allocator.get_kvcache()
|
||||
self.pool_group = resolve_hybrid_device_pool_group(
|
||||
@@ -384,9 +386,8 @@ class MooncakeDirectLinker(UnifiedCacheLinker):
|
||||
return True
|
||||
kv = next(transfer for transfer in transfers if transfer.name == PoolName.KV)
|
||||
tokens = len(kv.keys) * self.page_size
|
||||
ready_event = device_module.Event()
|
||||
ready_event.record()
|
||||
self.offload_queue.put((expanded, tokens, ready_event))
|
||||
ready_events = self.record_offload_ready()
|
||||
self.offload_queue.put((expanded, tokens, ready_events))
|
||||
return True
|
||||
|
||||
def offload_thread_func(self) -> None:
|
||||
@@ -395,8 +396,9 @@ class MooncakeDirectLinker(UnifiedCacheLinker):
|
||||
try:
|
||||
if task is None:
|
||||
return
|
||||
expanded, tokens, ready_event = task
|
||||
ready_event.synchronize()
|
||||
expanded, tokens, ready_events = task
|
||||
for event in ready_events:
|
||||
event.synchronize()
|
||||
results = self.storage.batch_set_v2(expanded)
|
||||
success = all(all(pool_results) for pool_results in results.values())
|
||||
if success:
|
||||
|
||||
@@ -91,8 +91,8 @@ class _PoolRangePlan:
|
||||
entries_per_page: int
|
||||
|
||||
|
||||
# One queued offload: the pools it resolved to, and the event guarding its KV.
|
||||
_OffloadTask = tuple[list[PoolTransfer], object]
|
||||
# One queued offload: the pools it resolved to, and the events gating the read.
|
||||
_OffloadTask = tuple[list[PoolTransfer], list[object]]
|
||||
|
||||
|
||||
def _offload_task_pages(expanded: list[PoolTransfer]) -> int:
|
||||
@@ -152,7 +152,9 @@ class UMBPDirectLinker(UnifiedCacheLinker):
|
||||
*,
|
||||
components,
|
||||
_storage=None,
|
||||
forward_stream=None,
|
||||
):
|
||||
self.forward_stream = forward_stream
|
||||
self.page_size = params.page_size
|
||||
# Group layers to amortize per-object RPC overhead; 8 is the measured default.
|
||||
self.layer_group = max(1, int(os.getenv("UMBP_LAYER_GROUP", "8")))
|
||||
@@ -856,9 +858,8 @@ class UMBPDirectLinker(UnifiedCacheLinker):
|
||||
if not expanded:
|
||||
return False
|
||||
self._freeze_gc_once()
|
||||
ready_event = device_module.Event()
|
||||
ready_event.record()
|
||||
self._offload_queue.put((expanded, ready_event))
|
||||
ready_events = self.record_offload_ready()
|
||||
self._offload_queue.put((expanded, ready_events))
|
||||
return True
|
||||
|
||||
def _take_offload_batch(self) -> tuple[list[_OffloadTask], bool]:
|
||||
@@ -912,8 +913,9 @@ class UMBPDirectLinker(UnifiedCacheLinker):
|
||||
self._offload_results.put(success)
|
||||
|
||||
def _run_offload(self, tasks: list[_OffloadTask]) -> bool:
|
||||
for _, ready_event in tasks:
|
||||
ready_event.synchronize()
|
||||
for _, ready_events in tasks:
|
||||
for event in ready_events:
|
||||
event.synchronize()
|
||||
next_slot = 0
|
||||
|
||||
def materialize_indices(indices: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -46,6 +46,7 @@ from sglang.srt.mem_cache.unified_cache.components import (
|
||||
TreeComponent,
|
||||
)
|
||||
from sglang.srt.mem_cache.utils import get_storage_hash_str
|
||||
from sglang.srt.utils import get_device_module
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
@@ -53,6 +54,7 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
device_module = get_device_module()
|
||||
|
||||
_EXTERNAL_LINKER_SUPPORTED_COMPONENTS = frozenset(
|
||||
{
|
||||
@@ -133,6 +135,25 @@ class UnifiedCacheLinker(ABC):
|
||||
|
||||
layer_done_counter: object
|
||||
pools: dict[PoolName, object]
|
||||
# The stream the forward writes the device pools on; None when the current
|
||||
# stream is the pages' only writer.
|
||||
forward_stream: Optional[torch.cuda.Stream] = None
|
||||
|
||||
def record_offload_ready(self) -> list[torch.cuda.Event]:
|
||||
"""Record the events an offload waits on before it reads device pages.
|
||||
|
||||
The current stream has computed which pages to offload, and the forward
|
||||
stream may still be writing them: prefill read-done lets the scheduler's
|
||||
stream run ahead of the forward that fills a stashed chunk.
|
||||
"""
|
||||
indices_ready = device_module.Event()
|
||||
indices_ready.record()
|
||||
events = [indices_ready]
|
||||
if self.forward_stream is not None:
|
||||
kv_written = device_module.Event()
|
||||
kv_written.record(self.forward_stream)
|
||||
events.append(kv_written)
|
||||
return events
|
||||
|
||||
@abstractmethod
|
||||
def lookup(self, rid: str, transfers: list[PoolTransfer]) -> list[int]:
|
||||
|
||||
Reference in New Issue
Block a user