[Perf] hicache: Publish prefill read-done when offload copies are fenced on the forward stream
The prefill shared-read-done fastpath lets the schedule stream run ahead of the forward that writes a stashed chunk's KV instead of waiting for the whole forward. It was disabled whenever hierarchical cache or a storage backend was enabled, because host and storage offload copies used to be ordered only behind the schedule stream and could have read unfinished KV. The cache controller now orders its device-to-host copies behind the forward stream through write_fence_stream, and its load-back H2D copies through load_fence_stream, so that reason no longer holds when every KV offload copy carries that fence. The scheduler records on the model runner whether that is the case: hierarchical cache with a cache controller and no decode offload manager, which has no such fence. The publish helper keeps the coarse whole-forward barrier for LMCache and for unfenced offload, and publishes read-done otherwise. With the coarse barrier, a synchronous page-index copy issued while preparing the next batch inherited the wait on the entire in-flight forward, leaving the GPU idle between batches. With read-done published, that copy only waits for the shared-table reads to finish; KV ordering is unchanged because the backup copies still wait on the forward stream. Tests: the CPU runner test covers the fenced hierarchical-cache and storage cases publishing, and LMCache keeping the barrier even when hierarchical cache is fenced. The GPU ordering test checks that a write-through backup of a chunk stashed before its forward finished still copies the final KV. Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com> (cherry picked from commit bb1cb2dded6fb9352d98076280342b10684e061b)
This commit is contained in:
@@ -645,6 +645,14 @@ class Scheduler(
|
||||
)
|
||||
else:
|
||||
self.decode_offload_manager = None
|
||||
# Prefill read-done lets the schedule stream run ahead of the forward that
|
||||
# writes a stashed chunk. The cache controller's fence streams above order
|
||||
# its copies behind that forward; the decode offload controller has none.
|
||||
self.tp_worker.model_runner.kv_offload_fenced_on_forward_stream = (
|
||||
self.enable_hierarchical_cache
|
||||
and self.tree_cache.cache_controller is not None
|
||||
and self.decode_offload_manager is None
|
||||
)
|
||||
|
||||
# Init running status
|
||||
self.init_running_status()
|
||||
|
||||
@@ -468,6 +468,8 @@ class ModelRunner:
|
||||
# Read-done mailbox: the scheduler's WAR barrier reads it from the runner
|
||||
# its worker names, and treats None as the coarse whole-forward fence.
|
||||
self.shared_read_done_event: Optional[torch.cuda.Event] = None
|
||||
# Set by the scheduler when every KV offload copy waits for forward_stream.
|
||||
self.kv_offload_fenced_on_forward_stream = False
|
||||
|
||||
# CPU offload
|
||||
set_offloader(create_offloader())
|
||||
|
||||
@@ -31,12 +31,12 @@ def maybe_publish_prefill_shared_read_done(
|
||||
if not model_runner.spec_algorithm.supports_prefill_shared_read_done():
|
||||
return
|
||||
# Host and storage offload copy a stashed chunk's KV ordered only behind the
|
||||
# schedule stream, so they need the whole-forward barrier.
|
||||
# schedule stream, so they need the whole-forward barrier unless every such
|
||||
# copy waits for the forward stream itself.
|
||||
memory = get_memory()
|
||||
if (
|
||||
memory.enable_hierarchical_cache
|
||||
or memory.hicache_storage_backend is not None
|
||||
or memory.enable_lmcache
|
||||
if memory.enable_lmcache or (
|
||||
(memory.enable_hierarchical_cache or memory.hicache_storage_backend is not None)
|
||||
and not model_runner.kv_offload_fenced_on_forward_stream
|
||||
):
|
||||
return
|
||||
logger.info_once(
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Write-back eviction must wait for the checkpoint's KV producer."""
|
||||
"""HiCache backups must wait for the checkpoint's KV producer."""
|
||||
|
||||
import unittest
|
||||
from array import array
|
||||
@@ -109,6 +109,72 @@ class TestHiCacheWriteBackOrdering(UnifiedRadixCacheFixtures, unittest.TestCase)
|
||||
finally:
|
||||
forward.synchronize()
|
||||
|
||||
def test_write_through_stashed_chunk_waits_for_its_producer(self):
|
||||
# Prefill read-done lets the scheduler stash a chunk while its forward
|
||||
# still writes the KV; the write-through backup must copy the final KV.
|
||||
self.cfg = CacheConfig(
|
||||
components=(ComponentType.FULL, ComponentType.SWA),
|
||||
page_size=1,
|
||||
sliding_window_size=4,
|
||||
num_layers=2,
|
||||
full_attention_layer_ids=(0,),
|
||||
kv_size=64,
|
||||
max_context_len=64,
|
||||
head_num=1,
|
||||
head_dim=64,
|
||||
)
|
||||
cache, allocator, req_pool = build_fixture(self.cfg)
|
||||
self._init_hicache(cache, write_policy="write_through")
|
||||
cache.write_through_threshold = 1
|
||||
controller = cache.cache_controller
|
||||
forward = torch.cuda.Stream()
|
||||
controller.write_fence_stream = forward
|
||||
req = self._request(cache, allocator, req_pool, 0, length=12)
|
||||
kv = allocator.get_kvcache()
|
||||
buffers = [
|
||||
buf
|
||||
for pool in (kv.full_kv_pool, kv.swa_kv_pool)
|
||||
for buf in pool.k_buffer + pool.v_buffer
|
||||
]
|
||||
for buf in buffers:
|
||||
buf.fill_(-1)
|
||||
read_done = torch.cuda.Event()
|
||||
torch.cuda.synchronize()
|
||||
engine = controller.l2_transfer_engine
|
||||
submit = engine.submit_device_to_host
|
||||
snapshots = []
|
||||
|
||||
def record_copy(transfers):
|
||||
completion = submit(transfers)
|
||||
completion.finish_event.synchronize()
|
||||
for transfer in transfers:
|
||||
if transfer.host_indices.numel():
|
||||
snapshots.append(
|
||||
transfer.host_pool.get_data_page(
|
||||
int(transfer.host_indices[0])
|
||||
).clone()
|
||||
)
|
||||
return completion
|
||||
|
||||
with torch.cuda.stream(forward):
|
||||
read_done.record()
|
||||
torch.cuda._sleep(600_000_000)
|
||||
for buf in buffers:
|
||||
buf.fill_(42)
|
||||
torch.cuda.current_stream().wait_event(read_done)
|
||||
try:
|
||||
with patch.object(engine, "submit_device_to_host", side_effect=record_copy):
|
||||
cache.checkpoint(req, up_to=12)
|
||||
cache.flush_pending_backups()
|
||||
self.assertTrue(snapshots)
|
||||
for snapshot in snapshots:
|
||||
self.assertTrue(
|
||||
torch.equal(snapshot, torch.full_like(snapshot, 42)),
|
||||
f"expected 42, copied {snapshot.unique().tolist()}",
|
||||
)
|
||||
finally:
|
||||
forward.synchronize()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -35,7 +35,11 @@ class _Event:
|
||||
|
||||
|
||||
def _model_runner(
|
||||
*, spec_algorithm=SpeculativeAlgorithm.NONE, compliant=True, declared=None
|
||||
*,
|
||||
spec_algorithm=SpeculativeAlgorithm.NONE,
|
||||
compliant=True,
|
||||
declared=None,
|
||||
offload_fenced=False,
|
||||
):
|
||||
if declared is None:
|
||||
declared = SharedReadEnds.PRE_REPLAY if compliant else SharedReadEnds.UNKNOWN
|
||||
@@ -45,6 +49,7 @@ def _model_runner(
|
||||
spec_algorithm=spec_algorithm,
|
||||
attn_backend=attn_backend,
|
||||
shared_read_done_event=None,
|
||||
kv_offload_fenced_on_forward_stream=offload_fenced,
|
||||
)
|
||||
return runner
|
||||
|
||||
@@ -92,6 +97,28 @@ def test_kv_offload_keeps_the_coarse_barrier(memory):
|
||||
assert runner.shared_read_done_event is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"memory",
|
||||
[
|
||||
{"enable_hierarchical_cache": True},
|
||||
{"enable_hierarchical_cache": True, "hicache_storage_backend": "mooncake"},
|
||||
],
|
||||
)
|
||||
def test_fenced_kv_offload_publishes(memory):
|
||||
runner = _model_runner(offload_fenced=True)
|
||||
with get_memory().override(**memory):
|
||||
_prepare_and_publish(runner, _batch())
|
||||
published = runner.shared_read_done_event
|
||||
assert isinstance(published, _Event) and published.recorded
|
||||
|
||||
|
||||
def test_lmcache_keeps_the_coarse_barrier_when_hicache_is_fenced():
|
||||
runner = _model_runner(offload_fenced=True)
|
||||
with get_memory().override(enable_hierarchical_cache=True, enable_lmcache=True):
|
||||
_prepare_and_publish(runner, _batch())
|
||||
assert runner.shared_read_done_event is None
|
||||
|
||||
|
||||
def test_dflash_target_prefill_publishes():
|
||||
runner = _model_runner(spec_algorithm=SpeculativeAlgorithm.DFLASH)
|
||||
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True):
|
||||
|
||||
Reference in New Issue
Block a user