[Perf] hicache: Publish prefill read-done when offload copies are fenced on the forward stream

The prefill shared-read-done fastpath lets the schedule stream run ahead of
the forward that writes a stashed chunk's KV instead of waiting for the whole
forward. It was disabled whenever hierarchical cache or a storage backend was
enabled, because host and storage offload copies used to be ordered only
behind the schedule stream and could have read unfinished KV.

The cache controller now orders its device-to-host copies behind the forward
stream through write_fence_stream, and its load-back H2D copies through
load_fence_stream, so that reason no longer holds when every KV offload copy
carries that fence. The scheduler records on the model runner whether that is
the case: hierarchical cache with a cache controller and no decode offload
manager, which has no such fence. The publish helper keeps the coarse
whole-forward barrier for LMCache and for unfenced offload, and publishes
read-done otherwise.

With the coarse barrier, a synchronous page-index copy issued while preparing
the next batch inherited the wait on the entire in-flight forward, leaving the
GPU idle between batches. With read-done published, that copy only waits for
the shared-table reads to finish; KV ordering is unchanged because the backup
copies still wait on the forward stream.

Tests: the CPU runner test covers the fenced hierarchical-cache and storage
cases publishing, and LMCache keeping the barrier even when hierarchical cache
is fenced. The GPU ordering test checks that a write-through backup of a chunk
stashed before its forward finished still copies the final KV.

Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
(cherry picked from commit bb1cb2dded6fb9352d98076280342b10684e061b)
This commit is contained in:
metamergebot
2026-11-07 14:36:38 +08:00
committed by minke.yu
parent 97c087778b
commit 5f7be7fab4
5 changed files with 110 additions and 7 deletions
+8
View File
@@ -645,6 +645,14 @@ class Scheduler(
)
else:
self.decode_offload_manager = None
# Prefill read-done lets the schedule stream run ahead of the forward that
# writes a stashed chunk. The cache controller's fence streams above order
# its copies behind that forward; the decode offload controller has none.
self.tp_worker.model_runner.kv_offload_fenced_on_forward_stream = (
self.enable_hierarchical_cache
and self.tree_cache.cache_controller is not None
and self.decode_offload_manager is None
)
# Init running status
self.init_running_status()
@@ -468,6 +468,8 @@ class ModelRunner:
# Read-done mailbox: the scheduler's WAR barrier reads it from the runner
# its worker names, and treats None as the coarse whole-forward fence.
self.shared_read_done_event: Optional[torch.cuda.Event] = None
# Set by the scheduler when every KV offload copy waits for forward_stream.
self.kv_offload_fenced_on_forward_stream = False
# CPU offload
set_offloader(create_offloader())
@@ -31,12 +31,12 @@ def maybe_publish_prefill_shared_read_done(
if not model_runner.spec_algorithm.supports_prefill_shared_read_done():
return
# Host and storage offload copy a stashed chunk's KV ordered only behind the
# schedule stream, so they need the whole-forward barrier.
# schedule stream, so they need the whole-forward barrier unless every such
# copy waits for the forward stream itself.
memory = get_memory()
if (
memory.enable_hierarchical_cache
or memory.hicache_storage_backend is not None
or memory.enable_lmcache
if memory.enable_lmcache or (
(memory.enable_hierarchical_cache or memory.hicache_storage_backend is not None)
and not model_runner.kv_offload_fenced_on_forward_stream
):
return
logger.info_once(
@@ -1,4 +1,4 @@
"""Write-back eviction must wait for the checkpoint's KV producer."""
"""HiCache backups must wait for the checkpoint's KV producer."""
import unittest
from array import array
@@ -109,6 +109,72 @@ class TestHiCacheWriteBackOrdering(UnifiedRadixCacheFixtures, unittest.TestCase)
finally:
forward.synchronize()
def test_write_through_stashed_chunk_waits_for_its_producer(self):
# Prefill read-done lets the scheduler stash a chunk while its forward
# still writes the KV; the write-through backup must copy the final KV.
self.cfg = CacheConfig(
components=(ComponentType.FULL, ComponentType.SWA),
page_size=1,
sliding_window_size=4,
num_layers=2,
full_attention_layer_ids=(0,),
kv_size=64,
max_context_len=64,
head_num=1,
head_dim=64,
)
cache, allocator, req_pool = build_fixture(self.cfg)
self._init_hicache(cache, write_policy="write_through")
cache.write_through_threshold = 1
controller = cache.cache_controller
forward = torch.cuda.Stream()
controller.write_fence_stream = forward
req = self._request(cache, allocator, req_pool, 0, length=12)
kv = allocator.get_kvcache()
buffers = [
buf
for pool in (kv.full_kv_pool, kv.swa_kv_pool)
for buf in pool.k_buffer + pool.v_buffer
]
for buf in buffers:
buf.fill_(-1)
read_done = torch.cuda.Event()
torch.cuda.synchronize()
engine = controller.l2_transfer_engine
submit = engine.submit_device_to_host
snapshots = []
def record_copy(transfers):
completion = submit(transfers)
completion.finish_event.synchronize()
for transfer in transfers:
if transfer.host_indices.numel():
snapshots.append(
transfer.host_pool.get_data_page(
int(transfer.host_indices[0])
).clone()
)
return completion
with torch.cuda.stream(forward):
read_done.record()
torch.cuda._sleep(600_000_000)
for buf in buffers:
buf.fill_(42)
torch.cuda.current_stream().wait_event(read_done)
try:
with patch.object(engine, "submit_device_to_host", side_effect=record_copy):
cache.checkpoint(req, up_to=12)
cache.flush_pending_backups()
self.assertTrue(snapshots)
for snapshot in snapshots:
self.assertTrue(
torch.equal(snapshot, torch.full_like(snapshot, 42)),
f"expected 42, copied {snapshot.unique().tolist()}",
)
finally:
forward.synchronize()
if __name__ == "__main__":
unittest.main()
@@ -35,7 +35,11 @@ class _Event:
def _model_runner(
*, spec_algorithm=SpeculativeAlgorithm.NONE, compliant=True, declared=None
*,
spec_algorithm=SpeculativeAlgorithm.NONE,
compliant=True,
declared=None,
offload_fenced=False,
):
if declared is None:
declared = SharedReadEnds.PRE_REPLAY if compliant else SharedReadEnds.UNKNOWN
@@ -45,6 +49,7 @@ def _model_runner(
spec_algorithm=spec_algorithm,
attn_backend=attn_backend,
shared_read_done_event=None,
kv_offload_fenced_on_forward_stream=offload_fenced,
)
return runner
@@ -92,6 +97,28 @@ def test_kv_offload_keeps_the_coarse_barrier(memory):
assert runner.shared_read_done_event is None
@pytest.mark.parametrize(
"memory",
[
{"enable_hierarchical_cache": True},
{"enable_hierarchical_cache": True, "hicache_storage_backend": "mooncake"},
],
)
def test_fenced_kv_offload_publishes(memory):
runner = _model_runner(offload_fenced=True)
with get_memory().override(**memory):
_prepare_and_publish(runner, _batch())
published = runner.shared_read_done_event
assert isinstance(published, _Event) and published.recorded
def test_lmcache_keeps_the_coarse_barrier_when_hicache_is_fenced():
runner = _model_runner(offload_fenced=True)
with get_memory().override(enable_hierarchical_cache=True, enable_lmcache=True):
_prepare_and_publish(runner, _batch())
assert runner.shared_read_done_event is None
def test_dflash_target_prefill_publishes():
runner = _model_runner(spec_algorithm=SpeculativeAlgorithm.DFLASH)
with envs.SGLANG_ENABLE_PREFILL_WAR_READ_DONE.override(True):