[Bugfix] fix(hicache): wait for decode offload before retraction (#30899)

Co-authored-by: agent <agent@local>
Co-authored-by: Kevin Flansburg <6134007+kflansburg@users.noreply.github.com>
Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com>
Co-authored-by: Shangming Cai <csmthu@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Kevin Flansburg
2026-09-27 23:24:19 -07:00
committed by GitHub
co-authored by agent Kevin Flansburg Jimmy Shong Shangming Cai Claude Fable 5
parent 6be9c78cf4
commit e75e3b8a98
3 changed files with 224 additions and 3 deletions
@@ -277,6 +277,28 @@ class DecodeKVCacheOffloadManager:
self._check_offload_progress(n_write)
self._check_backup_progress(n_backup)
def drain_before_retraction(self):
"""Drain offloads, then refuse retraction on every rank if any rank still tracks a D2H copy."""
self.check_offload_progress()
cc = self.cache_controller
# Retraction frees the device KV these copies read. Storage backups read only
# host memory, so they do not count.
pending = torch.tensor(
[bool(cc.ack_write_queue or self.ongoing_offload or self.offload_inflight)],
dtype=torch.int,
)
if self.tp_world_size > 1:
torch.distributed.all_reduce(
pending, op=torch.distributed.ReduceOp.MAX, group=self.tp_group
)
if pending.item():
raise RuntimeError(
"Decode KV offload copies are still pending on some rank; "
f"refusing to retract (local: {len(cc.ack_write_queue)} acks, "
f"{len(self.ongoing_offload)} copies, "
f"{len(self.offload_inflight)} requests)."
)
def _check_offload_progress(self, finish_count):
"""Check the progress of offload from device to host."""
while finish_count > 0:
+7
View File
@@ -4201,6 +4201,9 @@ class Scheduler(
if (kv_full_retract_flag := not batch.check_decode_mem()) or (
TEST_RETRACT and self.forward_ct % TEST_RETRACT_INTERVAL == 0
):
if self.decode_offload_manager is not None:
# Pending offload copies may still read the device KV that retraction frees.
self.decode_offload_manager.drain_before_retraction()
old_available_tokens = self.token_to_kv_pool_allocator.available_size()
old_ratio = self.new_token_ratio_tracker.current
mamba_allocator = getattr(
@@ -5622,6 +5625,10 @@ class Scheduler(
self.last_batch = None
self.cur_batch_for_debug = None
if self.decode_offload_manager is not None:
# Pending offload copies may still read the device KV that retraction frees,
# and the paused decode loop never drains them.
self.decode_offload_manager.drain_before_retraction()
if retract_reqs:
# Decode-side retract always rebootstraps (recomputes the KV from
# the prefill), so skip the device->host KV offload that release_req