[Bugfix] fix(hicache): wait for decode offload before retraction (#30899)
Co-authored-by: agent <agent@local> Co-authored-by: Kevin Flansburg <6134007+kflansburg@users.noreply.github.com> Co-authored-by: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> Co-authored-by: Shangming Cai <csmthu@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
agent
Kevin Flansburg
Jimmy Shong
Shangming Cai
Claude Fable 5
parent
6be9c78cf4
commit
e75e3b8a98
@@ -277,6 +277,28 @@ class DecodeKVCacheOffloadManager:
|
||||
self._check_offload_progress(n_write)
|
||||
self._check_backup_progress(n_backup)
|
||||
|
||||
def drain_before_retraction(self):
|
||||
"""Drain offloads, then refuse retraction on every rank if any rank still tracks a D2H copy."""
|
||||
self.check_offload_progress()
|
||||
cc = self.cache_controller
|
||||
# Retraction frees the device KV these copies read. Storage backups read only
|
||||
# host memory, so they do not count.
|
||||
pending = torch.tensor(
|
||||
[bool(cc.ack_write_queue or self.ongoing_offload or self.offload_inflight)],
|
||||
dtype=torch.int,
|
||||
)
|
||||
if self.tp_world_size > 1:
|
||||
torch.distributed.all_reduce(
|
||||
pending, op=torch.distributed.ReduceOp.MAX, group=self.tp_group
|
||||
)
|
||||
if pending.item():
|
||||
raise RuntimeError(
|
||||
"Decode KV offload copies are still pending on some rank; "
|
||||
f"refusing to retract (local: {len(cc.ack_write_queue)} acks, "
|
||||
f"{len(self.ongoing_offload)} copies, "
|
||||
f"{len(self.offload_inflight)} requests)."
|
||||
)
|
||||
|
||||
def _check_offload_progress(self, finish_count):
|
||||
"""Check the progress of offload from device to host."""
|
||||
while finish_count > 0:
|
||||
|
||||
@@ -4201,6 +4201,9 @@ class Scheduler(
|
||||
if (kv_full_retract_flag := not batch.check_decode_mem()) or (
|
||||
TEST_RETRACT and self.forward_ct % TEST_RETRACT_INTERVAL == 0
|
||||
):
|
||||
if self.decode_offload_manager is not None:
|
||||
# Pending offload copies may still read the device KV that retraction frees.
|
||||
self.decode_offload_manager.drain_before_retraction()
|
||||
old_available_tokens = self.token_to_kv_pool_allocator.available_size()
|
||||
old_ratio = self.new_token_ratio_tracker.current
|
||||
mamba_allocator = getattr(
|
||||
@@ -5622,6 +5625,10 @@ class Scheduler(
|
||||
self.last_batch = None
|
||||
self.cur_batch_for_debug = None
|
||||
|
||||
if self.decode_offload_manager is not None:
|
||||
# Pending offload copies may still read the device KV that retraction frees,
|
||||
# and the paused decode loop never drains them.
|
||||
self.decode_offload_manager.drain_before_retraction()
|
||||
if retract_reqs:
|
||||
# Decode-side retract always rebootstraps (recomputes the KV from
|
||||
# the prefill), so skip the device->host KV offload that release_req
|
||||
|
||||
Reference in New Issue
Block a user