[Fix] disaggregation: Synchronize deferred decode KV release across TP ranks (#45453)

This commit is contained in:
paulzhang-tm
2026-10-22 01:37:28 +08:00
committed by GitHub
parent ed268dd320
commit b57b5e2286
3 changed files with 237 additions and 56 deletions
+73 -53
View File
@@ -2775,6 +2775,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
# Queue-removed but held for deferred release; excluded from the metadata
# teardown below.
deferred_indices = set()
failed: List[Tuple[int, DecodeRequest, bool]] = []
for i, (decode_req, poll) in enumerate(zip(self.queue, polls)):
if rids_to_check is not None and decode_req.req.rid not in rids_to_check:
continue
@@ -2829,22 +2830,15 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
# already hold destinations; this path always reaches
# _do_release, which clears the tracker.
decode_req.kv_receiver.ensure_abort_notified(force_arm=True)
if requires_host_drain or (
deferrable and decode_req.kv_receiver.abort_notified
):
# Host pages always await a drain ack. A receiver that could
# not notify (metadata never published, so no prefill holds
# its destination info) has nothing in flight and releases
# immediately below.
self._defer_release(decode_req)
deferred_indices.add(i)
indices_to_remove.add(i)
else:
# release pre-allocated kv cache, but don't insert into the tree since it's failed
self._release_request(decode_req)
decode_req.kv_receiver.clear()
decode_req.kv_receiver = None
indices_to_remove.add(i)
failed.append(
(
i,
decode_req,
requires_host_drain
or (deferrable and decode_req.kv_receiver.abort_notified),
)
)
indices_to_remove.add(i)
self._record_transfer_failure(failure_reason)
continue
elif poll == KVPoll.Success:
@@ -2890,6 +2884,20 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
else:
raise ValueError(f"Unexpected poll case: {poll}")
# Notification state is rank-local. If any rank must hold a failed
# transfer, all must retain its capacity so admission stays identical.
held = self._all_reduce_release_flags(
[hold for _, _, hold in failed], torch.distributed.ReduceOp.MAX
)
for (i, decode_req, _), hold in zip(failed, held):
if hold:
self._defer_release(decode_req)
deferred_indices.add(i)
else:
self._release_request(decode_req)
decode_req.kv_receiver.clear()
decode_req.kv_receiver = None
for i in indices_to_remove:
if i in deferred_indices:
# Held for deferred release; metadata buffer freed at resolve time.
@@ -2918,9 +2926,11 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
deadline = start_time + self.deferred_kv_release_timeout
# Host-capable receivers own their drain tracking. Device destinations
# snapshot the number of prefill acknowledgements required by the manager.
# A rank that only holds for a peer has no local acknowledgements to await.
required_acks = (
0
if self.enable_host_receive and decode_req.host_staged
if (self.enable_host_receive and decode_req.host_staged)
or not decode_req.kv_receiver.abort_notified
else len(decode_req.kv_receiver.bootstrap_infos)
)
self._deferred_releases.append(
@@ -2961,52 +2971,62 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
outcome=outcome,
)
def _all_reduce_release_flags(
self, flags: List[bool], op: torch.distributed.ReduceOp
) -> List[bool]:
# Failures come from reduced polls and deferred entries from agreed holds,
# so every rank supplies the same entries in the same order, even if empty.
if not flags or torch.distributed.get_world_size(self.gloo_group) == 1:
return flags
reduced = torch.tensor(flags, dtype=torch.int32, device="cpu")
torch.distributed.all_reduce(reduced, op=op, group=self.gloo_group)
return [bool(flag) for flag in reduced.tolist()]
def resolve_deferred_releases(self) -> None:
"""Release drained transfers; device destinations may also time out."""
if not self._deferred_releases:
return
host_ready = torch.tensor(
[
req.kv_receiver.is_abort_release_safe()
for req, _, _, _, _ in self._deferred_releases
if self.enable_host_receive and req.host_staged
],
dtype=torch.int32,
)
if host_ready.numel() and torch.distributed.get_world_size(self.gloo_group) > 1:
# Admission must see identical host capacity and page order on every rank.
torch.distributed.all_reduce(
host_ready, op=torch.distributed.ReduceOp.MIN, group=self.gloo_group
)
host_ready = iter(host_ready.tolist())
now = time.monotonic()
drained = [
decode_req.kv_receiver.is_abort_release_safe()
if self.enable_host_receive and decode_req.host_staged
else decode_req.kv_receiver.kv_mgr.is_abort_release_safe(
decode_req.req.bootstrap_room, required_acks
)
for decode_req, _, _, _, required_acks in self._deferred_releases
]
# Acks and deadlines are rank-local. Releasing only on some ranks would
# diverge admission and the sizes of subsequent polling collectives.
# Host pages still require a drain: a timeout cannot rule out remote writes.
ready = self._all_reduce_release_flags(
[
is_drained
or (
not (self.enable_host_receive and decode_req.host_staged)
and now >= deadline
)
for (decode_req, _, deadline, _, _), is_drained in zip(
self._deferred_releases, drained
)
],
torch.distributed.ReduceOp.MIN,
)
still_held = []
to_release = []
for (
decode_req,
start_time,
deadline,
idx,
required_acks,
) in self._deferred_releases:
room = decode_req.req.bootstrap_room
if self.enable_host_receive and decode_req.host_staged:
if next(host_ready):
to_release.append((decode_req, start_time, idx, room, True))
else:
# A timeout cannot prove that a remote write has stopped.
still_held.append(
(decode_req, start_time, deadline, idx, required_acks)
for entry, is_ready, is_drained in zip(self._deferred_releases, ready, drained):
decode_req, start_time, _, idx, _ = entry
if is_ready:
to_release.append(
(
decode_req,
start_time,
idx,
decode_req.req.bootstrap_room,
is_drained,
)
continue
kv_mgr = decode_req.kv_receiver.kv_mgr
drained = kv_mgr.is_abort_release_safe(room, required_acks)
if not drained and now < deadline:
still_held.append(
(decode_req, start_time, deadline, idx, required_acks)
)
else:
to_release.append((decode_req, start_time, idx, room, drained))
still_held.append(entry)
# Commit the survivors before releasing so a _do_release exception can't
# leave a released entry in the list (double-free / None receiver on retry).
self._deferred_releases = still_held