[Fix] disaggregation: Synchronize deferred decode KV release across TP ranks (#45453)

This commit is contained in:
paulzhang-tm
2026-10-22 01:37:28 +08:00
committed by GitHub
parent ed268dd320
commit b57b5e2286
3 changed files with 237 additions and 56 deletions
+73 -53
View File
@@ -2775,6 +2775,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
# Queue-removed but held for deferred release; excluded from the metadata
# teardown below.
deferred_indices = set()
failed: List[Tuple[int, DecodeRequest, bool]] = []
for i, (decode_req, poll) in enumerate(zip(self.queue, polls)):
if rids_to_check is not None and decode_req.req.rid not in rids_to_check:
continue
@@ -2829,22 +2830,15 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
# already hold destinations; this path always reaches
# _do_release, which clears the tracker.
decode_req.kv_receiver.ensure_abort_notified(force_arm=True)
if requires_host_drain or (
deferrable and decode_req.kv_receiver.abort_notified
):
# Host pages always await a drain ack. A receiver that could
# not notify (metadata never published, so no prefill holds
# its destination info) has nothing in flight and releases
# immediately below.
self._defer_release(decode_req)
deferred_indices.add(i)
indices_to_remove.add(i)
else:
# release pre-allocated kv cache, but don't insert into the tree since it's failed
self._release_request(decode_req)
decode_req.kv_receiver.clear()
decode_req.kv_receiver = None
indices_to_remove.add(i)
failed.append(
(
i,
decode_req,
requires_host_drain
or (deferrable and decode_req.kv_receiver.abort_notified),
)
)
indices_to_remove.add(i)
self._record_transfer_failure(failure_reason)
continue
elif poll == KVPoll.Success:
@@ -2890,6 +2884,20 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
else:
raise ValueError(f"Unexpected poll case: {poll}")
# Notification state is rank-local. If any rank must hold a failed
# transfer, all must retain its capacity so admission stays identical.
held = self._all_reduce_release_flags(
[hold for _, _, hold in failed], torch.distributed.ReduceOp.MAX
)
for (i, decode_req, _), hold in zip(failed, held):
if hold:
self._defer_release(decode_req)
deferred_indices.add(i)
else:
self._release_request(decode_req)
decode_req.kv_receiver.clear()
decode_req.kv_receiver = None
for i in indices_to_remove:
if i in deferred_indices:
# Held for deferred release; metadata buffer freed at resolve time.
@@ -2918,9 +2926,11 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
deadline = start_time + self.deferred_kv_release_timeout
# Host-capable receivers own their drain tracking. Device destinations
# snapshot the number of prefill acknowledgements required by the manager.
# A rank that only holds for a peer has no local acknowledgements to await.
required_acks = (
0
if self.enable_host_receive and decode_req.host_staged
if (self.enable_host_receive and decode_req.host_staged)
or not decode_req.kv_receiver.abort_notified
else len(decode_req.kv_receiver.bootstrap_infos)
)
self._deferred_releases.append(
@@ -2961,52 +2971,62 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
outcome=outcome,
)
def _all_reduce_release_flags(
self, flags: List[bool], op: torch.distributed.ReduceOp
) -> List[bool]:
# Failures come from reduced polls and deferred entries from agreed holds,
# so every rank supplies the same entries in the same order, even if empty.
if not flags or torch.distributed.get_world_size(self.gloo_group) == 1:
return flags
reduced = torch.tensor(flags, dtype=torch.int32, device="cpu")
torch.distributed.all_reduce(reduced, op=op, group=self.gloo_group)
return [bool(flag) for flag in reduced.tolist()]
def resolve_deferred_releases(self) -> None:
"""Release drained transfers; device destinations may also time out."""
if not self._deferred_releases:
return
host_ready = torch.tensor(
[
req.kv_receiver.is_abort_release_safe()
for req, _, _, _, _ in self._deferred_releases
if self.enable_host_receive and req.host_staged
],
dtype=torch.int32,
)
if host_ready.numel() and torch.distributed.get_world_size(self.gloo_group) > 1:
# Admission must see identical host capacity and page order on every rank.
torch.distributed.all_reduce(
host_ready, op=torch.distributed.ReduceOp.MIN, group=self.gloo_group
)
host_ready = iter(host_ready.tolist())
now = time.monotonic()
drained = [
decode_req.kv_receiver.is_abort_release_safe()
if self.enable_host_receive and decode_req.host_staged
else decode_req.kv_receiver.kv_mgr.is_abort_release_safe(
decode_req.req.bootstrap_room, required_acks
)
for decode_req, _, _, _, required_acks in self._deferred_releases
]
# Acks and deadlines are rank-local. Releasing only on some ranks would
# diverge admission and the sizes of subsequent polling collectives.
# Host pages still require a drain: a timeout cannot rule out remote writes.
ready = self._all_reduce_release_flags(
[
is_drained
or (
not (self.enable_host_receive and decode_req.host_staged)
and now >= deadline
)
for (decode_req, _, deadline, _, _), is_drained in zip(
self._deferred_releases, drained
)
],
torch.distributed.ReduceOp.MIN,
)
still_held = []
to_release = []
for (
decode_req,
start_time,
deadline,
idx,
required_acks,
) in self._deferred_releases:
room = decode_req.req.bootstrap_room
if self.enable_host_receive and decode_req.host_staged:
if next(host_ready):
to_release.append((decode_req, start_time, idx, room, True))
else:
# A timeout cannot prove that a remote write has stopped.
still_held.append(
(decode_req, start_time, deadline, idx, required_acks)
for entry, is_ready, is_drained in zip(self._deferred_releases, ready, drained):
decode_req, start_time, _, idx, _ = entry
if is_ready:
to_release.append(
(
decode_req,
start_time,
idx,
decode_req.req.bootstrap_room,
is_drained,
)
continue
kv_mgr = decode_req.kv_receiver.kv_mgr
drained = kv_mgr.is_abort_release_safe(room, required_acks)
if not drained and now < deadline:
still_held.append(
(decode_req, start_time, deadline, idx, required_acks)
)
else:
to_release.append((decode_req, start_time, idx, room, drained))
still_held.append(entry)
# Commit the survivors before releasing so a _do_release exception can't
# leave a released entry in the list (double-free / None receiver on retry).
self._deferred_releases = still_held
@@ -45,6 +45,9 @@ class FakeReceiver:
class TestDecodeQueueCleanup(CustomTestCase):
def setUp(self):
world_size = patch("torch.distributed.get_world_size", return_value=1)
world_size.start()
self.addCleanup(world_size.stop)
# The code under test reads its config from the bags.
reset_context()
self.addCleanup(reset_context)
@@ -572,6 +575,12 @@ def _gated_decode_req(poll, restore_status):
class TestDecodeHiCacheRestoreGate(CustomTestCase):
def setUp(self):
super().setUp()
world_size = patch("torch.distributed.get_world_size", return_value=1)
world_size.start()
self.addCleanup(world_size.stop)
def test_gate_folds_restore_outcome_into_terminal_poll_only(self):
cases = [
(KVPoll.Success, HiCacheRestoreResult.READY, KVPoll.Success),
@@ -6,14 +6,21 @@ them immediately (which could let the still-in-flight write land on pages alread
reused by another request). The pages are released once every prefill rank acks
that its transfer drained (CommonKVManager.is_abort_release_safe). Device
destinations may also release on timeout; host destinations require the ack.
All decode attention-TP ranks must agree on holding and releasing each request.
"""
import datetime
import tempfile
import threading
import unittest
from pathlib import Path
from types import SimpleNamespace
from typing import NamedTuple
from unittest.mock import MagicMock, call, patch
import torch.distributed as dist
import torch.multiprocessing as mp
from sglang.srt.disaggregation import decode as decode_mod
from sglang.srt.disaggregation.base.conn import BaseKVManager, KVPoll
from sglang.srt.disaggregation.common.conn import (
@@ -27,10 +34,11 @@ from sglang.srt.disaggregation.common.conn import (
CommonKVSender,
)
from sglang.srt.disaggregation.decode import DecodeTransferQueue
from sglang.srt.disaggregation.decode_hicache_mixin import HiCacheRestoreResult
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
register_cpu_ci(est_time=45, suite="base-a-test-cpu")
ABORT_GENERATION = 7
@@ -661,14 +669,21 @@ class _FakeIdxAllocator:
def _make_queue(timeout=30.0, enable_metrics=False):
q = DecodeTransferQueue.__new__(DecodeTransferQueue)
q._deferred_releases = []
q.queue = []
q.gloo_group = dist.group.WORLD
q.tp_rank = 0
q.enable_deferred_kv_release = True
q.deferred_kv_release_timeout = timeout
q.enable_host_receive = False
q.enable_staging = False
q.staging_handler = None
q.tree_cache = object()
q.tree_cache = MagicMock()
q.metadata_buffers = SimpleNamespace(bootstrap_room={})
q.req_to_metadata_buffer_idx_allocator = _FakeIdxAllocator()
q.scheduler = SimpleNamespace(
enable_decode_hicache=False,
enable_hisparse=False,
output_streamer=MagicMock(),
metrics_reporter=SimpleNamespace(enable_metrics=enable_metrics),
metrics_collector=SimpleNamespace(
observe_decode_deferred_kv_release=MagicMock()
@@ -680,6 +695,10 @@ def _make_queue(timeout=30.0, enable_metrics=False):
def _make_decode_req(room, idx, mgr, n_prefill_ranks=1):
receiver = SimpleNamespace(
kv_mgr=mgr,
abort_notified=True,
ensure_abort_notified=MagicMock(),
failure_exception=lambda: None,
poll=lambda: KVPoll.Failed,
# One entry per prefill rank the decode notified of the abort; its length
# is the required drain-ack count (see DecodeTransferQueue._defer_release).
bootstrap_infos=[{"rank": r} for r in range(n_prefill_ranks)],
@@ -687,13 +706,23 @@ def _make_decode_req(room, idx, mgr, n_prefill_ranks=1):
clear=lambda: None,
)
return SimpleNamespace(
req=SimpleNamespace(bootstrap_room=room),
req=SimpleNamespace(bootstrap_room=room, rid=str(room), return_logprob=False),
host_staged=False,
hicache_restore_status=HiCacheRestoreResult.READY,
prefix_match=None,
hicache_restore_lock=None,
kv_receiver=receiver,
metadata_buffer_index=idx,
)
class TestResolveDeferredReleases(CustomTestCase):
def setUp(self):
super().setUp()
world_size = patch.object(dist, "get_world_size", return_value=1)
world_size.start()
self.addCleanup(world_size.stop)
def test_host_release_requires_drain_on_every_rank_even_after_timeout(self):
mgr = _make_manager()
q = _make_queue(timeout=-1)
@@ -866,6 +895,123 @@ class TestResolveDeferredReleases(CustomTestCase):
self.assertEqual(held_idx, 9)
self.assertEqual(required, 1)
def test_two_rank_release_consensus(self):
with tempfile.TemporaryDirectory() as tmp:
mp.spawn(
_run_release_consensus,
args=(str(Path(tmp) / "rendezvous"),),
nprocs=2,
join=True,
)
def _run_release_consensus(rank: int, rendezvous: str) -> None:
dist.init_process_group(
"gloo",
init_method=f"file://{rendezvous}",
rank=rank,
world_size=2,
timeout=datetime.timedelta(seconds=30),
)
try:
observations = []
expected = []
for scenario, host, drained, now, first_freed in (
("ack", False, rank == 0, 10.0, [1]),
("deadline", False, False, 31.0 if rank == 0 else 29.0, [1]),
("mixed", False, rank == 0, 31.0, [0, 1]),
("host_ack", True, rank == 0, 10.0, [1]),
("host_timeout", True, False, 31.0, [1]),
):
mgr = _make_manager()
q = _make_queue()
q.enable_host_receive = host
dreq = _make_decode_req(42, 0, mgr)
dreq.host_staged = host
dreq.kv_receiver.is_abort_release_safe = lambda: drained
generation = mgr.register_deferred_abort_room(42)
if drained:
mgr.note_abort_ack(42, 0, generation)
# An independently ready entry must be released even while the
# first entry waits for a peer; agreement is per request.
ready_req = _make_decode_req(44, 1, mgr)
ready_req.host_staged = host
ready_req.kv_receiver.is_abort_release_safe = lambda: True
ready_generation = mgr.register_deferred_abort_room(44)
mgr.note_abort_ack(44, 0, ready_generation)
with patch.object(decode_mod.time, "monotonic", return_value=0.0):
q._defer_release(dreq)
q._defer_release(ready_req)
with (
patch.object(decode_mod.time, "monotonic", return_value=now) as clock,
patch.object(decode_mod, "release_kv_cache"),
patch.object(decode_mod, "discard_kv_cache_backup"),
):
q.resolve_deferred_releases()
observations.append(
(scenario, list(q.req_to_metadata_buffer_idx_allocator.freed))
)
expected.append((scenario, first_freed))
drained = True
if scenario != "deadline":
mgr.note_abort_ack(42, 0, generation)
clock.return_value = 31.0
q.resolve_deferred_releases()
q.resolve_deferred_releases()
observations.append(
(
scenario + "_drained",
list(q.req_to_metadata_buffer_idx_allocator.freed),
)
)
expected.append(
(scenario + "_drained", [0, 1] if scenario == "mixed" else [1, 0])
)
for notification in ("some", "all", "none", "disabled"):
mgr = _make_manager()
q = _make_queue()
q.enable_deferred_kv_release = notification != "disabled"
dreq = _make_decode_req(43, 1, mgr)
receiver = dreq.kv_receiver
receiver.abort_notified = notification == "all" or (
notification == "some" and rank == 0
)
if not receiver.abort_notified:
receiver.bootstrap_infos = None
generation = mgr.register_deferred_abort_room(43)
q.queue = [dreq]
with (
patch.object(decode_mod, "prepare_abort"),
patch.object(decode_mod, "release_kv_cache"),
):
q.pop_transferred()
held = len(q._deferred_releases)
observations.append((notification, held))
expected.append((notification, int(notification in ("some", "all"))))
held_by_rank = [None, None]
dist.all_gather_object(held_by_rank, held)
# Keep a broken implementation from deadlocking the test when
# just one rank enters the release collective.
if held_by_rank == [1, 1]:
mgr.note_abort_ack(43, 0, generation)
q.resolve_deferred_releases()
observations.append(
(
notification + "_freed",
list(q.req_to_metadata_buffer_idx_allocator.freed),
)
)
expected.append((notification + "_freed", [1]))
# Both ranks reach the same assertion even if the old code diverges.
results = [None, None]
dist.all_gather_object(results, observations)
assert results == [expected, expected], results
finally:
dist.destroy_process_group()
def _make_pop_queue(mgr):
"""DecodeTransferQueue shell wired for pop_transferred's Failed branch."""
@@ -918,6 +1064,12 @@ class TestFailedTransfersDeferOnEveryFailure(CustomTestCase):
to prevent. pop_transferred must notify the prefill ranks and defer on
every failure kind, not only when abort_notified is already set."""
def setUp(self):
super().setUp()
world_size = patch.object(dist, "get_world_size", return_value=1)
world_size.start()
self.addCleanup(world_size.stop)
def _pop(self, q):
released = []
q._release_request = released.append