From b57b5e22867f110ff577a9da18a9ff7e6619100c Mon Sep 17 00:00:00 2001 From: paulzhang-tm Date: Wed, 21 Oct 2026 10:37:28 -0700 Subject: [PATCH] [Fix] disaggregation: Synchronize deferred decode KV release across TP ranks (#45453) --- python/sglang/srt/disaggregation/decode.py | 126 ++++++++------ .../test_decode_queue_cleanup.py | 9 + .../test_deferred_decode_kv_release.py | 158 +++++++++++++++++- 3 files changed, 237 insertions(+), 56 deletions(-) diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index fabdcfc1ac..2d417b5dae 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -2775,6 +2775,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): # Queue-removed but held for deferred release; excluded from the metadata # teardown below. deferred_indices = set() + failed: List[Tuple[int, DecodeRequest, bool]] = [] for i, (decode_req, poll) in enumerate(zip(self.queue, polls)): if rids_to_check is not None and decode_req.req.rid not in rids_to_check: continue @@ -2829,22 +2830,15 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): # already hold destinations; this path always reaches # _do_release, which clears the tracker. decode_req.kv_receiver.ensure_abort_notified(force_arm=True) - if requires_host_drain or ( - deferrable and decode_req.kv_receiver.abort_notified - ): - # Host pages always await a drain ack. A receiver that could - # not notify (metadata never published, so no prefill holds - # its destination info) has nothing in flight and releases - # immediately below. - self._defer_release(decode_req) - deferred_indices.add(i) - indices_to_remove.add(i) - else: - # release pre-allocated kv cache, but don't insert into the tree since it's failed - self._release_request(decode_req) - decode_req.kv_receiver.clear() - decode_req.kv_receiver = None - indices_to_remove.add(i) + failed.append( + ( + i, + decode_req, + requires_host_drain + or (deferrable and decode_req.kv_receiver.abort_notified), + ) + ) + indices_to_remove.add(i) self._record_transfer_failure(failure_reason) continue elif poll == KVPoll.Success: @@ -2890,6 +2884,20 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): else: raise ValueError(f"Unexpected poll case: {poll}") + # Notification state is rank-local. If any rank must hold a failed + # transfer, all must retain its capacity so admission stays identical. + held = self._all_reduce_release_flags( + [hold for _, _, hold in failed], torch.distributed.ReduceOp.MAX + ) + for (i, decode_req, _), hold in zip(failed, held): + if hold: + self._defer_release(decode_req) + deferred_indices.add(i) + else: + self._release_request(decode_req) + decode_req.kv_receiver.clear() + decode_req.kv_receiver = None + for i in indices_to_remove: if i in deferred_indices: # Held for deferred release; metadata buffer freed at resolve time. @@ -2918,9 +2926,11 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): deadline = start_time + self.deferred_kv_release_timeout # Host-capable receivers own their drain tracking. Device destinations # snapshot the number of prefill acknowledgements required by the manager. + # A rank that only holds for a peer has no local acknowledgements to await. required_acks = ( 0 - if self.enable_host_receive and decode_req.host_staged + if (self.enable_host_receive and decode_req.host_staged) + or not decode_req.kv_receiver.abort_notified else len(decode_req.kv_receiver.bootstrap_infos) ) self._deferred_releases.append( @@ -2961,52 +2971,62 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): outcome=outcome, ) + def _all_reduce_release_flags( + self, flags: List[bool], op: torch.distributed.ReduceOp + ) -> List[bool]: + # Failures come from reduced polls and deferred entries from agreed holds, + # so every rank supplies the same entries in the same order, even if empty. + if not flags or torch.distributed.get_world_size(self.gloo_group) == 1: + return flags + reduced = torch.tensor(flags, dtype=torch.int32, device="cpu") + torch.distributed.all_reduce(reduced, op=op, group=self.gloo_group) + return [bool(flag) for flag in reduced.tolist()] + def resolve_deferred_releases(self) -> None: """Release drained transfers; device destinations may also time out.""" if not self._deferred_releases: return - host_ready = torch.tensor( - [ - req.kv_receiver.is_abort_release_safe() - for req, _, _, _, _ in self._deferred_releases - if self.enable_host_receive and req.host_staged - ], - dtype=torch.int32, - ) - if host_ready.numel() and torch.distributed.get_world_size(self.gloo_group) > 1: - # Admission must see identical host capacity and page order on every rank. - torch.distributed.all_reduce( - host_ready, op=torch.distributed.ReduceOp.MIN, group=self.gloo_group - ) - host_ready = iter(host_ready.tolist()) now = time.monotonic() + drained = [ + decode_req.kv_receiver.is_abort_release_safe() + if self.enable_host_receive and decode_req.host_staged + else decode_req.kv_receiver.kv_mgr.is_abort_release_safe( + decode_req.req.bootstrap_room, required_acks + ) + for decode_req, _, _, _, required_acks in self._deferred_releases + ] + # Acks and deadlines are rank-local. Releasing only on some ranks would + # diverge admission and the sizes of subsequent polling collectives. + # Host pages still require a drain: a timeout cannot rule out remote writes. + ready = self._all_reduce_release_flags( + [ + is_drained + or ( + not (self.enable_host_receive and decode_req.host_staged) + and now >= deadline + ) + for (decode_req, _, deadline, _, _), is_drained in zip( + self._deferred_releases, drained + ) + ], + torch.distributed.ReduceOp.MIN, + ) still_held = [] to_release = [] - for ( - decode_req, - start_time, - deadline, - idx, - required_acks, - ) in self._deferred_releases: - room = decode_req.req.bootstrap_room - if self.enable_host_receive and decode_req.host_staged: - if next(host_ready): - to_release.append((decode_req, start_time, idx, room, True)) - else: - # A timeout cannot prove that a remote write has stopped. - still_held.append( - (decode_req, start_time, deadline, idx, required_acks) + for entry, is_ready, is_drained in zip(self._deferred_releases, ready, drained): + decode_req, start_time, _, idx, _ = entry + if is_ready: + to_release.append( + ( + decode_req, + start_time, + idx, + decode_req.req.bootstrap_room, + is_drained, ) - continue - kv_mgr = decode_req.kv_receiver.kv_mgr - drained = kv_mgr.is_abort_release_safe(room, required_acks) - if not drained and now < deadline: - still_held.append( - (decode_req, start_time, deadline, idx, required_acks) ) else: - to_release.append((decode_req, start_time, idx, room, drained)) + still_held.append(entry) # Commit the survivors before releasing so a _do_release exception can't # leave a released entry in the list (double-free / None receiver on retry). self._deferred_releases = still_held diff --git a/test/registered/unit/disaggregation/test_decode_queue_cleanup.py b/test/registered/unit/disaggregation/test_decode_queue_cleanup.py index a37ae2ad77..14f698df1c 100644 --- a/test/registered/unit/disaggregation/test_decode_queue_cleanup.py +++ b/test/registered/unit/disaggregation/test_decode_queue_cleanup.py @@ -45,6 +45,9 @@ class FakeReceiver: class TestDecodeQueueCleanup(CustomTestCase): def setUp(self): + world_size = patch("torch.distributed.get_world_size", return_value=1) + world_size.start() + self.addCleanup(world_size.stop) # The code under test reads its config from the bags. reset_context() self.addCleanup(reset_context) @@ -572,6 +575,12 @@ def _gated_decode_req(poll, restore_status): class TestDecodeHiCacheRestoreGate(CustomTestCase): + def setUp(self): + super().setUp() + world_size = patch("torch.distributed.get_world_size", return_value=1) + world_size.start() + self.addCleanup(world_size.stop) + def test_gate_folds_restore_outcome_into_terminal_poll_only(self): cases = [ (KVPoll.Success, HiCacheRestoreResult.READY, KVPoll.Success), diff --git a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py index 1c9673e4b6..e37e7d70fe 100644 --- a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py +++ b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py @@ -6,14 +6,21 @@ them immediately (which could let the still-in-flight write land on pages alread reused by another request). The pages are released once every prefill rank acks that its transfer drained (CommonKVManager.is_abort_release_safe). Device destinations may also release on timeout; host destinations require the ack. +All decode attention-TP ranks must agree on holding and releasing each request. """ +import datetime +import tempfile import threading import unittest +from pathlib import Path from types import SimpleNamespace from typing import NamedTuple from unittest.mock import MagicMock, call, patch +import torch.distributed as dist +import torch.multiprocessing as mp + from sglang.srt.disaggregation import decode as decode_mod from sglang.srt.disaggregation.base.conn import BaseKVManager, KVPoll from sglang.srt.disaggregation.common.conn import ( @@ -27,10 +34,11 @@ from sglang.srt.disaggregation.common.conn import ( CommonKVSender, ) from sglang.srt.disaggregation.decode import DecodeTransferQueue +from sglang.srt.disaggregation.decode_hicache_mixin import HiCacheRestoreResult from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase -register_cpu_ci(est_time=10, suite="base-a-test-cpu") +register_cpu_ci(est_time=45, suite="base-a-test-cpu") ABORT_GENERATION = 7 @@ -661,14 +669,21 @@ class _FakeIdxAllocator: def _make_queue(timeout=30.0, enable_metrics=False): q = DecodeTransferQueue.__new__(DecodeTransferQueue) q._deferred_releases = [] + q.queue = [] + q.gloo_group = dist.group.WORLD + q.tp_rank = 0 + q.enable_deferred_kv_release = True q.deferred_kv_release_timeout = timeout q.enable_host_receive = False q.enable_staging = False q.staging_handler = None - q.tree_cache = object() + q.tree_cache = MagicMock() q.metadata_buffers = SimpleNamespace(bootstrap_room={}) q.req_to_metadata_buffer_idx_allocator = _FakeIdxAllocator() q.scheduler = SimpleNamespace( + enable_decode_hicache=False, + enable_hisparse=False, + output_streamer=MagicMock(), metrics_reporter=SimpleNamespace(enable_metrics=enable_metrics), metrics_collector=SimpleNamespace( observe_decode_deferred_kv_release=MagicMock() @@ -680,6 +695,10 @@ def _make_queue(timeout=30.0, enable_metrics=False): def _make_decode_req(room, idx, mgr, n_prefill_ranks=1): receiver = SimpleNamespace( kv_mgr=mgr, + abort_notified=True, + ensure_abort_notified=MagicMock(), + failure_exception=lambda: None, + poll=lambda: KVPoll.Failed, # One entry per prefill rank the decode notified of the abort; its length # is the required drain-ack count (see DecodeTransferQueue._defer_release). bootstrap_infos=[{"rank": r} for r in range(n_prefill_ranks)], @@ -687,13 +706,23 @@ def _make_decode_req(room, idx, mgr, n_prefill_ranks=1): clear=lambda: None, ) return SimpleNamespace( - req=SimpleNamespace(bootstrap_room=room), + req=SimpleNamespace(bootstrap_room=room, rid=str(room), return_logprob=False), + host_staged=False, + hicache_restore_status=HiCacheRestoreResult.READY, + prefix_match=None, + hicache_restore_lock=None, kv_receiver=receiver, metadata_buffer_index=idx, ) class TestResolveDeferredReleases(CustomTestCase): + def setUp(self): + super().setUp() + world_size = patch.object(dist, "get_world_size", return_value=1) + world_size.start() + self.addCleanup(world_size.stop) + def test_host_release_requires_drain_on_every_rank_even_after_timeout(self): mgr = _make_manager() q = _make_queue(timeout=-1) @@ -866,6 +895,123 @@ class TestResolveDeferredReleases(CustomTestCase): self.assertEqual(held_idx, 9) self.assertEqual(required, 1) + def test_two_rank_release_consensus(self): + with tempfile.TemporaryDirectory() as tmp: + mp.spawn( + _run_release_consensus, + args=(str(Path(tmp) / "rendezvous"),), + nprocs=2, + join=True, + ) + + +def _run_release_consensus(rank: int, rendezvous: str) -> None: + dist.init_process_group( + "gloo", + init_method=f"file://{rendezvous}", + rank=rank, + world_size=2, + timeout=datetime.timedelta(seconds=30), + ) + try: + observations = [] + expected = [] + for scenario, host, drained, now, first_freed in ( + ("ack", False, rank == 0, 10.0, [1]), + ("deadline", False, False, 31.0 if rank == 0 else 29.0, [1]), + ("mixed", False, rank == 0, 31.0, [0, 1]), + ("host_ack", True, rank == 0, 10.0, [1]), + ("host_timeout", True, False, 31.0, [1]), + ): + mgr = _make_manager() + q = _make_queue() + q.enable_host_receive = host + dreq = _make_decode_req(42, 0, mgr) + dreq.host_staged = host + dreq.kv_receiver.is_abort_release_safe = lambda: drained + generation = mgr.register_deferred_abort_room(42) + if drained: + mgr.note_abort_ack(42, 0, generation) + # An independently ready entry must be released even while the + # first entry waits for a peer; agreement is per request. + ready_req = _make_decode_req(44, 1, mgr) + ready_req.host_staged = host + ready_req.kv_receiver.is_abort_release_safe = lambda: True + ready_generation = mgr.register_deferred_abort_room(44) + mgr.note_abort_ack(44, 0, ready_generation) + with patch.object(decode_mod.time, "monotonic", return_value=0.0): + q._defer_release(dreq) + q._defer_release(ready_req) + with ( + patch.object(decode_mod.time, "monotonic", return_value=now) as clock, + patch.object(decode_mod, "release_kv_cache"), + patch.object(decode_mod, "discard_kv_cache_backup"), + ): + q.resolve_deferred_releases() + observations.append( + (scenario, list(q.req_to_metadata_buffer_idx_allocator.freed)) + ) + expected.append((scenario, first_freed)) + + drained = True + if scenario != "deadline": + mgr.note_abort_ack(42, 0, generation) + clock.return_value = 31.0 + q.resolve_deferred_releases() + q.resolve_deferred_releases() + observations.append( + ( + scenario + "_drained", + list(q.req_to_metadata_buffer_idx_allocator.freed), + ) + ) + expected.append( + (scenario + "_drained", [0, 1] if scenario == "mixed" else [1, 0]) + ) + + for notification in ("some", "all", "none", "disabled"): + mgr = _make_manager() + q = _make_queue() + q.enable_deferred_kv_release = notification != "disabled" + dreq = _make_decode_req(43, 1, mgr) + receiver = dreq.kv_receiver + receiver.abort_notified = notification == "all" or ( + notification == "some" and rank == 0 + ) + if not receiver.abort_notified: + receiver.bootstrap_infos = None + generation = mgr.register_deferred_abort_room(43) + q.queue = [dreq] + with ( + patch.object(decode_mod, "prepare_abort"), + patch.object(decode_mod, "release_kv_cache"), + ): + q.pop_transferred() + held = len(q._deferred_releases) + observations.append((notification, held)) + expected.append((notification, int(notification in ("some", "all")))) + held_by_rank = [None, None] + dist.all_gather_object(held_by_rank, held) + # Keep a broken implementation from deadlocking the test when + # just one rank enters the release collective. + if held_by_rank == [1, 1]: + mgr.note_abort_ack(43, 0, generation) + q.resolve_deferred_releases() + observations.append( + ( + notification + "_freed", + list(q.req_to_metadata_buffer_idx_allocator.freed), + ) + ) + expected.append((notification + "_freed", [1])) + + # Both ranks reach the same assertion even if the old code diverges. + results = [None, None] + dist.all_gather_object(results, observations) + assert results == [expected, expected], results + finally: + dist.destroy_process_group() + def _make_pop_queue(mgr): """DecodeTransferQueue shell wired for pop_transferred's Failed branch.""" @@ -918,6 +1064,12 @@ class TestFailedTransfersDeferOnEveryFailure(CustomTestCase): to prevent. pop_transferred must notify the prefill ranks and defer on every failure kind, not only when abort_notified is already set.""" + def setUp(self): + super().setUp() + world_size = patch.object(dist, "get_world_size", return_value=1) + world_size.start() + self.addCleanup(world_size.stop) + def _pop(self, q): released = [] q._release_request = released.append