From a6a6beffac6fd1dc17516dcbab7ee2e17a81838e Mon Sep 17 00:00:00 2001 From: Shangming Cai Date: Thu, 22 Oct 2026 18:42:55 +1300 Subject: [PATCH] [Refactor] disaggregation: Share the flag all-reduce and host-destination predicate in the decode transfer queue (#45641) --- python/sglang/srt/disaggregation/decode.py | 72 ++++++++----------- python/sglang/srt/disaggregation/utils.py | 20 ++++-- .../test_decode_queue_cleanup.py | 16 +++-- .../test_deferred_decode_kv_release.py | 16 +++-- 4 files changed, 65 insertions(+), 59 deletions(-) diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 63450899ff..0ff153d571 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -57,6 +57,7 @@ from sglang.srt.disaggregation.utils import ( ReqToMetadataIdxAllocator, TransferBackend, _is_fake_transfer, + all_reduce_flags, build_kv_layer_ids, build_staging_slot_metadata, get_dsa_tail_state_indices, @@ -2747,12 +2748,15 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): return release_kv_cache(decode_req.req, self.tree_cache, checkpoint=False) + def _is_host_destination(self, decode_req: DecodeRequest) -> bool: + return self.enable_host_receive and decode_req.host_staged + def _record_transfer_failure(self, failure_reason: str) -> None: if self.scheduler.metrics_reporter.enable_metrics: self.scheduler.metrics_collector.increment_transfer_failed_reqs() def _abort_failed_transfer(self, decode_req: DecodeRequest) -> None: - if self.enable_host_receive and decode_req.host_staged: + if self._is_host_destination(decode_req): decode_req.kv_receiver.abort() def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]: @@ -2816,9 +2820,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): [decode_req.req], decode_req.req.return_logprob, ) - requires_host_drain = ( - self.enable_host_receive and decode_req.host_staged - ) + requires_host_drain = self._is_host_destination(decode_req) deferrable = ( self.enable_deferred_kv_release and decode_req.kv_receiver.kv_mgr.enable_deferred_decode_kv_release @@ -2843,7 +2845,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): self._record_transfer_failure(failure_reason) continue elif poll == KVPoll.Success: - if self.enable_host_receive and decode_req.host_staged: + if self._is_host_destination(decode_req): if isinstance(decode_req.req.finished_reason, FINISH_ABORT): self.scheduler.output_streamer.stream_output( [decode_req.req], decode_req.req.return_logprob @@ -2883,8 +2885,10 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): # Notification state is rank-local. If any rank must hold a failed # transfer, all must retain its capacity so admission stays identical. - held = self._all_reduce_release_flags( - [hold for _, _, hold in failed], torch.distributed.ReduceOp.MAX + held = all_reduce_flags( + flags=[hold for _, _, hold in failed], + group=self.gloo_group, + op=torch.distributed.ReduceOp.MAX, ) for (i, decode_req, _), hold in zip(failed, held): if hold: @@ -2926,7 +2930,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): # A rank that only holds for a peer has no local acknowledgements to await. required_acks = ( 0 - if (self.enable_host_receive and decode_req.host_staged) + if self._is_host_destination(decode_req) or not decode_req.kv_receiver.abort_notified else len(decode_req.kv_receiver.bootstrap_infos) ) @@ -2948,7 +2952,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): self._release_request(decode_req) self.metadata_buffers.bootstrap_room[idx] = 0 self.req_to_metadata_buffer_idx_allocator.free(idx) - if not (self.enable_host_receive and decode_req.host_staged): + if not self._is_host_destination(decode_req): decode_req.kv_receiver.kv_mgr.clear_deferred_abort_state(room) decode_req.kv_receiver.clear() decode_req.kv_receiver = None @@ -2968,45 +2972,31 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): outcome=outcome, ) - def _all_reduce_release_flags( - self, flags: List[bool], op: torch.distributed.ReduceOp - ) -> List[bool]: - # Failures come from reduced polls and deferred entries from agreed holds, - # so every rank supplies the same entries in the same order, even if empty. - if not flags or torch.distributed.get_world_size(self.gloo_group) == 1: - return flags - reduced = torch.tensor(flags, dtype=torch.int32, device="cpu") - torch.distributed.all_reduce(reduced, op=op, group=self.gloo_group) - return [bool(flag) for flag in reduced.tolist()] - def resolve_deferred_releases(self) -> None: """Release drained transfers; device destinations may also time out.""" if not self._deferred_releases: return now = time.monotonic() - drained = [ - decode_req.kv_receiver.is_abort_release_safe() - if self.enable_host_receive and decode_req.host_staged - else decode_req.kv_receiver.kv_mgr.is_abort_release_safe( - decode_req.req.bootstrap_room, required_acks - ) - for decode_req, _, _, _, required_acks in self._deferred_releases - ] + drained = [] + ready = [] + for decode_req, _, deadline, _, required_acks in self._deferred_releases: + if self._is_host_destination(decode_req): + # A timeout cannot prove that a remote write has stopped. + is_drained = decode_req.kv_receiver.is_abort_release_safe() + is_ready = is_drained + else: + is_drained = decode_req.kv_receiver.kv_mgr.is_abort_release_safe( + decode_req.req.bootstrap_room, required_acks + ) + is_ready = is_drained or now >= deadline + drained.append(is_drained) + ready.append(is_ready) # Acks and deadlines are rank-local. Releasing only on some ranks would # diverge admission and the sizes of subsequent polling collectives. - # Host pages still require a drain: a timeout cannot rule out remote writes. - ready = self._all_reduce_release_flags( - [ - is_drained - or ( - not (self.enable_host_receive and decode_req.host_staged) - and now >= deadline - ) - for (decode_req, _, deadline, _, _), is_drained in zip( - self._deferred_releases, drained - ) - ], - torch.distributed.ReduceOp.MIN, + # Entries are appended from agreed holds, so every rank reduces the same + # list in the same order. + ready = all_reduce_flags( + flags=ready, group=self.gloo_group, op=torch.distributed.ReduceOp.MIN ) still_held = [] to_release = [] diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index 44a4f111c2..8fe622b750 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -259,14 +259,22 @@ def _apply_metadata_gate(polls, decode_reqs, metadata_buffers) -> None: polls[i] = int(KVPoll.Transferring) +def all_reduce_flags( + flags: List[int], group: dist.ProcessGroup, op: dist.ReduceOp +) -> List[int]: + """Reduce one small int per entry across `group`; every rank must pass the + same number of entries in the same order. Empty or single-rank: unchanged.""" + if not flags or dist.get_world_size(group) == 1: + return flags + + tensor_to_reduce = torch.tensor(flags, dtype=torch.int32, device="cpu") + dist.all_reduce(tensor_to_reduce, op=op, group=group) + return tensor_to_reduce.tolist() + + def _all_reduce_polls(polls: List[int], group: dist.ProcessGroup) -> List[int]: """MIN-reduce poll states so no rank commits ahead of its peers.""" - if dist.get_world_size(group) == 1: - return polls - - tensor_to_reduce = torch.tensor(polls, dtype=torch.uint8, device="cpu") - dist.all_reduce(tensor_to_reduce, op=dist.ReduceOp.MIN, group=group) - return tensor_to_reduce.tolist() + return all_reduce_flags(flags=polls, group=group, op=dist.ReduceOp.MIN) def poll_and_all_reduce( diff --git a/test/registered/unit/disaggregation/test_decode_queue_cleanup.py b/test/registered/unit/disaggregation/test_decode_queue_cleanup.py index 01be6958e1..401790583f 100644 --- a/test/registered/unit/disaggregation/test_decode_queue_cleanup.py +++ b/test/registered/unit/disaggregation/test_decode_queue_cleanup.py @@ -21,11 +21,17 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=4, suite="base-a-test-cpu") +def _run_as_single_rank(test: unittest.TestCase) -> None: + # The transfer queue sizes its release collectives from the gloo group; + # these tests have no process group, so answer as one rank. + world_size = patch("torch.distributed.get_world_size", return_value=1) + world_size.start() + test.addCleanup(world_size.stop) + + class TestDecodeQueueCleanup(CustomTestCase): def setUp(self): - world_size = patch("torch.distributed.get_world_size", return_value=1) - world_size.start() - self.addCleanup(world_size.stop) + _run_as_single_rank(self) # The code under test reads its config from the bags. reset_context() self.addCleanup(reset_context) @@ -125,9 +131,7 @@ def _gated_decode_req(poll, restore_status): class TestDecodeHiCacheRestoreGate(CustomTestCase): def setUp(self): super().setUp() - world_size = patch("torch.distributed.get_world_size", return_value=1) - world_size.start() - self.addCleanup(world_size.stop) + _run_as_single_rank(self) def test_gate_folds_restore_outcome_into_terminal_poll_only(self): cases = [ diff --git a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py index 133906bc6a..5e914e68ff 100644 --- a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py +++ b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py @@ -528,12 +528,18 @@ def _make_decode_req(room, idx, mgr, n_prefill_ranks=1): ) +def _run_as_single_rank(test: unittest.TestCase) -> None: + # The transfer queue sizes its release collectives from the gloo group; + # these tests have no process group, so answer as one rank. + world_size = patch.object(dist, "get_world_size", return_value=1) + world_size.start() + test.addCleanup(world_size.stop) + + class TestResolveDeferredReleases(CustomTestCase): def setUp(self): super().setUp() - world_size = patch.object(dist, "get_world_size", return_value=1) - world_size.start() - self.addCleanup(world_size.stop) + _run_as_single_rank(self) def test_host_release_requires_drain_on_every_rank_even_after_timeout(self): mgr = _make_manager() @@ -855,9 +861,7 @@ class TestFailedTransfersDeferOnEveryFailure(CustomTestCase): def setUp(self): super().setUp() - world_size = patch.object(dist, "get_world_size", return_value=1) - world_size.start() - self.addCleanup(world_size.stop) + _run_as_single_rank(self) def _pop(self, q): released = []