From c5a5bf0d0bd3ff2d5d08bda41aa9273feea98584 Mon Sep 17 00:00:00 2001 From: Tong Chen <96029131+Robin930@users.noreply.github.com> Date: Fri, 16 Oct 2026 01:46:37 +0800 Subject: [PATCH] [Fix] pp: Order output send/recv by rank parity to fix cross-node hang (#44453) Co-authored-by: chuixue Co-authored-by: Shangming Cai --- .../sglang/srt/managers/scheduler_pp_mixin.py | 32 +++--- .../managers/test_pp_output_exchange_order.py | 97 +++++++++++++++++++ 2 files changed, 109 insertions(+), 20 deletions(-) create mode 100644 test/registered/unit/managers/test_pp_output_exchange_order.py diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 2c5a3e2830..76f79fa679 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -35,7 +35,7 @@ from sglang.srt.sampling.sampling_observer_pp import ( pop_auxiliary_output_from_pp_tensors, ) from sglang.srt.utils import DynamicGradMode, point_to_point_pyobj -from sglang.srt.utils.common import is_npu, is_xpu +from sglang.srt.utils.common import is_npu _is_npu = is_npu() logger = logging.getLogger(__name__) @@ -1456,25 +1456,17 @@ class SchedulerPPMixin: pp_outputs, ) - # On CUDA, isend is async: it enqueues to the stream and returns, - # so every rank can send first safely. On some backends isend is - # effectively blocking and does not return until the peer posts a - # matching recv; if every PP rank sends first, all ranks block - # waiting for a receiver and the ring deadlocks. Order send/recv - # by pp_rank parity (even: send->recv, odd: recv->send) so each - # adjacent pair has one sender and one receiver posted at the - # same time. - - # CUDA: send first - # XPU: even ranks send first, odd ranks recv first. - # PP+spec also pairs by parity: its relay carries several extra GPU - # tensors (accept_lens, new_seq_lens, bonus tokens, next chain), and - # device-side P2P stays ordered on the stream, so enqueueing that many - # sends before any recv can form the same ring wait on CUDA. Parity - # makes rank 1 post its recv first, which breaks the cycle for any - # pp_size > 1. - needs_pairing = is_xpu() or self._pp_spec_relay - send_first = (not needs_pairing) or ((get_parallel().pp_rank % 2) == 0) + # Order send/recv by pp_rank parity (even: send->recv, odd: recv->send) + # on every backend. A device send only completes on its own while the + # peer's per-connection NCCL buffer has free slots (~8 ops over NVLink, + # ~2 over NET/IB, i.e. any cross-node PP); beyond that it waits for the + # peer's matching recv, and P2P ops are serialized on one stream per + # peer pair. With two microbatches in flight, if every rank sends first + # each rank's output sends queue ahead of the recv its peer's sends + # need, and the ring deadlocks. Parity gives each adjacent pair one + # rank that posts its recv first, which breaks the cycle for any + # pp_size > 1 (an odd ring still has at least one recv-first rank). + send_first = (get_parallel().pp_rank % 2) == 0 def _do_send(): return self._pp_send_output_to_next_stage( diff --git a/test/registered/unit/managers/test_pp_output_exchange_order.py b/test/registered/unit/managers/test_pp_output_exchange_order.py new file mode 100644 index 0000000000..1f6cd8d37f --- /dev/null +++ b/test/registered/unit/managers/test_pp_output_exchange_order.py @@ -0,0 +1,97 @@ +"""PP output exchange must not cross-deadlock with two microbatches in flight. + +Each scheduler loop iteration, every PP rank sends one microbatch's output +downstream (the last rank returns it to rank 0, earlier ranks relay it on) and +receives another microbatch's output from upstream. NCCL p2p ops between a peer +pair are serialized on one stream, and a device send only completes on its own +while the peer's connection buffer has free slots (about two ops over NET/IB, so +any cross-node PP); after that it waits for the peer's matching recv. If every +rank posts its send first, each rank's send sits ahead of the recv its peer's +send is waiting for, and both GPUs hang. Model the worst case, where every send +must rendezvous with the peer's recv at the head of its stream. +""" + +import unittest +from collections import deque +from contextlib import nullcontext +from types import SimpleNamespace + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import ( + CustomTestCase, + maybe_stub_sgl_kernel, + published_topology, +) + +maybe_stub_sgl_kernel() + +from sglang.srt.managers.scheduler_pp_mixin import SchedulerPPMixin # noqa: E402 +from sglang.srt.model_executor.forward_batch_info import ForwardMode # noqa: E402 + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _posted_output_ops(pp_rank, spec_relay, send_msg, recv_msg): + """Device p2p ops one PP2 rank posts for one output exchange, in stream order.""" + ops = [] + scheduler = SchedulerPPMixin() + scheduler._pp_spec_relay = spec_relay + scheduler.schedule_stream = None + scheduler.copy_stream = SimpleNamespace( + wait_stream=lambda stream: None, wait_event=lambda event: None + ) + scheduler.copy_stream_ctx = nullcontext() + scheduler.device_module = SimpleNamespace( + Event=lambda: SimpleNamespace(record=lambda stream: None), + current_stream=lambda: None, + ) + scheduler._pp_prep_batch_result = lambda *args: SimpleNamespace(logits_output=None) + + def send(*args): + ops.append(("send", send_msg)) + return [] + + def recv(): + ops.append(("recv", recv_msg)) + return {}, None + + scheduler._pp_send_output_to_next_stage = send + scheduler._pp_recv_dict_from_prev_stage = recv + target = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, + reqs=[object(), object()], + contains_last_prefill_chunk=True, + return_logprob=True, + ) + with published_topology(role="scheduler", ranks={"world_rank": pp_rank}, pp_size=2): + scheduler._pp_send_recv_and_preprocess_output_tensors( + 0, 1, [None, target], [None, None], deque(), None + ) + return ops + + +class TestPPOutputExchangeOrder(CustomTestCase): + def test_two_in_flight_output_exchange_does_not_cross_deadlock(self): + for spec_relay in (False, True): + with self.subTest(spec_relay=spec_relay): + # Rank 1 (last) returns output A to rank 0 while rank 0 relays + # output B (received last iteration) on to rank 1. + streams = [ + deque(_posted_output_ops(0, spec_relay, "relay-B", "output-A")), + deque(_posted_output_ops(1, spec_relay, "output-A", "relay-B")), + ] + self.assertEqual([len(s) for s in streams], [2, 2]) + while streams[0] or streams[1]: + heads = {streams[0][0], streams[1][0]} + kinds = sorted(kind for kind, _ in heads) + self.assertTrue( + kinds == ["recv", "send"] and len({m for _, m in heads}) == 1, + f"PP output exchange deadlocks: rank0={list(streams[0])}, " + f"rank1={list(streams[1])}", + ) + streams[0].popleft() + streams[1].popleft() + + +if __name__ == "__main__": + unittest.main()