[Fix] pp: Order output send/recv by rank parity to fix cross-node hang (#44453)

Co-authored-by: chuixue <chuixue@minimaxi.com>
Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
Tong Chen
2026-10-16 01:46:37 +08:00
committed by GitHub
co-authored by chuixue Shangming Cai
parent c7fb57d990
commit c5a5bf0d0b
2 changed files with 109 additions and 20 deletions
@@ -35,7 +35,7 @@ from sglang.srt.sampling.sampling_observer_pp import (
pop_auxiliary_output_from_pp_tensors,
)
from sglang.srt.utils import DynamicGradMode, point_to_point_pyobj
from sglang.srt.utils.common import is_npu, is_xpu
from sglang.srt.utils.common import is_npu
_is_npu = is_npu()
logger = logging.getLogger(__name__)
@@ -1456,25 +1456,17 @@ class SchedulerPPMixin:
pp_outputs,
)
# On CUDA, isend is async: it enqueues to the stream and returns,
# so every rank can send first safely. On some backends isend is
# effectively blocking and does not return until the peer posts a
# matching recv; if every PP rank sends first, all ranks block
# waiting for a receiver and the ring deadlocks. Order send/recv
# by pp_rank parity (even: send->recv, odd: recv->send) so each
# adjacent pair has one sender and one receiver posted at the
# same time.
# CUDA: send first
# XPU: even ranks send first, odd ranks recv first.
# PP+spec also pairs by parity: its relay carries several extra GPU
# tensors (accept_lens, new_seq_lens, bonus tokens, next chain), and
# device-side P2P stays ordered on the stream, so enqueueing that many
# sends before any recv can form the same ring wait on CUDA. Parity
# makes rank 1 post its recv first, which breaks the cycle for any
# pp_size > 1.
needs_pairing = is_xpu() or self._pp_spec_relay
send_first = (not needs_pairing) or ((get_parallel().pp_rank % 2) == 0)
# Order send/recv by pp_rank parity (even: send->recv, odd: recv->send)
# on every backend. A device send only completes on its own while the
# peer's per-connection NCCL buffer has free slots (~8 ops over NVLink,
# ~2 over NET/IB, i.e. any cross-node PP); beyond that it waits for the
# peer's matching recv, and P2P ops are serialized on one stream per
# peer pair. With two microbatches in flight, if every rank sends first
# each rank's output sends queue ahead of the recv its peer's sends
# need, and the ring deadlocks. Parity gives each adjacent pair one
# rank that posts its recv first, which breaks the cycle for any
# pp_size > 1 (an odd ring still has at least one recv-first rank).
send_first = (get_parallel().pp_rank % 2) == 0
def _do_send():
return self._pp_send_output_to_next_stage(
@@ -0,0 +1,97 @@
"""PP output exchange must not cross-deadlock with two microbatches in flight.
Each scheduler loop iteration, every PP rank sends one microbatch's output
downstream (the last rank returns it to rank 0, earlier ranks relay it on) and
receives another microbatch's output from upstream. NCCL p2p ops between a peer
pair are serialized on one stream, and a device send only completes on its own
while the peer's connection buffer has free slots (about two ops over NET/IB, so
any cross-node PP); after that it waits for the peer's matching recv. If every
rank posts its send first, each rank's send sits ahead of the recv its peer's
send is waiting for, and both GPUs hang. Model the worst case, where every send
must rendezvous with the peer's recv at the head of its stream.
"""
import unittest
from collections import deque
from contextlib import nullcontext
from types import SimpleNamespace
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import (
CustomTestCase,
maybe_stub_sgl_kernel,
published_topology,
)
maybe_stub_sgl_kernel()
from sglang.srt.managers.scheduler_pp_mixin import SchedulerPPMixin # noqa: E402
from sglang.srt.model_executor.forward_batch_info import ForwardMode # noqa: E402
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
def _posted_output_ops(pp_rank, spec_relay, send_msg, recv_msg):
"""Device p2p ops one PP2 rank posts for one output exchange, in stream order."""
ops = []
scheduler = SchedulerPPMixin()
scheduler._pp_spec_relay = spec_relay
scheduler.schedule_stream = None
scheduler.copy_stream = SimpleNamespace(
wait_stream=lambda stream: None, wait_event=lambda event: None
)
scheduler.copy_stream_ctx = nullcontext()
scheduler.device_module = SimpleNamespace(
Event=lambda: SimpleNamespace(record=lambda stream: None),
current_stream=lambda: None,
)
scheduler._pp_prep_batch_result = lambda *args: SimpleNamespace(logits_output=None)
def send(*args):
ops.append(("send", send_msg))
return []
def recv():
ops.append(("recv", recv_msg))
return {}, None
scheduler._pp_send_output_to_next_stage = send
scheduler._pp_recv_dict_from_prev_stage = recv
target = SimpleNamespace(
forward_mode=ForwardMode.EXTEND,
reqs=[object(), object()],
contains_last_prefill_chunk=True,
return_logprob=True,
)
with published_topology(role="scheduler", ranks={"world_rank": pp_rank}, pp_size=2):
scheduler._pp_send_recv_and_preprocess_output_tensors(
0, 1, [None, target], [None, None], deque(), None
)
return ops
class TestPPOutputExchangeOrder(CustomTestCase):
def test_two_in_flight_output_exchange_does_not_cross_deadlock(self):
for spec_relay in (False, True):
with self.subTest(spec_relay=spec_relay):
# Rank 1 (last) returns output A to rank 0 while rank 0 relays
# output B (received last iteration) on to rank 1.
streams = [
deque(_posted_output_ops(0, spec_relay, "relay-B", "output-A")),
deque(_posted_output_ops(1, spec_relay, "output-A", "relay-B")),
]
self.assertEqual([len(s) for s in streams], [2, 2])
while streams[0] or streams[1]:
heads = {streams[0][0], streams[1][0]}
kinds = sorted(kind for kind, _ in heads)
self.assertTrue(
kinds == ["recv", "send"] and len({m for _, m in heads}) == 1,
f"PP output exchange deadlocks: rank0={list(streams[0])}, "
f"rank1={list(streams[1])}",
)
streams[0].popleft()
streams[1].popleft()
if __name__ == "__main__":
unittest.main()