[CI][PD] Pin nccl rendezvous port per side to fix flaky disaggregation tests (#34755)

This commit is contained in:
Shangming Cai
2026-08-14 01:44:20 +08:00
committed by GitHub
parent a82f8e1777
commit 0772e79ee7
2 changed files with 16 additions and 1 deletions
@@ -68,12 +68,19 @@ class PDDisaggregationServerBase(CustomTestCase):
cls.prefill_port = f"{int(base_port) + 100}"
cls.decode_port = f"{int(base_port) + 200}"
cls.bootstrap_port = f"{int(base_port) + 500}"
# Pin distinct nccl (torch.distributed rendezvous) ports below the
# ephemeral range; otherwise both sides derive them from get_free_port()
# and can race onto the same port, failing at init_process_group with
# EADDRINUSE.
cls.prefill_nccl_port = f"{int(base_port) + 300}"
cls.decode_nccl_port = f"{int(base_port) + 400}"
cls.prefill_url = f"http://{cls.base_host}:{cls.prefill_port}"
cls.decode_url = f"http://{cls.base_host}:{cls.decode_port}"
cls.lb_url = f"http://{cls.base_host}:{cls.lb_port}"
cls.base_url = cls.lb_url
print(
f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=} {cls.bootstrap_port=}"
f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=} "
f"{cls.bootstrap_port=} {cls.prefill_nccl_port=} {cls.decode_nccl_port=}"
)
cls.process_lb, cls.process_decode, cls.process_prefill = None, None, None
if cls.capture_per_side_logs:
@@ -116,6 +123,8 @@ class PDDisaggregationServerBase(CustomTestCase):
"prefill",
"--disaggregation-bootstrap-port",
cls.bootstrap_port,
"--nccl-port",
cls.prefill_nccl_port,
"--tp",
str(cls.prefill_tp_size),
] + list(cls.extra_prefill_args)
@@ -141,6 +150,8 @@ class PDDisaggregationServerBase(CustomTestCase):
"decode",
"--disaggregation-bootstrap-port",
cls.bootstrap_port,
"--nccl-port",
cls.decode_nccl_port,
"--tp",
str(cls.decode_tp_size),
"--base-gpu-id",
@@ -62,6 +62,8 @@ class TestDisaggregationDSV4(SpecDecodingMixin, PDDisaggregationServerBase, GSM8
"prefill",
"--disaggregation-bootstrap-port",
cls.bootstrap_port,
"--nccl-port",
cls.prefill_nccl_port,
"--tp",
4,
"--dp",
@@ -96,6 +98,8 @@ class TestDisaggregationDSV4(SpecDecodingMixin, PDDisaggregationServerBase, GSM8
"decode",
"--disaggregation-bootstrap-port",
cls.bootstrap_port,
"--nccl-port",
cls.decode_nccl_port,
"--tp",
4,
"--dp",