fix(mooncake): bind IntraNode batches to the bootstrap CUDA device

Sender and executor threads can enter synchronous IntraNode transfers with
another current CUDA device. Keep the explicit bootstrap ordinal, check it
against the published model device, and scope the native batch call to that
device while restoring the calling thread on success or failure.

Ordinary transports, force-TCP and non-CUDA devices keep their existing
path. Missing or inconsistent CUDA identity is rejected before native IO.

Add 13 CPU regressions importing the production wrapper, including real
constructor calls with a mocked SDK, concurrent thread-local device states,
error restoration and transport/device compatibility. All 13 pass; two
selected tests fail on the unchanged wrapper at behavior assertions. Native
and CUDA boundaries are mocked, so this does not certify physical CUDA IPC
or attribute the separate container-PID collision to this change.
This commit is contained in:
2026-10-30 11:42:19 +08:00
parent 17505e6cf6
commit ee9170e7bb
2 changed files with 327 additions and 3 deletions
@@ -3,10 +3,12 @@ from __future__ import annotations
import json
import logging
import os
from contextlib import nullcontext
from typing import TYPE_CHECKING, Dict, List, Optional, Union
from sglang.srt.environ import envs
from sglang.srt.runtime_context import (
get_device,
get_disagg,
get_exec,
get_memory,
@@ -128,6 +130,8 @@ class MooncakeTransferEngine:
self.engine = TransferEngine()
self.hostname = hostname
self.gpu_id = gpu_id if gpu_id is not None else 0
# Keep bootstrap's explicit local ordinal, not a new thread's default.
self._pd_cuda_gpu_id = gpu_id
# MC_FORCE_TCP=1 makes mooncake install TcpTransport instead of RDMA,
# in which case RDMA HCA selection is irrelevant; pass empty device.
if os.environ.get("MC_FORCE_TCP") == "1":
@@ -243,6 +247,40 @@ class MooncakeTransferEngine:
return ret
def _batch_cuda_device_context(self):
# Force-TCP initialization bypasses all GPU transports in this SDK.
if "MC_FORCE_TCP" in os.environ:
return nullcontext()
protocol = (os.environ.get("MOONCAKE_PROTOCOL") or "").strip().lower()
intra = "MC_INTRANODE_NVLINK" in os.environ or protocol in (
"nvlink_intra",
"nvlink-intra",
"intra_nvlink",
)
if not intra:
return nullcontext()
# CUDA device selection is thread-local. Sender and executor threads
# must use the model/bootstrap device, not their initial current_device.
try:
declared = get_device()
device_type = declared.device
except (ValueError, AttributeError) as exc:
raise RuntimeError("NVLink PD requires a published model device") from exc
if device_type in ("cpu", "npu", "xpu", "hpu", "musa", "mlu", "mps"):
return nullcontext()
gpu_id = self._pd_cuda_gpu_id
if (
device_type != "cuda"
or type(gpu_id) is not int
or gpu_id < 0
or self.gpu_id != gpu_id
or declared.gpu_id != gpu_id
):
raise RuntimeError("NVLink PD requires the explicit bootstrap CUDA gpu_id")
import torch
return torch.cuda.device(gpu_id)
def batch_transfer_sync(
self,
session_id: str,
@@ -251,10 +289,12 @@ class MooncakeTransferEngine:
lengths: List[int],
) -> int:
"""Synchronously transfer data to the specified addresses in batches."""
device_context = self._batch_cuda_device_context()
try:
ret = self.engine.batch_transfer_sync_write(
session_id, buffers, peer_buffer_addresses, lengths
)
with device_context:
ret = self.engine.batch_transfer_sync_write(
session_id, buffers, peer_buffer_addresses, lengths
)
except Exception:
ret = -1
if not hasattr(self.engine, "batch_transfer_sync_write"):
@@ -0,0 +1,284 @@
"""CPU regressions for the device context of an IntraNode NVLink sender."""
import os
import sys
import threading
import unittest
from concurrent.futures import ThreadPoolExecutor
from contextlib import ExitStack, contextmanager
from types import ModuleType, SimpleNamespace
from unittest.mock import patch
import torch
from sglang.srt.distributed.device_communicators import mooncake_transfer_engine
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
class _NativeEngine:
"""Only the native boundary is fake; the production wrapper runs unchanged."""
def __init__(self, devices):
self.devices = devices
self.calls = []
self.call_lock = threading.Lock()
self.result = 0
self.error = None
def initialize(self, *args):
return 0
def get_rpc_port(self):
return 12345
def batch_transfer_sync_write(self, *args):
with self.call_lock:
self.calls.append((threading.get_ident(), self.devices.current, args))
if self.error is not None:
raise self.error
return self.result
class TestMooncakeTransferDeviceContext(unittest.TestCase):
def setUp(self):
self.devices = threading.local()
self.devices.current = 0
self.native = _NativeEngine(self.devices)
self.model_device = SimpleNamespace(device="cuda", gpu_id=1)
self.patches = ExitStack()
self.addCleanup(self.patches.close)
self.device_snapshot = self.patches.enter_context(
patch.object(
mooncake_transfer_engine,
"get_device",
side_effect=lambda: self.model_device,
create=True,
)
)
self.cuda_context = self.patches.enter_context(
patch.object(torch.cuda, "device", side_effect=self._device_context)
)
self.patches.enter_context(
patch.object(
mooncake_transfer_engine.envs.ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE,
"get",
return_value=False,
)
)
@contextmanager
def _device_context(self, device):
previous = self.devices.current
self.devices.current = device
try:
yield
finally:
self.devices.current = previous
@contextmanager
def _environment(self, **values):
with patch.dict(os.environ):
for name in ("MC_FORCE_TCP", "MC_INTRANODE_NVLINK", "MOONCAKE_PROTOCOL"):
os.environ.pop(name, None)
os.environ.update(values)
yield
def _engine(self, gpu_id=1):
package = ModuleType("mooncake")
native_module = ModuleType("mooncake.engine")
native_module.TransferEngine = lambda: self.native
package.engine = native_module
with patch.dict(
sys.modules, {"mooncake": package, "mooncake.engine": native_module}
):
return mooncake_transfer_engine.MooncakeTransferEngine(
"127.0.0.1", gpu_id=gpu_id
)
@staticmethod
def _transfer(engine):
return engine.batch_transfer_sync("peer", [10, 20], [30, 40], [50, 60])
def test_each_batch_uses_bootstrap_device_and_restores_caller(self):
with self._environment(MC_INTRANODE_NVLINK="true"):
engine = self._engine()
for initial_device in (0, 3):
self.devices.current = initial_device
self.assertEqual(self._transfer(engine), 0)
self.assertEqual(self.devices.current, initial_device)
self.assertEqual([call[1] for call in self.native.calls], [1, 1])
self.assertTrue(
all(
call[2] == ("peer", [10, 20], [30, 40], [50, 60])
for call in self.native.calls
)
)
def test_sender_threads_restore_their_distinct_initial_devices(self):
barrier = threading.Barrier(3)
with self._environment(MC_INTRANODE_NVLINK="true"):
engine = self._engine()
def send(initial_device):
self.devices.current = initial_device
barrier.wait(timeout=5)
for _ in range(5):
self.assertEqual(self._transfer(engine), 0)
self.assertEqual(self.devices.current, initial_device)
return threading.get_ident()
with ThreadPoolExecutor(max_workers=3) as executor:
threads = list(executor.map(send, (0, 2, 3)))
self.assertEqual(len(set(threads)), 3)
self.assertEqual(len(self.native.calls), 15)
self.assertTrue(all(call[1] == 1 for call in self.native.calls))
def test_native_exception_restores_device_and_preserves_error_return(self):
with self._environment(MC_INTRANODE_NVLINK="true"):
engine = self._engine()
self.native.error = RuntimeError("native copy failed")
self.devices.current = 3
self.assertEqual(self._transfer(engine), -1)
self.assertEqual(self.devices.current, 3)
self.assertEqual(self.native.calls[-1][1], 1)
def test_negative_native_result_is_preserved(self):
with self._environment(MC_INTRANODE_NVLINK="true"):
engine = self._engine()
self.native.result = -7
self.devices.current = 2
self.assertEqual(self._transfer(engine), -7)
self.assertEqual(self.devices.current, 2)
def test_missing_native_api_still_reports_upgrade_after_context_exit(self):
with self._environment(MC_INTRANODE_NVLINK="true"):
engine = self._engine()
engine.engine = object()
self.devices.current = 3
with self.assertRaisesRegex(RuntimeError, "batch transfer requires"):
self._transfer(engine)
self.assertEqual(self.devices.current, 3)
def test_invalid_bootstrap_device_fails_before_native_call(self):
with self._environment(MC_INTRANODE_NVLINK="true"):
for gpu_id in (None, True, False, -1, "1"):
with self.subTest(gpu_id=gpu_id):
engine = self._engine(gpu_id)
with self.assertRaisesRegex(RuntimeError, "explicit bootstrap"):
self._transfer(engine)
self.assertEqual(self.native.calls, [])
self.cuda_context.assert_not_called()
def test_constructor_fallback_does_not_invent_an_explicit_device(self):
with self._environment(MC_INTRANODE_NVLINK="true"):
engine = self._engine(None)
self.model_device = SimpleNamespace(device="cuda", gpu_id=0)
self.assertEqual(engine.gpu_id, 0)
with self.assertRaisesRegex(RuntimeError, "explicit bootstrap"):
self._transfer(engine)
self.assertEqual(self.native.calls, [])
def test_mismatched_wrapper_or_published_device_fails_before_native(self):
with self._environment(MC_INTRANODE_NVLINK="true"):
for wrapper_device, published_device in ((0, 1), (1, 0)):
with self.subTest(
wrapper_device=wrapper_device, published_device=published_device
):
engine = self._engine()
engine.gpu_id = wrapper_device
self.model_device = SimpleNamespace(
device="cuda", gpu_id=published_device
)
with self.assertRaisesRegex(RuntimeError, "explicit bootstrap"):
self._transfer(engine)
self.assertEqual(self.native.calls, [])
self.cuda_context.assert_not_called()
def test_missing_or_unknown_runtime_device_fails_before_native(self):
with self._environment(MC_INTRANODE_NVLINK="true"):
engine = self._engine()
for error in (ValueError("not published"), AttributeError("no device")):
with self.subTest(error=type(error).__name__):
self.device_snapshot.side_effect = error
with self.assertRaisesRegex(RuntimeError, "published model device"):
self._transfer(engine)
self.device_snapshot.side_effect = lambda: self.model_device
self.model_device = SimpleNamespace(device=None, gpu_id=1)
with self.assertRaisesRegex(RuntimeError, "explicit bootstrap"):
self._transfer(engine)
self.assertEqual(self.native.calls, [])
self.cuda_context.assert_not_called()
def test_non_cuda_devices_keep_original_native_path(self):
with self._environment(MC_INTRANODE_NVLINK="true"):
engine = self._engine(None)
for device in ("cpu", "npu", "xpu", "hpu", "musa", "mlu", "mps"):
with self.subTest(device=device):
self.model_device = SimpleNamespace(device=device, gpu_id=None)
self.devices.current = 7
self.assertEqual(self._transfer(engine), 0)
self.assertEqual(self.devices.current, 7)
self.assertEqual(len(self.native.calls), 7)
self.assertTrue(all(call[1] == 7 for call in self.native.calls))
self.cuda_context.assert_not_called()
def test_ordinary_transports_do_not_require_cuda_device_snapshot(self):
for protocol in (None, "rdma", "tcp", "efa"):
with self.subTest(protocol=protocol):
values = {} if protocol is None else {"MOONCAKE_PROTOCOL": protocol}
with self._environment(**values):
engine = self._engine(None)
self.device_snapshot.side_effect = AssertionError(
"ordinary transport must not query the model device"
)
self.devices.current = 4
self.assertEqual(self._transfer(engine), 0)
self.assertEqual(self.devices.current, 4)
self.assertEqual(len(self.native.calls), 4)
self.device_snapshot.assert_not_called()
self.cuda_context.assert_not_called()
def test_force_tcp_presence_overrides_intra_nvlink_selection(self):
for value in ("", "0", "1"):
with self.subTest(value=value):
with self._environment(MC_FORCE_TCP=value, MC_INTRANODE_NVLINK="true"):
engine = self._engine(None)
self.devices.current = 4
self.assertEqual(self._transfer(engine), 0)
self.assertEqual(self.devices.current, 4)
self.assertEqual(len(self.native.calls), 3)
self.device_snapshot.assert_not_called()
self.cuda_context.assert_not_called()
def test_intra_nvlink_presence_and_protocol_aliases_select_context(self):
selections = (
{"MC_INTRANODE_NVLINK": ""},
{"MOONCAKE_PROTOCOL": "nvlink_intra"},
{"MOONCAKE_PROTOCOL": "nvlink-intra"},
{"MOONCAKE_PROTOCOL": "intra_nvlink"},
)
for selection in selections:
with self.subTest(selection=selection):
with self._environment(**selection):
engine = self._engine()
self.devices.current = 3
self.assertEqual(self._transfer(engine), 0)
self.assertEqual(self.devices.current, 3)
self.assertEqual(len(self.native.calls), 4)
self.assertTrue(all(call[1] == 1 for call in self.native.calls))
if __name__ == "__main__":
unittest.main()