diff --git a/python/sglang/srt/distributed/device_communicators/mooncake_transfer_engine.py b/python/sglang/srt/distributed/device_communicators/mooncake_transfer_engine.py index 9ae94543a5..6d2a884a77 100644 --- a/python/sglang/srt/distributed/device_communicators/mooncake_transfer_engine.py +++ b/python/sglang/srt/distributed/device_communicators/mooncake_transfer_engine.py @@ -3,10 +3,12 @@ from __future__ import annotations import json import logging import os +from contextlib import nullcontext from typing import TYPE_CHECKING, Dict, List, Optional, Union from sglang.srt.environ import envs from sglang.srt.runtime_context import ( + get_device, get_disagg, get_exec, get_memory, @@ -128,6 +130,8 @@ class MooncakeTransferEngine: self.engine = TransferEngine() self.hostname = hostname self.gpu_id = gpu_id if gpu_id is not None else 0 + # Keep bootstrap's explicit local ordinal, not a new thread's default. + self._pd_cuda_gpu_id = gpu_id # MC_FORCE_TCP=1 makes mooncake install TcpTransport instead of RDMA, # in which case RDMA HCA selection is irrelevant; pass empty device. if os.environ.get("MC_FORCE_TCP") == "1": @@ -243,6 +247,40 @@ class MooncakeTransferEngine: return ret + def _batch_cuda_device_context(self): + # Force-TCP initialization bypasses all GPU transports in this SDK. + if "MC_FORCE_TCP" in os.environ: + return nullcontext() + protocol = (os.environ.get("MOONCAKE_PROTOCOL") or "").strip().lower() + intra = "MC_INTRANODE_NVLINK" in os.environ or protocol in ( + "nvlink_intra", + "nvlink-intra", + "intra_nvlink", + ) + if not intra: + return nullcontext() + # CUDA device selection is thread-local. Sender and executor threads + # must use the model/bootstrap device, not their initial current_device. + try: + declared = get_device() + device_type = declared.device + except (ValueError, AttributeError) as exc: + raise RuntimeError("NVLink PD requires a published model device") from exc + if device_type in ("cpu", "npu", "xpu", "hpu", "musa", "mlu", "mps"): + return nullcontext() + gpu_id = self._pd_cuda_gpu_id + if ( + device_type != "cuda" + or type(gpu_id) is not int + or gpu_id < 0 + or self.gpu_id != gpu_id + or declared.gpu_id != gpu_id + ): + raise RuntimeError("NVLink PD requires the explicit bootstrap CUDA gpu_id") + import torch + + return torch.cuda.device(gpu_id) + def batch_transfer_sync( self, session_id: str, @@ -251,10 +289,12 @@ class MooncakeTransferEngine: lengths: List[int], ) -> int: """Synchronously transfer data to the specified addresses in batches.""" + device_context = self._batch_cuda_device_context() try: - ret = self.engine.batch_transfer_sync_write( - session_id, buffers, peer_buffer_addresses, lengths - ) + with device_context: + ret = self.engine.batch_transfer_sync_write( + session_id, buffers, peer_buffer_addresses, lengths + ) except Exception: ret = -1 if not hasattr(self.engine, "batch_transfer_sync_write"): diff --git a/test/registered/unit/disaggregation/test_mooncake_transfer_device_context.py b/test/registered/unit/disaggregation/test_mooncake_transfer_device_context.py new file mode 100644 index 0000000000..11081da648 --- /dev/null +++ b/test/registered/unit/disaggregation/test_mooncake_transfer_device_context.py @@ -0,0 +1,284 @@ +"""CPU regressions for the device context of an IntraNode NVLink sender.""" + +import os +import sys +import threading +import unittest +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack, contextmanager +from types import ModuleType, SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.distributed.device_communicators import mooncake_transfer_engine +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + + +class _NativeEngine: + """Only the native boundary is fake; the production wrapper runs unchanged.""" + + def __init__(self, devices): + self.devices = devices + self.calls = [] + self.call_lock = threading.Lock() + self.result = 0 + self.error = None + + def initialize(self, *args): + return 0 + + def get_rpc_port(self): + return 12345 + + def batch_transfer_sync_write(self, *args): + with self.call_lock: + self.calls.append((threading.get_ident(), self.devices.current, args)) + if self.error is not None: + raise self.error + return self.result + + +class TestMooncakeTransferDeviceContext(unittest.TestCase): + def setUp(self): + self.devices = threading.local() + self.devices.current = 0 + self.native = _NativeEngine(self.devices) + self.model_device = SimpleNamespace(device="cuda", gpu_id=1) + self.patches = ExitStack() + self.addCleanup(self.patches.close) + self.device_snapshot = self.patches.enter_context( + patch.object( + mooncake_transfer_engine, + "get_device", + side_effect=lambda: self.model_device, + create=True, + ) + ) + self.cuda_context = self.patches.enter_context( + patch.object(torch.cuda, "device", side_effect=self._device_context) + ) + self.patches.enter_context( + patch.object( + mooncake_transfer_engine.envs.ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE, + "get", + return_value=False, + ) + ) + + @contextmanager + def _device_context(self, device): + previous = self.devices.current + self.devices.current = device + try: + yield + finally: + self.devices.current = previous + + @contextmanager + def _environment(self, **values): + with patch.dict(os.environ): + for name in ("MC_FORCE_TCP", "MC_INTRANODE_NVLINK", "MOONCAKE_PROTOCOL"): + os.environ.pop(name, None) + os.environ.update(values) + yield + + def _engine(self, gpu_id=1): + package = ModuleType("mooncake") + native_module = ModuleType("mooncake.engine") + native_module.TransferEngine = lambda: self.native + package.engine = native_module + with patch.dict( + sys.modules, {"mooncake": package, "mooncake.engine": native_module} + ): + return mooncake_transfer_engine.MooncakeTransferEngine( + "127.0.0.1", gpu_id=gpu_id + ) + + @staticmethod + def _transfer(engine): + return engine.batch_transfer_sync("peer", [10, 20], [30, 40], [50, 60]) + + def test_each_batch_uses_bootstrap_device_and_restores_caller(self): + with self._environment(MC_INTRANODE_NVLINK="true"): + engine = self._engine() + for initial_device in (0, 3): + self.devices.current = initial_device + self.assertEqual(self._transfer(engine), 0) + self.assertEqual(self.devices.current, initial_device) + + self.assertEqual([call[1] for call in self.native.calls], [1, 1]) + self.assertTrue( + all( + call[2] == ("peer", [10, 20], [30, 40], [50, 60]) + for call in self.native.calls + ) + ) + + def test_sender_threads_restore_their_distinct_initial_devices(self): + barrier = threading.Barrier(3) + with self._environment(MC_INTRANODE_NVLINK="true"): + engine = self._engine() + + def send(initial_device): + self.devices.current = initial_device + barrier.wait(timeout=5) + for _ in range(5): + self.assertEqual(self._transfer(engine), 0) + self.assertEqual(self.devices.current, initial_device) + return threading.get_ident() + + with ThreadPoolExecutor(max_workers=3) as executor: + threads = list(executor.map(send, (0, 2, 3))) + + self.assertEqual(len(set(threads)), 3) + self.assertEqual(len(self.native.calls), 15) + self.assertTrue(all(call[1] == 1 for call in self.native.calls)) + + def test_native_exception_restores_device_and_preserves_error_return(self): + with self._environment(MC_INTRANODE_NVLINK="true"): + engine = self._engine() + self.native.error = RuntimeError("native copy failed") + self.devices.current = 3 + self.assertEqual(self._transfer(engine), -1) + self.assertEqual(self.devices.current, 3) + self.assertEqual(self.native.calls[-1][1], 1) + + def test_negative_native_result_is_preserved(self): + with self._environment(MC_INTRANODE_NVLINK="true"): + engine = self._engine() + self.native.result = -7 + self.devices.current = 2 + self.assertEqual(self._transfer(engine), -7) + self.assertEqual(self.devices.current, 2) + + def test_missing_native_api_still_reports_upgrade_after_context_exit(self): + with self._environment(MC_INTRANODE_NVLINK="true"): + engine = self._engine() + engine.engine = object() + self.devices.current = 3 + with self.assertRaisesRegex(RuntimeError, "batch transfer requires"): + self._transfer(engine) + self.assertEqual(self.devices.current, 3) + + def test_invalid_bootstrap_device_fails_before_native_call(self): + with self._environment(MC_INTRANODE_NVLINK="true"): + for gpu_id in (None, True, False, -1, "1"): + with self.subTest(gpu_id=gpu_id): + engine = self._engine(gpu_id) + with self.assertRaisesRegex(RuntimeError, "explicit bootstrap"): + self._transfer(engine) + + self.assertEqual(self.native.calls, []) + self.cuda_context.assert_not_called() + + def test_constructor_fallback_does_not_invent_an_explicit_device(self): + with self._environment(MC_INTRANODE_NVLINK="true"): + engine = self._engine(None) + self.model_device = SimpleNamespace(device="cuda", gpu_id=0) + self.assertEqual(engine.gpu_id, 0) + with self.assertRaisesRegex(RuntimeError, "explicit bootstrap"): + self._transfer(engine) + + self.assertEqual(self.native.calls, []) + + def test_mismatched_wrapper_or_published_device_fails_before_native(self): + with self._environment(MC_INTRANODE_NVLINK="true"): + for wrapper_device, published_device in ((0, 1), (1, 0)): + with self.subTest( + wrapper_device=wrapper_device, published_device=published_device + ): + engine = self._engine() + engine.gpu_id = wrapper_device + self.model_device = SimpleNamespace( + device="cuda", gpu_id=published_device + ) + with self.assertRaisesRegex(RuntimeError, "explicit bootstrap"): + self._transfer(engine) + + self.assertEqual(self.native.calls, []) + self.cuda_context.assert_not_called() + + def test_missing_or_unknown_runtime_device_fails_before_native(self): + with self._environment(MC_INTRANODE_NVLINK="true"): + engine = self._engine() + for error in (ValueError("not published"), AttributeError("no device")): + with self.subTest(error=type(error).__name__): + self.device_snapshot.side_effect = error + with self.assertRaisesRegex(RuntimeError, "published model device"): + self._transfer(engine) + self.device_snapshot.side_effect = lambda: self.model_device + self.model_device = SimpleNamespace(device=None, gpu_id=1) + with self.assertRaisesRegex(RuntimeError, "explicit bootstrap"): + self._transfer(engine) + + self.assertEqual(self.native.calls, []) + self.cuda_context.assert_not_called() + + def test_non_cuda_devices_keep_original_native_path(self): + with self._environment(MC_INTRANODE_NVLINK="true"): + engine = self._engine(None) + for device in ("cpu", "npu", "xpu", "hpu", "musa", "mlu", "mps"): + with self.subTest(device=device): + self.model_device = SimpleNamespace(device=device, gpu_id=None) + self.devices.current = 7 + self.assertEqual(self._transfer(engine), 0) + self.assertEqual(self.devices.current, 7) + + self.assertEqual(len(self.native.calls), 7) + self.assertTrue(all(call[1] == 7 for call in self.native.calls)) + self.cuda_context.assert_not_called() + + def test_ordinary_transports_do_not_require_cuda_device_snapshot(self): + for protocol in (None, "rdma", "tcp", "efa"): + with self.subTest(protocol=protocol): + values = {} if protocol is None else {"MOONCAKE_PROTOCOL": protocol} + with self._environment(**values): + engine = self._engine(None) + self.device_snapshot.side_effect = AssertionError( + "ordinary transport must not query the model device" + ) + self.devices.current = 4 + self.assertEqual(self._transfer(engine), 0) + self.assertEqual(self.devices.current, 4) + + self.assertEqual(len(self.native.calls), 4) + self.device_snapshot.assert_not_called() + self.cuda_context.assert_not_called() + + def test_force_tcp_presence_overrides_intra_nvlink_selection(self): + for value in ("", "0", "1"): + with self.subTest(value=value): + with self._environment(MC_FORCE_TCP=value, MC_INTRANODE_NVLINK="true"): + engine = self._engine(None) + self.devices.current = 4 + self.assertEqual(self._transfer(engine), 0) + self.assertEqual(self.devices.current, 4) + + self.assertEqual(len(self.native.calls), 3) + self.device_snapshot.assert_not_called() + self.cuda_context.assert_not_called() + + def test_intra_nvlink_presence_and_protocol_aliases_select_context(self): + selections = ( + {"MC_INTRANODE_NVLINK": ""}, + {"MOONCAKE_PROTOCOL": "nvlink_intra"}, + {"MOONCAKE_PROTOCOL": "nvlink-intra"}, + {"MOONCAKE_PROTOCOL": "intra_nvlink"}, + ) + for selection in selections: + with self.subTest(selection=selection): + with self._environment(**selection): + engine = self._engine() + self.devices.current = 3 + self.assertEqual(self._transfer(engine), 0) + self.assertEqual(self.devices.current, 3) + + self.assertEqual(len(self.native.calls), 4) + self.assertTrue(all(call[1] == 1 for call in self.native.calls)) + + +if __name__ == "__main__": + unittest.main()