fix(mooncake): bind IntraNode batches to the bootstrap CUDA device
Sender and executor threads can enter synchronous IntraNode transfers with another current CUDA device. Keep the explicit bootstrap ordinal, check it against the published model device, and scope the native batch call to that device while restoring the calling thread on success or failure. Ordinary transports, force-TCP and non-CUDA devices keep their existing path. Missing or inconsistent CUDA identity is rejected before native IO. Add 13 CPU regressions importing the production wrapper, including real constructor calls with a mocked SDK, concurrent thread-local device states, error restoration and transport/device compatibility. All 13 pass; two selected tests fail on the unchanged wrapper at behavior assertions. Native and CUDA boundaries are mocked, so this does not certify physical CUDA IPC or attribute the separate container-PID collision to this change.
This commit is contained in:
@@ -3,10 +3,12 @@ from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from contextlib import nullcontext
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional, Union
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import (
|
||||
get_device,
|
||||
get_disagg,
|
||||
get_exec,
|
||||
get_memory,
|
||||
@@ -128,6 +130,8 @@ class MooncakeTransferEngine:
|
||||
self.engine = TransferEngine()
|
||||
self.hostname = hostname
|
||||
self.gpu_id = gpu_id if gpu_id is not None else 0
|
||||
# Keep bootstrap's explicit local ordinal, not a new thread's default.
|
||||
self._pd_cuda_gpu_id = gpu_id
|
||||
# MC_FORCE_TCP=1 makes mooncake install TcpTransport instead of RDMA,
|
||||
# in which case RDMA HCA selection is irrelevant; pass empty device.
|
||||
if os.environ.get("MC_FORCE_TCP") == "1":
|
||||
@@ -243,6 +247,40 @@ class MooncakeTransferEngine:
|
||||
|
||||
return ret
|
||||
|
||||
def _batch_cuda_device_context(self):
|
||||
# Force-TCP initialization bypasses all GPU transports in this SDK.
|
||||
if "MC_FORCE_TCP" in os.environ:
|
||||
return nullcontext()
|
||||
protocol = (os.environ.get("MOONCAKE_PROTOCOL") or "").strip().lower()
|
||||
intra = "MC_INTRANODE_NVLINK" in os.environ or protocol in (
|
||||
"nvlink_intra",
|
||||
"nvlink-intra",
|
||||
"intra_nvlink",
|
||||
)
|
||||
if not intra:
|
||||
return nullcontext()
|
||||
# CUDA device selection is thread-local. Sender and executor threads
|
||||
# must use the model/bootstrap device, not their initial current_device.
|
||||
try:
|
||||
declared = get_device()
|
||||
device_type = declared.device
|
||||
except (ValueError, AttributeError) as exc:
|
||||
raise RuntimeError("NVLink PD requires a published model device") from exc
|
||||
if device_type in ("cpu", "npu", "xpu", "hpu", "musa", "mlu", "mps"):
|
||||
return nullcontext()
|
||||
gpu_id = self._pd_cuda_gpu_id
|
||||
if (
|
||||
device_type != "cuda"
|
||||
or type(gpu_id) is not int
|
||||
or gpu_id < 0
|
||||
or self.gpu_id != gpu_id
|
||||
or declared.gpu_id != gpu_id
|
||||
):
|
||||
raise RuntimeError("NVLink PD requires the explicit bootstrap CUDA gpu_id")
|
||||
import torch
|
||||
|
||||
return torch.cuda.device(gpu_id)
|
||||
|
||||
def batch_transfer_sync(
|
||||
self,
|
||||
session_id: str,
|
||||
@@ -251,10 +289,12 @@ class MooncakeTransferEngine:
|
||||
lengths: List[int],
|
||||
) -> int:
|
||||
"""Synchronously transfer data to the specified addresses in batches."""
|
||||
device_context = self._batch_cuda_device_context()
|
||||
try:
|
||||
ret = self.engine.batch_transfer_sync_write(
|
||||
session_id, buffers, peer_buffer_addresses, lengths
|
||||
)
|
||||
with device_context:
|
||||
ret = self.engine.batch_transfer_sync_write(
|
||||
session_id, buffers, peer_buffer_addresses, lengths
|
||||
)
|
||||
except Exception:
|
||||
ret = -1
|
||||
if not hasattr(self.engine, "batch_transfer_sync_write"):
|
||||
|
||||
@@ -0,0 +1,284 @@
|
||||
"""CPU regressions for the device context of an IntraNode NVLink sender."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.distributed.device_communicators import mooncake_transfer_engine
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class _NativeEngine:
|
||||
"""Only the native boundary is fake; the production wrapper runs unchanged."""
|
||||
|
||||
def __init__(self, devices):
|
||||
self.devices = devices
|
||||
self.calls = []
|
||||
self.call_lock = threading.Lock()
|
||||
self.result = 0
|
||||
self.error = None
|
||||
|
||||
def initialize(self, *args):
|
||||
return 0
|
||||
|
||||
def get_rpc_port(self):
|
||||
return 12345
|
||||
|
||||
def batch_transfer_sync_write(self, *args):
|
||||
with self.call_lock:
|
||||
self.calls.append((threading.get_ident(), self.devices.current, args))
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
return self.result
|
||||
|
||||
|
||||
class TestMooncakeTransferDeviceContext(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.devices = threading.local()
|
||||
self.devices.current = 0
|
||||
self.native = _NativeEngine(self.devices)
|
||||
self.model_device = SimpleNamespace(device="cuda", gpu_id=1)
|
||||
self.patches = ExitStack()
|
||||
self.addCleanup(self.patches.close)
|
||||
self.device_snapshot = self.patches.enter_context(
|
||||
patch.object(
|
||||
mooncake_transfer_engine,
|
||||
"get_device",
|
||||
side_effect=lambda: self.model_device,
|
||||
create=True,
|
||||
)
|
||||
)
|
||||
self.cuda_context = self.patches.enter_context(
|
||||
patch.object(torch.cuda, "device", side_effect=self._device_context)
|
||||
)
|
||||
self.patches.enter_context(
|
||||
patch.object(
|
||||
mooncake_transfer_engine.envs.ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE,
|
||||
"get",
|
||||
return_value=False,
|
||||
)
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def _device_context(self, device):
|
||||
previous = self.devices.current
|
||||
self.devices.current = device
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.devices.current = previous
|
||||
|
||||
@contextmanager
|
||||
def _environment(self, **values):
|
||||
with patch.dict(os.environ):
|
||||
for name in ("MC_FORCE_TCP", "MC_INTRANODE_NVLINK", "MOONCAKE_PROTOCOL"):
|
||||
os.environ.pop(name, None)
|
||||
os.environ.update(values)
|
||||
yield
|
||||
|
||||
def _engine(self, gpu_id=1):
|
||||
package = ModuleType("mooncake")
|
||||
native_module = ModuleType("mooncake.engine")
|
||||
native_module.TransferEngine = lambda: self.native
|
||||
package.engine = native_module
|
||||
with patch.dict(
|
||||
sys.modules, {"mooncake": package, "mooncake.engine": native_module}
|
||||
):
|
||||
return mooncake_transfer_engine.MooncakeTransferEngine(
|
||||
"127.0.0.1", gpu_id=gpu_id
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _transfer(engine):
|
||||
return engine.batch_transfer_sync("peer", [10, 20], [30, 40], [50, 60])
|
||||
|
||||
def test_each_batch_uses_bootstrap_device_and_restores_caller(self):
|
||||
with self._environment(MC_INTRANODE_NVLINK="true"):
|
||||
engine = self._engine()
|
||||
for initial_device in (0, 3):
|
||||
self.devices.current = initial_device
|
||||
self.assertEqual(self._transfer(engine), 0)
|
||||
self.assertEqual(self.devices.current, initial_device)
|
||||
|
||||
self.assertEqual([call[1] for call in self.native.calls], [1, 1])
|
||||
self.assertTrue(
|
||||
all(
|
||||
call[2] == ("peer", [10, 20], [30, 40], [50, 60])
|
||||
for call in self.native.calls
|
||||
)
|
||||
)
|
||||
|
||||
def test_sender_threads_restore_their_distinct_initial_devices(self):
|
||||
barrier = threading.Barrier(3)
|
||||
with self._environment(MC_INTRANODE_NVLINK="true"):
|
||||
engine = self._engine()
|
||||
|
||||
def send(initial_device):
|
||||
self.devices.current = initial_device
|
||||
barrier.wait(timeout=5)
|
||||
for _ in range(5):
|
||||
self.assertEqual(self._transfer(engine), 0)
|
||||
self.assertEqual(self.devices.current, initial_device)
|
||||
return threading.get_ident()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=3) as executor:
|
||||
threads = list(executor.map(send, (0, 2, 3)))
|
||||
|
||||
self.assertEqual(len(set(threads)), 3)
|
||||
self.assertEqual(len(self.native.calls), 15)
|
||||
self.assertTrue(all(call[1] == 1 for call in self.native.calls))
|
||||
|
||||
def test_native_exception_restores_device_and_preserves_error_return(self):
|
||||
with self._environment(MC_INTRANODE_NVLINK="true"):
|
||||
engine = self._engine()
|
||||
self.native.error = RuntimeError("native copy failed")
|
||||
self.devices.current = 3
|
||||
self.assertEqual(self._transfer(engine), -1)
|
||||
self.assertEqual(self.devices.current, 3)
|
||||
self.assertEqual(self.native.calls[-1][1], 1)
|
||||
|
||||
def test_negative_native_result_is_preserved(self):
|
||||
with self._environment(MC_INTRANODE_NVLINK="true"):
|
||||
engine = self._engine()
|
||||
self.native.result = -7
|
||||
self.devices.current = 2
|
||||
self.assertEqual(self._transfer(engine), -7)
|
||||
self.assertEqual(self.devices.current, 2)
|
||||
|
||||
def test_missing_native_api_still_reports_upgrade_after_context_exit(self):
|
||||
with self._environment(MC_INTRANODE_NVLINK="true"):
|
||||
engine = self._engine()
|
||||
engine.engine = object()
|
||||
self.devices.current = 3
|
||||
with self.assertRaisesRegex(RuntimeError, "batch transfer requires"):
|
||||
self._transfer(engine)
|
||||
self.assertEqual(self.devices.current, 3)
|
||||
|
||||
def test_invalid_bootstrap_device_fails_before_native_call(self):
|
||||
with self._environment(MC_INTRANODE_NVLINK="true"):
|
||||
for gpu_id in (None, True, False, -1, "1"):
|
||||
with self.subTest(gpu_id=gpu_id):
|
||||
engine = self._engine(gpu_id)
|
||||
with self.assertRaisesRegex(RuntimeError, "explicit bootstrap"):
|
||||
self._transfer(engine)
|
||||
|
||||
self.assertEqual(self.native.calls, [])
|
||||
self.cuda_context.assert_not_called()
|
||||
|
||||
def test_constructor_fallback_does_not_invent_an_explicit_device(self):
|
||||
with self._environment(MC_INTRANODE_NVLINK="true"):
|
||||
engine = self._engine(None)
|
||||
self.model_device = SimpleNamespace(device="cuda", gpu_id=0)
|
||||
self.assertEqual(engine.gpu_id, 0)
|
||||
with self.assertRaisesRegex(RuntimeError, "explicit bootstrap"):
|
||||
self._transfer(engine)
|
||||
|
||||
self.assertEqual(self.native.calls, [])
|
||||
|
||||
def test_mismatched_wrapper_or_published_device_fails_before_native(self):
|
||||
with self._environment(MC_INTRANODE_NVLINK="true"):
|
||||
for wrapper_device, published_device in ((0, 1), (1, 0)):
|
||||
with self.subTest(
|
||||
wrapper_device=wrapper_device, published_device=published_device
|
||||
):
|
||||
engine = self._engine()
|
||||
engine.gpu_id = wrapper_device
|
||||
self.model_device = SimpleNamespace(
|
||||
device="cuda", gpu_id=published_device
|
||||
)
|
||||
with self.assertRaisesRegex(RuntimeError, "explicit bootstrap"):
|
||||
self._transfer(engine)
|
||||
|
||||
self.assertEqual(self.native.calls, [])
|
||||
self.cuda_context.assert_not_called()
|
||||
|
||||
def test_missing_or_unknown_runtime_device_fails_before_native(self):
|
||||
with self._environment(MC_INTRANODE_NVLINK="true"):
|
||||
engine = self._engine()
|
||||
for error in (ValueError("not published"), AttributeError("no device")):
|
||||
with self.subTest(error=type(error).__name__):
|
||||
self.device_snapshot.side_effect = error
|
||||
with self.assertRaisesRegex(RuntimeError, "published model device"):
|
||||
self._transfer(engine)
|
||||
self.device_snapshot.side_effect = lambda: self.model_device
|
||||
self.model_device = SimpleNamespace(device=None, gpu_id=1)
|
||||
with self.assertRaisesRegex(RuntimeError, "explicit bootstrap"):
|
||||
self._transfer(engine)
|
||||
|
||||
self.assertEqual(self.native.calls, [])
|
||||
self.cuda_context.assert_not_called()
|
||||
|
||||
def test_non_cuda_devices_keep_original_native_path(self):
|
||||
with self._environment(MC_INTRANODE_NVLINK="true"):
|
||||
engine = self._engine(None)
|
||||
for device in ("cpu", "npu", "xpu", "hpu", "musa", "mlu", "mps"):
|
||||
with self.subTest(device=device):
|
||||
self.model_device = SimpleNamespace(device=device, gpu_id=None)
|
||||
self.devices.current = 7
|
||||
self.assertEqual(self._transfer(engine), 0)
|
||||
self.assertEqual(self.devices.current, 7)
|
||||
|
||||
self.assertEqual(len(self.native.calls), 7)
|
||||
self.assertTrue(all(call[1] == 7 for call in self.native.calls))
|
||||
self.cuda_context.assert_not_called()
|
||||
|
||||
def test_ordinary_transports_do_not_require_cuda_device_snapshot(self):
|
||||
for protocol in (None, "rdma", "tcp", "efa"):
|
||||
with self.subTest(protocol=protocol):
|
||||
values = {} if protocol is None else {"MOONCAKE_PROTOCOL": protocol}
|
||||
with self._environment(**values):
|
||||
engine = self._engine(None)
|
||||
self.device_snapshot.side_effect = AssertionError(
|
||||
"ordinary transport must not query the model device"
|
||||
)
|
||||
self.devices.current = 4
|
||||
self.assertEqual(self._transfer(engine), 0)
|
||||
self.assertEqual(self.devices.current, 4)
|
||||
|
||||
self.assertEqual(len(self.native.calls), 4)
|
||||
self.device_snapshot.assert_not_called()
|
||||
self.cuda_context.assert_not_called()
|
||||
|
||||
def test_force_tcp_presence_overrides_intra_nvlink_selection(self):
|
||||
for value in ("", "0", "1"):
|
||||
with self.subTest(value=value):
|
||||
with self._environment(MC_FORCE_TCP=value, MC_INTRANODE_NVLINK="true"):
|
||||
engine = self._engine(None)
|
||||
self.devices.current = 4
|
||||
self.assertEqual(self._transfer(engine), 0)
|
||||
self.assertEqual(self.devices.current, 4)
|
||||
|
||||
self.assertEqual(len(self.native.calls), 3)
|
||||
self.device_snapshot.assert_not_called()
|
||||
self.cuda_context.assert_not_called()
|
||||
|
||||
def test_intra_nvlink_presence_and_protocol_aliases_select_context(self):
|
||||
selections = (
|
||||
{"MC_INTRANODE_NVLINK": ""},
|
||||
{"MOONCAKE_PROTOCOL": "nvlink_intra"},
|
||||
{"MOONCAKE_PROTOCOL": "nvlink-intra"},
|
||||
{"MOONCAKE_PROTOCOL": "intra_nvlink"},
|
||||
)
|
||||
for selection in selections:
|
||||
with self.subTest(selection=selection):
|
||||
with self._environment(**selection):
|
||||
engine = self._engine()
|
||||
self.devices.current = 3
|
||||
self.assertEqual(self._transfer(engine), 0)
|
||||
self.assertEqual(self.devices.current, 3)
|
||||
|
||||
self.assertEqual(len(self.native.calls), 4)
|
||||
self.assertTrue(all(call[1] == 1 for call in self.native.calls))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user