fix(mooncake): bind IntraNode batches to the bootstrap CUDA device

Sender and executor threads can enter synchronous IntraNode transfers with
another current CUDA device. Keep the explicit bootstrap ordinal, check it
against the published model device, and scope the native batch call to that
device while restoring the calling thread on success or failure.

Ordinary transports, force-TCP and non-CUDA devices keep their existing
path. Missing or inconsistent CUDA identity is rejected before native IO.

Add 13 CPU regressions importing the production wrapper, including real
constructor calls with a mocked SDK, concurrent thread-local device states,
error restoration and transport/device compatibility. All 13 pass; two
selected tests fail on the unchanged wrapper at behavior assertions. Native
and CUDA boundaries are mocked, so this does not certify physical CUDA IPC
or attribute the separate container-PID collision to this change.
This commit is contained in:
2026-10-30 11:42:19 +08:00
parent 17505e6cf6
commit ee9170e7bb
2 changed files with 327 additions and 3 deletions
@@ -3,10 +3,12 @@ from __future__ import annotations
import json
import logging
import os
from contextlib import nullcontext
from typing import TYPE_CHECKING, Dict, List, Optional, Union
from sglang.srt.environ import envs
from sglang.srt.runtime_context import (
get_device,
get_disagg,
get_exec,
get_memory,
@@ -128,6 +130,8 @@ class MooncakeTransferEngine:
self.engine = TransferEngine()
self.hostname = hostname
self.gpu_id = gpu_id if gpu_id is not None else 0
# Keep bootstrap's explicit local ordinal, not a new thread's default.
self._pd_cuda_gpu_id = gpu_id
# MC_FORCE_TCP=1 makes mooncake install TcpTransport instead of RDMA,
# in which case RDMA HCA selection is irrelevant; pass empty device.
if os.environ.get("MC_FORCE_TCP") == "1":
@@ -243,6 +247,40 @@ class MooncakeTransferEngine:
return ret
def _batch_cuda_device_context(self):
# Force-TCP initialization bypasses all GPU transports in this SDK.
if "MC_FORCE_TCP" in os.environ:
return nullcontext()
protocol = (os.environ.get("MOONCAKE_PROTOCOL") or "").strip().lower()
intra = "MC_INTRANODE_NVLINK" in os.environ or protocol in (
"nvlink_intra",
"nvlink-intra",
"intra_nvlink",
)
if not intra:
return nullcontext()
# CUDA device selection is thread-local. Sender and executor threads
# must use the model/bootstrap device, not their initial current_device.
try:
declared = get_device()
device_type = declared.device
except (ValueError, AttributeError) as exc:
raise RuntimeError("NVLink PD requires a published model device") from exc
if device_type in ("cpu", "npu", "xpu", "hpu", "musa", "mlu", "mps"):
return nullcontext()
gpu_id = self._pd_cuda_gpu_id
if (
device_type != "cuda"
or type(gpu_id) is not int
or gpu_id < 0
or self.gpu_id != gpu_id
or declared.gpu_id != gpu_id
):
raise RuntimeError("NVLink PD requires the explicit bootstrap CUDA gpu_id")
import torch
return torch.cuda.device(gpu_id)
def batch_transfer_sync(
self,
session_id: str,
@@ -251,10 +289,12 @@ class MooncakeTransferEngine:
lengths: List[int],
) -> int:
"""Synchronously transfer data to the specified addresses in batches."""
device_context = self._batch_cuda_device_context()
try:
ret = self.engine.batch_transfer_sync_write(
session_id, buffers, peer_buffer_addresses, lengths
)
with device_context:
ret = self.engine.batch_transfer_sync_write(
session_id, buffers, peer_buffer_addresses, lengths
)
except Exception:
ret = -1
if not hasattr(self.engine, "batch_transfer_sync_write"):