fix(mooncake): bind IntraNode batches to the bootstrap CUDA device
Sender and executor threads can enter synchronous IntraNode transfers with another current CUDA device. Keep the explicit bootstrap ordinal, check it against the published model device, and scope the native batch call to that device while restoring the calling thread on success or failure. Ordinary transports, force-TCP and non-CUDA devices keep their existing path. Missing or inconsistent CUDA identity is rejected before native IO. Add 13 CPU regressions importing the production wrapper, including real constructor calls with a mocked SDK, concurrent thread-local device states, error restoration and transport/device compatibility. All 13 pass; two selected tests fail on the unchanged wrapper at behavior assertions. Native and CUDA boundaries are mocked, so this does not certify physical CUDA IPC or attribute the separate container-PID collision to this change.
This commit is contained in:
@@ -3,10 +3,12 @@ from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from contextlib import nullcontext
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional, Union
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import (
|
||||
get_device,
|
||||
get_disagg,
|
||||
get_exec,
|
||||
get_memory,
|
||||
@@ -128,6 +130,8 @@ class MooncakeTransferEngine:
|
||||
self.engine = TransferEngine()
|
||||
self.hostname = hostname
|
||||
self.gpu_id = gpu_id if gpu_id is not None else 0
|
||||
# Keep bootstrap's explicit local ordinal, not a new thread's default.
|
||||
self._pd_cuda_gpu_id = gpu_id
|
||||
# MC_FORCE_TCP=1 makes mooncake install TcpTransport instead of RDMA,
|
||||
# in which case RDMA HCA selection is irrelevant; pass empty device.
|
||||
if os.environ.get("MC_FORCE_TCP") == "1":
|
||||
@@ -243,6 +247,40 @@ class MooncakeTransferEngine:
|
||||
|
||||
return ret
|
||||
|
||||
def _batch_cuda_device_context(self):
|
||||
# Force-TCP initialization bypasses all GPU transports in this SDK.
|
||||
if "MC_FORCE_TCP" in os.environ:
|
||||
return nullcontext()
|
||||
protocol = (os.environ.get("MOONCAKE_PROTOCOL") or "").strip().lower()
|
||||
intra = "MC_INTRANODE_NVLINK" in os.environ or protocol in (
|
||||
"nvlink_intra",
|
||||
"nvlink-intra",
|
||||
"intra_nvlink",
|
||||
)
|
||||
if not intra:
|
||||
return nullcontext()
|
||||
# CUDA device selection is thread-local. Sender and executor threads
|
||||
# must use the model/bootstrap device, not their initial current_device.
|
||||
try:
|
||||
declared = get_device()
|
||||
device_type = declared.device
|
||||
except (ValueError, AttributeError) as exc:
|
||||
raise RuntimeError("NVLink PD requires a published model device") from exc
|
||||
if device_type in ("cpu", "npu", "xpu", "hpu", "musa", "mlu", "mps"):
|
||||
return nullcontext()
|
||||
gpu_id = self._pd_cuda_gpu_id
|
||||
if (
|
||||
device_type != "cuda"
|
||||
or type(gpu_id) is not int
|
||||
or gpu_id < 0
|
||||
or self.gpu_id != gpu_id
|
||||
or declared.gpu_id != gpu_id
|
||||
):
|
||||
raise RuntimeError("NVLink PD requires the explicit bootstrap CUDA gpu_id")
|
||||
import torch
|
||||
|
||||
return torch.cuda.device(gpu_id)
|
||||
|
||||
def batch_transfer_sync(
|
||||
self,
|
||||
session_id: str,
|
||||
@@ -251,10 +289,12 @@ class MooncakeTransferEngine:
|
||||
lengths: List[int],
|
||||
) -> int:
|
||||
"""Synchronously transfer data to the specified addresses in batches."""
|
||||
device_context = self._batch_cuda_device_context()
|
||||
try:
|
||||
ret = self.engine.batch_transfer_sync_write(
|
||||
session_id, buffers, peer_buffer_addresses, lengths
|
||||
)
|
||||
with device_context:
|
||||
ret = self.engine.batch_transfer_sync_write(
|
||||
session_id, buffers, peer_buffer_addresses, lengths
|
||||
)
|
||||
except Exception:
|
||||
ret = -1
|
||||
if not hasattr(self.engine, "batch_transfer_sync_write"):
|
||||
|
||||
Reference in New Issue
Block a user