[Feat] NIXL: Support per-GPU UCX device selection (#43482)
This commit is contained in:
@@ -80,7 +80,7 @@ class Disagg(msgspec.Struct):
|
||||
] = 8998
|
||||
disaggregation_ib_device: A[
|
||||
Optional[str],
|
||||
'The InfiniBand devices for disaggregation transfer. Supports a single device (e.g., --disaggregation-ib-device mlx5_0), a shared comma-separated list (e.g., --disaggregation-ib-device mlx5_0,mlx5_1), a per-GPU JSON mapping (e.g., --disaggregation-ib-device \'{"0": "mlx5_0,mlx5_1", "1": "mlx5_2"}\'), or a path to a JSON file containing that mapping. Default is None, which triggers automatic device detection when mooncake backend is enabled.',
|
||||
'The InfiniBand devices for disaggregation transfer. Supports a single device (e.g., --disaggregation-ib-device mlx5_0), a shared comma-separated list (e.g., --disaggregation-ib-device mlx5_0,mlx5_1), a per-GPU JSON mapping (e.g., --disaggregation-ib-device \'{"0": "mlx5_0,mlx5_1", "1": "mlx5_2"}\'), or a path to a JSON file containing that mapping. Supported by Mooncake and NIXL UCX. Mapping keys are process-visible GPU IDs (after CUDA_VISIBLE_DEVICES); use HCA names without port suffixes for NIXL UCX. Default is None, which preserves backend device selection.',
|
||||
] = None
|
||||
disaggregation_decode_enable_radix_cache: A[
|
||||
bool,
|
||||
|
||||
@@ -179,6 +179,24 @@ NIXL_ERR_SETTLE_TIMEOUT_S = 5.0
|
||||
NIXL_ERR_SETTLE_POLL_S = 0.001
|
||||
|
||||
|
||||
def _resolve_ucx_device_list(ib_device: str, gpu_id: int) -> Optional[str]:
|
||||
from sglang.srt.distributed.device_communicators.mooncake_transfer_engine import (
|
||||
get_ib_devices_for_gpu,
|
||||
)
|
||||
|
||||
devices = get_ib_devices_for_gpu(ib_device, gpu_id)
|
||||
if devices is None:
|
||||
return None
|
||||
names = [name.strip() for name in devices.split(",")]
|
||||
if any(not name or ":" in name for name in names):
|
||||
raise ValueError(
|
||||
"NIXL UCX --disaggregation-ib-device requires HCA names without "
|
||||
"port suffixes (e.g. mlx5_0); NIXL appends port 1."
|
||||
)
|
||||
# NIXL's UCX device_list parser splits on the literal delimiter ", ".
|
||||
return ", ".join(names)
|
||||
|
||||
|
||||
def _normalize_kv_mem_kinds(kinds: Optional[List[str]], expected_len: int) -> List[str]:
|
||||
if kinds is None:
|
||||
return ["VRAM"] * expected_len
|
||||
@@ -583,6 +601,31 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
"SGLANG_DISAGGREGATION_NIXL_BACKEND_PARAMS must be a JSON object "
|
||||
"with string keys and string values"
|
||||
)
|
||||
if backend == "UCX" and self.kv_args.ib_device:
|
||||
device_list = _resolve_ucx_device_list(
|
||||
self.kv_args.ib_device, self.kv_args.gpu_id
|
||||
)
|
||||
if device_list is not None:
|
||||
existing = backend_params.get("device_list")
|
||||
if (
|
||||
existing is not None
|
||||
and ", ".join(name.strip() for name in existing.split(","))
|
||||
!= device_list
|
||||
):
|
||||
raise ValueError(
|
||||
"Conflicting NIXL UCX device_list in "
|
||||
"SGLANG_DISAGGREGATION_NIXL_BACKEND_PARAMS and "
|
||||
"--disaggregation-ib-device. Configure only one "
|
||||
"device selection or make them agree."
|
||||
)
|
||||
backend_params["device_list"] = device_list
|
||||
logger.info(
|
||||
"NIXL UCX device binding: gpu_id=%s, engine_rank=%s, "
|
||||
"device_list=%s",
|
||||
self.kv_args.gpu_id,
|
||||
self.kv_args.engine_rank,
|
||||
device_list,
|
||||
)
|
||||
# self.transfer_worker and self._start_bootstrap_thread runs concurrently
|
||||
# so we cannot use sync_mode=None which is thread-unsafe.
|
||||
agent_config = nixl_agent_config(
|
||||
|
||||
@@ -29,9 +29,8 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
def configure_nixl_pd_backend(test_cls):
|
||||
test_cls.transfer_backend = ["--disaggregation-transfer-backend", "nixl"]
|
||||
# NIXL backend/network selection is driven by NIXL environment variables
|
||||
# such as SGLANG_DISAGGREGATION_NIXL_BACKEND and backend params, not by the
|
||||
# Mooncake-specific --disaggregation-ib-device argument.
|
||||
# Keep these tests on the configured NIXL/UCX device selection rather than
|
||||
# inheriting the Mooncake fixture's per-side RDMA device lists.
|
||||
test_cls.rdma_devices = []
|
||||
|
||||
|
||||
@@ -119,7 +118,7 @@ class PDDisaggregationServerBase(CustomTestCase):
|
||||
@classmethod
|
||||
def rdma_devices_for(cls, gpu_indices) -> list:
|
||||
"""`--disaggregation-ib-device` args for a server pinned to these GPUs."""
|
||||
if not is_in_ci():
|
||||
if not is_in_ci() or not cls.rdma_devices:
|
||||
return cls.rdma_devices
|
||||
return ["--disaggregation-ib-device", get_rdma_devices_args(gpu_indices)]
|
||||
|
||||
@@ -532,8 +531,8 @@ def _maybe_set_ucx_net_devices(ib_devices) -> bool:
|
||||
# createBackend and tcp is the only transport left that can serve it, so
|
||||
# keep loopback here.
|
||||
net_devices = ",".join([*(f"{d}:1" for d in devices), "lo"])
|
||||
# NIXL ignores --disaggregation-ib-device; without this UCX opens every RDMA
|
||||
# device on the host, and that full-device init can stall inside the driver.
|
||||
# Limit default UCX device discovery; opening every RDMA device on the host
|
||||
# can stall inside the driver.
|
||||
os.environ["UCX_NET_DEVICES"] = net_devices
|
||||
logger.warning("Set UCX_NET_DEVICES=%s for NIXL/UCX", net_devices)
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user