[Fix] NIXL: Preserve explicit UCX network device selection (#45722)
This commit is contained in:
@@ -80,7 +80,7 @@ class Disagg(msgspec.Struct):
|
||||
] = 8998
|
||||
disaggregation_ib_device: A[
|
||||
Optional[str],
|
||||
'The InfiniBand devices for disaggregation transfer. Supports a single device (e.g., --disaggregation-ib-device mlx5_0), a shared comma-separated list (e.g., --disaggregation-ib-device mlx5_0,mlx5_1), a per-GPU JSON mapping (e.g., --disaggregation-ib-device \'{"0": "mlx5_0,mlx5_1", "1": "mlx5_2"}\'), or a path to a JSON file containing that mapping. Supported by Mooncake and NIXL UCX. Mapping keys are process-visible GPU IDs (after CUDA_VISIBLE_DEVICES); use HCA names without port suffixes for NIXL UCX. Default is None, which preserves backend device selection.',
|
||||
'The InfiniBand devices for disaggregation transfer. Supports a single device (e.g., --disaggregation-ib-device mlx5_0), a shared comma-separated list (e.g., --disaggregation-ib-device mlx5_0,mlx5_1), a per-GPU JSON mapping (e.g., --disaggregation-ib-device \'{"0": "mlx5_0,mlx5_1", "1": "mlx5_2"}\'), or a path to a JSON file containing that mapping. Supported by Mooncake and NIXL UCX. Mapping keys are process-visible GPU IDs (after CUDA_VISIBLE_DEVICES); use HCA names without port suffixes for NIXL UCX. For NIXL UCX, a non-empty UCX_NET_DEVICES takes precedence over this option, including per-GPU mappings. Default is None, which preserves backend device selection.',
|
||||
] = None
|
||||
disaggregation_decode_enable_radix_cache: A[
|
||||
bool,
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import struct
|
||||
import threading
|
||||
import time
|
||||
@@ -604,30 +605,36 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
"with string keys and string values"
|
||||
)
|
||||
if backend == "UCX" and self.kv_args.ib_device:
|
||||
device_list = _resolve_ucx_device_list(
|
||||
self.kv_args.ib_device, self.kv_args.gpu_id
|
||||
)
|
||||
if device_list is not None:
|
||||
existing = backend_params.get("device_list")
|
||||
if (
|
||||
existing is not None
|
||||
and ", ".join(name.strip() for name in existing.split(","))
|
||||
!= device_list
|
||||
):
|
||||
raise ValueError(
|
||||
"Conflicting NIXL UCX device_list in "
|
||||
"SGLANG_DISAGGREGATION_NIXL_BACKEND_PARAMS and "
|
||||
"--disaggregation-ib-device. Configure only one "
|
||||
"device selection or make them agree."
|
||||
)
|
||||
backend_params["device_list"] = device_list
|
||||
logger.info(
|
||||
"NIXL UCX device binding: gpu_id=%s, engine_rank=%s, "
|
||||
"device_list=%s",
|
||||
self.kv_args.gpu_id,
|
||||
self.kv_args.engine_rank,
|
||||
device_list,
|
||||
if os.environ.get("UCX_NET_DEVICES"):
|
||||
logger.warning(
|
||||
"UCX_NET_DEVICES is set; ignoring --disaggregation-ib-device "
|
||||
"for NIXL UCX. Per-GPU CLI device mappings will not be applied."
|
||||
)
|
||||
else:
|
||||
device_list = _resolve_ucx_device_list(
|
||||
self.kv_args.ib_device, self.kv_args.gpu_id
|
||||
)
|
||||
if device_list is not None:
|
||||
existing = backend_params.get("device_list")
|
||||
if (
|
||||
existing is not None
|
||||
and ", ".join(name.strip() for name in existing.split(","))
|
||||
!= device_list
|
||||
):
|
||||
raise ValueError(
|
||||
"Conflicting NIXL UCX device_list in "
|
||||
"SGLANG_DISAGGREGATION_NIXL_BACKEND_PARAMS and "
|
||||
"--disaggregation-ib-device. Configure only one "
|
||||
"device selection or make them agree."
|
||||
)
|
||||
backend_params["device_list"] = device_list
|
||||
logger.info(
|
||||
"NIXL UCX device binding: gpu_id=%s, engine_rank=%s, "
|
||||
"device_list=%s",
|
||||
self.kv_args.gpu_id,
|
||||
self.kv_args.engine_rank,
|
||||
device_list,
|
||||
)
|
||||
# self.transfer_worker and self._start_bootstrap_thread runs concurrently
|
||||
# so we cannot use sync_mode=None which is thread-unsafe.
|
||||
agent_config = nixl_agent_config(
|
||||
|
||||
Reference in New Issue
Block a user