diff --git a/docs/docs/advanced_features/pd_disaggregation.mdx b/docs/docs/advanced_features/pd_disaggregation.mdx index b2abc92a1d..cf36d9ae59 100644 --- a/docs/docs/advanced_features/pd_disaggregation.mdx +++ b/docs/docs/advanced_features/pd_disaggregation.mdx @@ -585,6 +585,14 @@ python -m sglang.launch_server \ --port 30000 ``` +#### UCX device selection + +For NVIDIA CUDA and AMD ROCm deployments using NIXL UCX, `--disaggregation-ib-device` accepts a shared HCA list or a per-GPU JSON/file mapping. Mapping keys are process-visible GPU IDs; use HCA names without port suffixes, such as `mlx5_0`. + +A non-empty `UCX_NET_DEVICES` takes precedence over this CLI option. SGLang leaves the environment variable unchanged and skips the CLI mapping, including per-GPU mappings, with a warning. This preserves interfaces such as `lo` in an existing configuration like `rdma0:1,rdma4:1,lo`, which some RoCE hosts need for NIXL's self-endpoint. + +To use per-GPU CLI selection, unset `UCX_NET_DEVICES`; keep it configured on hosts that require the loopback entry. An explicit `device_list` in `SGLANG_DISAGGREGATION_NIXL_BACKEND_PARAMS` is left unchanged and still overrides `UCX_NET_DEVICES`; do not set it if you rely on the environment's complete device list. + ## ASCEND ### Usage diff --git a/python/sglang/srt/arg_groups/fields/disagg.py b/python/sglang/srt/arg_groups/fields/disagg.py index eab11e6cc7..8b3adf142c 100644 --- a/python/sglang/srt/arg_groups/fields/disagg.py +++ b/python/sglang/srt/arg_groups/fields/disagg.py @@ -80,7 +80,7 @@ class Disagg(msgspec.Struct): ] = 8998 disaggregation_ib_device: A[ Optional[str], - 'The InfiniBand devices for disaggregation transfer. Supports a single device (e.g., --disaggregation-ib-device mlx5_0), a shared comma-separated list (e.g., --disaggregation-ib-device mlx5_0,mlx5_1), a per-GPU JSON mapping (e.g., --disaggregation-ib-device \'{"0": "mlx5_0,mlx5_1", "1": "mlx5_2"}\'), or a path to a JSON file containing that mapping. Supported by Mooncake and NIXL UCX. Mapping keys are process-visible GPU IDs (after CUDA_VISIBLE_DEVICES); use HCA names without port suffixes for NIXL UCX. Default is None, which preserves backend device selection.', + 'The InfiniBand devices for disaggregation transfer. Supports a single device (e.g., --disaggregation-ib-device mlx5_0), a shared comma-separated list (e.g., --disaggregation-ib-device mlx5_0,mlx5_1), a per-GPU JSON mapping (e.g., --disaggregation-ib-device \'{"0": "mlx5_0,mlx5_1", "1": "mlx5_2"}\'), or a path to a JSON file containing that mapping. Supported by Mooncake and NIXL UCX. Mapping keys are process-visible GPU IDs (after CUDA_VISIBLE_DEVICES); use HCA names without port suffixes for NIXL UCX. For NIXL UCX, a non-empty UCX_NET_DEVICES takes precedence over this option, including per-GPU mappings. Default is None, which preserves backend device selection.', ] = None disaggregation_decode_enable_radix_cache: A[ bool, diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index 029478707a..a107f0f1a4 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -3,6 +3,7 @@ from __future__ import annotations import dataclasses import json import logging +import os import struct import threading import time @@ -604,30 +605,36 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager): "with string keys and string values" ) if backend == "UCX" and self.kv_args.ib_device: - device_list = _resolve_ucx_device_list( - self.kv_args.ib_device, self.kv_args.gpu_id - ) - if device_list is not None: - existing = backend_params.get("device_list") - if ( - existing is not None - and ", ".join(name.strip() for name in existing.split(",")) - != device_list - ): - raise ValueError( - "Conflicting NIXL UCX device_list in " - "SGLANG_DISAGGREGATION_NIXL_BACKEND_PARAMS and " - "--disaggregation-ib-device. Configure only one " - "device selection or make them agree." - ) - backend_params["device_list"] = device_list - logger.info( - "NIXL UCX device binding: gpu_id=%s, engine_rank=%s, " - "device_list=%s", - self.kv_args.gpu_id, - self.kv_args.engine_rank, - device_list, + if os.environ.get("UCX_NET_DEVICES"): + logger.warning( + "UCX_NET_DEVICES is set; ignoring --disaggregation-ib-device " + "for NIXL UCX. Per-GPU CLI device mappings will not be applied." ) + else: + device_list = _resolve_ucx_device_list( + self.kv_args.ib_device, self.kv_args.gpu_id + ) + if device_list is not None: + existing = backend_params.get("device_list") + if ( + existing is not None + and ", ".join(name.strip() for name in existing.split(",")) + != device_list + ): + raise ValueError( + "Conflicting NIXL UCX device_list in " + "SGLANG_DISAGGREGATION_NIXL_BACKEND_PARAMS and " + "--disaggregation-ib-device. Configure only one " + "device selection or make them agree." + ) + backend_params["device_list"] = device_list + logger.info( + "NIXL UCX device binding: gpu_id=%s, engine_rank=%s, " + "device_list=%s", + self.kv_args.gpu_id, + self.kv_args.engine_rank, + device_list, + ) # self.transfer_worker and self._start_bootstrap_thread runs concurrently # so we cannot use sync_mode=None which is thread-unsafe. agent_config = nixl_agent_config( diff --git a/test/registered/unit/disaggregation/test_nixl_backend_basic.py b/test/registered/unit/disaggregation/test_nixl_backend_basic.py index 1b1910e1c2..5684f2946f 100644 --- a/test/registered/unit/disaggregation/test_nixl_backend_basic.py +++ b/test/registered/unit/disaggregation/test_nixl_backend_basic.py @@ -1,5 +1,7 @@ """Basic CPU unit tests for NIXL disaggregation control paths.""" +import json +import os import sys import threading import types @@ -77,12 +79,23 @@ def _fake_staging_buffer_module(mock_gather=None): class TestNixlBackendInitialization(CustomTestCase): - def _initialize_manager(self, agent, device_module, mode, gpu_id): + def _initialize_manager( + self, + agent, + device_module, + mode, + gpu_id, + *, + backend="UCX", + ib_device=None, + backend_params=None, + ucx_net_devices="", + ): args = SimpleNamespace( pp_rank=0, engine_rank=0, gpu_id=gpu_id, - ib_device=None, + ib_device=ib_device, kv_data_ptrs=[], kv_item_lens=[], state_types=[], @@ -99,15 +112,18 @@ class TestNixlBackendInitialization(CustomTestCase): api.nixl_thread_sync_t = SimpleNamespace(NIXL_THREAD_SYNC_STRICT="strict") nixl = types.ModuleType("nixl") nixl._api = api - agent.get_plugin_list.return_value = ["UCX"] + agent.get_plugin_list.return_value = [backend] with ( patch.dict(sys.modules, {"nixl": nixl, "nixl._api": api}), patch.dict( "os.environ", { - "SGLANG_DISAGGREGATION_NIXL_BACKEND": "UCX", - "SGLANG_DISAGGREGATION_NIXL_BACKEND_PARAMS": "{}", + "SGLANG_DISAGGREGATION_NIXL_BACKEND": backend, + "SGLANG_DISAGGREGATION_NIXL_BACKEND_PARAMS": json.dumps( + backend_params or {} + ), + "UCX_NET_DEVICES": ucx_net_devices, "SGLANG_DISAGGREGATION_ENGINE_INIT_TIMEOUT": "5", "SGLANG_DISAGGREGATION_QUEUE_SIZE": "0", "SGLANG_DISAGG_STAGING_BUFFER": "false", @@ -131,6 +147,7 @@ class TestNixlBackendInitialization(CustomTestCase): self.assertIsNone(server_args.device) NixlKVManager.__init__(mgr, args, mode, server_args) get_device_module.assert_called_once_with("cuda") + self.assertEqual(os.environ["UCX_NET_DEVICES"], ucx_net_devices) def test_backend_initialization_selects_device_in_deadline_thread(self): caller_thread = threading.get_ident() @@ -166,6 +183,61 @@ class TestNixlBackendInitialization(CustomTestCase): {"num_threads": "8"} if mode == DisaggregationMode.PREFILL else {}, ) + def test_backend_initialization_respects_ucx_device_selection(self): + mapping = '{"1":"rdma0","3":"rdma4"}' + net_devices = "rdma0:1,rdma4:1,lo" + cases = ( + ("UCX", 1, mapping, "", {}, {"device_list": "rdma0"}), + ("UCX", 1, "rdma0,rdma4", "", {}, {"device_list": "rdma0, rdma4"}), + ("UCX", 3, mapping, "", {}, {"device_list": "rdma4"}), + ("UCX", 1, mapping, net_devices, {}, {}), + ("UCX", 3, mapping, net_devices, {}, {}), + ("UCX", 1, None, net_devices, {}, {}), + ( + "UCX", + 1, + mapping, + net_devices, + {"device_list": "rdma4"}, + {"device_list": "rdma4"}, + ), + ("LIBFABRIC", 1, mapping, net_devices, {}, {}), + ) + for backend, gpu_id, ib_device, net, params, expected in cases: + with ( + self.subTest(backend=backend, gpu_id=gpu_id, net=net, params=params), + patch("sglang.srt.disaggregation.nixl.conn.logger.warning") as warning, + ): + agent = MagicMock() + self._initialize_manager( + agent, + MagicMock(), + DisaggregationMode.DECODE, + gpu_id, + backend=backend, + ib_device=ib_device, + backend_params=params, + ucx_net_devices=net, + ) + agent.create_backend.assert_called_once_with(backend, expected) + if backend == "UCX" and ib_device and net: + warning.assert_called_once() + self.assertIn( + "Per-GPU CLI device mappings", warning.call_args[0][0] + ) + else: + warning.assert_not_called() + + with self.assertRaisesRegex(ValueError, "Conflicting NIXL UCX device_list"): + self._initialize_manager( + MagicMock(), + MagicMock(), + DisaggregationMode.DECODE, + 1, + ib_device=mapping, + backend_params={"device_list": "rdma4"}, + ) + def test_backend_initialization_propagates_error(self): error = RuntimeError("backend initialization failed") agent = MagicMock()