diff --git a/python/sglang/test/server_fixtures/disaggregation_fixture.py b/python/sglang/test/server_fixtures/disaggregation_fixture.py index 7bdf57cec8..2eab0c3e7b 100644 --- a/python/sglang/test/server_fixtures/disaggregation_fixture.py +++ b/python/sglang/test/server_fixtures/disaggregation_fixture.py @@ -1,3 +1,4 @@ +import ctypes import io import logging import os @@ -96,6 +97,7 @@ class PDDisaggregationServerBase(CustomTestCase): cls._ucx_net_devices_set = False if is_in_ci(): cls.transfer_backend = ["--disaggregation-transfer-backend", "mooncake"] + _check_rdma_openable() ib_devices = get_rdma_devices_args() cls.rdma_devices = ["--disaggregation-ib-device", ib_devices] cls._mc_gid_index_set = _maybe_set_roce_gid_index(ib_devices) @@ -404,6 +406,40 @@ def get_rdma_devices_args(gpu_indices=None) -> str: _IB_SYSFS = "/sys/class/infiniband" +# None means libibverbs is not loadable, not that no device is openable. +def _ibverbs_device_names() -> Optional[set]: + try: + lib = ctypes.CDLL("libibverbs.so.1") + except OSError: + return None + lib.ibv_get_device_list.restype = ctypes.POINTER(ctypes.c_void_p) + lib.ibv_get_device_list.argtypes = [ctypes.POINTER(ctypes.c_int)] + lib.ibv_get_device_name.restype = ctypes.c_char_p + lib.ibv_get_device_name.argtypes = [ctypes.c_void_p] + lib.ibv_free_device_list.argtypes = [ctypes.POINTER(ctypes.c_void_p)] + num = ctypes.c_int(0) + devices = lib.ibv_get_device_list(ctypes.byref(num)) + if not devices: + return set() + try: + return {lib.ibv_get_device_name(devices[i]).decode() for i in range(num.value)} + finally: + lib.ibv_free_device_list(devices) + + +# Without an openable device the PD transfer backend silently falls back to TCP, +# and transfers then fail under load instead of at setup. +def _check_rdma_openable() -> None: + if not _get_available_ib_devices(): + return + if _ibverbs_device_names() == set(): + raise RuntimeError( + f"Active RDMA devices are listed in {_IB_SYSFS} but ibverbs can open " + "none; the PD transfer backend would lose RDMA. " + "Check /dev/infiniband in the container." + ) + + def _roce_v2_gid_index(device: str): """Return a RoCEv2 GID index for a device, preferring a global (routable) GID over a link-local (fe80::) one, or None if the device has no RoCEv2 GID.