[Test] Fail fast when PD test RDMA devices are not openable by ibverbs (#41600)
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
import ctypes
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
@@ -96,6 +97,7 @@ class PDDisaggregationServerBase(CustomTestCase):
|
||||
cls._ucx_net_devices_set = False
|
||||
if is_in_ci():
|
||||
cls.transfer_backend = ["--disaggregation-transfer-backend", "mooncake"]
|
||||
_check_rdma_openable()
|
||||
ib_devices = get_rdma_devices_args()
|
||||
cls.rdma_devices = ["--disaggregation-ib-device", ib_devices]
|
||||
cls._mc_gid_index_set = _maybe_set_roce_gid_index(ib_devices)
|
||||
@@ -404,6 +406,40 @@ def get_rdma_devices_args(gpu_indices=None) -> str:
|
||||
_IB_SYSFS = "/sys/class/infiniband"
|
||||
|
||||
|
||||
# None means libibverbs is not loadable, not that no device is openable.
|
||||
def _ibverbs_device_names() -> Optional[set]:
|
||||
try:
|
||||
lib = ctypes.CDLL("libibverbs.so.1")
|
||||
except OSError:
|
||||
return None
|
||||
lib.ibv_get_device_list.restype = ctypes.POINTER(ctypes.c_void_p)
|
||||
lib.ibv_get_device_list.argtypes = [ctypes.POINTER(ctypes.c_int)]
|
||||
lib.ibv_get_device_name.restype = ctypes.c_char_p
|
||||
lib.ibv_get_device_name.argtypes = [ctypes.c_void_p]
|
||||
lib.ibv_free_device_list.argtypes = [ctypes.POINTER(ctypes.c_void_p)]
|
||||
num = ctypes.c_int(0)
|
||||
devices = lib.ibv_get_device_list(ctypes.byref(num))
|
||||
if not devices:
|
||||
return set()
|
||||
try:
|
||||
return {lib.ibv_get_device_name(devices[i]).decode() for i in range(num.value)}
|
||||
finally:
|
||||
lib.ibv_free_device_list(devices)
|
||||
|
||||
|
||||
# Without an openable device the PD transfer backend silently falls back to TCP,
|
||||
# and transfers then fail under load instead of at setup.
|
||||
def _check_rdma_openable() -> None:
|
||||
if not _get_available_ib_devices():
|
||||
return
|
||||
if _ibverbs_device_names() == set():
|
||||
raise RuntimeError(
|
||||
f"Active RDMA devices are listed in {_IB_SYSFS} but ibverbs can open "
|
||||
"none; the PD transfer backend would lose RDMA. "
|
||||
"Check /dev/infiniband in the container."
|
||||
)
|
||||
|
||||
|
||||
def _roce_v2_gid_index(device: str):
|
||||
"""Return a RoCEv2 GID index for a device, preferring a global (routable)
|
||||
GID over a link-local (fe80::) one, or None if the device has no RoCEv2 GID.
|
||||
|
||||
Reference in New Issue
Block a user