Fix RDMA device mapping for non-zero GPU indices in disaggregation tests (#21303)
Co-authored-by: Alison Shao <alison.shao@MacBook-Pro-D2W773R9CD.local> Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
co-authored by
Alison Shao
hnyls2002
parent
61a902ce88
commit
17e41cfb21
@@ -3234,9 +3234,15 @@ class ServerArgs:
|
|||||||
if len(devices) == 0:
|
if len(devices) == 0:
|
||||||
raise ValueError("No valid IB devices specified")
|
raise ValueError("No valid IB devices specified")
|
||||||
|
|
||||||
# Check for duplicates
|
# Deduplicate while preserving order
|
||||||
if len(devices) != len(set(devices)):
|
unique_devices = list(dict.fromkeys(devices))
|
||||||
raise ValueError(f"Duplicate IB devices specified: {device_str}")
|
if len(unique_devices) != len(devices):
|
||||||
|
logger.warning(
|
||||||
|
"Duplicate IB devices specified: %s. Deduplicating to: %s",
|
||||||
|
device_str,
|
||||||
|
",".join(unique_devices),
|
||||||
|
)
|
||||||
|
devices = unique_devices
|
||||||
|
|
||||||
# Get available IB devices from sysfs
|
# Get available IB devices from sysfs
|
||||||
ib_sysfs_path = "/sys/class/infiniband"
|
ib_sysfs_path = "/sys/class/infiniband"
|
||||||
|
|||||||
@@ -249,11 +249,13 @@ def get_rdma_devices_args():
|
|||||||
)
|
)
|
||||||
|
|
||||||
rdma_devices = []
|
rdma_devices = []
|
||||||
|
base_gpu = min(gpu_indices)
|
||||||
for gpu_idx in gpu_indices:
|
for gpu_idx in gpu_indices:
|
||||||
nic_index = min(gpu_idx // gpus_per_rdma, n_rdma - 1)
|
nic_index = min((gpu_idx - base_gpu) // gpus_per_rdma, n_rdma - 1)
|
||||||
rdma_devices.append(rdma_all_devices[nic_index])
|
rdma_devices.append(rdma_all_devices[nic_index])
|
||||||
|
|
||||||
if not rdma_devices:
|
if not rdma_devices:
|
||||||
return ",".join(_pick_default_pair(rdma_all_devices))
|
return ",".join(_pick_default_pair(rdma_all_devices))
|
||||||
|
|
||||||
return ",".join(rdma_devices)
|
# Deduplicate while preserving order
|
||||||
|
return ",".join(dict.fromkeys(rdma_devices))
|
||||||
|
|||||||
Reference in New Issue
Block a user