Fix RDMA device mapping for non-zero GPU indices in disaggregation tests (#21303)
Co-authored-by: Alison Shao <alison.shao@MacBook-Pro-D2W773R9CD.local> Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
co-authored by
Alison Shao
hnyls2002
parent
61a902ce88
commit
17e41cfb21
@@ -3234,9 +3234,15 @@ class ServerArgs:
|
||||
if len(devices) == 0:
|
||||
raise ValueError("No valid IB devices specified")
|
||||
|
||||
# Check for duplicates
|
||||
if len(devices) != len(set(devices)):
|
||||
raise ValueError(f"Duplicate IB devices specified: {device_str}")
|
||||
# Deduplicate while preserving order
|
||||
unique_devices = list(dict.fromkeys(devices))
|
||||
if len(unique_devices) != len(devices):
|
||||
logger.warning(
|
||||
"Duplicate IB devices specified: %s. Deduplicating to: %s",
|
||||
device_str,
|
||||
",".join(unique_devices),
|
||||
)
|
||||
devices = unique_devices
|
||||
|
||||
# Get available IB devices from sysfs
|
||||
ib_sysfs_path = "/sys/class/infiniband"
|
||||
|
||||
@@ -249,11 +249,13 @@ def get_rdma_devices_args():
|
||||
)
|
||||
|
||||
rdma_devices = []
|
||||
base_gpu = min(gpu_indices)
|
||||
for gpu_idx in gpu_indices:
|
||||
nic_index = min(gpu_idx // gpus_per_rdma, n_rdma - 1)
|
||||
nic_index = min((gpu_idx - base_gpu) // gpus_per_rdma, n_rdma - 1)
|
||||
rdma_devices.append(rdma_all_devices[nic_index])
|
||||
|
||||
if not rdma_devices:
|
||||
return ",".join(_pick_default_pair(rdma_all_devices))
|
||||
|
||||
return ",".join(rdma_devices)
|
||||
# Deduplicate while preserving order
|
||||
return ",".join(dict.fromkeys(rdma_devices))
|
||||
|
||||
Reference in New Issue
Block a user