Size the VMM graph-input exchange by the widest input across ranks (#42125)
Co-authored-by: merrymercy <merrymercy@users.noreply.github.com> Co-authored-by: oulgen <oulgen@users.noreply.github.com>
This commit is contained in:
co-authored by
merrymercy
oulgen
parent
4a4dad1cf1
commit
354ed2a7b6
@@ -989,7 +989,6 @@ class VmmGraphInputManager:
|
||||
"""
|
||||
FABRIC_HANDLE_BYTES = 64
|
||||
MAX_VMM_BASES = 4096
|
||||
MAX_CHUNKS_PER_INPUT = 16
|
||||
|
||||
t0 = time.perf_counter()
|
||||
|
||||
@@ -1025,12 +1024,15 @@ class VmmGraphInputManager:
|
||||
local_input_chunks = [
|
||||
[int(idx) for idx in indices] for indices in input_chunk_indices
|
||||
]
|
||||
for chunks in local_input_chunks:
|
||||
if len(chunks) > MAX_CHUNKS_PER_INPUT:
|
||||
raise RuntimeError(
|
||||
"Too many VMM chunks for graph input: "
|
||||
f"{len(chunks)} > {MAX_CHUNKS_PER_INPUT}"
|
||||
)
|
||||
# Every rank must pack the same input struct, so the per-input span
|
||||
# width is the widest input on any rank (floored at 16 slots).
|
||||
widest_input = torch.tensor(
|
||||
[max((len(c) for c in local_input_chunks), default=0)],
|
||||
dtype=torch.int64,
|
||||
device="cpu",
|
||||
)
|
||||
dist.all_reduce(widest_input, op=dist.ReduceOp.MAX, group=self.group)
|
||||
max_chunks_per_input = max(16, int(widest_input.item()))
|
||||
|
||||
# All-gather base metadata and per-input VMM spans. A captured tensor
|
||||
# can cross expandable-segment allocation boundaries, so peer mappings
|
||||
@@ -1041,7 +1043,7 @@ class VmmGraphInputManager:
|
||||
base_struct = struct.Struct(
|
||||
f"<QQ{FABRIC_HANDLE_BYTES}s" if use_fabric else "<QQ"
|
||||
)
|
||||
input_struct = struct.Struct(f"<QQ{MAX_CHUNKS_PER_INPUT}Q")
|
||||
input_struct = struct.Struct(f"<QQ{max_chunks_per_input}Q")
|
||||
base_offset = header_struct.size
|
||||
input_offset = base_offset + MAX_VMM_BASES * base_struct.size
|
||||
payload_size = input_offset + new_count * input_struct.size
|
||||
@@ -1067,7 +1069,7 @@ class VmmGraphInputManager:
|
||||
for i, (chunks, offset) in enumerate(
|
||||
zip(local_input_chunks, input_offsets)
|
||||
):
|
||||
padded_chunks = chunks + [0] * (MAX_CHUNKS_PER_INPUT - len(chunks))
|
||||
padded_chunks = chunks + [0] * (max_chunks_per_input - len(chunks))
|
||||
input_struct.pack_into(
|
||||
local_payload,
|
||||
input_offset + i * input_struct.size,
|
||||
|
||||
Reference in New Issue
Block a user