diff --git a/python/sglang/srt/utils/cuda_vmm_utils.py b/python/sglang/srt/utils/cuda_vmm_utils.py index 8f4cfe9ca0..c4bf4c2a5c 100644 --- a/python/sglang/srt/utils/cuda_vmm_utils.py +++ b/python/sglang/srt/utils/cuda_vmm_utils.py @@ -989,7 +989,6 @@ class VmmGraphInputManager: """ FABRIC_HANDLE_BYTES = 64 MAX_VMM_BASES = 4096 - MAX_CHUNKS_PER_INPUT = 16 t0 = time.perf_counter() @@ -1025,12 +1024,15 @@ class VmmGraphInputManager: local_input_chunks = [ [int(idx) for idx in indices] for indices in input_chunk_indices ] - for chunks in local_input_chunks: - if len(chunks) > MAX_CHUNKS_PER_INPUT: - raise RuntimeError( - "Too many VMM chunks for graph input: " - f"{len(chunks)} > {MAX_CHUNKS_PER_INPUT}" - ) + # Every rank must pack the same input struct, so the per-input span + # width is the widest input on any rank (floored at 16 slots). + widest_input = torch.tensor( + [max((len(c) for c in local_input_chunks), default=0)], + dtype=torch.int64, + device="cpu", + ) + dist.all_reduce(widest_input, op=dist.ReduceOp.MAX, group=self.group) + max_chunks_per_input = max(16, int(widest_input.item())) # All-gather base metadata and per-input VMM spans. A captured tensor # can cross expandable-segment allocation boundaries, so peer mappings @@ -1041,7 +1043,7 @@ class VmmGraphInputManager: base_struct = struct.Struct( f" exchange -> import/map across ranks for both transports (POSIX, FABRIC) and both mapping shapes (single base, multi-chunk span). The only in-tree consumer, ``register_graph_inputs``, reaches this path only under -``expandable_segments``, so the tests allocate shareable buffers directly. A +``expandable_segments``, so the tests allocate shareable buffers directly and +drive ``register_graph_inputs`` itself over hand-mapped spans. A POSIX-only allocation forces ``export_shareable_handles`` down its POSIX fallback (otherwise unreachable on FABRIC hardware); FABRIC cases need an NVLink fabric (GB200/GB300) and skip elsewhere. @@ -23,6 +24,7 @@ from cuda.bindings import driver as drv from sglang.kernels.jit.utils import cache_once from sglang.srt.utils import cuda_vmm_utils from sglang.srt.utils.cuda_vmm_utils import ( + VmmGraphInputManager, check_drv, exchange_posix_fds, export_shareable_handles, @@ -308,5 +310,93 @@ def test_handle_roundtrip(transport: str, n_chunks: int) -> None: check_drv(drv.cuMemRelease(handle), "cuMemRelease") +class _GraphInputHolder: + """Collective stand-in: serves one pre-mapped graph input and records the + per-rank pointers ``register_graph_inputs`` registers for it.""" + + def __init__(self, bases_info, chunk_indices, offsets) -> None: + self._bases = (bases_info, chunk_indices, offsets) + self.peer_ptrs = None + + def get_graph_capture_bases(self): + return self._bases + + def register_peer_mapped_inputs(self, peer_ptrs) -> None: + self.peer_ptrs = peer_ptrs + + +def _create_span(handle_type, n_chunks: int, chunk_hint: int): + """One reserved VA range backed by ``n_chunks`` separate allocations mapped + back to back, as expandable segments lay out a tensor that outgrows one + chunk. Returns (handles, span_va, chunk_size).""" + device_id = torch.cuda.current_device() + prop = _make_prop(handle_type, device_id) + gran = check_drv( + drv.cuMemGetAllocationGranularity(prop, _RECOMMENDED), + "cuMemGetAllocationGranularity", + ) + size = ((chunk_hint + gran - 1) // gran) * gran + span_va = int( + check_drv( + drv.cuMemAddressReserve(size * n_chunks, gran, 0, 0), + "cuMemAddressReserve", + ) + ) + handles = [] + for chunk in range(n_chunks): + handle = check_drv(drv.cuMemCreate(size, prop, 0), "cuMemCreate") + va = span_va + chunk * size + check_drv(drv.cuMemMap(va, size, 0, handle, 0), "cuMemMap") + check_drv( + drv.cuMemSetAccess(va, size, [make_rw_access_desc(device_id)], 1), + "cuMemSetAccess", + ) + handles.append(handle) + return handles, span_va, size + + +def test_register_graph_inputs_follows_widest_input() -> None: + """A graph input covering more than 16 allocator chunks on one rank must + still register on every rank: the exchanged per-input struct is sized to + the widest input across ranks, not to a fixed 16 slots.""" + group = _gloo_group() + rank = dist.get_rank(group) + world = dist.get_world_size(group) + chunks_on = [20 if peer % 2 else 1 for peer in range(world)] + handles, span_va, size = _create_span(_POSIX_FD, chunks_on[rank], _ALLOC_BYTES) + for chunk in range(chunks_on[rank]): + check_drv( + drv.cuMemsetD8(span_va + chunk * size, _byte(rank, chunk), size), + "cuMemsetD8", + ) + torch.cuda.synchronize() + + holder = _GraphInputHolder( + bases_info=[(span_va + chunk * size, size) for chunk in range(len(handles))], + chunk_indices=[list(range(len(handles)))], + offsets=[0], + ) + manager = VmmGraphInputManager(obj=holder, group=group, rank=rank, world_size=world) + try: + manager.register_graph_inputs() + (row,) = holder.peer_ptrs + assert row[rank] == span_va + for peer in range(world): + if peer == rank: + continue + for chunk in range(chunks_on[peer]): + _assert_region( + row[peer] + chunk * size, _byte(peer, chunk), peer, chunk + ) + finally: + manager.close() + for chunk, handle in enumerate(handles): + check_drv(drv.cuMemUnmap(span_va + chunk * size, size), "cuMemUnmap") + check_drv(drv.cuMemRelease(handle), "cuMemRelease") + check_drv( + drv.cuMemAddressFree(span_va, size * len(handles)), "cuMemAddressFree" + ) + + if __name__ == "__main__": multigpu_pytest_main(__name__, __file__, num_gpus=(2,))