From 79701a2549c3cf9935d41d753637e7443d016bfd Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Fri, 23 Oct 2026 19:01:26 -0700 Subject: [PATCH] [Perf] dsv41: Read Engram rows from peer device shards (#46180) Co-authored-by: kpham-sgl --- .../sglang/kernels/ops/embeddings/__init__.py | 8 + .../kernels/ops/embeddings/engram_gather.py | 89 ++++++++ python/sglang/srt/environ.py | 5 + python/sglang/srt/layers/engram.py | 184 +++++++++++++++- .../sglang/srt/layers/moe/dwdp/transport.py | 32 +-- python/sglang/srt/utils/cuda_vmm_utils.py | 44 ++++ test/manual/layers/test_engram_peer_shards.py | 196 ++++++++++++++++++ 7 files changed, 532 insertions(+), 26 deletions(-) create mode 100644 test/manual/layers/test_engram_peer_shards.py diff --git a/python/sglang/kernels/ops/embeddings/__init__.py b/python/sglang/kernels/ops/embeddings/__init__.py index d2f339e864..1db26b87e4 100644 --- a/python/sglang/kernels/ops/embeddings/__init__.py +++ b/python/sglang/kernels/ops/embeddings/__init__.py @@ -22,6 +22,14 @@ register_kernel( ) ) +register_kernel( + KernelSpec( + op="embeddings.engram_gather_peer", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.embeddings.engram_gather:engram_gather_peer", + ) +) + register_kernel( KernelSpec( op="embeddings.engram_hash_ids", diff --git a/python/sglang/kernels/ops/embeddings/engram_gather.py b/python/sglang/kernels/ops/embeddings/engram_gather.py index 0e2bd618b2..2e6e1c090d 100644 --- a/python/sglang/kernels/ops/embeddings/engram_gather.py +++ b/python/sglang/kernels/ops/embeddings/engram_gather.py @@ -96,3 +96,92 @@ def engram_gather( HAS_DST=dst is not None, ) return out + + +@triton.jit +def _engram_gather_peer_kernel( + ids_ptr, + ids_stride, + out_ptr, + w_bias_ptr, + s_bias_ptr, + starts_ptr, + n_cols, + DIM: tl.constexpr, + BLK: tl.constexpr, + ROWS: tl.constexpr, + LOG2_WORLD: tl.constexpr, + E8M0_ZERO: tl.constexpr, +): + tok = tl.program_id(0).to(tl.int64) + cols = tl.program_id(1) * ROWS + tl.arange(0, ROWS) + col_ok = cols < n_cols + idx = tl.load(ids_ptr + tok * ids_stride + cols, mask=col_ok, other=0).to(tl.int64) + # Owner = the last shard whose first row is <= idx; starts is padded to a + # power of two with int64 max, so the search never runs past a real shard. + owner = tl.zeros([ROWS], dtype=tl.int32) + for step in tl.static_range(LOG2_WORLD): + probe = owner + (1 << (LOG2_WORLD - 1 - step)) + start = tl.load(starts_ptr + probe) + owner = tl.where(start <= idx, probe, owner) + # Bases are pre-biased by each shard's first row, so a global id indexes them. + w_row = tl.load(w_bias_ptr + owner) + idx * DIM + s_row = tl.load(s_bias_ptr + owner) + idx * (DIM // BLK) + offs = tl.arange(0, DIM) + w = (w_row[:, None] + offs[None, :]).to(tl.pointer_type(tl.float8e4nv)) + s = (s_row[:, None] + (offs // BLK)[None, :]).to(tl.pointer_type(tl.uint8)) + mask = col_ok[:, None] + vals = tl.load(w, mask=mask, other=0.0).to(tl.float32) + exps = tl.load(s, mask=mask, other=0).to(tl.int32) + scale = (exps << 23).to(tl.float32, bitcast=True) + scale = tl.where(exps == 0, E8M0_ZERO, scale) + # -0 becomes +0, as the zero-fill + sum of the sharded lookup leaves it, so the + # two lookups agree bit for bit. + out = vals * scale + out = tl.where(out == 0.0, 0.0, out) + dst = out_ptr + (tok * n_cols + cols)[:, None].to(tl.int64) * DIM + offs[None, :] + tl.store(dst, out.to(tl.bfloat16), mask=mask) + + +def engram_gather_peer( + ids: torch.Tensor, + out: torch.Tensor, + w_bias: torch.Tensor, + s_bias: torch.Tensor, + starts: torch.Tensor, + dim: int, + block_size: int, +) -> torch.Tensor: + """Gather ``ids`` ([T, C] int, unit column stride) into ``out`` ([T, C, dim] + bf16, contiguous), reading each id from the shard that owns it. + + ``starts`` ([P] int64, P a power of two) holds each shard's first global row, + padded with int64 max; ``w_bias`` / ``s_bias`` ([P] int64) are each shard's fp8 + row / e8m0 scale base address minus its first row's offset, so any rank's + shard (local or a peer mapping over NVLink) is addressed by the global id. + """ + assert ids.dim() == 2 and ids.stride(1) == 1, (ids.shape, ids.stride()) + assert ids.dtype in (torch.int32, torch.int64) + assert out.is_contiguous() and out.dtype == torch.bfloat16 + assert out.shape == (*ids.shape, dim), (out.shape, ids.shape) + world = starts.numel() + assert world & (world - 1) == 0, world + t, c = ids.shape + if t and c: + rows = 8 + _engram_gather_peer_kernel[(t, triton.cdiv(c, rows))]( + ids, + ids.stride(0), + out, + w_bias, + s_bias, + starts, + c, + DIM=dim, + BLK=block_size, + ROWS=rows, + LOG2_WORLD=world.bit_length() - 1, + E8M0_ZERO=_E8M0_ZERO, + num_warps=4, + ) + return out diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index d5d71bda98..4510e77d9c 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1777,6 +1777,11 @@ class Envs: # Scheduler-side staging of shared host-table rows on GPUs without ATS # (engram_staging.py); pays off for large prefill batches. SGLANG_OPT_DSV41_ENGRAM_HOST_STAGING = EnvBool(False) + # Keep each rank's engram row shard in exportable device memory (CUDA VMM, + # FABRIC handles across an NVLink domain) mapped by every rank of the TP group: + # a rank reads its own tokens' rows from the owning shard, with no id gather + # and no reduce-scatter. Takes precedence over the host table. + SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS = EnvBool(False) # Kernels and indexer SGLANG_OPT_DEEPGEMM_HC_PRENORM = EnvBool(True) diff --git a/python/sglang/srt/layers/engram.py b/python/sglang/srt/layers/engram.py index 85c6672599..d32f9a5dc2 100644 --- a/python/sglang/srt/layers/engram.py +++ b/python/sglang/srt/layers/engram.py @@ -23,7 +23,10 @@ import torch from torch import nn from sglang.kernels.ops.embeddings.engram_gate import fused_engram_gate -from sglang.kernels.ops.embeddings.engram_gather import engram_gather +from sglang.kernels.ops.embeddings.engram_gather import ( + engram_gather, + engram_gather_peer, +) from sglang.kernels.ops.embeddings.engram_hash import ( MODE_DECODE, MODE_EXTEND, @@ -795,6 +798,161 @@ class _HostTable: logger.info(msg) +@functools.cache +def _warn_host_table_ignored() -> None: + logger.warning( + "SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS takes precedence: the engram tables " + "stay in device memory and SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE is ignored" + ) + + +class _PeerShards: + """One rank's engram row shard in exportable device memory (CUDA VMM), with + every TP peer's shard mapped into this process after `connect`. + + Inside one NVLink domain (MNNVL included, through FABRIC handles) a kernel + reads any peer's rows in place, so a rank looks up only its own tokens: no + id gather, no [DP x tokens] buffer, no reduce-scatter. The shard holds the + fp8 rows followed by their e8m0 scales. Lives for the whole process. + """ + + def __init__(self, rows: int, dim: int, group): + from sglang.srt.utils.cuda_vmm_utils import ( + VmmReservation, + align_up, + get_device_granularity, + make_device_allocation_prop, + ) + + self.rows = rows + self.dim = dim + self.group = group + self.device_id = torch.cuda.current_device() + self.w_bytes = rows * dim + self.s_bytes = rows * (dim // FP8_BLOCK_SIZE) + gran = get_device_granularity(self.device_id) + self.size = align_up(max(1, self.w_bytes + self.s_bytes), gran) + self.local = VmmReservation( + self.size, + make_device_allocation_prop(self.device_id), + self.device_id, + alignment=gran, + ) + self.handle = self.local.map(0, self.size, retain_handle=True) + self.base = self.local.base + self.w_bias: Optional[torch.Tensor] = None + self.s_bias: Optional[torch.Tensor] = None + self.starts: Optional[torch.Tensor] = None + + def views(self) -> tuple[torch.Tensor, torch.Tensor]: + from sglang.srt.utils.cuda_vmm_utils import tensor_from_pointer + + raw = tensor_from_pointer(self.base, self.size, device_id=self.device_id) + weight = raw[: self.w_bytes].view(torch.float8_e4m3fn).view(self.rows, self.dim) + scale = ( + raw[self.w_bytes : self.w_bytes + self.s_bytes] + .view(torch.float8_e8m0fnu) + .view(self.rows, self.dim // FP8_BLOCK_SIZE) + ) + return weight, scale + + @property + def ready(self) -> bool: + return self.starts is not None + + def connect(self, row_start: int, label: str) -> None: + """Collective over the TP group: export this shard, map every peer's.""" + if self.ready: + return + import torch.distributed as dist + + from sglang.srt.utils.cuda_vmm_utils import ( + exchange_shareable_handles, + import_and_map_alloc, + ) + + t0 = time.perf_counter() + torch.cuda.synchronize() + cpu_group = self.group.cpu_group + rank, world = self.group.rank_in_group, self.group.world_size + use_fabric, peer_handles = exchange_shareable_handles( + [self.handle], cpu_group, rank, world + ) + meta = [None] * world + dist.all_gather_object(meta, (row_start, self.rows, self.size), group=cpu_group) + bases = [] + try: + for r, (_, _, size) in enumerate(meta): + if r == rank: + bases.append(self.base) + continue + handle = peer_handles[(r, 0)] + bases.append( + import_and_map_alloc( + handle if use_fabric else None, + None if use_fabric else handle, + size, + self.device_id, + use_fabric=use_fabric, + peer_rank=r, + ) + ) + finally: + if not use_fabric: + for fd in peer_handles.values(): + try: + os.close(fd) + except OSError: + pass + padded = 1 << max(0, (world - 1).bit_length()) + int64_max = torch.iinfo(torch.int64).max + starts, w_bias, s_bias = [], [], [] + n_scales = self.dim // FP8_BLOCK_SIZE + for r in range(padded): + if r < world: + start, rows = meta[r][0], meta[r][1] + starts.append(start) + w_bias.append(bases[r] - start * self.dim) + s_bias.append(bases[r] + rows * self.dim - start * n_scales) + else: + starts.append(int64_max) + w_bias.append(0) + s_bias.append(0) + assert starts[0] == 0 and starts[:world] == sorted(starts[:world]), starts + device = torch.device("cuda", self.device_id) + self.w_bias = torch.tensor(w_bias, dtype=torch.int64, device=device) + self.s_bias = torch.tensor(s_bias, dtype=torch.int64, device=device) + self.starts = torch.tensor(starts, dtype=torch.int64, device=device) + torch.cuda.synchronize() + dist.barrier(group=cpu_group) + logger.info( + "engram peer shards %s: %d ranks over %s, %.2f GiB local, mapped in %.1f s", + label, + world, + "FABRIC" if use_fabric else "POSIX fd", + self.size / 2**30, + time.perf_counter() - t0, + ) + + def lookup(self, indices: torch.Tensor) -> torch.Tensor: + flat = indices.reshape(-1, indices.shape[-1]) + if flat.stride(-1) != 1: + flat = flat.contiguous() + out = torch.empty( + *flat.shape, self.dim, dtype=torch.bfloat16, device=indices.device + ) + engram_gather_peer( + flat, + out, + self.w_bias, + self.s_bias, + self.starts, + self.dim, + FP8_BLOCK_SIZE, + ) + return out.view(*indices.shape, self.dim) + + def shard_range( num_embeddings: int, column_bounds: tuple[int, ...], @@ -848,7 +1006,19 @@ class EngramEmbedding(nn.Module): self.host_table: Optional[_HostTable] = None self._sort_host_reads = False self.staged_by_scheduler = False - if envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get(): + self.peer: Optional[_PeerShards] = None + if ( + envs.SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS.get() + and self.tp_size > 1 + and is_cuda() + ): + if envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get() and tp_rank == 0: + _warn_host_table_ignored() + self.peer = _PeerShards(self.rows, dim, get_parallel().tp_group) + weight, scale = self.peer.views() + self.weight = nn.Parameter(weight, requires_grad=False) + self.scale = nn.Parameter(scale, requires_grad=False) + elif envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get(): self._init_host_table(num_embeddings, dim, layer_id) else: self.weight = nn.Parameter( @@ -914,7 +1084,10 @@ class EngramEmbedding(nn.Module): self.host_table.dirty = True def finish_load(self, label: str): - """Collective in the shared layout: every rank calls it after loading.""" + """Collective in the shared layout and with peer shards: every rank calls + it after loading.""" + if self.peer is not None: + self.peer.connect(self.row_start, label) if self.host_table is not None: self.host_table.finish_load(label) @@ -925,6 +1098,11 @@ class EngramEmbedding(nn.Module): *, cp_all_tokens: bool = False, ) -> torch.Tensor: + if self.peer is not None: + assert self.peer.ready, "engram peer shards are mapped in finish_load" + # Every shard is mapped here: read this rank's own tokens only, which + # also covers the prefill CP and DP paths below without a collective. + return self.peer.lookup(indices) if self._shared: if indices.shape[0] == 0: return self._empty(indices) diff --git a/python/sglang/srt/layers/moe/dwdp/transport.py b/python/sglang/srt/layers/moe/dwdp/transport.py index e325160ec2..81c20c95dc 100644 --- a/python/sglang/srt/layers/moe/dwdp/transport.py +++ b/python/sglang/srt/layers/moe/dwdp/transport.py @@ -21,8 +21,7 @@ from sglang.srt.utils.cuda_vmm_utils import ( align_down, align_up, check_drv, - exchange_posix_fds, - export_shareable_handles, + exchange_shareable_handles, get_device_granularity, import_peer_handle, make_device_allocation_prop, @@ -138,11 +137,6 @@ class DWDPTransport: prop = make_device_allocation_prop(device_id) handle_list = [self._handle_set.get_handle(li, n) for li, n in sorted_keys] - fabric_handles, local_posix_fds, use_fabric = export_shareable_handles( - handle_list, cpu_group, layout.dwdp_rank - ) - peer_fds: Dict[Tuple[int, int], int] = {} - key_counts = [None] * layout.dwdp_size dist.all_gather_object(key_counts, len(sorted_keys), group=cpu_group) if any(count != len(sorted_keys) for count in key_counts): @@ -150,18 +144,9 @@ class DWDPTransport: f"Mismatched DWDP weight handle counts across ranks: {key_counts}" ) - if use_fabric: - all_fabric = [None] * layout.dwdp_size - dist.all_gather_object(all_fabric, fabric_handles, group=cpu_group) - else: - all_fabric = None - peer_fds = exchange_posix_fds( - cpu_group, - layout.dwdp_rank, - layout.dwdp_size, - local_posix_fds, - key_counts, - ) + use_fabric, peer_handles = exchange_shareable_handles( + handle_list, cpu_group, layout.dwdp_rank, layout.dwdp_size + ) logger.info( "DWDP handle exchange via %s (%d handles)", "FABRIC" if use_fabric else "POSIX fd", @@ -175,8 +160,9 @@ class DWDPTransport: if peer_rank == layout.dwdp_rank: continue - fabric_handle = all_fabric[peer_rank][key_idx] if use_fabric else None - fd = None if use_fabric else peer_fds[(peer_rank, key_idx)] + handle = peer_handles[(peer_rank, key_idx)] + fabric_handle = handle if use_fabric else None + fd = None if use_fabric else handle peer_handle = import_peer_handle( fabric_handle, fd, use_fabric=use_fabric, peer_rank=peer_rank ) @@ -209,8 +195,8 @@ class DWDPTransport: ) self._peer_views[(peer_rank, layer_idx, name)] = peer_tensor - _close_fds(local_posix_fds) - _close_fds(peer_fds.values()) + if not use_fabric: + _close_fds(peer_handles.values()) @property def handle_set(self) -> MnnvlHandleSet: diff --git a/python/sglang/srt/utils/cuda_vmm_utils.py b/python/sglang/srt/utils/cuda_vmm_utils.py index c4bf4c2a5c..746a5fe2c2 100644 --- a/python/sglang/srt/utils/cuda_vmm_utils.py +++ b/python/sglang/srt/utils/cuda_vmm_utils.py @@ -880,6 +880,50 @@ def exchange_posix_fds( pass +def exchange_shareable_handles( + retained_handles, group: ProcessGroup, rank: int, world_size: int +): + """Export this rank's retained VMM handles and collect every peer's. + + FABRIC handles when every rank can export them; otherwise POSIX fds over UNIX + sockets, which only reach ranks on this host, so a group spanning hosts raises. + Returns ``(use_fabric, peer_handles)``: ``peer_handles[(peer_rank, idx)]`` is a + FABRIC handle or a received fd the caller must close. This rank's exported fds + are closed here. + """ + fabric_handles, local_fds, use_fabric = export_shareable_handles( + retained_handles, group, rank + ) + if use_fabric: + all_fabric = [None] * world_size + dist.all_gather_object(all_fabric, fabric_handles, group=group) + return True, { + (r, i): handle + for r, handles in enumerate(all_fabric) + if r != rank + for i, handle in enumerate(handles) + } + try: + info = [None] * world_size + dist.all_gather_object( + info, (socket.gethostname(), len(local_fds)), group=group + ) + if len({host for host, _ in info}) > 1: + raise RuntimeError( + "VMM handles cannot reach ranks on other hosts without FABRIC " + "handles (one NVLink domain with IMEX)" + ) + return False, exchange_posix_fds( + group, rank, world_size, local_fds, [n for _, n in info] + ) + finally: + for fd in local_fds: + try: + os.close(fd) + except OSError: + pass + + def import_peer_handle(fabric_handle, fd, *, use_fabric: bool, peer_rank: int): """Import a peer allocation handle (FABRIC or POSIX fd). Returns the handle. diff --git a/test/manual/layers/test_engram_peer_shards.py b/test/manual/layers/test_engram_peer_shards.py new file mode 100644 index 0000000000..50e6638eaa --- /dev/null +++ b/test/manual/layers/test_engram_peer_shards.py @@ -0,0 +1,196 @@ +"""Multi-rank check of the DeepSeek-V4.1 engram peer-shard lookup. + +Each rank owns a real-size row shard of one engram table (random fp8 rows and +e8m0 scales), looks up random ids for its own tokens through the peer mappings, +and compares bitwise with the DP-sharded path it replaces (all-gather ids, zero- +filled owned rows, reduce-scatter). Launch one process per GPU, e.g. under srun: + + MASTER_ADDR= MASTER_PORT=29511 python test_engram_peer_shards.py +""" + +import os +import time +from types import SimpleNamespace + +import torch +import torch.distributed as dist + +from sglang.kernels.ops.embeddings.engram_gather import engram_gather +from sglang.kernels.ops.quantization.fp4_fake_quant import FP8_BLOCK_SIZE +from sglang.srt.layers.engram import EngramLayout, _PeerShards, shard_range + +DIM = 256 +CONFIG = SimpleNamespace( + engram_layer_ids=[1, 14], + engram_num_embeddings=[384006168, 384016682], + engram_max_ngram_size=4, + engram_n_heads=8, + engram_vocab_size=16000000, + engram_head_dim=DIM, +) + + +def fill_shard(weight, scale, seed): + g = torch.Generator(device=weight.device).manual_seed(seed) + chunk = 1 << 22 + w8 = weight.view(torch.uint8) + for i in range(0, w8.shape[0], chunk): + part = torch.randint( + 0, + 256, + w8[i : i + chunk].shape, + generator=g, + device=w8.device, + dtype=torch.uint8, + ) + # 0x7f / 0xff are e4m3 NaN; keep every value finite. + part = torch.where((part & 0x7F) == 0x7F, part - 1, part) + w8[i : i + chunk] = part + s8 = scale.view(torch.uint8) + s8.copy_( + torch.randint( + 100, 150, s8.shape, generator=g, device=s8.device, dtype=torch.uint8 + ) + ) + + +def random_ids(bounds, tokens, seed, device): + g = torch.Generator(device=device).manual_seed(seed) + lo = torch.tensor(bounds[:-1], device=device, dtype=torch.int64) + hi = torch.tensor(bounds[1:], device=device, dtype=torch.int64) + u = torch.rand(tokens, len(lo), generator=g, device=device, dtype=torch.float64) + ids = lo + (u * (hi - lo).double()).long() + return torch.minimum(ids, hi - 1) + + +def reference(ids, weight, scale, row_start, rows, world): + """The current _dp_sharded_lookup mechanics with equal per-rank token counts.""" + t = ids.shape[0] + gathered = torch.empty(world * t, ids.shape[1], dtype=ids.dtype, device=ids.device) + dist.all_gather_into_tensor(gathered, ids.contiguous()) + values = torch.empty( + world * t * ids.shape[1], DIM, dtype=torch.bfloat16, device=ids.device + ) + engram_gather( + weight.data_ptr(), + scale.data_ptr(), + gathered.reshape(-1), + values, + DIM, + FP8_BLOCK_SIZE, + row_lo=row_start, + row_hi=row_start + rows, + ) + values = values.view(world * t, -1) + local = torch.empty(t, values.shape[1], dtype=values.dtype, device=values.device) + dist.reduce_scatter_tensor(local, values) + return local.view(t, ids.shape[1], DIM) + + +def bench(fn, calls=20, replays=20): + """GPU time per call (us) of ``fn`` captured ``calls`` times in one CUDA graph.""" + s = torch.cuda.Stream() + s.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(s): + for _ in range(3): + fn() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=s): + for _ in range(calls): + fn() + torch.cuda.current_stream().wait_stream(s) + graph.replay() + torch.cuda.synchronize() + dist.barrier() + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(replays): + graph.replay() + end.record() + torch.cuda.synchronize() + return start.elapsed_time(end) / (calls * replays) * 1000.0 + + +def main(): + rank = int(os.environ.get("SLURM_PROCID", os.environ.get("RANK", 0))) + world = int(os.environ.get("SLURM_NTASKS", os.environ.get("WORLD_SIZE", 1))) + local_rank = int(os.environ.get("SLURM_LOCALID", os.environ.get("LOCAL_RANK", 0))) + torch.cuda.set_device(local_rank) + dist.init_process_group( + "nccl", rank=rank, world_size=world, device_id=torch.device("cuda", local_rank) + ) + cpu_group = dist.new_group(backend="gloo") + group = SimpleNamespace(cpu_group=cpu_group, rank_in_group=rank, world_size=world) + layout = EngramLayout.from_config(CONFIG) + n = layout.num_embeddings[0] + col_bounds = layout.column_bounds(0) + token_list = [ + int(x) for x in os.environ.get("TOKENS", "1 64 256 1024 16384").split() + ] + split = os.environ.get("SPLIT", "auto") # auto (column if it divides) | rows + shard_bounds = col_bounds if split == "auto" else (0, n) + row_start, row_end = shard_range(n, shard_bounds, rank, world) + rows = row_end - row_start + t0 = time.perf_counter() + peer = _PeerShards(rows, DIM, group) + weight, scale = peer.views() + fill_shard(weight, scale, seed=1000 + rank) + torch.cuda.synchronize() + t_alloc = time.perf_counter() - t0 + t0 = time.perf_counter() + peer.connect(row_start, "test") + t_connect = time.perf_counter() - t0 + if rank == 0: + print( + f"world={world} split={split} rows/rank~{rows} shard={peer.size / 2**30:.2f} GiB " + f"alloc+fill={t_alloc:.1f}s connect={t_connect:.1f}s", + flush=True, + ) + ok = True + for tokens in token_list: + ids = random_ids(col_bounds, tokens, seed=7 * tokens + rank, device="cuda") + # The model passes a strided [T, 24] slice of [T, n_layers, 24]. + both = torch.stack([ids, ids], dim=1) + got = peer.lookup(both[:, 0]) + ref = reference(ids, weight, scale, row_start, rows, world) + g16, r16 = got.view(torch.int16), ref.view(torch.int16) + bit_diff = int((g16 != r16).sum()) + val_diff = int((got != ref).sum()) + zero_sign = int(((g16 != r16) & (got == 0) & (ref == 0)).sum()) + # The same lookup inside a CUDA graph. + static_ids = both[:, 0].clone() + graph = torch.cuda.CUDAGraph() + s = torch.cuda.Stream() + s.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(s): + peer.lookup(static_ids) + with torch.cuda.graph(graph, stream=s): + graph_out = peer.lookup(static_ids) + torch.cuda.current_stream().wait_stream(s) + graph.replay() + torch.cuda.synchronize() + graph_diff = int((graph_out.view(torch.int16) != g16).sum()) + t_peer = bench(lambda: peer.lookup(both[:, 0])) + t_ref = bench(lambda: reference(ids, weight, scale, row_start, rows, world)) + res = torch.tensor([bit_diff, val_diff, zero_sign, graph_diff], device="cuda") + dist.all_reduce(res) + times = torch.tensor([t_peer, t_ref], device="cuda") + tmax = times.clone() + dist.all_reduce(tmax, op=dist.ReduceOp.MAX) + if rank == 0: + print( + f"tokens/rank={tokens}: bit_mismatch={int(res[0])} value_mismatch={int(res[1])} " + f"zero_sign_only={int(res[2])} graph_mismatch={int(res[3])} " + f"peer={t_peer:.1f}us (max {tmax[0]:.1f}) current={t_ref:.1f}us (max {tmax[1]:.1f})", + flush=True, + ) + ok &= int(res[1]) == 0 and int(res[3]) == 0 + dist.barrier() + if rank == 0: + print("PASS" if ok else "FAIL", flush=True) + dist.destroy_process_group() + + +if __name__ == "__main__": + main()