[Perf] dsv41: Read Engram rows from peer device shards (#46180)
Co-authored-by: kpham-sgl <khoa.pham@radixark.ai>
This commit is contained in:
co-authored by
kpham-sgl
parent
327c906764
commit
79701a2549
@@ -22,6 +22,14 @@ register_kernel(
|
||||
)
|
||||
)
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="embeddings.engram_gather_peer",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.embeddings.engram_gather:engram_gather_peer",
|
||||
)
|
||||
)
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="embeddings.engram_hash_ids",
|
||||
|
||||
@@ -96,3 +96,92 @@ def engram_gather(
|
||||
HAS_DST=dst is not None,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _engram_gather_peer_kernel(
|
||||
ids_ptr,
|
||||
ids_stride,
|
||||
out_ptr,
|
||||
w_bias_ptr,
|
||||
s_bias_ptr,
|
||||
starts_ptr,
|
||||
n_cols,
|
||||
DIM: tl.constexpr,
|
||||
BLK: tl.constexpr,
|
||||
ROWS: tl.constexpr,
|
||||
LOG2_WORLD: tl.constexpr,
|
||||
E8M0_ZERO: tl.constexpr,
|
||||
):
|
||||
tok = tl.program_id(0).to(tl.int64)
|
||||
cols = tl.program_id(1) * ROWS + tl.arange(0, ROWS)
|
||||
col_ok = cols < n_cols
|
||||
idx = tl.load(ids_ptr + tok * ids_stride + cols, mask=col_ok, other=0).to(tl.int64)
|
||||
# Owner = the last shard whose first row is <= idx; starts is padded to a
|
||||
# power of two with int64 max, so the search never runs past a real shard.
|
||||
owner = tl.zeros([ROWS], dtype=tl.int32)
|
||||
for step in tl.static_range(LOG2_WORLD):
|
||||
probe = owner + (1 << (LOG2_WORLD - 1 - step))
|
||||
start = tl.load(starts_ptr + probe)
|
||||
owner = tl.where(start <= idx, probe, owner)
|
||||
# Bases are pre-biased by each shard's first row, so a global id indexes them.
|
||||
w_row = tl.load(w_bias_ptr + owner) + idx * DIM
|
||||
s_row = tl.load(s_bias_ptr + owner) + idx * (DIM // BLK)
|
||||
offs = tl.arange(0, DIM)
|
||||
w = (w_row[:, None] + offs[None, :]).to(tl.pointer_type(tl.float8e4nv))
|
||||
s = (s_row[:, None] + (offs // BLK)[None, :]).to(tl.pointer_type(tl.uint8))
|
||||
mask = col_ok[:, None]
|
||||
vals = tl.load(w, mask=mask, other=0.0).to(tl.float32)
|
||||
exps = tl.load(s, mask=mask, other=0).to(tl.int32)
|
||||
scale = (exps << 23).to(tl.float32, bitcast=True)
|
||||
scale = tl.where(exps == 0, E8M0_ZERO, scale)
|
||||
# -0 becomes +0, as the zero-fill + sum of the sharded lookup leaves it, so the
|
||||
# two lookups agree bit for bit.
|
||||
out = vals * scale
|
||||
out = tl.where(out == 0.0, 0.0, out)
|
||||
dst = out_ptr + (tok * n_cols + cols)[:, None].to(tl.int64) * DIM + offs[None, :]
|
||||
tl.store(dst, out.to(tl.bfloat16), mask=mask)
|
||||
|
||||
|
||||
def engram_gather_peer(
|
||||
ids: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
w_bias: torch.Tensor,
|
||||
s_bias: torch.Tensor,
|
||||
starts: torch.Tensor,
|
||||
dim: int,
|
||||
block_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Gather ``ids`` ([T, C] int, unit column stride) into ``out`` ([T, C, dim]
|
||||
bf16, contiguous), reading each id from the shard that owns it.
|
||||
|
||||
``starts`` ([P] int64, P a power of two) holds each shard's first global row,
|
||||
padded with int64 max; ``w_bias`` / ``s_bias`` ([P] int64) are each shard's fp8
|
||||
row / e8m0 scale base address minus its first row's offset, so any rank's
|
||||
shard (local or a peer mapping over NVLink) is addressed by the global id.
|
||||
"""
|
||||
assert ids.dim() == 2 and ids.stride(1) == 1, (ids.shape, ids.stride())
|
||||
assert ids.dtype in (torch.int32, torch.int64)
|
||||
assert out.is_contiguous() and out.dtype == torch.bfloat16
|
||||
assert out.shape == (*ids.shape, dim), (out.shape, ids.shape)
|
||||
world = starts.numel()
|
||||
assert world & (world - 1) == 0, world
|
||||
t, c = ids.shape
|
||||
if t and c:
|
||||
rows = 8
|
||||
_engram_gather_peer_kernel[(t, triton.cdiv(c, rows))](
|
||||
ids,
|
||||
ids.stride(0),
|
||||
out,
|
||||
w_bias,
|
||||
s_bias,
|
||||
starts,
|
||||
c,
|
||||
DIM=dim,
|
||||
BLK=block_size,
|
||||
ROWS=rows,
|
||||
LOG2_WORLD=world.bit_length() - 1,
|
||||
E8M0_ZERO=_E8M0_ZERO,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -1777,6 +1777,11 @@ class Envs:
|
||||
# Scheduler-side staging of shared host-table rows on GPUs without ATS
|
||||
# (engram_staging.py); pays off for large prefill batches.
|
||||
SGLANG_OPT_DSV41_ENGRAM_HOST_STAGING = EnvBool(False)
|
||||
# Keep each rank's engram row shard in exportable device memory (CUDA VMM,
|
||||
# FABRIC handles across an NVLink domain) mapped by every rank of the TP group:
|
||||
# a rank reads its own tokens' rows from the owning shard, with no id gather
|
||||
# and no reduce-scatter. Takes precedence over the host table.
|
||||
SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS = EnvBool(False)
|
||||
|
||||
# Kernels and indexer
|
||||
SGLANG_OPT_DEEPGEMM_HC_PRENORM = EnvBool(True)
|
||||
|
||||
@@ -23,7 +23,10 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.embeddings.engram_gate import fused_engram_gate
|
||||
from sglang.kernels.ops.embeddings.engram_gather import engram_gather
|
||||
from sglang.kernels.ops.embeddings.engram_gather import (
|
||||
engram_gather,
|
||||
engram_gather_peer,
|
||||
)
|
||||
from sglang.kernels.ops.embeddings.engram_hash import (
|
||||
MODE_DECODE,
|
||||
MODE_EXTEND,
|
||||
@@ -795,6 +798,161 @@ class _HostTable:
|
||||
logger.info(msg)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _warn_host_table_ignored() -> None:
|
||||
logger.warning(
|
||||
"SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS takes precedence: the engram tables "
|
||||
"stay in device memory and SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE is ignored"
|
||||
)
|
||||
|
||||
|
||||
class _PeerShards:
|
||||
"""One rank's engram row shard in exportable device memory (CUDA VMM), with
|
||||
every TP peer's shard mapped into this process after `connect`.
|
||||
|
||||
Inside one NVLink domain (MNNVL included, through FABRIC handles) a kernel
|
||||
reads any peer's rows in place, so a rank looks up only its own tokens: no
|
||||
id gather, no [DP x tokens] buffer, no reduce-scatter. The shard holds the
|
||||
fp8 rows followed by their e8m0 scales. Lives for the whole process.
|
||||
"""
|
||||
|
||||
def __init__(self, rows: int, dim: int, group):
|
||||
from sglang.srt.utils.cuda_vmm_utils import (
|
||||
VmmReservation,
|
||||
align_up,
|
||||
get_device_granularity,
|
||||
make_device_allocation_prop,
|
||||
)
|
||||
|
||||
self.rows = rows
|
||||
self.dim = dim
|
||||
self.group = group
|
||||
self.device_id = torch.cuda.current_device()
|
||||
self.w_bytes = rows * dim
|
||||
self.s_bytes = rows * (dim // FP8_BLOCK_SIZE)
|
||||
gran = get_device_granularity(self.device_id)
|
||||
self.size = align_up(max(1, self.w_bytes + self.s_bytes), gran)
|
||||
self.local = VmmReservation(
|
||||
self.size,
|
||||
make_device_allocation_prop(self.device_id),
|
||||
self.device_id,
|
||||
alignment=gran,
|
||||
)
|
||||
self.handle = self.local.map(0, self.size, retain_handle=True)
|
||||
self.base = self.local.base
|
||||
self.w_bias: Optional[torch.Tensor] = None
|
||||
self.s_bias: Optional[torch.Tensor] = None
|
||||
self.starts: Optional[torch.Tensor] = None
|
||||
|
||||
def views(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from sglang.srt.utils.cuda_vmm_utils import tensor_from_pointer
|
||||
|
||||
raw = tensor_from_pointer(self.base, self.size, device_id=self.device_id)
|
||||
weight = raw[: self.w_bytes].view(torch.float8_e4m3fn).view(self.rows, self.dim)
|
||||
scale = (
|
||||
raw[self.w_bytes : self.w_bytes + self.s_bytes]
|
||||
.view(torch.float8_e8m0fnu)
|
||||
.view(self.rows, self.dim // FP8_BLOCK_SIZE)
|
||||
)
|
||||
return weight, scale
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
return self.starts is not None
|
||||
|
||||
def connect(self, row_start: int, label: str) -> None:
|
||||
"""Collective over the TP group: export this shard, map every peer's."""
|
||||
if self.ready:
|
||||
return
|
||||
import torch.distributed as dist
|
||||
|
||||
from sglang.srt.utils.cuda_vmm_utils import (
|
||||
exchange_shareable_handles,
|
||||
import_and_map_alloc,
|
||||
)
|
||||
|
||||
t0 = time.perf_counter()
|
||||
torch.cuda.synchronize()
|
||||
cpu_group = self.group.cpu_group
|
||||
rank, world = self.group.rank_in_group, self.group.world_size
|
||||
use_fabric, peer_handles = exchange_shareable_handles(
|
||||
[self.handle], cpu_group, rank, world
|
||||
)
|
||||
meta = [None] * world
|
||||
dist.all_gather_object(meta, (row_start, self.rows, self.size), group=cpu_group)
|
||||
bases = []
|
||||
try:
|
||||
for r, (_, _, size) in enumerate(meta):
|
||||
if r == rank:
|
||||
bases.append(self.base)
|
||||
continue
|
||||
handle = peer_handles[(r, 0)]
|
||||
bases.append(
|
||||
import_and_map_alloc(
|
||||
handle if use_fabric else None,
|
||||
None if use_fabric else handle,
|
||||
size,
|
||||
self.device_id,
|
||||
use_fabric=use_fabric,
|
||||
peer_rank=r,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
if not use_fabric:
|
||||
for fd in peer_handles.values():
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
padded = 1 << max(0, (world - 1).bit_length())
|
||||
int64_max = torch.iinfo(torch.int64).max
|
||||
starts, w_bias, s_bias = [], [], []
|
||||
n_scales = self.dim // FP8_BLOCK_SIZE
|
||||
for r in range(padded):
|
||||
if r < world:
|
||||
start, rows = meta[r][0], meta[r][1]
|
||||
starts.append(start)
|
||||
w_bias.append(bases[r] - start * self.dim)
|
||||
s_bias.append(bases[r] + rows * self.dim - start * n_scales)
|
||||
else:
|
||||
starts.append(int64_max)
|
||||
w_bias.append(0)
|
||||
s_bias.append(0)
|
||||
assert starts[0] == 0 and starts[:world] == sorted(starts[:world]), starts
|
||||
device = torch.device("cuda", self.device_id)
|
||||
self.w_bias = torch.tensor(w_bias, dtype=torch.int64, device=device)
|
||||
self.s_bias = torch.tensor(s_bias, dtype=torch.int64, device=device)
|
||||
self.starts = torch.tensor(starts, dtype=torch.int64, device=device)
|
||||
torch.cuda.synchronize()
|
||||
dist.barrier(group=cpu_group)
|
||||
logger.info(
|
||||
"engram peer shards %s: %d ranks over %s, %.2f GiB local, mapped in %.1f s",
|
||||
label,
|
||||
world,
|
||||
"FABRIC" if use_fabric else "POSIX fd",
|
||||
self.size / 2**30,
|
||||
time.perf_counter() - t0,
|
||||
)
|
||||
|
||||
def lookup(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
flat = indices.reshape(-1, indices.shape[-1])
|
||||
if flat.stride(-1) != 1:
|
||||
flat = flat.contiguous()
|
||||
out = torch.empty(
|
||||
*flat.shape, self.dim, dtype=torch.bfloat16, device=indices.device
|
||||
)
|
||||
engram_gather_peer(
|
||||
flat,
|
||||
out,
|
||||
self.w_bias,
|
||||
self.s_bias,
|
||||
self.starts,
|
||||
self.dim,
|
||||
FP8_BLOCK_SIZE,
|
||||
)
|
||||
return out.view(*indices.shape, self.dim)
|
||||
|
||||
|
||||
def shard_range(
|
||||
num_embeddings: int,
|
||||
column_bounds: tuple[int, ...],
|
||||
@@ -848,7 +1006,19 @@ class EngramEmbedding(nn.Module):
|
||||
self.host_table: Optional[_HostTable] = None
|
||||
self._sort_host_reads = False
|
||||
self.staged_by_scheduler = False
|
||||
if envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get():
|
||||
self.peer: Optional[_PeerShards] = None
|
||||
if (
|
||||
envs.SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS.get()
|
||||
and self.tp_size > 1
|
||||
and is_cuda()
|
||||
):
|
||||
if envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get() and tp_rank == 0:
|
||||
_warn_host_table_ignored()
|
||||
self.peer = _PeerShards(self.rows, dim, get_parallel().tp_group)
|
||||
weight, scale = self.peer.views()
|
||||
self.weight = nn.Parameter(weight, requires_grad=False)
|
||||
self.scale = nn.Parameter(scale, requires_grad=False)
|
||||
elif envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get():
|
||||
self._init_host_table(num_embeddings, dim, layer_id)
|
||||
else:
|
||||
self.weight = nn.Parameter(
|
||||
@@ -914,7 +1084,10 @@ class EngramEmbedding(nn.Module):
|
||||
self.host_table.dirty = True
|
||||
|
||||
def finish_load(self, label: str):
|
||||
"""Collective in the shared layout: every rank calls it after loading."""
|
||||
"""Collective in the shared layout and with peer shards: every rank calls
|
||||
it after loading."""
|
||||
if self.peer is not None:
|
||||
self.peer.connect(self.row_start, label)
|
||||
if self.host_table is not None:
|
||||
self.host_table.finish_load(label)
|
||||
|
||||
@@ -925,6 +1098,11 @@ class EngramEmbedding(nn.Module):
|
||||
*,
|
||||
cp_all_tokens: bool = False,
|
||||
) -> torch.Tensor:
|
||||
if self.peer is not None:
|
||||
assert self.peer.ready, "engram peer shards are mapped in finish_load"
|
||||
# Every shard is mapped here: read this rank's own tokens only, which
|
||||
# also covers the prefill CP and DP paths below without a collective.
|
||||
return self.peer.lookup(indices)
|
||||
if self._shared:
|
||||
if indices.shape[0] == 0:
|
||||
return self._empty(indices)
|
||||
|
||||
@@ -21,8 +21,7 @@ from sglang.srt.utils.cuda_vmm_utils import (
|
||||
align_down,
|
||||
align_up,
|
||||
check_drv,
|
||||
exchange_posix_fds,
|
||||
export_shareable_handles,
|
||||
exchange_shareable_handles,
|
||||
get_device_granularity,
|
||||
import_peer_handle,
|
||||
make_device_allocation_prop,
|
||||
@@ -138,11 +137,6 @@ class DWDPTransport:
|
||||
prop = make_device_allocation_prop(device_id)
|
||||
|
||||
handle_list = [self._handle_set.get_handle(li, n) for li, n in sorted_keys]
|
||||
fabric_handles, local_posix_fds, use_fabric = export_shareable_handles(
|
||||
handle_list, cpu_group, layout.dwdp_rank
|
||||
)
|
||||
peer_fds: Dict[Tuple[int, int], int] = {}
|
||||
|
||||
key_counts = [None] * layout.dwdp_size
|
||||
dist.all_gather_object(key_counts, len(sorted_keys), group=cpu_group)
|
||||
if any(count != len(sorted_keys) for count in key_counts):
|
||||
@@ -150,18 +144,9 @@ class DWDPTransport:
|
||||
f"Mismatched DWDP weight handle counts across ranks: {key_counts}"
|
||||
)
|
||||
|
||||
if use_fabric:
|
||||
all_fabric = [None] * layout.dwdp_size
|
||||
dist.all_gather_object(all_fabric, fabric_handles, group=cpu_group)
|
||||
else:
|
||||
all_fabric = None
|
||||
peer_fds = exchange_posix_fds(
|
||||
cpu_group,
|
||||
layout.dwdp_rank,
|
||||
layout.dwdp_size,
|
||||
local_posix_fds,
|
||||
key_counts,
|
||||
)
|
||||
use_fabric, peer_handles = exchange_shareable_handles(
|
||||
handle_list, cpu_group, layout.dwdp_rank, layout.dwdp_size
|
||||
)
|
||||
logger.info(
|
||||
"DWDP handle exchange via %s (%d handles)",
|
||||
"FABRIC" if use_fabric else "POSIX fd",
|
||||
@@ -175,8 +160,9 @@ class DWDPTransport:
|
||||
if peer_rank == layout.dwdp_rank:
|
||||
continue
|
||||
|
||||
fabric_handle = all_fabric[peer_rank][key_idx] if use_fabric else None
|
||||
fd = None if use_fabric else peer_fds[(peer_rank, key_idx)]
|
||||
handle = peer_handles[(peer_rank, key_idx)]
|
||||
fabric_handle = handle if use_fabric else None
|
||||
fd = None if use_fabric else handle
|
||||
peer_handle = import_peer_handle(
|
||||
fabric_handle, fd, use_fabric=use_fabric, peer_rank=peer_rank
|
||||
)
|
||||
@@ -209,8 +195,8 @@ class DWDPTransport:
|
||||
)
|
||||
self._peer_views[(peer_rank, layer_idx, name)] = peer_tensor
|
||||
|
||||
_close_fds(local_posix_fds)
|
||||
_close_fds(peer_fds.values())
|
||||
if not use_fabric:
|
||||
_close_fds(peer_handles.values())
|
||||
|
||||
@property
|
||||
def handle_set(self) -> MnnvlHandleSet:
|
||||
|
||||
@@ -880,6 +880,50 @@ def exchange_posix_fds(
|
||||
pass
|
||||
|
||||
|
||||
def exchange_shareable_handles(
|
||||
retained_handles, group: ProcessGroup, rank: int, world_size: int
|
||||
):
|
||||
"""Export this rank's retained VMM handles and collect every peer's.
|
||||
|
||||
FABRIC handles when every rank can export them; otherwise POSIX fds over UNIX
|
||||
sockets, which only reach ranks on this host, so a group spanning hosts raises.
|
||||
Returns ``(use_fabric, peer_handles)``: ``peer_handles[(peer_rank, idx)]`` is a
|
||||
FABRIC handle or a received fd the caller must close. This rank's exported fds
|
||||
are closed here.
|
||||
"""
|
||||
fabric_handles, local_fds, use_fabric = export_shareable_handles(
|
||||
retained_handles, group, rank
|
||||
)
|
||||
if use_fabric:
|
||||
all_fabric = [None] * world_size
|
||||
dist.all_gather_object(all_fabric, fabric_handles, group=group)
|
||||
return True, {
|
||||
(r, i): handle
|
||||
for r, handles in enumerate(all_fabric)
|
||||
if r != rank
|
||||
for i, handle in enumerate(handles)
|
||||
}
|
||||
try:
|
||||
info = [None] * world_size
|
||||
dist.all_gather_object(
|
||||
info, (socket.gethostname(), len(local_fds)), group=group
|
||||
)
|
||||
if len({host for host, _ in info}) > 1:
|
||||
raise RuntimeError(
|
||||
"VMM handles cannot reach ranks on other hosts without FABRIC "
|
||||
"handles (one NVLink domain with IMEX)"
|
||||
)
|
||||
return False, exchange_posix_fds(
|
||||
group, rank, world_size, local_fds, [n for _, n in info]
|
||||
)
|
||||
finally:
|
||||
for fd in local_fds:
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def import_peer_handle(fabric_handle, fd, *, use_fabric: bool, peer_rank: int):
|
||||
"""Import a peer allocation handle (FABRIC or POSIX fd). Returns the handle.
|
||||
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Multi-rank check of the DeepSeek-V4.1 engram peer-shard lookup.
|
||||
|
||||
Each rank owns a real-size row shard of one engram table (random fp8 rows and
|
||||
e8m0 scales), looks up random ids for its own tokens through the peer mappings,
|
||||
and compares bitwise with the DP-sharded path it replaces (all-gather ids, zero-
|
||||
filled owned rows, reduce-scatter). Launch one process per GPU, e.g. under srun:
|
||||
|
||||
MASTER_ADDR=<node0> MASTER_PORT=29511 python test_engram_peer_shards.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from sglang.kernels.ops.embeddings.engram_gather import engram_gather
|
||||
from sglang.kernels.ops.quantization.fp4_fake_quant import FP8_BLOCK_SIZE
|
||||
from sglang.srt.layers.engram import EngramLayout, _PeerShards, shard_range
|
||||
|
||||
DIM = 256
|
||||
CONFIG = SimpleNamespace(
|
||||
engram_layer_ids=[1, 14],
|
||||
engram_num_embeddings=[384006168, 384016682],
|
||||
engram_max_ngram_size=4,
|
||||
engram_n_heads=8,
|
||||
engram_vocab_size=16000000,
|
||||
engram_head_dim=DIM,
|
||||
)
|
||||
|
||||
|
||||
def fill_shard(weight, scale, seed):
|
||||
g = torch.Generator(device=weight.device).manual_seed(seed)
|
||||
chunk = 1 << 22
|
||||
w8 = weight.view(torch.uint8)
|
||||
for i in range(0, w8.shape[0], chunk):
|
||||
part = torch.randint(
|
||||
0,
|
||||
256,
|
||||
w8[i : i + chunk].shape,
|
||||
generator=g,
|
||||
device=w8.device,
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
# 0x7f / 0xff are e4m3 NaN; keep every value finite.
|
||||
part = torch.where((part & 0x7F) == 0x7F, part - 1, part)
|
||||
w8[i : i + chunk] = part
|
||||
s8 = scale.view(torch.uint8)
|
||||
s8.copy_(
|
||||
torch.randint(
|
||||
100, 150, s8.shape, generator=g, device=s8.device, dtype=torch.uint8
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def random_ids(bounds, tokens, seed, device):
|
||||
g = torch.Generator(device=device).manual_seed(seed)
|
||||
lo = torch.tensor(bounds[:-1], device=device, dtype=torch.int64)
|
||||
hi = torch.tensor(bounds[1:], device=device, dtype=torch.int64)
|
||||
u = torch.rand(tokens, len(lo), generator=g, device=device, dtype=torch.float64)
|
||||
ids = lo + (u * (hi - lo).double()).long()
|
||||
return torch.minimum(ids, hi - 1)
|
||||
|
||||
|
||||
def reference(ids, weight, scale, row_start, rows, world):
|
||||
"""The current _dp_sharded_lookup mechanics with equal per-rank token counts."""
|
||||
t = ids.shape[0]
|
||||
gathered = torch.empty(world * t, ids.shape[1], dtype=ids.dtype, device=ids.device)
|
||||
dist.all_gather_into_tensor(gathered, ids.contiguous())
|
||||
values = torch.empty(
|
||||
world * t * ids.shape[1], DIM, dtype=torch.bfloat16, device=ids.device
|
||||
)
|
||||
engram_gather(
|
||||
weight.data_ptr(),
|
||||
scale.data_ptr(),
|
||||
gathered.reshape(-1),
|
||||
values,
|
||||
DIM,
|
||||
FP8_BLOCK_SIZE,
|
||||
row_lo=row_start,
|
||||
row_hi=row_start + rows,
|
||||
)
|
||||
values = values.view(world * t, -1)
|
||||
local = torch.empty(t, values.shape[1], dtype=values.dtype, device=values.device)
|
||||
dist.reduce_scatter_tensor(local, values)
|
||||
return local.view(t, ids.shape[1], DIM)
|
||||
|
||||
|
||||
def bench(fn, calls=20, replays=20):
|
||||
"""GPU time per call (us) of ``fn`` captured ``calls`` times in one CUDA graph."""
|
||||
s = torch.cuda.Stream()
|
||||
s.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(s):
|
||||
for _ in range(3):
|
||||
fn()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=s):
|
||||
for _ in range(calls):
|
||||
fn()
|
||||
torch.cuda.current_stream().wait_stream(s)
|
||||
graph.replay()
|
||||
torch.cuda.synchronize()
|
||||
dist.barrier()
|
||||
start = torch.cuda.Event(enable_timing=True)
|
||||
end = torch.cuda.Event(enable_timing=True)
|
||||
start.record()
|
||||
for _ in range(replays):
|
||||
graph.replay()
|
||||
end.record()
|
||||
torch.cuda.synchronize()
|
||||
return start.elapsed_time(end) / (calls * replays) * 1000.0
|
||||
|
||||
|
||||
def main():
|
||||
rank = int(os.environ.get("SLURM_PROCID", os.environ.get("RANK", 0)))
|
||||
world = int(os.environ.get("SLURM_NTASKS", os.environ.get("WORLD_SIZE", 1)))
|
||||
local_rank = int(os.environ.get("SLURM_LOCALID", os.environ.get("LOCAL_RANK", 0)))
|
||||
torch.cuda.set_device(local_rank)
|
||||
dist.init_process_group(
|
||||
"nccl", rank=rank, world_size=world, device_id=torch.device("cuda", local_rank)
|
||||
)
|
||||
cpu_group = dist.new_group(backend="gloo")
|
||||
group = SimpleNamespace(cpu_group=cpu_group, rank_in_group=rank, world_size=world)
|
||||
layout = EngramLayout.from_config(CONFIG)
|
||||
n = layout.num_embeddings[0]
|
||||
col_bounds = layout.column_bounds(0)
|
||||
token_list = [
|
||||
int(x) for x in os.environ.get("TOKENS", "1 64 256 1024 16384").split()
|
||||
]
|
||||
split = os.environ.get("SPLIT", "auto") # auto (column if it divides) | rows
|
||||
shard_bounds = col_bounds if split == "auto" else (0, n)
|
||||
row_start, row_end = shard_range(n, shard_bounds, rank, world)
|
||||
rows = row_end - row_start
|
||||
t0 = time.perf_counter()
|
||||
peer = _PeerShards(rows, DIM, group)
|
||||
weight, scale = peer.views()
|
||||
fill_shard(weight, scale, seed=1000 + rank)
|
||||
torch.cuda.synchronize()
|
||||
t_alloc = time.perf_counter() - t0
|
||||
t0 = time.perf_counter()
|
||||
peer.connect(row_start, "test")
|
||||
t_connect = time.perf_counter() - t0
|
||||
if rank == 0:
|
||||
print(
|
||||
f"world={world} split={split} rows/rank~{rows} shard={peer.size / 2**30:.2f} GiB "
|
||||
f"alloc+fill={t_alloc:.1f}s connect={t_connect:.1f}s",
|
||||
flush=True,
|
||||
)
|
||||
ok = True
|
||||
for tokens in token_list:
|
||||
ids = random_ids(col_bounds, tokens, seed=7 * tokens + rank, device="cuda")
|
||||
# The model passes a strided [T, 24] slice of [T, n_layers, 24].
|
||||
both = torch.stack([ids, ids], dim=1)
|
||||
got = peer.lookup(both[:, 0])
|
||||
ref = reference(ids, weight, scale, row_start, rows, world)
|
||||
g16, r16 = got.view(torch.int16), ref.view(torch.int16)
|
||||
bit_diff = int((g16 != r16).sum())
|
||||
val_diff = int((got != ref).sum())
|
||||
zero_sign = int(((g16 != r16) & (got == 0) & (ref == 0)).sum())
|
||||
# The same lookup inside a CUDA graph.
|
||||
static_ids = both[:, 0].clone()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
s = torch.cuda.Stream()
|
||||
s.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(s):
|
||||
peer.lookup(static_ids)
|
||||
with torch.cuda.graph(graph, stream=s):
|
||||
graph_out = peer.lookup(static_ids)
|
||||
torch.cuda.current_stream().wait_stream(s)
|
||||
graph.replay()
|
||||
torch.cuda.synchronize()
|
||||
graph_diff = int((graph_out.view(torch.int16) != g16).sum())
|
||||
t_peer = bench(lambda: peer.lookup(both[:, 0]))
|
||||
t_ref = bench(lambda: reference(ids, weight, scale, row_start, rows, world))
|
||||
res = torch.tensor([bit_diff, val_diff, zero_sign, graph_diff], device="cuda")
|
||||
dist.all_reduce(res)
|
||||
times = torch.tensor([t_peer, t_ref], device="cuda")
|
||||
tmax = times.clone()
|
||||
dist.all_reduce(tmax, op=dist.ReduceOp.MAX)
|
||||
if rank == 0:
|
||||
print(
|
||||
f"tokens/rank={tokens}: bit_mismatch={int(res[0])} value_mismatch={int(res[1])} "
|
||||
f"zero_sign_only={int(res[2])} graph_mismatch={int(res[3])} "
|
||||
f"peer={t_peer:.1f}us (max {tmax[0]:.1f}) current={t_ref:.1f}us (max {tmax[1]:.1f})",
|
||||
flush=True,
|
||||
)
|
||||
ok &= int(res[1]) == 0 and int(res[3]) == 0
|
||||
dist.barrier()
|
||||
if rank == 0:
|
||||
print("PASS" if ok else "FAIL", flush=True)
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user