[Perf] dsv41: Read Engram rows from peer device shards (#46180)

Co-authored-by: kpham-sgl <khoa.pham@radixark.ai>
This commit is contained in:
Liangsheng Yin
2026-10-23 19:01:26 -07:00
committed by GitHub
co-authored by kpham-sgl
parent 327c906764
commit 79701a2549
7 changed files with 532 additions and 26 deletions
@@ -22,6 +22,14 @@ register_kernel(
)
)
register_kernel(
KernelSpec(
op="embeddings.engram_gather_peer",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.embeddings.engram_gather:engram_gather_peer",
)
)
register_kernel(
KernelSpec(
op="embeddings.engram_hash_ids",
@@ -96,3 +96,92 @@ def engram_gather(
HAS_DST=dst is not None,
)
return out
@triton.jit
def _engram_gather_peer_kernel(
ids_ptr,
ids_stride,
out_ptr,
w_bias_ptr,
s_bias_ptr,
starts_ptr,
n_cols,
DIM: tl.constexpr,
BLK: tl.constexpr,
ROWS: tl.constexpr,
LOG2_WORLD: tl.constexpr,
E8M0_ZERO: tl.constexpr,
):
tok = tl.program_id(0).to(tl.int64)
cols = tl.program_id(1) * ROWS + tl.arange(0, ROWS)
col_ok = cols < n_cols
idx = tl.load(ids_ptr + tok * ids_stride + cols, mask=col_ok, other=0).to(tl.int64)
# Owner = the last shard whose first row is <= idx; starts is padded to a
# power of two with int64 max, so the search never runs past a real shard.
owner = tl.zeros([ROWS], dtype=tl.int32)
for step in tl.static_range(LOG2_WORLD):
probe = owner + (1 << (LOG2_WORLD - 1 - step))
start = tl.load(starts_ptr + probe)
owner = tl.where(start <= idx, probe, owner)
# Bases are pre-biased by each shard's first row, so a global id indexes them.
w_row = tl.load(w_bias_ptr + owner) + idx * DIM
s_row = tl.load(s_bias_ptr + owner) + idx * (DIM // BLK)
offs = tl.arange(0, DIM)
w = (w_row[:, None] + offs[None, :]).to(tl.pointer_type(tl.float8e4nv))
s = (s_row[:, None] + (offs // BLK)[None, :]).to(tl.pointer_type(tl.uint8))
mask = col_ok[:, None]
vals = tl.load(w, mask=mask, other=0.0).to(tl.float32)
exps = tl.load(s, mask=mask, other=0).to(tl.int32)
scale = (exps << 23).to(tl.float32, bitcast=True)
scale = tl.where(exps == 0, E8M0_ZERO, scale)
# -0 becomes +0, as the zero-fill + sum of the sharded lookup leaves it, so the
# two lookups agree bit for bit.
out = vals * scale
out = tl.where(out == 0.0, 0.0, out)
dst = out_ptr + (tok * n_cols + cols)[:, None].to(tl.int64) * DIM + offs[None, :]
tl.store(dst, out.to(tl.bfloat16), mask=mask)
def engram_gather_peer(
ids: torch.Tensor,
out: torch.Tensor,
w_bias: torch.Tensor,
s_bias: torch.Tensor,
starts: torch.Tensor,
dim: int,
block_size: int,
) -> torch.Tensor:
"""Gather ``ids`` ([T, C] int, unit column stride) into ``out`` ([T, C, dim]
bf16, contiguous), reading each id from the shard that owns it.
``starts`` ([P] int64, P a power of two) holds each shard's first global row,
padded with int64 max; ``w_bias`` / ``s_bias`` ([P] int64) are each shard's fp8
row / e8m0 scale base address minus its first row's offset, so any rank's
shard (local or a peer mapping over NVLink) is addressed by the global id.
"""
assert ids.dim() == 2 and ids.stride(1) == 1, (ids.shape, ids.stride())
assert ids.dtype in (torch.int32, torch.int64)
assert out.is_contiguous() and out.dtype == torch.bfloat16
assert out.shape == (*ids.shape, dim), (out.shape, ids.shape)
world = starts.numel()
assert world & (world - 1) == 0, world
t, c = ids.shape
if t and c:
rows = 8
_engram_gather_peer_kernel[(t, triton.cdiv(c, rows))](
ids,
ids.stride(0),
out,
w_bias,
s_bias,
starts,
c,
DIM=dim,
BLK=block_size,
ROWS=rows,
LOG2_WORLD=world.bit_length() - 1,
E8M0_ZERO=_E8M0_ZERO,
num_warps=4,
)
return out
+5
View File
@@ -1777,6 +1777,11 @@ class Envs:
# Scheduler-side staging of shared host-table rows on GPUs without ATS
# (engram_staging.py); pays off for large prefill batches.
SGLANG_OPT_DSV41_ENGRAM_HOST_STAGING = EnvBool(False)
# Keep each rank's engram row shard in exportable device memory (CUDA VMM,
# FABRIC handles across an NVLink domain) mapped by every rank of the TP group:
# a rank reads its own tokens' rows from the owning shard, with no id gather
# and no reduce-scatter. Takes precedence over the host table.
SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS = EnvBool(False)
# Kernels and indexer
SGLANG_OPT_DEEPGEMM_HC_PRENORM = EnvBool(True)
+181 -3
View File
@@ -23,7 +23,10 @@ import torch
from torch import nn
from sglang.kernels.ops.embeddings.engram_gate import fused_engram_gate
from sglang.kernels.ops.embeddings.engram_gather import engram_gather
from sglang.kernels.ops.embeddings.engram_gather import (
engram_gather,
engram_gather_peer,
)
from sglang.kernels.ops.embeddings.engram_hash import (
MODE_DECODE,
MODE_EXTEND,
@@ -795,6 +798,161 @@ class _HostTable:
logger.info(msg)
@functools.cache
def _warn_host_table_ignored() -> None:
logger.warning(
"SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS takes precedence: the engram tables "
"stay in device memory and SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE is ignored"
)
class _PeerShards:
"""One rank's engram row shard in exportable device memory (CUDA VMM), with
every TP peer's shard mapped into this process after `connect`.
Inside one NVLink domain (MNNVL included, through FABRIC handles) a kernel
reads any peer's rows in place, so a rank looks up only its own tokens: no
id gather, no [DP x tokens] buffer, no reduce-scatter. The shard holds the
fp8 rows followed by their e8m0 scales. Lives for the whole process.
"""
def __init__(self, rows: int, dim: int, group):
from sglang.srt.utils.cuda_vmm_utils import (
VmmReservation,
align_up,
get_device_granularity,
make_device_allocation_prop,
)
self.rows = rows
self.dim = dim
self.group = group
self.device_id = torch.cuda.current_device()
self.w_bytes = rows * dim
self.s_bytes = rows * (dim // FP8_BLOCK_SIZE)
gran = get_device_granularity(self.device_id)
self.size = align_up(max(1, self.w_bytes + self.s_bytes), gran)
self.local = VmmReservation(
self.size,
make_device_allocation_prop(self.device_id),
self.device_id,
alignment=gran,
)
self.handle = self.local.map(0, self.size, retain_handle=True)
self.base = self.local.base
self.w_bias: Optional[torch.Tensor] = None
self.s_bias: Optional[torch.Tensor] = None
self.starts: Optional[torch.Tensor] = None
def views(self) -> tuple[torch.Tensor, torch.Tensor]:
from sglang.srt.utils.cuda_vmm_utils import tensor_from_pointer
raw = tensor_from_pointer(self.base, self.size, device_id=self.device_id)
weight = raw[: self.w_bytes].view(torch.float8_e4m3fn).view(self.rows, self.dim)
scale = (
raw[self.w_bytes : self.w_bytes + self.s_bytes]
.view(torch.float8_e8m0fnu)
.view(self.rows, self.dim // FP8_BLOCK_SIZE)
)
return weight, scale
@property
def ready(self) -> bool:
return self.starts is not None
def connect(self, row_start: int, label: str) -> None:
"""Collective over the TP group: export this shard, map every peer's."""
if self.ready:
return
import torch.distributed as dist
from sglang.srt.utils.cuda_vmm_utils import (
exchange_shareable_handles,
import_and_map_alloc,
)
t0 = time.perf_counter()
torch.cuda.synchronize()
cpu_group = self.group.cpu_group
rank, world = self.group.rank_in_group, self.group.world_size
use_fabric, peer_handles = exchange_shareable_handles(
[self.handle], cpu_group, rank, world
)
meta = [None] * world
dist.all_gather_object(meta, (row_start, self.rows, self.size), group=cpu_group)
bases = []
try:
for r, (_, _, size) in enumerate(meta):
if r == rank:
bases.append(self.base)
continue
handle = peer_handles[(r, 0)]
bases.append(
import_and_map_alloc(
handle if use_fabric else None,
None if use_fabric else handle,
size,
self.device_id,
use_fabric=use_fabric,
peer_rank=r,
)
)
finally:
if not use_fabric:
for fd in peer_handles.values():
try:
os.close(fd)
except OSError:
pass
padded = 1 << max(0, (world - 1).bit_length())
int64_max = torch.iinfo(torch.int64).max
starts, w_bias, s_bias = [], [], []
n_scales = self.dim // FP8_BLOCK_SIZE
for r in range(padded):
if r < world:
start, rows = meta[r][0], meta[r][1]
starts.append(start)
w_bias.append(bases[r] - start * self.dim)
s_bias.append(bases[r] + rows * self.dim - start * n_scales)
else:
starts.append(int64_max)
w_bias.append(0)
s_bias.append(0)
assert starts[0] == 0 and starts[:world] == sorted(starts[:world]), starts
device = torch.device("cuda", self.device_id)
self.w_bias = torch.tensor(w_bias, dtype=torch.int64, device=device)
self.s_bias = torch.tensor(s_bias, dtype=torch.int64, device=device)
self.starts = torch.tensor(starts, dtype=torch.int64, device=device)
torch.cuda.synchronize()
dist.barrier(group=cpu_group)
logger.info(
"engram peer shards %s: %d ranks over %s, %.2f GiB local, mapped in %.1f s",
label,
world,
"FABRIC" if use_fabric else "POSIX fd",
self.size / 2**30,
time.perf_counter() - t0,
)
def lookup(self, indices: torch.Tensor) -> torch.Tensor:
flat = indices.reshape(-1, indices.shape[-1])
if flat.stride(-1) != 1:
flat = flat.contiguous()
out = torch.empty(
*flat.shape, self.dim, dtype=torch.bfloat16, device=indices.device
)
engram_gather_peer(
flat,
out,
self.w_bias,
self.s_bias,
self.starts,
self.dim,
FP8_BLOCK_SIZE,
)
return out.view(*indices.shape, self.dim)
def shard_range(
num_embeddings: int,
column_bounds: tuple[int, ...],
@@ -848,7 +1006,19 @@ class EngramEmbedding(nn.Module):
self.host_table: Optional[_HostTable] = None
self._sort_host_reads = False
self.staged_by_scheduler = False
if envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get():
self.peer: Optional[_PeerShards] = None
if (
envs.SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS.get()
and self.tp_size > 1
and is_cuda()
):
if envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get() and tp_rank == 0:
_warn_host_table_ignored()
self.peer = _PeerShards(self.rows, dim, get_parallel().tp_group)
weight, scale = self.peer.views()
self.weight = nn.Parameter(weight, requires_grad=False)
self.scale = nn.Parameter(scale, requires_grad=False)
elif envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get():
self._init_host_table(num_embeddings, dim, layer_id)
else:
self.weight = nn.Parameter(
@@ -914,7 +1084,10 @@ class EngramEmbedding(nn.Module):
self.host_table.dirty = True
def finish_load(self, label: str):
"""Collective in the shared layout: every rank calls it after loading."""
"""Collective in the shared layout and with peer shards: every rank calls
it after loading."""
if self.peer is not None:
self.peer.connect(self.row_start, label)
if self.host_table is not None:
self.host_table.finish_load(label)
@@ -925,6 +1098,11 @@ class EngramEmbedding(nn.Module):
*,
cp_all_tokens: bool = False,
) -> torch.Tensor:
if self.peer is not None:
assert self.peer.ready, "engram peer shards are mapped in finish_load"
# Every shard is mapped here: read this rank's own tokens only, which
# also covers the prefill CP and DP paths below without a collective.
return self.peer.lookup(indices)
if self._shared:
if indices.shape[0] == 0:
return self._empty(indices)
+9 -23
View File
@@ -21,8 +21,7 @@ from sglang.srt.utils.cuda_vmm_utils import (
align_down,
align_up,
check_drv,
exchange_posix_fds,
export_shareable_handles,
exchange_shareable_handles,
get_device_granularity,
import_peer_handle,
make_device_allocation_prop,
@@ -138,11 +137,6 @@ class DWDPTransport:
prop = make_device_allocation_prop(device_id)
handle_list = [self._handle_set.get_handle(li, n) for li, n in sorted_keys]
fabric_handles, local_posix_fds, use_fabric = export_shareable_handles(
handle_list, cpu_group, layout.dwdp_rank
)
peer_fds: Dict[Tuple[int, int], int] = {}
key_counts = [None] * layout.dwdp_size
dist.all_gather_object(key_counts, len(sorted_keys), group=cpu_group)
if any(count != len(sorted_keys) for count in key_counts):
@@ -150,18 +144,9 @@ class DWDPTransport:
f"Mismatched DWDP weight handle counts across ranks: {key_counts}"
)
if use_fabric:
all_fabric = [None] * layout.dwdp_size
dist.all_gather_object(all_fabric, fabric_handles, group=cpu_group)
else:
all_fabric = None
peer_fds = exchange_posix_fds(
cpu_group,
layout.dwdp_rank,
layout.dwdp_size,
local_posix_fds,
key_counts,
)
use_fabric, peer_handles = exchange_shareable_handles(
handle_list, cpu_group, layout.dwdp_rank, layout.dwdp_size
)
logger.info(
"DWDP handle exchange via %s (%d handles)",
"FABRIC" if use_fabric else "POSIX fd",
@@ -175,8 +160,9 @@ class DWDPTransport:
if peer_rank == layout.dwdp_rank:
continue
fabric_handle = all_fabric[peer_rank][key_idx] if use_fabric else None
fd = None if use_fabric else peer_fds[(peer_rank, key_idx)]
handle = peer_handles[(peer_rank, key_idx)]
fabric_handle = handle if use_fabric else None
fd = None if use_fabric else handle
peer_handle = import_peer_handle(
fabric_handle, fd, use_fabric=use_fabric, peer_rank=peer_rank
)
@@ -209,8 +195,8 @@ class DWDPTransport:
)
self._peer_views[(peer_rank, layer_idx, name)] = peer_tensor
_close_fds(local_posix_fds)
_close_fds(peer_fds.values())
if not use_fabric:
_close_fds(peer_handles.values())
@property
def handle_set(self) -> MnnvlHandleSet:
+44
View File
@@ -880,6 +880,50 @@ def exchange_posix_fds(
pass
def exchange_shareable_handles(
retained_handles, group: ProcessGroup, rank: int, world_size: int
):
"""Export this rank's retained VMM handles and collect every peer's.
FABRIC handles when every rank can export them; otherwise POSIX fds over UNIX
sockets, which only reach ranks on this host, so a group spanning hosts raises.
Returns ``(use_fabric, peer_handles)``: ``peer_handles[(peer_rank, idx)]`` is a
FABRIC handle or a received fd the caller must close. This rank's exported fds
are closed here.
"""
fabric_handles, local_fds, use_fabric = export_shareable_handles(
retained_handles, group, rank
)
if use_fabric:
all_fabric = [None] * world_size
dist.all_gather_object(all_fabric, fabric_handles, group=group)
return True, {
(r, i): handle
for r, handles in enumerate(all_fabric)
if r != rank
for i, handle in enumerate(handles)
}
try:
info = [None] * world_size
dist.all_gather_object(
info, (socket.gethostname(), len(local_fds)), group=group
)
if len({host for host, _ in info}) > 1:
raise RuntimeError(
"VMM handles cannot reach ranks on other hosts without FABRIC "
"handles (one NVLink domain with IMEX)"
)
return False, exchange_posix_fds(
group, rank, world_size, local_fds, [n for _, n in info]
)
finally:
for fd in local_fds:
try:
os.close(fd)
except OSError:
pass
def import_peer_handle(fabric_handle, fd, *, use_fabric: bool, peer_rank: int):
"""Import a peer allocation handle (FABRIC or POSIX fd). Returns the handle.
@@ -0,0 +1,196 @@
"""Multi-rank check of the DeepSeek-V4.1 engram peer-shard lookup.
Each rank owns a real-size row shard of one engram table (random fp8 rows and
e8m0 scales), looks up random ids for its own tokens through the peer mappings,
and compares bitwise with the DP-sharded path it replaces (all-gather ids, zero-
filled owned rows, reduce-scatter). Launch one process per GPU, e.g. under srun:
MASTER_ADDR=<node0> MASTER_PORT=29511 python test_engram_peer_shards.py
"""
import os
import time
from types import SimpleNamespace
import torch
import torch.distributed as dist
from sglang.kernels.ops.embeddings.engram_gather import engram_gather
from sglang.kernels.ops.quantization.fp4_fake_quant import FP8_BLOCK_SIZE
from sglang.srt.layers.engram import EngramLayout, _PeerShards, shard_range
DIM = 256
CONFIG = SimpleNamespace(
engram_layer_ids=[1, 14],
engram_num_embeddings=[384006168, 384016682],
engram_max_ngram_size=4,
engram_n_heads=8,
engram_vocab_size=16000000,
engram_head_dim=DIM,
)
def fill_shard(weight, scale, seed):
g = torch.Generator(device=weight.device).manual_seed(seed)
chunk = 1 << 22
w8 = weight.view(torch.uint8)
for i in range(0, w8.shape[0], chunk):
part = torch.randint(
0,
256,
w8[i : i + chunk].shape,
generator=g,
device=w8.device,
dtype=torch.uint8,
)
# 0x7f / 0xff are e4m3 NaN; keep every value finite.
part = torch.where((part & 0x7F) == 0x7F, part - 1, part)
w8[i : i + chunk] = part
s8 = scale.view(torch.uint8)
s8.copy_(
torch.randint(
100, 150, s8.shape, generator=g, device=s8.device, dtype=torch.uint8
)
)
def random_ids(bounds, tokens, seed, device):
g = torch.Generator(device=device).manual_seed(seed)
lo = torch.tensor(bounds[:-1], device=device, dtype=torch.int64)
hi = torch.tensor(bounds[1:], device=device, dtype=torch.int64)
u = torch.rand(tokens, len(lo), generator=g, device=device, dtype=torch.float64)
ids = lo + (u * (hi - lo).double()).long()
return torch.minimum(ids, hi - 1)
def reference(ids, weight, scale, row_start, rows, world):
"""The current _dp_sharded_lookup mechanics with equal per-rank token counts."""
t = ids.shape[0]
gathered = torch.empty(world * t, ids.shape[1], dtype=ids.dtype, device=ids.device)
dist.all_gather_into_tensor(gathered, ids.contiguous())
values = torch.empty(
world * t * ids.shape[1], DIM, dtype=torch.bfloat16, device=ids.device
)
engram_gather(
weight.data_ptr(),
scale.data_ptr(),
gathered.reshape(-1),
values,
DIM,
FP8_BLOCK_SIZE,
row_lo=row_start,
row_hi=row_start + rows,
)
values = values.view(world * t, -1)
local = torch.empty(t, values.shape[1], dtype=values.dtype, device=values.device)
dist.reduce_scatter_tensor(local, values)
return local.view(t, ids.shape[1], DIM)
def bench(fn, calls=20, replays=20):
"""GPU time per call (us) of ``fn`` captured ``calls`` times in one CUDA graph."""
s = torch.cuda.Stream()
s.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(s):
for _ in range(3):
fn()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=s):
for _ in range(calls):
fn()
torch.cuda.current_stream().wait_stream(s)
graph.replay()
torch.cuda.synchronize()
dist.barrier()
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
start.record()
for _ in range(replays):
graph.replay()
end.record()
torch.cuda.synchronize()
return start.elapsed_time(end) / (calls * replays) * 1000.0
def main():
rank = int(os.environ.get("SLURM_PROCID", os.environ.get("RANK", 0)))
world = int(os.environ.get("SLURM_NTASKS", os.environ.get("WORLD_SIZE", 1)))
local_rank = int(os.environ.get("SLURM_LOCALID", os.environ.get("LOCAL_RANK", 0)))
torch.cuda.set_device(local_rank)
dist.init_process_group(
"nccl", rank=rank, world_size=world, device_id=torch.device("cuda", local_rank)
)
cpu_group = dist.new_group(backend="gloo")
group = SimpleNamespace(cpu_group=cpu_group, rank_in_group=rank, world_size=world)
layout = EngramLayout.from_config(CONFIG)
n = layout.num_embeddings[0]
col_bounds = layout.column_bounds(0)
token_list = [
int(x) for x in os.environ.get("TOKENS", "1 64 256 1024 16384").split()
]
split = os.environ.get("SPLIT", "auto") # auto (column if it divides) | rows
shard_bounds = col_bounds if split == "auto" else (0, n)
row_start, row_end = shard_range(n, shard_bounds, rank, world)
rows = row_end - row_start
t0 = time.perf_counter()
peer = _PeerShards(rows, DIM, group)
weight, scale = peer.views()
fill_shard(weight, scale, seed=1000 + rank)
torch.cuda.synchronize()
t_alloc = time.perf_counter() - t0
t0 = time.perf_counter()
peer.connect(row_start, "test")
t_connect = time.perf_counter() - t0
if rank == 0:
print(
f"world={world} split={split} rows/rank~{rows} shard={peer.size / 2**30:.2f} GiB "
f"alloc+fill={t_alloc:.1f}s connect={t_connect:.1f}s",
flush=True,
)
ok = True
for tokens in token_list:
ids = random_ids(col_bounds, tokens, seed=7 * tokens + rank, device="cuda")
# The model passes a strided [T, 24] slice of [T, n_layers, 24].
both = torch.stack([ids, ids], dim=1)
got = peer.lookup(both[:, 0])
ref = reference(ids, weight, scale, row_start, rows, world)
g16, r16 = got.view(torch.int16), ref.view(torch.int16)
bit_diff = int((g16 != r16).sum())
val_diff = int((got != ref).sum())
zero_sign = int(((g16 != r16) & (got == 0) & (ref == 0)).sum())
# The same lookup inside a CUDA graph.
static_ids = both[:, 0].clone()
graph = torch.cuda.CUDAGraph()
s = torch.cuda.Stream()
s.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(s):
peer.lookup(static_ids)
with torch.cuda.graph(graph, stream=s):
graph_out = peer.lookup(static_ids)
torch.cuda.current_stream().wait_stream(s)
graph.replay()
torch.cuda.synchronize()
graph_diff = int((graph_out.view(torch.int16) != g16).sum())
t_peer = bench(lambda: peer.lookup(both[:, 0]))
t_ref = bench(lambda: reference(ids, weight, scale, row_start, rows, world))
res = torch.tensor([bit_diff, val_diff, zero_sign, graph_diff], device="cuda")
dist.all_reduce(res)
times = torch.tensor([t_peer, t_ref], device="cuda")
tmax = times.clone()
dist.all_reduce(tmax, op=dist.ReduceOp.MAX)
if rank == 0:
print(
f"tokens/rank={tokens}: bit_mismatch={int(res[0])} value_mismatch={int(res[1])} "
f"zero_sign_only={int(res[2])} graph_mismatch={int(res[3])} "
f"peer={t_peer:.1f}us (max {tmax[0]:.1f}) current={t_ref:.1f}us (max {tmax[1]:.1f})",
flush=True,
)
ok &= int(res[1]) == 0 and int(res[3]) == 0
dist.barrier()
if rank == 0:
print("PASS" if ok else "FAIL", flush=True)
dist.destroy_process_group()
if __name__ == "__main__":
main()