[Perf] dsv41: Read Engram rows from peer device shards (#46180)

Co-authored-by: kpham-sgl <khoa.pham@radixark.ai>
This commit is contained in:
Liangsheng Yin
2026-10-23 19:01:26 -07:00
committed by GitHub
co-authored by kpham-sgl
parent 327c906764
commit 79701a2549
7 changed files with 532 additions and 26 deletions
@@ -22,6 +22,14 @@ register_kernel(
)
)
register_kernel(
KernelSpec(
op="embeddings.engram_gather_peer",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.embeddings.engram_gather:engram_gather_peer",
)
)
register_kernel(
KernelSpec(
op="embeddings.engram_hash_ids",
@@ -96,3 +96,92 @@ def engram_gather(
HAS_DST=dst is not None,
)
return out
@triton.jit
def _engram_gather_peer_kernel(
ids_ptr,
ids_stride,
out_ptr,
w_bias_ptr,
s_bias_ptr,
starts_ptr,
n_cols,
DIM: tl.constexpr,
BLK: tl.constexpr,
ROWS: tl.constexpr,
LOG2_WORLD: tl.constexpr,
E8M0_ZERO: tl.constexpr,
):
tok = tl.program_id(0).to(tl.int64)
cols = tl.program_id(1) * ROWS + tl.arange(0, ROWS)
col_ok = cols < n_cols
idx = tl.load(ids_ptr + tok * ids_stride + cols, mask=col_ok, other=0).to(tl.int64)
# Owner = the last shard whose first row is <= idx; starts is padded to a
# power of two with int64 max, so the search never runs past a real shard.
owner = tl.zeros([ROWS], dtype=tl.int32)
for step in tl.static_range(LOG2_WORLD):
probe = owner + (1 << (LOG2_WORLD - 1 - step))
start = tl.load(starts_ptr + probe)
owner = tl.where(start <= idx, probe, owner)
# Bases are pre-biased by each shard's first row, so a global id indexes them.
w_row = tl.load(w_bias_ptr + owner) + idx * DIM
s_row = tl.load(s_bias_ptr + owner) + idx * (DIM // BLK)
offs = tl.arange(0, DIM)
w = (w_row[:, None] + offs[None, :]).to(tl.pointer_type(tl.float8e4nv))
s = (s_row[:, None] + (offs // BLK)[None, :]).to(tl.pointer_type(tl.uint8))
mask = col_ok[:, None]
vals = tl.load(w, mask=mask, other=0.0).to(tl.float32)
exps = tl.load(s, mask=mask, other=0).to(tl.int32)
scale = (exps << 23).to(tl.float32, bitcast=True)
scale = tl.where(exps == 0, E8M0_ZERO, scale)
# -0 becomes +0, as the zero-fill + sum of the sharded lookup leaves it, so the
# two lookups agree bit for bit.
out = vals * scale
out = tl.where(out == 0.0, 0.0, out)
dst = out_ptr + (tok * n_cols + cols)[:, None].to(tl.int64) * DIM + offs[None, :]
tl.store(dst, out.to(tl.bfloat16), mask=mask)
def engram_gather_peer(
ids: torch.Tensor,
out: torch.Tensor,
w_bias: torch.Tensor,
s_bias: torch.Tensor,
starts: torch.Tensor,
dim: int,
block_size: int,
) -> torch.Tensor:
"""Gather ``ids`` ([T, C] int, unit column stride) into ``out`` ([T, C, dim]
bf16, contiguous), reading each id from the shard that owns it.
``starts`` ([P] int64, P a power of two) holds each shard's first global row,
padded with int64 max; ``w_bias`` / ``s_bias`` ([P] int64) are each shard's fp8
row / e8m0 scale base address minus its first row's offset, so any rank's
shard (local or a peer mapping over NVLink) is addressed by the global id.
"""
assert ids.dim() == 2 and ids.stride(1) == 1, (ids.shape, ids.stride())
assert ids.dtype in (torch.int32, torch.int64)
assert out.is_contiguous() and out.dtype == torch.bfloat16
assert out.shape == (*ids.shape, dim), (out.shape, ids.shape)
world = starts.numel()
assert world & (world - 1) == 0, world
t, c = ids.shape
if t and c:
rows = 8
_engram_gather_peer_kernel[(t, triton.cdiv(c, rows))](
ids,
ids.stride(0),
out,
w_bias,
s_bias,
starts,
c,
DIM=dim,
BLK=block_size,
ROWS=rows,
LOG2_WORLD=world.bit_length() - 1,
E8M0_ZERO=_E8M0_ZERO,
num_warps=4,
)
return out
+5
View File
@@ -1777,6 +1777,11 @@ class Envs:
# Scheduler-side staging of shared host-table rows on GPUs without ATS
# (engram_staging.py); pays off for large prefill batches.
SGLANG_OPT_DSV41_ENGRAM_HOST_STAGING = EnvBool(False)
# Keep each rank's engram row shard in exportable device memory (CUDA VMM,
# FABRIC handles across an NVLink domain) mapped by every rank of the TP group:
# a rank reads its own tokens' rows from the owning shard, with no id gather
# and no reduce-scatter. Takes precedence over the host table.
SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS = EnvBool(False)
# Kernels and indexer
SGLANG_OPT_DEEPGEMM_HC_PRENORM = EnvBool(True)
+181 -3
View File
@@ -23,7 +23,10 @@ import torch
from torch import nn
from sglang.kernels.ops.embeddings.engram_gate import fused_engram_gate
from sglang.kernels.ops.embeddings.engram_gather import engram_gather
from sglang.kernels.ops.embeddings.engram_gather import (
engram_gather,
engram_gather_peer,
)
from sglang.kernels.ops.embeddings.engram_hash import (
MODE_DECODE,
MODE_EXTEND,
@@ -795,6 +798,161 @@ class _HostTable:
logger.info(msg)
@functools.cache
def _warn_host_table_ignored() -> None:
logger.warning(
"SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS takes precedence: the engram tables "
"stay in device memory and SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE is ignored"
)
class _PeerShards:
"""One rank's engram row shard in exportable device memory (CUDA VMM), with
every TP peer's shard mapped into this process after `connect`.
Inside one NVLink domain (MNNVL included, through FABRIC handles) a kernel
reads any peer's rows in place, so a rank looks up only its own tokens: no
id gather, no [DP x tokens] buffer, no reduce-scatter. The shard holds the
fp8 rows followed by their e8m0 scales. Lives for the whole process.
"""
def __init__(self, rows: int, dim: int, group):
from sglang.srt.utils.cuda_vmm_utils import (
VmmReservation,
align_up,
get_device_granularity,
make_device_allocation_prop,
)
self.rows = rows
self.dim = dim
self.group = group
self.device_id = torch.cuda.current_device()
self.w_bytes = rows * dim
self.s_bytes = rows * (dim // FP8_BLOCK_SIZE)
gran = get_device_granularity(self.device_id)
self.size = align_up(max(1, self.w_bytes + self.s_bytes), gran)
self.local = VmmReservation(
self.size,
make_device_allocation_prop(self.device_id),
self.device_id,
alignment=gran,
)
self.handle = self.local.map(0, self.size, retain_handle=True)
self.base = self.local.base
self.w_bias: Optional[torch.Tensor] = None
self.s_bias: Optional[torch.Tensor] = None
self.starts: Optional[torch.Tensor] = None
def views(self) -> tuple[torch.Tensor, torch.Tensor]:
from sglang.srt.utils.cuda_vmm_utils import tensor_from_pointer
raw = tensor_from_pointer(self.base, self.size, device_id=self.device_id)
weight = raw[: self.w_bytes].view(torch.float8_e4m3fn).view(self.rows, self.dim)
scale = (
raw[self.w_bytes : self.w_bytes + self.s_bytes]
.view(torch.float8_e8m0fnu)
.view(self.rows, self.dim // FP8_BLOCK_SIZE)
)
return weight, scale
@property
def ready(self) -> bool:
return self.starts is not None
def connect(self, row_start: int, label: str) -> None:
"""Collective over the TP group: export this shard, map every peer's."""
if self.ready:
return
import torch.distributed as dist
from sglang.srt.utils.cuda_vmm_utils import (
exchange_shareable_handles,
import_and_map_alloc,
)
t0 = time.perf_counter()
torch.cuda.synchronize()
cpu_group = self.group.cpu_group
rank, world = self.group.rank_in_group, self.group.world_size
use_fabric, peer_handles = exchange_shareable_handles(
[self.handle], cpu_group, rank, world
)
meta = [None] * world
dist.all_gather_object(meta, (row_start, self.rows, self.size), group=cpu_group)
bases = []
try:
for r, (_, _, size) in enumerate(meta):
if r == rank:
bases.append(self.base)
continue
handle = peer_handles[(r, 0)]
bases.append(
import_and_map_alloc(
handle if use_fabric else None,
None if use_fabric else handle,
size,
self.device_id,
use_fabric=use_fabric,
peer_rank=r,
)
)
finally:
if not use_fabric:
for fd in peer_handles.values():
try:
os.close(fd)
except OSError:
pass
padded = 1 << max(0, (world - 1).bit_length())
int64_max = torch.iinfo(torch.int64).max
starts, w_bias, s_bias = [], [], []
n_scales = self.dim // FP8_BLOCK_SIZE
for r in range(padded):
if r < world:
start, rows = meta[r][0], meta[r][1]
starts.append(start)
w_bias.append(bases[r] - start * self.dim)
s_bias.append(bases[r] + rows * self.dim - start * n_scales)
else:
starts.append(int64_max)
w_bias.append(0)
s_bias.append(0)
assert starts[0] == 0 and starts[:world] == sorted(starts[:world]), starts
device = torch.device("cuda", self.device_id)
self.w_bias = torch.tensor(w_bias, dtype=torch.int64, device=device)
self.s_bias = torch.tensor(s_bias, dtype=torch.int64, device=device)
self.starts = torch.tensor(starts, dtype=torch.int64, device=device)
torch.cuda.synchronize()
dist.barrier(group=cpu_group)
logger.info(
"engram peer shards %s: %d ranks over %s, %.2f GiB local, mapped in %.1f s",
label,
world,
"FABRIC" if use_fabric else "POSIX fd",
self.size / 2**30,
time.perf_counter() - t0,
)
def lookup(self, indices: torch.Tensor) -> torch.Tensor:
flat = indices.reshape(-1, indices.shape[-1])
if flat.stride(-1) != 1:
flat = flat.contiguous()
out = torch.empty(
*flat.shape, self.dim, dtype=torch.bfloat16, device=indices.device
)
engram_gather_peer(
flat,
out,
self.w_bias,
self.s_bias,
self.starts,
self.dim,
FP8_BLOCK_SIZE,
)
return out.view(*indices.shape, self.dim)
def shard_range(
num_embeddings: int,
column_bounds: tuple[int, ...],
@@ -848,7 +1006,19 @@ class EngramEmbedding(nn.Module):
self.host_table: Optional[_HostTable] = None
self._sort_host_reads = False
self.staged_by_scheduler = False
if envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get():
self.peer: Optional[_PeerShards] = None
if (
envs.SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS.get()
and self.tp_size > 1
and is_cuda()
):
if envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get() and tp_rank == 0:
_warn_host_table_ignored()
self.peer = _PeerShards(self.rows, dim, get_parallel().tp_group)
weight, scale = self.peer.views()
self.weight = nn.Parameter(weight, requires_grad=False)
self.scale = nn.Parameter(scale, requires_grad=False)
elif envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get():
self._init_host_table(num_embeddings, dim, layer_id)
else:
self.weight = nn.Parameter(
@@ -914,7 +1084,10 @@ class EngramEmbedding(nn.Module):
self.host_table.dirty = True
def finish_load(self, label: str):
"""Collective in the shared layout: every rank calls it after loading."""
"""Collective in the shared layout and with peer shards: every rank calls
it after loading."""
if self.peer is not None:
self.peer.connect(self.row_start, label)
if self.host_table is not None:
self.host_table.finish_load(label)
@@ -925,6 +1098,11 @@ class EngramEmbedding(nn.Module):
*,
cp_all_tokens: bool = False,
) -> torch.Tensor:
if self.peer is not None:
assert self.peer.ready, "engram peer shards are mapped in finish_load"
# Every shard is mapped here: read this rank's own tokens only, which
# also covers the prefill CP and DP paths below without a collective.
return self.peer.lookup(indices)
if self._shared:
if indices.shape[0] == 0:
return self._empty(indices)
+9 -23
View File
@@ -21,8 +21,7 @@ from sglang.srt.utils.cuda_vmm_utils import (
align_down,
align_up,
check_drv,
exchange_posix_fds,
export_shareable_handles,
exchange_shareable_handles,
get_device_granularity,
import_peer_handle,
make_device_allocation_prop,
@@ -138,11 +137,6 @@ class DWDPTransport:
prop = make_device_allocation_prop(device_id)
handle_list = [self._handle_set.get_handle(li, n) for li, n in sorted_keys]
fabric_handles, local_posix_fds, use_fabric = export_shareable_handles(
handle_list, cpu_group, layout.dwdp_rank
)
peer_fds: Dict[Tuple[int, int], int] = {}
key_counts = [None] * layout.dwdp_size
dist.all_gather_object(key_counts, len(sorted_keys), group=cpu_group)
if any(count != len(sorted_keys) for count in key_counts):
@@ -150,18 +144,9 @@ class DWDPTransport:
f"Mismatched DWDP weight handle counts across ranks: {key_counts}"
)
if use_fabric:
all_fabric = [None] * layout.dwdp_size
dist.all_gather_object(all_fabric, fabric_handles, group=cpu_group)
else:
all_fabric = None
peer_fds = exchange_posix_fds(
cpu_group,
layout.dwdp_rank,
layout.dwdp_size,
local_posix_fds,
key_counts,
)
use_fabric, peer_handles = exchange_shareable_handles(
handle_list, cpu_group, layout.dwdp_rank, layout.dwdp_size
)
logger.info(
"DWDP handle exchange via %s (%d handles)",
"FABRIC" if use_fabric else "POSIX fd",
@@ -175,8 +160,9 @@ class DWDPTransport:
if peer_rank == layout.dwdp_rank:
continue
fabric_handle = all_fabric[peer_rank][key_idx] if use_fabric else None
fd = None if use_fabric else peer_fds[(peer_rank, key_idx)]
handle = peer_handles[(peer_rank, key_idx)]
fabric_handle = handle if use_fabric else None
fd = None if use_fabric else handle
peer_handle = import_peer_handle(
fabric_handle, fd, use_fabric=use_fabric, peer_rank=peer_rank
)
@@ -209,8 +195,8 @@ class DWDPTransport:
)
self._peer_views[(peer_rank, layer_idx, name)] = peer_tensor
_close_fds(local_posix_fds)
_close_fds(peer_fds.values())
if not use_fabric:
_close_fds(peer_handles.values())
@property
def handle_set(self) -> MnnvlHandleSet:
+44
View File
@@ -880,6 +880,50 @@ def exchange_posix_fds(
pass
def exchange_shareable_handles(
retained_handles, group: ProcessGroup, rank: int, world_size: int
):
"""Export this rank's retained VMM handles and collect every peer's.
FABRIC handles when every rank can export them; otherwise POSIX fds over UNIX
sockets, which only reach ranks on this host, so a group spanning hosts raises.
Returns ``(use_fabric, peer_handles)``: ``peer_handles[(peer_rank, idx)]`` is a
FABRIC handle or a received fd the caller must close. This rank's exported fds
are closed here.
"""
fabric_handles, local_fds, use_fabric = export_shareable_handles(
retained_handles, group, rank
)
if use_fabric:
all_fabric = [None] * world_size
dist.all_gather_object(all_fabric, fabric_handles, group=group)
return True, {
(r, i): handle
for r, handles in enumerate(all_fabric)
if r != rank
for i, handle in enumerate(handles)
}
try:
info = [None] * world_size
dist.all_gather_object(
info, (socket.gethostname(), len(local_fds)), group=group
)
if len({host for host, _ in info}) > 1:
raise RuntimeError(
"VMM handles cannot reach ranks on other hosts without FABRIC "
"handles (one NVLink domain with IMEX)"
)
return False, exchange_posix_fds(
group, rank, world_size, local_fds, [n for _, n in info]
)
finally:
for fd in local_fds:
try:
os.close(fd)
except OSError:
pass
def import_peer_handle(fabric_handle, fd, *, use_fabric: bool, peer_rank: int):
"""Import a peer allocation handle (FABRIC or POSIX fd). Returns the handle.