[Perf] dsv41: Read Engram rows from peer device shards (#46180)
Co-authored-by: kpham-sgl <khoa.pham@radixark.ai>
This commit is contained in:
co-authored by
kpham-sgl
parent
327c906764
commit
79701a2549
@@ -22,6 +22,14 @@ register_kernel(
|
||||
)
|
||||
)
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="embeddings.engram_gather_peer",
|
||||
backend=KernelBackend.TRITON,
|
||||
target="sglang.kernels.ops.embeddings.engram_gather:engram_gather_peer",
|
||||
)
|
||||
)
|
||||
|
||||
register_kernel(
|
||||
KernelSpec(
|
||||
op="embeddings.engram_hash_ids",
|
||||
|
||||
@@ -96,3 +96,92 @@ def engram_gather(
|
||||
HAS_DST=dst is not None,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _engram_gather_peer_kernel(
|
||||
ids_ptr,
|
||||
ids_stride,
|
||||
out_ptr,
|
||||
w_bias_ptr,
|
||||
s_bias_ptr,
|
||||
starts_ptr,
|
||||
n_cols,
|
||||
DIM: tl.constexpr,
|
||||
BLK: tl.constexpr,
|
||||
ROWS: tl.constexpr,
|
||||
LOG2_WORLD: tl.constexpr,
|
||||
E8M0_ZERO: tl.constexpr,
|
||||
):
|
||||
tok = tl.program_id(0).to(tl.int64)
|
||||
cols = tl.program_id(1) * ROWS + tl.arange(0, ROWS)
|
||||
col_ok = cols < n_cols
|
||||
idx = tl.load(ids_ptr + tok * ids_stride + cols, mask=col_ok, other=0).to(tl.int64)
|
||||
# Owner = the last shard whose first row is <= idx; starts is padded to a
|
||||
# power of two with int64 max, so the search never runs past a real shard.
|
||||
owner = tl.zeros([ROWS], dtype=tl.int32)
|
||||
for step in tl.static_range(LOG2_WORLD):
|
||||
probe = owner + (1 << (LOG2_WORLD - 1 - step))
|
||||
start = tl.load(starts_ptr + probe)
|
||||
owner = tl.where(start <= idx, probe, owner)
|
||||
# Bases are pre-biased by each shard's first row, so a global id indexes them.
|
||||
w_row = tl.load(w_bias_ptr + owner) + idx * DIM
|
||||
s_row = tl.load(s_bias_ptr + owner) + idx * (DIM // BLK)
|
||||
offs = tl.arange(0, DIM)
|
||||
w = (w_row[:, None] + offs[None, :]).to(tl.pointer_type(tl.float8e4nv))
|
||||
s = (s_row[:, None] + (offs // BLK)[None, :]).to(tl.pointer_type(tl.uint8))
|
||||
mask = col_ok[:, None]
|
||||
vals = tl.load(w, mask=mask, other=0.0).to(tl.float32)
|
||||
exps = tl.load(s, mask=mask, other=0).to(tl.int32)
|
||||
scale = (exps << 23).to(tl.float32, bitcast=True)
|
||||
scale = tl.where(exps == 0, E8M0_ZERO, scale)
|
||||
# -0 becomes +0, as the zero-fill + sum of the sharded lookup leaves it, so the
|
||||
# two lookups agree bit for bit.
|
||||
out = vals * scale
|
||||
out = tl.where(out == 0.0, 0.0, out)
|
||||
dst = out_ptr + (tok * n_cols + cols)[:, None].to(tl.int64) * DIM + offs[None, :]
|
||||
tl.store(dst, out.to(tl.bfloat16), mask=mask)
|
||||
|
||||
|
||||
def engram_gather_peer(
|
||||
ids: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
w_bias: torch.Tensor,
|
||||
s_bias: torch.Tensor,
|
||||
starts: torch.Tensor,
|
||||
dim: int,
|
||||
block_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Gather ``ids`` ([T, C] int, unit column stride) into ``out`` ([T, C, dim]
|
||||
bf16, contiguous), reading each id from the shard that owns it.
|
||||
|
||||
``starts`` ([P] int64, P a power of two) holds each shard's first global row,
|
||||
padded with int64 max; ``w_bias`` / ``s_bias`` ([P] int64) are each shard's fp8
|
||||
row / e8m0 scale base address minus its first row's offset, so any rank's
|
||||
shard (local or a peer mapping over NVLink) is addressed by the global id.
|
||||
"""
|
||||
assert ids.dim() == 2 and ids.stride(1) == 1, (ids.shape, ids.stride())
|
||||
assert ids.dtype in (torch.int32, torch.int64)
|
||||
assert out.is_contiguous() and out.dtype == torch.bfloat16
|
||||
assert out.shape == (*ids.shape, dim), (out.shape, ids.shape)
|
||||
world = starts.numel()
|
||||
assert world & (world - 1) == 0, world
|
||||
t, c = ids.shape
|
||||
if t and c:
|
||||
rows = 8
|
||||
_engram_gather_peer_kernel[(t, triton.cdiv(c, rows))](
|
||||
ids,
|
||||
ids.stride(0),
|
||||
out,
|
||||
w_bias,
|
||||
s_bias,
|
||||
starts,
|
||||
c,
|
||||
DIM=dim,
|
||||
BLK=block_size,
|
||||
ROWS=rows,
|
||||
LOG2_WORLD=world.bit_length() - 1,
|
||||
E8M0_ZERO=_E8M0_ZERO,
|
||||
num_warps=4,
|
||||
)
|
||||
return out
|
||||
|
||||
@@ -1777,6 +1777,11 @@ class Envs:
|
||||
# Scheduler-side staging of shared host-table rows on GPUs without ATS
|
||||
# (engram_staging.py); pays off for large prefill batches.
|
||||
SGLANG_OPT_DSV41_ENGRAM_HOST_STAGING = EnvBool(False)
|
||||
# Keep each rank's engram row shard in exportable device memory (CUDA VMM,
|
||||
# FABRIC handles across an NVLink domain) mapped by every rank of the TP group:
|
||||
# a rank reads its own tokens' rows from the owning shard, with no id gather
|
||||
# and no reduce-scatter. Takes precedence over the host table.
|
||||
SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS = EnvBool(False)
|
||||
|
||||
# Kernels and indexer
|
||||
SGLANG_OPT_DEEPGEMM_HC_PRENORM = EnvBool(True)
|
||||
|
||||
@@ -23,7 +23,10 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from sglang.kernels.ops.embeddings.engram_gate import fused_engram_gate
|
||||
from sglang.kernels.ops.embeddings.engram_gather import engram_gather
|
||||
from sglang.kernels.ops.embeddings.engram_gather import (
|
||||
engram_gather,
|
||||
engram_gather_peer,
|
||||
)
|
||||
from sglang.kernels.ops.embeddings.engram_hash import (
|
||||
MODE_DECODE,
|
||||
MODE_EXTEND,
|
||||
@@ -795,6 +798,161 @@ class _HostTable:
|
||||
logger.info(msg)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _warn_host_table_ignored() -> None:
|
||||
logger.warning(
|
||||
"SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS takes precedence: the engram tables "
|
||||
"stay in device memory and SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE is ignored"
|
||||
)
|
||||
|
||||
|
||||
class _PeerShards:
|
||||
"""One rank's engram row shard in exportable device memory (CUDA VMM), with
|
||||
every TP peer's shard mapped into this process after `connect`.
|
||||
|
||||
Inside one NVLink domain (MNNVL included, through FABRIC handles) a kernel
|
||||
reads any peer's rows in place, so a rank looks up only its own tokens: no
|
||||
id gather, no [DP x tokens] buffer, no reduce-scatter. The shard holds the
|
||||
fp8 rows followed by their e8m0 scales. Lives for the whole process.
|
||||
"""
|
||||
|
||||
def __init__(self, rows: int, dim: int, group):
|
||||
from sglang.srt.utils.cuda_vmm_utils import (
|
||||
VmmReservation,
|
||||
align_up,
|
||||
get_device_granularity,
|
||||
make_device_allocation_prop,
|
||||
)
|
||||
|
||||
self.rows = rows
|
||||
self.dim = dim
|
||||
self.group = group
|
||||
self.device_id = torch.cuda.current_device()
|
||||
self.w_bytes = rows * dim
|
||||
self.s_bytes = rows * (dim // FP8_BLOCK_SIZE)
|
||||
gran = get_device_granularity(self.device_id)
|
||||
self.size = align_up(max(1, self.w_bytes + self.s_bytes), gran)
|
||||
self.local = VmmReservation(
|
||||
self.size,
|
||||
make_device_allocation_prop(self.device_id),
|
||||
self.device_id,
|
||||
alignment=gran,
|
||||
)
|
||||
self.handle = self.local.map(0, self.size, retain_handle=True)
|
||||
self.base = self.local.base
|
||||
self.w_bias: Optional[torch.Tensor] = None
|
||||
self.s_bias: Optional[torch.Tensor] = None
|
||||
self.starts: Optional[torch.Tensor] = None
|
||||
|
||||
def views(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from sglang.srt.utils.cuda_vmm_utils import tensor_from_pointer
|
||||
|
||||
raw = tensor_from_pointer(self.base, self.size, device_id=self.device_id)
|
||||
weight = raw[: self.w_bytes].view(torch.float8_e4m3fn).view(self.rows, self.dim)
|
||||
scale = (
|
||||
raw[self.w_bytes : self.w_bytes + self.s_bytes]
|
||||
.view(torch.float8_e8m0fnu)
|
||||
.view(self.rows, self.dim // FP8_BLOCK_SIZE)
|
||||
)
|
||||
return weight, scale
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
return self.starts is not None
|
||||
|
||||
def connect(self, row_start: int, label: str) -> None:
|
||||
"""Collective over the TP group: export this shard, map every peer's."""
|
||||
if self.ready:
|
||||
return
|
||||
import torch.distributed as dist
|
||||
|
||||
from sglang.srt.utils.cuda_vmm_utils import (
|
||||
exchange_shareable_handles,
|
||||
import_and_map_alloc,
|
||||
)
|
||||
|
||||
t0 = time.perf_counter()
|
||||
torch.cuda.synchronize()
|
||||
cpu_group = self.group.cpu_group
|
||||
rank, world = self.group.rank_in_group, self.group.world_size
|
||||
use_fabric, peer_handles = exchange_shareable_handles(
|
||||
[self.handle], cpu_group, rank, world
|
||||
)
|
||||
meta = [None] * world
|
||||
dist.all_gather_object(meta, (row_start, self.rows, self.size), group=cpu_group)
|
||||
bases = []
|
||||
try:
|
||||
for r, (_, _, size) in enumerate(meta):
|
||||
if r == rank:
|
||||
bases.append(self.base)
|
||||
continue
|
||||
handle = peer_handles[(r, 0)]
|
||||
bases.append(
|
||||
import_and_map_alloc(
|
||||
handle if use_fabric else None,
|
||||
None if use_fabric else handle,
|
||||
size,
|
||||
self.device_id,
|
||||
use_fabric=use_fabric,
|
||||
peer_rank=r,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
if not use_fabric:
|
||||
for fd in peer_handles.values():
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
padded = 1 << max(0, (world - 1).bit_length())
|
||||
int64_max = torch.iinfo(torch.int64).max
|
||||
starts, w_bias, s_bias = [], [], []
|
||||
n_scales = self.dim // FP8_BLOCK_SIZE
|
||||
for r in range(padded):
|
||||
if r < world:
|
||||
start, rows = meta[r][0], meta[r][1]
|
||||
starts.append(start)
|
||||
w_bias.append(bases[r] - start * self.dim)
|
||||
s_bias.append(bases[r] + rows * self.dim - start * n_scales)
|
||||
else:
|
||||
starts.append(int64_max)
|
||||
w_bias.append(0)
|
||||
s_bias.append(0)
|
||||
assert starts[0] == 0 and starts[:world] == sorted(starts[:world]), starts
|
||||
device = torch.device("cuda", self.device_id)
|
||||
self.w_bias = torch.tensor(w_bias, dtype=torch.int64, device=device)
|
||||
self.s_bias = torch.tensor(s_bias, dtype=torch.int64, device=device)
|
||||
self.starts = torch.tensor(starts, dtype=torch.int64, device=device)
|
||||
torch.cuda.synchronize()
|
||||
dist.barrier(group=cpu_group)
|
||||
logger.info(
|
||||
"engram peer shards %s: %d ranks over %s, %.2f GiB local, mapped in %.1f s",
|
||||
label,
|
||||
world,
|
||||
"FABRIC" if use_fabric else "POSIX fd",
|
||||
self.size / 2**30,
|
||||
time.perf_counter() - t0,
|
||||
)
|
||||
|
||||
def lookup(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
flat = indices.reshape(-1, indices.shape[-1])
|
||||
if flat.stride(-1) != 1:
|
||||
flat = flat.contiguous()
|
||||
out = torch.empty(
|
||||
*flat.shape, self.dim, dtype=torch.bfloat16, device=indices.device
|
||||
)
|
||||
engram_gather_peer(
|
||||
flat,
|
||||
out,
|
||||
self.w_bias,
|
||||
self.s_bias,
|
||||
self.starts,
|
||||
self.dim,
|
||||
FP8_BLOCK_SIZE,
|
||||
)
|
||||
return out.view(*indices.shape, self.dim)
|
||||
|
||||
|
||||
def shard_range(
|
||||
num_embeddings: int,
|
||||
column_bounds: tuple[int, ...],
|
||||
@@ -848,7 +1006,19 @@ class EngramEmbedding(nn.Module):
|
||||
self.host_table: Optional[_HostTable] = None
|
||||
self._sort_host_reads = False
|
||||
self.staged_by_scheduler = False
|
||||
if envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get():
|
||||
self.peer: Optional[_PeerShards] = None
|
||||
if (
|
||||
envs.SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS.get()
|
||||
and self.tp_size > 1
|
||||
and is_cuda()
|
||||
):
|
||||
if envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get() and tp_rank == 0:
|
||||
_warn_host_table_ignored()
|
||||
self.peer = _PeerShards(self.rows, dim, get_parallel().tp_group)
|
||||
weight, scale = self.peer.views()
|
||||
self.weight = nn.Parameter(weight, requires_grad=False)
|
||||
self.scale = nn.Parameter(scale, requires_grad=False)
|
||||
elif envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get():
|
||||
self._init_host_table(num_embeddings, dim, layer_id)
|
||||
else:
|
||||
self.weight = nn.Parameter(
|
||||
@@ -914,7 +1084,10 @@ class EngramEmbedding(nn.Module):
|
||||
self.host_table.dirty = True
|
||||
|
||||
def finish_load(self, label: str):
|
||||
"""Collective in the shared layout: every rank calls it after loading."""
|
||||
"""Collective in the shared layout and with peer shards: every rank calls
|
||||
it after loading."""
|
||||
if self.peer is not None:
|
||||
self.peer.connect(self.row_start, label)
|
||||
if self.host_table is not None:
|
||||
self.host_table.finish_load(label)
|
||||
|
||||
@@ -925,6 +1098,11 @@ class EngramEmbedding(nn.Module):
|
||||
*,
|
||||
cp_all_tokens: bool = False,
|
||||
) -> torch.Tensor:
|
||||
if self.peer is not None:
|
||||
assert self.peer.ready, "engram peer shards are mapped in finish_load"
|
||||
# Every shard is mapped here: read this rank's own tokens only, which
|
||||
# also covers the prefill CP and DP paths below without a collective.
|
||||
return self.peer.lookup(indices)
|
||||
if self._shared:
|
||||
if indices.shape[0] == 0:
|
||||
return self._empty(indices)
|
||||
|
||||
@@ -21,8 +21,7 @@ from sglang.srt.utils.cuda_vmm_utils import (
|
||||
align_down,
|
||||
align_up,
|
||||
check_drv,
|
||||
exchange_posix_fds,
|
||||
export_shareable_handles,
|
||||
exchange_shareable_handles,
|
||||
get_device_granularity,
|
||||
import_peer_handle,
|
||||
make_device_allocation_prop,
|
||||
@@ -138,11 +137,6 @@ class DWDPTransport:
|
||||
prop = make_device_allocation_prop(device_id)
|
||||
|
||||
handle_list = [self._handle_set.get_handle(li, n) for li, n in sorted_keys]
|
||||
fabric_handles, local_posix_fds, use_fabric = export_shareable_handles(
|
||||
handle_list, cpu_group, layout.dwdp_rank
|
||||
)
|
||||
peer_fds: Dict[Tuple[int, int], int] = {}
|
||||
|
||||
key_counts = [None] * layout.dwdp_size
|
||||
dist.all_gather_object(key_counts, len(sorted_keys), group=cpu_group)
|
||||
if any(count != len(sorted_keys) for count in key_counts):
|
||||
@@ -150,18 +144,9 @@ class DWDPTransport:
|
||||
f"Mismatched DWDP weight handle counts across ranks: {key_counts}"
|
||||
)
|
||||
|
||||
if use_fabric:
|
||||
all_fabric = [None] * layout.dwdp_size
|
||||
dist.all_gather_object(all_fabric, fabric_handles, group=cpu_group)
|
||||
else:
|
||||
all_fabric = None
|
||||
peer_fds = exchange_posix_fds(
|
||||
cpu_group,
|
||||
layout.dwdp_rank,
|
||||
layout.dwdp_size,
|
||||
local_posix_fds,
|
||||
key_counts,
|
||||
)
|
||||
use_fabric, peer_handles = exchange_shareable_handles(
|
||||
handle_list, cpu_group, layout.dwdp_rank, layout.dwdp_size
|
||||
)
|
||||
logger.info(
|
||||
"DWDP handle exchange via %s (%d handles)",
|
||||
"FABRIC" if use_fabric else "POSIX fd",
|
||||
@@ -175,8 +160,9 @@ class DWDPTransport:
|
||||
if peer_rank == layout.dwdp_rank:
|
||||
continue
|
||||
|
||||
fabric_handle = all_fabric[peer_rank][key_idx] if use_fabric else None
|
||||
fd = None if use_fabric else peer_fds[(peer_rank, key_idx)]
|
||||
handle = peer_handles[(peer_rank, key_idx)]
|
||||
fabric_handle = handle if use_fabric else None
|
||||
fd = None if use_fabric else handle
|
||||
peer_handle = import_peer_handle(
|
||||
fabric_handle, fd, use_fabric=use_fabric, peer_rank=peer_rank
|
||||
)
|
||||
@@ -209,8 +195,8 @@ class DWDPTransport:
|
||||
)
|
||||
self._peer_views[(peer_rank, layer_idx, name)] = peer_tensor
|
||||
|
||||
_close_fds(local_posix_fds)
|
||||
_close_fds(peer_fds.values())
|
||||
if not use_fabric:
|
||||
_close_fds(peer_handles.values())
|
||||
|
||||
@property
|
||||
def handle_set(self) -> MnnvlHandleSet:
|
||||
|
||||
@@ -880,6 +880,50 @@ def exchange_posix_fds(
|
||||
pass
|
||||
|
||||
|
||||
def exchange_shareable_handles(
|
||||
retained_handles, group: ProcessGroup, rank: int, world_size: int
|
||||
):
|
||||
"""Export this rank's retained VMM handles and collect every peer's.
|
||||
|
||||
FABRIC handles when every rank can export them; otherwise POSIX fds over UNIX
|
||||
sockets, which only reach ranks on this host, so a group spanning hosts raises.
|
||||
Returns ``(use_fabric, peer_handles)``: ``peer_handles[(peer_rank, idx)]`` is a
|
||||
FABRIC handle or a received fd the caller must close. This rank's exported fds
|
||||
are closed here.
|
||||
"""
|
||||
fabric_handles, local_fds, use_fabric = export_shareable_handles(
|
||||
retained_handles, group, rank
|
||||
)
|
||||
if use_fabric:
|
||||
all_fabric = [None] * world_size
|
||||
dist.all_gather_object(all_fabric, fabric_handles, group=group)
|
||||
return True, {
|
||||
(r, i): handle
|
||||
for r, handles in enumerate(all_fabric)
|
||||
if r != rank
|
||||
for i, handle in enumerate(handles)
|
||||
}
|
||||
try:
|
||||
info = [None] * world_size
|
||||
dist.all_gather_object(
|
||||
info, (socket.gethostname(), len(local_fds)), group=group
|
||||
)
|
||||
if len({host for host, _ in info}) > 1:
|
||||
raise RuntimeError(
|
||||
"VMM handles cannot reach ranks on other hosts without FABRIC "
|
||||
"handles (one NVLink domain with IMEX)"
|
||||
)
|
||||
return False, exchange_posix_fds(
|
||||
group, rank, world_size, local_fds, [n for _, n in info]
|
||||
)
|
||||
finally:
|
||||
for fd in local_fds:
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def import_peer_handle(fabric_handle, fd, *, use_fabric: bool, peer_rank: int):
|
||||
"""Import a peer allocation handle (FABRIC or POSIX fd). Returns the handle.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user