[DSV4.1] Reduce eager Engram extend preparation (#42159)

Co-authored-by: Shangming Cai <csmthu@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yuwei An
2026-10-11 21:31:55 -07:00
committed by GitHub
co-authored by Shangming Cai Claude Opus 5.5
parent a1001bcf3a
commit d434ea444d
4 changed files with 343 additions and 2 deletions
@@ -98,6 +98,8 @@ def _engram_hash_kernel(
out_ptr,
num_tokens,
num_real,
extend_bs,
search_steps,
pad_id,
image_token_id,
mm_pad_shift,
@@ -111,6 +113,7 @@ def _engram_hash_kernel(
L: tl.constexpr,
H: tl.constexpr,
BLOCK_T: tl.constexpr,
ROWS_FROM_STARTS: tl.constexpr = False,
):
COLS: tl.constexpr = (N - 1) * H
t = tl.program_id(0) * BLOCK_T + tl.arange(0, BLOCK_T)
@@ -124,7 +127,25 @@ def _engram_hash_kernel(
r = (t // BLOCK).to(tl.int64)
off = t - (t // BLOCK) * BLOCK
else:
r = tl.load(row_ptr + t, mask=real, other=0).to(tl.int64)
if ROWS_FROM_STARTS:
# Upper bound handles adjacent starts from zero-length requests.
# Resolve rows in registers instead of materializing repeat_interleave.
# The batch size and step count are runtime values: as constexprs every
# new batch size would compile and device-load a fresh specialization
# mid-serving.
lo = tl.zeros((BLOCK_T,), tl.int32)
hi = tl.zeros((BLOCK_T,), tl.int32) + extend_bs
for _ in range(search_steps):
mid = (lo + hi) // 2
start = tl.load(
starts_ptr + mid, mask=real & (mid < extend_bs), other=0
)
right = (mid < extend_bs) & (start <= t)
lo = tl.where(right, mid + 1, lo)
hi = tl.where(right, hi, mid)
r = tl.maximum(lo - 1, 0).to(tl.int64)
else:
r = tl.load(row_ptr + t, mask=real, other=0).to(tl.int64)
off = t - tl.load(starts_ptr + r, mask=real, other=0).to(tl.int32)
if HIST_VIA_SLOTS:
hrow = tl.load(slots_ptr + r, mask=real, other=0).to(tl.int64)
@@ -207,6 +228,7 @@ def _launch_hash_kernel(
out_cache_loc: Optional[torch.Tensor],
write_tokens: bool,
block_t: int,
extend_bs: int = 0,
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
num_tokens = input_ids.shape[0]
L, N = multipliers.shape
@@ -215,7 +237,7 @@ def _launch_hash_kernel(
assert primes.shape == (L, N - 1, H) and offsets.shape == (L, (N - 1) * H)
assert history.dim() == 2 and history.shape[1] == N - 1, history.shape
if mode == MODE_EXTEND:
assert row is not None and starts is not None
assert (row is not None or extend_bs > 0) and starts is not None
if out_cache_loc is not None:
assert mode == MODE_DECODE and req_slots is not None, "commit is decode-only"
assert out_cache_loc.shape[0] == num_tokens, out_cache_loc.shape
@@ -247,6 +269,8 @@ def _launch_hash_kernel(
out,
num_tokens,
num_real,
extend_bs,
extend_bs.bit_length(),
pad_id,
image_token_id if image_token_id is not None else -1,
mm_pad_shift,
@@ -260,11 +284,111 @@ def _launch_hash_kernel(
L=L,
H=H,
BLOCK_T=block_t,
ROWS_FROM_STARTS=extend_bs > 0,
num_warps=4,
)
return out, tokens
@triton.jit
def _engram_commit_extend_history_kernel(
history_ptr,
tokens_ptr,
slots_ptr,
starts_ptr,
lens_ptr,
out_loc_ptr,
num_tokens,
N: tl.constexpr,
HAS_OUT_LOC: tl.constexpr,
):
r = tl.program_id(0)
length = tl.load(lens_ptr + r)
start = tl.load(starts_ptr + r)
last = tl.minimum(tl.maximum(start + length - 1, 0), num_tokens - 1)
live = length > 0
if HAS_OUT_LOC:
live = live & (tl.load(out_loc_ptr + last) != 0)
slot = tl.load(slots_ptr + r).to(tl.int64)
col = tl.arange(0, N)
value = tl.load(
tokens_ptr + last * N + (N - 2 - col),
mask=live & (col < N - 1),
other=0,
)
tl.store(history_ptr + slot * (N - 1) + col, value, mask=live & (col < N - 1))
def engram_hash_extend_and_commit(
input_ids: torch.Tensor,
positions: torch.Tensor,
*,
history: torch.Tensor,
commit_history: torch.Tensor,
req_slots: torch.Tensor,
starts: torch.Tensor,
lengths: torch.Tensor,
num_real: int,
history_via_slots: bool,
out_cache_loc: Optional[torch.Tensor],
token_map: torch.Tensor,
multipliers: torch.Tensor,
primes: torch.Tensor,
offsets: torch.Tensor,
pad_id: int,
image_token_id: Optional[int] = None,
mm_pad_shift: int = 0,
) -> torch.Tensor:
"""Eager extend: hash without a row-map tensor, then commit in one launch.
Hashing must finish before history is updated: different token programs may
still read a request's old history. Keep these as two ordered kernels.
Empty/padded requests leave live history untouched; the spare padding row is
intentionally not updated. Request slots must be distinct for live requests.
"""
bs = req_slots.numel()
assert 0 <= num_real <= input_ids.numel()
assert starts.numel() == lengths.numel() == bs
assert history.is_contiguous() and commit_history.is_contiguous()
assert all(t.ndim == 1 and t.stride(0) == 1 for t in (req_slots, starts, lengths))
out, tokens = _launch_hash_kernel(
input_ids,
positions,
mode=MODE_EXTEND,
history=history,
token_map=token_map,
multipliers=multipliers,
primes=primes,
offsets=offsets,
pad_id=pad_id,
num_real=num_real,
req_slots=req_slots if history_via_slots else None,
block=1,
row=None,
starts=starts,
image_token_id=image_token_id,
mm_pad_shift=mm_pad_shift,
out_cache_loc=None,
write_tokens=True,
block_t=32,
extend_bs=bs,
)
if input_ids.numel() and bs:
_engram_commit_extend_history_kernel[(bs,)](
commit_history,
tokens,
req_slots,
starts,
lengths,
out_cache_loc if out_cache_loc is not None else req_slots,
input_ids.numel(),
N=multipliers.shape[1],
HAS_OUT_LOC=out_cache_loc is not None,
num_warps=4,
)
return out
def engram_hash_ids(
input_ids: torch.Tensor,
positions: torch.Tensor,
+1
View File
@@ -1599,6 +1599,7 @@ class Envs:
# DeepSeek-V4.1 engram host table: keep the tables in host memory (layout
# below) and gather rows from the GPU instead of sharding them over HBM.
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE = EnvBool(False)
SGLANG_OPT_ENGRAM_EXTEND_HASH = EnvBool(False)
SGLANG_OPT_DSV41_MEGA_MHC_PREFILL = EnvBool(False)
# "shared" is one buffer for the whole TP group, mapped by every rank, with no
# lookup all-reduce (the ranks must share a PID namespace); "per_rank" is one
+28
View File
@@ -30,6 +30,7 @@ from sglang.kernels.ops.embeddings.engram_hash import (
MODE_EXTEND,
MODE_VERIFY,
engram_commit_history,
engram_hash_extend_and_commit,
engram_hash_ids,
engram_hash_ids_and_commit,
)
@@ -298,6 +299,33 @@ class EngramHasher(nn.Module):
device=input_ids.device,
)
mode = forward_batch.forward_mode
if (
mode.is_extend()
and not mode.is_target_verify()
and envs.SGLANG_OPT_ENGRAM_EXTEND_HASH.get()
and _cuda_kernels(input_ids)
and forward_batch.extend_seq_lens_cpu is not None
):
history = forward_batch.engram_history
return engram_hash_extend_and_commit(
input_ids,
forward_batch.positions,
history=self.history if history is None else history,
commit_history=self.history,
req_slots=forward_batch.req_pool_indices,
starts=forward_batch.extend_start_loc,
lengths=forward_batch.extend_seq_lens,
num_real=sum(forward_batch.extend_seq_lens_cpu),
history_via_slots=history is None,
out_cache_loc=forward_batch.out_cache_loc,
token_map=self.token_map,
multipliers=self.multipliers,
primes=self.primes,
offsets=self.offsets,
pad_id=self.pad_id,
image_token_id=self.image_token_id,
mm_pad_shift=MM_PAD_SHIFT_VALUE,
)
req_slots = forward_batch.req_pool_indices
bs = req_slots.shape[0]
device = input_ids.device