[Perf] dsv4: Optionally stage V4.1 engram rows from the shared host table on the scheduler (#45921)

This commit is contained in:
Liangsheng Yin
2026-10-22 18:35:55 -07:00
committed by GitHub
parent f36ee5de93
commit b588aa39e9
8 changed files with 276 additions and 13 deletions
+3
View File
@@ -1730,6 +1730,9 @@ class Envs:
# anonymous mapping per rank holding only its rows, gathered with the
# all-reduce, and the only layout that gets huge pages without shmem THP.
SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT = EnvStr("shared")
# Scheduler-side staging of shared host-table rows on GPUs without ATS
# (engram_staging.py); pays off for large prefill batches.
SGLANG_OPT_DSV41_ENGRAM_HOST_STAGING = EnvBool(False)
# Kernels and indexer
SGLANG_OPT_DEEPGEMM_HC_PRENORM = EnvBool(True)
+103 -12
View File
@@ -52,6 +52,15 @@ from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.managers.schedule_batch import MM_PAD_SHIFT_VALUE
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.breakable_cuda_graph import (
eager_on_graph,
)
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import (
is_in_breakable_cuda_graph,
)
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import (
get_tc_piecewise_forward_context,
)
from sglang.srt.runtime_context import get_model, get_parallel, get_serving
from sglang.srt.utils import add_prefix, is_cuda, is_gfx95_supported, is_hip
from sglang.srt.utils.hf_transformers.tokenizer import get_tokenizer
@@ -257,6 +266,8 @@ class EngramHasher(nn.Module):
self.image_token_id: Optional[int] = None
self.history: Optional[torch.Tensor] = None
self.pad_row = 0
# Host copies of the hash buffers, for windows built on the CPU.
self._cpu_hash_params: Optional[tuple[torch.Tensor, ...]] = None
# typing only
self.token_map: torch.Tensor
self.multipliers: torch.Tensor
@@ -453,7 +464,9 @@ class EngramHasher(nn.Module):
starts: Optional[torch.Tensor],
) -> tuple[torch.Tensor, torch.Tensor]:
"""Non-CUDA fallback of the hash kernel: predecessor table [T, n] and hash ids.
``history`` is already the per-request [bs, n - 1] rows of this batch."""
``history`` is already the per-request [bs, n - 1] rows of this batch.
EngramRowStager builds the same windows on the host from the requests'
fill ids."""
n = self.max_ngram_size
num_tokens = input_ids.shape[0]
device = input_ids.device
@@ -485,6 +498,15 @@ class EngramHasher(nn.Module):
blocked = positions.unsqueeze(-1) < shifts
if num_real < num_tokens:
blocked[num_real:] = True
return self.hash_windows(tokens, blocked)
def hash_windows(
self, tokens: torch.Tensor, blocked: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""Hash ids [T, n_engram_layers, n_hash_cols] of predecessor windows on the
device of ``tokens``: tokens [T, n] holds the s-th predecessor in column s,
blocked [T, n] the look-back past the sequence start. Also returns tokens
with multimodal pad ids read as the image token."""
if self.image_token_id is not None:
# Scheduler-provided history still carries the multimodal pad ids.
tokens = tokens.masked_fill(
@@ -497,16 +519,12 @@ class EngramHasher(nn.Module):
.cummax(-1)
.values.bool()
)
hash_ids = compute_engram_hash_ids(
tokens,
blocked,
self.pad_id,
self.token_map,
self.multipliers,
self.primes,
self.offsets,
)
return hash_ids, tokens
params = (self.token_map, self.multipliers, self.primes, self.offsets)
if tokens.device != self.token_map.device:
if self._cpu_hash_params is None:
self._cpu_hash_params = tuple(t.cpu() for t in params)
params = self._cpu_hash_params
return compute_engram_hash_ids(tokens, blocked, self.pad_id, *params), tokens
def commit_after_verify(
self,
@@ -829,6 +847,7 @@ class EngramEmbedding(nn.Module):
self.rows = row_end - self.row_start
self.host_table: Optional[_HostTable] = None
self._sort_host_reads = False
self.staged_by_scheduler = False
if envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get():
self._init_host_table(num_embeddings, dim, layer_id)
else:
@@ -869,6 +888,17 @@ class EngramEmbedding(nn.Module):
def _shared(self) -> bool:
return self.host_table is not None and self.host_table.layout == "shared"
@property
def shared_host_table_without_ats(self) -> bool:
"""The shared host layout (every row mapped here) on a GPU without ATS."""
return self._shared and self._sort_host_reads
def enable_scheduler_staging(self) -> None:
"""Read the rows the scheduler stages (EngramRowStager) instead of
gathering them in the forward."""
assert self.shared_host_table_without_ats
self.staged_by_scheduler = True
def _table_ptrs(self) -> tuple[int, int]:
if self.host_table is None:
return self.weight.data_ptr(), self.scale.data_ptr()
@@ -923,6 +953,35 @@ class EngramEmbedding(nn.Module):
return self._dp_sharded_lookup(indices, forward_batch)
return self._lookup(indices)
def dequant_staged(
self,
rows: torch.Tensor,
scales: torch.Tensor,
copied: torch.cuda.Event,
hash_shape: torch.Size,
) -> torch.Tensor:
"""Dequantize this layer's staged rows into [*hash_shape, dim]; rows past the
staged ones (prefill graph padding) are zero."""
stream = torch.cuda.current_stream()
stream.wait_event(copied)
rows.record_stream(stream)
scales.record_stream(stream)
out = torch.empty(
*hash_shape, self.dim, dtype=torch.bfloat16, device=rows.device
)
flat = out.view(-1, self.dim)
assert rows.shape[0] <= flat.shape[0], (rows.shape, flat.shape)
engram_gather(
rows.data_ptr(),
scales.data_ptr(),
torch.arange(flat.shape[0], dtype=torch.int32, device=rows.device),
flat,
self.dim,
FP8_BLOCK_SIZE,
row_hi=rows.shape[0],
)
return out
def _lookup(self, indices: torch.Tensor) -> torch.Tensor:
"""Lookup when every TP rank holds the same indices: the device and
per-rank host shards zero unowned rows and the all-reduce reassembles."""
@@ -1100,7 +1159,16 @@ class Engram(nn.Module):
layer."""
# The lookup runs first even for an idle DP-attention batch: under DP
# attention it is a collective every rank has to join.
emb = self.embed(hash_ids, forward_batch, cp_all_tokens=cp_all_tokens)
if (
self.embed.staged_by_scheduler
and forward_batch is not None
and forward_batch.forward_mode.is_extend_without_speculative()
and is_in_breakable_cuda_graph()
):
# Every replay stages new rows, so the lookup runs between segments.
emb = _bcg_engram_embed(self, hash_ids, cp_all_tokens)
else:
emb = self.embed_rows(hash_ids, forward_batch, cp_all_tokens)
if x.shape[0] == 0:
# Nothing to gate, and the MXFP8 quantize behind wkv rejects an
# empty M.
@@ -1117,3 +1185,26 @@ class Engram(nn.Module):
image_ids,
self.image_token_id,
)
def embed_rows(
self,
hash_ids: torch.Tensor,
forward_batch: Optional[ForwardBatch],
cp_all_tokens: bool,
) -> torch.Tensor:
"""The rows the scheduler staged for this batch, else the table lookup."""
staged = None if forward_batch is None else forward_batch.engram_staged_rows
if staged is not None:
return self.embed.dequant_staged(
*staged[self.layer_hash_index], hash_ids.shape
)
return self.embed(hash_ids, forward_batch, cp_all_tokens=cp_all_tokens)
@eager_on_graph
def _bcg_engram_embed(
engram: Engram, hash_ids: torch.Tensor, cp_all_tokens: bool
) -> torch.Tensor:
# A replay builds a fresh batch; the capture-time one stages no rows.
forward_batch = get_tc_piecewise_forward_context().forward_batch
return engram.embed_rows(hash_ids, forward_batch, cp_all_tokens)
+149
View File
@@ -0,0 +1,149 @@
"""Scheduler-side staging of DeepSeek-V4.1 engram rows from the shared host table.
Without ATS the forward's gather of random host rows is slow (_sorted_host_reads).
The scheduler knows an extend batch's tokens before its forward, so it hashes them
on the CPU, gathers each layer's rows into pinned memory, and copies them to the GPU
on a side stream while the previous forward runs; the layer only dequantizes them.
"""
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor
from typing import TYPE_CHECKING, Optional
import numpy as np
import torch
from sglang.srt.environ import envs
from sglang.srt.layers.engram import (
Engram,
EngramEmbedding,
EngramHasher,
)
from sglang.srt.model_executor.encoder_swa_replay import replay_len
from sglang.srt.runtime_context import get_parallel
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import ScheduleBatch
# Per engram layer: device fp8 rows, device e8m0 scales, and the event of their copy.
StagedRows = list[tuple[torch.Tensor, torch.Tensor, torch.cuda.Event]]
_GATHER_THREADS = 16
class EngramRowStager:
def __init__(
self,
hasher: EngramHasher,
embeds: list[EngramEmbedding],
device: torch.device,
):
self.hasher = hasher
# fp8 and e8m0 have no CPU index_select; the gather moves bytes.
self.tables = [
(e.weight.data.view(torch.uint8), e.scale.data.view(torch.uint8))
for e in embeds
]
self.device = device
self.pool = ThreadPoolExecutor(_GATHER_THREADS)
# The schedule stream waits on the running forward (the WAR barrier), so
# copies there would not overlap it; each layer waits on its copy's event.
self.copy_stream = torch.cuda.Stream(device)
@classmethod
def from_model(
cls, model: torch.nn.Module, hasher: EngramHasher, device: torch.device
) -> Optional[EngramRowStager]:
"""A stager when every engram table is shared_host_table_without_ats."""
if not envs.SGLANG_OPT_DSV41_ENGRAM_HOST_STAGING.get():
return None
engrams = sorted(
(m for m in model.modules() if isinstance(m, Engram)),
key=lambda m: m.layer_hash_index,
)
embeds = [m.embed for m in engrams]
if (
not embeds
# Prefill CP hashes the whole prompt and keeps this rank's rows.
or get_parallel().attn_cp_size > 1
or not all(e.shared_host_table_without_ats for e in embeds)
):
return None
for e in embeds:
e.enable_scheduler_staging()
return cls(hasher, embeds, device)
def stage(self, batch: ScheduleBatch) -> Optional[StagedRows]:
"""Stage every engram layer's rows for an extend batch, None to leave the
lookup to the forward."""
if not batch.forward_mode.is_extend_without_speculative():
return None
windows = self._windows(batch)
if windows is None:
return None
ids, _ = self.hasher.hash_windows(*windows)
staged = []
for layer, (weight, scale) in enumerate(self.tables):
layer_ids = ids[:, layer].reshape(-1)
rows = self._gather(weight, layer_ids)
scales = self._gather(scale, layer_ids)
with torch.cuda.stream(self.copy_stream):
rows = rows.to(self.device, non_blocking=True)
scales = scales.to(self.device, non_blocking=True)
copied = torch.cuda.Event()
copied.record()
staged.append((rows, scales, copied))
return staged
def _windows(
self, batch: ScheduleBatch
) -> Optional[tuple[torch.Tensor, torch.Tensor]]:
"""The host twin of the windows EngramHasher builds on the device: tokens
[T, n], column s the s-th predecessor, and blocked [T, n], the look-back
past the sequence start. None unless every extend token is on the host."""
max_ngram_size = self.hasher.max_ngram_size
lookback = max_ngram_size - 1
tokens, positions = [], []
# The forward's rows: an encoder SWA replay hit extends from its replay
# start (fold_encoder_swa_replay).
resets = batch.encoder_swa_reset or []
replay = [replay_len(p) if r else 0 for r, p in zip(resets, batch.prefix_lens)]
replay += [0] * (len(batch.reqs) - len(replay))
for req, prefix, extend, head in zip(
batch.reqs, batch.prefix_lens, batch.extend_lens, replay, strict=True
):
start, length = prefix - head, extend + head
fill_ids = req.full_untruncated_fill_ids
if start + length > len(fill_ids):
# A decode row's newest token can still be on the device.
return None
if length == 0:
continue
lo = max(0, start - lookback)
seg = np.asarray(fill_ids[lo : start + length], dtype=np.int64)
seg = np.concatenate([np.zeros(lookback - (start - lo), np.int64), seg])
tokens.append(
np.lib.stride_tricks.sliding_window_view(seg, max_ngram_size)[:, ::-1]
)
positions.append(np.arange(start, start + length))
if not tokens:
return None
positions = torch.from_numpy(np.concatenate(positions))
blocked = positions.unsqueeze(-1) < torch.arange(max_ngram_size)
return torch.from_numpy(np.concatenate(tokens)), blocked
def _gather(self, table: torch.Tensor, ids: torch.Tensor) -> torch.Tensor:
out = torch.empty(
(ids.numel(), table.shape[1]), dtype=table.dtype, pin_memory=True
)
step = -(-ids.numel() // _GATHER_THREADS)
chunks = [
self.pool.submit(
torch.index_select, table, 0, ids[i : i + step], out=out[i : i + step]
)
for i in range(0, ids.numel(), step)
]
for chunk in chunks:
chunk.result()
return out
@@ -152,6 +152,7 @@ if TYPE_CHECKING:
from typing import Any, Dict
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.layers.engram_staging import StagedRows
from sglang.srt.managers.hisparse_coordinator import HiSparseCoordinator
from sglang.srt.managers.scheduler_components.metrics_reporter import PrefillStats
from sglang.srt.mem_cache.storage_prefetch import StagedPrefetchPlan
@@ -2535,6 +2536,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# DeepSeek-V4.1 engram, extend batches only: [bs, n - 1] int32 predecessors
# of each request's first extend token (NgramEmbeddingManager).
engram_history: Optional[torch.Tensor] = None
# Per engram layer, the rows the scheduler staged (EngramRowStager).
engram_staged_rows: Optional[StagedRows] = None
encoder_swa_reset: Optional[List[bool]] = None
req_pool_indices: torch.Tensor = None # shape: [b], int64
@@ -103,6 +103,7 @@ def run_encoder_swa_replay(worker, batch):
rows = _reset_windows_and_collect_hits(batch=batch, window=window)
if not rows:
return
assert batch.engram_staged_rows is None, "staged engram rows need a folded replay"
# One forward replays every hitting request; each request's window is
# floored at its own start, so the result matches per-request replays.
replay = _build_replay_batch(batch=batch, runner=runner, rows=rows)
@@ -75,6 +75,7 @@ if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.layers.cp.base import BaseContextParallelMetadata
from sglang.srt.layers.dcp.metadata import DecodeContextParallelMetadata
from sglang.srt.layers.engram_staging import StagedRows
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch
@@ -790,6 +791,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# DeepSeek-V4.1 engram, extend only: the n - 1 tokens before each request's
# first extend token, oldest first, [bs, n - 1] int32 (see EngramHasher).
engram_history: Optional[torch.Tensor] = None
engram_staged_rows: Optional[StagedRows] = None
# For dumper: int-hashed request / bootstrap-room IDs (derived from rids)
rids_int: Optional[torch.Tensor] = None
@@ -1048,6 +1050,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
orig_seq_lens=batch.orig_seq_lens,
out_cache_loc_dsv4=batch.out_cache_loc_dsv4,
engram_history=batch.engram_history,
engram_staged_rows=batch.engram_staged_rows,
mamba_track_indices=batch.mamba_track_indices,
mamba_track_mask=batch.mamba_track_mask,
mamba_track_seqlens=batch.mamba_track_seqlens,
@@ -16,6 +16,7 @@ from sglang.srt.utils.common import async_h2d
if TYPE_CHECKING:
from sglang.srt.layers.engram import EngramHasher
from sglang.srt.layers.engram_staging import EngramRowStager
from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
@@ -27,6 +28,7 @@ class NgramEmbeddingManager:
n: int
# Draft runners have no hasher even when their config has engram layers.
engram_hasher: Optional[EngramHasher] = None
engram_stager: Optional[EngramRowStager] = None
@classmethod
def from_model(
@@ -59,7 +61,7 @@ class NgramEmbeddingManager:
module.init_buffers(
max_running_requests, chunked_prefill_size, device
)
engram_hasher = None
engram_hasher = engram_stager = None
if model_config.use_engram:
from sglang.srt.layers.engram import EngramHasher
@@ -68,11 +70,18 @@ class NgramEmbeddingManager:
assert engram_hasher is None, "one engram hasher per model"
module.init_history(req_to_token_pool.req_to_token.shape[0], device)
engram_hasher = module
if engram_hasher is not None:
from sglang.srt.layers.engram_staging import EngramRowStager
engram_stager = EngramRowStager.from_model(
model, engram_hasher, torch.device(device)
)
return cls(
enabled=use_ngram_embedding,
table=token_table,
n=model_config.ngram_embedding_n,
engram_hasher=engram_hasher,
engram_stager=engram_stager,
)
def update_after_decode(
@@ -116,6 +125,8 @@ class NgramEmbeddingManager:
return batch
if self.engram_hasher is not None:
self._prepare_engram_history(batch)
if self.engram_stager is not None:
batch.engram_staged_rows = self.engram_stager.stage(batch)
if not self.enabled:
return batch
batch.ne_token_table = self.table
@@ -2042,6 +2042,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
# The n-gram hasher runs outside the graph and reads this at replay.
static_forward_batch.ngram_embedding_info = forward_batch.ngram_embedding_info
static_forward_batch.engram_history = forward_batch.engram_history
# The engram lookup break reads the staged rows at replay.
static_forward_batch.engram_staged_rows = forward_batch.engram_staged_rows
# The encoder replay fold: a replay builds the same window floors and
# compressor slots as an eager prefill.
static_forward_batch.encoder_swa_replay = forward_batch.encoder_swa_replay