[Perf] dsv4: Optionally stage V4.1 engram rows from the shared host table on the scheduler (#45921)
This commit is contained in:
@@ -1730,6 +1730,9 @@ class Envs:
|
||||
# anonymous mapping per rank holding only its rows, gathered with the
|
||||
# all-reduce, and the only layout that gets huge pages without shmem THP.
|
||||
SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT = EnvStr("shared")
|
||||
# Scheduler-side staging of shared host-table rows on GPUs without ATS
|
||||
# (engram_staging.py); pays off for large prefill batches.
|
||||
SGLANG_OPT_DSV41_ENGRAM_HOST_STAGING = EnvBool(False)
|
||||
|
||||
# Kernels and indexer
|
||||
SGLANG_OPT_DEEPGEMM_HC_PRENORM = EnvBool(True)
|
||||
|
||||
@@ -52,6 +52,15 @@ from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.schedule_batch import MM_PAD_SHIFT_VALUE
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.breakable_cuda_graph import (
|
||||
eager_on_graph,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import (
|
||||
is_in_breakable_cuda_graph,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import (
|
||||
get_tc_piecewise_forward_context,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_model, get_parallel, get_serving
|
||||
from sglang.srt.utils import add_prefix, is_cuda, is_gfx95_supported, is_hip
|
||||
from sglang.srt.utils.hf_transformers.tokenizer import get_tokenizer
|
||||
@@ -257,6 +266,8 @@ class EngramHasher(nn.Module):
|
||||
self.image_token_id: Optional[int] = None
|
||||
self.history: Optional[torch.Tensor] = None
|
||||
self.pad_row = 0
|
||||
# Host copies of the hash buffers, for windows built on the CPU.
|
||||
self._cpu_hash_params: Optional[tuple[torch.Tensor, ...]] = None
|
||||
# typing only
|
||||
self.token_map: torch.Tensor
|
||||
self.multipliers: torch.Tensor
|
||||
@@ -453,7 +464,9 @@ class EngramHasher(nn.Module):
|
||||
starts: Optional[torch.Tensor],
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Non-CUDA fallback of the hash kernel: predecessor table [T, n] and hash ids.
|
||||
``history`` is already the per-request [bs, n - 1] rows of this batch."""
|
||||
``history`` is already the per-request [bs, n - 1] rows of this batch.
|
||||
EngramRowStager builds the same windows on the host from the requests'
|
||||
fill ids."""
|
||||
n = self.max_ngram_size
|
||||
num_tokens = input_ids.shape[0]
|
||||
device = input_ids.device
|
||||
@@ -485,6 +498,15 @@ class EngramHasher(nn.Module):
|
||||
blocked = positions.unsqueeze(-1) < shifts
|
||||
if num_real < num_tokens:
|
||||
blocked[num_real:] = True
|
||||
return self.hash_windows(tokens, blocked)
|
||||
|
||||
def hash_windows(
|
||||
self, tokens: torch.Tensor, blocked: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Hash ids [T, n_engram_layers, n_hash_cols] of predecessor windows on the
|
||||
device of ``tokens``: tokens [T, n] holds the s-th predecessor in column s,
|
||||
blocked [T, n] the look-back past the sequence start. Also returns tokens
|
||||
with multimodal pad ids read as the image token."""
|
||||
if self.image_token_id is not None:
|
||||
# Scheduler-provided history still carries the multimodal pad ids.
|
||||
tokens = tokens.masked_fill(
|
||||
@@ -497,16 +519,12 @@ class EngramHasher(nn.Module):
|
||||
.cummax(-1)
|
||||
.values.bool()
|
||||
)
|
||||
hash_ids = compute_engram_hash_ids(
|
||||
tokens,
|
||||
blocked,
|
||||
self.pad_id,
|
||||
self.token_map,
|
||||
self.multipliers,
|
||||
self.primes,
|
||||
self.offsets,
|
||||
)
|
||||
return hash_ids, tokens
|
||||
params = (self.token_map, self.multipliers, self.primes, self.offsets)
|
||||
if tokens.device != self.token_map.device:
|
||||
if self._cpu_hash_params is None:
|
||||
self._cpu_hash_params = tuple(t.cpu() for t in params)
|
||||
params = self._cpu_hash_params
|
||||
return compute_engram_hash_ids(tokens, blocked, self.pad_id, *params), tokens
|
||||
|
||||
def commit_after_verify(
|
||||
self,
|
||||
@@ -829,6 +847,7 @@ class EngramEmbedding(nn.Module):
|
||||
self.rows = row_end - self.row_start
|
||||
self.host_table: Optional[_HostTable] = None
|
||||
self._sort_host_reads = False
|
||||
self.staged_by_scheduler = False
|
||||
if envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get():
|
||||
self._init_host_table(num_embeddings, dim, layer_id)
|
||||
else:
|
||||
@@ -869,6 +888,17 @@ class EngramEmbedding(nn.Module):
|
||||
def _shared(self) -> bool:
|
||||
return self.host_table is not None and self.host_table.layout == "shared"
|
||||
|
||||
@property
|
||||
def shared_host_table_without_ats(self) -> bool:
|
||||
"""The shared host layout (every row mapped here) on a GPU without ATS."""
|
||||
return self._shared and self._sort_host_reads
|
||||
|
||||
def enable_scheduler_staging(self) -> None:
|
||||
"""Read the rows the scheduler stages (EngramRowStager) instead of
|
||||
gathering them in the forward."""
|
||||
assert self.shared_host_table_without_ats
|
||||
self.staged_by_scheduler = True
|
||||
|
||||
def _table_ptrs(self) -> tuple[int, int]:
|
||||
if self.host_table is None:
|
||||
return self.weight.data_ptr(), self.scale.data_ptr()
|
||||
@@ -923,6 +953,35 @@ class EngramEmbedding(nn.Module):
|
||||
return self._dp_sharded_lookup(indices, forward_batch)
|
||||
return self._lookup(indices)
|
||||
|
||||
def dequant_staged(
|
||||
self,
|
||||
rows: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
copied: torch.cuda.Event,
|
||||
hash_shape: torch.Size,
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize this layer's staged rows into [*hash_shape, dim]; rows past the
|
||||
staged ones (prefill graph padding) are zero."""
|
||||
stream = torch.cuda.current_stream()
|
||||
stream.wait_event(copied)
|
||||
rows.record_stream(stream)
|
||||
scales.record_stream(stream)
|
||||
out = torch.empty(
|
||||
*hash_shape, self.dim, dtype=torch.bfloat16, device=rows.device
|
||||
)
|
||||
flat = out.view(-1, self.dim)
|
||||
assert rows.shape[0] <= flat.shape[0], (rows.shape, flat.shape)
|
||||
engram_gather(
|
||||
rows.data_ptr(),
|
||||
scales.data_ptr(),
|
||||
torch.arange(flat.shape[0], dtype=torch.int32, device=rows.device),
|
||||
flat,
|
||||
self.dim,
|
||||
FP8_BLOCK_SIZE,
|
||||
row_hi=rows.shape[0],
|
||||
)
|
||||
return out
|
||||
|
||||
def _lookup(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
"""Lookup when every TP rank holds the same indices: the device and
|
||||
per-rank host shards zero unowned rows and the all-reduce reassembles."""
|
||||
@@ -1100,7 +1159,16 @@ class Engram(nn.Module):
|
||||
layer."""
|
||||
# The lookup runs first even for an idle DP-attention batch: under DP
|
||||
# attention it is a collective every rank has to join.
|
||||
emb = self.embed(hash_ids, forward_batch, cp_all_tokens=cp_all_tokens)
|
||||
if (
|
||||
self.embed.staged_by_scheduler
|
||||
and forward_batch is not None
|
||||
and forward_batch.forward_mode.is_extend_without_speculative()
|
||||
and is_in_breakable_cuda_graph()
|
||||
):
|
||||
# Every replay stages new rows, so the lookup runs between segments.
|
||||
emb = _bcg_engram_embed(self, hash_ids, cp_all_tokens)
|
||||
else:
|
||||
emb = self.embed_rows(hash_ids, forward_batch, cp_all_tokens)
|
||||
if x.shape[0] == 0:
|
||||
# Nothing to gate, and the MXFP8 quantize behind wkv rejects an
|
||||
# empty M.
|
||||
@@ -1117,3 +1185,26 @@ class Engram(nn.Module):
|
||||
image_ids,
|
||||
self.image_token_id,
|
||||
)
|
||||
|
||||
def embed_rows(
|
||||
self,
|
||||
hash_ids: torch.Tensor,
|
||||
forward_batch: Optional[ForwardBatch],
|
||||
cp_all_tokens: bool,
|
||||
) -> torch.Tensor:
|
||||
"""The rows the scheduler staged for this batch, else the table lookup."""
|
||||
staged = None if forward_batch is None else forward_batch.engram_staged_rows
|
||||
if staged is not None:
|
||||
return self.embed.dequant_staged(
|
||||
*staged[self.layer_hash_index], hash_ids.shape
|
||||
)
|
||||
return self.embed(hash_ids, forward_batch, cp_all_tokens=cp_all_tokens)
|
||||
|
||||
|
||||
@eager_on_graph
|
||||
def _bcg_engram_embed(
|
||||
engram: Engram, hash_ids: torch.Tensor, cp_all_tokens: bool
|
||||
) -> torch.Tensor:
|
||||
# A replay builds a fresh batch; the capture-time one stages no rows.
|
||||
forward_batch = get_tc_piecewise_forward_context().forward_batch
|
||||
return engram.embed_rows(hash_ids, forward_batch, cp_all_tokens)
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Scheduler-side staging of DeepSeek-V4.1 engram rows from the shared host table.
|
||||
|
||||
Without ATS the forward's gather of random host rows is slow (_sorted_host_reads).
|
||||
The scheduler knows an extend batch's tokens before its forward, so it hashes them
|
||||
on the CPU, gathers each layer's rows into pinned memory, and copies them to the GPU
|
||||
on a side stream while the previous forward runs; the layer only dequantizes them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.engram import (
|
||||
Engram,
|
||||
EngramEmbedding,
|
||||
EngramHasher,
|
||||
)
|
||||
from sglang.srt.model_executor.encoder_swa_replay import replay_len
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
|
||||
# Per engram layer: device fp8 rows, device e8m0 scales, and the event of their copy.
|
||||
StagedRows = list[tuple[torch.Tensor, torch.Tensor, torch.cuda.Event]]
|
||||
|
||||
_GATHER_THREADS = 16
|
||||
|
||||
|
||||
class EngramRowStager:
|
||||
def __init__(
|
||||
self,
|
||||
hasher: EngramHasher,
|
||||
embeds: list[EngramEmbedding],
|
||||
device: torch.device,
|
||||
):
|
||||
self.hasher = hasher
|
||||
# fp8 and e8m0 have no CPU index_select; the gather moves bytes.
|
||||
self.tables = [
|
||||
(e.weight.data.view(torch.uint8), e.scale.data.view(torch.uint8))
|
||||
for e in embeds
|
||||
]
|
||||
self.device = device
|
||||
self.pool = ThreadPoolExecutor(_GATHER_THREADS)
|
||||
# The schedule stream waits on the running forward (the WAR barrier), so
|
||||
# copies there would not overlap it; each layer waits on its copy's event.
|
||||
self.copy_stream = torch.cuda.Stream(device)
|
||||
|
||||
@classmethod
|
||||
def from_model(
|
||||
cls, model: torch.nn.Module, hasher: EngramHasher, device: torch.device
|
||||
) -> Optional[EngramRowStager]:
|
||||
"""A stager when every engram table is shared_host_table_without_ats."""
|
||||
if not envs.SGLANG_OPT_DSV41_ENGRAM_HOST_STAGING.get():
|
||||
return None
|
||||
engrams = sorted(
|
||||
(m for m in model.modules() if isinstance(m, Engram)),
|
||||
key=lambda m: m.layer_hash_index,
|
||||
)
|
||||
embeds = [m.embed for m in engrams]
|
||||
if (
|
||||
not embeds
|
||||
# Prefill CP hashes the whole prompt and keeps this rank's rows.
|
||||
or get_parallel().attn_cp_size > 1
|
||||
or not all(e.shared_host_table_without_ats for e in embeds)
|
||||
):
|
||||
return None
|
||||
for e in embeds:
|
||||
e.enable_scheduler_staging()
|
||||
return cls(hasher, embeds, device)
|
||||
|
||||
def stage(self, batch: ScheduleBatch) -> Optional[StagedRows]:
|
||||
"""Stage every engram layer's rows for an extend batch, None to leave the
|
||||
lookup to the forward."""
|
||||
if not batch.forward_mode.is_extend_without_speculative():
|
||||
return None
|
||||
windows = self._windows(batch)
|
||||
if windows is None:
|
||||
return None
|
||||
ids, _ = self.hasher.hash_windows(*windows)
|
||||
staged = []
|
||||
for layer, (weight, scale) in enumerate(self.tables):
|
||||
layer_ids = ids[:, layer].reshape(-1)
|
||||
rows = self._gather(weight, layer_ids)
|
||||
scales = self._gather(scale, layer_ids)
|
||||
with torch.cuda.stream(self.copy_stream):
|
||||
rows = rows.to(self.device, non_blocking=True)
|
||||
scales = scales.to(self.device, non_blocking=True)
|
||||
copied = torch.cuda.Event()
|
||||
copied.record()
|
||||
staged.append((rows, scales, copied))
|
||||
return staged
|
||||
|
||||
def _windows(
|
||||
self, batch: ScheduleBatch
|
||||
) -> Optional[tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""The host twin of the windows EngramHasher builds on the device: tokens
|
||||
[T, n], column s the s-th predecessor, and blocked [T, n], the look-back
|
||||
past the sequence start. None unless every extend token is on the host."""
|
||||
max_ngram_size = self.hasher.max_ngram_size
|
||||
lookback = max_ngram_size - 1
|
||||
tokens, positions = [], []
|
||||
# The forward's rows: an encoder SWA replay hit extends from its replay
|
||||
# start (fold_encoder_swa_replay).
|
||||
resets = batch.encoder_swa_reset or []
|
||||
replay = [replay_len(p) if r else 0 for r, p in zip(resets, batch.prefix_lens)]
|
||||
replay += [0] * (len(batch.reqs) - len(replay))
|
||||
for req, prefix, extend, head in zip(
|
||||
batch.reqs, batch.prefix_lens, batch.extend_lens, replay, strict=True
|
||||
):
|
||||
start, length = prefix - head, extend + head
|
||||
fill_ids = req.full_untruncated_fill_ids
|
||||
if start + length > len(fill_ids):
|
||||
# A decode row's newest token can still be on the device.
|
||||
return None
|
||||
if length == 0:
|
||||
continue
|
||||
lo = max(0, start - lookback)
|
||||
seg = np.asarray(fill_ids[lo : start + length], dtype=np.int64)
|
||||
seg = np.concatenate([np.zeros(lookback - (start - lo), np.int64), seg])
|
||||
tokens.append(
|
||||
np.lib.stride_tricks.sliding_window_view(seg, max_ngram_size)[:, ::-1]
|
||||
)
|
||||
positions.append(np.arange(start, start + length))
|
||||
if not tokens:
|
||||
return None
|
||||
positions = torch.from_numpy(np.concatenate(positions))
|
||||
blocked = positions.unsqueeze(-1) < torch.arange(max_ngram_size)
|
||||
return torch.from_numpy(np.concatenate(tokens)), blocked
|
||||
|
||||
def _gather(self, table: torch.Tensor, ids: torch.Tensor) -> torch.Tensor:
|
||||
out = torch.empty(
|
||||
(ids.numel(), table.shape[1]), dtype=table.dtype, pin_memory=True
|
||||
)
|
||||
step = -(-ids.numel() // _GATHER_THREADS)
|
||||
chunks = [
|
||||
self.pool.submit(
|
||||
torch.index_select, table, 0, ids[i : i + step], out=out[i : i + step]
|
||||
)
|
||||
for i in range(0, ids.numel(), step)
|
||||
]
|
||||
for chunk in chunks:
|
||||
chunk.result()
|
||||
return out
|
||||
@@ -152,6 +152,7 @@ if TYPE_CHECKING:
|
||||
from typing import Any, Dict
|
||||
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.layers.engram_staging import StagedRows
|
||||
from sglang.srt.managers.hisparse_coordinator import HiSparseCoordinator
|
||||
from sglang.srt.managers.scheduler_components.metrics_reporter import PrefillStats
|
||||
from sglang.srt.mem_cache.storage_prefetch import StagedPrefetchPlan
|
||||
@@ -2535,6 +2536,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# DeepSeek-V4.1 engram, extend batches only: [bs, n - 1] int32 predecessors
|
||||
# of each request's first extend token (NgramEmbeddingManager).
|
||||
engram_history: Optional[torch.Tensor] = None
|
||||
# Per engram layer, the rows the scheduler staged (EngramRowStager).
|
||||
engram_staged_rows: Optional[StagedRows] = None
|
||||
encoder_swa_reset: Optional[List[bool]] = None
|
||||
|
||||
req_pool_indices: torch.Tensor = None # shape: [b], int64
|
||||
|
||||
@@ -103,6 +103,7 @@ def run_encoder_swa_replay(worker, batch):
|
||||
rows = _reset_windows_and_collect_hits(batch=batch, window=window)
|
||||
if not rows:
|
||||
return
|
||||
assert batch.engram_staged_rows is None, "staged engram rows need a folded replay"
|
||||
# One forward replays every hitting request; each request's window is
|
||||
# floored at its own start, so the result matches per-request replays.
|
||||
replay = _build_replay_batch(batch=batch, runner=runner, rows=rows)
|
||||
|
||||
@@ -75,6 +75,7 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.layers.cp.base import BaseContextParallelMetadata
|
||||
from sglang.srt.layers.dcp.metadata import DecodeContextParallelMetadata
|
||||
from sglang.srt.layers.engram_staging import StagedRows
|
||||
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch
|
||||
@@ -790,6 +791,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
# DeepSeek-V4.1 engram, extend only: the n - 1 tokens before each request's
|
||||
# first extend token, oldest first, [bs, n - 1] int32 (see EngramHasher).
|
||||
engram_history: Optional[torch.Tensor] = None
|
||||
engram_staged_rows: Optional[StagedRows] = None
|
||||
|
||||
# For dumper: int-hashed request / bootstrap-room IDs (derived from rids)
|
||||
rids_int: Optional[torch.Tensor] = None
|
||||
@@ -1048,6 +1050,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
orig_seq_lens=batch.orig_seq_lens,
|
||||
out_cache_loc_dsv4=batch.out_cache_loc_dsv4,
|
||||
engram_history=batch.engram_history,
|
||||
engram_staged_rows=batch.engram_staged_rows,
|
||||
mamba_track_indices=batch.mamba_track_indices,
|
||||
mamba_track_mask=batch.mamba_track_mask,
|
||||
mamba_track_seqlens=batch.mamba_track_seqlens,
|
||||
|
||||
+12
-1
@@ -16,6 +16,7 @@ from sglang.srt.utils.common import async_h2d
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.engram import EngramHasher
|
||||
from sglang.srt.layers.engram_staging import EngramRowStager
|
||||
from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
@@ -27,6 +28,7 @@ class NgramEmbeddingManager:
|
||||
n: int
|
||||
# Draft runners have no hasher even when their config has engram layers.
|
||||
engram_hasher: Optional[EngramHasher] = None
|
||||
engram_stager: Optional[EngramRowStager] = None
|
||||
|
||||
@classmethod
|
||||
def from_model(
|
||||
@@ -59,7 +61,7 @@ class NgramEmbeddingManager:
|
||||
module.init_buffers(
|
||||
max_running_requests, chunked_prefill_size, device
|
||||
)
|
||||
engram_hasher = None
|
||||
engram_hasher = engram_stager = None
|
||||
if model_config.use_engram:
|
||||
from sglang.srt.layers.engram import EngramHasher
|
||||
|
||||
@@ -68,11 +70,18 @@ class NgramEmbeddingManager:
|
||||
assert engram_hasher is None, "one engram hasher per model"
|
||||
module.init_history(req_to_token_pool.req_to_token.shape[0], device)
|
||||
engram_hasher = module
|
||||
if engram_hasher is not None:
|
||||
from sglang.srt.layers.engram_staging import EngramRowStager
|
||||
|
||||
engram_stager = EngramRowStager.from_model(
|
||||
model, engram_hasher, torch.device(device)
|
||||
)
|
||||
return cls(
|
||||
enabled=use_ngram_embedding,
|
||||
table=token_table,
|
||||
n=model_config.ngram_embedding_n,
|
||||
engram_hasher=engram_hasher,
|
||||
engram_stager=engram_stager,
|
||||
)
|
||||
|
||||
def update_after_decode(
|
||||
@@ -116,6 +125,8 @@ class NgramEmbeddingManager:
|
||||
return batch
|
||||
if self.engram_hasher is not None:
|
||||
self._prepare_engram_history(batch)
|
||||
if self.engram_stager is not None:
|
||||
batch.engram_staged_rows = self.engram_stager.stage(batch)
|
||||
if not self.enabled:
|
||||
return batch
|
||||
batch.ne_token_table = self.table
|
||||
|
||||
@@ -2042,6 +2042,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
# The n-gram hasher runs outside the graph and reads this at replay.
|
||||
static_forward_batch.ngram_embedding_info = forward_batch.ngram_embedding_info
|
||||
static_forward_batch.engram_history = forward_batch.engram_history
|
||||
# The engram lookup break reads the staged rows at replay.
|
||||
static_forward_batch.engram_staged_rows = forward_batch.engram_staged_rows
|
||||
# The encoder replay fold: a replay builds the same window floors and
|
||||
# compressor slots as an eager prefill.
|
||||
static_forward_batch.encoder_swa_replay = forward_batch.encoder_swa_replay
|
||||
|
||||
Reference in New Issue
Block a user