From b588aa39e95795e86a1d9ec3afb56c01b2623bfa Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Thu, 22 Oct 2026 18:35:55 -0700 Subject: [PATCH] [Perf] dsv4: Optionally stage V4.1 engram rows from the shared host table on the scheduler (#45921) --- python/sglang/srt/environ.py | 3 + python/sglang/srt/layers/engram.py | 115 ++++++++++++-- python/sglang/srt/layers/engram_staging.py | 149 ++++++++++++++++++ python/sglang/srt/managers/schedule_batch.py | 3 + .../srt/model_executor/encoder_swa_replay.py | 1 + .../srt/model_executor/forward_batch_info.py | 3 + .../ngram_embedding_manager.py | 13 +- .../runner/prefill_cuda_graph_runner.py | 2 + 8 files changed, 276 insertions(+), 13 deletions(-) create mode 100644 python/sglang/srt/layers/engram_staging.py diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 10f20eb799..3696350a83 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1730,6 +1730,9 @@ class Envs: # anonymous mapping per rank holding only its rows, gathered with the # all-reduce, and the only layout that gets huge pages without shmem THP. SGLANG_DSV41_ENGRAM_HOST_TABLE_LAYOUT = EnvStr("shared") + # Scheduler-side staging of shared host-table rows on GPUs without ATS + # (engram_staging.py); pays off for large prefill batches. + SGLANG_OPT_DSV41_ENGRAM_HOST_STAGING = EnvBool(False) # Kernels and indexer SGLANG_OPT_DEEPGEMM_HC_PRENORM = EnvBool(True) diff --git a/python/sglang/srt/layers/engram.py b/python/sglang/srt/layers/engram.py index c7be800c73..85c6672599 100644 --- a/python/sglang/srt/layers/engram.py +++ b/python/sglang/srt/layers/engram.py @@ -52,6 +52,15 @@ from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.managers.schedule_batch import MM_PAD_SHIFT_VALUE from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.breakable_cuda_graph import ( + eager_on_graph, +) +from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context import ( + is_in_breakable_cuda_graph, +) +from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import ( + get_tc_piecewise_forward_context, +) from sglang.srt.runtime_context import get_model, get_parallel, get_serving from sglang.srt.utils import add_prefix, is_cuda, is_gfx95_supported, is_hip from sglang.srt.utils.hf_transformers.tokenizer import get_tokenizer @@ -257,6 +266,8 @@ class EngramHasher(nn.Module): self.image_token_id: Optional[int] = None self.history: Optional[torch.Tensor] = None self.pad_row = 0 + # Host copies of the hash buffers, for windows built on the CPU. + self._cpu_hash_params: Optional[tuple[torch.Tensor, ...]] = None # typing only self.token_map: torch.Tensor self.multipliers: torch.Tensor @@ -453,7 +464,9 @@ class EngramHasher(nn.Module): starts: Optional[torch.Tensor], ) -> tuple[torch.Tensor, torch.Tensor]: """Non-CUDA fallback of the hash kernel: predecessor table [T, n] and hash ids. - ``history`` is already the per-request [bs, n - 1] rows of this batch.""" + ``history`` is already the per-request [bs, n - 1] rows of this batch. + EngramRowStager builds the same windows on the host from the requests' + fill ids.""" n = self.max_ngram_size num_tokens = input_ids.shape[0] device = input_ids.device @@ -485,6 +498,15 @@ class EngramHasher(nn.Module): blocked = positions.unsqueeze(-1) < shifts if num_real < num_tokens: blocked[num_real:] = True + return self.hash_windows(tokens, blocked) + + def hash_windows( + self, tokens: torch.Tensor, blocked: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + """Hash ids [T, n_engram_layers, n_hash_cols] of predecessor windows on the + device of ``tokens``: tokens [T, n] holds the s-th predecessor in column s, + blocked [T, n] the look-back past the sequence start. Also returns tokens + with multimodal pad ids read as the image token.""" if self.image_token_id is not None: # Scheduler-provided history still carries the multimodal pad ids. tokens = tokens.masked_fill( @@ -497,16 +519,12 @@ class EngramHasher(nn.Module): .cummax(-1) .values.bool() ) - hash_ids = compute_engram_hash_ids( - tokens, - blocked, - self.pad_id, - self.token_map, - self.multipliers, - self.primes, - self.offsets, - ) - return hash_ids, tokens + params = (self.token_map, self.multipliers, self.primes, self.offsets) + if tokens.device != self.token_map.device: + if self._cpu_hash_params is None: + self._cpu_hash_params = tuple(t.cpu() for t in params) + params = self._cpu_hash_params + return compute_engram_hash_ids(tokens, blocked, self.pad_id, *params), tokens def commit_after_verify( self, @@ -829,6 +847,7 @@ class EngramEmbedding(nn.Module): self.rows = row_end - self.row_start self.host_table: Optional[_HostTable] = None self._sort_host_reads = False + self.staged_by_scheduler = False if envs.SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE.get(): self._init_host_table(num_embeddings, dim, layer_id) else: @@ -869,6 +888,17 @@ class EngramEmbedding(nn.Module): def _shared(self) -> bool: return self.host_table is not None and self.host_table.layout == "shared" + @property + def shared_host_table_without_ats(self) -> bool: + """The shared host layout (every row mapped here) on a GPU without ATS.""" + return self._shared and self._sort_host_reads + + def enable_scheduler_staging(self) -> None: + """Read the rows the scheduler stages (EngramRowStager) instead of + gathering them in the forward.""" + assert self.shared_host_table_without_ats + self.staged_by_scheduler = True + def _table_ptrs(self) -> tuple[int, int]: if self.host_table is None: return self.weight.data_ptr(), self.scale.data_ptr() @@ -923,6 +953,35 @@ class EngramEmbedding(nn.Module): return self._dp_sharded_lookup(indices, forward_batch) return self._lookup(indices) + def dequant_staged( + self, + rows: torch.Tensor, + scales: torch.Tensor, + copied: torch.cuda.Event, + hash_shape: torch.Size, + ) -> torch.Tensor: + """Dequantize this layer's staged rows into [*hash_shape, dim]; rows past the + staged ones (prefill graph padding) are zero.""" + stream = torch.cuda.current_stream() + stream.wait_event(copied) + rows.record_stream(stream) + scales.record_stream(stream) + out = torch.empty( + *hash_shape, self.dim, dtype=torch.bfloat16, device=rows.device + ) + flat = out.view(-1, self.dim) + assert rows.shape[0] <= flat.shape[0], (rows.shape, flat.shape) + engram_gather( + rows.data_ptr(), + scales.data_ptr(), + torch.arange(flat.shape[0], dtype=torch.int32, device=rows.device), + flat, + self.dim, + FP8_BLOCK_SIZE, + row_hi=rows.shape[0], + ) + return out + def _lookup(self, indices: torch.Tensor) -> torch.Tensor: """Lookup when every TP rank holds the same indices: the device and per-rank host shards zero unowned rows and the all-reduce reassembles.""" @@ -1100,7 +1159,16 @@ class Engram(nn.Module): layer.""" # The lookup runs first even for an idle DP-attention batch: under DP # attention it is a collective every rank has to join. - emb = self.embed(hash_ids, forward_batch, cp_all_tokens=cp_all_tokens) + if ( + self.embed.staged_by_scheduler + and forward_batch is not None + and forward_batch.forward_mode.is_extend_without_speculative() + and is_in_breakable_cuda_graph() + ): + # Every replay stages new rows, so the lookup runs between segments. + emb = _bcg_engram_embed(self, hash_ids, cp_all_tokens) + else: + emb = self.embed_rows(hash_ids, forward_batch, cp_all_tokens) if x.shape[0] == 0: # Nothing to gate, and the MXFP8 quantize behind wkv rejects an # empty M. @@ -1117,3 +1185,26 @@ class Engram(nn.Module): image_ids, self.image_token_id, ) + + def embed_rows( + self, + hash_ids: torch.Tensor, + forward_batch: Optional[ForwardBatch], + cp_all_tokens: bool, + ) -> torch.Tensor: + """The rows the scheduler staged for this batch, else the table lookup.""" + staged = None if forward_batch is None else forward_batch.engram_staged_rows + if staged is not None: + return self.embed.dequant_staged( + *staged[self.layer_hash_index], hash_ids.shape + ) + return self.embed(hash_ids, forward_batch, cp_all_tokens=cp_all_tokens) + + +@eager_on_graph +def _bcg_engram_embed( + engram: Engram, hash_ids: torch.Tensor, cp_all_tokens: bool +) -> torch.Tensor: + # A replay builds a fresh batch; the capture-time one stages no rows. + forward_batch = get_tc_piecewise_forward_context().forward_batch + return engram.embed_rows(hash_ids, forward_batch, cp_all_tokens) diff --git a/python/sglang/srt/layers/engram_staging.py b/python/sglang/srt/layers/engram_staging.py new file mode 100644 index 0000000000..2d701218e1 --- /dev/null +++ b/python/sglang/srt/layers/engram_staging.py @@ -0,0 +1,149 @@ +"""Scheduler-side staging of DeepSeek-V4.1 engram rows from the shared host table. + +Without ATS the forward's gather of random host rows is slow (_sorted_host_reads). +The scheduler knows an extend batch's tokens before its forward, so it hashes them +on the CPU, gathers each layer's rows into pinned memory, and copies them to the GPU +on a side stream while the previous forward runs; the layer only dequantizes them. +""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from typing import TYPE_CHECKING, Optional + +import numpy as np +import torch + +from sglang.srt.environ import envs +from sglang.srt.layers.engram import ( + Engram, + EngramEmbedding, + EngramHasher, +) +from sglang.srt.model_executor.encoder_swa_replay import replay_len +from sglang.srt.runtime_context import get_parallel + +if TYPE_CHECKING: + from sglang.srt.managers.schedule_batch import ScheduleBatch + +# Per engram layer: device fp8 rows, device e8m0 scales, and the event of their copy. +StagedRows = list[tuple[torch.Tensor, torch.Tensor, torch.cuda.Event]] + +_GATHER_THREADS = 16 + + +class EngramRowStager: + def __init__( + self, + hasher: EngramHasher, + embeds: list[EngramEmbedding], + device: torch.device, + ): + self.hasher = hasher + # fp8 and e8m0 have no CPU index_select; the gather moves bytes. + self.tables = [ + (e.weight.data.view(torch.uint8), e.scale.data.view(torch.uint8)) + for e in embeds + ] + self.device = device + self.pool = ThreadPoolExecutor(_GATHER_THREADS) + # The schedule stream waits on the running forward (the WAR barrier), so + # copies there would not overlap it; each layer waits on its copy's event. + self.copy_stream = torch.cuda.Stream(device) + + @classmethod + def from_model( + cls, model: torch.nn.Module, hasher: EngramHasher, device: torch.device + ) -> Optional[EngramRowStager]: + """A stager when every engram table is shared_host_table_without_ats.""" + if not envs.SGLANG_OPT_DSV41_ENGRAM_HOST_STAGING.get(): + return None + engrams = sorted( + (m for m in model.modules() if isinstance(m, Engram)), + key=lambda m: m.layer_hash_index, + ) + embeds = [m.embed for m in engrams] + if ( + not embeds + # Prefill CP hashes the whole prompt and keeps this rank's rows. + or get_parallel().attn_cp_size > 1 + or not all(e.shared_host_table_without_ats for e in embeds) + ): + return None + for e in embeds: + e.enable_scheduler_staging() + return cls(hasher, embeds, device) + + def stage(self, batch: ScheduleBatch) -> Optional[StagedRows]: + """Stage every engram layer's rows for an extend batch, None to leave the + lookup to the forward.""" + if not batch.forward_mode.is_extend_without_speculative(): + return None + windows = self._windows(batch) + if windows is None: + return None + ids, _ = self.hasher.hash_windows(*windows) + staged = [] + for layer, (weight, scale) in enumerate(self.tables): + layer_ids = ids[:, layer].reshape(-1) + rows = self._gather(weight, layer_ids) + scales = self._gather(scale, layer_ids) + with torch.cuda.stream(self.copy_stream): + rows = rows.to(self.device, non_blocking=True) + scales = scales.to(self.device, non_blocking=True) + copied = torch.cuda.Event() + copied.record() + staged.append((rows, scales, copied)) + return staged + + def _windows( + self, batch: ScheduleBatch + ) -> Optional[tuple[torch.Tensor, torch.Tensor]]: + """The host twin of the windows EngramHasher builds on the device: tokens + [T, n], column s the s-th predecessor, and blocked [T, n], the look-back + past the sequence start. None unless every extend token is on the host.""" + max_ngram_size = self.hasher.max_ngram_size + lookback = max_ngram_size - 1 + tokens, positions = [], [] + # The forward's rows: an encoder SWA replay hit extends from its replay + # start (fold_encoder_swa_replay). + resets = batch.encoder_swa_reset or [] + replay = [replay_len(p) if r else 0 for r, p in zip(resets, batch.prefix_lens)] + replay += [0] * (len(batch.reqs) - len(replay)) + for req, prefix, extend, head in zip( + batch.reqs, batch.prefix_lens, batch.extend_lens, replay, strict=True + ): + start, length = prefix - head, extend + head + fill_ids = req.full_untruncated_fill_ids + if start + length > len(fill_ids): + # A decode row's newest token can still be on the device. + return None + if length == 0: + continue + lo = max(0, start - lookback) + seg = np.asarray(fill_ids[lo : start + length], dtype=np.int64) + seg = np.concatenate([np.zeros(lookback - (start - lo), np.int64), seg]) + tokens.append( + np.lib.stride_tricks.sliding_window_view(seg, max_ngram_size)[:, ::-1] + ) + positions.append(np.arange(start, start + length)) + if not tokens: + return None + positions = torch.from_numpy(np.concatenate(positions)) + blocked = positions.unsqueeze(-1) < torch.arange(max_ngram_size) + return torch.from_numpy(np.concatenate(tokens)), blocked + + def _gather(self, table: torch.Tensor, ids: torch.Tensor) -> torch.Tensor: + out = torch.empty( + (ids.numel(), table.shape[1]), dtype=table.dtype, pin_memory=True + ) + step = -(-ids.numel() // _GATHER_THREADS) + chunks = [ + self.pool.submit( + torch.index_select, table, 0, ids[i : i + step], out=out[i : i + step] + ) + for i in range(0, ids.numel(), step) + ] + for chunk in chunks: + chunk.result() + return out diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 0133caafcf..6cab5b8066 100644 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -152,6 +152,7 @@ if TYPE_CHECKING: from typing import Any, Dict from sglang.srt.configs.model_config import ModelConfig + from sglang.srt.layers.engram_staging import StagedRows from sglang.srt.managers.hisparse_coordinator import HiSparseCoordinator from sglang.srt.managers.scheduler_components.metrics_reporter import PrefillStats from sglang.srt.mem_cache.storage_prefetch import StagedPrefetchPlan @@ -2535,6 +2536,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): # DeepSeek-V4.1 engram, extend batches only: [bs, n - 1] int32 predecessors # of each request's first extend token (NgramEmbeddingManager). engram_history: Optional[torch.Tensor] = None + # Per engram layer, the rows the scheduler staged (EngramRowStager). + engram_staged_rows: Optional[StagedRows] = None encoder_swa_reset: Optional[List[bool]] = None req_pool_indices: torch.Tensor = None # shape: [b], int64 diff --git a/python/sglang/srt/model_executor/encoder_swa_replay.py b/python/sglang/srt/model_executor/encoder_swa_replay.py index d904378af4..81ed4e15f0 100644 --- a/python/sglang/srt/model_executor/encoder_swa_replay.py +++ b/python/sglang/srt/model_executor/encoder_swa_replay.py @@ -103,6 +103,7 @@ def run_encoder_swa_replay(worker, batch): rows = _reset_windows_and_collect_hits(batch=batch, window=window) if not rows: return + assert batch.engram_staged_rows is None, "staged engram rows need a folded replay" # One forward replays every hitting request; each request's window is # floored at its own start, so the result matches per-request replays. replay = _build_replay_batch(batch=batch, runner=runner, rows=rows) diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index af2c5ca9c0..41ca963bfe 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -75,6 +75,7 @@ if TYPE_CHECKING: from sglang.srt.configs.model_config import ModelConfig from sglang.srt.layers.cp.base import BaseContextParallelMetadata from sglang.srt.layers.dcp.metadata import DecodeContextParallelMetadata + from sglang.srt.layers.engram_staging import StagedRows from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch @@ -790,6 +791,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # DeepSeek-V4.1 engram, extend only: the n - 1 tokens before each request's # first extend token, oldest first, [bs, n - 1] int32 (see EngramHasher). engram_history: Optional[torch.Tensor] = None + engram_staged_rows: Optional[StagedRows] = None # For dumper: int-hashed request / bootstrap-room IDs (derived from rids) rids_int: Optional[torch.Tensor] = None @@ -1048,6 +1050,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): orig_seq_lens=batch.orig_seq_lens, out_cache_loc_dsv4=batch.out_cache_loc_dsv4, engram_history=batch.engram_history, + engram_staged_rows=batch.engram_staged_rows, mamba_track_indices=batch.mamba_track_indices, mamba_track_mask=batch.mamba_track_mask, mamba_track_seqlens=batch.mamba_track_seqlens, diff --git a/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py b/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py index bb90840a36..1d44cb50db 100644 --- a/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py +++ b/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py @@ -16,6 +16,7 @@ from sglang.srt.utils.common import async_h2d if TYPE_CHECKING: from sglang.srt.layers.engram import EngramHasher + from sglang.srt.layers.engram_staging import EngramRowStager from sglang.srt.managers.schedule_batch import Req, ScheduleBatch from sglang.srt.model_executor.forward_batch_info import ForwardBatch @@ -27,6 +28,7 @@ class NgramEmbeddingManager: n: int # Draft runners have no hasher even when their config has engram layers. engram_hasher: Optional[EngramHasher] = None + engram_stager: Optional[EngramRowStager] = None @classmethod def from_model( @@ -59,7 +61,7 @@ class NgramEmbeddingManager: module.init_buffers( max_running_requests, chunked_prefill_size, device ) - engram_hasher = None + engram_hasher = engram_stager = None if model_config.use_engram: from sglang.srt.layers.engram import EngramHasher @@ -68,11 +70,18 @@ class NgramEmbeddingManager: assert engram_hasher is None, "one engram hasher per model" module.init_history(req_to_token_pool.req_to_token.shape[0], device) engram_hasher = module + if engram_hasher is not None: + from sglang.srt.layers.engram_staging import EngramRowStager + + engram_stager = EngramRowStager.from_model( + model, engram_hasher, torch.device(device) + ) return cls( enabled=use_ngram_embedding, table=token_table, n=model_config.ngram_embedding_n, engram_hasher=engram_hasher, + engram_stager=engram_stager, ) def update_after_decode( @@ -116,6 +125,8 @@ class NgramEmbeddingManager: return batch if self.engram_hasher is not None: self._prepare_engram_history(batch) + if self.engram_stager is not None: + batch.engram_staged_rows = self.engram_stager.stage(batch) if not self.enabled: return batch batch.ne_token_table = self.table diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index fb17965a83..7122fe86f7 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -2042,6 +2042,8 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): # The n-gram hasher runs outside the graph and reads this at replay. static_forward_batch.ngram_embedding_info = forward_batch.ngram_embedding_info static_forward_batch.engram_history = forward_batch.engram_history + # The engram lookup break reads the staged rows at replay. + static_forward_batch.engram_staged_rows = forward_batch.engram_staged_rows # The encoder replay fold: a replay builds the same window floors and # compressor slots as an eager prefill. static_forward_batch.encoder_swa_replay = forward_batch.encoder_swa_replay