From da414adda71957676dcbb434a60326bd3a14600a Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sun, 18 Oct 2026 12:24:05 -0700 Subject: [PATCH] [Fix] engram: Clamp multimodal pad ids in host-built history rows (#45038) --- python/sglang/srt/layers/engram.py | 7 ++ .../srt/model_executor/encoder_swa_replay.py | 4 +- .../ngram_embedding_manager.py | 5 +- .../unit/layers/test_engram_history.py | 89 +++++++++++++++++++ 4 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 test/registered/unit/layers/test_engram_history.py diff --git a/python/sglang/srt/layers/engram.py b/python/sglang/srt/layers/engram.py index 024a4b9fb2..8d401670a7 100644 --- a/python/sglang/srt/layers/engram.py +++ b/python/sglang/srt/layers/engram.py @@ -273,6 +273,13 @@ class EngramHasher(nn.Module): ) self.pad_row = num_req_slots + def history_row(self, ids: list[int]) -> list[int]: + """Left-pad up to n - 1 predecessors into a history row. The hash reads every + id at or above MM_PAD_SHIFT_VALUE as an image token, so multimodal pad ids + clamp to it and fit the int32 history.""" + n1 = self.max_ngram_size - 1 + return [0] * (n1 - len(ids)) + [min(t, MM_PAD_SHIFT_VALUE) for t in ids] + @classmethod def from_config( cls, config, layout: EngramLayout, *, image_token_id: Optional[int] = None diff --git a/python/sglang/srt/model_executor/encoder_swa_replay.py b/python/sglang/srt/model_executor/encoder_swa_replay.py index 10df46d669..cad9522d57 100644 --- a/python/sglang/srt/model_executor/encoder_swa_replay.py +++ b/python/sglang/srt/model_executor/encoder_swa_replay.py @@ -173,8 +173,8 @@ def _engram_history(*, reqs, starts, runner): n = hasher.max_ngram_size - 1 hist = [] for r, s in zip(reqs, starts): - ids = list(r.full_untruncated_fill_ids[max(0, s - n) : s]) - hist.append([0] * (n - len(ids)) + ids) + ids = r.full_untruncated_fill_ids[max(0, s - n) : s] + hist.append(hasher.history_row(ids)) return torch.tensor(hist, dtype=torch.int32, device=runner.device) diff --git a/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py b/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py index c76f6b2091..fc4a574a53 100644 --- a/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py +++ b/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py @@ -183,8 +183,7 @@ class NgramEmbeddingManager: # decode paths, which a PD decode request has not run yet. # Slice the tail first: origin_input_ids can be ~100k tokens. tail = (req.origin_input_ids[-(n1 + 1) :] + req.output_ids)[-(n1 + 1) :] - ids = tail[:-1] - rows.append([0] * (n1 - len(ids)) + list(ids)) + rows.append(self.engram_hasher.history_row(tail[:-1])) # Pinned copies keep the admission fence from stalling the CPU. slots = async_h2d( [req.kv.req_pool_idx for req in batch.reqs], @@ -201,7 +200,7 @@ class NgramEmbeddingManager: for req, start in zip(batch.reqs, batch.prefix_lens, strict=True): lo = max(0, start - n1) ids = req.full_untruncated_fill_ids[lo:start] - rows.append([0] * (n1 - len(ids)) + list(ids)) + rows.append(self.engram_hasher.history_row(ids)) # The forward stream waits on the schedule stream, which orders this copy. batch.engram_history = async_h2d( rows, dtype=torch.int32, device=self.engram_hasher.history.device diff --git a/test/registered/unit/layers/test_engram_history.py b/test/registered/unit/layers/test_engram_history.py new file mode 100644 index 0000000000..ba1da026f5 --- /dev/null +++ b/test/registered/unit/layers/test_engram_history.py @@ -0,0 +1,89 @@ +"""Engram history rows the scheduler builds for multimodal requests. CPU only.""" + +import unittest +from types import SimpleNamespace + +import torch +from torch import nn + +from sglang.kernels.ops.embeddings.engram_hash import MODE_EXTEND +from sglang.srt.layers.engram import EngramHasher +from sglang.srt.managers.schedule_batch import MM_PAD_SHIFT_VALUE +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.model_executor.model_runner_components.ngram_embedding_manager import ( + NgramEmbeddingManager, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +IMAGE_TOKEN_ID = 9 +# Multimodal pad ids span 62 bits, past what the int32 history holds. +WIDE_PAD = MM_PAD_SHIFT_VALUE + (1 << 61) + + +def _make_manager(): + """A manager whose trigram hasher has a 16-token vocab, one layer and two + heads, built without a tokenizer.""" + hasher = EngramHasher.__new__(EngramHasher) + nn.Module.__init__(hasher) + hasher.max_ngram_size = 3 + hasher.pad_id = 0 + hasher.image_token_id = IMAGE_TOKEN_ID + hasher.token_map = torch.arange(16) + hasher.multipliers = torch.tensor([[3, 5, 7]]) + hasher.primes = torch.tensor([[[11, 13], [17, 19]]]) + hasher.offsets = torch.zeros(1, 4, dtype=torch.int64) + hasher.init_history(num_req_slots=2, device="cpu") + return NgramEmbeddingManager(enabled=False, table=None, n=0, engram_hasher=hasher) + + +class TestEngramHistoryMultimodalPads(CustomTestCase): + def test_extend_history_clamps_pads_and_hashes_them_as_images(self): + manager = _make_manager() + hasher = manager.engram_hasher + # The prefix hit ends right after an image, so both predecessors are pads. + req = SimpleNamespace( + full_untruncated_fill_ids=[1, 2, WIDE_PAD, WIDE_PAD, 3, 4] + ) + batch = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, reqs=[req], prefix_lens=[4] + ) + manager._prepare_engram_history(batch) + self.assertEqual(batch.engram_history.tolist(), [[MM_PAD_SHIFT_VALUE] * 2]) + + def hash_ids(history): + return hasher._torch_hash_ids( + torch.tensor([3, 4]), + torch.tensor([4, 5]), + MODE_EXTEND, + history, + num_real=2, + block=1, + row=torch.tensor([0, 0]), + starts=torch.tensor([0]), + )[0] + + image_history = torch.full((1, 2), IMAGE_TOKEN_ID, dtype=torch.int32) + torch.testing.assert_close( + hash_ids(batch.engram_history), hash_ids(image_history) + ) + + def test_pd_prebuilt_history_clamps_pads(self): + manager = _make_manager() + # The prompt ends with an image and prefill sampled token 7. + req = SimpleNamespace( + origin_input_ids=[1, WIDE_PAD, WIDE_PAD], + output_ids=[7], + kv=SimpleNamespace(req_pool_idx=1), + ) + batch = SimpleNamespace(forward_mode=ForwardMode.PREBUILT, reqs=[req]) + manager._prepare_engram_history(batch) + self.assertEqual( + manager.engram_hasher.history[1].tolist(), [MM_PAD_SHIFT_VALUE] * 2 + ) + + +if __name__ == "__main__": + unittest.main()