diff --git a/python/sglang/kernels/ops/embeddings/engram_hash.py b/python/sglang/kernels/ops/embeddings/engram_hash.py index a8e49e5f5c..deb58658f7 100644 --- a/python/sglang/kernels/ops/embeddings/engram_hash.py +++ b/python/sglang/kernels/ops/embeddings/engram_hash.py @@ -98,6 +98,8 @@ def _engram_hash_kernel( out_ptr, num_tokens, num_real, + extend_bs, + search_steps, pad_id, image_token_id, mm_pad_shift, @@ -111,6 +113,7 @@ def _engram_hash_kernel( L: tl.constexpr, H: tl.constexpr, BLOCK_T: tl.constexpr, + ROWS_FROM_STARTS: tl.constexpr = False, ): COLS: tl.constexpr = (N - 1) * H t = tl.program_id(0) * BLOCK_T + tl.arange(0, BLOCK_T) @@ -124,7 +127,25 @@ def _engram_hash_kernel( r = (t // BLOCK).to(tl.int64) off = t - (t // BLOCK) * BLOCK else: - r = tl.load(row_ptr + t, mask=real, other=0).to(tl.int64) + if ROWS_FROM_STARTS: + # Upper bound handles adjacent starts from zero-length requests. + # Resolve rows in registers instead of materializing repeat_interleave. + # The batch size and step count are runtime values: as constexprs every + # new batch size would compile and device-load a fresh specialization + # mid-serving. + lo = tl.zeros((BLOCK_T,), tl.int32) + hi = tl.zeros((BLOCK_T,), tl.int32) + extend_bs + for _ in range(search_steps): + mid = (lo + hi) // 2 + start = tl.load( + starts_ptr + mid, mask=real & (mid < extend_bs), other=0 + ) + right = (mid < extend_bs) & (start <= t) + lo = tl.where(right, mid + 1, lo) + hi = tl.where(right, hi, mid) + r = tl.maximum(lo - 1, 0).to(tl.int64) + else: + r = tl.load(row_ptr + t, mask=real, other=0).to(tl.int64) off = t - tl.load(starts_ptr + r, mask=real, other=0).to(tl.int32) if HIST_VIA_SLOTS: hrow = tl.load(slots_ptr + r, mask=real, other=0).to(tl.int64) @@ -207,6 +228,7 @@ def _launch_hash_kernel( out_cache_loc: Optional[torch.Tensor], write_tokens: bool, block_t: int, + extend_bs: int = 0, ) -> tuple[torch.Tensor, Optional[torch.Tensor]]: num_tokens = input_ids.shape[0] L, N = multipliers.shape @@ -215,7 +237,7 @@ def _launch_hash_kernel( assert primes.shape == (L, N - 1, H) and offsets.shape == (L, (N - 1) * H) assert history.dim() == 2 and history.shape[1] == N - 1, history.shape if mode == MODE_EXTEND: - assert row is not None and starts is not None + assert (row is not None or extend_bs > 0) and starts is not None if out_cache_loc is not None: assert mode == MODE_DECODE and req_slots is not None, "commit is decode-only" assert out_cache_loc.shape[0] == num_tokens, out_cache_loc.shape @@ -247,6 +269,8 @@ def _launch_hash_kernel( out, num_tokens, num_real, + extend_bs, + extend_bs.bit_length(), pad_id, image_token_id if image_token_id is not None else -1, mm_pad_shift, @@ -260,11 +284,111 @@ def _launch_hash_kernel( L=L, H=H, BLOCK_T=block_t, + ROWS_FROM_STARTS=extend_bs > 0, num_warps=4, ) return out, tokens +@triton.jit +def _engram_commit_extend_history_kernel( + history_ptr, + tokens_ptr, + slots_ptr, + starts_ptr, + lens_ptr, + out_loc_ptr, + num_tokens, + N: tl.constexpr, + HAS_OUT_LOC: tl.constexpr, +): + r = tl.program_id(0) + length = tl.load(lens_ptr + r) + start = tl.load(starts_ptr + r) + last = tl.minimum(tl.maximum(start + length - 1, 0), num_tokens - 1) + live = length > 0 + if HAS_OUT_LOC: + live = live & (tl.load(out_loc_ptr + last) != 0) + slot = tl.load(slots_ptr + r).to(tl.int64) + col = tl.arange(0, N) + value = tl.load( + tokens_ptr + last * N + (N - 2 - col), + mask=live & (col < N - 1), + other=0, + ) + tl.store(history_ptr + slot * (N - 1) + col, value, mask=live & (col < N - 1)) + + +def engram_hash_extend_and_commit( + input_ids: torch.Tensor, + positions: torch.Tensor, + *, + history: torch.Tensor, + commit_history: torch.Tensor, + req_slots: torch.Tensor, + starts: torch.Tensor, + lengths: torch.Tensor, + num_real: int, + history_via_slots: bool, + out_cache_loc: Optional[torch.Tensor], + token_map: torch.Tensor, + multipliers: torch.Tensor, + primes: torch.Tensor, + offsets: torch.Tensor, + pad_id: int, + image_token_id: Optional[int] = None, + mm_pad_shift: int = 0, +) -> torch.Tensor: + """Eager extend: hash without a row-map tensor, then commit in one launch. + + Hashing must finish before history is updated: different token programs may + still read a request's old history. Keep these as two ordered kernels. + Empty/padded requests leave live history untouched; the spare padding row is + intentionally not updated. Request slots must be distinct for live requests. + """ + bs = req_slots.numel() + assert 0 <= num_real <= input_ids.numel() + assert starts.numel() == lengths.numel() == bs + assert history.is_contiguous() and commit_history.is_contiguous() + assert all(t.ndim == 1 and t.stride(0) == 1 for t in (req_slots, starts, lengths)) + out, tokens = _launch_hash_kernel( + input_ids, + positions, + mode=MODE_EXTEND, + history=history, + token_map=token_map, + multipliers=multipliers, + primes=primes, + offsets=offsets, + pad_id=pad_id, + num_real=num_real, + req_slots=req_slots if history_via_slots else None, + block=1, + row=None, + starts=starts, + image_token_id=image_token_id, + mm_pad_shift=mm_pad_shift, + out_cache_loc=None, + write_tokens=True, + block_t=32, + extend_bs=bs, + ) + if input_ids.numel() and bs: + _engram_commit_extend_history_kernel[(bs,)]( + commit_history, + tokens, + req_slots, + starts, + lengths, + out_cache_loc if out_cache_loc is not None else req_slots, + input_ids.numel(), + N=multipliers.shape[1], + HAS_OUT_LOC=out_cache_loc is not None, + num_warps=4, + ) + return out + + def engram_hash_ids( input_ids: torch.Tensor, positions: torch.Tensor, diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index c1f436eee5..01cdb859bb 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1599,6 +1599,7 @@ class Envs: # DeepSeek-V4.1 engram host table: keep the tables in host memory (layout # below) and gather rows from the GPU instead of sharding them over HBM. SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE = EnvBool(False) + SGLANG_OPT_ENGRAM_EXTEND_HASH = EnvBool(False) SGLANG_OPT_DSV41_MEGA_MHC_PREFILL = EnvBool(False) # "shared" is one buffer for the whole TP group, mapped by every rank, with no # lookup all-reduce (the ranks must share a PID namespace); "per_rank" is one diff --git a/python/sglang/srt/layers/engram.py b/python/sglang/srt/layers/engram.py index 68c1299336..71fc553dce 100644 --- a/python/sglang/srt/layers/engram.py +++ b/python/sglang/srt/layers/engram.py @@ -30,6 +30,7 @@ from sglang.kernels.ops.embeddings.engram_hash import ( MODE_EXTEND, MODE_VERIFY, engram_commit_history, + engram_hash_extend_and_commit, engram_hash_ids, engram_hash_ids_and_commit, ) @@ -298,6 +299,33 @@ class EngramHasher(nn.Module): device=input_ids.device, ) mode = forward_batch.forward_mode + if ( + mode.is_extend() + and not mode.is_target_verify() + and envs.SGLANG_OPT_ENGRAM_EXTEND_HASH.get() + and _cuda_kernels(input_ids) + and forward_batch.extend_seq_lens_cpu is not None + ): + history = forward_batch.engram_history + return engram_hash_extend_and_commit( + input_ids, + forward_batch.positions, + history=self.history if history is None else history, + commit_history=self.history, + req_slots=forward_batch.req_pool_indices, + starts=forward_batch.extend_start_loc, + lengths=forward_batch.extend_seq_lens, + num_real=sum(forward_batch.extend_seq_lens_cpu), + history_via_slots=history is None, + out_cache_loc=forward_batch.out_cache_loc, + token_map=self.token_map, + multipliers=self.multipliers, + primes=self.primes, + offsets=self.offsets, + pad_id=self.pad_id, + image_token_id=self.image_token_id, + mm_pad_shift=MM_PAD_SHIFT_VALUE, + ) req_slots = forward_batch.req_pool_indices bs = req_slots.shape[0] device = input_ids.device diff --git a/test/manual/quantization/test_engram_extend_hash.py b/test/manual/quantization/test_engram_extend_hash.py new file mode 100644 index 0000000000..560d9c0b64 --- /dev/null +++ b/test/manual/quantization/test_engram_extend_hash.py @@ -0,0 +1,188 @@ +"""Standalone CUDA checks; imports only the kernel module, not a model/server.""" + +import importlib.util +import unittest +from pathlib import Path + +import torch + +_path = ( + Path(__file__).resolve().parents[3] + / "python/sglang/kernels/ops/embeddings/engram_hash.py" +) +_spec = importlib.util.spec_from_file_location("engram_hash_test_kernel", _path) +kernel = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(kernel) + + +def fixture(lengths, padding=0, explicit_history=False, image=False, dtype=torch.int32): + torch.manual_seed(27) + bs, n, layers, heads, vocab = len(lengths), 4, 2, 8, 256 + real = sum(lengths) + ids = torch.randint(0, vocab, (real + padding,), device="cuda") + lens = torch.tensor(lengths, dtype=dtype, device="cuda") + starts = (lens.cumsum(0) - lens).to(dtype) + slots = torch.randperm(bs + 3, device="cuda")[:bs].to(dtype) + history = torch.randint(0, vocab, (bs + 4, n - 1), device="cuda", dtype=torch.int32) + source_history = ( + torch.randint_like(history[:bs], 0, vocab) if explicit_history else history + ) + positions = torch.zeros_like(ids) + offset = 0 + for r, length in enumerate(lengths): + positions[offset : offset + length] = torch.arange(length, device="cuda") + ( + 0 if r % 2 else 19 + ) + offset += length + if image: + if real: + ids[0] = 4096 + if real > 2: + ids[2] = 7 + source_history[:, -1] = 4096 + primes = torch.arange( + 16000001, 16000001 + layers * (n - 1) * heads, device="cuda" + ).reshape(layers, n - 1, heads) + offsets = primes.flatten(1).cumsum(1) - primes.flatten(1) + return dict( + input_ids=ids, + positions=positions, + history=source_history, + commit_history=history, + req_slots=slots, + starts=starts, + lengths=lens, + num_real=real, + history_via_slots=not explicit_history, + out_cache_loc=torch.ones_like(ids), + token_map=torch.randperm(vocab, device="cuda"), + multipliers=torch.randint(0, 2**48, (layers, n), device="cuda") * 2 + 1, + primes=primes, + offsets=offsets, + pad_id=2, + image_token_id=7 if image else None, + mm_pad_shift=4096, + ) + + +def reference(kwargs): + """Original eager row preparation, hash call, and PyTorch history update.""" + lens = kwargs["lengths"].to(torch.int64) + starts = kwargs["starts"].to(torch.int64) + slots = kwargs["req_slots"] + row = torch.repeat_interleave( + torch.arange(slots.numel(), device="cuda"), lens, output_size=kwargs["num_real"] + ) + hash_ids, tokens = kernel.engram_hash_ids( + kwargs["input_ids"], + kwargs["positions"], + mode=kernel.MODE_EXTEND, + history=kwargs["history"], + req_slots=slots if kwargs["history_via_slots"] else None, + row=row, + starts=starts, + **{ + k: kwargs[k] + for k in ( + "num_real", + "token_map", + "multipliers", + "primes", + "offsets", + "pad_id", + "image_token_id", + "mm_pad_shift", + ) + }, + ) + if kwargs["input_ids"].numel(): + pad_row = kwargs["commit_history"].shape[0] - 1 + commit_rows = torch.where(lens > 0, slots, pad_row) + last = (starts + lens - 1).clamp(0, kwargs["input_ids"].numel() - 1) + if kwargs["out_cache_loc"] is not None: + commit_rows = torch.where( + kwargs["out_cache_loc"][last] == 0, pad_row, commit_rows + ) + kwargs["commit_history"][commit_rows] = ( + tokens[last, :3].flip(-1).to(torch.int32) + ) + return hash_ids + + +def clone_inputs(kwargs): + result = { + k: v.clone() if isinstance(v, torch.Tensor) else v for k, v in kwargs.items() + } + if kwargs["history"] is kwargs["commit_history"]: + result["history"] = result["commit_history"] + return result + + +@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") +class ExtendHashTest(unittest.TestCase): + def check_case(self, lengths, **options): + inputs = fixture(lengths, **options) + if inputs["num_real"] > 1: + # A request with an invalid final output slot must not update history. + inputs["out_cache_loc"][inputs["num_real"] - 1] = 0 + baseline, candidate = clone_inputs(inputs), clone_inputs(inputs) + for _ in range(2): + expected = reference(baseline) + actual = kernel.engram_hash_extend_and_commit(**candidate) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + # The reserved padding row is deliberately no longer written. + torch.testing.assert_close( + candidate["commit_history"][:-1], + baseline["commit_history"][:-1], + rtol=0, + atol=0, + ) + baseline["out_cache_loc"].fill_(1) + candidate["out_cache_loc"].fill_(1) + + def test_shapes_history_padding_and_multimodal(self): + for lengths in ([8192], [1], [2], [0, 3, 0, 5, 0], [3, 1, 7], [1] * 64, [0, 0]): + for explicit_history in (False, True): + for image in (False, True): + with self.subTest( + lengths=lengths, explicit_history=explicit_history, image=image + ): + self.check_case( + lengths, + padding=7, + explicit_history=explicit_history, + image=image, + ) + + def test_int64_metadata_and_no_output_mask(self): + inputs = fixture([5, 0, 9], dtype=torch.int64) + inputs["out_cache_loc"] = None + a, b = clone_inputs(inputs), clone_inputs(inputs) + torch.testing.assert_close( + kernel.engram_hash_extend_and_commit(**a), reference(b), rtol=0, atol=0 + ) + torch.testing.assert_close( + a["commit_history"][:-1], b["commit_history"][:-1], rtol=0, atol=0 + ) + + def test_batch_shapes_reuse_compiled_kernels(self): + # Batch size and token count must stay runtime arguments: a constexpr would + # compile and device-load a new specialization for every new prefill shape + # while serving, where the load can hit CUDA OOM. Shapes avoid Triton's + # == 1 and % 16 == 0 integer specializations so the keys must all match. + def num_compiled(fn): + return sum(len(cache[0]) for cache in fn.device_caches.values()) + + kernels = ( + kernel._engram_hash_kernel, + kernel._engram_commit_extend_history_kernel, + ) + kernel.engram_hash_extend_and_commit(**fixture([3, 4])) + before = [num_compiled(fn) for fn in kernels] + for lengths in ([2, 5, 6], [3, 9, 2, 5, 4], [7, 2, 11, 3, 5, 9, 6]): + kernel.engram_hash_extend_and_commit(**fixture(lengths)) + self.assertEqual([num_compiled(fn) for fn in kernels], before) + + +if __name__ == "__main__": + unittest.main()