diff --git a/python/sglang/kernels/ops/speculative/__init__.py b/python/sglang/kernels/ops/speculative/__init__.py index 644296c508..fa44cf5c9f 100644 --- a/python/sglang/kernels/ops/speculative/__init__.py +++ b/python/sglang/kernels/ops/speculative/__init__.py @@ -23,6 +23,7 @@ _TRITON_KERNELS = [ ("ragged_verify_kernels", "build_qo_indptr"), ("reject_sampling", "chain_speculative_sampling_triton"), ("row_argmax", "row_argmax"), + ("reconstruct_tree", "reconstruct_indices_from_tree_mask_triton"), ] for _mod, _fn in _TRITON_KERNELS: register_kernel( diff --git a/python/sglang/kernels/ops/speculative/reconstruct_tree.py b/python/sglang/kernels/ops/speculative/reconstruct_tree.py new file mode 100644 index 0000000000..3a9d650c7a --- /dev/null +++ b/python/sglang/kernels/ops/speculative/reconstruct_tree.py @@ -0,0 +1,153 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Fused Triton kernel: reconstruct NGRAM verify metadata from a tree mask. + +Device-native fallback used where the compiled +``sgl_kernel.speculative.reconstruct_indices_from_tree_mask`` op is unavailable, +e.g. Intel XPU. It reproduces that op's contract exactly, mutating +``positions`` / ``retrieve_*`` in place. + +Semantics (per batch ``b``, per node ``tid``; ``n = draft_token_num``): + ``tree_mask[b, i, j]`` (row-major, flat) marks node ``j`` an ancestor of ``i``. + ``ancestors[i, j] = tree_mask[b, i, j] & (j < i)`` (strict ancestors) + ``positions[b,tid] = (#ancestors of tid) + verified_seq_len[b]`` + ``retrieve_index[b,tid] = b * n + tid`` + ``parent[tid] = max{ jtid : ancestors[k, tid] } else -1`` (first child) + ``retrieve_next_sibling[b,tid] = min{ k>tid : parent[k]==parent[tid] }`` + else -1, and -1 when parent[tid] < 0. + +The whole reduction is over one ``n x n`` bool matrix, so a single program per +batch loads it once into registers and emits all four outputs -- collapsing the +~15-launch, multi-``[bs,n,n]``-HBM-roundtrip torch fallback into one kernel with +zero materialized intermediates. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from sglang.srt.utils import next_power_of_2 + +# Sentinel for "no ancestor / not-yet-found child": any value >= n. Reductions use +# n for min-searches (first child/sibling) and -1 for max-searches (parent). + + +@triton.jit +def _reconstruct_tree_mask_kernel( + tree_mask_ptr, # bool [bs * n * n], row-major (b, i, j) + verified_seq_len_ptr, # int [bs] + positions_ptr, # int64 [bs * n] (out) + retrieve_index_ptr, # int64 [bs * n] (out) + retrieve_next_token_ptr, # int64 [bs * n] (out) + retrieve_next_sibling_ptr, # int64 [bs * n] (out) + n: tl.constexpr, # draft_token_num + BLOCK_N: tl.constexpr, # next_power_of_2(n) +): + b = tl.program_id(axis=0) + + idx = tl.arange(0, BLOCK_N) + row = idx[:, None] # i -> the node being described + col = idx[None, :] # j -> ancestor candidate + valid = (row < n) & (col < n) + + # Load this batch's n x n ancestor mask exactly once. + m = tl.load(tree_mask_ptr + b * (n * n) + row * n + col, mask=valid, other=0).to( + tl.int1 + ) + + # ancestors[i, j] = mask[i, j] & (j < i): strict lower-triangular ancestors. + ancestors = m & (col < row) + ancestors_i = ancestors.to(tl.int32) + + # depth[i] = #ancestors (reduce columns); positions = depth + verified_seq_len. + depth = tl.sum(ancestors_i, axis=1) + seq_len = tl.load(verified_seq_len_ptr + b).to(tl.int64) + positions = depth.to(tl.int64) + seq_len + + # parent[i] = largest jtid with ancestors[k, tid] (reduce rows over the + # tid-th column). ancestors already encodes k>tid via (col < row). + has_child = tl.max(ancestors_i, axis=0) > 0 + first_child = tl.min(tl.where(ancestors, row, n), axis=0) # indexed by column tid + next_token = tl.where(has_child, first_child, -1) + + # next sibling of tid = smallest k>tid sharing tid's parent; -1 if tid is a + # root (parent < 0). parent[k] over rows vs parent[tid] over columns. + parent_k = parent[:, None] + parent_tid = parent[None, :] + same_parent = (parent_k == parent_tid) & (row > col) & (row < n) + first_sibling = tl.min(tl.where(same_parent, row, n), axis=0) # indexed by tid + has_parent = parent >= 0 + next_sibling = tl.where(has_parent & (first_sibling < n), first_sibling, -1) + + # Store the n valid entries of row b. + out = b * n + idx + store_mask = idx < n + tl.store(positions_ptr + out, positions, mask=store_mask) + tl.store(retrieve_index_ptr + out, (b * n + idx).to(tl.int64), mask=store_mask) + tl.store(retrieve_next_token_ptr + out, next_token.to(tl.int64), mask=store_mask) + tl.store( + retrieve_next_sibling_ptr + out, next_sibling.to(tl.int64), mask=store_mask + ) + + +def _num_warps_for(block_n: int) -> int: + # One program owns an n x n tile. Small trees (the common decode case) waste + # EUs with wide warp groups; scale warps with the tile so each subgroup keeps + # useful lanes. Tuned for Xe2 SIMD16 subgroups. + if block_n <= 16: + return 1 + if block_n <= 32: + return 2 + return 4 + + +def reconstruct_indices_from_tree_mask_triton( + tree_mask: torch.Tensor, + verified_seq_len: torch.Tensor, + positions: torch.Tensor, + retrieve_index: torch.Tensor, + retrieve_next_token: torch.Tensor, + retrieve_next_sibling: torch.Tensor, + batch_size: int, + draft_token_num: int, +) -> None: + """Fused device kernel for NGRAM verify-metadata reconstruction. + + Drop-in for ``reconstruct_indices_from_tree_mask`` on GPU-like devices + (XPU/CUDA/HIP). Mutates ``positions`` / ``retrieve_*`` in place; identical + numerical contract to the compiled op and the torch fallback. + """ + if batch_size == 0: + return + + n = draft_token_num + block_n = next_power_of_2(n) + + _reconstruct_tree_mask_kernel[(batch_size,)]( + tree_mask, + verified_seq_len, + positions, + retrieve_index, + retrieve_next_token, + retrieve_next_sibling, + n, + block_n, + num_warps=_num_warps_for(block_n), + ) diff --git a/python/sglang/srt/arg_groups/speculative_hook.py b/python/sglang/srt/arg_groups/speculative_hook.py index 957c90689f..1da0defffc 100644 --- a/python/sglang/srt/arg_groups/speculative_hook.py +++ b/python/sglang/srt/arg_groups/speculative_hook.py @@ -1114,9 +1114,9 @@ def _handle_eagle_family(server_args: ServerArgs) -> None: def _handle_ngram(server_args: ServerArgs) -> None: cfg = resolving_view(server_args) - if cfg.device not in ("cuda", "cpu"): + if cfg.device not in ("cuda", "cpu", "xpu"): raise ValueError( - "Ngram speculative decoding only supports CUDA or CPU devices." + "Ngram speculative decoding only supports CUDA, CPU, or XPU devices." ) _disable_overlap_schedule_for_cpu(server_args) diff --git a/python/sglang/srt/speculative/ngram_worker.py b/python/sglang/srt/speculative/ngram_worker.py index 66429bbe0b..109d5cfbf7 100644 --- a/python/sglang/srt/speculative/ngram_worker.py +++ b/python/sglang/srt/speculative/ngram_worker.py @@ -31,12 +31,16 @@ from sglang.srt.speculative.spec_utils import ( prepare_mamba_track_for_verify, record_stream_for_v2_verify, ) -from sglang.srt.utils import is_cpu, is_cuda +from sglang.srt.utils import is_cpu, is_cuda, is_xpu from sglang.srt.utils.async_probe import maybe_detect_inf, maybe_detect_nan _is_cpu = is_cpu() -if is_cuda(): +if is_xpu(): + from sglang.kernels.ops.speculative.reconstruct_tree import ( + reconstruct_indices_from_tree_mask_triton as reconstruct_indices_from_tree_mask, + ) +elif is_cuda(): from sglang.kernels.ops.speculative.tree import reconstruct_indices_from_tree_mask else: from sgl_kernel.speculative import reconstruct_indices_from_tree_mask diff --git a/test/manual/kernels/test_reconstruct_tree_mask.py b/test/manual/kernels/test_reconstruct_tree_mask.py new file mode 100644 index 0000000000..e77d322dec --- /dev/null +++ b/test/manual/kernels/test_reconstruct_tree_mask.py @@ -0,0 +1,182 @@ +"""Test the Triton implementation of ``reconstruct_indices_from_tree_mask``. + +``reconstruct_indices_from_tree_mask_triton`` is the device-native fallback for +the compiled ``sgl_kernel.speculative.reconstruct_indices_from_tree_mask`` op on +platforms that ship no such op (e.g. Intel XPU). It reconstructs NGRAM verify +metadata (``positions`` / ``retrieve_index`` / ``retrieve_next_token`` / +``retrieve_next_sibling``) from a per-batch ``n x n`` tree mask. Run manually; +it exercises whichever GPU-like backends (CUDA, XPU) are locally available. + +This test pins the kernel to the op's documented contract two ways: + +1. **External-literal anchor** -- the worked example from the compiled op's own + test (``kernels/aot/tests/speculative/test_ngram_utils.py``). Those output + values are the CUDA/CPU op's ground truth, independent of any torch port. + +2. **Differential sweep vs an independent oracle** -- a plain-python + reconstruction (no torch/triton vectorization shared with the kernel) over + randomly generated *valid* trees, across power-of-two and non-power-of-two + ``n`` (exercises the ``BLOCK_N`` padding mask) and multiple batch sizes. + +Failure modes guarded (no other test covers these): + - root nodes (``parent < 0``) must NOT link as siblings to one another; + - ``n`` not a power of two must mask out the padded ``BLOCK_N`` tail; + - the two reduction axes (parent/depth over columns, child/sibling over rows) + must not be transposed. +""" + +import unittest + +import numpy as np +import torch + +from sglang.test.test_utils import CustomTestCase + +try: + from sglang.kernels.ops.speculative.reconstruct_tree import ( + reconstruct_indices_from_tree_mask_triton, + ) + + _HAS_KERNEL = True +except Exception: # pragma: no cover - import guarded for hardware-less CI shards + _HAS_KERNEL = False + +_AVAILABLE_DEVICES = [ + device + for device, available in ( + ("cuda", torch.cuda.is_available()), + ("xpu", hasattr(torch, "xpu") and torch.xpu.is_available()), + ) + if available +] + + +def _make_valid_tree_mask(bs: int, n: int, seed: int) -> np.ndarray: + """Random *valid* tree mask [bs, n, n]: mask[b, i, j] == node j is an + ancestor of node i (transitive closure; the diagonal is set). Each node + either roots (~30%) or attaches under a uniformly random earlier node.""" + rng = np.random.default_rng(seed) + mask = np.zeros((bs, n, n), dtype=bool) + for b in range(bs): + ancestors = [set() for _ in range(n)] + for i in range(n): + ancestors[i].add(i) + if i > 0 and rng.random() >= 0.3: + parent = int(rng.integers(0, i)) + ancestors[i] |= ancestors[parent] + for j in ancestors[i]: + mask[b, i, j] = True + return mask + + +def _reference(mask: np.ndarray, seq_lens: np.ndarray, bs: int, n: int): + """Plain-python oracle for the documented reconstruction contract. Returns + (positions, retrieve_index, next_token, next_sibling) as int64 arrays.""" + positions = np.empty(bs * n, dtype=np.int64) + retrieve_index = np.empty(bs * n, dtype=np.int64) + next_token = np.full(bs * n, -1, dtype=np.int64) + next_sibling = np.full(bs * n, -1, dtype=np.int64) + + for b in range(bs): + parent = [-1] * n + for tid in range(n): + ancestors = [j for j in range(tid) if mask[b, tid, j]] + positions[b * n + tid] = len(ancestors) + int(seq_lens[b]) + retrieve_index[b * n + tid] = b * n + tid + parent[tid] = max(ancestors) if ancestors else -1 + + for tid in range(n): + children = [k for k in range(tid + 1, n) if mask[b, k, tid]] + if children: + next_token[b * n + tid] = min(children) + if parent[tid] >= 0: + siblings = [k for k in range(tid + 1, n) if parent[k] == parent[tid]] + if siblings: + next_sibling[b * n + tid] = min(siblings) + + return positions, retrieve_index, next_token, next_sibling + + +@unittest.skipUnless(_HAS_KERNEL, "reconstruct_tree Triton kernel import required") +@unittest.skipUnless(_AVAILABLE_DEVICES, "CUDA or XPU device required") +class TestReconstructTreeMask(CustomTestCase): + def _run_kernel(self, device, mask_bool_cpu, seq_lens_cpu, bs, n): + tree_mask = mask_bool_cpu.reshape(-1).contiguous().to(device) + seq_lens = seq_lens_cpu.to(device) + positions = torch.empty(bs * n, dtype=torch.int64, device=device) + retrieve_index = torch.full((bs, n), -1, dtype=torch.int64, device=device) + next_token = torch.full((bs, n), -1, dtype=torch.int64, device=device) + next_sibling = torch.full((bs, n), -1, dtype=torch.int64, device=device) + reconstruct_indices_from_tree_mask_triton( + tree_mask, + seq_lens, + positions, + retrieve_index, + next_token, + next_sibling, + bs, + n, + ) + if device == "cuda": + torch.cuda.synchronize() + else: + torch.xpu.synchronize() + return ( + positions.cpu().numpy(), + retrieve_index.reshape(-1).cpu().numpy(), + next_token.reshape(-1).cpu().numpy(), + next_sibling.reshape(-1).cpu().numpy(), + ) + + def test_matches_compiled_op_worked_example(self): + # Literal contract from the compiled op's test (test_ngram_utils.py): + # tree over 4 nodes, verified_seq_len == 12. + bs, n = 1, 4 + tree_mask = torch.tensor( + [1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 1], + dtype=torch.bool, + ) + seq_lens = torch.tensor([12], dtype=torch.int64) + for device in _AVAILABLE_DEVICES: + with self.subTest(device=device): + positions, retrieve_index, next_token, next_sibling = self._run_kernel( + device, tree_mask.reshape(bs, n, n), seq_lens, bs, n + ) + self.assertEqual(positions.tolist(), [12, 13, 13, 14]) + self.assertEqual(retrieve_index.tolist(), [0, 1, 2, 3]) + self.assertEqual(next_token.tolist(), [1, -1, 3, -1]) + self.assertEqual(next_sibling.tolist(), [-1, 2, -1, -1]) + + def test_matches_reference_over_random_trees(self): + # Power-of-two and non-power-of-two n (BLOCK_N tail masking) x batch sizes. + for device in _AVAILABLE_DEVICES: + for n in (1, 2, 3, 7, 8, 16, 17, 32, 63, 64): + for bs in (1, 3, 16, 64): + for seed in range(3): + mask = _make_valid_tree_mask(bs, n, seed * 1000 + bs * 100 + n) + seq_lens = torch.from_numpy( + np.random.default_rng(seed).integers(1, 200, size=bs) + ).to(torch.int64) + got = self._run_kernel( + device, torch.from_numpy(mask), seq_lens, bs, n + ) + ref = _reference(mask, seq_lens.numpy(), bs, n) + names = ( + "positions", + "retrieve_index", + "retrieve_next_token", + "retrieve_next_sibling", + ) + for name, g, r in zip(names, got, ref): + np.testing.assert_array_equal( + g, + r, + err_msg=( + f"{name} mismatch at device={device} " + f"bs={bs} n={n} seed={seed}" + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/spec/test_ngram_corpus.py b/test/registered/unit/spec/test_ngram_corpus.py index 5c0c45a056..c13bf19ea9 100644 --- a/test/registered/unit/spec/test_ngram_corpus.py +++ b/test/registered/unit/spec/test_ngram_corpus.py @@ -10,10 +10,11 @@ from sglang.srt.speculative.cpp_ngram.external_corpus import ( iter_external_corpus_chunks, ) from sglang.srt.speculative.cpp_ngram.ngram_corpus import NgramCorpus -from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.ci.ci_register import register_cpu_ci, register_xpu_ci from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=20, suite="base-a-test-cpu") +register_xpu_ci(est_time=20, suite="stage-b-test-1-gpu-xpu") def _make_corpus(match_type="BFS", **kwargs): @@ -350,6 +351,44 @@ class TestFrequencyBoosting(CustomTestCase): ) +class TestFrequencyTieSensitivity(CustomTestCase): + """A single additional insertion -- not the forced 10x majority in + TestFrequencyBoosting -- is enough to flip which candidate Prob-mode + returns for a query that two independent, freshly-stated requests both + send unchanged. Reproduces the mechanism behind an observed divergence: + two requests generating from the same repeated prompt got different + NGRAM drafts because one more generation's output landed in the shared + corpus between them, flipping a near-tied frequency count. + """ + + def test_single_insert_flips_tied_candidate(self): + corpus = _make_corpus( + "PROB", + draft_token_num=2, + max_bfs_breadth=1, + min_bfs_breadth=1, + max_trie_depth=5, + ) + corpus.batch_put([[1, 2, 3, 10, 11]]) + corpus.batch_put([[1, 2, 3, 20, 21]]) + corpus.synchronize() + + # Two unrelated requests, each with fresh state, send the identical + # query; only the shared corpus differs between them. + ids_before, _ = _batch_get_with_state(corpus, "req-before", [1, 2, 3], 3) + self.assertEqual(ids_before.tolist(), [3, 10], ids_before.tolist()) + + corpus.batch_put([[1, 2, 3, 20, 21]]) + corpus.synchronize() + + ids_after, _ = _batch_get_with_state(corpus, "req-after", [1, 2, 3], 3) + self.assertEqual( + ids_after.tolist(), + [3, 20], + f"one more insertion should flip the tie, got {ids_after.tolist()}", + ) + + class TestRecencyOrdering(CustomTestCase): """Verify that BFS mode respects LRU recency."""