minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Feature] [Ngram spec] Support ngram spec v2 (#17260)

Browse Source 
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
Co-authored-by: Ratish P <114130421+Ratish1@users.noreply.github.com>
This commit is contained in:
Siyuan Chen
2026-06-10 02:46:00 -07:00
committed by GitHub
co-authored by hnyls2002 Ratish P
parent 255843d454
commit 111009ea54
13 changed files with 323 additions and 502 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/spec/test_spec_ngram.py
+3 -2
View File
@@ -2,15 +2,16 @@ import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.kits.spec_server_kits import SpecLogprobKit
from sglang.test.server_fixtures.ngram_fixture import NgramServerBase
# Per-commit: Paged backend only.
# - FA3 base test archived to test/manual/spec/test_spec_ngram_fa3.py
# - Triton + Flashinfer moved to test_spec_ngram_extra.py
register_cuda_ci(est_time=254, stage="base-b", runner_config="1-gpu-large")
register_cuda_ci(est_time=400, stage="base-b", runner_config="1-gpu-large")
class TestNgramSpeculativeDecodingPaged(NgramServerBase, GSM8KMixin):
class TestNgramSpeculativeDecodingPaged(NgramServerBase, GSM8KMixin, SpecLogprobKit):
attention_backend = "flashinfer"
extra_args = ["--page-size", "64"]
test/registered/spec/test_spec_ngram_extra.py
+12 -3
View File
@@ -6,15 +6,24 @@ from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
from sglang.test.server_fixtures.ngram_fixture import NgramServerBase
# Extra: Triton + Flashinfer NGRAM backends. Sibling per-commit file
# (test_spec_ngram.py) keeps the Paged variant.
register_cuda_ci(est_time=254, stage="extra-a", runner_config="1-gpu-large")
# Extra: Triton + Flashinfer NGRAM backends + non-overlap (sync V2) variant.
# Sibling per-commit file (test_spec_ngram.py) keeps the Paged variant.
register_cuda_ci(est_time=400, stage="extra-a", runner_config="1-gpu-large")
class TestNgramSpeculativeDecodingTriton(NgramServerBase, GSM8KMixin):
attention_backend = "triton"
class TestNgramSpeculativeDecodingNoOverlap(NgramServerBase, GSM8KMixin):
"""Non-overlap path: the scheduler drives the V2 worker synchronously
(seq_lens advance via GenerationBatchResult.new_seq_lens, accepted tokens
come from req.output_ids instead of the spec_info splice)."""
attention_backend = "flashinfer"
extra_args = ["--disable-overlap-schedule"]
class TestNgramSpeculativeDecodingFlashinfer(NgramServerBase, GSM8KMixin):
attention_backend = "flashinfer"
extra_args = ["--speculative-ngram-external-sam-budget", "8"]