[Spec][Ngram] Support multiple SAMs with dynamic HTTP API (#22203)

This commit is contained in:
Liangsheng Yin
2026-04-06 18:49:22 -07:00
committed by GitHub
parent 49cb7d546e
commit e4b1366a46
14 changed files with 685 additions and 121 deletions
+1 -6
View File
@@ -3132,11 +3132,6 @@ class ServerArgs:
"speculative_ngram_external_sam_budget must be less than or equal to "
f"speculative_num_draft_tokens - 1 ({self.speculative_num_draft_tokens - 1})."
)
elif self.speculative_ngram_external_sam_budget != 0:
raise ValueError(
"--speculative-ngram-external-sam-budget requires "
"--speculative-ngram-external-corpus-path."
)
logger.warning(
"The overlap scheduler and mixed chunked prefill are disabled because of "
"using ngram speculative decoding."
@@ -4921,7 +4916,7 @@ class ServerArgs:
"--speculative-ngram-external-corpus-path",
type=str,
default=ServerArgs.speculative_ngram_external_corpus_path,
help="Optional path to an external corpus used to build a read-only SAM for ngram speculative decoding.",
help="Path to an external JSONL corpus to pre-load into SAM at startup. Additional corpora can be added at runtime via POST /add_external_corpus.",
)
parser.add_argument(
"--speculative-ngram-external-sam-budget",