[Spec][Ngram] 4/N: Remove max_match_window_size and min_match_window_size, matching all suffixes of the Trie (#21225)
This commit is contained in:
@@ -506,8 +506,6 @@ class ServerArgs:
|
||||
speculative_draft_model_quantization: Optional[str] = None
|
||||
|
||||
# Speculative decoding (ngram)
|
||||
speculative_ngram_min_match_window_size: int = 1
|
||||
speculative_ngram_max_match_window_size: int = 12
|
||||
speculative_ngram_min_bfs_breadth: int = 1
|
||||
speculative_ngram_max_bfs_breadth: int = 10
|
||||
speculative_ngram_match_type: Literal["BFS", "PROB"] = "BFS"
|
||||
@@ -3108,8 +3106,10 @@ class ServerArgs:
|
||||
self.enable_mixed_chunk = False
|
||||
self.speculative_eagle_topk = self.speculative_ngram_max_bfs_breadth
|
||||
if self.speculative_num_draft_tokens is None:
|
||||
self.speculative_num_draft_tokens = (
|
||||
self.speculative_ngram_max_match_window_size
|
||||
self.speculative_num_draft_tokens = 12
|
||||
logger.warning(
|
||||
"speculative_num_draft_tokens is set to 12 by default for ngram speculative decoding. "
|
||||
"You can override this by explicitly setting --speculative-num-draft-tokens."
|
||||
)
|
||||
logger.warning(
|
||||
"The overlap scheduler and mixed chunked prefill are disabled because of "
|
||||
@@ -4851,18 +4851,6 @@ class ServerArgs:
|
||||
)
|
||||
|
||||
# Speculative decoding (ngram)
|
||||
parser.add_argument(
|
||||
"--speculative-ngram-min-match-window-size",
|
||||
type=int,
|
||||
default=ServerArgs.speculative_ngram_min_match_window_size,
|
||||
help="The minimum window size for pattern matching in ngram speculative decoding.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--speculative-ngram-max-match-window-size",
|
||||
type=int,
|
||||
default=ServerArgs.speculative_ngram_max_match_window_size,
|
||||
help="The maximum window size for pattern matching in ngram speculative decoding.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--speculative-ngram-min-bfs-breadth",
|
||||
type=int,
|
||||
|
||||
Reference in New Issue
Block a user