From 68706e615a3f6ac52920b4002b3314055cbca378 Mon Sep 17 00:00:00 2001 From: shuwenn <47200617+alphabetc1@users.noreply.github.com> Date: Fri, 29 May 2026 04:33:11 +0800 Subject: [PATCH] =?UTF-8?q?[SPEC]=20fix:=20use=20effective=20max=20draft?= =?UTF-8?q?=20tokens=20for=20adaptive=20spec=20initiali=E2=80=A6=20(#26354?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: maodoudou168 --- python/sglang/srt/managers/tokenizer_manager.py | 2 +- python/sglang/srt/managers/utils.py | 2 +- .../model_executor/model_runner_kv_cache_mixin.py | 4 +--- python/sglang/srt/server_args.py | 12 ++++++++---- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index cae9ed06f..bc33c29ac 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -297,7 +297,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): # so we need to reserve the space for the draft tokens. self.num_reserved_tokens = max( server_args.speculative_eagle_topk * server_args.speculative_num_steps, - server_args.speculative_num_draft_tokens, + server_args.max_speculative_num_draft_tokens, ) else: self.num_reserved_tokens = 0 diff --git a/python/sglang/srt/managers/utils.py b/python/sglang/srt/managers/utils.py index 67de78596..0dae5b27d 100644 --- a/python/sglang/srt/managers/utils.py +++ b/python/sglang/srt/managers/utils.py @@ -240,7 +240,7 @@ def get_alloc_len_per_decode(server_args: Optional[ServerArgs] = None) -> int: spec_steps = server_args.speculative_num_steps or 1 spec_topk = server_args.speculative_eagle_topk or 1 - spec_tokens = server_args.speculative_num_draft_tokens + spec_tokens = server_args.max_speculative_num_draft_tokens page_size = server_args.page_size if page_size == 1 or spec_topk == 1: diff --git a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py index 0198a1919..5ee04b2ad 100644 --- a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py +++ b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py @@ -285,9 +285,7 @@ class ModelRunnerKVCacheMixin: # Initialize req_to_token_pool if self.req_to_token_pool is None: # FIXME(lsyin): this is the temporary fix for the context length issue when using speculative decoding - max_spec_draft_tokens = ( - self.server_args.effective_max_speculative_num_draft_tokens() - ) + max_spec_draft_tokens = self.server_args.max_speculative_num_draft_tokens extra_max_context_len = 4 if max_spec_draft_tokens is not None: extra_max_context_len += max_spec_draft_tokens diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 9d27b3366..4f5dd3456 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -25,6 +25,7 @@ import logging import os import random import tempfile +from functools import cached_property from typing import Any, Callable, Dict, List, Literal, Optional, Union from sglang.srt.arg_groups.argparse_actions import ( @@ -590,8 +591,6 @@ class ServerArgs: speculative_moe_runner_backend: Optional[str] = None speculative_moe_a2a_backend: Optional[str] = None speculative_draft_model_quantization: Optional[str] = None - speculative_adaptive: bool = False - speculative_adaptive_config: Optional[str] = None speculative_skip_dp_mlp_sync: bool = False # Speculative decoding (ngram) @@ -605,6 +604,10 @@ class ServerArgs: speculative_ngram_external_corpus_max_tokens: int = 10000000 enable_multi_layer_eagle: bool = False + # Adaptive speculative decoding + speculative_adaptive: bool = False + speculative_adaptive_config: Optional[str] = None + # Expert parallelism ep_size: int = 1 moe_a2a_backend: Literal[ @@ -7050,8 +7053,9 @@ class ServerArgs: def enable_mamba_extra_buffer(self) -> bool: return self.mamba_scheduler_strategy == "extra_buffer" - def effective_max_speculative_num_draft_tokens(self) -> Optional[int]: - """Return the maximum draft-token count runtime speculative decoding may use.""" + @cached_property + def max_speculative_num_draft_tokens(self) -> Optional[int]: + """Return the maximum draft-token count speculative decoding may use.""" if self.speculative_num_draft_tokens is None: return None if not self.speculative_adaptive: