minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Fix default_max_tokens compute error in responses api when mtp is opened (#18932)

Browse Source
This commit is contained in:
0xNullPath
2026-03-12 16:00:48 +08:00
committed by GitHub
parent ac1310b300
commit 46b558445d
1 changed files with 8 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/openai/serving_responses.py
+8 -2
View File
@@ -258,8 +258,11 @@ class OpenAIServingResponses(OpenAIServingChat):
if hasattr(self.tokenizer_manager.model_config, "context_len") if hasattr(self.tokenizer_manager.model_config, "context_len")
else 4096 else 4096
) )
# Account for reserved tokens (e.g., EAGLE speculative decoding slots)
# that the tokenizer_manager adds during validation
num_reserved_tokens = self.tokenizer_manager.num_reserved_tokens
default_max_tokens = max( default_max_tokens = max(
context_len - prompt_length, 512 context_len - prompt_length - num_reserved_tokens, 512
) # Ensure minimum 512 tokens ) # Ensure minimum 512 tokens
sampling_params = request.to_sampling_params( sampling_params = request.to_sampling_params(
default_max_tokens, self.default_sampling_params default_max_tokens, self.default_sampling_params
@@ -1313,7 +1316,10 @@ class OpenAIServingResponses(OpenAIServingChat):
context_len = getattr( context_len = getattr(
self.tokenizer_manager.model_config, "context_len", 4096 self.tokenizer_manager.model_config, "context_len", 4096
) )
remaining_tokens = context_len - len(prompt_token_ids) - 1 num_reserved_tokens = self.tokenizer_manager.num_reserved_tokens
remaining_tokens = (
context_len - len(prompt_token_ids) - num_reserved_tokens
)
if isinstance(sampling_params, dict): if isinstance(sampling_params, dict):
sampling_params["max_new_tokens"] = max(remaining_tokens, 1) sampling_params["max_new_tokens"] = max(remaining_tokens, 1)