minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

fix: handle Jinja2 template errors as client errors in OpenAIServingChat (#14748)

Browse Source 
Signed-off-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
Xinyuan Tong
2025-12-11 17:06:26 -08:00
committed by GitHub
parent f832994c32
commit 4885f8b9c1
1 changed files with 22 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/openai/serving_chat.py
+22 -16
View File
@@ -7,6 +7,7 @@ import time
import uuid import uuid
from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional, Union from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional, Union
import jinja2
import orjson import orjson
from fastapi import Request from fastapi import Request
from fastapi.responses import ORJSONResponse, StreamingResponse from fastapi.responses import ORJSONResponse, StreamingResponse
@@ -362,27 +363,32 @@ class OpenAIServingChat(OpenAIServingBase):
else {} else {}
), ),
) )
except Exception: except Exception as e:
# This except branch will be triggered when the chosen model # If the first attempt fails, try transforming the tools format
# has a different tools input format that is not compatible # This handles models like Mistral that have a different tools input format
# with openAI's apply_chat_template tool_call format, like Mistral. # that is not compatible with OpenAI's apply_chat_template tool_call format
tools = ( tools = (
[t if "function" in t else {"function": t} for t in tools] [t if "function" in t else {"function": t} for t in tools]
if tools if tools
else None else None
) )
prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template( try:
openai_compatible_messages, prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template(
tokenize=True, openai_compatible_messages,
add_generation_prompt=True, tokenize=True,
tools=tools, add_generation_prompt=True,
reasoning_effort=request.reasoning_effort, tools=tools,
**( reasoning_effort=request.reasoning_effort,
request.chat_template_kwargs **(
if request.chat_template_kwargs request.chat_template_kwargs
else {} if request.chat_template_kwargs
), else {}
) ),
)
except jinja2.TemplateError as template_error:
# Template errors (e.g., from raise_exception in Jinja templates)
# should be treated as client errors (400 BadRequest)
raise ValueError(str(template_error)) from template_error
if assistant_prefix: if assistant_prefix:
encoded = self.tokenizer_manager.tokenizer.encode(assistant_prefix) encoded = self.tokenizer_manager.tokenizer.encode(assistant_prefix)