fix(openai): reject request-supplied chat_template by default (#28135)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
@@ -136,6 +136,7 @@ POSITIONAL_FIELD_ORDER = (
|
||||
"served_model_name",
|
||||
"weight_version",
|
||||
"chat_template",
|
||||
"trust_request_chat_template",
|
||||
"hf_chat_template_name",
|
||||
"completion_template",
|
||||
"file_storage_path",
|
||||
|
||||
@@ -170,6 +170,10 @@ class Serving(msgspec.Struct):
|
||||
Optional[str],
|
||||
"The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server.",
|
||||
] = None
|
||||
trust_request_chat_template: A[
|
||||
bool,
|
||||
"Allow a request to override the server chat template via its chat_template_kwargs. Off by default for safety.",
|
||||
] = False
|
||||
hf_chat_template_name: A[
|
||||
Optional[str],
|
||||
"When the HuggingFace tokenizer has multiple chat templates (e.g., 'default', 'tool_use', 'rag'), specify which named template to use. If not set, the first available template is used.",
|
||||
|
||||
@@ -1014,6 +1014,16 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
if not request.messages:
|
||||
return "Messages cannot be empty."
|
||||
|
||||
if (
|
||||
request.chat_template_kwargs
|
||||
and "chat_template" in request.chat_template_kwargs
|
||||
and not get_serving().trust_request_chat_template
|
||||
):
|
||||
return (
|
||||
"Request-supplied chat_template is not allowed. Start the server "
|
||||
"with --trust-request-chat-template to permit it."
|
||||
)
|
||||
|
||||
if request.return_sampling_mask and not request.return_meta_info:
|
||||
return "return_sampling_mask requires return_meta_info=true."
|
||||
|
||||
|
||||
Reference in New Issue
Block a user