[Feat] glm-5.3: Render GLM-5.3 and GLM-5.3-Flash chat prompts with a Python chat template (#45154)
Signed-off-by: Shijin Zhang <75300765+Dovis01@users.noreply.github.com> Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
co-authored by
Xinyuan Tong
parent
800e60e345
commit
ca473ef3e4
@@ -467,6 +467,9 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
return chat_encoding.spec_supports_inline_system(self.chat_encoding_spec)
|
||||
if self.template_manager.chat_template_name is not None:
|
||||
return False
|
||||
python_template = self.template_manager.python_chat_template
|
||||
if python_template is not None:
|
||||
return python_template.supports_inline_system
|
||||
tokenizer = self.tokenizer_manager.tokenizer
|
||||
return tokenizer is not None and detect_inline_system_support(
|
||||
tokenizer.chat_template
|
||||
|
||||
@@ -0,0 +1,438 @@
|
||||
# Adapted from the GLM-5.3 / GLM-5.3-Flash reference chat_template.py by Z.ai.
|
||||
"""Python version of the GLM-5.3 and GLM-5.3-Flash chat templates.
|
||||
|
||||
The two templates are the same apart from the multimodal block, so a single
|
||||
renderer serves both. The output is byte-identical to the Jinja templates.
|
||||
Where a template fails on malformed input, the renderer fails at the same
|
||||
point, raising ``ValueError`` for Jinja's ``UndefinedError`` and the very
|
||||
``TypeError`` Jinja would raise, so the serving layer answers both with 400
|
||||
exactly as it did for the template. Jinja semantics that are easy to get wrong
|
||||
when reading the template source are called out inline; ``_UNDEFINED`` stands
|
||||
for Jinja's ``Undefined``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Optional
|
||||
|
||||
_UNDEFINED = object()
|
||||
|
||||
_TOOLS_PREFIX = (
|
||||
"<|system|>\n"
|
||||
"# Tools\n\n"
|
||||
"You may call one or more functions to assist with the user query.\n\n"
|
||||
"You are provided with function signatures within <tools></tools> XML tags:\n"
|
||||
"<tools>\n"
|
||||
)
|
||||
_TOOLS_SUFFIX = (
|
||||
"</tools>\n\n"
|
||||
"For each function call, output the function name and arguments within the "
|
||||
"following XML format:\n"
|
||||
"<tool_call>{function-name}<arg_key>{arg-key-1}</arg_key>"
|
||||
"<arg_value>{arg-value-1}</arg_value><arg_key>{arg-key-2}</arg_key>"
|
||||
"<arg_value>{arg-value-2}</arg_value>...</tool_call>"
|
||||
)
|
||||
|
||||
# Tuples rather than sets: Jinja's `in` compares with ==, so an unhashable
|
||||
# part type must not raise here.
|
||||
_IMAGE_PART_TYPES = ("image", "image_url")
|
||||
_VIDEO_PART_TYPES = ("video", "video_url")
|
||||
_AUDIO_PART_TYPES = ("audio", "audio_url", "input_audio")
|
||||
_MEDIA_PART_TYPES = _IMAGE_PART_TYPES + _VIDEO_PART_TYPES + _AUDIO_PART_TYPES
|
||||
|
||||
_MEDIA_REMINDER = (
|
||||
"<reminder>You are unable to process this {} because you don't have "
|
||||
"multi-modal input ability. Try different methods.</reminder>"
|
||||
)
|
||||
_IMAGE_TOKENS = "<|begin_of_image|><|image|><|end_of_image|>"
|
||||
_VIDEO_TOKENS = "<|begin_of_video|><|video|><|end_of_video|>"
|
||||
_AUDIO_TOKENS = "<|begin_of_audio|><|end_of_audio|>"
|
||||
|
||||
|
||||
def _get(obj: Any, key: str, default: Any = _UNDEFINED) -> Any:
|
||||
"""``obj.key`` in Jinja: an item of a mapping, an attribute otherwise."""
|
||||
if isinstance(obj, Mapping):
|
||||
return obj[key] if key in obj else default
|
||||
return getattr(obj, key, default)
|
||||
|
||||
|
||||
def _first(value: Any) -> Any:
|
||||
"""``value.0`` in Jinja, which is Undefined when there is no item 0."""
|
||||
try:
|
||||
return value[0]
|
||||
except (AttributeError, TypeError, LookupError):
|
||||
return _UNDEFINED
|
||||
|
||||
|
||||
def _string(value: Any) -> str:
|
||||
"""``{{ value }}`` in Jinja: Undefined renders empty, None renders "None"."""
|
||||
return "" if value is _UNDEFINED else str(value)
|
||||
|
||||
|
||||
def _is_iterable(value: Any) -> bool:
|
||||
try:
|
||||
iter(value)
|
||||
except TypeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _render_media(part_type: str, multimodal: bool) -> str:
|
||||
if not multimodal:
|
||||
media_type = part_type.replace("_url", "").replace("input_", "")
|
||||
return _MEDIA_REMINDER.format(media_type)
|
||||
if part_type in _IMAGE_PART_TYPES:
|
||||
return _IMAGE_TOKENS
|
||||
if part_type in _VIDEO_PART_TYPES:
|
||||
return _VIDEO_TOKENS
|
||||
return _AUDIO_TOKENS
|
||||
|
||||
|
||||
def _visible_text(content: Any, multimodal: bool) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if _is_iterable(content) and not isinstance(content, Mapping):
|
||||
parts = []
|
||||
for item in content:
|
||||
if isinstance(item, Mapping):
|
||||
part_type = _get(item, "type")
|
||||
if part_type == "text":
|
||||
parts.append(_string(_get(item, "text")))
|
||||
elif part_type in _MEDIA_PART_TYPES:
|
||||
parts.append(_render_media(part_type, multimodal))
|
||||
elif isinstance(item, str):
|
||||
parts.append(item)
|
||||
return "".join(parts)
|
||||
# The template guards this branch with `content is not none`, so None
|
||||
# renders nothing instead of "None".
|
||||
return "" if content is None else _string(content)
|
||||
|
||||
|
||||
def _unwrap_tool(tool: Any) -> Any:
|
||||
# `'function' in tool` is a membership test, so a present but empty
|
||||
# `function` still replaces the tool.
|
||||
if isinstance(tool, Mapping) and "function" in tool:
|
||||
return tool["function"]
|
||||
return tool
|
||||
|
||||
|
||||
def _tool_to_json(tool: Any) -> str:
|
||||
# The tool_to_json macro calls `tool.items()`, which fails on a non-mapping.
|
||||
if not isinstance(tool, Mapping):
|
||||
raise ValueError(f"Each tool must be a JSON object, got {type(tool).__name__}.")
|
||||
# The macro writes `"{{ k }}": {{ v | tojson }}` by hand: values go through
|
||||
# tojson (json.dumps under transformers), but keys are interpolated raw and
|
||||
# are not JSON-escaped, so json.dumps on the whole dict would diverge.
|
||||
fields = [
|
||||
f'"{key!s}": {json.dumps(value, ensure_ascii=False)}'
|
||||
for key, value in tool.items()
|
||||
if key != "defer_loading" and key != "strict"
|
||||
]
|
||||
return "{" + ", ".join(fields) + "}"
|
||||
|
||||
|
||||
def _tool_references_to_response(refs: Any, tools: Any) -> str:
|
||||
# The template defines this macro under `{%- if tools -%}`, so rendering a
|
||||
# tool_reference without tools called an undefined macro.
|
||||
if not tools:
|
||||
raise ValueError(
|
||||
"Rendering a tool_reference requires `tools`; the GLM-5.3 chat "
|
||||
"template only defines its tool macros when tools are given."
|
||||
)
|
||||
parts = ["<tool_response><tools>\n"]
|
||||
for ref in refs:
|
||||
ref_name = _get(ref, "name")
|
||||
for tool in tools:
|
||||
tool = _unwrap_tool(tool)
|
||||
# Undefined equals Undefined in Jinja, so a reference without a
|
||||
# name matches every tool without one, as it does here.
|
||||
if _get(tool, "name") == ref_name:
|
||||
parts.append(_tool_to_json(tool) + "\n")
|
||||
parts.append("</tools></tool_response>")
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def _tool_response(text: str) -> str:
|
||||
return "<tool_response>" + text + "</tool_response>"
|
||||
|
||||
|
||||
def _id_of(obj: Any) -> str:
|
||||
tool_call_id = _get(obj, "tool_call_id", None)
|
||||
if tool_call_id:
|
||||
return str(tool_call_id)
|
||||
obj_id = _get(obj, "id", None)
|
||||
if obj_id:
|
||||
return str(obj_id)
|
||||
return ""
|
||||
|
||||
|
||||
def _is_list_of_outputs(message: Any) -> bool:
|
||||
# `m.content and m.content.0.output is defined`: falsy content
|
||||
# short-circuits, while reading `.output` from a missing item 0 (a mapping
|
||||
# or a scalar content) raises UndefinedError.
|
||||
content = _get(message, "content", None)
|
||||
if not content:
|
||||
return False
|
||||
first = _first(content)
|
||||
if first is _UNDEFINED:
|
||||
raise ValueError(
|
||||
"A tool message `content` must be a string or a list, got "
|
||||
f"{type(content).__name__}."
|
||||
)
|
||||
return _get(first, "output") is not _UNDEFINED
|
||||
|
||||
|
||||
def _is_tool_reference_list(value: Any) -> bool:
|
||||
# `v is iterable and v is not string and v is not mapping and v and
|
||||
# v.0.type == "tool_reference"`; reading `.type` from a missing item 0
|
||||
# raises UndefinedError like `_is_list_of_outputs`.
|
||||
if isinstance(value, (str, Mapping)) or not _is_iterable(value) or not value:
|
||||
return False
|
||||
first = _first(value)
|
||||
if first is _UNDEFINED:
|
||||
raise ValueError(
|
||||
f"A tool_reference list must be a list, got {type(value).__name__}."
|
||||
)
|
||||
return _get(first, "type") == "tool_reference"
|
||||
|
||||
|
||||
def _render_output(entry: Any, tools: Any, multimodal: bool) -> str:
|
||||
output = _get(entry, "output")
|
||||
if _is_tool_reference_list(output):
|
||||
return _tool_references_to_response(output, tools)
|
||||
return _tool_response(_visible_text(output, multimodal))
|
||||
|
||||
|
||||
def _render_tool_response(message: Any, tools: Any, multimodal: bool) -> str:
|
||||
content = _get(message, "content", None)
|
||||
if isinstance(content, str):
|
||||
return _tool_response(content)
|
||||
if not isinstance(content, Mapping) and _is_tool_reference_list(content):
|
||||
return _tool_references_to_response(content, tools)
|
||||
if _is_list_of_outputs(message):
|
||||
return "".join(_render_output(entry, tools, multimodal) for entry in content)
|
||||
return _tool_response(_visible_text(content, multimodal))
|
||||
|
||||
|
||||
def _render_tool_block(
|
||||
messages: list[Any], start: int, end: int, tools: Any, multimodal: bool
|
||||
) -> str:
|
||||
"""Render the consecutive tool messages ``messages[start:end + 1]``.
|
||||
|
||||
Results are reordered to follow the driving assistant's ``tool_calls`` when
|
||||
every id is non-empty and unique on both sides and every result id names
|
||||
one of the calls; otherwise they render in message order.
|
||||
"""
|
||||
# One (message index, entry index or -1, id) per tool result. The template
|
||||
# collects these for the whole block before it looks at the tool calls, so
|
||||
# a malformed tool message fails here whether or not results get sorted.
|
||||
results = []
|
||||
for index in range(start, end + 1):
|
||||
message = messages[index]
|
||||
if _is_list_of_outputs(message):
|
||||
for entry_index, entry in enumerate(_get(message, "content")):
|
||||
results.append((index, entry_index, _id_of(entry)))
|
||||
else:
|
||||
results.append((index, -1, _id_of(message)))
|
||||
|
||||
assistant = messages[start - 1] if start > 0 else None
|
||||
tool_calls = None
|
||||
if assistant is not None and _get(assistant, "role") == "assistant":
|
||||
tool_calls = _get(assistant, "tool_calls", None)
|
||||
|
||||
can_sort = bool(tool_calls)
|
||||
if can_sort:
|
||||
call_ids = [_id_of(tool_call) for tool_call in tool_calls]
|
||||
result_ids = [result_id for _, _, result_id in results]
|
||||
can_sort = (
|
||||
"" not in result_ids
|
||||
and "" not in call_ids
|
||||
and len(set(result_ids)) == len(result_ids)
|
||||
and len(set(call_ids)) == len(call_ids)
|
||||
and all(result_id in call_ids for result_id in result_ids)
|
||||
)
|
||||
|
||||
if not can_sort:
|
||||
return "".join(
|
||||
_render_tool_response(messages[index], tools, multimodal)
|
||||
for index in range(start, end + 1)
|
||||
)
|
||||
|
||||
call_position = {call_id: position for position, call_id in enumerate(call_ids)}
|
||||
parts = []
|
||||
for index, entry_index, _ in sorted(
|
||||
results, key=lambda result: call_position[result[2]]
|
||||
):
|
||||
message = messages[index]
|
||||
if entry_index >= 0:
|
||||
entry = _get(message, "content")[entry_index]
|
||||
parts.append(_render_output(entry, tools, multimodal))
|
||||
else:
|
||||
parts.append(_render_tool_response(message, tools, multimodal))
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def _render_tool_call(tool_call: Any, multimodal: bool) -> str:
|
||||
# `{%- if tc.function %}{%- set tc = tc.function %}`: a truthiness test.
|
||||
function = _get(tool_call, "function", None)
|
||||
if function:
|
||||
tool_call = function
|
||||
name = _get(tool_call, "name")
|
||||
if multimodal:
|
||||
# GLM-5.3-Flash: `'<tool_call>' ~ tc.name` stringifies the name.
|
||||
parts = ["<tool_call>" + _string(name)]
|
||||
else:
|
||||
# GLM-5.3: `'<tool_call>' + tc.name` is Python's +, so a non-string
|
||||
# name raises the very same TypeError; do not coerce it.
|
||||
if name is _UNDEFINED:
|
||||
raise ValueError(
|
||||
"Each tool_call needs a `name`; the GLM-5.3 chat template "
|
||||
"reads `tc.name` without a guard."
|
||||
)
|
||||
parts = ["<tool_call>" + name]
|
||||
# The name is concatenated first, so it fails before bad arguments do.
|
||||
arguments = _get(tool_call, "arguments", None)
|
||||
if not isinstance(arguments, Mapping):
|
||||
raise ValueError(
|
||||
"tool_call `arguments` must be a JSON object, got "
|
||||
f"{type(arguments).__name__}."
|
||||
)
|
||||
for key, value in arguments.items():
|
||||
if not isinstance(value, str):
|
||||
value = json.dumps(value, ensure_ascii=False)
|
||||
parts.append(f"<arg_key>{key!s}</arg_key><arg_value>{value}</arg_value>")
|
||||
parts.append("</tool_call>")
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def _render(
|
||||
messages: list[Any],
|
||||
tools: Optional[list[Any]],
|
||||
add_generation_prompt: bool,
|
||||
reasoning_effort: Any,
|
||||
clear_thinking: Any,
|
||||
reasoning_content: Any,
|
||||
multimodal: bool,
|
||||
) -> str:
|
||||
effective_reasoning_effort = (
|
||||
reasoning_effort if reasoning_effort in ("low", "high") else "max"
|
||||
)
|
||||
parts = [
|
||||
"[gMASK]<sop>",
|
||||
f"<|system|>Reasoning Effort: {effective_reasoning_effort.capitalize()}",
|
||||
]
|
||||
|
||||
if tools:
|
||||
parts.append(_TOOLS_PREFIX)
|
||||
for tool in tools:
|
||||
tool = _unwrap_tool(tool)
|
||||
defer_loading = _get(tool, "defer_loading")
|
||||
if defer_loading is _UNDEFINED or not defer_loading:
|
||||
parts.append(_tool_to_json(tool) + "\n")
|
||||
parts.append(_TOOLS_SUFFIX)
|
||||
|
||||
last_user_index = -1
|
||||
for index, message in enumerate(messages):
|
||||
if _get(message, "role") == "user":
|
||||
last_user_index = index
|
||||
|
||||
index = 0
|
||||
while index < len(messages):
|
||||
message = messages[index]
|
||||
role = _get(message, "role")
|
||||
if role == "user":
|
||||
parts.append(
|
||||
"<|user|>" + _visible_text(_get(message, "content"), multimodal)
|
||||
)
|
||||
elif role == "assistant":
|
||||
parts.append("<|assistant|>")
|
||||
content = _visible_text(_get(message, "content"), multimodal)
|
||||
reasoning = _get(message, "reasoning_content")
|
||||
if not isinstance(reasoning, str):
|
||||
if "</think>" in content:
|
||||
reasoning = content.split("</think>")[0].split("<think>")[-1]
|
||||
content = content.split("</think>")[-1]
|
||||
else:
|
||||
# The template only assigns reasoning_content in the two
|
||||
# branches above, so otherwise the name resolves to the
|
||||
# template variable of that name, which is Undefined unless
|
||||
# the caller passed `reasoning_content` as a template kwarg.
|
||||
reasoning = reasoning_content
|
||||
if (not clear_thinking or index > last_user_index) and (
|
||||
reasoning is not _UNDEFINED
|
||||
):
|
||||
parts.append("<think>" + reasoning + "</think>")
|
||||
else:
|
||||
parts.append("<think></think>")
|
||||
if content.strip():
|
||||
parts.append(content.strip())
|
||||
tool_calls = _get(message, "tool_calls", None)
|
||||
if tool_calls:
|
||||
for tool_call in tool_calls:
|
||||
parts.append(_render_tool_call(tool_call, multimodal))
|
||||
elif role == "tool":
|
||||
if index == 0 or _get(messages[index - 1], "role") != "tool":
|
||||
end = index
|
||||
while (
|
||||
end + 1 < len(messages)
|
||||
and _get(messages[end + 1], "role") == "tool"
|
||||
):
|
||||
end += 1
|
||||
parts.append("<|observation|>")
|
||||
parts.append(
|
||||
_render_tool_block(messages, index, end, tools, multimodal)
|
||||
)
|
||||
index = end
|
||||
elif role == "system":
|
||||
parts.append(
|
||||
"<|system|>" + _visible_text(_get(message, "content"), multimodal)
|
||||
)
|
||||
index += 1
|
||||
|
||||
if add_generation_prompt:
|
||||
parts.append("<|assistant|><think>")
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def render_glm53(
|
||||
messages: list[Any],
|
||||
tools: Optional[list[Any]] = None,
|
||||
add_generation_prompt: bool = False,
|
||||
reasoning_effort: Any = None,
|
||||
clear_thinking: Any = False,
|
||||
reasoning_content: Any = _UNDEFINED,
|
||||
**_: Any,
|
||||
) -> str:
|
||||
"""Render a GLM-5.3 (text-only) chat prompt."""
|
||||
return _render(
|
||||
messages,
|
||||
tools,
|
||||
add_generation_prompt,
|
||||
reasoning_effort,
|
||||
clear_thinking,
|
||||
reasoning_content,
|
||||
multimodal=False,
|
||||
)
|
||||
|
||||
|
||||
def render_glm53_flash(
|
||||
messages: list[Any],
|
||||
tools: Optional[list[Any]] = None,
|
||||
add_generation_prompt: bool = False,
|
||||
reasoning_effort: Any = None,
|
||||
clear_thinking: Any = False,
|
||||
reasoning_content: Any = _UNDEFINED,
|
||||
**_: Any,
|
||||
) -> str:
|
||||
"""Render a GLM-5.3-Flash (multimodal) chat prompt."""
|
||||
return _render(
|
||||
messages,
|
||||
tools,
|
||||
add_generation_prompt,
|
||||
reasoning_effort,
|
||||
clear_thinking,
|
||||
reasoning_content,
|
||||
multimodal=True,
|
||||
)
|
||||
@@ -0,0 +1,224 @@
|
||||
"""Chat templates rendered in Python instead of the checkpoint's Jinja template.
|
||||
|
||||
When one applies to the served checkpoint, the serving tokenizer's
|
||||
``apply_chat_template`` renders through Python, so every caller keeps the
|
||||
transformers calling convention and the request pipeline in front of it is
|
||||
unchanged; the features sglang would otherwise derive from the Jinja source
|
||||
(content format, reasoning mode, tool-result reordering, inline system
|
||||
messages, the parsers ``auto`` resolves to) are declared on the template
|
||||
instead.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
|
||||
from transformers.utils import get_json_schema
|
||||
|
||||
from sglang.srt.parser.glm53_chat_template import render_glm53, render_glm53_flash
|
||||
from sglang.srt.parser.template_detection import (
|
||||
ReasoningToggleConfig,
|
||||
is_glm53_template,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PythonChatTemplate:
|
||||
"""A chat template implemented in Python, with the features sglang needs.
|
||||
|
||||
``render(messages, tools=None, add_generation_prompt=False, **kwargs)``
|
||||
receives the same variables a Jinja template does and returns the prompt.
|
||||
"""
|
||||
|
||||
name: str
|
||||
render: Callable[..., str]
|
||||
content_format: str
|
||||
reasoning_config: Optional[ReasoningToggleConfig]
|
||||
may_reorder_tool_results: bool
|
||||
supports_inline_system: bool
|
||||
reasoning_parser: Optional[str] = None
|
||||
tool_call_parser: Optional[str] = None
|
||||
|
||||
@property
|
||||
def force_reasoning(self) -> bool:
|
||||
return self.reasoning_config is not None and self.reasoning_config.always_on
|
||||
|
||||
|
||||
# Matching what template detection derives from both models' Jinja source.
|
||||
_GLM53_FEATURES: Dict[str, Any] = dict(
|
||||
content_format="openai",
|
||||
reasoning_config=ReasoningToggleConfig(special_case="always"),
|
||||
may_reorder_tool_results=True,
|
||||
supports_inline_system=True,
|
||||
reasoning_parser="glm45",
|
||||
tool_call_parser="glm47",
|
||||
)
|
||||
|
||||
GLM53 = PythonChatTemplate(name="glm-5.3", render=render_glm53, **_GLM53_FEATURES)
|
||||
GLM53_FLASH = PythonChatTemplate(
|
||||
name="glm-5.3-flash", render=render_glm53_flash, **_GLM53_FEATURES
|
||||
)
|
||||
|
||||
PYTHON_CHAT_TEMPLATES: Dict[str, PythonChatTemplate] = {
|
||||
template.name: template for template in (GLM53, GLM53_FLASH)
|
||||
}
|
||||
|
||||
_GLM53_TEMPLATES_BY_ARCH = {
|
||||
"GlmMoeDsaForCausalLM": GLM53,
|
||||
"Glm5NextForConditionalGeneration": GLM53_FLASH,
|
||||
}
|
||||
|
||||
|
||||
def get_python_chat_template(name: str) -> Optional[PythonChatTemplate]:
|
||||
"""Look up a built-in Python chat template by name (``--chat-template``)."""
|
||||
return PYTHON_CHAT_TEMPLATES.get(name)
|
||||
|
||||
|
||||
def may_replace_jinja_template(hf_chat_template: Optional[str]) -> bool:
|
||||
"""Whether a built-in Python template can replace this Jinja template.
|
||||
|
||||
Only the checkpoint's template is inspected, so callers can skip loading
|
||||
the model config for every other template.
|
||||
"""
|
||||
return isinstance(hf_chat_template, str) and is_glm53_template(hf_chat_template)
|
||||
|
||||
|
||||
def resolve_python_chat_template(
|
||||
hf_config: Any, hf_chat_template: Optional[str]
|
||||
) -> Optional[PythonChatTemplate]:
|
||||
"""The Python template that replaces this checkpoint's Jinja template.
|
||||
|
||||
GLM-5, GLM-5.1 and GLM-5.2 share ``GlmMoeDsaForCausalLM`` with GLM-5.3 but
|
||||
ship different templates, so the architecture alone does not decide: the
|
||||
checkpoint's template must be a GLM-5.3 one, and its multimodal block must
|
||||
match the architecture (GLM-5.3 is text-only, GLM-5.3-Flash multimodal).
|
||||
"""
|
||||
architectures = getattr(hf_config, "architectures", None) or []
|
||||
template = _GLM53_TEMPLATES_BY_ARCH.get(architectures[0] if architectures else "")
|
||||
if template is None or not may_replace_jinja_template(hf_chat_template):
|
||||
return None
|
||||
multimodal = "<|begin_of_image|>" in hf_chat_template
|
||||
if multimodal != (template is GLM53_FLASH):
|
||||
return None
|
||||
return template
|
||||
|
||||
|
||||
def _tool_schemas(tools):
|
||||
# Same conversion transformers applies before rendering a Jinja template.
|
||||
if tools is None:
|
||||
return None
|
||||
schemas = []
|
||||
for tool in tools:
|
||||
if isinstance(tool, dict):
|
||||
schemas.append(tool)
|
||||
elif inspect.isfunction(tool) or inspect.ismethod(tool):
|
||||
schemas.append(get_json_schema(tool))
|
||||
else:
|
||||
raise ValueError(
|
||||
"Tools should either be a JSON schema, or a callable function with "
|
||||
"type hints and a docstring suitable for auto-conversion to a schema."
|
||||
)
|
||||
return schemas
|
||||
|
||||
|
||||
def install_python_chat_template(tokenizer: Any, template: PythonChatTemplate) -> None:
|
||||
"""Make ``tokenizer.apply_chat_template`` render with ``template``.
|
||||
|
||||
Mirrors transformers' ``apply_chat_template``: tool schemas, special tokens
|
||||
as template variables, batched conversations, and tokenization without
|
||||
added special tokens. A caller-supplied ``chat_template`` and the Jinja-only
|
||||
``continue_final_message`` / ``return_assistant_tokens_mask`` modes still go
|
||||
to the original Jinja implementation.
|
||||
"""
|
||||
jinja_apply_chat_template = getattr(
|
||||
tokenizer, "_sglang_jinja_apply_chat_template", tokenizer.apply_chat_template
|
||||
)
|
||||
|
||||
def apply_chat_template(
|
||||
conversation,
|
||||
tools=None,
|
||||
documents=None,
|
||||
chat_template=None,
|
||||
add_generation_prompt=False,
|
||||
continue_final_message=False,
|
||||
tokenize=True,
|
||||
padding=False,
|
||||
truncation=False,
|
||||
max_length=None,
|
||||
return_tensors=None,
|
||||
return_dict=True,
|
||||
return_assistant_tokens_mask=False,
|
||||
tokenizer_kwargs=None,
|
||||
**kwargs,
|
||||
):
|
||||
if (
|
||||
chat_template is not None
|
||||
or continue_final_message
|
||||
or return_assistant_tokens_mask
|
||||
):
|
||||
return jinja_apply_chat_template(
|
||||
conversation,
|
||||
tools=tools,
|
||||
documents=documents,
|
||||
chat_template=chat_template,
|
||||
add_generation_prompt=add_generation_prompt,
|
||||
continue_final_message=continue_final_message,
|
||||
tokenize=tokenize,
|
||||
padding=padding,
|
||||
truncation=truncation,
|
||||
max_length=max_length,
|
||||
return_tensors=return_tensors,
|
||||
return_dict=return_dict,
|
||||
return_assistant_tokens_mask=return_assistant_tokens_mask,
|
||||
tokenizer_kwargs=tokenizer_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
if isinstance(conversation, (list, tuple)) and len(conversation) == 0:
|
||||
raise ValueError(
|
||||
"Cannot apply chat template to an empty conversation. Provide at "
|
||||
"least one message."
|
||||
)
|
||||
is_batched = isinstance(conversation, (list, tuple)) and (
|
||||
isinstance(conversation[0], (list, tuple))
|
||||
or hasattr(conversation[0], "messages")
|
||||
)
|
||||
conversations = conversation if is_batched else [conversation]
|
||||
tool_schemas = _tool_schemas(tools)
|
||||
if documents is not None:
|
||||
for document in documents:
|
||||
if not isinstance(document, dict):
|
||||
raise TypeError(
|
||||
"Documents should be a list of dicts with 'title' and "
|
||||
"'text' keys!"
|
||||
)
|
||||
|
||||
# As for Jinja, kwargs override special tokens of the same name.
|
||||
template_kwargs = {**tokenizer.special_tokens_map, **kwargs}
|
||||
rendered = [
|
||||
template.render(
|
||||
messages=getattr(chat, "messages", chat),
|
||||
tools=tool_schemas,
|
||||
documents=documents,
|
||||
add_generation_prompt=add_generation_prompt,
|
||||
**template_kwargs,
|
||||
)
|
||||
for chat in conversations
|
||||
]
|
||||
rendered_chat = rendered if is_batched else rendered[0]
|
||||
if not tokenize:
|
||||
return rendered_chat
|
||||
|
||||
out = tokenizer(
|
||||
rendered_chat,
|
||||
padding=padding,
|
||||
truncation=truncation,
|
||||
max_length=max_length,
|
||||
add_special_tokens=False,
|
||||
return_tensors=return_tensors,
|
||||
**(tokenizer_kwargs or {}),
|
||||
)
|
||||
return out if return_dict else out["input_ids"]
|
||||
|
||||
tokenizer._sglang_jinja_apply_chat_template = jinja_apply_chat_template
|
||||
tokenizer.apply_chat_template = apply_chat_template
|
||||
@@ -718,6 +718,17 @@ def detect_tool_call_parser(
|
||||
return match_rules(ctx, TOOL_CALL_PARSER_RULES, "tool-call parser")
|
||||
|
||||
|
||||
def is_glm53_template(template: Optional[str]) -> bool:
|
||||
"""True if ``template`` is a GLM-5.3 family chat template."""
|
||||
if not template:
|
||||
return False
|
||||
return _is_glm53(
|
||||
TemplateDetectionContext(
|
||||
template=template, reasoning_config=None, force_reasoning=False, vocab=set()
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def detect_inline_system_support(chat_template: Optional[str]) -> bool:
|
||||
"""True if mid-conversation ``role: "system"`` renders inline; False if the
|
||||
template raises or silently drops it (then merge into the leading block).
|
||||
@@ -784,6 +795,48 @@ def _log_undetected_parser(attr: str, label: str) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _declare_python_template_parsers(server_args, template, needs) -> None:
|
||||
"""Resolve the ``auto`` parsers to the ones a Python chat template declares."""
|
||||
detected = {attr: getattr(template, attr) for attr in needs}
|
||||
for attr, value in detected.items():
|
||||
logger.info(
|
||||
f"Auto-detected --{attr.replace('_', '-')} as '{value}' from "
|
||||
f"Python chat template '{template.name}'"
|
||||
)
|
||||
declare_resolution(server_args, "template-detection", **detected)
|
||||
|
||||
|
||||
def _checkpoint_python_chat_template(server_args, template):
|
||||
"""The Python chat template replacing the checkpoint's ``template``, if any.
|
||||
|
||||
Mirrors TemplateManager's selection when ``--chat-template`` is not set.
|
||||
"""
|
||||
from sglang.srt.parser.python_chat_template import (
|
||||
may_replace_jinja_template,
|
||||
resolve_python_chat_template,
|
||||
)
|
||||
|
||||
if not may_replace_jinja_template(template):
|
||||
return None
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
try:
|
||||
from sglang.srt.utils.hf_transformers_utils import get_config
|
||||
|
||||
config = get_config(
|
||||
cfg.model_path,
|
||||
trust_remote_code=cfg.trust_remote_code,
|
||||
revision=getattr(cfg, "revision", None),
|
||||
model_config_parser=getattr(cfg, "model_config_parser", "auto"),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to load model config to resolve a Python chat template: %s", e
|
||||
)
|
||||
return None
|
||||
return resolve_python_chat_template(config, template)
|
||||
|
||||
|
||||
def _architecture_auto_parsers(server_args, needs: Tuple[str, ...]) -> Dict[str, str]:
|
||||
"""The parsers the model architecture implies, for the fields still on auto."""
|
||||
from sglang.srt.entrypoints.openai.chat_encoding import is_deepseek_v41_arch
|
||||
@@ -904,9 +957,19 @@ def resolve_auto_parsers(server_args) -> None:
|
||||
if not needs:
|
||||
return
|
||||
|
||||
chat_template_arg = getattr(cfg, "chat_template", None)
|
||||
if isinstance(chat_template_arg, str):
|
||||
from sglang.srt.parser.python_chat_template import get_python_chat_template
|
||||
|
||||
# A built-in Python chat template declares its parsers, so neither the
|
||||
# tokenizer nor a Jinja source needs loading.
|
||||
python_template = get_python_chat_template(chat_template_arg)
|
||||
if python_template is not None:
|
||||
_declare_python_template_parsers(server_args, python_template, needs)
|
||||
return
|
||||
|
||||
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
||||
|
||||
chat_template_arg = getattr(cfg, "chat_template", None)
|
||||
try:
|
||||
explicit_jinja_template = _load_explicit_jinja_template(chat_template_arg)
|
||||
except Exception as e:
|
||||
@@ -929,6 +992,14 @@ def resolve_auto_parsers(server_args) -> None:
|
||||
if template is None and tokenizer is not None:
|
||||
template = getattr(tokenizer, "chat_template", None)
|
||||
|
||||
# A Python chat template replacing the checkpoint's renders this model's
|
||||
# prompts, so its declared parsers decide; the Jinja source is not consulted.
|
||||
if chat_template_arg is None:
|
||||
python_template = _checkpoint_python_chat_template(server_args, template)
|
||||
if python_template is not None:
|
||||
_declare_python_template_parsers(server_args, python_template, needs)
|
||||
return
|
||||
|
||||
force_reasoning, reasoning_config = detect_reasoning_pattern(template)
|
||||
ctx = build_detection_context(
|
||||
template, tokenizer, reasoning_config, force_reasoning
|
||||
|
||||
@@ -42,6 +42,12 @@ from sglang.srt.parser.jinja_template_utils import (
|
||||
detect_jinja_template_content_format,
|
||||
jinja_template_may_reorder_tool_results,
|
||||
)
|
||||
from sglang.srt.parser.python_chat_template import (
|
||||
PythonChatTemplate,
|
||||
get_python_chat_template,
|
||||
install_python_chat_template,
|
||||
resolve_python_chat_template,
|
||||
)
|
||||
from sglang.srt.parser.template_detection import (
|
||||
REASONING_PARSER_RULES,
|
||||
TOOL_CALL_PARSER_RULES,
|
||||
@@ -73,12 +79,18 @@ class TemplateManager:
|
||||
self._suggested_reasoning_parser: Optional[str] = None
|
||||
self._suggested_tool_call_parser: Optional[str] = None
|
||||
self._jinja_template_may_reorder_tool_results: bool = False
|
||||
self._python_chat_template: Optional[PythonChatTemplate] = None
|
||||
|
||||
@property
|
||||
def chat_template_name(self) -> Optional[str]:
|
||||
"""Get the current chat template name."""
|
||||
return self._chat_template_name
|
||||
|
||||
@property
|
||||
def python_chat_template(self) -> Optional[PythonChatTemplate]:
|
||||
"""The Python chat template rendering prompts in place of Jinja, if any."""
|
||||
return self._python_chat_template
|
||||
|
||||
@property
|
||||
def completion_template_name(self) -> Optional[str]:
|
||||
"""Get the current completion template name."""
|
||||
@@ -140,6 +152,16 @@ class TemplateManager:
|
||||
ctx, TOOL_CALL_PARSER_RULES, "tool-call parser"
|
||||
)
|
||||
|
||||
def _use_python_template_features(self, template: PythonChatTemplate) -> None:
|
||||
"""Take the features detection derives from a Jinja source from ``template``."""
|
||||
self._jinja_template_may_reorder_tool_results = (
|
||||
template.may_reorder_tool_results
|
||||
)
|
||||
self._force_reasoning = template.force_reasoning
|
||||
self._reasoning_config = template.reasoning_config
|
||||
self._suggested_reasoning_parser = template.reasoning_parser
|
||||
self._suggested_tool_call_parser = template.tool_call_parser
|
||||
|
||||
def load_chat_template(
|
||||
self,
|
||||
tokenizer_manager: TokenizerManager,
|
||||
@@ -164,10 +186,15 @@ class TemplateManager:
|
||||
if self._chat_template_name is None:
|
||||
# Try HuggingFace template first
|
||||
hf_template = self._resolve_hf_chat_template(tokenizer_manager)
|
||||
if hf_template:
|
||||
if hf_template and tokenizer_manager.tokenizer:
|
||||
# override the chat template
|
||||
if tokenizer_manager.tokenizer:
|
||||
tokenizer_manager.tokenizer.chat_template = hf_template
|
||||
tokenizer_manager.tokenizer.chat_template = hf_template
|
||||
python_template = resolve_python_chat_template(
|
||||
tokenizer_manager.model_config.hf_config, hf_template
|
||||
)
|
||||
if python_template is not None:
|
||||
self._load_python_chat_template(tokenizer_manager, python_template)
|
||||
elif hf_template:
|
||||
self._jinja_template_content_format = (
|
||||
detect_jinja_template_content_format(hf_template)
|
||||
)
|
||||
@@ -183,8 +210,11 @@ class TemplateManager:
|
||||
|
||||
# Detect reasoning pattern and suggest parser from chat template
|
||||
if tokenizer_manager.tokenizer:
|
||||
template = tokenizer_manager.tokenizer.chat_template
|
||||
self._run_template_detection(template, tokenizer_manager.tokenizer)
|
||||
if self._python_chat_template is not None:
|
||||
self._use_python_template_features(self._python_chat_template)
|
||||
else:
|
||||
template = tokenizer_manager.tokenizer.chat_template
|
||||
self._run_template_detection(template, tokenizer_manager.tokenizer)
|
||||
parts = []
|
||||
if self._reasoning_config:
|
||||
parts.append(f"reasoning_config={self._reasoning_config}")
|
||||
@@ -205,6 +235,11 @@ class TemplateManager:
|
||||
self._chat_template_name = chat_template_arg
|
||||
return
|
||||
|
||||
python_template = get_python_chat_template(chat_template_arg)
|
||||
if python_template is not None:
|
||||
self._load_python_chat_template(tokenizer_manager, python_template)
|
||||
return
|
||||
|
||||
if not os.path.exists(chat_template_arg):
|
||||
raise RuntimeError(
|
||||
f"Chat template {chat_template_arg} is not a built-in template name "
|
||||
@@ -273,6 +308,20 @@ class TemplateManager:
|
||||
if completion_template:
|
||||
self.load_completion_template(completion_template)
|
||||
|
||||
def _load_python_chat_template(
|
||||
self, tokenizer_manager: TokenizerManager, template: PythonChatTemplate
|
||||
) -> None:
|
||||
"""Render chat prompts with a built-in Python chat template."""
|
||||
if tokenizer_manager.tokenizer is not None:
|
||||
install_python_chat_template(tokenizer_manager.tokenizer, template)
|
||||
self._python_chat_template = template
|
||||
self._chat_template_name = None
|
||||
self._jinja_template_content_format = template.content_format
|
||||
logger.info(
|
||||
f"Using Python chat template '{template.name}' in place of the Jinja "
|
||||
f"chat template, content format: {template.content_format}"
|
||||
)
|
||||
|
||||
def _load_jinja_template(
|
||||
self, tokenizer_manager: TokenizerManager, template_path: str
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user