[Feat] glm-5.3: Render GLM-5.3 and GLM-5.3-Flash chat prompts with a Python chat template (#45154)

Signed-off-by: Shijin Zhang <75300765+Dovis01@users.noreply.github.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
Shijin Zhang
2026-10-20 19:11:40 +08:00
committed by GitHub
co-authored by Xinyuan Tong
parent 800e60e345
commit ca473ef3e4
8 changed files with 1438 additions and 6 deletions
@@ -467,6 +467,9 @@ class OpenAIServingChat(OpenAIServingBase):
return chat_encoding.spec_supports_inline_system(self.chat_encoding_spec)
if self.template_manager.chat_template_name is not None:
return False
python_template = self.template_manager.python_chat_template
if python_template is not None:
return python_template.supports_inline_system
tokenizer = self.tokenizer_manager.tokenizer
return tokenizer is not None and detect_inline_system_support(
tokenizer.chat_template
@@ -0,0 +1,438 @@
# Adapted from the GLM-5.3 / GLM-5.3-Flash reference chat_template.py by Z.ai.
"""Python version of the GLM-5.3 and GLM-5.3-Flash chat templates.
The two templates are the same apart from the multimodal block, so a single
renderer serves both. The output is byte-identical to the Jinja templates.
Where a template fails on malformed input, the renderer fails at the same
point, raising ``ValueError`` for Jinja's ``UndefinedError`` and the very
``TypeError`` Jinja would raise, so the serving layer answers both with 400
exactly as it did for the template. Jinja semantics that are easy to get wrong
when reading the template source are called out inline; ``_UNDEFINED`` stands
for Jinja's ``Undefined``.
"""
from __future__ import annotations
import json
from collections.abc import Mapping
from typing import Any, Optional
_UNDEFINED = object()
_TOOLS_PREFIX = (
"<|system|>\n"
"# Tools\n\n"
"You may call one or more functions to assist with the user query.\n\n"
"You are provided with function signatures within <tools></tools> XML tags:\n"
"<tools>\n"
)
_TOOLS_SUFFIX = (
"</tools>\n\n"
"For each function call, output the function name and arguments within the "
"following XML format:\n"
"<tool_call>{function-name}<arg_key>{arg-key-1}</arg_key>"
"<arg_value>{arg-value-1}</arg_value><arg_key>{arg-key-2}</arg_key>"
"<arg_value>{arg-value-2}</arg_value>...</tool_call>"
)
# Tuples rather than sets: Jinja's `in` compares with ==, so an unhashable
# part type must not raise here.
_IMAGE_PART_TYPES = ("image", "image_url")
_VIDEO_PART_TYPES = ("video", "video_url")
_AUDIO_PART_TYPES = ("audio", "audio_url", "input_audio")
_MEDIA_PART_TYPES = _IMAGE_PART_TYPES + _VIDEO_PART_TYPES + _AUDIO_PART_TYPES
_MEDIA_REMINDER = (
"<reminder>You are unable to process this {} because you don't have "
"multi-modal input ability. Try different methods.</reminder>"
)
_IMAGE_TOKENS = "<|begin_of_image|><|image|><|end_of_image|>"
_VIDEO_TOKENS = "<|begin_of_video|><|video|><|end_of_video|>"
_AUDIO_TOKENS = "<|begin_of_audio|><|end_of_audio|>"
def _get(obj: Any, key: str, default: Any = _UNDEFINED) -> Any:
"""``obj.key`` in Jinja: an item of a mapping, an attribute otherwise."""
if isinstance(obj, Mapping):
return obj[key] if key in obj else default
return getattr(obj, key, default)
def _first(value: Any) -> Any:
"""``value.0`` in Jinja, which is Undefined when there is no item 0."""
try:
return value[0]
except (AttributeError, TypeError, LookupError):
return _UNDEFINED
def _string(value: Any) -> str:
"""``{{ value }}`` in Jinja: Undefined renders empty, None renders "None"."""
return "" if value is _UNDEFINED else str(value)
def _is_iterable(value: Any) -> bool:
try:
iter(value)
except TypeError:
return False
return True
def _render_media(part_type: str, multimodal: bool) -> str:
if not multimodal:
media_type = part_type.replace("_url", "").replace("input_", "")
return _MEDIA_REMINDER.format(media_type)
if part_type in _IMAGE_PART_TYPES:
return _IMAGE_TOKENS
if part_type in _VIDEO_PART_TYPES:
return _VIDEO_TOKENS
return _AUDIO_TOKENS
def _visible_text(content: Any, multimodal: bool) -> str:
if isinstance(content, str):
return content
if _is_iterable(content) and not isinstance(content, Mapping):
parts = []
for item in content:
if isinstance(item, Mapping):
part_type = _get(item, "type")
if part_type == "text":
parts.append(_string(_get(item, "text")))
elif part_type in _MEDIA_PART_TYPES:
parts.append(_render_media(part_type, multimodal))
elif isinstance(item, str):
parts.append(item)
return "".join(parts)
# The template guards this branch with `content is not none`, so None
# renders nothing instead of "None".
return "" if content is None else _string(content)
def _unwrap_tool(tool: Any) -> Any:
# `'function' in tool` is a membership test, so a present but empty
# `function` still replaces the tool.
if isinstance(tool, Mapping) and "function" in tool:
return tool["function"]
return tool
def _tool_to_json(tool: Any) -> str:
# The tool_to_json macro calls `tool.items()`, which fails on a non-mapping.
if not isinstance(tool, Mapping):
raise ValueError(f"Each tool must be a JSON object, got {type(tool).__name__}.")
# The macro writes `"{{ k }}": {{ v | tojson }}` by hand: values go through
# tojson (json.dumps under transformers), but keys are interpolated raw and
# are not JSON-escaped, so json.dumps on the whole dict would diverge.
fields = [
f'"{key!s}": {json.dumps(value, ensure_ascii=False)}'
for key, value in tool.items()
if key != "defer_loading" and key != "strict"
]
return "{" + ", ".join(fields) + "}"
def _tool_references_to_response(refs: Any, tools: Any) -> str:
# The template defines this macro under `{%- if tools -%}`, so rendering a
# tool_reference without tools called an undefined macro.
if not tools:
raise ValueError(
"Rendering a tool_reference requires `tools`; the GLM-5.3 chat "
"template only defines its tool macros when tools are given."
)
parts = ["<tool_response><tools>\n"]
for ref in refs:
ref_name = _get(ref, "name")
for tool in tools:
tool = _unwrap_tool(tool)
# Undefined equals Undefined in Jinja, so a reference without a
# name matches every tool without one, as it does here.
if _get(tool, "name") == ref_name:
parts.append(_tool_to_json(tool) + "\n")
parts.append("</tools></tool_response>")
return "".join(parts)
def _tool_response(text: str) -> str:
return "<tool_response>" + text + "</tool_response>"
def _id_of(obj: Any) -> str:
tool_call_id = _get(obj, "tool_call_id", None)
if tool_call_id:
return str(tool_call_id)
obj_id = _get(obj, "id", None)
if obj_id:
return str(obj_id)
return ""
def _is_list_of_outputs(message: Any) -> bool:
# `m.content and m.content.0.output is defined`: falsy content
# short-circuits, while reading `.output` from a missing item 0 (a mapping
# or a scalar content) raises UndefinedError.
content = _get(message, "content", None)
if not content:
return False
first = _first(content)
if first is _UNDEFINED:
raise ValueError(
"A tool message `content` must be a string or a list, got "
f"{type(content).__name__}."
)
return _get(first, "output") is not _UNDEFINED
def _is_tool_reference_list(value: Any) -> bool:
# `v is iterable and v is not string and v is not mapping and v and
# v.0.type == "tool_reference"`; reading `.type` from a missing item 0
# raises UndefinedError like `_is_list_of_outputs`.
if isinstance(value, (str, Mapping)) or not _is_iterable(value) or not value:
return False
first = _first(value)
if first is _UNDEFINED:
raise ValueError(
f"A tool_reference list must be a list, got {type(value).__name__}."
)
return _get(first, "type") == "tool_reference"
def _render_output(entry: Any, tools: Any, multimodal: bool) -> str:
output = _get(entry, "output")
if _is_tool_reference_list(output):
return _tool_references_to_response(output, tools)
return _tool_response(_visible_text(output, multimodal))
def _render_tool_response(message: Any, tools: Any, multimodal: bool) -> str:
content = _get(message, "content", None)
if isinstance(content, str):
return _tool_response(content)
if not isinstance(content, Mapping) and _is_tool_reference_list(content):
return _tool_references_to_response(content, tools)
if _is_list_of_outputs(message):
return "".join(_render_output(entry, tools, multimodal) for entry in content)
return _tool_response(_visible_text(content, multimodal))
def _render_tool_block(
messages: list[Any], start: int, end: int, tools: Any, multimodal: bool
) -> str:
"""Render the consecutive tool messages ``messages[start:end + 1]``.
Results are reordered to follow the driving assistant's ``tool_calls`` when
every id is non-empty and unique on both sides and every result id names
one of the calls; otherwise they render in message order.
"""
# One (message index, entry index or -1, id) per tool result. The template
# collects these for the whole block before it looks at the tool calls, so
# a malformed tool message fails here whether or not results get sorted.
results = []
for index in range(start, end + 1):
message = messages[index]
if _is_list_of_outputs(message):
for entry_index, entry in enumerate(_get(message, "content")):
results.append((index, entry_index, _id_of(entry)))
else:
results.append((index, -1, _id_of(message)))
assistant = messages[start - 1] if start > 0 else None
tool_calls = None
if assistant is not None and _get(assistant, "role") == "assistant":
tool_calls = _get(assistant, "tool_calls", None)
can_sort = bool(tool_calls)
if can_sort:
call_ids = [_id_of(tool_call) for tool_call in tool_calls]
result_ids = [result_id for _, _, result_id in results]
can_sort = (
"" not in result_ids
and "" not in call_ids
and len(set(result_ids)) == len(result_ids)
and len(set(call_ids)) == len(call_ids)
and all(result_id in call_ids for result_id in result_ids)
)
if not can_sort:
return "".join(
_render_tool_response(messages[index], tools, multimodal)
for index in range(start, end + 1)
)
call_position = {call_id: position for position, call_id in enumerate(call_ids)}
parts = []
for index, entry_index, _ in sorted(
results, key=lambda result: call_position[result[2]]
):
message = messages[index]
if entry_index >= 0:
entry = _get(message, "content")[entry_index]
parts.append(_render_output(entry, tools, multimodal))
else:
parts.append(_render_tool_response(message, tools, multimodal))
return "".join(parts)
def _render_tool_call(tool_call: Any, multimodal: bool) -> str:
# `{%- if tc.function %}{%- set tc = tc.function %}`: a truthiness test.
function = _get(tool_call, "function", None)
if function:
tool_call = function
name = _get(tool_call, "name")
if multimodal:
# GLM-5.3-Flash: `'<tool_call>' ~ tc.name` stringifies the name.
parts = ["<tool_call>" + _string(name)]
else:
# GLM-5.3: `'<tool_call>' + tc.name` is Python's +, so a non-string
# name raises the very same TypeError; do not coerce it.
if name is _UNDEFINED:
raise ValueError(
"Each tool_call needs a `name`; the GLM-5.3 chat template "
"reads `tc.name` without a guard."
)
parts = ["<tool_call>" + name]
# The name is concatenated first, so it fails before bad arguments do.
arguments = _get(tool_call, "arguments", None)
if not isinstance(arguments, Mapping):
raise ValueError(
"tool_call `arguments` must be a JSON object, got "
f"{type(arguments).__name__}."
)
for key, value in arguments.items():
if not isinstance(value, str):
value = json.dumps(value, ensure_ascii=False)
parts.append(f"<arg_key>{key!s}</arg_key><arg_value>{value}</arg_value>")
parts.append("</tool_call>")
return "".join(parts)
def _render(
messages: list[Any],
tools: Optional[list[Any]],
add_generation_prompt: bool,
reasoning_effort: Any,
clear_thinking: Any,
reasoning_content: Any,
multimodal: bool,
) -> str:
effective_reasoning_effort = (
reasoning_effort if reasoning_effort in ("low", "high") else "max"
)
parts = [
"[gMASK]<sop>",
f"<|system|>Reasoning Effort: {effective_reasoning_effort.capitalize()}",
]
if tools:
parts.append(_TOOLS_PREFIX)
for tool in tools:
tool = _unwrap_tool(tool)
defer_loading = _get(tool, "defer_loading")
if defer_loading is _UNDEFINED or not defer_loading:
parts.append(_tool_to_json(tool) + "\n")
parts.append(_TOOLS_SUFFIX)
last_user_index = -1
for index, message in enumerate(messages):
if _get(message, "role") == "user":
last_user_index = index
index = 0
while index < len(messages):
message = messages[index]
role = _get(message, "role")
if role == "user":
parts.append(
"<|user|>" + _visible_text(_get(message, "content"), multimodal)
)
elif role == "assistant":
parts.append("<|assistant|>")
content = _visible_text(_get(message, "content"), multimodal)
reasoning = _get(message, "reasoning_content")
if not isinstance(reasoning, str):
if "</think>" in content:
reasoning = content.split("</think>")[0].split("<think>")[-1]
content = content.split("</think>")[-1]
else:
# The template only assigns reasoning_content in the two
# branches above, so otherwise the name resolves to the
# template variable of that name, which is Undefined unless
# the caller passed `reasoning_content` as a template kwarg.
reasoning = reasoning_content
if (not clear_thinking or index > last_user_index) and (
reasoning is not _UNDEFINED
):
parts.append("<think>" + reasoning + "</think>")
else:
parts.append("<think></think>")
if content.strip():
parts.append(content.strip())
tool_calls = _get(message, "tool_calls", None)
if tool_calls:
for tool_call in tool_calls:
parts.append(_render_tool_call(tool_call, multimodal))
elif role == "tool":
if index == 0 or _get(messages[index - 1], "role") != "tool":
end = index
while (
end + 1 < len(messages)
and _get(messages[end + 1], "role") == "tool"
):
end += 1
parts.append("<|observation|>")
parts.append(
_render_tool_block(messages, index, end, tools, multimodal)
)
index = end
elif role == "system":
parts.append(
"<|system|>" + _visible_text(_get(message, "content"), multimodal)
)
index += 1
if add_generation_prompt:
parts.append("<|assistant|><think>")
return "".join(parts)
def render_glm53(
messages: list[Any],
tools: Optional[list[Any]] = None,
add_generation_prompt: bool = False,
reasoning_effort: Any = None,
clear_thinking: Any = False,
reasoning_content: Any = _UNDEFINED,
**_: Any,
) -> str:
"""Render a GLM-5.3 (text-only) chat prompt."""
return _render(
messages,
tools,
add_generation_prompt,
reasoning_effort,
clear_thinking,
reasoning_content,
multimodal=False,
)
def render_glm53_flash(
messages: list[Any],
tools: Optional[list[Any]] = None,
add_generation_prompt: bool = False,
reasoning_effort: Any = None,
clear_thinking: Any = False,
reasoning_content: Any = _UNDEFINED,
**_: Any,
) -> str:
"""Render a GLM-5.3-Flash (multimodal) chat prompt."""
return _render(
messages,
tools,
add_generation_prompt,
reasoning_effort,
clear_thinking,
reasoning_content,
multimodal=True,
)
@@ -0,0 +1,224 @@
"""Chat templates rendered in Python instead of the checkpoint's Jinja template.
When one applies to the served checkpoint, the serving tokenizer's
``apply_chat_template`` renders through Python, so every caller keeps the
transformers calling convention and the request pipeline in front of it is
unchanged; the features sglang would otherwise derive from the Jinja source
(content format, reasoning mode, tool-result reordering, inline system
messages, the parsers ``auto`` resolves to) are declared on the template
instead.
"""
import inspect
from dataclasses import dataclass
from typing import Any, Callable, Dict, Optional
from transformers.utils import get_json_schema
from sglang.srt.parser.glm53_chat_template import render_glm53, render_glm53_flash
from sglang.srt.parser.template_detection import (
ReasoningToggleConfig,
is_glm53_template,
)
@dataclass(frozen=True)
class PythonChatTemplate:
"""A chat template implemented in Python, with the features sglang needs.
``render(messages, tools=None, add_generation_prompt=False, **kwargs)``
receives the same variables a Jinja template does and returns the prompt.
"""
name: str
render: Callable[..., str]
content_format: str
reasoning_config: Optional[ReasoningToggleConfig]
may_reorder_tool_results: bool
supports_inline_system: bool
reasoning_parser: Optional[str] = None
tool_call_parser: Optional[str] = None
@property
def force_reasoning(self) -> bool:
return self.reasoning_config is not None and self.reasoning_config.always_on
# Matching what template detection derives from both models' Jinja source.
_GLM53_FEATURES: Dict[str, Any] = dict(
content_format="openai",
reasoning_config=ReasoningToggleConfig(special_case="always"),
may_reorder_tool_results=True,
supports_inline_system=True,
reasoning_parser="glm45",
tool_call_parser="glm47",
)
GLM53 = PythonChatTemplate(name="glm-5.3", render=render_glm53, **_GLM53_FEATURES)
GLM53_FLASH = PythonChatTemplate(
name="glm-5.3-flash", render=render_glm53_flash, **_GLM53_FEATURES
)
PYTHON_CHAT_TEMPLATES: Dict[str, PythonChatTemplate] = {
template.name: template for template in (GLM53, GLM53_FLASH)
}
_GLM53_TEMPLATES_BY_ARCH = {
"GlmMoeDsaForCausalLM": GLM53,
"Glm5NextForConditionalGeneration": GLM53_FLASH,
}
def get_python_chat_template(name: str) -> Optional[PythonChatTemplate]:
"""Look up a built-in Python chat template by name (``--chat-template``)."""
return PYTHON_CHAT_TEMPLATES.get(name)
def may_replace_jinja_template(hf_chat_template: Optional[str]) -> bool:
"""Whether a built-in Python template can replace this Jinja template.
Only the checkpoint's template is inspected, so callers can skip loading
the model config for every other template.
"""
return isinstance(hf_chat_template, str) and is_glm53_template(hf_chat_template)
def resolve_python_chat_template(
hf_config: Any, hf_chat_template: Optional[str]
) -> Optional[PythonChatTemplate]:
"""The Python template that replaces this checkpoint's Jinja template.
GLM-5, GLM-5.1 and GLM-5.2 share ``GlmMoeDsaForCausalLM`` with GLM-5.3 but
ship different templates, so the architecture alone does not decide: the
checkpoint's template must be a GLM-5.3 one, and its multimodal block must
match the architecture (GLM-5.3 is text-only, GLM-5.3-Flash multimodal).
"""
architectures = getattr(hf_config, "architectures", None) or []
template = _GLM53_TEMPLATES_BY_ARCH.get(architectures[0] if architectures else "")
if template is None or not may_replace_jinja_template(hf_chat_template):
return None
multimodal = "<|begin_of_image|>" in hf_chat_template
if multimodal != (template is GLM53_FLASH):
return None
return template
def _tool_schemas(tools):
# Same conversion transformers applies before rendering a Jinja template.
if tools is None:
return None
schemas = []
for tool in tools:
if isinstance(tool, dict):
schemas.append(tool)
elif inspect.isfunction(tool) or inspect.ismethod(tool):
schemas.append(get_json_schema(tool))
else:
raise ValueError(
"Tools should either be a JSON schema, or a callable function with "
"type hints and a docstring suitable for auto-conversion to a schema."
)
return schemas
def install_python_chat_template(tokenizer: Any, template: PythonChatTemplate) -> None:
"""Make ``tokenizer.apply_chat_template`` render with ``template``.
Mirrors transformers' ``apply_chat_template``: tool schemas, special tokens
as template variables, batched conversations, and tokenization without
added special tokens. A caller-supplied ``chat_template`` and the Jinja-only
``continue_final_message`` / ``return_assistant_tokens_mask`` modes still go
to the original Jinja implementation.
"""
jinja_apply_chat_template = getattr(
tokenizer, "_sglang_jinja_apply_chat_template", tokenizer.apply_chat_template
)
def apply_chat_template(
conversation,
tools=None,
documents=None,
chat_template=None,
add_generation_prompt=False,
continue_final_message=False,
tokenize=True,
padding=False,
truncation=False,
max_length=None,
return_tensors=None,
return_dict=True,
return_assistant_tokens_mask=False,
tokenizer_kwargs=None,
**kwargs,
):
if (
chat_template is not None
or continue_final_message
or return_assistant_tokens_mask
):
return jinja_apply_chat_template(
conversation,
tools=tools,
documents=documents,
chat_template=chat_template,
add_generation_prompt=add_generation_prompt,
continue_final_message=continue_final_message,
tokenize=tokenize,
padding=padding,
truncation=truncation,
max_length=max_length,
return_tensors=return_tensors,
return_dict=return_dict,
return_assistant_tokens_mask=return_assistant_tokens_mask,
tokenizer_kwargs=tokenizer_kwargs,
**kwargs,
)
if isinstance(conversation, (list, tuple)) and len(conversation) == 0:
raise ValueError(
"Cannot apply chat template to an empty conversation. Provide at "
"least one message."
)
is_batched = isinstance(conversation, (list, tuple)) and (
isinstance(conversation[0], (list, tuple))
or hasattr(conversation[0], "messages")
)
conversations = conversation if is_batched else [conversation]
tool_schemas = _tool_schemas(tools)
if documents is not None:
for document in documents:
if not isinstance(document, dict):
raise TypeError(
"Documents should be a list of dicts with 'title' and "
"'text' keys!"
)
# As for Jinja, kwargs override special tokens of the same name.
template_kwargs = {**tokenizer.special_tokens_map, **kwargs}
rendered = [
template.render(
messages=getattr(chat, "messages", chat),
tools=tool_schemas,
documents=documents,
add_generation_prompt=add_generation_prompt,
**template_kwargs,
)
for chat in conversations
]
rendered_chat = rendered if is_batched else rendered[0]
if not tokenize:
return rendered_chat
out = tokenizer(
rendered_chat,
padding=padding,
truncation=truncation,
max_length=max_length,
add_special_tokens=False,
return_tensors=return_tensors,
**(tokenizer_kwargs or {}),
)
return out if return_dict else out["input_ids"]
tokenizer._sglang_jinja_apply_chat_template = jinja_apply_chat_template
tokenizer.apply_chat_template = apply_chat_template
+72 -1
View File
@@ -718,6 +718,17 @@ def detect_tool_call_parser(
return match_rules(ctx, TOOL_CALL_PARSER_RULES, "tool-call parser")
def is_glm53_template(template: Optional[str]) -> bool:
"""True if ``template`` is a GLM-5.3 family chat template."""
if not template:
return False
return _is_glm53(
TemplateDetectionContext(
template=template, reasoning_config=None, force_reasoning=False, vocab=set()
)
)
def detect_inline_system_support(chat_template: Optional[str]) -> bool:
"""True if mid-conversation ``role: "system"`` renders inline; False if the
template raises or silently drops it (then merge into the leading block).
@@ -784,6 +795,48 @@ def _log_undetected_parser(attr: str, label: str) -> None:
)
def _declare_python_template_parsers(server_args, template, needs) -> None:
"""Resolve the ``auto`` parsers to the ones a Python chat template declares."""
detected = {attr: getattr(template, attr) for attr in needs}
for attr, value in detected.items():
logger.info(
f"Auto-detected --{attr.replace('_', '-')} as '{value}' from "
f"Python chat template '{template.name}'"
)
declare_resolution(server_args, "template-detection", **detected)
def _checkpoint_python_chat_template(server_args, template):
"""The Python chat template replacing the checkpoint's ``template``, if any.
Mirrors TemplateManager's selection when ``--chat-template`` is not set.
"""
from sglang.srt.parser.python_chat_template import (
may_replace_jinja_template,
resolve_python_chat_template,
)
if not may_replace_jinja_template(template):
return None
cfg = resolving_view(server_args)
try:
from sglang.srt.utils.hf_transformers_utils import get_config
config = get_config(
cfg.model_path,
trust_remote_code=cfg.trust_remote_code,
revision=getattr(cfg, "revision", None),
model_config_parser=getattr(cfg, "model_config_parser", "auto"),
)
except Exception as e:
logger.warning(
"Failed to load model config to resolve a Python chat template: %s", e
)
return None
return resolve_python_chat_template(config, template)
def _architecture_auto_parsers(server_args, needs: Tuple[str, ...]) -> Dict[str, str]:
"""The parsers the model architecture implies, for the fields still on auto."""
from sglang.srt.entrypoints.openai.chat_encoding import is_deepseek_v41_arch
@@ -904,9 +957,19 @@ def resolve_auto_parsers(server_args) -> None:
if not needs:
return
chat_template_arg = getattr(cfg, "chat_template", None)
if isinstance(chat_template_arg, str):
from sglang.srt.parser.python_chat_template import get_python_chat_template
# A built-in Python chat template declares its parsers, so neither the
# tokenizer nor a Jinja source needs loading.
python_template = get_python_chat_template(chat_template_arg)
if python_template is not None:
_declare_python_template_parsers(server_args, python_template, needs)
return
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
chat_template_arg = getattr(cfg, "chat_template", None)
try:
explicit_jinja_template = _load_explicit_jinja_template(chat_template_arg)
except Exception as e:
@@ -929,6 +992,14 @@ def resolve_auto_parsers(server_args) -> None:
if template is None and tokenizer is not None:
template = getattr(tokenizer, "chat_template", None)
# A Python chat template replacing the checkpoint's renders this model's
# prompts, so its declared parsers decide; the Jinja source is not consulted.
if chat_template_arg is None:
python_template = _checkpoint_python_chat_template(server_args, template)
if python_template is not None:
_declare_python_template_parsers(server_args, python_template, needs)
return
force_reasoning, reasoning_config = detect_reasoning_pattern(template)
ctx = build_detection_context(
template, tokenizer, reasoning_config, force_reasoning
+54 -5
View File
@@ -42,6 +42,12 @@ from sglang.srt.parser.jinja_template_utils import (
detect_jinja_template_content_format,
jinja_template_may_reorder_tool_results,
)
from sglang.srt.parser.python_chat_template import (
PythonChatTemplate,
get_python_chat_template,
install_python_chat_template,
resolve_python_chat_template,
)
from sglang.srt.parser.template_detection import (
REASONING_PARSER_RULES,
TOOL_CALL_PARSER_RULES,
@@ -73,12 +79,18 @@ class TemplateManager:
self._suggested_reasoning_parser: Optional[str] = None
self._suggested_tool_call_parser: Optional[str] = None
self._jinja_template_may_reorder_tool_results: bool = False
self._python_chat_template: Optional[PythonChatTemplate] = None
@property
def chat_template_name(self) -> Optional[str]:
"""Get the current chat template name."""
return self._chat_template_name
@property
def python_chat_template(self) -> Optional[PythonChatTemplate]:
"""The Python chat template rendering prompts in place of Jinja, if any."""
return self._python_chat_template
@property
def completion_template_name(self) -> Optional[str]:
"""Get the current completion template name."""
@@ -140,6 +152,16 @@ class TemplateManager:
ctx, TOOL_CALL_PARSER_RULES, "tool-call parser"
)
def _use_python_template_features(self, template: PythonChatTemplate) -> None:
"""Take the features detection derives from a Jinja source from ``template``."""
self._jinja_template_may_reorder_tool_results = (
template.may_reorder_tool_results
)
self._force_reasoning = template.force_reasoning
self._reasoning_config = template.reasoning_config
self._suggested_reasoning_parser = template.reasoning_parser
self._suggested_tool_call_parser = template.tool_call_parser
def load_chat_template(
self,
tokenizer_manager: TokenizerManager,
@@ -164,10 +186,15 @@ class TemplateManager:
if self._chat_template_name is None:
# Try HuggingFace template first
hf_template = self._resolve_hf_chat_template(tokenizer_manager)
if hf_template:
if hf_template and tokenizer_manager.tokenizer:
# override the chat template
if tokenizer_manager.tokenizer:
tokenizer_manager.tokenizer.chat_template = hf_template
tokenizer_manager.tokenizer.chat_template = hf_template
python_template = resolve_python_chat_template(
tokenizer_manager.model_config.hf_config, hf_template
)
if python_template is not None:
self._load_python_chat_template(tokenizer_manager, python_template)
elif hf_template:
self._jinja_template_content_format = (
detect_jinja_template_content_format(hf_template)
)
@@ -183,8 +210,11 @@ class TemplateManager:
# Detect reasoning pattern and suggest parser from chat template
if tokenizer_manager.tokenizer:
template = tokenizer_manager.tokenizer.chat_template
self._run_template_detection(template, tokenizer_manager.tokenizer)
if self._python_chat_template is not None:
self._use_python_template_features(self._python_chat_template)
else:
template = tokenizer_manager.tokenizer.chat_template
self._run_template_detection(template, tokenizer_manager.tokenizer)
parts = []
if self._reasoning_config:
parts.append(f"reasoning_config={self._reasoning_config}")
@@ -205,6 +235,11 @@ class TemplateManager:
self._chat_template_name = chat_template_arg
return
python_template = get_python_chat_template(chat_template_arg)
if python_template is not None:
self._load_python_chat_template(tokenizer_manager, python_template)
return
if not os.path.exists(chat_template_arg):
raise RuntimeError(
f"Chat template {chat_template_arg} is not a built-in template name "
@@ -273,6 +308,20 @@ class TemplateManager:
if completion_template:
self.load_completion_template(completion_template)
def _load_python_chat_template(
self, tokenizer_manager: TokenizerManager, template: PythonChatTemplate
) -> None:
"""Render chat prompts with a built-in Python chat template."""
if tokenizer_manager.tokenizer is not None:
install_python_chat_template(tokenizer_manager.tokenizer, template)
self._python_chat_template = template
self._chat_template_name = None
self._jinja_template_content_format = template.content_format
logger.info(
f"Using Python chat template '{template.name}' in place of the Jinja "
f"chat template, content format: {template.content_format}"
)
def _load_jinja_template(
self, tokenizer_manager: TokenizerManager, template_path: str
) -> None: