diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index f195b837dd..c183edab23 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -467,6 +467,9 @@ class OpenAIServingChat(OpenAIServingBase): return chat_encoding.spec_supports_inline_system(self.chat_encoding_spec) if self.template_manager.chat_template_name is not None: return False + python_template = self.template_manager.python_chat_template + if python_template is not None: + return python_template.supports_inline_system tokenizer = self.tokenizer_manager.tokenizer return tokenizer is not None and detect_inline_system_support( tokenizer.chat_template diff --git a/python/sglang/srt/parser/glm53_chat_template.py b/python/sglang/srt/parser/glm53_chat_template.py new file mode 100644 index 0000000000..de1c335ba6 --- /dev/null +++ b/python/sglang/srt/parser/glm53_chat_template.py @@ -0,0 +1,438 @@ +# Adapted from the GLM-5.3 / GLM-5.3-Flash reference chat_template.py by Z.ai. +"""Python version of the GLM-5.3 and GLM-5.3-Flash chat templates. + +The two templates are the same apart from the multimodal block, so a single +renderer serves both. The output is byte-identical to the Jinja templates. +Where a template fails on malformed input, the renderer fails at the same +point, raising ``ValueError`` for Jinja's ``UndefinedError`` and the very +``TypeError`` Jinja would raise, so the serving layer answers both with 400 +exactly as it did for the template. Jinja semantics that are easy to get wrong +when reading the template source are called out inline; ``_UNDEFINED`` stands +for Jinja's ``Undefined``. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from typing import Any, Optional + +_UNDEFINED = object() + +_TOOLS_PREFIX = ( + "<|system|>\n" + "# Tools\n\n" + "You may call one or more functions to assist with the user query.\n\n" + "You are provided with function signatures within XML tags:\n" + "\n" +) +_TOOLS_SUFFIX = ( + "\n\n" + "For each function call, output the function name and arguments within the " + "following XML format:\n" + "{function-name}{arg-key-1}" + "{arg-value-1}{arg-key-2}" + "{arg-value-2}..." +) + +# Tuples rather than sets: Jinja's `in` compares with ==, so an unhashable +# part type must not raise here. +_IMAGE_PART_TYPES = ("image", "image_url") +_VIDEO_PART_TYPES = ("video", "video_url") +_AUDIO_PART_TYPES = ("audio", "audio_url", "input_audio") +_MEDIA_PART_TYPES = _IMAGE_PART_TYPES + _VIDEO_PART_TYPES + _AUDIO_PART_TYPES + +_MEDIA_REMINDER = ( + "You are unable to process this {} because you don't have " + "multi-modal input ability. Try different methods." +) +_IMAGE_TOKENS = "<|begin_of_image|><|image|><|end_of_image|>" +_VIDEO_TOKENS = "<|begin_of_video|><|video|><|end_of_video|>" +_AUDIO_TOKENS = "<|begin_of_audio|><|end_of_audio|>" + + +def _get(obj: Any, key: str, default: Any = _UNDEFINED) -> Any: + """``obj.key`` in Jinja: an item of a mapping, an attribute otherwise.""" + if isinstance(obj, Mapping): + return obj[key] if key in obj else default + return getattr(obj, key, default) + + +def _first(value: Any) -> Any: + """``value.0`` in Jinja, which is Undefined when there is no item 0.""" + try: + return value[0] + except (AttributeError, TypeError, LookupError): + return _UNDEFINED + + +def _string(value: Any) -> str: + """``{{ value }}`` in Jinja: Undefined renders empty, None renders "None".""" + return "" if value is _UNDEFINED else str(value) + + +def _is_iterable(value: Any) -> bool: + try: + iter(value) + except TypeError: + return False + return True + + +def _render_media(part_type: str, multimodal: bool) -> str: + if not multimodal: + media_type = part_type.replace("_url", "").replace("input_", "") + return _MEDIA_REMINDER.format(media_type) + if part_type in _IMAGE_PART_TYPES: + return _IMAGE_TOKENS + if part_type in _VIDEO_PART_TYPES: + return _VIDEO_TOKENS + return _AUDIO_TOKENS + + +def _visible_text(content: Any, multimodal: bool) -> str: + if isinstance(content, str): + return content + if _is_iterable(content) and not isinstance(content, Mapping): + parts = [] + for item in content: + if isinstance(item, Mapping): + part_type = _get(item, "type") + if part_type == "text": + parts.append(_string(_get(item, "text"))) + elif part_type in _MEDIA_PART_TYPES: + parts.append(_render_media(part_type, multimodal)) + elif isinstance(item, str): + parts.append(item) + return "".join(parts) + # The template guards this branch with `content is not none`, so None + # renders nothing instead of "None". + return "" if content is None else _string(content) + + +def _unwrap_tool(tool: Any) -> Any: + # `'function' in tool` is a membership test, so a present but empty + # `function` still replaces the tool. + if isinstance(tool, Mapping) and "function" in tool: + return tool["function"] + return tool + + +def _tool_to_json(tool: Any) -> str: + # The tool_to_json macro calls `tool.items()`, which fails on a non-mapping. + if not isinstance(tool, Mapping): + raise ValueError(f"Each tool must be a JSON object, got {type(tool).__name__}.") + # The macro writes `"{{ k }}": {{ v | tojson }}` by hand: values go through + # tojson (json.dumps under transformers), but keys are interpolated raw and + # are not JSON-escaped, so json.dumps on the whole dict would diverge. + fields = [ + f'"{key!s}": {json.dumps(value, ensure_ascii=False)}' + for key, value in tool.items() + if key != "defer_loading" and key != "strict" + ] + return "{" + ", ".join(fields) + "}" + + +def _tool_references_to_response(refs: Any, tools: Any) -> str: + # The template defines this macro under `{%- if tools -%}`, so rendering a + # tool_reference without tools called an undefined macro. + if not tools: + raise ValueError( + "Rendering a tool_reference requires `tools`; the GLM-5.3 chat " + "template only defines its tool macros when tools are given." + ) + parts = ["\n"] + for ref in refs: + ref_name = _get(ref, "name") + for tool in tools: + tool = _unwrap_tool(tool) + # Undefined equals Undefined in Jinja, so a reference without a + # name matches every tool without one, as it does here. + if _get(tool, "name") == ref_name: + parts.append(_tool_to_json(tool) + "\n") + parts.append("") + return "".join(parts) + + +def _tool_response(text: str) -> str: + return "" + text + "" + + +def _id_of(obj: Any) -> str: + tool_call_id = _get(obj, "tool_call_id", None) + if tool_call_id: + return str(tool_call_id) + obj_id = _get(obj, "id", None) + if obj_id: + return str(obj_id) + return "" + + +def _is_list_of_outputs(message: Any) -> bool: + # `m.content and m.content.0.output is defined`: falsy content + # short-circuits, while reading `.output` from a missing item 0 (a mapping + # or a scalar content) raises UndefinedError. + content = _get(message, "content", None) + if not content: + return False + first = _first(content) + if first is _UNDEFINED: + raise ValueError( + "A tool message `content` must be a string or a list, got " + f"{type(content).__name__}." + ) + return _get(first, "output") is not _UNDEFINED + + +def _is_tool_reference_list(value: Any) -> bool: + # `v is iterable and v is not string and v is not mapping and v and + # v.0.type == "tool_reference"`; reading `.type` from a missing item 0 + # raises UndefinedError like `_is_list_of_outputs`. + if isinstance(value, (str, Mapping)) or not _is_iterable(value) or not value: + return False + first = _first(value) + if first is _UNDEFINED: + raise ValueError( + f"A tool_reference list must be a list, got {type(value).__name__}." + ) + return _get(first, "type") == "tool_reference" + + +def _render_output(entry: Any, tools: Any, multimodal: bool) -> str: + output = _get(entry, "output") + if _is_tool_reference_list(output): + return _tool_references_to_response(output, tools) + return _tool_response(_visible_text(output, multimodal)) + + +def _render_tool_response(message: Any, tools: Any, multimodal: bool) -> str: + content = _get(message, "content", None) + if isinstance(content, str): + return _tool_response(content) + if not isinstance(content, Mapping) and _is_tool_reference_list(content): + return _tool_references_to_response(content, tools) + if _is_list_of_outputs(message): + return "".join(_render_output(entry, tools, multimodal) for entry in content) + return _tool_response(_visible_text(content, multimodal)) + + +def _render_tool_block( + messages: list[Any], start: int, end: int, tools: Any, multimodal: bool +) -> str: + """Render the consecutive tool messages ``messages[start:end + 1]``. + + Results are reordered to follow the driving assistant's ``tool_calls`` when + every id is non-empty and unique on both sides and every result id names + one of the calls; otherwise they render in message order. + """ + # One (message index, entry index or -1, id) per tool result. The template + # collects these for the whole block before it looks at the tool calls, so + # a malformed tool message fails here whether or not results get sorted. + results = [] + for index in range(start, end + 1): + message = messages[index] + if _is_list_of_outputs(message): + for entry_index, entry in enumerate(_get(message, "content")): + results.append((index, entry_index, _id_of(entry))) + else: + results.append((index, -1, _id_of(message))) + + assistant = messages[start - 1] if start > 0 else None + tool_calls = None + if assistant is not None and _get(assistant, "role") == "assistant": + tool_calls = _get(assistant, "tool_calls", None) + + can_sort = bool(tool_calls) + if can_sort: + call_ids = [_id_of(tool_call) for tool_call in tool_calls] + result_ids = [result_id for _, _, result_id in results] + can_sort = ( + "" not in result_ids + and "" not in call_ids + and len(set(result_ids)) == len(result_ids) + and len(set(call_ids)) == len(call_ids) + and all(result_id in call_ids for result_id in result_ids) + ) + + if not can_sort: + return "".join( + _render_tool_response(messages[index], tools, multimodal) + for index in range(start, end + 1) + ) + + call_position = {call_id: position for position, call_id in enumerate(call_ids)} + parts = [] + for index, entry_index, _ in sorted( + results, key=lambda result: call_position[result[2]] + ): + message = messages[index] + if entry_index >= 0: + entry = _get(message, "content")[entry_index] + parts.append(_render_output(entry, tools, multimodal)) + else: + parts.append(_render_tool_response(message, tools, multimodal)) + return "".join(parts) + + +def _render_tool_call(tool_call: Any, multimodal: bool) -> str: + # `{%- if tc.function %}{%- set tc = tc.function %}`: a truthiness test. + function = _get(tool_call, "function", None) + if function: + tool_call = function + name = _get(tool_call, "name") + if multimodal: + # GLM-5.3-Flash: `'' ~ tc.name` stringifies the name. + parts = ["" + _string(name)] + else: + # GLM-5.3: `'' + tc.name` is Python's +, so a non-string + # name raises the very same TypeError; do not coerce it. + if name is _UNDEFINED: + raise ValueError( + "Each tool_call needs a `name`; the GLM-5.3 chat template " + "reads `tc.name` without a guard." + ) + parts = ["" + name] + # The name is concatenated first, so it fails before bad arguments do. + arguments = _get(tool_call, "arguments", None) + if not isinstance(arguments, Mapping): + raise ValueError( + "tool_call `arguments` must be a JSON object, got " + f"{type(arguments).__name__}." + ) + for key, value in arguments.items(): + if not isinstance(value, str): + value = json.dumps(value, ensure_ascii=False) + parts.append(f"{key!s}{value}") + parts.append("") + return "".join(parts) + + +def _render( + messages: list[Any], + tools: Optional[list[Any]], + add_generation_prompt: bool, + reasoning_effort: Any, + clear_thinking: Any, + reasoning_content: Any, + multimodal: bool, +) -> str: + effective_reasoning_effort = ( + reasoning_effort if reasoning_effort in ("low", "high") else "max" + ) + parts = [ + "[gMASK]", + f"<|system|>Reasoning Effort: {effective_reasoning_effort.capitalize()}", + ] + + if tools: + parts.append(_TOOLS_PREFIX) + for tool in tools: + tool = _unwrap_tool(tool) + defer_loading = _get(tool, "defer_loading") + if defer_loading is _UNDEFINED or not defer_loading: + parts.append(_tool_to_json(tool) + "\n") + parts.append(_TOOLS_SUFFIX) + + last_user_index = -1 + for index, message in enumerate(messages): + if _get(message, "role") == "user": + last_user_index = index + + index = 0 + while index < len(messages): + message = messages[index] + role = _get(message, "role") + if role == "user": + parts.append( + "<|user|>" + _visible_text(_get(message, "content"), multimodal) + ) + elif role == "assistant": + parts.append("<|assistant|>") + content = _visible_text(_get(message, "content"), multimodal) + reasoning = _get(message, "reasoning_content") + if not isinstance(reasoning, str): + if "" in content: + reasoning = content.split("")[0].split("")[-1] + content = content.split("")[-1] + else: + # The template only assigns reasoning_content in the two + # branches above, so otherwise the name resolves to the + # template variable of that name, which is Undefined unless + # the caller passed `reasoning_content` as a template kwarg. + reasoning = reasoning_content + if (not clear_thinking or index > last_user_index) and ( + reasoning is not _UNDEFINED + ): + parts.append("" + reasoning + "") + else: + parts.append("") + if content.strip(): + parts.append(content.strip()) + tool_calls = _get(message, "tool_calls", None) + if tool_calls: + for tool_call in tool_calls: + parts.append(_render_tool_call(tool_call, multimodal)) + elif role == "tool": + if index == 0 or _get(messages[index - 1], "role") != "tool": + end = index + while ( + end + 1 < len(messages) + and _get(messages[end + 1], "role") == "tool" + ): + end += 1 + parts.append("<|observation|>") + parts.append( + _render_tool_block(messages, index, end, tools, multimodal) + ) + index = end + elif role == "system": + parts.append( + "<|system|>" + _visible_text(_get(message, "content"), multimodal) + ) + index += 1 + + if add_generation_prompt: + parts.append("<|assistant|>") + return "".join(parts) + + +def render_glm53( + messages: list[Any], + tools: Optional[list[Any]] = None, + add_generation_prompt: bool = False, + reasoning_effort: Any = None, + clear_thinking: Any = False, + reasoning_content: Any = _UNDEFINED, + **_: Any, +) -> str: + """Render a GLM-5.3 (text-only) chat prompt.""" + return _render( + messages, + tools, + add_generation_prompt, + reasoning_effort, + clear_thinking, + reasoning_content, + multimodal=False, + ) + + +def render_glm53_flash( + messages: list[Any], + tools: Optional[list[Any]] = None, + add_generation_prompt: bool = False, + reasoning_effort: Any = None, + clear_thinking: Any = False, + reasoning_content: Any = _UNDEFINED, + **_: Any, +) -> str: + """Render a GLM-5.3-Flash (multimodal) chat prompt.""" + return _render( + messages, + tools, + add_generation_prompt, + reasoning_effort, + clear_thinking, + reasoning_content, + multimodal=True, + ) diff --git a/python/sglang/srt/parser/python_chat_template.py b/python/sglang/srt/parser/python_chat_template.py new file mode 100644 index 0000000000..adc9609f99 --- /dev/null +++ b/python/sglang/srt/parser/python_chat_template.py @@ -0,0 +1,224 @@ +"""Chat templates rendered in Python instead of the checkpoint's Jinja template. + +When one applies to the served checkpoint, the serving tokenizer's +``apply_chat_template`` renders through Python, so every caller keeps the +transformers calling convention and the request pipeline in front of it is +unchanged; the features sglang would otherwise derive from the Jinja source +(content format, reasoning mode, tool-result reordering, inline system +messages, the parsers ``auto`` resolves to) are declared on the template +instead. +""" + +import inspect +from dataclasses import dataclass +from typing import Any, Callable, Dict, Optional + +from transformers.utils import get_json_schema + +from sglang.srt.parser.glm53_chat_template import render_glm53, render_glm53_flash +from sglang.srt.parser.template_detection import ( + ReasoningToggleConfig, + is_glm53_template, +) + + +@dataclass(frozen=True) +class PythonChatTemplate: + """A chat template implemented in Python, with the features sglang needs. + + ``render(messages, tools=None, add_generation_prompt=False, **kwargs)`` + receives the same variables a Jinja template does and returns the prompt. + """ + + name: str + render: Callable[..., str] + content_format: str + reasoning_config: Optional[ReasoningToggleConfig] + may_reorder_tool_results: bool + supports_inline_system: bool + reasoning_parser: Optional[str] = None + tool_call_parser: Optional[str] = None + + @property + def force_reasoning(self) -> bool: + return self.reasoning_config is not None and self.reasoning_config.always_on + + +# Matching what template detection derives from both models' Jinja source. +_GLM53_FEATURES: Dict[str, Any] = dict( + content_format="openai", + reasoning_config=ReasoningToggleConfig(special_case="always"), + may_reorder_tool_results=True, + supports_inline_system=True, + reasoning_parser="glm45", + tool_call_parser="glm47", +) + +GLM53 = PythonChatTemplate(name="glm-5.3", render=render_glm53, **_GLM53_FEATURES) +GLM53_FLASH = PythonChatTemplate( + name="glm-5.3-flash", render=render_glm53_flash, **_GLM53_FEATURES +) + +PYTHON_CHAT_TEMPLATES: Dict[str, PythonChatTemplate] = { + template.name: template for template in (GLM53, GLM53_FLASH) +} + +_GLM53_TEMPLATES_BY_ARCH = { + "GlmMoeDsaForCausalLM": GLM53, + "Glm5NextForConditionalGeneration": GLM53_FLASH, +} + + +def get_python_chat_template(name: str) -> Optional[PythonChatTemplate]: + """Look up a built-in Python chat template by name (``--chat-template``).""" + return PYTHON_CHAT_TEMPLATES.get(name) + + +def may_replace_jinja_template(hf_chat_template: Optional[str]) -> bool: + """Whether a built-in Python template can replace this Jinja template. + + Only the checkpoint's template is inspected, so callers can skip loading + the model config for every other template. + """ + return isinstance(hf_chat_template, str) and is_glm53_template(hf_chat_template) + + +def resolve_python_chat_template( + hf_config: Any, hf_chat_template: Optional[str] +) -> Optional[PythonChatTemplate]: + """The Python template that replaces this checkpoint's Jinja template. + + GLM-5, GLM-5.1 and GLM-5.2 share ``GlmMoeDsaForCausalLM`` with GLM-5.3 but + ship different templates, so the architecture alone does not decide: the + checkpoint's template must be a GLM-5.3 one, and its multimodal block must + match the architecture (GLM-5.3 is text-only, GLM-5.3-Flash multimodal). + """ + architectures = getattr(hf_config, "architectures", None) or [] + template = _GLM53_TEMPLATES_BY_ARCH.get(architectures[0] if architectures else "") + if template is None or not may_replace_jinja_template(hf_chat_template): + return None + multimodal = "<|begin_of_image|>" in hf_chat_template + if multimodal != (template is GLM53_FLASH): + return None + return template + + +def _tool_schemas(tools): + # Same conversion transformers applies before rendering a Jinja template. + if tools is None: + return None + schemas = [] + for tool in tools: + if isinstance(tool, dict): + schemas.append(tool) + elif inspect.isfunction(tool) or inspect.ismethod(tool): + schemas.append(get_json_schema(tool)) + else: + raise ValueError( + "Tools should either be a JSON schema, or a callable function with " + "type hints and a docstring suitable for auto-conversion to a schema." + ) + return schemas + + +def install_python_chat_template(tokenizer: Any, template: PythonChatTemplate) -> None: + """Make ``tokenizer.apply_chat_template`` render with ``template``. + + Mirrors transformers' ``apply_chat_template``: tool schemas, special tokens + as template variables, batched conversations, and tokenization without + added special tokens. A caller-supplied ``chat_template`` and the Jinja-only + ``continue_final_message`` / ``return_assistant_tokens_mask`` modes still go + to the original Jinja implementation. + """ + jinja_apply_chat_template = getattr( + tokenizer, "_sglang_jinja_apply_chat_template", tokenizer.apply_chat_template + ) + + def apply_chat_template( + conversation, + tools=None, + documents=None, + chat_template=None, + add_generation_prompt=False, + continue_final_message=False, + tokenize=True, + padding=False, + truncation=False, + max_length=None, + return_tensors=None, + return_dict=True, + return_assistant_tokens_mask=False, + tokenizer_kwargs=None, + **kwargs, + ): + if ( + chat_template is not None + or continue_final_message + or return_assistant_tokens_mask + ): + return jinja_apply_chat_template( + conversation, + tools=tools, + documents=documents, + chat_template=chat_template, + add_generation_prompt=add_generation_prompt, + continue_final_message=continue_final_message, + tokenize=tokenize, + padding=padding, + truncation=truncation, + max_length=max_length, + return_tensors=return_tensors, + return_dict=return_dict, + return_assistant_tokens_mask=return_assistant_tokens_mask, + tokenizer_kwargs=tokenizer_kwargs, + **kwargs, + ) + + if isinstance(conversation, (list, tuple)) and len(conversation) == 0: + raise ValueError( + "Cannot apply chat template to an empty conversation. Provide at " + "least one message." + ) + is_batched = isinstance(conversation, (list, tuple)) and ( + isinstance(conversation[0], (list, tuple)) + or hasattr(conversation[0], "messages") + ) + conversations = conversation if is_batched else [conversation] + tool_schemas = _tool_schemas(tools) + if documents is not None: + for document in documents: + if not isinstance(document, dict): + raise TypeError( + "Documents should be a list of dicts with 'title' and " + "'text' keys!" + ) + + # As for Jinja, kwargs override special tokens of the same name. + template_kwargs = {**tokenizer.special_tokens_map, **kwargs} + rendered = [ + template.render( + messages=getattr(chat, "messages", chat), + tools=tool_schemas, + documents=documents, + add_generation_prompt=add_generation_prompt, + **template_kwargs, + ) + for chat in conversations + ] + rendered_chat = rendered if is_batched else rendered[0] + if not tokenize: + return rendered_chat + + out = tokenizer( + rendered_chat, + padding=padding, + truncation=truncation, + max_length=max_length, + add_special_tokens=False, + return_tensors=return_tensors, + **(tokenizer_kwargs or {}), + ) + return out if return_dict else out["input_ids"] + + tokenizer._sglang_jinja_apply_chat_template = jinja_apply_chat_template + tokenizer.apply_chat_template = apply_chat_template diff --git a/python/sglang/srt/parser/template_detection.py b/python/sglang/srt/parser/template_detection.py index e2cb5b2b03..42384d97ed 100644 --- a/python/sglang/srt/parser/template_detection.py +++ b/python/sglang/srt/parser/template_detection.py @@ -718,6 +718,17 @@ def detect_tool_call_parser( return match_rules(ctx, TOOL_CALL_PARSER_RULES, "tool-call parser") +def is_glm53_template(template: Optional[str]) -> bool: + """True if ``template`` is a GLM-5.3 family chat template.""" + if not template: + return False + return _is_glm53( + TemplateDetectionContext( + template=template, reasoning_config=None, force_reasoning=False, vocab=set() + ) + ) + + def detect_inline_system_support(chat_template: Optional[str]) -> bool: """True if mid-conversation ``role: "system"`` renders inline; False if the template raises or silently drops it (then merge into the leading block). @@ -784,6 +795,48 @@ def _log_undetected_parser(attr: str, label: str) -> None: ) +def _declare_python_template_parsers(server_args, template, needs) -> None: + """Resolve the ``auto`` parsers to the ones a Python chat template declares.""" + detected = {attr: getattr(template, attr) for attr in needs} + for attr, value in detected.items(): + logger.info( + f"Auto-detected --{attr.replace('_', '-')} as '{value}' from " + f"Python chat template '{template.name}'" + ) + declare_resolution(server_args, "template-detection", **detected) + + +def _checkpoint_python_chat_template(server_args, template): + """The Python chat template replacing the checkpoint's ``template``, if any. + + Mirrors TemplateManager's selection when ``--chat-template`` is not set. + """ + from sglang.srt.parser.python_chat_template import ( + may_replace_jinja_template, + resolve_python_chat_template, + ) + + if not may_replace_jinja_template(template): + return None + + cfg = resolving_view(server_args) + try: + from sglang.srt.utils.hf_transformers_utils import get_config + + config = get_config( + cfg.model_path, + trust_remote_code=cfg.trust_remote_code, + revision=getattr(cfg, "revision", None), + model_config_parser=getattr(cfg, "model_config_parser", "auto"), + ) + except Exception as e: + logger.warning( + "Failed to load model config to resolve a Python chat template: %s", e + ) + return None + return resolve_python_chat_template(config, template) + + def _architecture_auto_parsers(server_args, needs: Tuple[str, ...]) -> Dict[str, str]: """The parsers the model architecture implies, for the fields still on auto.""" from sglang.srt.entrypoints.openai.chat_encoding import is_deepseek_v41_arch @@ -904,9 +957,19 @@ def resolve_auto_parsers(server_args) -> None: if not needs: return + chat_template_arg = getattr(cfg, "chat_template", None) + if isinstance(chat_template_arg, str): + from sglang.srt.parser.python_chat_template import get_python_chat_template + + # A built-in Python chat template declares its parsers, so neither the + # tokenizer nor a Jinja source needs loading. + python_template = get_python_chat_template(chat_template_arg) + if python_template is not None: + _declare_python_template_parsers(server_args, python_template, needs) + return + from sglang.srt.utils.hf_transformers_utils import get_tokenizer - chat_template_arg = getattr(cfg, "chat_template", None) try: explicit_jinja_template = _load_explicit_jinja_template(chat_template_arg) except Exception as e: @@ -929,6 +992,14 @@ def resolve_auto_parsers(server_args) -> None: if template is None and tokenizer is not None: template = getattr(tokenizer, "chat_template", None) + # A Python chat template replacing the checkpoint's renders this model's + # prompts, so its declared parsers decide; the Jinja source is not consulted. + if chat_template_arg is None: + python_template = _checkpoint_python_chat_template(server_args, template) + if python_template is not None: + _declare_python_template_parsers(server_args, python_template, needs) + return + force_reasoning, reasoning_config = detect_reasoning_pattern(template) ctx = build_detection_context( template, tokenizer, reasoning_config, force_reasoning diff --git a/python/sglang/srt/parser/template_manager.py b/python/sglang/srt/parser/template_manager.py index b033ff5ccd..70eedbb2d2 100644 --- a/python/sglang/srt/parser/template_manager.py +++ b/python/sglang/srt/parser/template_manager.py @@ -42,6 +42,12 @@ from sglang.srt.parser.jinja_template_utils import ( detect_jinja_template_content_format, jinja_template_may_reorder_tool_results, ) +from sglang.srt.parser.python_chat_template import ( + PythonChatTemplate, + get_python_chat_template, + install_python_chat_template, + resolve_python_chat_template, +) from sglang.srt.parser.template_detection import ( REASONING_PARSER_RULES, TOOL_CALL_PARSER_RULES, @@ -73,12 +79,18 @@ class TemplateManager: self._suggested_reasoning_parser: Optional[str] = None self._suggested_tool_call_parser: Optional[str] = None self._jinja_template_may_reorder_tool_results: bool = False + self._python_chat_template: Optional[PythonChatTemplate] = None @property def chat_template_name(self) -> Optional[str]: """Get the current chat template name.""" return self._chat_template_name + @property + def python_chat_template(self) -> Optional[PythonChatTemplate]: + """The Python chat template rendering prompts in place of Jinja, if any.""" + return self._python_chat_template + @property def completion_template_name(self) -> Optional[str]: """Get the current completion template name.""" @@ -140,6 +152,16 @@ class TemplateManager: ctx, TOOL_CALL_PARSER_RULES, "tool-call parser" ) + def _use_python_template_features(self, template: PythonChatTemplate) -> None: + """Take the features detection derives from a Jinja source from ``template``.""" + self._jinja_template_may_reorder_tool_results = ( + template.may_reorder_tool_results + ) + self._force_reasoning = template.force_reasoning + self._reasoning_config = template.reasoning_config + self._suggested_reasoning_parser = template.reasoning_parser + self._suggested_tool_call_parser = template.tool_call_parser + def load_chat_template( self, tokenizer_manager: TokenizerManager, @@ -164,10 +186,15 @@ class TemplateManager: if self._chat_template_name is None: # Try HuggingFace template first hf_template = self._resolve_hf_chat_template(tokenizer_manager) - if hf_template: + if hf_template and tokenizer_manager.tokenizer: # override the chat template - if tokenizer_manager.tokenizer: - tokenizer_manager.tokenizer.chat_template = hf_template + tokenizer_manager.tokenizer.chat_template = hf_template + python_template = resolve_python_chat_template( + tokenizer_manager.model_config.hf_config, hf_template + ) + if python_template is not None: + self._load_python_chat_template(tokenizer_manager, python_template) + elif hf_template: self._jinja_template_content_format = ( detect_jinja_template_content_format(hf_template) ) @@ -183,8 +210,11 @@ class TemplateManager: # Detect reasoning pattern and suggest parser from chat template if tokenizer_manager.tokenizer: - template = tokenizer_manager.tokenizer.chat_template - self._run_template_detection(template, tokenizer_manager.tokenizer) + if self._python_chat_template is not None: + self._use_python_template_features(self._python_chat_template) + else: + template = tokenizer_manager.tokenizer.chat_template + self._run_template_detection(template, tokenizer_manager.tokenizer) parts = [] if self._reasoning_config: parts.append(f"reasoning_config={self._reasoning_config}") @@ -205,6 +235,11 @@ class TemplateManager: self._chat_template_name = chat_template_arg return + python_template = get_python_chat_template(chat_template_arg) + if python_template is not None: + self._load_python_chat_template(tokenizer_manager, python_template) + return + if not os.path.exists(chat_template_arg): raise RuntimeError( f"Chat template {chat_template_arg} is not a built-in template name " @@ -273,6 +308,20 @@ class TemplateManager: if completion_template: self.load_completion_template(completion_template) + def _load_python_chat_template( + self, tokenizer_manager: TokenizerManager, template: PythonChatTemplate + ) -> None: + """Render chat prompts with a built-in Python chat template.""" + if tokenizer_manager.tokenizer is not None: + install_python_chat_template(tokenizer_manager.tokenizer, template) + self._python_chat_template = template + self._chat_template_name = None + self._jinja_template_content_format = template.content_format + logger.info( + f"Using Python chat template '{template.name}' in place of the Jinja " + f"chat template, content format: {template.content_format}" + ) + def _load_jinja_template( self, tokenizer_manager: TokenizerManager, template_path: str ) -> None: diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index 8679166e42..e777b459b5 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -196,6 +196,7 @@ class _MockTemplateManager: self.reasoning_config = None self.force_reasoning = False self.jinja_template_may_reorder_tool_results = False + self.python_chat_template = None class TestChatTemplateCache(CustomTestCase): diff --git a/test/registered/unit/entrypoints/openai/utils.py b/test/registered/unit/entrypoints/openai/utils.py index 1d248945bc..fa0d7df00e 100644 --- a/test/registered/unit/entrypoints/openai/utils.py +++ b/test/registered/unit/entrypoints/openai/utils.py @@ -87,6 +87,7 @@ class MockTemplateManager: self.reasoning_config = None self.force_reasoning = False self.jinja_template_may_reorder_tool_results = False + self.python_chat_template = None def make_serving(*, is_multimodal: bool = False) -> OpenAIServingResponses: diff --git a/test/registered/unit/parser/test_glm53_chat_template.py b/test/registered/unit/parser/test_glm53_chat_template.py new file mode 100644 index 0000000000..bdf63d0aeb --- /dev/null +++ b/test/registered/unit/parser/test_glm53_chat_template.py @@ -0,0 +1,645 @@ +"""GLM-5.3 / GLM-5.3-Flash Python chat templates. + +The expected prompts below were rendered with the checkpoints' chat_template.jinja. +Set SGLANG_TEST_GLM53_CHECKPOINT / SGLANG_TEST_GLM53_FLASH_CHECKPOINT to model +directories to also compare against their Jinja templates directly. +""" + +import inspect +import os +import sys +import unittest +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock, patch + +from tokenizers import Tokenizer, models, pre_tokenizers +from transformers import PreTrainedTokenizerFast +from transformers.utils.chat_template_utils import _compile_jinja_template + +from sglang.srt.parser.glm53_chat_template import render_glm53, render_glm53_flash +from sglang.srt.parser.jinja_template_utils import detect_jinja_template_content_format +from sglang.srt.parser.python_chat_template import ( + GLM53, + GLM53_FLASH, + get_python_chat_template, + install_python_chat_template, + resolve_python_chat_template, +) +from sglang.srt.parser.template_detection import ( + detect_inline_system_support, + resolve_auto_parsers, +) +from sglang.srt.parser.template_manager import TemplateManager +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=3.0, suite="base-a-test-cpu") + +PREFIX = "[gMASK]<|system|>Reasoning Effort: Max" +TOOLS_HEADER = ( + "<|system|>\n# Tools\n\nYou may call one or more functions to assist with the user " + "query.\n\nYou are provided with function signatures within XML " + "tags:\n\n" +) +TOOLS_FOOTER = ( + "\n\nFor each function call, output the function name and arguments " + "within the following XML format:\n{function-name}{arg-key-1}" + "{arg-value-1}{arg-key-2}" + "{arg-value-2}..." +) +WEATHER = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather.", + "strict": True, + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, +} +WEATHER_JSON = ( + '{"name": "get_weather", "description": "Get weather.", "parameters": ' + '{"type": "object", "properties": {"city": {"type": "string"}}}}' +) +DEFERRED = {"type": "function", "function": {"name": "later", "defer_loading": True}} +CALLS = [ + { + "id": "a", + "type": "function", + "function": {"name": "get_weather", "arguments": {"city": "Paris", "days": 2}}, + }, + { + "id": "b", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"city": "Rome", "opts": {"metric": True}}, + }, + }, +] +CALLS_TEXT = ( + "get_weathercityParis" + "days2" + "get_weathercityRome" + 'opts{"metric": true}' +) + +# name -> (render kwargs, expected GLM-5.3 prompt, expected GLM-5.3-Flash prompt) +CASES = { + "basic": ( + dict( + messages=[ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hi"}, + ], + add_generation_prompt=True, + ), + PREFIX + "<|system|>You are helpful.<|user|>Hi<|assistant|>", + None, + ), + "reasoning_history": ( + dict( + messages=[ + {"role": "user", "content": "q1"}, + {"role": "assistant", "content": "a1", "reasoning_content": "r1"}, + {"role": "user", "content": "q2"}, + {"role": "assistant", "content": "r2 a2 "}, + ], + reasoning_effort="low", + ), + "[gMASK]<|system|>Reasoning Effort: Low<|user|>q1<|assistant|>" + "r1a1<|user|>q2<|assistant|>r2a2", + None, + ), + "reasoning_effort_fallback": ( + dict( + messages=[{"role": "user", "content": "q"}], + reasoning_effort="medium", + ), + PREFIX + "<|user|>q", + None, + ), + "clear_thinking": ( + dict( + messages=[ + {"role": "user", "content": "q1"}, + {"role": "assistant", "content": "a1", "reasoning_content": "r1"}, + {"role": "user", "content": "q2"}, + {"role": "assistant", "content": "a2", "reasoning_content": "r2"}, + ], + clear_thinking=True, + reasoning_effort="high", + ), + "[gMASK]<|system|>Reasoning Effort: High<|user|>q1<|assistant|>" + "a1<|user|>q2<|assistant|>r2a2", + None, + ), + "tool_results_follow_tool_calls": ( + dict( + messages=[ + {"role": "user", "content": "Weather?"}, + {"role": "assistant", "content": "", "tool_calls": CALLS}, + {"role": "tool", "tool_call_id": "b", "content": "Rome: sun"}, + {"role": "tool", "tool_call_id": "a", "content": "Paris: rain"}, + ], + tools=[WEATHER, DEFERRED], + add_generation_prompt=True, + ), + PREFIX + + TOOLS_HEADER + + WEATHER_JSON + + "\n" + + TOOLS_FOOTER + + "<|user|>Weather?<|assistant|>" + + CALLS_TEXT + + "<|observation|>Paris: rain" + "Rome: sun<|assistant|>", + None, + ), + "unmatched_tool_result_keeps_order": ( + dict( + messages=[ + {"role": "user", "content": "Weather?"}, + {"role": "assistant", "content": "", "tool_calls": CALLS}, + {"role": "tool", "tool_call_id": "b", "content": "Rome: sun"}, + {"role": "tool", "tool_call_id": "zzz", "content": "?"}, + ], + ), + PREFIX + + "<|user|>Weather?<|assistant|>" + + CALLS_TEXT + + "<|observation|>Rome: sun" + "?", + None, + ), + "media": ( + dict( + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Look:"}, + {"type": "image"}, + {"type": "video_url"}, + {"type": "input_audio"}, + ], + } + ], + ), + PREFIX + + "<|user|>Look:" + + "".join( + f"You are unable to process this {media} because you don't " + "have multi-modal input ability. Try different methods." + for media in ("image", "video", "audio") + ), + PREFIX + "<|user|>Look:<|begin_of_image|><|image|><|end_of_image|>" + "<|begin_of_video|><|video|><|end_of_video|><|begin_of_audio|><|end_of_audio|>", + ), + "tool_reference": ( + dict( + messages=[ + {"role": "user", "content": "q"}, + { + "role": "tool", + "content": [ + { + "output": [ + {"type": "tool_reference", "name": "get_weather"} + ], + "tool_call_id": "a", + } + ], + }, + ], + tools=[WEATHER], + ), + PREFIX + + TOOLS_HEADER + + WEATHER_JSON + + "\n" + + TOOLS_FOOTER + + "<|user|>q<|observation|>\n" + + WEATHER_JSON + + "\n", + None, + ), + # The template only assigns reasoning_content in its own branches, so a + # template variable of that name fills in for turns without reasoning. + "reasoning_content_variable": ( + dict( + messages=[ + {"role": "user", "content": "q"}, + {"role": "assistant", "content": "A1"}, + ], + reasoning_content="X", + ), + PREFIX + "<|user|>q<|assistant|>XA1", + None, + ), +} + + +def _render(render, kwargs): + kwargs = dict(kwargs) + return render(kwargs.pop("messages"), **kwargs) + + +def _glm53_like_template(multimodal): + """A reduced GLM-5.3 template with every construct template detection keys on.""" + concat = "~" if multimodal else "+" + image = ( + "{%- elif item.type == 'image' -%}<|begin_of_image|><|image|><|end_of_image|>" + if multimodal + else "" + ) + return ( + "[gMASK]\n" + "{%- set effective_reasoning_effort = reasoning_effort if reasoning_effort is " + "defined and reasoning_effort in ['low', 'high'] else 'max' -%}" + "<|system|>Reasoning Effort: {{ effective_reasoning_effort | capitalize }}" + "{%- for m in messages -%}" + "{%- if m.role == 'system' -%}<|system|>{{ m.content }}" + "{%- elif m.role == 'user' -%}<|user|>" + "{%- if m.content is string -%}{{ m.content }}{%- else -%}" + "{%- for item in m.content -%}{%- if item.type == 'text' -%}{{ item.text }}" + f"{image}{{%- endif -%}}{{%- endfor -%}}{{%- endif -%}}" + "{%- elif m.role == 'tool' -%}<|observation|>{{ m.tool_call_id }}" + "{%- elif m.role == 'assistant' -%}<|assistant|>" + "{% for tc in m.tool_calls %}" + f"{{{{- '' {concat} tc.name -}}}}" + "{% for k, v in tc.arguments.items() %}{{ k }}" + "{{ v }}{% endfor %}{% endfor %}" + "{%- endif -%}{%- endfor -%}" + "{%- if add_generation_prompt -%}<|assistant|>{{- '' -}}{%- endif -%}" + ) + + +GLM52_LIKE_TEMPLATE = ( + "[gMASK]{%- if enable_thinking is defined and not enable_thinking -%}" + "/nothink{%- endif -%}" +) + + +def _hf_config(architecture): + return SimpleNamespace(architectures=[architecture]) + + +def _tiny_tokenizer(chat_template=None): + backend = Tokenizer(models.WordLevel(vocab={"[UNK]": 0}, unk_token="[UNK]")) + backend.pre_tokenizer = pre_tokenizers.Whitespace() + tokenizer = PreTrainedTokenizerFast( + tokenizer_object=backend, unk_token="[UNK]", eos_token="<|endoftext|>" + ) + tokenizer.add_tokens(["Hi", "q", "<|user|>", "<|assistant|>", ""]) + tokenizer.chat_template = chat_template + return tokenizer + + +class TestGlm53ChatTemplate(CustomTestCase): + def test_renders_like_the_jinja_templates(self): + for name, (kwargs, text_prompt, flash_prompt) in CASES.items(): + with self.subTest(case=name, model="GLM-5.3"): + self.assertEqual(_render(render_glm53, kwargs), text_prompt) + with self.subTest(case=name, model="GLM-5.3-Flash"): + self.assertEqual( + _render(render_glm53_flash, kwargs), flash_prompt or text_prompt + ) + + def test_tool_keys_are_interpolated_raw(self): + tool = {"name": "f", 'we"ird': 1, "strict": False} + prompt = render_glm53([{"role": "user", "content": "q"}], tools=[tool]) + self.assertIn('{"name": "f", "we"ird": 1}\n', prompt) + + def test_tool_call_name_concatenation(self): + messages = [ + { + "role": "assistant", + "content": "", + "tool_calls": [{"function": {"name": None, "arguments": {}}}], + } + ] + # GLM-5.3 concatenates with `+`, GLM-5.3-Flash with `~`. + with self.assertRaisesRegex(TypeError, "can only concatenate str"): + render_glm53(messages) + self.assertTrue( + render_glm53_flash(messages).endswith("None") + ) + + def test_malformed_inputs_raise_like_the_template(self): + user = {"role": "user", "content": "q"} + reference = { + "role": "tool", + "content": [{"type": "tool_reference", "name": "f"}], + } + malformed = { + "string arguments": [ + { + "role": "assistant", + "content": "", + "tool_calls": [{"function": {"name": "f", "arguments": "{}"}}], + } + ], + "missing name": [ + { + "role": "assistant", + "content": "", + "tool_calls": [{"function": {"arguments": {}}}], + } + ], + "tool_reference without tools": [user, reference], + "mapping tool content": [user, {"role": "tool", "content": {"a": 1}}], + } + for name, messages in malformed.items(): + with self.subTest(case=name): + with self.assertRaises(ValueError): + render_glm53(messages) + with self.assertRaises(ValueError): + render_glm53([user], tools=["not a tool"]) + + +class TestPythonChatTemplateSelection(CustomTestCase): + def test_features_match_jinja_template_detection(self): + for multimodal, template in ((False, GLM53), (True, GLM53_FLASH)): + jinja_template = _glm53_like_template(multimodal) + detected = TemplateManager() + detected._run_template_detection(jinja_template, None) + with self.subTest(template=template.name): + self.assertEqual( + detect_jinja_template_content_format(jinja_template), + template.content_format, + ) + self.assertEqual(detected.force_reasoning, template.force_reasoning) + self.assertEqual(detected.reasoning_config, template.reasoning_config) + self.assertEqual( + detected.jinja_template_may_reorder_tool_results, + template.may_reorder_tool_results, + ) + self.assertEqual( + detected.suggested_reasoning_parser, template.reasoning_parser + ) + self.assertEqual( + detected.suggested_tool_call_parser, template.tool_call_parser + ) + self.assertEqual( + detect_inline_system_support(jinja_template), + template.supports_inline_system, + ) + + def test_resolves_by_architecture_and_template(self): + text, flash = _glm53_like_template(False), _glm53_like_template(True) + cases = [ + ("GlmMoeDsaForCausalLM", text, GLM53), + ("Glm5NextForConditionalGeneration", flash, GLM53_FLASH), + # GLM-5.2 shares the architecture but not the template. + ("GlmMoeDsaForCausalLM", GLM52_LIKE_TEMPLATE, None), + # The multimodal block must match the architecture. + ("GlmMoeDsaForCausalLM", flash, None), + ("Glm5NextForConditionalGeneration", text, None), + ("GlmMoeDsaForCausalLM", None, None), + ("LlamaForCausalLM", text, None), + ] + for architecture, jinja_template, expected in cases: + with self.subTest( + architecture=architecture, expected=expected and expected.name + ): + self.assertIs( + resolve_python_chat_template( + _hf_config(architecture), jinja_template + ), + expected, + ) + + def test_builtin_names(self): + self.assertIs(get_python_chat_template("glm-5.3"), GLM53) + self.assertIs(get_python_chat_template("glm-5.3-flash"), GLM53_FLASH) + self.assertIsNone(get_python_chat_template("chatml")) + + +class TestTemplateManagerPythonChatTemplate(CustomTestCase): + def _tokenizer_manager(self, architecture, chat_template): + return SimpleNamespace( + tokenizer=_tiny_tokenizer(chat_template), + processor=None, + model_config=SimpleNamespace(hf_config=_hf_config(architecture)), + ) + + def test_replaces_the_jinja_template(self): + tokenizer_manager = self._tokenizer_manager( + "GlmMoeDsaForCausalLM", _glm53_like_template(False) + ) + manager = TemplateManager() + manager.load_chat_template(tokenizer_manager, None, "zai-org/GLM-5.3") + + self.assertIs(manager.python_chat_template, GLM53) + self.assertIsNone(manager.chat_template_name) + self.assertEqual(manager.jinja_template_content_format, "openai") + self.assertTrue(manager.force_reasoning) + self.assertEqual(manager.reasoning_config, GLM53.reasoning_config) + self.assertTrue(manager.jinja_template_may_reorder_tool_results) + self.assertEqual(manager.suggested_reasoning_parser, "glm45") + self.assertEqual(manager.suggested_tool_call_parser, "glm47") + + messages = CASES["media"][0]["messages"] + self.assertEqual( + tokenizer_manager.tokenizer.apply_chat_template(messages, tokenize=False), + render_glm53(messages), + ) + + def test_other_templates_keep_the_jinja_template(self): + # GLM-5.2 shares GLM-5.3's architecture but not its template. + tokenizer_manager = self._tokenizer_manager( + "GlmMoeDsaForCausalLM", GLM52_LIKE_TEMPLATE + ) + manager = TemplateManager() + manager.load_chat_template(tokenizer_manager, None, "zai-org/GLM-5.2") + + self.assertIsNone(manager.python_chat_template) + messages = [{"role": "user", "content": "q"}] + self.assertEqual( + tokenizer_manager.tokenizer.apply_chat_template(messages, tokenize=False), + _compile_jinja_template(GLM52_LIKE_TEMPLATE).render( + messages=messages, add_generation_prompt=False + ), + ) + + def test_explicit_builtin_name(self): + tokenizer_manager = self._tokenizer_manager("LlamaForCausalLM", None) + manager = TemplateManager() + manager.load_chat_template(tokenizer_manager, "glm-5.3-flash", "some/model") + + self.assertIs(manager.python_chat_template, GLM53_FLASH) + messages = CASES["media"][0]["messages"] + self.assertEqual( + tokenizer_manager.tokenizer.apply_chat_template(messages, tokenize=False), + CASES["media"][2], + ) + + +def _patch_hf_transformers_utils(get_tokenizer, get_config): + module = ModuleType("sglang.srt.utils.hf_transformers_utils") + module.get_tokenizer = get_tokenizer + module.get_config = get_config + return patch.dict(sys.modules, {module.__name__: module}) + + +class TestAutoParsersFromPythonChatTemplate(CustomTestCase): + def _resolve(self, architecture, jinja_template, chat_template=None): + server_args = ServerArgs( + model_path="dummy", + reasoning_parser="auto", + tool_call_parser="auto", + trust_remote_code=False, + chat_template=chat_template, + ) + tokenizer = SimpleNamespace(chat_template=jinja_template, get_vocab=dict) + self.get_tokenizer = Mock(return_value=tokenizer) + self.get_config = Mock(return_value=_hf_config(architecture)) + with ( + _patch_hf_transformers_utils(self.get_tokenizer, self.get_config), + self.assertLogs("sglang.srt.parser.template_detection", "INFO") as logs, + ): + resolve_auto_parsers(server_args) + + from sglang.srt.arg_groups.overrides import resolution_result + + parsers = tuple( + resolution_result(server_args, field) + for field in ("reasoning_parser", "tool_call_parser") + ) + return parsers, "\n".join(logs.output) + + def test_auto_parsers_come_from_the_python_template(self): + for architecture, multimodal, name in ( + ("GlmMoeDsaForCausalLM", False, "glm-5.3"), + ("Glm5NextForConditionalGeneration", True, "glm-5.3-flash"), + ): + with self.subTest(template=name): + parsers, logs = self._resolve( + architecture, _glm53_like_template(multimodal) + ) + self.assertEqual(parsers, ("glm45", "glm47")) + self.assertIn(f"from Python chat template '{name}'", logs) + + def test_explicit_builtin_name_loads_nothing(self): + parsers, logs = self._resolve("LlamaForCausalLM", None, "glm-5.3-flash") + self.assertEqual(parsers, ("glm45", "glm47")) + self.assertIn("from Python chat template 'glm-5.3-flash'", logs) + self.get_tokenizer.assert_not_called() + self.get_config.assert_not_called() + + def test_other_templates_use_jinja_detection(self): + _, logs = self._resolve("GlmMoeDsaForCausalLM", GLM52_LIKE_TEMPLATE) + self.assertNotIn("Python chat template", logs) + + +class TestInstallPythonChatTemplate(CustomTestCase): + def setUp(self): + super().setUp() + self.tokenizer = _tiny_tokenizer("{{ messages[0]['content'] }}!") + install_python_chat_template(self.tokenizer, GLM53) + self.messages = [{"role": "user", "content": "Hi"}] + self.rendered = render_glm53(self.messages, add_generation_prompt=True) + + def test_tokenizes_like_transformers(self): + ids = self.tokenizer(self.rendered, add_special_tokens=False)["input_ids"] + apply = self.tokenizer.apply_chat_template + self.assertEqual( + apply(self.messages, add_generation_prompt=True, tokenize=False), + self.rendered, + ) + self.assertEqual( + apply(self.messages, add_generation_prompt=True, return_dict=False), ids + ) + self.assertEqual( + apply(self.messages, add_generation_prompt=True)["input_ids"], ids + ) + self.assertEqual( + apply( + [self.messages, self.messages], + add_generation_prompt=True, + tokenize=False, + ), + [self.rendered, self.rendered], + ) + with self.assertRaises(ValueError): + apply([], tokenize=False) + + def test_converts_callable_tools(self): + def get_weather(city: str): + """Get the weather. + + Args: + city: The city. + """ + + prompt = self.tokenizer.apply_chat_template( + self.messages, tools=[get_weather], tokenize=False + ) + self.assertIn('"name": "get_weather"', prompt) + + def test_jinja_only_features_use_the_original_implementation(self): + apply = self.tokenizer.apply_chat_template + self.assertEqual( + apply( + self.messages, + chat_template="{{ messages[0]['content'] }}?", + tokenize=False, + ), + "Hi?", + ) + self.assertEqual( + apply(self.messages, continue_final_message=True, tokenize=False), "Hi" + ) + + def test_signature_matches_transformers(self): + original = inspect.signature(PreTrainedTokenizerFast.apply_chat_template) + installed = inspect.signature(self.tokenizer.apply_chat_template) + expected = [ + (name, parameter.default) + for name, parameter in original.parameters.items() + if name != "self" + ] + self.assertEqual( + [ + (name, parameter.default) + for name, parameter in installed.parameters.items() + ], + expected, + ) + + +@unittest.skipUnless( + os.environ.get("SGLANG_TEST_GLM53_CHECKPOINT") + or os.environ.get("SGLANG_TEST_GLM53_FLASH_CHECKPOINT"), + "set SGLANG_TEST_GLM53_CHECKPOINT / SGLANG_TEST_GLM53_FLASH_CHECKPOINT", +) +class TestAgainstCheckpointTemplates(CustomTestCase): + def test_matches_checkpoint_jinja(self): + for env, render in ( + ("SGLANG_TEST_GLM53_CHECKPOINT", render_glm53), + ("SGLANG_TEST_GLM53_FLASH_CHECKPOINT", render_glm53_flash), + ): + model_dir = os.environ.get(env) + if not model_dir: + continue + with open( + os.path.join(model_dir, "chat_template.jinja"), encoding="utf-8" + ) as f: + compiled = _compile_jinja_template(f.read()) + for name, (kwargs, _, _) in CASES.items(): + for add_generation_prompt in (False, True): + with self.subTest( + model=model_dir, case=name, gen=add_generation_prompt + ): + kwargs = { + **kwargs, + "add_generation_prompt": add_generation_prompt, + } + self.assertEqual( + _render(render, kwargs), compiled.render(**kwargs) + ) + + +if __name__ == "__main__": + unittest.main()