fix: Handle multiple named chat templates in HuggingFace tokenizers (#17236)

Signed-off-by: Xinyuan Tong <xinyuantong.cs@gmail.com>

docs/advanced_features/server_arguments.md

@@ -56,7 +56,7 @@ You can find all arguments by `python3 -m sglang.launch_server --help`
 - To enable fp8 weight quantization, add `--quantization fp8` on a fp16 checkpoint or directly load a fp8 checkpoint without specifying any arguments.
 - To enable fp8 kv cache quantization, add `--kv-cache-dtype fp8_e5m2`.
 - To enable deterministic inference and batch invariant operations, add `--enable-deterministic-inference`. More details can be found in [deterministic inference document](../advanced_features/deterministic_inference.md).
-- If the model does not have a chat template in the Hugging Face tokenizer, you can specify a [custom chat template](../references/custom_chat_template.md).
+- If the model does not have a chat template in the Hugging Face tokenizer, you can specify a [custom chat template](../references/custom_chat_template.md). If the tokenizer has multiple named templates (e.g., 'default', 'tool_use'), you can select one using `--hf-chat-template-name tool_use`.
 - To run tensor parallelism on multiple nodes, add `--nnodes 2`. If you have two nodes with two GPUs on each node and want to run TP=4, let `sgl-dev-0` be the hostname of the first node and `50000` be an available port, you can use the following commands. If you meet deadlock, please try to add `--disable-cuda-graph`
 
   ```bash
@@ -216,6 +216,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s
 | `--served-model-name` | Override the model name returned by the v1/models endpoint in OpenAI API server. | `None` | Type: str |
 | `--weight-version` | Version identifier for the model weights. Defaults to 'default' if not specified. | `default` | Type: str |
 | `--chat-template` | The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server. | `None` | Type: str |
+| `--hf-chat-template-name` | When the HuggingFace tokenizer has multiple chat templates (e.g., 'default', 'tool_use', 'rag'), specify which named template to use. If not set, the first available template is used. | `None` | Type: str |
 | `--completion-template` | The buliltin completion template name or the path of the completion template file. This is only used for OpenAI-compatible API server. only for code completion currently. | `None` | Type: str |
 | `--file-storage-path` | The path of the file storage in backend. | `sglang_storage` | Type: str |
 | `--enable-cache-report` | Return number of cached tokens in usage.prompt_tokens_details for each openai request. | `False` | bool flag (set to enable) |

python/sglang/srt/managers/template_manager.py

@@ -22,8 +22,9 @@ import json
 import logging
 import os
 import re
-from typing import Optional
+from typing import Dict, Optional
 
+from sglang.srt.managers.tokenizer_manager import TokenizerManager
 from sglang.srt.parser.code_completion_parser import (
     CompletionTemplate,
     FimPosition,
@@ -99,7 +100,10 @@ class TemplateManager:
         return has_reasoning
 
     def load_chat_template(
-        self, tokenizer_manager, chat_template_arg: Optional[str], model_path: str
+        self,
+        tokenizer_manager: TokenizerManager,
+        chat_template_arg: Optional[str],
+        model_path: str,
     ) -> None:
         """
         Load a chat template from various sources.
@@ -143,7 +147,7 @@ class TemplateManager:
             )
 
     def _load_explicit_chat_template(
-        self, tokenizer_manager, chat_template_arg: str
+        self, tokenizer_manager: TokenizerManager, chat_template_arg: str
     ) -> None:
         """Load explicitly specified chat template."""
         logger.info(f"Loading chat template from argument: {chat_template_arg}")
@@ -197,7 +201,7 @@ class TemplateManager:
 
     def initialize_templates(
         self,
-        tokenizer_manager,
+        tokenizer_manager: TokenizerManager,
         model_path: str,
         chat_template: Optional[str] = None,
         completion_template: Optional[str] = None,
@@ -218,7 +222,9 @@ class TemplateManager:
         if completion_template:
             self.load_completion_template(completion_template)
 
-    def _load_jinja_template(self, tokenizer_manager, template_path: str) -> None:
+    def _load_jinja_template(
+        self, tokenizer_manager: TokenizerManager, template_path: str
+    ) -> None:
         """Load a Jinja template file."""
         with open(template_path, "r") as f:
             chat_template = "".join(f.readlines()).strip("\n")
@@ -288,21 +294,56 @@ class TemplateManager:
             )
         self._completion_template_name = template["name"]
 
-    def _resolve_hf_chat_template(self, tokenizer_manager) -> Optional[str]:
+    def _resolve_hf_chat_template(
-        """
+        self, tokenizer_manager: TokenizerManager
-        Resolve HuggingFace chat template.
+    ) -> Optional[str]:
-
-        Returns the chat template string if found, None otherwise.
-        """
         try:
-            if processor := tokenizer_manager.processor:
+            # Try (mm-)processor first, then tokenizer
-                if hasattr(processor, "chat_template") and processor.chat_template:
+            template = (
-                    return processor.chat_template
+                getattr(tokenizer_manager.processor, "chat_template", None)
-            if tokenizer := tokenizer_manager.tokenizer:
+                if tokenizer_manager.processor
-                if hasattr(tokenizer, "chat_template") and tokenizer.chat_template:
+                else None
-                    return tokenizer.chat_template
+            ) or (
-        except Exception as e:
+                getattr(tokenizer_manager.tokenizer, "chat_template", None)
-            logger.debug(f"Error getting chat template: {e}")
+                if tokenizer_manager.tokenizer
+                else None
+            )
 
-        logger.debug("No HuggingFace chat template found")
+            if template is None:
-        return None
+                logger.warning("No HuggingFace chat template found")
+                return None
+
+            # Handle dict templates (multiple named templates)
+            if isinstance(template, dict):
+                return self._select_named_template(template, tokenizer_manager)
+
+            # Single string template
+            return template
+
+        except Exception as e:
+            logger.warning(f"Error getting chat template: {e}")
+            return None
+
+    def _select_named_template(
+        self, templates: Dict[str, str], tokenizer_manager: TokenizerManager
+    ) -> str:
+        if not templates:
+            raise ValueError("Empty templates dict provided")
+
+        available_names = list(templates.keys())
+        logger.info(f"Multiple HuggingFace chat templates available: {available_names}")
+
+        # Use specified template if provided
+        if preferred_name := tokenizer_manager.server_args.hf_chat_template_name:
+            if preferred_name not in templates:
+                raise ValueError(
+                    f"Specified template '{preferred_name}' not found. "
+                    f"Available templates: {available_names}"
+                )
+            logger.info(f"Using specified chat template: '{preferred_name}'")
+            return templates[preferred_name]
+
+        # Fallback: Use first available template
+        first_name = available_names[0]
+        logger.info(f"Using first available template: '{first_name}'")
+        return templates[first_name]

python/sglang/srt/server_args.py

@@ -380,6 +380,7 @@ class ServerArgs:
     served_model_name: Optional[str] = None
     weight_version: str = "default"
     chat_template: Optional[str] = None
+    hf_chat_template_name: Optional[str] = None
     completion_template: Optional[str] = None
     file_storage_path: str = "sglang_storage"
     enable_cache_report: bool = False
@@ -3291,6 +3292,13 @@ class ServerArgs:
             default=ServerArgs.chat_template,
             help="The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server.",
         )
+        parser.add_argument(
+            "--hf-chat-template-name",
+            type=str,
+            default=ServerArgs.hf_chat_template_name,
+            help="When the HuggingFace tokenizer has multiple chat templates (e.g., 'default', 'tool_use', 'rag'), "
+            "specify which named template to use. If not set, the first available template is used.",
+        )
         parser.add_argument(
             "--completion-template",
             type=str,