diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index b14be748e..11baf6abf 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -171,6 +171,9 @@ class OpenAIServingChat(OpenAIServingBase): self.template_manager = template_manager self.tool_call_parser = self.tokenizer_manager.server_args.tool_call_parser self.reasoning_parser = self.tokenizer_manager.server_args.reasoning_parser + self.default_chat_template_kwargs = ( + self.tokenizer_manager.server_args.default_chat_template_kwargs or {} + ) self._reasoning_detector = None if self.reasoning_parser: try: @@ -633,6 +636,15 @@ class OpenAIServingChat(OpenAIServingBase): self, request: ChatCompletionRequest, is_multimodal: bool ) -> MessageProcessingResult: """Process chat messages and apply chat template""" + if self.default_chat_template_kwargs: + ctk = dict(request.chat_template_kwargs or {}) + for k, v in self.default_chat_template_kwargs.items(): + ctk.setdefault(k, v) + request.chat_template_kwargs = ctk + effort = ctk.get("reasoning_effort") + if effort is not None and request.reasoning_effort is None: + request.reasoning_effort = effort + # GptOss model needs to keep special tokens for harmony parsing if self.is_gpt_oss or self.is_gemma4: request.skip_special_tokens = False diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 395e5260b..85024538a 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -1116,6 +1116,16 @@ class ServerArgs: "Return number of cached tokens in usage.prompt_tokens_details for each openai request.", ] = False reasoning_parser: Optional[str] = None + default_chat_template_kwargs: A[ + Optional[Dict[str, Any]], + Arg( + help="Default chat template kwargs applied to every request when not " + "overridden per-request. Keys must match what the model's chat template " + "expects (e.g. enable_thinking, thinking, reasoning_effort). Per-request " + "chat_template_kwargs takes precedence.", + type_parser=json.loads, + ), + ] = None strip_thinking_cache: A[ bool, "Skip caching reasoning-model output (thinking + answer) in the radix tree on finish; keep only the prompt prefix. Opt-in: changes cache contents.", @@ -2810,6 +2820,13 @@ class ServerArgs: # Validate transcription/ASR-specific server args. self._handle_asr_validation() + if self.default_chat_template_kwargs is not None and not isinstance( + self.default_chat_template_kwargs, dict + ): + raise ValueError( + "--default-chat-template-kwargs must decode to a JSON object" + ) + # Handle deprecated arguments. self._handle_deprecated_args() diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index 8e3ba3b66..ab7f9469b 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -45,6 +45,7 @@ class _MockTokenizerManager: tool_call_parser="hermes", reasoning_parser=None, stream_response_default_include_usage=False, + default_chat_template_kwargs=None, ) # Mock hf_config for _resolve_chat_encoding_spec check mock_hf_config = Mock() @@ -283,6 +284,54 @@ class ServingChatTestCase(unittest.TestCase): self.assertFalse(adapted.require_reasoning) + def test_default_chat_template_kwargs_applied_when_request_unset(self): + self.template_manager.chat_template_name = None + self.template_manager.jinja_template_content_format = "string" + self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3] + self.chat.default_chat_template_kwargs = {"enable_thinking": False} + + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "What is 2+2?"}], + ) + + self.chat._process_messages(req, is_multimodal=False) + + kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs + self.assertIs(kwargs["enable_thinking"], False) + + def test_default_chat_template_kwargs_overridden_per_request(self): + self.template_manager.chat_template_name = None + self.template_manager.jinja_template_content_format = "string" + self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3] + self.chat.default_chat_template_kwargs = {"enable_thinking": False} + + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "What is 2+2?"}], + chat_template_kwargs={"enable_thinking": True}, + ) + + self.chat._process_messages(req, is_multimodal=False) + + kwargs = self.tm.tokenizer.apply_chat_template.call_args.kwargs + self.assertIs(kwargs["enable_thinking"], True) + + def test_default_chat_template_kwargs_mirrors_reasoning_effort(self): + self.template_manager.chat_template_name = None + self.template_manager.jinja_template_content_format = "string" + self.tm.tokenizer.apply_chat_template.return_value = [1, 2, 3] + self.chat.default_chat_template_kwargs = {"reasoning_effort": "high"} + + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "What is 2+2?"}], + ) + + self.chat._process_messages(req, is_multimodal=False) + + self.assertEqual(req.reasoning_effort, "high") + def test_kimi_tool_call_keeps_template_default_thinking(self): self.template_manager.chat_template_name = None self.template_manager.jinja_template_content_format = "string"