config: template-detected parsers go to the engine's control-plane overlay
`init_tokenizer_manager` wrote the chat-template-detected `reasoning_parser` / `tool_call_parser` onto the published `ServerArgs`, after `TokenizerManager.__init__` had already projected the config bags — so the namespace readers and the resolved-config readback disagreed with the instance, and a second `Engine` in the same process would inherit the first one's detection through the shared bags. Detection is per-engine control-plane state, which the manager already models: `record_config_updates` records it, the readback endpoints overlay it, and `config_value` reports what is in effect. `OpenAIServingChat` — the only reader of these two fields in the tokenizer process — follows the overlay. The architecture pass (`resolve_auto_parsers`, before the schedulers fork) is unchanged: the scheduler resolves its own bags from the instance it receives.
This commit is contained in:
@@ -175,10 +175,12 @@ def init_tokenizer_manager(
|
|||||||
"tool-call parser",
|
"tool-call parser",
|
||||||
),
|
),
|
||||||
):
|
):
|
||||||
if getattr(server_args, attr) != "auto":
|
if tokenizer_manager.config_value(attr) != "auto":
|
||||||
continue
|
continue
|
||||||
if suggested is not None:
|
if suggested is not None:
|
||||||
server_args.override(source="template-detection", **{attr: suggested})
|
tokenizer_manager.record_config_updates(
|
||||||
|
"template-detection", **{attr: suggested}
|
||||||
|
)
|
||||||
logger.info(
|
logger.info(
|
||||||
f"Auto-detected --{attr.replace('_', '-')} as '{suggested}' from chat template"
|
f"Auto-detected --{attr.replace('_', '-')} as '{suggested}' from chat template"
|
||||||
)
|
)
|
||||||
@@ -187,7 +189,9 @@ def init_tokenizer_manager(
|
|||||||
f"--{attr.replace('_', '-')}=auto specified but could not detect "
|
f"--{attr.replace('_', '-')}=auto specified but could not detect "
|
||||||
f"{label} from chat template. Disabling {label}."
|
f"{label} from chat template. Disabling {label}."
|
||||||
)
|
)
|
||||||
server_args.override(source="template-detection", **{attr: None})
|
tokenizer_manager.record_config_updates(
|
||||||
|
"template-detection", **{attr: None}
|
||||||
|
)
|
||||||
|
|
||||||
return tokenizer_manager, template_manager
|
return tokenizer_manager, template_manager
|
||||||
|
|
||||||
|
|||||||
@@ -203,8 +203,8 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
):
|
):
|
||||||
super().__init__(tokenizer_manager)
|
super().__init__(tokenizer_manager)
|
||||||
self.template_manager = template_manager
|
self.template_manager = template_manager
|
||||||
self.tool_call_parser = self.tokenizer_manager.server_args.tool_call_parser
|
self.tool_call_parser = self.tokenizer_manager.config_value("tool_call_parser")
|
||||||
self.reasoning_parser = self.tokenizer_manager.server_args.reasoning_parser
|
self.reasoning_parser = self.tokenizer_manager.config_value("reasoning_parser")
|
||||||
self.default_chat_template_kwargs = (
|
self.default_chat_template_kwargs = (
|
||||||
self.tokenizer_manager.server_args.default_chat_template_kwargs or {}
|
self.tokenizer_manager.server_args.default_chat_template_kwargs or {}
|
||||||
)
|
)
|
||||||
@@ -1049,9 +1049,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
# SGLang's ReasonerGrammarBackend owns the reasoning prefix
|
# SGLang's ReasonerGrammarBackend owns the reasoning prefix
|
||||||
# when --reasoning-parser is configured, so builtin xgrammar
|
# when --reasoning-parser is configured, so builtin xgrammar
|
||||||
# tags must describe only the post-reasoning tool-call suffix.
|
# tags must describe only the post-reasoning tool-call suffix.
|
||||||
xgrammar_reasoning = thinking_mode and (
|
xgrammar_reasoning = thinking_mode and (self.reasoning_parser is None)
|
||||||
self.tokenizer_manager.server_args.reasoning_parser is None
|
|
||||||
)
|
|
||||||
tool_call_constraint = None
|
tool_call_constraint = None
|
||||||
|
|
||||||
# Apply chat template and its stop strings
|
# Apply chat template and its stop strings
|
||||||
|
|||||||
@@ -97,8 +97,8 @@ class OpenAIServingResponses(OpenAIServingChat):
|
|||||||
) -> None:
|
) -> None:
|
||||||
super().__init__(tokenizer_manager, template_manager)
|
super().__init__(tokenizer_manager, template_manager)
|
||||||
|
|
||||||
# template_manager is already set by parent class
|
# template_manager is already set by parent class; reasoning_parser comes
|
||||||
self.reasoning_parser = self.tokenizer_manager.server_args.reasoning_parser
|
# from the parent, which reads the manager's control-plane overlay.
|
||||||
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
||||||
|
|
||||||
# Parent OpenAIServingChat.__init__ already populated default_sampling_params.
|
# Parent OpenAIServingChat.__init__ already populated default_sampling_params.
|
||||||
|
|||||||
@@ -74,6 +74,7 @@ class _MockTokenizerManager:
|
|||||||
self.model_path = self.server_args.model_path
|
self.model_path = self.server_args.model_path
|
||||||
# The manager tracks the served name itself; a weight update rewrites it.
|
# The manager tracks the served name itself; a weight update rewrites it.
|
||||||
self.served_model_name = "test-model"
|
self.served_model_name = "test-model"
|
||||||
|
self._config_updates = []
|
||||||
|
|
||||||
# Mock hf_config for _resolve_chat_encoding_spec check
|
# Mock hf_config for _resolve_chat_encoding_spec check
|
||||||
mock_hf_config = Mock()
|
mock_hf_config = Mock()
|
||||||
@@ -109,6 +110,13 @@ class _MockTokenizerManager:
|
|||||||
self.generate_request = Mock(return_value=_mock_generate())
|
self.generate_request = Mock(return_value=_mock_generate())
|
||||||
self.create_abort_task = Mock()
|
self.create_abort_task = Mock()
|
||||||
|
|
||||||
|
def config_value(self, name: str):
|
||||||
|
"""The manager's overlay accessor: no control-plane update recorded."""
|
||||||
|
for _source, fields in reversed(self._config_updates):
|
||||||
|
if name in fields:
|
||||||
|
return fields[name]
|
||||||
|
return getattr(self.server_args, name)
|
||||||
|
|
||||||
|
|
||||||
class _MockTemplateManager:
|
class _MockTemplateManager:
|
||||||
"""Minimal mock for TemplateManager."""
|
"""Minimal mock for TemplateManager."""
|
||||||
@@ -148,6 +156,35 @@ class ServingChatTestCase(unittest.TestCase):
|
|||||||
self.fastapi_request = Mock(spec=Request)
|
self.fastapi_request = Mock(spec=Request)
|
||||||
self.fastapi_request.headers = {}
|
self.fastapi_request.headers = {}
|
||||||
|
|
||||||
|
def test_parsers_follow_the_control_plane_overlay(self):
|
||||||
|
"""Template detection records the parsers on the manager, not on its
|
||||||
|
ServerArgs — the instance keeps what the launcher passed."""
|
||||||
|
self.tm.server_args.tool_call_parser = "auto"
|
||||||
|
self.tm.server_args.reasoning_parser = "auto"
|
||||||
|
self.tm._config_updates.append(
|
||||||
|
(
|
||||||
|
"template-detection",
|
||||||
|
{"tool_call_parser": "qwen25", "reasoning_parser": None},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
chat = OpenAIServingChat(self.tm, self.template_manager)
|
||||||
|
|
||||||
|
self.assertEqual(chat.tool_call_parser, "qwen25")
|
||||||
|
self.assertIsNone(chat.reasoning_parser)
|
||||||
|
self.assertEqual(self.tm.server_args.tool_call_parser, "auto")
|
||||||
|
|
||||||
|
def test_the_xgrammar_gate_follows_the_overlay(self):
|
||||||
|
"""A detected `reasoning_parser` must gate xgrammar, not the seed's "auto"."""
|
||||||
|
self.tm.server_args.reasoning_parser = "auto"
|
||||||
|
self.tm._config_updates.append(
|
||||||
|
("template-detection", {"reasoning_parser": "qwen3"})
|
||||||
|
)
|
||||||
|
chat = OpenAIServingChat(self.tm, self.template_manager)
|
||||||
|
self.assertEqual(chat.reasoning_parser, "qwen3")
|
||||||
|
# the gate reads the same value the parser was built from
|
||||||
|
self.assertIsNotNone(chat.reasoning_parser)
|
||||||
|
|
||||||
def test_text_only_model_rejects_media_before_generation(self):
|
def test_text_only_model_rejects_media_before_generation(self):
|
||||||
media_parts = {
|
media_parts = {
|
||||||
"image_url": {
|
"image_url": {
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ class MockTokenizerManager:
|
|||||||
tool_call_parser=None,
|
tool_call_parser=None,
|
||||||
incremental_streaming_output=False,
|
incremental_streaming_output=False,
|
||||||
)
|
)
|
||||||
|
self._config_updates = []
|
||||||
self.tokenizer = Mock()
|
self.tokenizer = Mock()
|
||||||
self.tokenizer.encode.return_value = [1, 2, 3]
|
self.tokenizer.encode.return_value = [1, 2, 3]
|
||||||
self.tokenizer.chat_template = None
|
self.tokenizer.chat_template = None
|
||||||
@@ -60,6 +61,13 @@ class MockTokenizerManager:
|
|||||||
self.generate_request = Mock()
|
self.generate_request = Mock()
|
||||||
self.create_abort_task = Mock()
|
self.create_abort_task = Mock()
|
||||||
|
|
||||||
|
def config_value(self, name: str):
|
||||||
|
"""The manager's overlay accessor: no control-plane update recorded."""
|
||||||
|
for _source, fields in reversed(self._config_updates):
|
||||||
|
if name in fields:
|
||||||
|
return fields[name]
|
||||||
|
return getattr(self.server_args, name)
|
||||||
|
|
||||||
|
|
||||||
class MockTemplateManager:
|
class MockTemplateManager:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
|||||||
Reference in New Issue
Block a user