config: template-detected parsers go to the engine's control-plane overlay
`init_tokenizer_manager` wrote the chat-template-detected `reasoning_parser` / `tool_call_parser` onto the published `ServerArgs`, after `TokenizerManager.__init__` had already projected the config bags — so the namespace readers and the resolved-config readback disagreed with the instance, and a second `Engine` in the same process would inherit the first one's detection through the shared bags. Detection is per-engine control-plane state, which the manager already models: `record_config_updates` records it, the readback endpoints overlay it, and `config_value` reports what is in effect. `OpenAIServingChat` — the only reader of these two fields in the tokenizer process — follows the overlay. The architecture pass (`resolve_auto_parsers`, before the schedulers fork) is unchanged: the scheduler resolves its own bags from the instance it receives.
This commit is contained in:
@@ -74,6 +74,7 @@ class _MockTokenizerManager:
|
||||
self.model_path = self.server_args.model_path
|
||||
# The manager tracks the served name itself; a weight update rewrites it.
|
||||
self.served_model_name = "test-model"
|
||||
self._config_updates = []
|
||||
|
||||
# Mock hf_config for _resolve_chat_encoding_spec check
|
||||
mock_hf_config = Mock()
|
||||
@@ -109,6 +110,13 @@ class _MockTokenizerManager:
|
||||
self.generate_request = Mock(return_value=_mock_generate())
|
||||
self.create_abort_task = Mock()
|
||||
|
||||
def config_value(self, name: str):
|
||||
"""The manager's overlay accessor: no control-plane update recorded."""
|
||||
for _source, fields in reversed(self._config_updates):
|
||||
if name in fields:
|
||||
return fields[name]
|
||||
return getattr(self.server_args, name)
|
||||
|
||||
|
||||
class _MockTemplateManager:
|
||||
"""Minimal mock for TemplateManager."""
|
||||
@@ -148,6 +156,35 @@ class ServingChatTestCase(unittest.TestCase):
|
||||
self.fastapi_request = Mock(spec=Request)
|
||||
self.fastapi_request.headers = {}
|
||||
|
||||
def test_parsers_follow_the_control_plane_overlay(self):
|
||||
"""Template detection records the parsers on the manager, not on its
|
||||
ServerArgs — the instance keeps what the launcher passed."""
|
||||
self.tm.server_args.tool_call_parser = "auto"
|
||||
self.tm.server_args.reasoning_parser = "auto"
|
||||
self.tm._config_updates.append(
|
||||
(
|
||||
"template-detection",
|
||||
{"tool_call_parser": "qwen25", "reasoning_parser": None},
|
||||
)
|
||||
)
|
||||
|
||||
chat = OpenAIServingChat(self.tm, self.template_manager)
|
||||
|
||||
self.assertEqual(chat.tool_call_parser, "qwen25")
|
||||
self.assertIsNone(chat.reasoning_parser)
|
||||
self.assertEqual(self.tm.server_args.tool_call_parser, "auto")
|
||||
|
||||
def test_the_xgrammar_gate_follows_the_overlay(self):
|
||||
"""A detected `reasoning_parser` must gate xgrammar, not the seed's "auto"."""
|
||||
self.tm.server_args.reasoning_parser = "auto"
|
||||
self.tm._config_updates.append(
|
||||
("template-detection", {"reasoning_parser": "qwen3"})
|
||||
)
|
||||
chat = OpenAIServingChat(self.tm, self.template_manager)
|
||||
self.assertEqual(chat.reasoning_parser, "qwen3")
|
||||
# the gate reads the same value the parser was built from
|
||||
self.assertIsNotNone(chat.reasoning_parser)
|
||||
|
||||
def test_text_only_model_rejects_media_before_generation(self):
|
||||
media_parts = {
|
||||
"image_url": {
|
||||
|
||||
@@ -52,6 +52,7 @@ class MockTokenizerManager:
|
||||
tool_call_parser=None,
|
||||
incremental_streaming_output=False,
|
||||
)
|
||||
self._config_updates = []
|
||||
self.tokenizer = Mock()
|
||||
self.tokenizer.encode.return_value = [1, 2, 3]
|
||||
self.tokenizer.chat_template = None
|
||||
@@ -60,6 +61,13 @@ class MockTokenizerManager:
|
||||
self.generate_request = Mock()
|
||||
self.create_abort_task = Mock()
|
||||
|
||||
def config_value(self, name: str):
|
||||
"""The manager's overlay accessor: no control-plane update recorded."""
|
||||
for _source, fields in reversed(self._config_updates):
|
||||
if name in fields:
|
||||
return fields[name]
|
||||
return getattr(self.server_args, name)
|
||||
|
||||
|
||||
class MockTemplateManager:
|
||||
def __init__(self):
|
||||
|
||||
Reference in New Issue
Block a user