minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

config: one control-plane log for the process (#35028)

Browse Source
This commit is contained in:
Cheng Wan
2026-08-17 16:19:00 -07:00
committed by GitHub
parent c70c7d72a8
commit b3c8f0d923
8 changed files with 245 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/unit/entrypoints/openai/test_serving_chat.py
+11 -15
View File
@@ -75,7 +75,9 @@ class _MockTokenizerManager:
self.model_path = self.server_args.model_path
# The manager tracks the served name itself; a weight update rewrites it.
self.served_model_name = "test-model"
self._config_updates = []
# Stands in for the context's resolved leaves: an override replaces the
# field's one live value, the seed stays on server_args.
self._config_overrides = {}
# Mock hf_config for _resolve_chat_encoding_spec check
mock_hf_config = Mock()
@@ -114,10 +116,9 @@ class _MockTokenizerManager:
self.request_logger = Mock(log_requests=False, log_requests_level=0)
def config_value(self, name: str):
"""The manager's overlay accessor: no control-plane update recorded."""
for _source, fields in reversed(self._config_updates):
if name in fields:
return fields[name]
"""The value in effect for one config field."""
if name in self._config_overrides:
return self._config_overrides[name]
return getattr(self.server_args, name)
@@ -160,15 +161,12 @@ class ServingChatTestCase(unittest.TestCase):
self.fastapi_request.headers = {}
def test_parsers_follow_the_control_plane_overlay(self):
"""Template detection records the parsers on the manager, not on its
ServerArgs — the instance keeps what the launcher passed."""
"""Template detection records the parsers through `override`, so they
answer from the bags; `ServerArgs` keeps the launcher's seed."""
self.tm.server_args.tool_call_parser = "auto"
self.tm.server_args.reasoning_parser = "auto"
self.tm._config_updates.append(
(
"template-detection",
{"tool_call_parser": "qwen25", "reasoning_parser": None},
)
self.tm._config_overrides.update(
{"tool_call_parser": "qwen25", "reasoning_parser": None}
)
chat = OpenAIServingChat(self.tm, self.template_manager)
@@ -180,9 +178,7 @@ class ServingChatTestCase(unittest.TestCase):
def test_the_xgrammar_gate_follows_the_overlay(self):
"""A detected `reasoning_parser` must gate xgrammar, not the seed's "auto"."""
self.tm.server_args.reasoning_parser = "auto"
self.tm._config_updates.append(
("template-detection", {"reasoning_parser": "qwen3"})
)
self.tm._config_overrides["reasoning_parser"] = "qwen3"
chat = OpenAIServingChat(self.tm, self.template_manager)
self.assertEqual(chat.reasoning_parser, "qwen3")
# the gate reads the same value the parser was built from
test/registered/unit/entrypoints/openai/utils.py
+6 -5
View File
@@ -54,7 +54,9 @@ class MockTokenizerManager:
tool_call_parser=None,
incremental_streaming_output=False,
)
self._config_updates = []
# Stands in for the context's resolved leaves: an override replaces the
# field's one live value, the seed stays on server_args.
self._config_overrides = {}
self.tokenizer = Mock()
self.tokenizer.encode.return_value = [1, 2, 3]
self.tokenizer.chat_template = None
@@ -64,10 +66,9 @@ class MockTokenizerManager:
self.create_abort_task = Mock()
def config_value(self, name: str):
"""The manager's overlay accessor: no control-plane update recorded."""
for _source, fields in reversed(self._config_updates):
if name in fields:
return fields[name]
"""The value in effect for one config field."""
if name in self._config_overrides:
return self._config_overrides[name]
return getattr(self.server_args, name)
test/registered/unit/entrypoints/test_server_info.py
+1 -3
View File
@@ -43,15 +43,13 @@ def _stub_tokenizer_manager(
"""A manager carrying the state `/server_info` and its writers read.
`__new__` skips `__init__`, which would open the ZMQ sockets and start
the handle loop; `_config_updates` is the log `record_config_updates`
appends to.
the handle loop.
"""
tokenizer_manager = TokenizerManager.__new__(TokenizerManager)
tokenizer_manager.server_args = server_args
tokenizer_manager.model_path = server_args.model_path
tokenizer_manager.served_model_name = server_args.served_model_name
tokenizer_manager.startup_time = None
tokenizer_manager._config_updates = []
tokenizer_manager.get_internal_state = get_internal_state
return tokenizer_manager
test/registered/unit/managers/test_tokenizer_config_updates.py
+35 -21
View File
@@ -1,9 +1,9 @@
"""Control-plane config updates stay on the tokenizer manager.
"""Control-plane config updates go into the process log, not onto the record.
Regression: runtime updates (weight version, model path, HiCache attach) were
written onto the manager's ServerArgs instance so that the readback endpoints
would show them. The record stays pristine; the updates live in a separate log
that the endpoints overlay on top of it.
would show them. The record stays pristine; the update lands in the runtime
context, which is where a reader of any field asks for the value in effect.
"""
import re
@@ -12,6 +12,7 @@ from pathlib import Path
import sglang
from sglang.srt.managers.tokenizer_manager import TokenizerManager
from sglang.srt.runtime_context import get_context, publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -19,28 +20,31 @@ from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
def _manager(**fields):
def _manager(case, **fields):
"""A manager over a published config: the updates it records go to the bags."""
server_args = ServerArgs(model_path="dummy", **fields)
publish(server_args, role="tokenizer")
case.addCleanup(reset_context)
manager = TokenizerManager.__new__(TokenizerManager)
manager.server_args = ServerArgs(model_path="dummy", **fields)
manager._config_updates = []
manager.server_args = server_args
return manager
class TestTokenizerConfigUpdates(CustomTestCase):
def test_startup_config_shows_through_until_something_updates_it(self):
manager = _manager(weight_version="v1")
manager = _manager(self, weight_version="v1")
self.assertEqual(manager.config_value("weight_version"), "v1")
manager.record_config_updates("test", weight_version="v2")
self.assertEqual(manager.config_value("weight_version"), "v2")
def test_the_serverargs_instance_is_not_written(self):
manager = _manager(weight_version="v1")
manager = _manager(self, weight_version="v1")
manager.record_config_updates("test", weight_version="v2")
self.assertEqual(manager.server_args.weight_version, "v1")
def test_the_readback_dict_carries_the_updates(self):
manager = _manager(hicache_storage_backend=None)
manager = _manager(self, hicache_storage_backend=None)
manager.record_config_updates(
"test", hicache_storage_backend="file", hicache_write_policy="write_through"
)
@@ -54,7 +58,7 @@ class TestTokenizerConfigUpdates(CustomTestCase):
self.assertEqual(resolved["model_path"], "dummy")
def test_detach_reports_the_backend_as_gone(self):
manager = _manager(hicache_storage_backend="file")
manager = _manager(self, hicache_storage_backend="file")
manager.record_config_updates(
"test",
hicache_storage_backend=None,
@@ -63,22 +67,27 @@ class TestTokenizerConfigUpdates(CustomTestCase):
self.assertIsNone(manager.config_value("hicache_storage_backend"))
def test_an_unknown_field_is_refused(self):
manager = _manager()
with self.assertRaisesRegex(ValueError, "not ServerArgs fields"):
manager = _manager(self)
with self.assertRaisesRegex(ValueError, "not a resolved config leaf"):
manager.record_config_updates("test", waight_version="v2")
def test_a_name_that_is_not_a_config_leaf_is_refused(self):
manager = _manager(self)
with self.assertRaisesRegex(ValueError, "not a config leaf"):
manager.config_value("waight_version")
def test_the_source_is_kept_for_provenance(self):
manager = _manager(weight_version="v1")
manager = _manager(self, weight_version="v1")
manager.record_config_updates("http.update_weight_version", weight_version="v2")
self.assertEqual(
manager._config_updates,
get_context().overrides_log(),
[("http.update_weight_version", {"weight_version": "v2"})],
)
def test_the_dump_snapshot_identifies_the_running_checkpoint(self):
import dataclasses
manager = _manager(load_format="auto")
manager = _manager(self, load_format="auto")
manager.model_path = "at-startup"
manager.served_model_name = "at-startup"
manager._update_model_path_info("after-reload", "dummy")
@@ -87,6 +96,10 @@ class TestTokenizerConfigUpdates(CustomTestCase):
self.assertEqual(snapshot["model_path"], "after-reload")
self.assertEqual(snapshot["served_model_name"], "after-reload")
self.assertEqual(snapshot["load_format"], "dummy")
self.assertEqual(
get_context().overrides_log(),
[("tokenizer.update_weights", {"load_format": "dummy"})],
)
self.assertEqual(manager.server_args.model_path, "dummy")
def test_an_unsnapshotable_config_does_not_lose_the_dump(self):
@@ -94,7 +107,7 @@ class TestTokenizerConfigUpdates(CustomTestCase):
def __deepcopy__(self, memo):
raise RuntimeError("refuses to be copied")
manager = _manager()
manager = _manager(self)
manager.model_path = "dummy"
manager.served_model_name = "dummy"
manager.server_args.custom_sigquit_handler = Hostile()
@@ -105,7 +118,7 @@ class TestTokenizerConfigUpdates(CustomTestCase):
import dataclasses
import pickle
manager = _manager()
manager = _manager(self)
manager.model_path = "dummy"
manager.served_model_name = "dummy"
# What --custom-sigquit-handler leaves on a real ServerArgs.
@@ -113,7 +126,7 @@ class TestTokenizerConfigUpdates(CustomTestCase):
payload = {
"server_args": manager.server_args,
"config_updates": list(manager._config_updates),
"config_updates": get_context().overrides_log(),
"resolved_config": manager.resolved_config_dict(
dataclasses.asdict(manager.server_args)
),
@@ -128,7 +141,7 @@ class TestTokenizerConfigUpdates(CustomTestCase):
self.assertTrue(pickle.dumps(payload))
def test_the_model_path_readback_follows_the_manager(self):
manager = _manager()
manager = _manager(self)
manager.model_path = "after-update"
manager.served_model_name = "after-update"
resolved = manager.resolved_config_dict({"model_path": "at-startup"})
@@ -200,8 +213,9 @@ class TestControlPlaneFieldsAreNotReadFromTheInstance(CustomTestCase):
self.assertEqual(
stale,
[],
"control-plane fields change at runtime and the update lives on the "
"TokenizerManager; read them with config_value() / "
"control-plane fields change at runtime and the update lands in "
"the process bags (model_path / served_model_name stay manager "
"attributes); read them with config_value() / "
"resolved_config_dict() so the readback reflects the change:\n"
+ "\n".join(stale),
)
test/registered/unit/test_supplied_instance_exposure_ratchet.py
+72 -16
View File
@@ -367,6 +367,8 @@ _EXPOSED_CUDA_ONLY: frozenset = frozenset()
# some code overrides post-publish. Each needs an ordering judgment, not a blanket
# conversion; the list exists so a new one is a decision made when it is written.
_OVERRIDDEN_AND_READ = {
("entrypoints/engine.py", "reasoning_parser"),
("entrypoints/engine.py", "tool_call_parser"),
("configs/model_config.py", "dtype"),
("configs/model_config.py", "model_path"),
("disaggregation/decode_kvcache_offload_manager.py", "hicache_storage_backend"),
@@ -442,11 +444,53 @@ def _expanded_override_keys(rel, tree, call, kw) -> set:
):
return set()
def loop_variable_values(name: str) -> set:
"""The values a `for name, ... in (<literal tuples>)` loop binds.
A handler that records one field per loop iteration spells the field
names in the loop's own literal, so they are still static.
"""
values = set()
for node in ast.walk(tree):
if not isinstance(node, ast.For):
continue
target = node.target
names = (
[target]
if isinstance(target, ast.Name)
else list(getattr(target, "elts", []))
)
if not names or not isinstance(names[0], ast.Name) or names[0].id != name:
continue
if not (node.lineno <= call.lineno <= (node.end_lineno or node.lineno)):
continue
for item in getattr(node.iter, "elts", []):
first = (
item.elts[0] if isinstance(item, ast.Tuple) and item.elts else item
)
if isinstance(first, ast.Constant) and isinstance(first.value, str):
values.add(first.value)
return values
def dict_keys(node) -> set:
assert isinstance(node, ast.Dict) and all(
isinstance(key, ast.Constant) for key in node.keys
assert isinstance(
node, ast.Dict
), f"non-literal dict in override expansion at {rel}:{call.lineno}"
return {key.value for key in node.keys}
keys = set()
for key in node.keys:
if isinstance(key, ast.Constant):
keys.add(key.value)
continue
assert isinstance(
key, ast.Name
), f"non-literal dict key in override expansion at {rel}:{call.lineno}"
bound = loop_variable_values(key.id)
assert bound, (
f"dict key {key.id!r} at {rel}:{call.lineno} is not bound by a "
"literal loop; extend the resolver"
)
keys |= bound
return keys
if isinstance(kw.value, ast.Dict):
return dict_keys(kw.value)
@@ -941,25 +985,37 @@ class TestSuppliedInstanceExposure(CustomTestCase):
raise AssertionError(f"unparsable module in the census: {rel}")
for node in ast.walk(tree):
if not (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "override"
isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)
):
continue
base = node.func.value
if (
is_override = node.func.attr == "override" and (
isinstance(base, ast.Call)
and isinstance(base.func, ast.Name)
and base.func.id == "get_context"
):
for kw in node.keywords:
if kw.arg == "source":
# Override metadata, not a config field.
continue
if kw.arg:
written.add(kw.arg)
else:
written |= _expanded_override_keys(rel, tree, node, kw)
)
# `record_config_updates` is a named wrapper over override, so
# its call sites are override sites. Its body forwards **kwargs
# and names no field, so skip the forwarding call itself.
is_wrapper = node.func.attr == "record_config_updates"
if not (is_override or is_wrapper):
continue
inside_wrapper = any(
isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef))
and fn.name == "record_config_updates"
and fn.lineno <= node.lineno <= (fn.end_lineno or fn.lineno)
for fn in ast.walk(tree)
)
if inside_wrapper:
continue
for kw in node.keywords:
if kw.arg == "source":
# Override metadata, not a config field.
continue
if kw.arg:
written.add(kw.arg)
else:
written |= _expanded_override_keys(rel, tree, node, kw)
return written
def test_the_post_publish_override_surface_matches_the_pinned_list(self):