Files
sglang/test/registered/unit/managers/test_tokenizer_config_updates.py
T

229 lines
8.8 KiB
Python

"""Control-plane config updates go into the process log, not onto the record.
Regression: runtime updates (weight version, model path, HiCache attach) were
written onto the manager's ServerArgs instance so that the readback endpoints
would show them. The record stays pristine; the update lands in the runtime
context, which is where a reader of any field asks for the value in effect.
"""
import re
import unittest
from pathlib import Path
import msgspec
import msgspec.structs
import sglang
from sglang.srt.managers.tokenizer_manager import TokenizerManager
from sglang.srt.runtime_context import get_context, publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=12, suite="base-a-test-cpu")
def _manager(case, **fields):
"""A manager over a published config: the updates it records go to the bags."""
server_args = ServerArgs(model_path="dummy", **fields)
publish(server_args, role="tokenizer")
case.addCleanup(reset_context)
manager = TokenizerManager.__new__(TokenizerManager)
manager.server_args = server_args
return manager
class TestTokenizerConfigUpdates(CustomTestCase):
def test_startup_config_shows_through_until_something_updates_it(self):
manager = _manager(self, weight_version="v1")
self.assertEqual(manager.config_value("weight_version"), "v1")
manager.record_config_updates("test", weight_version="v2")
self.assertEqual(manager.config_value("weight_version"), "v2")
def test_the_serverargs_instance_is_not_written(self):
manager = _manager(self, weight_version="v1")
manager.record_config_updates("test", weight_version="v2")
self.assertEqual(manager.server_args.weight_version, "v1")
def test_the_readback_dict_carries_the_updates(self):
manager = _manager(self, hicache_storage_backend=None)
manager.record_config_updates(
"test", hicache_storage_backend="file", hicache_write_policy="write_through"
)
manager.model_path = "dummy"
manager.served_model_name = "dummy"
resolved = manager.resolved_config_dict(
{"hicache_storage_backend": None, "model_path": "dummy"}
)
self.assertEqual(resolved["hicache_storage_backend"], "file")
self.assertEqual(resolved["hicache_write_policy"], "write_through")
self.assertEqual(resolved["model_path"], "dummy")
def test_detach_reports_the_backend_as_gone(self):
manager = _manager(self, hicache_storage_backend="file")
manager.record_config_updates(
"test",
hicache_storage_backend=None,
hicache_storage_backend_extra_config=None,
)
self.assertIsNone(manager.config_value("hicache_storage_backend"))
def test_an_unknown_field_is_refused(self):
manager = _manager(self)
with self.assertRaisesRegex(ValueError, "not a resolved config leaf"):
manager.record_config_updates("test", waight_version="v2")
def test_a_name_that_is_not_a_config_leaf_is_refused(self):
manager = _manager(self)
with self.assertRaisesRegex(ValueError, "not a config leaf"):
manager.config_value("waight_version")
def test_the_source_is_kept_for_provenance(self):
manager = _manager(self, weight_version="v1")
manager.record_config_updates("http.update_weight_version", weight_version="v2")
self.assertEqual(
get_context().overrides_log(),
[("http.update_weight_version", {"weight_version": "v2"})],
)
def test_the_dump_snapshot_identifies_the_running_checkpoint(self):
manager = _manager(self, load_format="auto")
manager.model_path = "at-startup"
manager.served_model_name = "at-startup"
manager._update_model_path_info("after-reload", "dummy")
snapshot = manager.resolved_config_dict(
msgspec.structs.asdict(manager.server_args)
)
self.assertEqual(snapshot["model_path"], "after-reload")
self.assertEqual(snapshot["served_model_name"], "after-reload")
self.assertEqual(snapshot["load_format"], "dummy")
self.assertEqual(
get_context().overrides_log(),
[("tokenizer.update_weights", {"load_format": "dummy"})],
)
self.assertEqual(manager.server_args.model_path, "dummy")
def test_an_unsnapshotable_config_does_not_lose_the_dump(self):
class Hostile:
def __deepcopy__(self, memo):
raise RuntimeError("refuses to be copied")
# Through the constructor: the field is raw input, and a resolved
# record refuses to be written.
manager = _manager(self, custom_sigquit_handler=Hostile())
manager.model_path = "dummy"
manager.served_model_name = "dummy"
self.assertIsNone(manager._dump_config_snapshot())
def test_an_unpickleable_field_does_not_lose_the_dump(self):
import pickle
# What --custom-sigquit-handler leaves on a real ServerArgs.
manager = _manager(self, custom_sigquit_handler=lambda *_: None)
manager.model_path = "dummy"
manager.served_model_name = "dummy"
payload = {
"server_args": manager.server_args,
"config_updates": get_context().overrides_log(),
"resolved_config": manager.resolved_config_dict(
msgspec.structs.asdict(manager.server_args)
),
"requests": [],
}
with self.assertRaises(Exception):
pickle.dumps(payload)
# The fallback drops both copies of the offending object, not just one.
payload["server_args"] = None
payload["resolved_config"] = None
self.assertTrue(pickle.dumps(payload))
def test_the_model_path_readback_follows_the_manager(self):
manager = _manager(self)
manager.model_path = "after-update"
manager.served_model_name = "after-update"
resolved = manager.resolved_config_dict({"model_path": "at-startup"})
self.assertEqual(resolved["model_path"], "after-update")
self.assertEqual(resolved["served_model_name"], "after-update")
CONTROL_PLANE_FIELDS = (
"weight_version",
"model_path",
"served_model_name",
"load_format",
"hicache_storage_backend",
"hicache_storage_backend_extra_config",
"hicache_storage_prefetch_policy",
"hicache_write_policy",
)
# Modules that answer readbacks or fill responses; the tokenizer manager's own
# __init__ seeds attributes from the constructor argument, which is not a
# readback and not matched by the patterns below. Prometheus label sets are
# exempt: a label must stay fixed for the lifetime of the series, so the metrics
# collector keeps the name the server started with.
EXEMPT_LINES = (
(
"srt/managers/tokenizer_manager.py",
'"model_name": self.server_args.served_model_name',
),
)
READBACK_MODULES = (
"srt/managers/tokenizer_manager.py",
"srt/managers/tokenizer_control_mixin.py",
"srt/managers/multi_tokenizer_mixin.py",
"srt/entrypoints/http_server.py",
"srt/entrypoints/grpc_bridge.py",
"srt/entrypoints/engine.py",
"srt/entrypoints/openai",
)
class TestControlPlaneFieldsAreNotReadFromTheInstance(CustomTestCase):
def test_readbacks_go_through_the_manager(self):
root = Path(next(iter(sglang.__path__)))
patterns = [
re.compile(
rf"self\.server_args\.{f}\b|tokenizer_manager\.server_args\.{f}\b"
)
for f in CONTROL_PLANE_FIELDS
]
stale = []
for rel in READBACK_MODULES:
paths = (
sorted((root / rel).rglob("*.py"))
if (root / rel).is_dir()
else [root / rel]
)
for path in paths:
for number, line in enumerate(path.read_text().split("\n"), 1):
if any(
rel_exempt == path.relative_to(root).as_posix()
and needle in line
for rel_exempt, needle in EXEMPT_LINES
):
continue
if any(p.search(line) for p in patterns):
stale.append(
f"{path.relative_to(root)}:{number}: {line.strip()}"
)
self.assertEqual(
stale,
[],
"control-plane fields change at runtime and the update lands in "
"the process bags (model_path / served_model_name stay manager "
"attributes); read them with config_value() / "
"resolved_config_dict() so the readback reflects the change:\n"
+ "\n".join(stale),
)
if __name__ == "__main__":
unittest.main()