config: keep runtime hicache and weight-version updates off ServerArgs (#33336)

The scheduler's runtime HiCache attach/detach wrote its own ServerArgs so the
internal-state readback would show the change; that readback already reports
the resolved config, so the writes become get_context().override(...) and the
namespace readers see them too.

The tokenizer side is per-engine — several Engines can share one process — so
its control-plane updates (weight version, model path + load format, HiCache
attach/detach) stay with the manager instead of moving to the process-global
bags. TokenizerManager gains record_config_updates / config_value /
resolved_config_dict, and the readbacks that used to observe the instance write
(/server_info, /model_info, the HiCache status endpoint, the gRPC bridge) now
overlay those updates onto the startup config.

test_server_info's stub grew the real manager instead of a SimpleNamespace, so
the overlay it now exercises cannot drift from production.

Writer ratchet 26 -> 19.
This commit is contained in:
Cheng Wan
2026-08-02 21:23:38 -07:00
committed by GitHub
parent 9bc8848fcf
commit 0b3e8bedd1
16 changed files with 432 additions and 42 deletions
@@ -48,6 +48,9 @@ class _MockTokenizerManager:
stream_response_default_include_usage=False,
default_chat_template_kwargs=None,
)
# The manager tracks the served name itself; a weight update rewrites it.
self.served_model_name = "test-model"
# Mock hf_config for _resolve_chat_encoding_spec check
mock_hf_config = Mock()
mock_hf_config.architectures = ["LlamaForCausalLM"]
@@ -28,6 +28,7 @@ from types import SimpleNamespace
from sglang.srt.entrypoints import http_server
from sglang.srt.lora.lora_registry import LoRARef
from sglang.srt.managers.tokenizer_manager import TokenizerManager
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -36,7 +37,9 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu")
def _call_server_info_with(
server_args: ServerArgs, internal_states: list[dict] | None = None
server_args: ServerArgs,
internal_states: list[dict] | None = None,
config_updates: dict | None = None,
) -> dict:
"""Invoke `http_server.server_info()` against a stub global state.
@@ -50,11 +53,16 @@ def _call_server_info_with(
async def _fake_internal_state():
return internal_states or [{"max_req_input_len": 1024}]
tokenizer_manager = TokenizerManager.__new__(TokenizerManager)
tokenizer_manager.server_args = server_args
tokenizer_manager.model_path = server_args.model_path
tokenizer_manager.served_model_name = server_args.served_model_name
tokenizer_manager._config_updates = (
[("test", dict(config_updates))] if config_updates else []
)
tokenizer_manager.get_internal_state = _fake_internal_state
stub_state = SimpleNamespace(
tokenizer_manager=SimpleNamespace(
server_args=server_args,
get_internal_state=_fake_internal_state,
),
tokenizer_manager=tokenizer_manager,
scheduler_info={"max_req_input_len": 1024},
)
prior_state = http_server.get_global_state()
@@ -233,6 +241,18 @@ class TestServerInfoKvEventsField(CustomTestCase):
self.assertIsNone(info["kv_events"])
class TestServerInfoControlPlaneUpdates(CustomTestCase):
"""Runtime control-plane updates live on the manager, not on ServerArgs."""
def test_recorded_updates_win_over_the_startup_config(self):
server_args = ServerArgs(model_path="dummy", weight_version="v1")
payload = _call_server_info_with(
server_args, config_updates={"weight_version": "v2"}
)
self.assertEqual(payload["weight_version"], "v2")
self.assertEqual(server_args.weight_version, "v1")
class TestServerInfoExistingFieldsPreserved(CustomTestCase):
"""Regression guard: the new `kv_events` field is additive — none of
the fields existing consumers depend on may be silently dropped.