diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index c8e785bca..0a512f6bc 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -1271,7 +1271,9 @@ class Engine(EngineScoreMixin, EngineBase): ) return msgspec_to_builtins( { - **dataclasses.asdict(self.tokenizer_manager.server_args), + **self.tokenizer_manager.resolved_config_dict( + dataclasses.asdict(self.tokenizer_manager.server_args) + ), **self._scheduler_init_result.scheduler_infos[0], "internal_states": internal_states, "version": __version__, diff --git a/python/sglang/srt/entrypoints/grpc_bridge.py b/python/sglang/srt/entrypoints/grpc_bridge.py index f2adfb8f8..dfaf2840f 100644 --- a/python/sglang/srt/entrypoints/grpc_bridge.py +++ b/python/sglang/srt/entrypoints/grpc_bridge.py @@ -379,7 +379,7 @@ class RuntimeHandle: "model_path": self.tokenizer_manager.model_path, "tokenizer_path": self.tokenizer_manager.server_args.tokenizer_path, "is_generation": self.tokenizer_manager.is_generation, - "weight_version": self.tokenizer_manager.server_args.weight_version, + "weight_version": self.tokenizer_manager.config_value("weight_version"), "model_type": getattr(model_config.hf_config, "model_type", None), "architectures": getattr(model_config.hf_config, "architectures", None), } @@ -393,7 +393,9 @@ class RuntimeHandle: return json.dumps(result, default=str) def get_server_info(self) -> str: - result: Dict[str, Any] = dataclasses.asdict(self.server_args) + result: Dict[str, Any] = self.tokenizer_manager.resolved_config_dict( + dataclasses.asdict(self.tokenizer_manager.server_args) + ) result.update(self.scheduler_info) return json.dumps(msgspec_to_builtins(result), default=str) diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index ed4ffb817..5962f1edc 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -710,12 +710,13 @@ async def model_info(): "tokenizer_path": _global_state.tokenizer_manager.server_args.tokenizer_path, "is_generation": _global_state.tokenizer_manager.is_generation, "preferred_sampling_params": _global_state.tokenizer_manager.server_args.preferred_sampling_params, - "weight_version": _global_state.tokenizer_manager.server_args.weight_version, + "weight_version": _global_state.tokenizer_manager.config_value( + "weight_version" + ), "has_image_understanding": model_config.is_image_understandable_model, "has_audio_understanding": model_config.is_audio_understandable_model, "model_type": getattr(model_config.hf_config, "model_type", None), "architectures": getattr(model_config.hf_config, "architectures", None), - "weight_version": _global_state.tokenizer_manager.server_args.weight_version, # "hf_config": model_config.hf_config.to_dict(), } embedding_model_spec = getattr(model_config, "embedding_model_spec", None) @@ -761,7 +762,9 @@ async def server_info(): # server_args.model_config is not serializable but should be excluded by asdict. return msgspec_to_builtins( { - **dataclasses.asdict(server_args), + **_global_state.tokenizer_manager.resolved_config_dict( + dataclasses.asdict(server_args) + ), **_global_state.scheduler_info, "internal_states": internal_states, "version": __version__, @@ -1091,10 +1094,13 @@ async def hicache_storage_backend_status(): return _admin_api_key_missing_response() return { - "hicache_storage_backend": _global_state.tokenizer_manager.server_args.hicache_storage_backend, - "hicache_storage_backend_extra_config": _global_state.tokenizer_manager.server_args.hicache_storage_backend_extra_config, - "hicache_storage_prefetch_policy": _global_state.tokenizer_manager.server_args.hicache_storage_prefetch_policy, - "hicache_write_policy": _global_state.tokenizer_manager.server_args.hicache_write_policy, + name: _global_state.tokenizer_manager.config_value(name) + for name in ( + "hicache_storage_backend", + "hicache_storage_backend_extra_config", + "hicache_storage_prefetch_policy", + "hicache_write_policy", + ) } @@ -1385,8 +1391,7 @@ async def update_weight_version( # Use a simple approach without the complex lock mechanism for now # since weight_version update is a simple operation that doesn't affect model weights try: - # Update the weight version in server args (the single source of truth) - _global_state.tokenizer_manager.server_args.override( + _global_state.tokenizer_manager.record_config_updates( "http.update_weight_version", weight_version=obj.new_version ) diff --git a/python/sglang/srt/entrypoints/openai/realtime/session.py b/python/sglang/srt/entrypoints/openai/realtime/session.py index c5951993e..f7a52b08f 100644 --- a/python/sglang/srt/entrypoints/openai/realtime/session.py +++ b/python/sglang/srt/entrypoints/openai/realtime/session.py @@ -338,12 +338,12 @@ class RealtimeConnection: if ( transcription is not None and transcription.model - and transcription.model != self.server_args.served_model_name + and transcription.model != self.tokenizer_manager.served_model_name ): await self._send_error( "not_supported", f"Model {transcription.model!r} is not served by this endpoint " - f"(serving {self.server_args.served_model_name!r}); set " + f"(serving {self.tokenizer_manager.served_model_name!r}); set " f"transcription.model to null or to the server's model name.", param="session.audio.input.transcription.model", ) diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index b12615e60..cb0256d0d 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -1279,7 +1279,7 @@ class OpenAIServingChat(OpenAIServingBase): logger.warning( "Model '%s' supports only 'low' reasoning effort; " "requested '%s' treated as default thinking", - self.tokenizer_manager.server_args.served_model_name, + self.tokenizer_manager.served_model_name, request.reasoning_effort, ) diff --git a/python/sglang/srt/entrypoints/openai/serving_classify.py b/python/sglang/srt/entrypoints/openai/serving_classify.py index c5feff867..da80bc90a 100644 --- a/python/sglang/srt/entrypoints/openai/serving_classify.py +++ b/python/sglang/srt/entrypoints/openai/serving_classify.py @@ -39,7 +39,7 @@ class OpenAIServingClassify(OpenAIServingBase): self.model_name = ( self.tokenizer_manager.served_model_name if self.tokenizer_manager.served_model_name - else self.tokenizer_manager.server_args.model_path + else self.tokenizer_manager.model_path ) if not self.id2label: raise ValueError("id2label mapping is missing") diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 09e838e57..69705dc72 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -4009,7 +4009,7 @@ class Scheduler( ) if recv_req.hicache_write_policy is not None: hicache_fields["hicache_write_policy"] = recv_req.hicache_write_policy - self.server_args.override("scheduler.attach_hicache", **hicache_fields) + get_context().override("scheduler.attach_hicache", **hicache_fields) logger.info( f"Attached HiCache storage backend: {recv_req.hicache_storage_backend}" ) @@ -4050,7 +4050,7 @@ class Scheduler( if ok or (not self.enable_hicache_storage): # Treat "already disabled / nothing to do" as success for idempotence. self.enable_hicache_storage = False - self.server_args.override( + get_context().override( "scheduler.detach_hicache", hicache_storage_backend=None, hicache_storage_backend_extra_config=None, diff --git a/python/sglang/srt/managers/tokenizer_control_mixin.py b/python/sglang/srt/managers/tokenizer_control_mixin.py index 5cca6e1b9..7cfd98b8f 100644 --- a/python/sglang/srt/managers/tokenizer_control_mixin.py +++ b/python/sglang/srt/managers/tokenizer_control_mixin.py @@ -343,7 +343,7 @@ class TokenizerControlMixin: ) if hicache_write_policy is not None: hicache_fields["hicache_write_policy"] = hicache_write_policy - self.server_args.override("tokenizer.attach_hicache", **hicache_fields) + self.record_config_updates("tokenizer.attach_hicache", **hicache_fields) return out async def detach_hicache_storage( @@ -359,7 +359,7 @@ class TokenizerControlMixin: out = DetachHiCacheStorageReqOutput(success=all_success, message=all_message) # TODO: partial rollback if failed if all_success: - self.server_args.override( + self.record_config_updates( "tokenizer.detach_hicache", hicache_storage_backend=None, hicache_storage_backend_extra_config=None, @@ -920,6 +920,6 @@ class TokenizerControlMixin: ) -> None: """Update weight version if provided.""" if weight_version is not None: - self.server_args.override( + self.record_config_updates( "tokenizer.weight_version", weight_version=weight_version ) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 4a80ce938..f4516b290 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -364,6 +364,11 @@ class InputFormat(Enum): CROSS_ENCODER_PAIRS = 3 # Cross-encoder pairs like [["query", "document"]] +_SERVER_ARGS_FIELDS = frozenset(f.name for f in dataclasses.fields(ServerArgs)) + +_MANAGER_OWNED_FIELDS = ("model_path", "served_model_name") + + class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): """TokenizerManager is a process that tokenizes the text.""" @@ -386,6 +391,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): ): # Parse args self.server_args = server_args + self._config_updates: List[Tuple[str, Dict[str, Any]]] = [] self.elastic_worker_count = server_args.dp_size self.elastic_pending_ep_size = None self.elastic_scale_phase = "idle" @@ -1888,9 +1894,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): ) -> Tuple[bool, str]: self.auto_create_handle_loop() - # default the load format to the server_args if obj.load_format is None: - obj.load_format = self.server_args.load_format + obj.load_format = self.config_value("load_format") logger.info("Start update_weights. Load format=%s", obj.load_format) if obj.abort_all_requests: @@ -1914,11 +1919,57 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): return success, message, num_paused_requests + def record_config_updates(self, source: str, **fields) -> None: + """Record a control-plane config change for this engine. + + Per-engine state: several ``Engine``s can share one tokenizer process. + The readback endpoints overlay these onto the startup config. The + process-global sibling is ``RuntimeContext.override`` / + ``resolved_server_args_dict``, which writes the config bags every + process shares. + """ + unknown = sorted(f for f in fields if f not in _SERVER_ARGS_FIELDS) + if unknown: + raise ValueError( + f"{unknown} are not ServerArgs fields; the readback endpoints " + "overlay these onto a serialized ServerArgs, so an unknown key " + "would surface as a phantom config entry." + ) + self._config_updates.append((source, dict(fields))) + + def config_value(self, name: str): + """The value in effect for one config field, control-plane updates first.""" + if name in _MANAGER_OWNED_FIELDS: + return getattr(self, name) + for _source, fields in reversed(self._config_updates): + if name in fields: + return fields[name] + return getattr(self.server_args, name) + + def _dump_config_snapshot(self) -> Optional[Dict[str, Any]]: + """The config in effect, or None when it cannot be serialized. + + A dump is worth having even when the config is not: request data is the + part that cannot be reconstructed afterwards. + """ + try: + return self.resolved_config_dict(dataclasses.asdict(self.server_args)) + except Exception as e: + logger.error(f"Failed to snapshot the resolved config for the dump: {e!r}") + return None + + def resolved_config_dict(self, base: Dict[str, Any]) -> Dict[str, Any]: + """``base`` (a serialized ``ServerArgs``) with the control-plane updates on top.""" + resolved = dict(base) + for _source, fields in self._config_updates: + resolved.update(fields) + for name in _MANAGER_OWNED_FIELDS: + resolved[name] = getattr(self, name) + return resolved + def _update_model_path_info(self, model_path: str, load_format: str): self.served_model_name = model_path - self.server_args.override( - "tokenizer.update_weights", model_path=model_path, load_format=load_format - ) + self.record_config_updates("tokenizer.update_weights", load_format=load_format) self.model_path = model_path async def _wait_for_model_update_from_disk( @@ -2060,7 +2111,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): "id": rid, "finish_reason": recv_obj.finished_reasons[i], "prompt_tokens": recv_obj.prompt_tokens[i], - "weight_version": self.server_args.weight_version, + "weight_version": self.config_value("weight_version"), "num_retractions": recv_obj.retraction_counts[i], } @@ -2789,6 +2840,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): logger.info(log_message) to_dump_with_server_args = { "server_args": self.server_args, + "config_updates": list(self._config_updates), + "resolved_config": self._dump_config_snapshot(), "requests": data_list.copy(), } @@ -2808,6 +2861,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): f.seek(0) f.truncate() to_dump_with_server_args["server_args"] = None + # The snapshot copies the same object field by field. + to_dump_with_server_args["resolved_config"] = None pickle.dump(to_dump_with_server_args, f) asyncio.create_task(asyncio.to_thread(background_task)) @@ -2870,6 +2925,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): # Write the data to the file data_to_dump_with_server_args = { "server_args": self.server_args, + "config_updates": list(self._config_updates), + "resolved_config": self._dump_config_snapshot(), "requests": data_to_dump, "launch_command": " ".join(sys.argv), } @@ -2887,6 +2944,8 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): f.seek(0) f.truncate() data_to_dump_with_server_args["server_args"] = None + # The snapshot copies the same object field by field. + data_to_dump_with_server_args["resolved_config"] = None pickle.dump(data_to_dump_with_server_args, f) logger.error( f"Dumped {len(self.crash_dump_request_list)} finished and {len(unfinished_requests)} unfinished requests before crash to {filename}" @@ -2999,7 +3058,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): meta_info = { "id": recv_obj.rid, "finish_reason": finish_reason, - "weight_version": self.server_args.weight_version, + "weight_version": self.config_value("weight_version"), "e2e_latency": state.time_stats.get_e2e_latency(), } is_stream = getattr(state.obj, "stream", False) diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index 617d96489..33cd4780b 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -912,17 +912,23 @@ class RuntimeContext: """Serialize the *resolved* config: the pristine ``server_args`` fields with every post-publish ``override`` overlaid. - Reporting endpoints (``/server_info``, ``get_internal_state``) surface - the config the process is *currently* running, not the startup record, - so they read this rather than serializing ``server_args`` directly — - otherwise runtime updates (weight version, model path, tunables set via - ``/set_internal_state``) never show up in the readback. + ``get_internal_state`` reports this, and ``/server_info`` carries it in + the ``internal_states`` block, so scheduler-side runtime changes show up + in a readback: HiCache attach/detach, the generated forward-pass-metrics + endpoint, tunables set via ``/set_internal_state``. ``base`` defaults to ``dict(vars(server_args))`` (matching the legacy ``vars`` dump); pass ``dataclasses.asdict(server_args)`` when nested - dataclass fields must be expanded first (``/server_info``). Override - leaves are flat ``ServerArgs`` field names, so overlaying them onto the - top level of either base is exact. + dataclass fields must be expanded first. Override leaves are flat + ``ServerArgs`` field names, so overlaying them onto the top level of + either base is exact. + + This covers the process-global bags only. Per-engine control-plane + changes (weight version, model path, the tokenizer's HiCache mirror) + live on the tokenizer manager — several ``Engine``s can share one + process — and ``TokenizerManager.resolved_config_dict`` overlays those + for the top-level ``/server_info`` body. The two are separate logs, not + one merged dict. """ d = dict(vars(self.server_args)) if base is None else dict(base) for _source, fields in self._overrides_log: diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index 234ee19fb..2496f7def 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -48,6 +48,9 @@ class _MockTokenizerManager: stream_response_default_include_usage=False, default_chat_template_kwargs=None, ) + # The manager tracks the served name itself; a weight update rewrites it. + self.served_model_name = "test-model" + # Mock hf_config for _resolve_chat_encoding_spec check mock_hf_config = Mock() mock_hf_config.architectures = ["LlamaForCausalLM"] diff --git a/test/registered/unit/entrypoints/test_server_info.py b/test/registered/unit/entrypoints/test_server_info.py index 3f348dc78..393025d65 100644 --- a/test/registered/unit/entrypoints/test_server_info.py +++ b/test/registered/unit/entrypoints/test_server_info.py @@ -28,6 +28,7 @@ from types import SimpleNamespace from sglang.srt.entrypoints import http_server from sglang.srt.lora.lora_registry import LoRARef +from sglang.srt.managers.tokenizer_manager import TokenizerManager from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -36,7 +37,9 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu") def _call_server_info_with( - server_args: ServerArgs, internal_states: list[dict] | None = None + server_args: ServerArgs, + internal_states: list[dict] | None = None, + config_updates: dict | None = None, ) -> dict: """Invoke `http_server.server_info()` against a stub global state. @@ -50,11 +53,16 @@ def _call_server_info_with( async def _fake_internal_state(): return internal_states or [{"max_req_input_len": 1024}] + tokenizer_manager = TokenizerManager.__new__(TokenizerManager) + tokenizer_manager.server_args = server_args + tokenizer_manager.model_path = server_args.model_path + tokenizer_manager.served_model_name = server_args.served_model_name + tokenizer_manager._config_updates = ( + [("test", dict(config_updates))] if config_updates else [] + ) + tokenizer_manager.get_internal_state = _fake_internal_state stub_state = SimpleNamespace( - tokenizer_manager=SimpleNamespace( - server_args=server_args, - get_internal_state=_fake_internal_state, - ), + tokenizer_manager=tokenizer_manager, scheduler_info={"max_req_input_len": 1024}, ) prior_state = http_server.get_global_state() @@ -233,6 +241,18 @@ class TestServerInfoKvEventsField(CustomTestCase): self.assertIsNone(info["kv_events"]) +class TestServerInfoControlPlaneUpdates(CustomTestCase): + """Runtime control-plane updates live on the manager, not on ServerArgs.""" + + def test_recorded_updates_win_over_the_startup_config(self): + server_args = ServerArgs(model_path="dummy", weight_version="v1") + payload = _call_server_info_with( + server_args, config_updates={"weight_version": "v2"} + ) + self.assertEqual(payload["weight_version"], "v2") + self.assertEqual(server_args.weight_version, "v1") + + class TestServerInfoExistingFieldsPreserved(CustomTestCase): """Regression guard: the new `kv_events` field is additive — none of the fields existing consumers depend on may be silently dropped. diff --git a/test/registered/unit/managers/test_scheduler_hicache_attach.py b/test/registered/unit/managers/test_scheduler_hicache_attach.py new file mode 100644 index 000000000..fda42d5aa --- /dev/null +++ b/test/registered/unit/managers/test_scheduler_hicache_attach.py @@ -0,0 +1,76 @@ +"""Runtime HiCache attach/detach lands on the config bags. + +The attach RPC used to mutate the scheduler's ServerArgs so the readback would +show the change; the namespace readers never saw it. Both now go through +get_context().override, so get_memory() and the resolved-config readback agree +and the published instance stays as the launcher left it. +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.managers.io_struct import ( + AttachHiCacheStorageReqInput, + DetachHiCacheStorageReqInput, +) +from sglang.srt.managers.scheduler import Scheduler +from sglang.srt.runtime_context import get_context, get_memory +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestSchedulerHiCacheAttach(CustomTestCase): + def _scheduler(self, **fields): + override = get_context().override_server_args( + enable_hierarchical_cache=True, **fields + ) + self.server_args = override.install() + self.addCleanup(override.restore) + + scheduler = Scheduler.__new__(Scheduler) + scheduler.server_args = self.server_args + scheduler.enable_hierarchical_cache = True + scheduler.enable_hicache_storage = False + scheduler.is_fully_idle = lambda: True + scheduler.tree_cache = SimpleNamespace( + attach_storage_backend=lambda **kwargs: (True, "attached"), + detach_storage_backend=lambda: (True, "detached"), + ) + return scheduler + + def test_attach_reaches_the_namespace_readers(self): + scheduler = self._scheduler(hicache_storage_backend=None) + out = scheduler.attach_hicache_storage_wrapped( + AttachHiCacheStorageReqInput( + hicache_storage_backend="file", + hicache_write_policy="write_through", + ) + ) + + self.assertTrue(out.success) + self.assertEqual(get_memory().hicache_storage_backend, "file") + self.assertEqual(get_memory().hicache_write_policy, "write_through") + self.assertEqual( + get_context().resolved_server_args_dict()["hicache_storage_backend"], + "file", + ) + self.assertIsNone(self.server_args.hicache_storage_backend) + + def test_detach_clears_the_backend_for_the_same_readers(self): + scheduler = self._scheduler(hicache_storage_backend="file") + scheduler.enable_hicache_storage = True + + out = scheduler.detach_hicache_storage_wrapped(DetachHiCacheStorageReqInput()) + + self.assertTrue(out.success) + self.assertIsNone(get_memory().hicache_storage_backend) + self.assertIsNone( + get_context().resolved_server_args_dict()["hicache_storage_backend"] + ) + self.assertEqual(self.server_args.hicache_storage_backend, "file") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/managers/test_tokenizer_config_updates.py b/test/registered/unit/managers/test_tokenizer_config_updates.py new file mode 100644 index 000000000..d4950280b --- /dev/null +++ b/test/registered/unit/managers/test_tokenizer_config_updates.py @@ -0,0 +1,216 @@ +"""Control-plane config updates stay on the tokenizer manager. + +Regression: runtime updates (weight version, model path, HiCache attach) were +written onto the manager's ServerArgs instance so that the readback endpoints +would show them. They are per-engine — several Engines can share a tokenizer +process — so they live on the manager and the endpoints overlay them. +""" + +import re +import unittest +from pathlib import Path + +import sglang +from sglang.srt.managers.tokenizer_manager import TokenizerManager +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _manager(**fields): + manager = TokenizerManager.__new__(TokenizerManager) + manager.server_args = ServerArgs(model_path="dummy", **fields) + manager._config_updates = [] + return manager + + +class TestTokenizerConfigUpdates(CustomTestCase): + def test_startup_config_shows_through_until_something_updates_it(self): + manager = _manager(weight_version="v1") + self.assertEqual(manager.config_value("weight_version"), "v1") + + manager.record_config_updates("test", weight_version="v2") + self.assertEqual(manager.config_value("weight_version"), "v2") + + def test_the_serverargs_instance_is_not_written(self): + manager = _manager(weight_version="v1") + manager.record_config_updates("test", weight_version="v2") + self.assertEqual(manager.server_args.weight_version, "v1") + + def test_two_engines_keep_their_own_updates(self): + first, second = _manager(weight_version="v1"), _manager(weight_version="v1") + first.record_config_updates("test", weight_version="v2") + self.assertEqual(second.config_value("weight_version"), "v1") + + def test_the_readback_dict_carries_the_updates(self): + manager = _manager(hicache_storage_backend=None) + manager.record_config_updates( + "test", hicache_storage_backend="file", hicache_write_policy="write_through" + ) + manager.model_path = "dummy" + manager.served_model_name = "dummy" + resolved = manager.resolved_config_dict( + {"hicache_storage_backend": None, "model_path": "dummy"} + ) + self.assertEqual(resolved["hicache_storage_backend"], "file") + self.assertEqual(resolved["hicache_write_policy"], "write_through") + self.assertEqual(resolved["model_path"], "dummy") + + def test_detach_reports_the_backend_as_gone(self): + manager = _manager(hicache_storage_backend="file") + manager.record_config_updates( + "test", + hicache_storage_backend=None, + hicache_storage_backend_extra_config=None, + ) + self.assertIsNone(manager.config_value("hicache_storage_backend")) + + def test_an_unknown_field_is_refused(self): + manager = _manager() + with self.assertRaisesRegex(ValueError, "not ServerArgs fields"): + manager.record_config_updates("test", waight_version="v2") + + def test_the_source_is_kept_for_provenance(self): + manager = _manager(weight_version="v1") + manager.record_config_updates("http.update_weight_version", weight_version="v2") + self.assertEqual( + manager._config_updates, + [("http.update_weight_version", {"weight_version": "v2"})], + ) + + def test_the_dump_snapshot_identifies_the_running_checkpoint(self): + import dataclasses + + manager = _manager(load_format="auto") + manager.model_path = "at-startup" + manager.served_model_name = "at-startup" + manager._update_model_path_info("after-reload", "dummy") + + snapshot = manager.resolved_config_dict(dataclasses.asdict(manager.server_args)) + self.assertEqual(snapshot["model_path"], "after-reload") + self.assertEqual(snapshot["served_model_name"], "after-reload") + self.assertEqual(snapshot["load_format"], "dummy") + self.assertEqual(manager.server_args.model_path, "dummy") + + def test_an_unsnapshotable_config_does_not_lose_the_dump(self): + class Hostile: + def __deepcopy__(self, memo): + raise RuntimeError("refuses to be copied") + + manager = _manager() + manager.model_path = "dummy" + manager.served_model_name = "dummy" + manager.server_args.custom_sigquit_handler = Hostile() + + self.assertIsNone(manager._dump_config_snapshot()) + + def test_an_unpickleable_field_does_not_lose_the_dump(self): + import dataclasses + import pickle + + manager = _manager() + manager.model_path = "dummy" + manager.served_model_name = "dummy" + # What --custom-sigquit-handler leaves on a real ServerArgs. + manager.server_args.custom_sigquit_handler = lambda *_: None + + payload = { + "server_args": manager.server_args, + "config_updates": list(manager._config_updates), + "resolved_config": manager.resolved_config_dict( + dataclasses.asdict(manager.server_args) + ), + "requests": [], + } + with self.assertRaises(Exception): + pickle.dumps(payload) + + # The fallback drops both copies of the offending object, not just one. + payload["server_args"] = None + payload["resolved_config"] = None + self.assertTrue(pickle.dumps(payload)) + + def test_the_model_path_readback_follows_the_manager(self): + manager = _manager() + manager.model_path = "after-update" + manager.served_model_name = "after-update" + resolved = manager.resolved_config_dict({"model_path": "at-startup"}) + self.assertEqual(resolved["model_path"], "after-update") + self.assertEqual(resolved["served_model_name"], "after-update") + + +CONTROL_PLANE_FIELDS = ( + "weight_version", + "model_path", + "served_model_name", + "load_format", + "hicache_storage_backend", + "hicache_storage_backend_extra_config", + "hicache_storage_prefetch_policy", + "hicache_write_policy", +) + +# Modules that answer readbacks or fill responses; the tokenizer manager's own +# __init__ seeds attributes from the constructor argument, which is not a +# readback and not matched by the patterns below. Prometheus label sets are +# exempt: a label must stay fixed for the lifetime of the series, so the metrics +# collector keeps the name the server started with. +EXEMPT_LINES = ( + ( + "srt/managers/tokenizer_manager.py", + '"model_name": self.server_args.served_model_name', + ), +) +READBACK_MODULES = ( + "srt/managers/tokenizer_manager.py", + "srt/managers/tokenizer_control_mixin.py", + "srt/managers/multi_tokenizer_mixin.py", + "srt/entrypoints/http_server.py", + "srt/entrypoints/grpc_bridge.py", + "srt/entrypoints/engine.py", + "srt/entrypoints/openai", +) + + +class TestControlPlaneFieldsAreNotReadFromTheInstance(CustomTestCase): + def test_readbacks_go_through_the_manager(self): + root = Path(next(iter(sglang.__path__))) + patterns = [ + re.compile( + rf"self\.server_args\.{f}\b|tokenizer_manager\.server_args\.{f}\b" + ) + for f in CONTROL_PLANE_FIELDS + ] + stale = [] + for rel in READBACK_MODULES: + paths = ( + sorted((root / rel).rglob("*.py")) + if (root / rel).is_dir() + else [root / rel] + ) + for path in paths: + for number, line in enumerate(path.read_text().split("\n"), 1): + if any( + rel_exempt == path.relative_to(root).as_posix() + and needle in line + for rel_exempt, needle in EXEMPT_LINES + ): + continue + if any(p.search(line) for p in patterns): + stale.append( + f"{path.relative_to(root)}:{number}: {line.strip()}" + ) + self.assertEqual( + stale, + [], + "control-plane fields change at runtime and the update lives on the " + "TokenizerManager; read them with config_value() / " + "resolved_config_dict() so the readback reflects the change:\n" + + "\n".join(stale), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py b/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py index 0adb79942..2926a22a5 100644 --- a/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py +++ b/test/registered/unit/managers/test_tokenizer_manager_rid_cleanup.py @@ -98,6 +98,7 @@ def _make_tokenizer_manager() -> TokenizerManager: """Create a TokenizerManager with mocked dependencies, bypassing __init__.""" tm = TokenizerManager.__new__(TokenizerManager) tm.server_args = MagicMock() + tm._config_updates = [] tm.server_args.enable_trace = False tm.server_args.enable_metrics = False tm.server_args.enable_lora = False diff --git a/test/registered/unit/test_server_args_writer_ratchet.py b/test/registered/unit/test_server_args_writer_ratchet.py index f8d26bc21..600341397 100644 --- a/test/registered/unit/test_server_args_writer_ratchet.py +++ b/test/registered/unit/test_server_args_writer_ratchet.py @@ -49,7 +49,7 @@ _EXCLUDED = ( "multimodal_gen", ) -_BASELINE = 26 +_BASELINE = 19 class TestServerArgsWriterRatchet(CustomTestCase):