minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

config: the readback and the resolving view say what they are (#35027)

Browse Source
This commit is contained in:
Cheng Wan
2026-08-17 16:18:19 -07:00
committed by GitHub
parent cba3c5d5ac
commit c70c7d72a8
12 changed files with 775 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/engine.py
+22 -3
View File
@@ -1364,9 +1364,7 @@ class Engine(EngineScoreMixin, EngineBase):
)
return msgspec_to_builtins(
{
**self.tokenizer_manager.resolved_config_dict(
dataclasses.asdict(self.tokenizer_manager.server_args)
),
**dataclasses.asdict(self.tokenizer_manager.server_args),
**self._scheduler_init_result.scheduler_infos[0],
"startup_time": self.tokenizer_manager.startup_time,
"internal_states": internal_states,
@@ -1374,6 +1372,27 @@ class Engine(EngineScoreMixin, EngineBase):
}
)
def get_model_info(self):
"""What this engine is serving right now.
`get_server_info` answers with the record: the launch configuration,
parsers included -- `auto` resolves into the record before the config
is published. This surface adds what the control plane changed after
publication: the model a weight update swapped in, its load format, an
operator-set weight version. The HTTP and gRPC model-info endpoints
answer with the same fields.
"""
tm = self.tokenizer_manager
return {
"model_path": tm.model_path,
"served_model_name": tm.served_model_name,
"is_generation": tm.is_generation,
"weight_version": tm.config_value("weight_version"),
"load_format": tm.config_value("load_format"),
"reasoning_parser": tm.config_value("reasoning_parser"),
"tool_call_parser": tm.config_value("tool_call_parser"),
}
def init_weights_update_group(
self,
master_address: str,
python/sglang/srt/entrypoints/grpc_bridge.py
+5 -3
View File
@@ -397,9 +397,13 @@ class RuntimeHandle:
model_config = self.tokenizer_manager.model_config
result = {
"model_path": self.tokenizer_manager.model_path,
"served_model_name": self.tokenizer_manager.served_model_name,
"tokenizer_path": self.tokenizer_manager.server_args.tokenizer_path,
"is_generation": self.tokenizer_manager.is_generation,
"weight_version": self.tokenizer_manager.config_value("weight_version"),
"load_format": self.tokenizer_manager.config_value("load_format"),
"reasoning_parser": self.tokenizer_manager.config_value("reasoning_parser"),
"tool_call_parser": self.tokenizer_manager.config_value("tool_call_parser"),
"model_type": getattr(model_config.hf_config, "model_type", None),
"architectures": getattr(model_config.hf_config, "architectures", None),
}
@@ -413,9 +417,7 @@ class RuntimeHandle:
return json.dumps(result, default=str)
def get_server_info(self) -> str:
result: Dict[str, Any] = self.tokenizer_manager.resolved_config_dict(
dataclasses.asdict(self.tokenizer_manager.server_args)
)
result: Dict[str, Any] = dataclasses.asdict(self.tokenizer_manager.server_args)
result.update(self.scheduler_info)
return json.dumps(msgspec_to_builtins(result), default=str)
python/sglang/srt/entrypoints/http_server.py
+19 -4
View File
@@ -741,12 +741,22 @@ async def model_info():
model_config = _global_state.tokenizer_manager.model_config
result = {
"model_path": _global_state.tokenizer_manager.model_path,
# Manager-owned, and moved by a weight update alongside `model_path`:
# this is where a client reads the identity the server answers under.
"served_model_name": _global_state.tokenizer_manager.served_model_name,
"tokenizer_path": _global_state.tokenizer_manager.server_args.tokenizer_path,
"is_generation": _global_state.tokenizer_manager.is_generation,
"preferred_sampling_params": _global_state.tokenizer_manager.server_args.preferred_sampling_params,
"weight_version": _global_state.tokenizer_manager.config_value(
"weight_version"
),
"load_format": _global_state.tokenizer_manager.config_value("load_format"),
"reasoning_parser": _global_state.tokenizer_manager.config_value(
"reasoning_parser"
),
"tool_call_parser": _global_state.tokenizer_manager.config_value(
"tool_call_parser"
),
"has_image_understanding": model_config.is_image_understandable_model,
"has_audio_understanding": model_config.is_audio_understandable_model,
"model_type": getattr(model_config.hf_config, "model_type", None),
@@ -785,7 +795,14 @@ async def get_server_info():
@app.get("/server_info")
async def server_info():
"""Get the server information."""
"""The startup configuration, plus live scheduler state.
The `ServerArgs` fields here are the record: what the launcher was given,
with resolution written back into it. Fields the control plane changes
after publication -- the model a weight update swapped in, its load format,
an operator-set weight version -- are reported by `/model_info`, and the
HiCache mirror by `GET /hicache/storage-backend`.
"""
# Returns internal states per DP.
internal_states: List[Dict[Any, Any]] = (
await _global_state.tokenizer_manager.get_internal_state()
@@ -796,9 +813,7 @@ async def server_info():
# server_args.model_config is not serializable but should be excluded by asdict.
return msgspec_to_builtins(
{
**_global_state.tokenizer_manager.resolved_config_dict(
dataclasses.asdict(server_args)
),
**dataclasses.asdict(server_args),
**_global_state.scheduler_info,
"startup_time": _global_state.tokenizer_manager.startup_time,
"internal_states": internal_states,
python/sglang/srt/utils/common.py
+14 -8
View File
@@ -98,7 +98,11 @@ from typing_extensions import Literal
from sglang.srt.environ import envs
from sglang.srt.observability.func_timer import enable_func_timer
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import get_parallel
from sglang.srt.runtime_context import (
configured_tp_size,
get_exec,
get_parallel,
)
from sglang.srt.utils.video_decoder import _BACKEND, VideoDecoderWrapper
if TYPE_CHECKING:
@@ -3578,10 +3582,12 @@ def bind_or_assign(target, source):
# TODO(hebiao064): Accelerate FA3 Spec Decode with topk > 1.
# TODO(hebiao064): Improve the acc rate for FA3 Spec Decode with topk == 1 and page_size > 1.
def is_no_spec_infer_or_topk_one(server_args):
return server_args.speculative_eagle_topk is None or (
server_args.speculative_eagle_topk == 1
and (server_args.page_size == 1 or server_args.page_size is None)
def is_no_spec_infer_or_topk_one(cfg):
"""``cfg`` is a resolving config view, not the published record: the
resolution pipeline is the only caller, and it asks mid-resolution."""
return cfg.speculative_eagle_topk is None or (
cfg.speculative_eagle_topk == 1
and (cfg.page_size == 1 or cfg.page_size is None)
)
@@ -3752,7 +3758,7 @@ def require_mlp_tp_gather(server_args: ServerArgs):
else:
return (
get_parallel().moe_dense_tp_size
> server_args.tp_size // get_parallel().dp_size
> configured_tp_size() // get_parallel().dp_size
)
else:
return False
@@ -3778,7 +3784,7 @@ def require_attn_tp_gather(server_args: ServerArgs):
or get_parallel().moe_dense_tp_size is not None
):
if get_parallel().enable_dp_attention:
return get_parallel().dp_size < server_args.tp_size
return get_parallel().dp_size < configured_tp_size()
else:
return True
else:
@@ -3797,7 +3803,7 @@ def require_mlp_sync(server_args: ServerArgs):
def get_cuda_graph_batch_size_alignment(server_args: ServerArgs) -> int:
alignment = 1
if server_args.enable_two_batch_overlap:
if get_exec().overlap.enable_two_batch_overlap:
alignment *= 2
if require_gathered_buffer(server_args):
alignment *= get_parallel().attn_tp_size