minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

config: publish before the launcher reads effective configuration (#35910)

Browse Source 
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Cheng Wan
2026-08-23 01:20:20 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent a43592dce5
commit 340391a297
8 changed files with 591 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/engine.py
+64 -33
View File
@@ -100,12 +100,16 @@ from sglang.srt.parser.template_detection import resolve_auto_parsers
from sglang.srt.parser.template_manager import TemplateManager
from sglang.srt.plugins import load_plugins
from sglang.srt.runtime_context import (
configured_attn_cp_size,
configured_moe_dp_size,
configured_pp_size,
get_exec,
get_model,
get_parallel,
get_serving,
publish,
restore_context,
snapshot_context,
)
from sglang.srt.server_args import PortArgs, ServerArgs
from sglang.srt.utils import (
@@ -1089,43 +1093,63 @@ class Engine(EngineScoreMixin, EngineBase):
# Engine.__init__ or CLI entry).
load_plugins()
# Not read-only: the LoRA checks normalize adapter paths through late
# resolution, which a published config refuses. Hence before publish --
# and before the parser detection below, which consumes the "auto"
# sentinel: a record rejected here has to stay retryable.
server_args.check_server_args()
# Allocate ports for inter-process communications
if port_args is None:
port_args = PortArgs.init_new(server_args)
logger.info(f"{server_args=}")
# Start the engine info bootstrap server if per-rank info is needed.
engine_info_bootstrap_server = None
if (
server_args.remote_instance_weight_loader_start_seed_via_transfer_engine
and server_args.node_rank == 0
):
bootstrap_port = server_args.engine_info_bootstrap_port
if not is_port_available(bootstrap_port):
raise RuntimeError(
f"engine_info_bootstrap_port {bootstrap_port} is already in use. "
f"When running multiple instances on the same node, each instance must use a "
f"different --engine-info-bootstrap-port."
)
engine_info_bootstrap_server = EngineInfoBootstrapServer(
host=server_args.host, port=bootstrap_port
)
# Needs a tokenizer and a chat template, so it cannot live in the
# pipeline; after the plugins, which may register the parser detected.
if (
server_args.reasoning_parser == "auto"
or server_args.tool_call_parser == "auto"
):
resolve_auto_parsers(server_args)
# This publish replaces whatever was published before it, so the
# rollback below restores that rather than clearing the process: a
# caller that catches the launch error still has the context it had.
context_before_publish = snapshot_context()
publish(server_args, role="tokenizer")
# Launch daemons (daemon mode only). The handles travel back to the
# Engine that spawned them; shutdown() reaps from there.
weight_cache_daemon_procs: List = []
if server_args.weight_cache_mode == "daemon":
weight_cache_daemon_procs = cls._launch_weight_cache_daemons(server_args)
# Nothing below has spawned yet, so a failure here leaves a record the
# caller can hand back -- but only once the publication goes with it:
# the validation stage writes through late resolution, which refuses a
# record that is already published.
try:
# Allocate ports for inter-process communications
if port_args is None:
port_args = PortArgs.init_new(server_args)
logger.info(f"{server_args=}")
# Start the engine info bootstrap server if per-rank info is needed.
engine_info_bootstrap_server = None
if (
get_model().remote_instance_weight_loader_start_seed_via_transfer_engine
and server_args.node_rank == 0
):
bootstrap_port = server_args.engine_info_bootstrap_port
if not is_port_available(bootstrap_port):
raise RuntimeError(
f"engine_info_bootstrap_port {bootstrap_port} is already in use. "
f"When running multiple instances on the same node, each instance must use a "
f"different --engine-info-bootstrap-port."
)
engine_info_bootstrap_server = EngineInfoBootstrapServer(
host=server_args.host, port=bootstrap_port
)
# Launch daemons (daemon mode only). The handles travel back to the
# Engine that spawned them; shutdown() reaps from there.
weight_cache_daemon_procs: List = []
if server_args.weight_cache_mode == "daemon":
weight_cache_daemon_procs = cls._launch_weight_cache_daemons(
server_args
)
except BaseException:
restore_context(context_before_publish)
raise
# Launch scheduler processes
# Passed only when there is one: this hook is an override point, and a
@@ -1858,18 +1882,25 @@ def _calculate_rank_ranges(
def _compute_parallelism_ranks(
server_args: ServerArgs, tp_rank: int
) -> Tuple[int, int, int]:
"""Compute attention-CP, MoE-DP, and MoE-EP ranks for a TP rank."""
"""Compute attention-CP, MoE-DP, and MoE-EP ranks for a TP rank.
Called while the launcher is deciding what to spawn, so the sizes are the
configured ones -- the groups this is laying out do not exist yet.
"""
attn_dp_size = get_parallel().dp_size if get_parallel().enable_dp_attention else 1
tp_size = server_args.tp_size
attn_cp_size = configured_attn_cp_size()
moe_dp_size = configured_moe_dp_size()
# Parallelism hierarchy (outermost to innermost):
# - Attention: Global(TP) -> DP -> ATTN_CP -> ATTN_TP (innermost)
# - MoE: Global(TP) -> MOE_DP -> EP -> MOE_TP (innermost)
attn_tp_size = server_args.tp_size // attn_dp_size // server_args.attn_cp_size
attn_cp_rank = (tp_rank // attn_tp_size) % server_args.attn_cp_size
moe_dp_rank = tp_rank // (server_args.tp_size // server_args.moe_dp_size)
attn_tp_size = tp_size // attn_dp_size // attn_cp_size
attn_cp_rank = (tp_rank // attn_tp_size) % attn_cp_size
moe_dp_rank = tp_rank // (tp_size // moe_dp_size)
moe_ep_rank = (
tp_rank
% (server_args.tp_size // server_args.moe_dp_size)
// (server_args.tp_size // server_args.moe_dp_size // get_parallel().ep_size)
% (tp_size // moe_dp_size)
// (tp_size // moe_dp_size // get_parallel().ep_size)
)
return attn_cp_rank, moe_dp_rank, moe_ep_rank
python/sglang/srt/managers/data_parallel_controller.py
+23 -14
View File
@@ -49,7 +49,16 @@ from sglang.srt.observability.cpu_monitor import start_cpu_monitor_thread
from sglang.srt.observability.req_time_stats import DPControllerReqTimeStats
from sglang.srt.observability.startup_time import aggregate_scheduler_startup_times
from sglang.srt.observability.trace import process_tracing_init, trace_set_thread_info
from sglang.srt.runtime_context import get_exec, get_parallel, publish
from sglang.srt.runtime_context import (
configured_attn_cp_size,
configured_moe_dp_size,
configured_pp_size,
get_device,
get_disagg,
get_exec,
get_parallel,
publish,
)
from sglang.srt.server_args import (
DP_ATTENTION_HANDSHAKE_PORT_DELTA,
PortArgs,
@@ -142,7 +151,7 @@ class DataParallelController:
self.server_args = server_args
self.port_args = port_args
self.load_balance_method = LoadBalanceMethod.from_str(
server_args.load_balance_method
get_parallel().load_balance_method
)
self.run_scheduler_process_func = run_scheduler_process_func
@@ -212,7 +221,7 @@ class DataParallelController:
self.soft_watchdog = Watchdog.create(
debug_name="DataParallelController",
watchdog_timeout=server_args.soft_watchdog_timeout,
watchdog_timeout=get_device().soft_watchdog_timeout,
soft=True,
test_stuck_time=envs.SGLANG_TEST_STUCK_DP_CONTROLLER.get(),
)
@@ -386,7 +395,7 @@ class DataParallelController:
)
threads.append(thread)
base_gpu_id += (
server_args.tp_size * server_args.pp_size * server_args.gpu_id_step
server_args.tp_size * configured_pp_size() * server_args.gpu_id_step
)
if server_args.node_rank == 0:
@@ -607,8 +616,8 @@ class DataParallelController:
scheduler_pipe_readers = []
pp_size_per_node = max(server_args.pp_size // server_args.nnodes, 1)
nnodes_per_pp_rank = max(server_args.nnodes // server_args.pp_size, 1)
pp_size_per_node = max(configured_pp_size() // server_args.nnodes, 1)
nnodes_per_pp_rank = max(server_args.nnodes // configured_pp_size(), 1)
pp_rank_range = range(
pp_size_per_node * (server_args.node_rank // nnodes_per_pp_rank),
pp_size_per_node * (server_args.node_rank // nnodes_per_pp_rank + 1),
@@ -639,7 +648,7 @@ class DataParallelController:
tp_rank,
server_args.tp_size,
get_parallel().dp_size,
server_args.attn_cp_size,
configured_attn_cp_size(),
)
# compute zmq ports for this dp rank
rank_port_args = PortArgs.init_new(
@@ -675,18 +684,18 @@ class DataParallelController:
# - Attention: Global(TP) -> DP -> ATTN_CP -> ATTN_TP (innermost)
# - MoE: Global(TP) -> MOE_DP -> EP -> MOE_TP (innermost)
attn_tp_size = (
server_args.tp_size // attn_dp_size // server_args.attn_cp_size
server_args.tp_size // attn_dp_size // configured_attn_cp_size()
)
attn_cp_rank = (tp_rank // attn_tp_size) % server_args.attn_cp_size
attn_cp_rank = (tp_rank // attn_tp_size) % configured_attn_cp_size()
moe_dp_rank = tp_rank // (
server_args.tp_size // server_args.moe_dp_size
server_args.tp_size // configured_moe_dp_size()
)
moe_ep_rank = (
tp_rank
% (server_args.tp_size // server_args.moe_dp_size)
% (server_args.tp_size // configured_moe_dp_size())
// (
server_args.tp_size
// server_args.moe_dp_size
// configured_moe_dp_size()
// get_parallel().ep_size
)
)
@@ -829,9 +838,9 @@ def run_data_parallel_controller_process(
trace_modules=server_args.trace_modules,
)
thread_label = "DP Controller"
if server_args.disaggregation_mode == "prefill":
if get_disagg().disaggregation_mode == "prefill":
thread_label = "Prefill DP Controller"
elif server_args.disaggregation_mode == "decode":
elif get_disagg().disaggregation_mode == "decode":
thread_label = "Decode DP Controller"
trace_set_thread_info(thread_label)
python/sglang/srt/runtime_context.py
+79 -3
View File
@@ -1187,9 +1187,10 @@ ROLE_NAMESPACE_SETS: dict[str, frozenset[str] | None] = {
# Reads (almost) everything by design — the model-executing process.
"scheduler": None,
"test": None,
# Audited (record-mode smokes, plain + DP-attention): the DP controller
# reads only the elastic-EP gate; its module's static read set agrees.
"dp_controller": frozenset({"exec"}),
# The DP controller's static read set, checked against the module: the
# elastic-EP gate, the load-balance method, the watchdog timeout, and the
# disaggregation mode.
"dp_controller": frozenset({"exec", "parallel", "device", "disagg"}),
# Record-mode audit (2026-08-06, text model, /generate + /get_server_info +
# /v1/models): reads exactly {"serving"} — the per-instance managers read
# self.server_args by design. Still declared full, because that run did not
@@ -1407,6 +1408,81 @@ def set_global_dwdp_manager(manager: Any) -> None:
_GLOBAL_DWDP_MANAGER = manager
def _group_leaves(group: _FlagGroupBase) -> dict[str, Any]:
"""The leaf values of a flag group, recursively."""
leaves: dict[str, Any] = {}
for name in type(group).__dataclass_fields__:
value = getattr(group, name)
if isinstance(value, _FlagGroupBase):
leaves[name] = _group_leaves(value)
elif isinstance(value, (dict, list)):
leaves[name] = type(value)(value)
else:
leaves[name] = value
return leaves
def _restore_leaves(group: _FlagGroupBase, leaves: dict[str, Any]) -> None:
for name, value in leaves.items():
current = getattr(group, name)
if isinstance(current, _FlagGroupBase):
_restore_leaves(current, value)
elif isinstance(current, dict):
current.clear()
current.update(value)
elif isinstance(current, list):
current[:] = value
else:
setattr(group, name, value)
def snapshot_context() -> dict[str, Any]:
"""Everything a publish replaces, so a failed launch can put it back.
Enumerated from ``__slots__`` rather than listed by hand: a hand-picked copy
of context state is one field behind the day a slot is added, and the copy
that silently drops one is worse than none. Flag groups are snapshotted by
leaf, not by reference: publish writes *into* the same ``Flags`` object
(``capture.enable_torch_compile``), so a reference held here would already
carry the failed launch's value by the time it is put back.
"""
state: dict[str, Any] = {}
for name in RuntimeContext.__slots__:
if name == "parallel":
continue
value = getattr(_CONTEXT, name)
if isinstance(value, _FlagGroupBase):
state[name] = (value, _group_leaves(value))
elif isinstance(value, list):
state[name] = list(value)
else:
state[name] = value
state["__parallel__"] = {
name: getattr(_CONTEXT.parallel, name)
for name in type(_CONTEXT.parallel).__slots__
}
state["__dwdp__"] = get_global_dwdp_manager()
return state
def restore_context(state: dict[str, Any]) -> None:
"""Put back what ``snapshot_context`` captured."""
for name in RuntimeContext.__slots__:
if name == "parallel":
continue
value = state[name]
if isinstance(value, tuple) and isinstance(value[0], _FlagGroupBase):
group, leaves = value
setattr(_CONTEXT, name, group)
_restore_leaves(group, leaves)
else:
setattr(_CONTEXT, name, value)
for name, value in state["__parallel__"].items():
setattr(_CONTEXT.parallel, name, value)
_adaptive_draft_token_bound.cache_clear()
set_global_dwdp_manager(state["__dwdp__"])
def reset_context() -> None:
"""Clear the context-owned store (unit-test teardown): drop the published
``server_args`` and install fresh ``Flags`` and ``Resources``.