config: publish before a process reads configuration (#35023)

This commit is contained in:
Cheng Wan
2026-08-17 16:16:20 -07:00
committed by GitHub
parent 2b278b4ac4
commit 3d7ec00179
9 changed files with 504 additions and 16 deletions
@@ -27,10 +27,13 @@ resolved configuration lives in the namespace bags.**
- Every publishing process entry calls `publish(server_args, role=...)`
(`run_scheduler_process`, the Ray `SchedulerActor`, the DP controller, tokenizer,
encoder, weight-cache daemon, launcher, ...). The one deliberate exception is the
detokenizer: its processes never publish and read only the raw config handed to
their constructors — code that can run detokenizer-side must not use the
namespace accessors. `publish` snapshots the resolved field
detokenizer, encoder, weight-cache daemon, ...); the roles are enumerated once,
as the keys of `ROLE_NAMESPACE_SETS` — there is no `launcher` role, the launch
path publishes as `tokenizer`. The remaining non-publisher is
`run_multi_detokenizer_router_process`: it *is* handed a `ServerArgs`, and uses
it only for `configure_logger(server_args)` today, so it has nothing to publish
for — a bag read added under that entry needs a `publish` at the entry first.
`publish` snapshots the resolved field
values into the config bags; the accessors (`get_exec()` etc.) fail closed before it
runs. `role` records which process type published, and keys per-role namespace
enforcement: `SGLANG_ROLE_NAMESPACES=record` audits which namespaces each role's
@@ -309,13 +309,15 @@ class MMEncoder:
``base_gpu_id + rank`` — the DP launcher's per-worker placement. It is
this instance's value, not a config change, so it travels as an
argument."""
# The DP and TP encoder workers are spawned, so this constructor is
# the first publish in those processes.
publish(server_args, role="encoder")
logger.info(f"init MMEncoder {rank}/{server_args.tp_size}")
self.server_args = server_args
configure_media_url_security(
server_args.allowed_media_domains,
server_args.media_url_max_file_size_mb,
)
publish(server_args, role="encoder")
self.rank = rank
# DP rank for metric labels; overridden by run_dp_worker in DP mode.
# 0 in the single-instance (non-DP) path.
@@ -3952,6 +3954,9 @@ def _unregister_encoder_url_from_bootstrap(server_args: ServerArgs):
def launch_server(server_args: ServerArgs):
configure_logger(server_args, prefix=" encode_server")
# Publish before the launch path reads configuration; the encoder built
# below re-projects the same object.
publish(server_args, role="encoder")
if server_args.dp_size > 1:
_launch_server_dp(server_args)
return
+6
View File
@@ -99,6 +99,7 @@ from sglang.srt.observability.trace import process_tracing_init, trace_set_threa
from sglang.srt.parser.template_detection import resolve_auto_parsers
from sglang.srt.parser.template_manager import TemplateManager
from sglang.srt.plugins import load_plugins
from sglang.srt.runtime_context import publish
from sglang.srt.server_args import PortArgs, ServerArgs
from sglang.srt.utils import (
MultiprocessingSerializer,
@@ -1109,6 +1110,11 @@ class Engine(EngineScoreMixin, EngineBase):
):
resolve_auto_parsers(server_args)
# Resolution is complete here; this process goes on to host the
# tokenizer manager or the multi-tokenizer router, whose own publish
# re-projects the same object.
publish(server_args, role="tokenizer")
# Launch daemons (daemon mode only). The handles travel back to the
# Engine that spawned them; shutdown() reaps from there.
weight_cache_daemon_procs: List = []
@@ -39,6 +39,7 @@ from sglang.srt.managers.io_struct import (
)
from sglang.srt.managers.multi_tokenizer_mixin import MultiHttpWorkerDetokenizerMixin
from sglang.srt.observability.cpu_monitor import start_cpu_monitor_thread
from sglang.srt.runtime_context import publish
from sglang.srt.server_args import PortArgs, ServerArgs
from sglang.srt.utils import configure_logger, freeze_gc, kill_itself_when_parent_died
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
@@ -520,6 +521,7 @@ def run_detokenizer_process(
kill_itself_when_parent_died()
setproctitle.setproctitle("sglang::detokenizer")
configure_logger(server_args)
publish(server_args, role="detokenizer")
parent_process = psutil.Process().parent()
manager = None
+2 -3
View File
@@ -5016,6 +5016,8 @@ def run_scheduler_process(
):
# Load plugins so hooks can override Scheduler and its dependencies.
load_plugins()
# Publish before anything in this process reads configuration.
publish(server_args, role="scheduler")
dp_rank = configure_scheduler_process(
server_args,
gpu_id,
@@ -5029,9 +5031,6 @@ def run_scheduler_process(
display_dp_rank=display_dp_rank,
display_moe_ep_rank=display_moe_ep_rank,
)
# Scheduler.__init__ reads the config namespaces before the model
# worker's own publish.
publish(server_args, role="scheduler")
parent_process = psutil.Process().parent()
# Set up tracing
@@ -393,6 +393,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
):
# Parse args
self.server_args = server_args
# In a tokenizer-worker process this is the process's first publish;
# the in-process path re-projects the object the launcher published.
set_global_server_args_for_tokenizer(server_args)
self.startup_time: Optional[Dict[str, Any]] = None
self._config_updates: List[Tuple[str, Dict[str, Any]]] = []
self.elastic_worker_count = server_args.dp_size
@@ -407,7 +410,6 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
self.skip_tokenizer_init = server_args.skip_tokenizer_init
self.preferred_sampling_params = server_args.preferred_sampling_params
self.crash_dump_folder = server_args.crash_dump_folder
set_global_server_args_for_tokenizer(server_args)
# Init model config
self.init_model_config()
+4 -4
View File
@@ -75,6 +75,10 @@ class SchedulerActor:
actual_gpu_id = gpu_id
logger.info(f"[TP{tp_rank}] Using passed gpu_id: {gpu_id}")
# This actor takes the place of run_scheduler_process, which is where
# a forked scheduler publishes.
publish(server_args, role="scheduler")
# Configure worker (logging, process title, etc.)
dp_rank = configure_scheduler_process(
server_args,
@@ -99,10 +103,6 @@ class SchedulerActor:
f"[TP{tp_rank}] Bound to NUMA node {numa_node} for GPU {actual_gpu_id}"
)
# This actor constructs Scheduler directly (no run_scheduler_process),
# which reads the config namespaces before the model worker's publish.
publish(server_args, role="scheduler")
# Create scheduler (loads model into GPU, initializes NCCL)
self.scheduler = Scheduler(
server_args=server_args,
+3 -3
View File
@@ -1155,6 +1155,7 @@ ROLE_NAMESPACE_SETS: dict[str, frozenset[str] | None] = {
# a wrong set fails a request rather than a test.
"tokenizer": None,
# Deployment shapes not exercised locally; audit before restricting.
"detokenizer": None,
"encoder": None,
"expert_backup": None,
"weight_cache_daemon": None,
@@ -1259,9 +1260,8 @@ def _dump_recorded_namespace_reads() -> None:
def publish(server_args, *, role: str, hf_config: Any = None) -> RuntimeContext:
"""Install process-wide config for this OS process.
Records the process ``role`` (``tokenizer`` / ``scheduler`` /
``dp_controller`` / ``encoder`` / ``expert_backup`` /
``weight_cache_daemon`` / ``test``) and
Records the process ``role`` one of the ``ROLE_NAMESPACE_SETS`` keys,
which is the one place the roles are enumerated and
projects the config bags. Draft workers skip publish (they must not clobber
the target). ``role`` is provenance, and when ``SGLANG_ROLE_NAMESPACES``
is ``enforce`` the key into ``ROLE_NAMESPACE_SETS`` for fail-closed
@@ -0,0 +1,471 @@
"""A process entry publishes before it reads a config namespace.
The functions checked here are found by walking the package for `publish`
calls, not by listing them: a hand-kept list can name a function that no longer
exists and still pass, which is how the Ray actor entry went unchecked.
Every such function starts a process (or is the first thing a spawned worker
runs), so the runtime context it inherits is empty. A bag read placed above the
publish raises `config namespace ... not published` -- in a spawned worker,
which no unit test starts, so the failure only shows up as a server that never
comes up.
A process entry reaches its bag reads through what it calls -- `Scheduler(...)`,
`configure_scheduler_process(...)`, `self.init_tokenizer_and_processor()` -- not
by naming an accessor itself, so a scan of the entry's own body sees nothing to
order and passes whatever the code does. The read line is therefore taken over
what the entry calls: a call resolved inside the module, through a parameter's
default (`detokenizer_manager_class=DetokenizerManager`), or one hop out through
that module's import table, followed the same way at every depth. Following the
import table only out of the entry's own body would stop one call short of the
expert-backup read, which is reached as `ExpertBackupManager(...)` ->
`backup_weights_from_disk` -> imported loader code -> `get_model()`.
Reaching no read is not a pass. The walk is a static one, and every call it
cannot resolve -- a callable handed in as a parameter, an attribute off
something other than `self` -- turns into "reaches no accessor", which is also
what a defect looks like. So every publisher that reaches none is named in
`_UNREAD_ENTRIES` with the reason, and that map is asserted against the whole
set the walk finds: a publisher that stops reaching a read, or a new one that
never reached any, fails here instead of becoming an entry with nothing to
check. Restricting the comparison to `_KNOWN_ENTRIES` would exempt exactly the
newly discovered entry the derivation exists to catch.
What this cannot pin: a publish moving across code that reads only the handed
`server_args` instance. Such code names no accessor, so there is no read for the
walk to order it against.
"""
import ast
import pathlib
import unittest
import sglang
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=40, suite="base-a-test-cpu")
_PACKAGE_ROOT = pathlib.Path(sglang.__file__).resolve().parent
_ACCESSORS = frozenset(
{
"get_exec",
"get_memory",
"get_schedule",
"get_model",
"get_spec",
"get_serving",
"get_observability",
"get_disagg",
"get_lora",
"get_mm",
"get_device",
"get_parallel",
}
)
# The process entries that must be found by the walk. A derivation that stops
# matching -- an import rewritten, a publish moved behind a helper -- would
# otherwise leave this test green over an empty set.
_KNOWN_ENTRIES = frozenset(
{
("srt/managers/scheduler.py", "run_scheduler_process"),
("srt/managers/detokenizer_manager.py", "run_detokenizer_process"),
(
"srt/managers/data_parallel_controller.py",
"run_data_parallel_controller_process",
),
("srt/ray/scheduler_actor.py", "__init__"),
("srt/disaggregation/encode_server.py", "__init__"),
("srt/disaggregation/encode_server.py", "launch_server"),
("srt/managers/tokenizer_manager.py", "__init__"),
("srt/entrypoints/engine.py", "_launch_subprocesses"),
(
"srt/elastic_ep/expert_backup_manager.py",
"run_expert_backup_manager_process",
),
("srt/weight_cache/daemon.py", "load"),
}
)
# Every publisher the walk finds whose callees reach no bag accessor at this
# revision, and why. Asserted exactly against what the walk finds -- not
# intersected with `_KNOWN_ENTRIES`, which would drop a newly discovered entry
# and check no ordering for the one case the derivation exists to catch.
# "Reaches none" is also what the walk answers when it cannot resolve a call, so
# every one of them is named. An entry leaves this map in the commit that gives
# it a bag read.
_UNREAD_ENTRIES: dict = {
# Not process entries: these publish to set up a context for themselves.
("kernels/aot/tests/test_fused_qk_norm_rope.py", "test_fused_qk_norm_rope"): (
"a kernel test publishing its own context"
),
("multimodal_gen/test/unit/test_disagg_trace.py", "_srt_trace_server_args"): (
"a trace fixture publishing its own context"
),
("srt/entrypoints/engine.py", "_launch_subprocesses"): (
"its subprocess targets and its tokenizer-manager factory arrive as "
"parameters, so the walk resolves none of them"
),
("srt/managers/detokenizer_manager.py", "run_detokenizer_process"): (
"DetokenizerManager reads the handed instance at this revision"
),
("srt/managers/tokenizer_manager.py", "__init__"): (
"the constructor and the init_* helpers it calls read the handed "
"instance at this revision"
),
}
# `publish` itself and its named wrappers live here; a call inside them is the
# definition, not a process entry.
_PUBLISH_HOMES = frozenset({"srt/runtime_context.py", "srt/server_args.py"})
_CONFIG_MODULES = frozenset({"sglang.srt.runtime_context", "sglang.srt.server_args"})
def _module_path(dotted: str):
"""The package-relative file a `sglang.` import names, if it is one."""
if not dotted.startswith("sglang."):
return None
parts = dotted.split(".")[1:]
for candidate in ("/".join(parts) + ".py", "/".join(parts) + "/__init__.py"):
if (_PACKAGE_ROOT / candidate).exists():
return candidate
return None
_CALLS = {}
def _calls(fn):
"""(callee key, line) per call in this body.
`self.f()` / `cls.f()` are keyed to the owning class; a bare name is a
module-level def, an imported name, or a class -- for a class the call runs
its `__init__`. `x.f()` for any other `x` yields two keys: the bare `x`,
which resolves when `x` is a class (`PortArgs.init_new()`), and a
module-qualified one that keeps `f`, which resolves when `x` is a module
the file imported. Keeping only the bare name loses `f` entirely, so a
helper called as `foo.initialize()` contributes nothing to the walk.
Anything deeper (`a.b.c()`, a callable off an attribute) is not resolved.
"""
if id(fn) in _CALLS:
return _CALLS[id(fn)]
out = []
for node in ast.walk(fn):
if not isinstance(node, ast.Call):
continue
func = node.func
if isinstance(func, ast.Name):
out.append((("name", func.id), node.lineno))
elif isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name):
if func.value.id in ("self", "cls"):
out.append((("self", func.attr), node.lineno))
else:
out.append((("name", func.value.id), node.lineno))
out.append(((f"module:{func.value.id}", func.attr), node.lineno))
_CALLS[id(fn)] = out
return out
class _Module:
"""One parsed module: what it calls the config API, and what it defines.
The publisher and accessor names are resolved from the imports rather than
matched by name: a model's ``index_topk_share.publish()`` and a platform's
``get_device()`` are unrelated methods that a name-only match reports as
config calls.
"""
def __init__(self, rel: str, tree):
self.rel, self.tree = rel, tree
self.publishers, self.accessors, self.qualified = set(), set(), set()
self.imported = {}
# Names bound to another sglang module rather than to a symbol in one:
# `import sglang.srt.foo as foo` / `from sglang.srt import foo`. Without
# these, `foo.initialize()` reaches nothing.
self.modules = {}
self.functions = {}
self.classes = {}
self.owner = {}
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node.module:
if node.module in _CONFIG_MODULES:
for alias in node.names:
local = alias.asname or alias.name
if alias.name == "publish" or alias.name.startswith(
"set_global_server_args"
):
self.publishers.add(local)
elif alias.name in _ACCESSORS:
self.accessors.add(local)
target = _module_path(node.module)
if target is not None and target != rel:
for alias in node.names:
self.imported[alias.asname or alias.name] = (target, alias.name)
for alias in node.names:
# `from sglang.srt import runtime_context` binds the module
# itself; `runtime_context.get_serving()` is the same read.
dotted = f"{node.module}.{alias.name}"
if dotted in _CONFIG_MODULES:
self.qualified.add(alias.asname or alias.name)
bound = _module_path(dotted)
if bound is not None and bound != rel:
self.modules[alias.asname or alias.name] = bound
elif isinstance(node, ast.Import):
for alias in node.names:
if alias.name in _CONFIG_MODULES:
self.qualified.add(alias.asname or alias.name.split(".")[0])
# Only `import a.b.c as name` binds a usable name; without
# the alias the call reads `a.b.c.f()`, which is deeper than
# this walk resolves.
bound = _module_path(alias.name) if alias.asname else None
if bound is not None and bound != rel:
self.modules[alias.asname] = bound
elif isinstance(node, ast.ClassDef):
methods = self.classes.setdefault(node.name, {})
for stmt in node.body:
if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef)):
methods[stmt.name] = stmt
self.owner[id(stmt)] = node.name
for node in ast.walk(tree):
if (
isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
and id(node) not in self.owner
):
self.functions.setdefault(node.name, node)
self._direct = {}
def is_publish(self, node) -> bool:
if not isinstance(node, ast.Call):
return False
if isinstance(node.func, ast.Name) and node.func.id in self.publishers:
return True
return (
isinstance(node.func, ast.Attribute)
and isinstance(node.func.value, ast.Name)
and node.func.value.id in self.qualified
)
def is_read(self, node) -> bool:
if not isinstance(node, ast.Call):
return False
if isinstance(node.func, ast.Name) and node.func.id in self.accessors:
return True
return (
isinstance(node.func, ast.Attribute)
and isinstance(node.func.value, ast.Name)
and node.func.value.id in self.qualified
and node.func.attr in _ACCESSORS
)
def resolve(self, key, cls):
"""The def in this module a callee key names, if it names one.
A module-qualified key names nothing here: its def lives in the module
the alias is bound to, which `_targets` follows.
"""
kind, name = key
if kind == "self":
return self.classes.get(cls, {}).get(name)
if kind.startswith("module:"):
return None
if name in self.functions:
return self.functions[name]
return self.classes.get(name, {}).get("__init__")
def direct_read(self, fn) -> bool:
"""Whether this body names an accessor itself."""
if id(fn) not in self._direct:
self._direct[id(fn)] = any(self.is_read(n) for n in ast.walk(fn))
return self._direct[id(fn)]
_MODULES = {}
def _module(rel: str):
# A module this walk cannot parse would resolve to "reaches no config",
# which is the answer that hides a defect. utf-8-sig because a file in the
# package carries a BOM; anything still unparsable fails the test.
if rel not in _MODULES:
_MODULES[rel] = _Module(
rel, ast.parse((_PACKAGE_ROOT / rel).read_text(encoding="utf-8-sig"))
)
return _MODULES[rel]
def _defaulted_parameters(fn):
"""`{parameter: default name}` for the parameters this def gives a plain
name as default. `run_detokenizer_process` reaches `DetokenizerManager`
only this way -- the body calls the parameter, and the class it is really
handed is written once, as that parameter's default."""
if id(fn) in _DEFAULTS:
return _DEFAULTS[id(fn)]
arguments = fn.args
positional = arguments.posonlyargs + arguments.args
pairs = list(
zip(positional[len(positional) - len(arguments.defaults) :], arguments.defaults)
)
pairs += zip(arguments.kwonlyargs, arguments.kw_defaults)
_DEFAULTS[id(fn)] = {
parameter.arg: default.id
for parameter, default in pairs
if isinstance(default, ast.Name)
}
return _DEFAULTS[id(fn)]
_DEFAULTS = {}
def _targets(mod, key, cls, fn=None):
"""The (module, def, owning class) a callee key names, here and one hop out.
A bare name that is one of `fn`'s parameters resolves through that
parameter's default as well, which is how a factory handed in as an
argument is followed to the class the entry actually constructs.
A module-qualified key (`foo.initialize()`) resolves in the module `foo` is
bound to, so a helper reached that way joins the walk instead of dropping
out of it.
"""
keys = [key]
if fn is not None and key[0] == "name":
default = _defaulted_parameters(fn).get(key[1])
if default is not None:
keys.append(("name", default))
for key in keys:
target = mod.resolve(key, cls)
if target is not None:
yield mod, target, mod.owner.get(id(target), cls)
if key[0].startswith("module:"):
home = mod.modules.get(key[0][len("module:") :])
if home is None:
continue
other = _module(home)
target = other.resolve(("name", key[1]), None)
if target is not None:
yield other, target, other.owner.get(id(target))
continue
hop = mod.imported.get(key[1]) if key[0] == "name" else None
if hop is None:
continue
other = _module(hop[0])
target = other.resolve(("name", hop[1]), None)
if target is not None:
yield other, target, other.owner.get(id(target))
# The callee names from a def down to the read it reaches, for the defs a
# witness has been found for. Only "reaches a read" is carried between
# questions: a witness path stays one, while "reaches none" can be the answer a
# recursive call gets when it re-enters a def the walk is still inside, which
# is that call's answer and not the def's own.
_WITNESS = {}
def _witness(mod, fn, cls, seen):
"""The callees from this def down to a bag read, ending in the file that
reads, or None for a def that reaches no read.
`seen` belongs to one question. Skipping a def already on this walk keeps
the answer for the def the question was asked about -- that def opened the
skipped one, so a read below it still comes back up the path that opened
it -- and bounds the walk on a call cycle.
"""
key = (mod.rel, id(fn))
if key in _WITNESS:
return _WITNESS[key]
if key in seen:
return None
seen.add(key)
if mod.direct_read(fn):
_WITNESS[key] = [mod.rel]
return _WITNESS[key]
for call, _ in _calls(fn):
for target in _targets(mod, call, cls, fn):
below = _witness(*target, seen)
if below is not None:
_WITNESS[key] = [call[1]] + below
return _WITNESS[key]
return None
def _publishing_functions():
"""(relative path, function node, module) per publisher."""
for path in sorted(_PACKAGE_ROOT.rglob("*.py")):
rel = path.relative_to(_PACKAGE_ROOT).as_posix()
if rel in _PUBLISH_HOMES:
continue
mod = _module(rel)
if mod is None or not mod.publishers:
continue
for fn in ast.walk(mod.tree):
if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
continue
if any(mod.is_publish(n) for n in ast.walk(fn)):
yield rel, fn, mod
def _first_read(fn, mod):
"""(line, what read it) of the earliest config read this entry reaches."""
cls = mod.owner.get(id(fn))
marks = [(n.lineno, "a bag accessor") for n in ast.walk(fn) if mod.is_read(n)]
for call, line in _calls(fn):
for target in _targets(mod, call, cls, fn):
if target[1] is fn:
continue
below = _witness(*target, set())
if below is not None:
chain = [call[1]] + below
marks.append(
(line, " -> ".join(chain[:-1]) + f", which reads in {chain[-1]}")
)
break
return min(marks) if marks else None
class TestPublishPrecedesBagReads(CustomTestCase):
def test_every_publishing_entry_publishes_first(self):
offenders = []
found = set()
unread = set()
for rel, fn, mod in _publishing_functions():
found.add((rel, fn.name))
# ast.walk yields breadth-first, so the first match is not the
# earliest line; take the minimum.
publish_line = min(n.lineno for n in ast.walk(fn) if mod.is_publish(n))
read = _first_read(fn, mod)
if read is None:
unread.add((rel, fn.name))
elif read[0] < publish_line:
offenders.append(
f"{rel}:{fn.name} reaches a config namespace at line "
f"{read[0]} through {read[1]}, before its publish at "
f"{publish_line}"
)
self.assertEqual(
sorted(_KNOWN_ENTRIES - found),
[],
"the walk stopped finding known process entries; the derivation "
"is broken, not the tree",
)
self.assertEqual(
sorted(unread),
sorted(_UNREAD_ENTRIES),
"a publisher reaching no bag accessor checks nothing; either the "
"walk stopped resolving a call, or a publisher appeared or moved "
"its reads and _UNREAD_ENTRIES has to say so",
)
self.assertEqual(
offenders,
[],
"a spawned worker starts with an empty context:\n "
+ "\n ".join(offenders),
)
if __name__ == "__main__":
unittest.main()