minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

config: spell the parallel config tier at the call site (#36250)

Browse Source
This commit is contained in:
Cheng Wan
2026-08-26 03:00:28 -07:00
committed by GitHub
parent 689ade69d1
commit 8005df61d3
136 changed files with 1033 additions and 795 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/unit/disaggregation/test_disaggregation_wire.py
+8 -2
View File
@@ -158,8 +158,12 @@ class TestCPReplicatedStateTransfer(unittest.TestCase):
manager = object.__new__(CommonKVManager)
manager.attn_cp_size = cp_size
manager.attn_cp_rank = cp_rank
# The policy reads the configured tier, so the stand-in
# carries the leaf under `config`, where the bag serves it.
parallel = SimpleNamespace(
enable_dsa_cache_layer_split=layer_split,
config=SimpleNamespace(
enable_dsa_cache_layer_split=layer_split,
),
)
with patch(
"sglang.srt.disaggregation.common.conn.get_parallel",
@@ -178,7 +182,9 @@ class TestCPReplicatedStateTransfer(unittest.TestCase):
with patch(
"sglang.srt.disaggregation.common.conn.get_parallel",
return_value=SimpleNamespace(enable_dsa_cache_layer_split=False),
return_value=SimpleNamespace(
config=SimpleNamespace(enable_dsa_cache_layer_split=False)
),
):
self.assertEqual(
manager._get_dsa_cache_transfer_skip_flags(None),
test/registered/unit/model_loader/test_presharded_loader.py
+7 -5
View File
@@ -803,18 +803,20 @@ class TestShardConfig(unittest.TestCase):
"init_expert_location",
"structural_signature",
}
# Both tiers on one stand-in: bare names are the live groups, `config`
# is the published parallel bag.
parallel = SimpleNamespace(
tp_size=8,
moe_dp_size=2,
moe_ep_size=4,
pp_size=1,
moe_dense_tp_size=1,
enable_dp_lm_head=True,
config=SimpleNamespace(
moe_dp_size=2,
moe_dense_tp_size=1,
enable_dp_lm_head=True,
),
)
with mock.patch(
"sglang.srt.model_loader.loader.configured_moe_dp_size",
return_value=2,
), mock.patch(
"sglang.srt.model_loader.loader.get_parallel",
return_value=parallel,
), mock.patch(
test/registered/unit/models/test_kimi_k25.py
+9 -3
View File
@@ -66,7 +66,13 @@ from sglang.srt.multimodal.transport.cuda_ipc import (
DEFER_CUDA_IPC_FEATURE_RECONSTRUCTION_KEY,
CudaIpcTensorTransportProxy,
)
from sglang.srt.runtime_context import get_context, get_parallel, publish, reset_context
from sglang.srt.runtime_context import (
ParallelContext,
get_context,
get_parallel,
publish,
reset_context,
)
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import ImageData
from sglang.test.ci.ci_register import register_cpu_ci
@@ -903,7 +909,7 @@ def test_kimi_k3_normal_cache_path_connects_real_producer_to_model_consumer():
hot_items = pickle.loads(pickle.dumps(hot.mm_items))
with (
patch("sglang.srt.models.kimi_k3.configured_tp_size", return_value=1),
patch.object(ParallelContext, "config", SimpleNamespace(tp_size=1)),
patch(
"sglang.srt.multimodal.processors.kimi_k25._gpu_preprocess_images",
return_value=(
@@ -967,7 +973,7 @@ def test_kimi_k3_model_accepts_mixed_cached_eager_and_deferred_artifacts():
)
with (
patch("sglang.srt.models.kimi_k3.configured_tp_size", return_value=1),
patch.object(ParallelContext, "config", SimpleNamespace(tp_size=1)),
patch(
"sglang.srt.multimodal.processors.kimi_k25._gpu_preprocess_images",
return_value=(torch.full((1, 3), 2.0), torch.tensor([[1, 1, 1]])),
test/registered/unit/multimodal/test_tensor_transport_mode.py
+6 -2
View File
@@ -22,9 +22,13 @@ class TestTensorTransportMode(CustomTestCase):
for nnodes, dist_init_addr, expected in cases:
with self.subTest(nnodes=nnodes, dist_init_addr=dist_init_addr):
# `nnodes` is a config-only leaf, so the stand-in carries it
# under `config`, where the published bag serves it.
parallel = SimpleNamespace(
nnodes=nnodes,
dist_init_addr=dist_init_addr,
config=SimpleNamespace(
nnodes=nnodes,
dist_init_addr=dist_init_addr,
),
)
with patch(
"sglang.srt.multimodal.transport.get_parallel",
test/registered/unit/server_args/test_resolution_declarations.py
+8 -9
View File
@@ -396,9 +396,10 @@ class TestResolutionDeclarations(CustomTestCase):
mapping = namespace_of(ServerArgs)
self.assertGreater(len(mapping), 400, "the namespace mapping collapsed")
shadowed = _live_topology_leaves()
# The five sizes keep a live property shadowing the bare name; the
# comparison below reaches them anyway, through `get_parallel().config`.
self.assertGreaterEqual(
shadowed
_live_topology_leaves()
& {
"tp_size",
"pp_size",
@@ -407,8 +408,7 @@ class TestResolutionDeclarations(CustomTestCase):
"dcp_size",
},
{"tp_size", "pp_size", "moe_dp_size", "attn_cp_size", "dcp_size"},
"a parallel size stopped being served from the live topology; if it "
"is a plain config leaf now, it belongs in the comparison below",
"a parallel size stopped being served from the live topology",
)
compared = 0
@@ -418,17 +418,16 @@ class TestResolutionDeclarations(CustomTestCase):
server_args = self._resolve(shape)
publish(server_args, role="scheduler")
for field, path in mapping.items():
if field in shadowed:
# Served from the process groups by design; `configured_*()`
# is what answers with the configured value, and
# test_launch_path_reads_configured_sizes pins that.
continue
groups = path.split(".")
accessor = getattr(runtime_context, f"get_{groups[0]}", None)
if accessor is None:
unreachable.append(f"no get_{groups[0]}() for {path}.{field}")
continue
node = accessor()
if groups[0] == "parallel":
# Bare names there are the live topology; the published
# leaves are one hop down, so the reader takes that hop.
node = node.config
try:
for group in groups[1:]:
node = getattr(node, group)
test/registered/unit/server_args/test_resolution_is_reproducible.py
+1 -1
View File
@@ -844,7 +844,7 @@ class TestACopyStaysResolved(_RestoresProcessState, CustomTestCase):
self.addCleanup(reset_context)
reset_context()
publish(copy_, role="scheduler")
self.assertEqual(get_parallel().dist_init_addr, "1.2.3.4:5000")
self.assertEqual(get_parallel().config.dist_init_addr, "1.2.3.4:5000")
self.assertEqual(
get_schedule().chunked_prefill_size,
parent.chunked_prefill_size,
test/registered/unit/server_args/test_resolution_reads_no_bag.py
+2 -3
View File
@@ -53,8 +53,7 @@ def _accessor_names():
names = {
node.name
for node in tree.body
if isinstance(node, ast.FunctionDef)
and (node.name.startswith("get_") or node.name.startswith("configured_"))
if isinstance(node, ast.FunctionDef) and node.name.startswith("get_")
}
# The context object itself is not a bag: it exists before anything is
# published, and `declare_late_resolution` calls it deliberately to find
@@ -232,7 +231,7 @@ class TestResolutionReadsNoBag(CustomTestCase):
"""A shrunken accessor set would make every other check pass quietly."""
self.assertGreaterEqual(
len(_BAG_ACCESSORS),
20,
15,
f"only {len(_BAG_ACCESSORS)} accessors were derived from "
"runtime_context; the derivation broke",
)
test/registered/unit/test_global_config_read_ratchet.py
+249 -84
View File
@@ -12,10 +12,10 @@ slot.
The reads that remain live in ``runtime_context.py`` (exempt by module): the
``@property`` / method members computed from several fields plus the HF config,
which are not namespace leaves and have no home but ``ServerArgs``, and the
``configured_*_size()`` accessors for the sizes ``get_parallel()`` shadows with
the live topology. ``_CONFIGURED_SIZE_CALL_SITES`` registers every one of the
latter with the reason the live property cannot serve it.
which are not namespace leaves and have no home but ``ServerArgs``.
Separately, ``_CONFIGURED_SIZE_CALL_SITES`` registers every business read of
``get_parallel().config.<size>`` — the config tier of a size whose bare name is
the live topology — with the reason the live property cannot serve it.
What the scan sees: ``get_server_args().field``, an alias (``sa =
get_server_args()`` then ``sa.field`` -- function-local, module-level, or parked
@@ -48,133 +48,134 @@ _PACKAGE_ROOT = Path(next(iter(sglang.__path__)))
# resolution pipeline.
_SLOT_OWNERS = ("srt/runtime_context.py", "srt/server_args.py", "srt/arg_groups/")
# Every call site of a ``configured_*_size()`` accessor, with the reason the
# live topology cannot answer there. The test below asserts this map is exactly
# the set of call sites, so the reasons cannot drift away from the code.
# Every configured read of a live-shadowed size (``get_parallel().config.pp_size``
# and its four siblings), with the reason the live topology cannot answer there.
# The test below asserts this map is exactly the set of such reads, so the
# reasons cannot drift away from the code.
_CONFIGURED_SIZE_CALL_SITES = {
("srt/entrypoints/engine.py", "configured_pp_size"): (
("srt/entrypoints/engine.py", "pp_size"): (
"the launch path decides how many scheduler processes to spawn; it runs "
"before any of them exists, so there is no group to ask"
),
("srt/entrypoints/engine.py", "configured_attn_cp_size"): (
("srt/entrypoints/engine.py", "attn_cp_size"): (
"the launcher's per-TP-rank layout, computed while deciding what to "
"spawn -- the groups it is laying out do not exist yet"
),
("srt/entrypoints/engine.py", "configured_moe_dp_size"): (
("srt/entrypoints/engine.py", "moe_dp_size"): (
"the MoE factor of that same pre-spawn layout"
),
("srt/ray/engine.py", "configured_pp_size"): (
("srt/ray/engine.py", "pp_size"): (
"the Ray driver sizes the actor placement group; the actors it is about "
"to create are the ones that will hold the process groups"
),
("srt/ray/data_parallel_controller.py", "configured_pp_size"): (
("srt/ray/data_parallel_controller.py", "pp_size"): (
"same placement arithmetic on the DP path -- ranks per TP group, "
"computed in the driver before the actors start"
),
("srt/ray/data_parallel_controller.py", "configured_attn_cp_size"): (
("srt/ray/data_parallel_controller.py", "attn_cp_size"): (
"the attention-CP factor of that same placement arithmetic, and the one "
"size whose live value cannot express the configured intent when "
"attn_cp_size > moe_dp_size aliases the groups"
),
("srt/layers/attention/dsa/dsa_indexer.py", "configured_pp_size"): (
("srt/layers/attention/dsa/dsa_indexer.py", "pp_size"): (
"gates `pp_size > 1 and not get_pp_group()...`; the short circuit is the "
"point, since with PP off the group is never touched, which is what lets "
"the Indexer be constructed before distributed init"
),
("srt/managers/scheduler.py", "configured_pp_size"): (
("srt/managers/scheduler.py", "pp_size"): (
"dispatch_event_loop picks the PP event loop; the MLX runner stub never "
"initializes torch.distributed, so the live property asserts before the "
"MLX loop can start -- the configured leaf answers the same value "
"wherever the live groups exist"
),
("srt/mem_cache/kv_cache_configurator.py", "configured_pp_size"): (
("srt/mem_cache/kv_cache_configurator.py", "pp_size"): (
"decides whether the token capacity needs a cross-PP all-reduce at all; "
"asking the configured size keeps that decision independent of whether a "
"PP group is installed in this process"
),
("srt/layers/dp_attention.py", "configured_attn_cp_size"): (
("srt/layers/dp_attention.py", "attn_cp_size"): (
"compared against the configured moe_dp_size below"
),
("srt/layers/dp_attention.py", "configured_moe_dp_size"): (
("srt/layers/dp_attention.py", "moe_dp_size"): (
"the configuration this predicate detects (attn_cp_size > moe_dp_size) is "
"the one where initialize_model_parallel aliases _MOE_DP to _ATTN_CP, so "
"the live sizes are equal there and a live comparison is always false"
),
("srt/managers/scheduler.py", "configured_tp_size"): (
("srt/managers/scheduler.py", "tp_size"): (
"configure_scheduler_process runs before the scheduler's own process "
"groups exist -- configuring the process is what it is for -- so there "
"is nothing live to ask yet"
),
("srt/managers/scheduler.py", "configured_moe_dp_size"): (
("srt/managers/scheduler.py", "moe_dp_size"): (
"same pre-distributed-init arithmetic in configure_scheduler_process"
),
("srt/managers/scheduler.py", "configured_attn_cp_size"): (
("srt/managers/scheduler.py", "attn_cp_size"): (
"same pre-distributed-init arithmetic in configure_scheduler_process"
),
("srt/managers/scheduler.py", "configured_dcp_size"): (
("srt/managers/scheduler.py", "dcp_size"): (
"same pre-distributed-init arithmetic in configure_scheduler_process"
),
("srt/model_executor/runner/base_runner.py", "configured_pp_size"): (
("srt/model_executor/runner/base_runner.py", "pp_size"): (
"the runner's layer window is arithmetic over the configured stage "
"count; a draft runner shares the target's groups, so the live "
"property would answer for the wrong runner"
),
("srt/model_executor/cpu_graph_runner.py", "configured_pp_size"): (
("srt/model_executor/cpu_graph_runner.py", "pp_size"): (
"the same window, on the CPU graph path"
),
(
"srt/managers/scheduler_components/metrics_reporter.py",
"configured_pp_size",
"pp_size",
): (
"the reporter labels its metrics with the stage count it was launched "
"with, which is configuration; the live group answers per process"
),
("srt/speculative/eagle_draft_cuda_graph_runner.py", "configured_pp_size"): (
("srt/speculative/eagle_draft_cuda_graph_runner.py", "pp_size"): (
"the draft runner's window over the target's stages: its own groups are "
"the target's, so the configured count is the one that describes it"
),
(
"srt/speculative/eagle_draft_extend_cuda_graph_runner.py",
"configured_pp_size",
"pp_size",
): ("the same draft window, on the extend path"),
(
"srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py",
"configured_pp_size",
"pp_size",
): ("the same draft window, multi-layer extend"),
("srt/speculative/frozen_kv_mtp_cuda_graph_runner.py", "configured_pp_size"): (
("srt/speculative/frozen_kv_mtp_cuda_graph_runner.py", "pp_size"): (
"the same draft window, frozen-KV MTP"
),
("srt/managers/data_parallel_controller.py", "configured_pp_size"): (
("srt/managers/data_parallel_controller.py", "pp_size"): (
"the controller lays out its schedulers' ranks before spawning them, so "
"the groups it is sizing for do not exist yet"
),
("srt/managers/data_parallel_controller.py", "configured_attn_cp_size"): (
("srt/managers/data_parallel_controller.py", "attn_cp_size"): (
"the same pre-spawn rank arithmetic"
),
("srt/managers/data_parallel_controller.py", "configured_moe_dp_size"): (
("srt/managers/data_parallel_controller.py", "moe_dp_size"): (
"the same pre-spawn rank arithmetic"
),
("srt/entrypoints/v1_loads.py", "configured_pp_size"): (
("srt/entrypoints/v1_loads.py", "pp_size"): (
"the /v1/loads accelerator count is arithmetic over the launch shape, "
"reported from the tokenizer process, which holds no model groups"
),
("srt/disaggregation/common/conn.py", "configured_pp_size"): (
("srt/disaggregation/common/conn.py", "pp_size"): (
"the bootstrap connection is built by the KV manager on the transfer "
"path, which the CPU-only conn tests exercise without ever starting "
"torch.distributed"
),
("srt/elastic_ep/elastic_ep.py", "configured_tp_size"): (
("srt/elastic_ep/elastic_ep.py", "tp_size"): (
"the joiner's rank window is computed against the size the process was "
"configured with, not the size of the group it is about to join"
),
("srt/elastic_ep/expert_backup_manager.py", "configured_tp_size"): (
("srt/elastic_ep/expert_backup_manager.py", "tp_size"): (
"the backup server counts the clients it expects to report in, which "
"is how many the launch configured -- the live group is what they are "
"still joining"
),
(
"srt/model_executor/model_runner_components/startup_weight_load.py",
"configured_tp_size",
"tp_size",
): (
"the load options are assembled in ModelRunner.__init__ for a runner "
"that may be a draft, whose groups are the target's; the configured "
@@ -182,51 +183,51 @@ _CONFIGURED_SIZE_CALL_SITES = {
),
(
"srt/model_executor/model_runner_components/startup_weight_load.py",
"configured_pp_size",
"pp_size",
): ("same options object, same reason"),
(
"srt/model_executor/model_runner_components/startup_weight_load.py",
"configured_attn_cp_size",
"attn_cp_size",
): ("same options object, same reason"),
(
"srt/model_executor/model_runner_components/startup_weight_load.py",
"configured_dcp_size",
"dcp_size",
): ("same options object, same reason"),
(
"srt/model_executor/model_runner_components/spec_aux_hidden_state.py",
"configured_tp_size",
"tp_size",
): (
"the draft KV bytes/token estimate sizes the memory pool before the "
"draft runner exists, so its shard count is configuration"
),
("srt/eplb/expert_location.py", "configured_tp_size"): (
("srt/eplb/expert_location.py", "tp_size"): (
"the elastic-EP joiner window, used to size the expert layout: the "
"size the process was configured with, not the group it is joining"
),
("srt/utils/cuda_vmm_transport_utils.py", "configured_tp_size"): (
("srt/utils/cuda_vmm_transport_utils.py", "tp_size"): (
"the consumer count is configured fan-out arithmetic (tp_size // "
"dp_size), which is what the record answered before"
),
("srt/disaggregation/encoder/runtime.py", "configured_tp_size"): (
("srt/disaggregation/encoder/runtime.py", "tp_size"): (
"the encode server's launch entry sizes its workers before it has "
"spawned any of them"
),
("srt/utils/common.py", "configured_tp_size"): (
("srt/utils/common.py", "tp_size"): (
"the require_*_tp_gather predicates compared the configured tp_size "
"when they read the record; the live property answers a different "
"question wherever the groups alias, so the configured accessor is the "
"mechanical substitution and the live one would be a semantic change"
),
("srt/model_loader/loader.py", "configured_moe_dp_size"): (
("srt/model_loader/loader.py", "moe_dp_size"): (
"the same dict already carries the live moe_dp_size under 'dp'; this entry "
"is the configured intent"
),
("srt/models/kimi_k25.py", "configured_tp_size"): (
("srt/models/kimi_k25.py", "tp_size"): (
"the IPC refcount must match the configured TP consumer count captured "
"when the tokenizer creates MmItemMemoryPool; a live attention subgroup "
"size could strand leases in the bounded pool"
),
("srt/models/kimi_k3.py", "configured_tp_size"): (
("srt/models/kimi_k3.py", "tp_size"): (
"same as kimi_k25: the IPC refcount must agree with the recycler's waiter"
),
}
@@ -533,69 +534,236 @@ class TestGlobalConfigReadRatchet(CustomTestCase):
self._check("alias-form", alias, _ALIAS_BASELINE)
def _live_shadowed_sizes() -> frozenset:
"""Names that are BOTH a live ``ParallelContext`` property and a ``parallel``
config leaf.
Derived from the two sides themselves: a size that gains a live property, or
a live property that gains a leaf, joins the registry's subject set without a
list here.
"""
from sglang.srt.arg_groups.arg_utils import namespace_of
from sglang.srt.runtime_context import ParallelContext
from sglang.srt.server_args import ServerArgs
live = {
name
for name, value in vars(ParallelContext).items()
if isinstance(value, property)
}
leaves = {
field for field, path in namespace_of(ServerArgs).items() if path == "parallel"
}
shadowed = frozenset(live & leaves)
assert shadowed, "no live-shadowed size found; the derivation is broken"
return shadowed
def _parallel_config_reads(tree, subjects):
"""Names in ``subjects`` read through the parallel bag's ``config`` hop.
Sees ``get_parallel().config.pp_size``, the module-qualified spelling, a
local bound to either hop (``p = get_parallel()`` / ``cfg = p.config``), and
the ``getattr`` form of each.
"""
fns, modules = set(), set()
for node in ast.walk(tree):
if (
isinstance(node, ast.ImportFrom)
and node.module
and node.module.endswith("runtime_context")
):
fns |= {a.asname or a.name for a in node.names if a.name == "get_parallel"}
elif isinstance(node, ast.ImportFrom) and node.module:
# `from sglang.srt import runtime_context as rc` binds the module.
for a in node.names:
if f"{node.module}.{a.name}".endswith("runtime_context"):
modules.add(a.asname or a.name)
elif isinstance(node, ast.Import):
for a in node.names:
if a.name.endswith("runtime_context"):
# Unaliased, the call site spells the whole dotted path.
modules.add(a.asname or a.name)
def dotted(node):
parts = []
while isinstance(node, ast.Attribute):
parts.append(node.attr)
node = node.value
if not isinstance(node, ast.Name):
return None
parts.append(node.id)
return ".".join(reversed(parts))
def is_bag_call(node):
if not isinstance(node, ast.Call):
return False
func = node.func
if isinstance(func, ast.Name):
return func.id in fns
return (
isinstance(func, ast.Attribute)
and func.attr == "get_parallel"
and dotted(func.value) in modules
)
bag_aliases, config_aliases = set(), set()
for _ in range(2): # a local copy of a local is still the same object
for node in ast.walk(tree):
if not isinstance(node, ast.Assign):
continue
value = node.value
if is_bag_call(value) or (
isinstance(value, ast.Name) and value.id in bag_aliases
):
bucket = bag_aliases
elif (
isinstance(value, ast.Attribute)
and value.attr == "config"
and (
is_bag_call(value.value)
or (
isinstance(value.value, ast.Name)
and value.value.id in bag_aliases
)
)
) or (isinstance(value, ast.Name) and value.id in config_aliases):
bucket = config_aliases
else:
continue
bucket |= {t.id for t in node.targets if isinstance(t, ast.Name)}
def is_config_hop(node):
return (
isinstance(node, ast.Attribute)
and node.attr == "config"
and (
is_bag_call(node.value)
or (isinstance(node.value, ast.Name) and node.value.id in bag_aliases)
)
) or (isinstance(node, ast.Name) and node.id in config_aliases)
found = set()
for node in ast.walk(tree):
if isinstance(node, ast.Attribute) and node.attr in subjects:
base, name = node.value, node.attr
elif (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id == "getattr"
and len(node.args) >= 2
and isinstance(node.args[1], ast.Constant)
and node.args[1].value in subjects
):
base, name = node.args[0], node.args[1].value
else:
continue
if is_config_hop(base):
found.add(name)
return found
_READ_SPELLINGS = (
"from sglang.srt.runtime_context import get_parallel\nx = get_parallel().config.tp_size",
"from sglang.srt.runtime_context import get_parallel as gp\nx = gp().config.tp_size",
"from sglang.srt import runtime_context as rc\nx = rc.get_parallel().config.tp_size",
"import sglang.srt.runtime_context\nx = sglang.srt.runtime_context.get_parallel().config.tp_size",
"from sglang.srt.runtime_context import get_parallel\np = get_parallel()\nx = p.config.tp_size",
"from sglang.srt.runtime_context import get_parallel\nc = get_parallel().config\nx = c.tp_size",
'from sglang.srt.runtime_context import get_parallel\nx = getattr(get_parallel().config, "tp_size")',
)
class TestParallelConfigReadSpellings(CustomTestCase):
"""``_parallel_config_reads`` resolves every spelling it claims to.
The scan below decides what the documented set is compared against, so a
spelling it cannot resolve does not fail anything -- it drops the read.
"""
def test_every_documented_spelling_resolves(self):
for source in _READ_SPELLINGS:
with self.subTest(source=source):
found = _parallel_config_reads(ast.parse(source), {"tp_size"})
self.assertEqual({"tp_size"}, set(found))
def test_the_live_property_is_not_a_config_read(self):
source = (
"from sglang.srt.runtime_context import get_parallel\n"
"x = get_parallel().tp_size"
)
self.assertEqual(
set(), set(_parallel_config_reads(ast.parse(source), {"tp_size"}))
)
class TestConfiguredSizeCallSites(CustomTestCase):
"""The configured-vs-live exceptions are enumerated, with reasons.
``configured_*_size()`` answers what the user asked for where
``get_parallel()`` would answer what the process ended up with. Each such
exception is listed above with why the live property cannot serve it, and
this case fails if the code and that list disagree.
``get_parallel().config.tp_size`` answers what the process was configured
with where the bare ``get_parallel().tp_size`` answers what the process ended
up with. Each site that needs the former is listed above with why the live
property cannot serve it, and this case fails if the code and that list
disagree.
The unit is **(file, accessor)**, not the individual call: a second
`configured_pp_size()` in a file already registered for it collapses into
the same entry, so the reason has to cover the file's use of that accessor
rather than one line. A new file, or a new accessor in a listed file, is
what this catches -- in either call form (bare or module-qualified).
The unit is **(file, size)**, not the individual read: a second
``.config.pp_size`` in a file already registered for it collapses into the
same entry, so the reason has to cover the file's use of that size rather
than one line. A new file, or a new size in a listed file, is what this
catches -- through any spelling of the hop.
"""
def test_the_call_sites_match_the_documented_set(self):
subjects = _live_shadowed_sizes()
found = set()
scanned = 0
for path in sorted(_PACKAGE_ROOT.rglob("*.py")):
rel = path.relative_to(_PACKAGE_ROOT).as_posix()
if rel.startswith(_SLOT_OWNERS):
continue
source = path.read_text()
if "configured_" not in source:
# Every spelling `_parallel_config_reads` resolves -- the direct
# call, an aliased import, a module-qualified call, a local bound to
# either hop -- needs the name in the source, so skipping the rest is
# free. Filtering on anything narrower silently empties the scan.
if "get_parallel" not in source:
continue
scanned += 1
try:
tree = ast.parse(source)
except SyntaxError:
continue
for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
func = node.func
name = (
func.id
if isinstance(func, ast.Name)
else (func.attr if isinstance(func, ast.Attribute) else None)
)
if name and name.startswith("configured_") and name.endswith("_size"):
found.add((rel, name))
found |= {(rel, name) for name in _parallel_config_reads(tree, subjects)}
self.assertGreater(
scanned,
50,
f"the pre-filter left only {scanned} files to scan; the derivation "
"is broken, not the tree",
)
documented = set(_CONFIGURED_SIZE_CALL_SITES)
self.assertEqual(
documented,
found,
"configured-size call sites drifted from their documented reasons.\n"
"configured-size reads drifted from their documented reasons.\n"
f" undocumented: {sorted(found - documented)}\n"
f" stale entries: {sorted(documented - found)}",
)
class TestNoRenamedAccessorImports(CustomTestCase):
"""The scanners above match ``get_server_args`` and ``configured_*_size``
by their literal names, so an ``import ... as`` rename would walk a read
straight past both the zero baseline and the call-site registry. Renaming
these accessors buys nothing (the names are already short and unambiguous),
so it is banned outright — which is exactly what makes literal-name
matching sound."""
"""The baseline scanner matches ``get_server_args`` by its literal name, so
an ``import ... as`` rename would walk a read straight past the zero
baseline. Renaming the accessor buys nothing (the name is already short and
unambiguous), so it is banned outright — which is exactly what makes
literal-name matching sound. (The configured-size registry resolves
``get_parallel`` aliases itself, so it needs no such ban.)"""
def test_the_scanned_accessors_are_never_import_renamed(self):
offenders = []
for path in sorted(_PACKAGE_ROOT.rglob("*.py")):
rel = path.relative_to(_PACKAGE_ROOT).as_posix()
source = path.read_text()
if "get_server_args" not in source and "configured_" not in source:
if "get_server_args" not in source:
continue
try:
tree = ast.parse(source)
@@ -608,19 +776,16 @@ class TestNoRenamedAccessorImports(CustomTestCase):
if imported.asname is None or imported.asname == imported.name:
continue
base = imported.name.rsplit(".", 1)[-1]
if base == "get_server_args" or (
base.startswith("configured_") and base.endswith("_size")
):
if base == "get_server_args":
offenders.append(
f"{rel}:{node.lineno}: {imported.name} as "
f"{imported.asname}"
)
self.assertFalse(
offenders,
"get_server_args / configured_*_size imported under another name; "
"the read ratchet and the configured-size registry match these "
"accessors by their literal names, so a rename silently escapes "
"both:\n" + "\n".join(offenders),
"get_server_args imported under another name; the read ratchet "
"matches it by its literal name, so a rename silently escapes the "
"baseline:\n" + "\n".join(offenders),
)
test/registered/unit/test_launch_path_reads_configured_sizes.py
+165 -150
View File
@@ -4,11 +4,12 @@
the process groups, so they answer only after distributed init. The launcher
decides how many processes to spawn *before* that, and a live read there raises
`Distributed environment is not initialized` -- a startup crash no unit test
reaches, because nothing short of booting a server runs the launcher.
reaches, because nothing short of booting a server runs the launcher. The
configured answer is one hop away on the same object,
`get_parallel().config.pp_size`, which reads the published `parallel` bag.
"""
import ast
import functools
import pathlib
import unittest
@@ -20,17 +21,39 @@ register_cpu_ci(est_time=9, suite="base-a-test-cpu")
_PACKAGE_ROOT = pathlib.Path(sglang.__file__).resolve().parent
# Live-shadowed sizes a launch path is known to have read. ParallelContext
# shadows more properties than these (every `_v(name, ...)` one raises the same
# "Distributed environment is not initialized"); this dict carries the ones a
# `configured_*` accessor answers, so it is a remedy map, not a census.
_LIVE_SHADOWED = {
"tp_size": "configured_tp_size()",
"pp_size": "configured_pp_size()",
"moe_dp_size": "configured_moe_dp_size()",
"attn_cp_size": "configured_attn_cp_size()",
"dcp_size": "configured_dcp_size()",
}
def _live_shadowed() -> dict:
"""{name: remedy} for every name that is BOTH a live ParallelContext
property and a `parallel` config leaf.
Derived from the two sides themselves, so a new size that gains a live
property (or a live property that gains a leaf) is watched without a second
list here. ParallelContext shadows more properties than these -- every
`_v(name, ...)` one raises the same "Distributed environment is not
initialized" -- but only a shadowed name has a configured answer to point a
launcher at.
"""
from sglang.srt.arg_groups.arg_utils import namespace_of
from sglang.srt.runtime_context import ParallelContext
from sglang.srt.server_args import ServerArgs
live = {
name
for name, value in vars(ParallelContext).items()
if isinstance(value, property)
}
leaves = {
field for field, path in namespace_of(ServerArgs).items() if path == "parallel"
}
shadowed = live & leaves
assert shadowed, (
"no live-shadowed parallel size found; the derivation is broken, not "
"the tree"
)
return {name: f"get_parallel().config.{name}" for name in sorted(shadowed)}
_LIVE_SHADOWED = _live_shadowed()
# Launch paths that decide how many children to spawn are derived below
# from the spawn itself. These launch without a size-driven spawn, so no
@@ -83,81 +106,41 @@ def _multiprocessing_names(tree):
return modules, constructors
@functools.lru_cache(maxsize=None)
def _configured_accessors() -> frozenset:
"""The `configured_*_size()` names `runtime_context` exports.
Derived from that module, so a new accessor keeps its launcher watched
without a second list here.
"""
tree = ast.parse(
(_PACKAGE_ROOT / "srt/runtime_context.py").read_text(encoding="utf-8-sig")
)
names = frozenset(
node.name
for node in tree.body
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
and node.name.startswith("configured_")
and node.name.endswith("_size")
)
assert names, (
"no configured_*_size accessors found in runtime_context; the "
"derivation is broken, not the tree"
)
return names
def _spawns_from_a_size(tree) -> bool:
"""Does any function here construct a child process *and* read one of the
five sizes -- live off the parallel bag, or through its `configured_*_size()`
answer? That is a spawn count decided from the topology.
"""Does a function here spawn a child *and* read a live-shadowed size?
Counting the configured read too is what keeps a launcher watched after it
is converted. Deriving on the live read alone means the file drops out of
the scan the moment it stops offending, so the guard would only ever watch
the launchers that already fail it.
Both tiers count: deriving on the live read alone drops a launcher from the
scan the moment it is converted, so the guard would only watch the ones
that already fail it.
"""
configured = _configured_accessors()
modules, constructors = _multiprocessing_names(tree)
names, qualified = _parallel_bag_names(tree)
aliases = _bag_aliases(tree, names, qualified)
for fn in ast.walk(tree):
if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
continue
spawns = reads = False
spawns = False
for node in ast.walk(fn):
if isinstance(node, ast.Call):
func = node.func
if isinstance(func, ast.Attribute) and func.attr in (
"Process",
"ProcessPoolExecutor",
"Popen",
"spawn",
):
# `mp.Process(`, `mp.get_context("spawn").Process(` and
# `subprocess.Popen(` all reach a child process; the
# receiver of a chained call is itself a call, so this
# cannot require a bare Name.
spawns = True
elif isinstance(func, ast.Name) and func.id in constructors:
spawns = True
if (isinstance(func, ast.Name) and func.id in configured) or (
isinstance(func, ast.Attribute) and func.attr in configured
):
reads = True
elif (
isinstance(node, ast.Attribute)
and node.attr in _LIVE_SHADOWED
and (
_is_parallel_bag_call(node.value, names, qualified)
or (isinstance(node.value, ast.Name) and node.value.id in aliases)
)
if not isinstance(node, ast.Call):
continue
func = node.func
if isinstance(func, ast.Attribute) and func.attr in (
"Process",
"ProcessPoolExecutor",
"Popen",
"spawn",
):
# A record read (`server_args.tp_size`) sizes a spawn too, but
# it cannot raise pre-dist; only the bag read is this guard's
# subject, so only it forces a module into _PRE_DIST.
reads = True
if spawns and reads:
# `mp.Process(`, `mp.get_context("spawn").Process(` and
# `subprocess.Popen(` all reach a child process; the receiver of
# a chained call is itself a call, so this cannot require a bare
# Name.
spawns = True
elif isinstance(func, ast.Name) and func.id in constructors:
spawns = True
if not spawns:
continue
# A record read (`server_args.tp_size`) sizes a spawn too, but it cannot
# raise pre-dist; only a bag read is this guard's subject.
live, configured = _shadowed_size_reads(tree, scope=fn)
if live or configured:
return True
return False
@@ -178,6 +161,12 @@ def _parallel_bag_names(tree):
names |= {
a.asname or a.name for a in node.names if a.name == "get_parallel"
}
elif isinstance(node, ast.ImportFrom) and node.module:
# `from sglang.srt import runtime_context as rc` binds the module,
# so `rc.get_parallel()` is the same call under another spelling.
for a in node.names:
if f"{node.module}.{a.name}".endswith("runtime_context"):
modules.add(a.asname or a.name)
elif isinstance(node, ast.Import):
for a in node.names:
if a.name.endswith("runtime_context"):
@@ -199,16 +188,75 @@ def _is_parallel_bag_call(node, names, modules) -> bool:
def _bag_aliases(tree, names, qualified):
"""Locals bound to the parallel bag: `p = get_parallel()` then `p.pp_size`
is the same read one line later."""
return {
target.id
for node in ast.walk(tree)
if isinstance(node, ast.Assign)
and _is_parallel_bag_call(node.value, names, qualified)
for target in node.targets
if isinstance(target, ast.Name)
}
"""Locals bound to either tier: `p = get_parallel()` then `p.pp_size` is the
same live read one line later, and `cfg = get_parallel().config` then
`cfg.pp_size` is the same configured read."""
live, config = set(), set()
for node in ast.walk(tree):
if not isinstance(node, ast.Assign):
continue
value = node.value
if _is_parallel_bag_call(value, names, qualified):
bucket = live
elif (
isinstance(value, ast.Attribute)
and value.attr == "config"
and _is_parallel_bag_call(value.value, names, qualified)
):
bucket = config
else:
continue
bucket |= {t.id for t in node.targets if isinstance(t, ast.Name)}
return live, config
def _shadowed_size_reads(module_tree, scope=None):
"""(live, configured) reads of a live-shadowed size in `scope`.
`<parallel bag>.tp_size` is the live group; `<parallel bag>.config.tp_size`
is the published leaf. Both spellings are reported so a caller can tell a
launcher that reads the topology at all from one that reads it live.
What binds the bag -- the import, a module-level alias -- lives at module
scope, so those names always come from `module_tree` even when only one
function is being walked. Deriving them from the function alone finds no
import, reports no reads, and quietly answers "this launcher reads nothing".
"""
names, qualified = _parallel_bag_names(module_tree)
live_aliases, config_aliases = _bag_aliases(module_tree, names, qualified)
def is_live_bag(node):
return _is_parallel_bag_call(node, names, qualified) or (
isinstance(node, ast.Name) and node.id in live_aliases
)
def is_config_bag(node):
return (
isinstance(node, ast.Attribute)
and node.attr == "config"
and is_live_bag(node.value)
) or (isinstance(node, ast.Name) and node.id in config_aliases)
live, configured = [], []
for node in ast.walk(scope if scope is not None else module_tree):
if isinstance(node, ast.Attribute) and node.attr in _LIVE_SHADOWED:
base, name, spelling = node.value, node.attr, "attribute"
elif (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id == "getattr"
and len(node.args) >= 2
and isinstance(node.args[1], ast.Constant)
and node.args[1].value in _LIVE_SHADOWED
):
base, name, spelling = node.args[0], node.args[1].value, "getattr"
else:
continue
if is_config_bag(base):
configured.append((node.lineno, name, spelling))
elif is_live_bag(base):
live.append((node.lineno, name, spelling))
return live, configured
def _launch_paths():
@@ -219,7 +267,7 @@ def _launch_paths():
nothing, which no derivation can reach.
"""
seen = {}
sizes = frozenset(_LIVE_SHADOWED) | _configured_accessors()
sizes = frozenset(_LIVE_SHADOWED)
for path in sorted(_PACKAGE_ROOT.rglob("*.py")):
source = path.read_text()
# Every spawn shape below names Process, ProcessPoolExecutor or Popen.
@@ -243,12 +291,12 @@ class TestLaunchPathsReadConfiguredSizes(CustomTestCase):
"""The other direction: groups exist and answer something else.
The check above proves nobody reads a live size too early. It says
nothing about what `configured_*()` returns once the groups *are* up
and answering a different number -- which is not hypothetical: elastic
EP scales the live topology away from what the operator configured, and
that divergence is the entire reason these five helpers exist. With
only the early-read direction covered, a helper that quietly delegated
to the live property would look correct.
nothing about what `.config.<size>` returns once the groups *are* up and
answering a different number -- which is not hypothetical: elastic EP
scales the live topology away from what the operator configured, and
that divergence is the entire reason the two tiers are separate. With
only the early-read direction covered, a `config` hop that quietly
delegated to the live property would look correct.
"""
import json
import os
@@ -256,11 +304,6 @@ class TestLaunchPathsReadConfiguredSizes(CustomTestCase):
from unittest.mock import patch
from sglang.srt.runtime_context import (
configured_attn_cp_size,
configured_dcp_size,
configured_moe_dp_size,
configured_pp_size,
configured_tp_size,
get_parallel,
publish,
reset_context,
@@ -318,26 +361,16 @@ class TestLaunchPathsReadConfiguredSizes(CustomTestCase):
if isinstance(getter, ast.Attribute):
live_getter[call.args[0].value] = getter.attr
state = "sglang.srt.distributed.parallel_state"
helpers = {
"tp_size": configured_tp_size,
"pp_size": configured_pp_size,
"moe_dp_size": configured_moe_dp_size,
"attn_cp_size": configured_attn_cp_size,
"dcp_size": configured_dcp_size,
}
missing = sorted(set(helpers) - set(live_getter))
missing = sorted(set(_LIVE_SHADOWED) - set(live_getter))
self.assertEqual(
missing,
[],
f"these sizes no longer have a live property to diverge from: {missing}",
)
cases = tuple(
(name, helper, f"{state}.{live_getter[name]}")
for name, helper in helpers.items()
)
for name, helper, target in cases:
for name in sorted(_LIVE_SHADOWED):
with self.subTest(size=name):
configured = helper()
target = f"{state}.{live_getter[name]}"
configured = getattr(get_parallel().config, name)
with patch(target, return_value=configured + 41):
self.assertEqual(
get_parallel().__getattribute__(name),
@@ -345,48 +378,30 @@ class TestLaunchPathsReadConfiguredSizes(CustomTestCase):
f"{name} no longer follows the live topology",
)
self.assertEqual(
helper(),
getattr(get_parallel().config, name),
configured,
f"configured_{name}() followed the live topology instead "
"of the published configuration",
f"get_parallel().config.{name} followed the live topology "
"instead of the published configuration",
)
# A bare read of a leaf with no live property is not a config read any
# more, and the error says where it went. Spelled through `getattr` so a
# mechanical `.config` sweep cannot "fix" the very read under test.
with self.assertRaisesRegex(
AttributeError, r"read it as get_parallel\(\)\.config\.nccl_port"
):
getattr(get_parallel(), "nccl_port")
reset_context()
def test_no_live_topology_read_before_distributed_init(self):
offenders = []
for rel, tree in _launch_paths():
names, modules = _parallel_bag_names(tree)
aliases = _bag_aliases(tree, names, modules)
for node in ast.walk(tree):
if isinstance(node, ast.Attribute) and node.attr in _LIVE_SHADOWED:
base = node.value
if _is_parallel_bag_call(base, names, modules) or (
isinstance(base, ast.Name) and base.id in aliases
):
offenders.append(
f"{rel}:{node.lineno} reads the live {node.attr}; "
f"use {_LIVE_SHADOWED[node.attr]}"
)
elif (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id == "getattr"
and len(node.args) >= 2
and isinstance(node.args[1], ast.Constant)
and node.args[1].value in _LIVE_SHADOWED
and (
_is_parallel_bag_call(node.args[0], names, modules)
or (
isinstance(node.args[0], ast.Name)
and node.args[0].id in aliases
)
)
):
offenders.append(
f"{rel}:{node.lineno} reads the live "
f"{node.args[1].value} through getattr; "
f"use {_LIVE_SHADOWED[node.args[1].value]}"
)
live, _ = _shadowed_size_reads(tree)
for lineno, name, spelling in live:
through = " through getattr" if spelling == "getattr" else ""
offenders.append(
f"{rel}:{lineno} reads the live {name}{through}; "
f"use {_LIVE_SHADOWED[name]}"
)
self.assertEqual(
offenders,
[],
test/registered/unit/test_runtime_context.py
+8 -9
View File
@@ -907,12 +907,11 @@ class TestForwardFlags(_IsolatedServerArgs):
self.assertEqual(probe(torch.zeros(())).item(), 0)
def test_parallel_config_leaves_trace_under_torch_compile(self):
# Regression: parallel config leaves resolve through
# ``ParallelContext.__getattr__`` (the bag fallback), and gate helpers
# such as ``enable_moe_dense_fully_dp()`` read them inside compiled
# model forwards — the fallback body must stay dynamo-traceable
# (``object.__getattribute__`` graph-breaks). fullgraph=True turns any
# graph break back into a failure.
# Regression: gate helpers such as ``enable_moe_dense_fully_dp()`` read
# parallel config leaves inside compiled model forwards through the
# `config` property, which must stay dynamo-traceable
# (``object.__getattribute__`` graph-breaks).
# fullgraph=True turns any graph break back into a failure.
import torch
from sglang.srt.runtime_context import get_parallel
@@ -923,11 +922,11 @@ class TestForwardFlags(_IsolatedServerArgs):
@torch.compile(fullgraph=True, backend="eager", dynamic=False)
def probe(x):
par = get_parallel()
if par.enable_prefill_context_parallel:
if par.config.enable_prefill_context_parallel:
x = x + 1
if par.moe_dense_tp_size == 1:
if par.config.moe_dense_tp_size == 1:
x = x + 2
if par.dwdp_size > 1:
if par.config.dwdp_size > 1:
x = x + 4
return x
test/registered/unit/test_runtime_context_override.py
+1 -1
View File
@@ -87,7 +87,7 @@ class TestContextOverride(CustomTestCase):
speculative_accept_threshold_single=0.5,
speculative_accept_threshold_acc=0.9,
)
self.assertEqual(rc.get_parallel().pp_max_micro_batch_size, 8)
self.assertEqual(rc.get_parallel().config.pp_max_micro_batch_size, 8)
self.assertEqual(rc.get_spec().speculative_accept_threshold_single, 0.5)
self.assertEqual(rc.get_spec().speculative_accept_threshold_acc, 0.9)
test/registered/unit/test_server_args_namespaces.py
+6 -3
View File
@@ -72,10 +72,9 @@ class TestServerArgsNamespaces(CustomTestCase):
accessors = {
node.name
for node in context_module.body
if isinstance(node, ast.FunctionDef)
and (node.name.startswith("get_") or node.name.startswith("configured_"))
if isinstance(node, ast.FunctionDef) and node.name.startswith("get_")
}
self.assertGreater(len(accessors), 20, "the accessor derivation broke")
self.assertGreater(len(accessors), 15, "the accessor derivation broke")
shadowed = []
for path in sorted(srt.rglob("*.py")):
@@ -184,6 +183,10 @@ class TestServerArgsNamespaces(CustomTestCase):
continue
sites += 1
read = [cursor.func.id[len("get_") :]] + chain[:-1]
if read[:2] == ["parallel", "config"]:
# `config` on `get_parallel()` is the tier hop, not a
# sub-namespace: bare names there are the live topology.
del read[1]
if mapping[field].split(".") != read:
disagreements.append(
f"{path.relative_to(srt)}:{node.lineno} reads "
test/registered/unit/test_supplied_instance_exposure_ratchet.py
+16 -2
View File
@@ -160,6 +160,8 @@ _EXPOSED = {
("configs/model_config.py", "quantization"),
("configs/model_config.py", "speculative_algorithm"),
("configs/model_config.py", "speculative_draft_model_quantization"),
("dllm/config.py", "max_running_requests"),
("dllm/config.py", "model_path"),
("entrypoints/engine.py", "enable_symm_mem"),
("entrypoints/engine.py", "reasoning_parser"),
("entrypoints/engine.py", "tool_call_parser"),
@@ -171,20 +173,25 @@ _EXPOSED = {
("layers/cp/bcg.py", "cp_strategy"),
("layers/cp/bcg.py", "enable_prefill_cp"),
("layers/flashinfer_comm_fusion.py", "flashinfer_allreduce_fusion_backend"),
("layers/moe/utils.py", "deepep_mode"),
("layers/moe/utils.py", "moe_a2a_backend"),
("layers/moe/utils.py", "moe_runner_backend"),
("layers/moe/utils.py", "quantization"),
("layers/moe/utils.py", "speculative_moe_runner_backend"),
("lora/lora_manager.py", "enable_lora_overlap_loading"),
("lora/marlin_lora_temp/policy.py", "lora_paths"),
("model_loader/expert_pack_runtime.py", "model_path"),
("model_loader/expert_pack_runtime.py", "tokenizer_path"),
("multimodal/processors/base_processor.py", "image_processor_backend"),
("parser/template_detection.py", "model_path"),
("speculative/adaptive_spec_params.py", "speculative_algorithm"),
("speculative/adaptive_spec_params.py", "speculative_eagle_topk"),
("speculative/draft_worker_common.py", "speculative_draft_attention_backend"),
("speculative/spec_info.py", "enable_multi_layer_eagle"),
("speculative/spec_registry.py", "disable_overlap_schedule"),
("utils/common.py", "speculative_num_draft_tokens"),
("utils/common.py", "speculative_num_steps"),
("utils/hf_transformers/processor.py", "image_processor_backend"),
# The daemon command and constructor snapshot the resolved startup layout
# before the daemon's loading lifecycle can apply any runtime overrides.
("weight_cache/daemon.py", "attn_cp_size"),
("weight_cache/daemon.py", "deepep_mode"),
("weight_cache/daemon.py", "dp_size"),
@@ -214,6 +221,8 @@ _EXPOSED_CUDA_ONLY: frozenset = frozenset()
# some code overrides post-publish. Each needs an ordering judgment, not a blanket
# conversion; the list exists so a new one is a decision made when it is written.
_OVERRIDDEN_AND_READ = {
("configs/model_config.py", "dtype"),
("configs/model_config.py", "model_path"),
("dllm/config.py", "model_path"),
("entrypoints/engine.py", "reasoning_parser"),
("entrypoints/engine.py", "tool_call_parser"),
@@ -234,6 +243,11 @@ _OVERRIDDEN_AND_READ = {
("parser/template_detection.py", "model_path"),
("utils/common.py", "speculative_num_draft_tokens"),
("utils/common.py", "speculative_num_steps"),
("weight_cache/daemon.py", "dp_size"),
("weight_cache/daemon.py", "dtype"),
("weight_cache/daemon.py", "ep_size"),
("weight_cache/daemon.py", "load_format"),
("weight_cache/daemon.py", "model_path"),
}