config: project the config bags from the resolution result (#35906)
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Cheng Wan
co-authored by
Claude Opus 5
Claude Opus 5
parent
0e22777572
commit
4bc79a1b49
@@ -1,7 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.kv_canary.capacities import CanaryLaunchCapacities
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
@@ -9,6 +8,7 @@ from sglang.srt.model_executor.cuda_graph_config import (
|
||||
CudaGraphConfig,
|
||||
PhaseConfig,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -16,28 +16,33 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestComputeLaunchCapacities(CustomTestCase):
|
||||
@staticmethod
|
||||
def _make_server_args(*, max_bs: int) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
cuda_graph_config=CudaGraphConfig(
|
||||
decode=PhaseConfig(backend=Backend.FULL, max_bs=max_bs)
|
||||
),
|
||||
speculative_num_draft_tokens=0,
|
||||
chunked_prefill_size=None,
|
||||
max_prefill_tokens=128,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _from_args(
|
||||
self,
|
||||
*,
|
||||
max_bs: int,
|
||||
max_seq_len: int,
|
||||
max_total_num_tokens: int | None = None,
|
||||
speculative_num_draft_tokens: int | None = 0,
|
||||
) -> CanaryLaunchCapacities:
|
||||
"""`from_args` reads the published configuration, so publish one.
|
||||
|
||||
Handing it a stand-in object stopped meaning anything when the reads
|
||||
moved to the config bags: the parameter was ignored and the values
|
||||
under test came from whatever the process had published.
|
||||
"""
|
||||
if max_total_num_tokens is None:
|
||||
max_total_num_tokens = max_bs * max_seq_len
|
||||
override = get_context().override_server_args(
|
||||
cuda_graph_config=CudaGraphConfig(
|
||||
decode=PhaseConfig(backend=Backend.FULL, max_bs=max_bs)
|
||||
),
|
||||
speculative_num_draft_tokens=speculative_num_draft_tokens,
|
||||
chunked_prefill_size=None,
|
||||
max_prefill_tokens=128,
|
||||
)
|
||||
override.install()
|
||||
self.addCleanup(override.restore)
|
||||
return CanaryLaunchCapacities.from_args(
|
||||
server_args=TestComputeLaunchCapacities._make_server_args(max_bs=max_bs),
|
||||
req_to_token_pool_size=max_bs,
|
||||
max_seq_len_per_req=max_seq_len,
|
||||
pool_slot_count=max_total_num_tokens,
|
||||
@@ -60,14 +65,11 @@ class TestComputeLaunchCapacities(CustomTestCase):
|
||||
|
||||
def test_from_args_treats_missing_speculative_draft_tokens_as_zero(self) -> None:
|
||||
"""per_forward_write_entry_capacity is floored by max_prefill_tokens when batch * tokens_per_req is smaller."""
|
||||
server_args = self._make_server_args(max_bs=2)
|
||||
server_args.speculative_num_draft_tokens = None
|
||||
|
||||
capacities = CanaryLaunchCapacities.from_args(
|
||||
server_args=server_args,
|
||||
req_to_token_pool_size=2,
|
||||
max_seq_len_per_req=32,
|
||||
pool_slot_count=64,
|
||||
capacities = self._from_args(
|
||||
max_bs=2,
|
||||
max_seq_len=32,
|
||||
max_total_num_tokens=64,
|
||||
speculative_num_draft_tokens=None,
|
||||
)
|
||||
|
||||
self.assertEqual(capacities.per_forward_write_entry_capacity, 128)
|
||||
@@ -84,12 +86,7 @@ class TestComputeLaunchCapacities(CustomTestCase):
|
||||
def test_from_args_rejects_empty_pool_capacity(self) -> None:
|
||||
"""Verify derived launch capacities reject invalid pool sizing."""
|
||||
with self.assertRaisesRegex(ValueError, "pool_slot_count"):
|
||||
CanaryLaunchCapacities.from_args(
|
||||
server_args=self._make_server_args(max_bs=1),
|
||||
req_to_token_pool_size=1,
|
||||
max_seq_len_per_req=1,
|
||||
pool_slot_count=0,
|
||||
)
|
||||
self._from_args(max_bs=1, max_seq_len=1, max_total_num_tokens=0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -2,6 +2,8 @@ import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.runtime_context import publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -17,14 +19,21 @@ def _req(output_len: int, input_len: int = 8, priority=None):
|
||||
|
||||
|
||||
def _args(policy: str = "length", low_first: bool = False):
|
||||
return SimpleNamespace(
|
||||
"""The retraction order reads the schedule bag, so the policy has to be
|
||||
published rather than handed in."""
|
||||
return ServerArgs(
|
||||
model_path="dummy",
|
||||
retraction_policy=policy,
|
||||
schedule_low_priority_values_first=low_first,
|
||||
)
|
||||
|
||||
|
||||
def _order(reqs, args):
|
||||
return ScheduleBatch._get_decode_retraction_order(reqs, args)
|
||||
publish(args, role="test")
|
||||
try:
|
||||
return ScheduleBatch._get_decode_retraction_order(reqs)
|
||||
finally:
|
||||
reset_context()
|
||||
|
||||
|
||||
class TestRetractionOrder(CustomTestCase):
|
||||
|
||||
@@ -20,6 +20,7 @@ from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
|
||||
from sglang.srt.managers.scheduler import Scheduler
|
||||
from sglang.srt.managers.scheduler_components.pool_stats_observer import PoolStats
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.runtime_context import publish, reset_context
|
||||
from sglang.srt.sampling.sampling_params import SamplingParams
|
||||
|
||||
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
|
||||
@@ -27,6 +28,15 @@ register_cpu_ci(est_time=9, suite="base-c-test-cpu")
|
||||
|
||||
|
||||
class TestSchedulerPauseGeneration(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# The scheduler runs after its process publishes; retraction reads the
|
||||
# disaggregation and schedule bags rather than the record it is handed.
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
super().setUp()
|
||||
publish(ServerArgs(model_path="dummy"), role="test")
|
||||
self.addCleanup(reset_context)
|
||||
|
||||
def _new_scheduler(self) -> Scheduler:
|
||||
scheduler = Scheduler.__new__(Scheduler)
|
||||
scheduler._engine_paused = False
|
||||
|
||||
@@ -15,6 +15,7 @@ see.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import copy
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
@@ -22,8 +23,11 @@ import pathlib
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
import unittest.mock
|
||||
|
||||
import sglang
|
||||
from sglang.srt import server_args as server_args_module
|
||||
from sglang.srt.arg_groups.overrides import resolution_result
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
@@ -203,6 +207,36 @@ def _stash_overlay(server_args):
|
||||
return overlay
|
||||
|
||||
|
||||
def _live_topology_leaves():
|
||||
"""Names `ParallelContext` serves from the live topology, not the config.
|
||||
|
||||
Read out of the class: each shadowed name arrives as `self._v("<name>",
|
||||
<getter>)`. Inferring them from "did the read raise" is wrong -- it only
|
||||
raises while the process groups are missing, so in a process where an
|
||||
earlier test built them the property answers the *live* size and a leaf
|
||||
check reads it as a config mismatch (`parallel.tp_size: bag=1
|
||||
resolution=2`). Whether they are shadowed is a property of the class, not
|
||||
of the process.
|
||||
"""
|
||||
tree = ast.parse((_SRT / "runtime_context.py").read_text(encoding="utf-8-sig"))
|
||||
parallel = next(
|
||||
node
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.ClassDef) and node.name == "ParallelContext"
|
||||
)
|
||||
names = set()
|
||||
for node in ast.walk(parallel):
|
||||
if (
|
||||
isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr == "_v"
|
||||
and node.args
|
||||
and isinstance(node.args[0], ast.Constant)
|
||||
):
|
||||
names.add(node.args[0].value)
|
||||
return frozenset(names)
|
||||
|
||||
|
||||
class TestResolutionDeclarations(CustomTestCase):
|
||||
def setUp(self):
|
||||
# Resolution writes environment variables, and those outlive the
|
||||
@@ -275,6 +309,224 @@ class TestResolutionDeclarations(CustomTestCase):
|
||||
+ "\n ".join(unexplained),
|
||||
)
|
||||
|
||||
def test_the_projection_input_is_the_resolved_configuration(self):
|
||||
"""What the bags are built from equals what the record ends up holding.
|
||||
|
||||
The projection reads `raw input + declarations` rather than the
|
||||
fields, so that it keeps working when the declarations stop
|
||||
materializing. While they still do, the two have to agree leaf for
|
||||
leaf -- a difference means the projection would publish something the
|
||||
record does not say, which is the failure this whole transition is
|
||||
meant to avoid.
|
||||
"""
|
||||
from sglang.srt.arg_groups.arg_utils import namespace_of
|
||||
from sglang.srt.arg_groups.overrides import resolution_result
|
||||
|
||||
differences = []
|
||||
for shape in _SHAPES:
|
||||
server_args = self._resolve(shape)
|
||||
for field in namespace_of(type(server_args)):
|
||||
projected = resolution_result(server_args, field)
|
||||
on_record = getattr(server_args, field)
|
||||
if projected != on_record:
|
||||
differences.append(
|
||||
f"{shape} -> {field}: projection={projected!r} "
|
||||
f"record={on_record!r}"
|
||||
)
|
||||
self.assertEqual(
|
||||
differences,
|
||||
[],
|
||||
"the projection and the record disagree about a config leaf:\n "
|
||||
+ "\n ".join(differences),
|
||||
)
|
||||
|
||||
def test_every_published_leaf_is_what_resolution_decided(self):
|
||||
"""One hop further than the check above: the leaf a reader reads.
|
||||
|
||||
The projection's *input* agreeing with the record says nothing about
|
||||
the last hop: whether the leaf is reachable through the path the
|
||||
metadata declares, and whether it carries the resolved value once it
|
||||
is. Both sides here come from that metadata, so this cannot tell that
|
||||
a field is assigned to the *wrong* group -- the readers are the
|
||||
independent source for that, and
|
||||
`test_server_args_namespaces.py::test_the_readers_agree_with_the_namespace_metadata`
|
||||
is where the two are compared.
|
||||
"""
|
||||
import sglang.srt.runtime_context as runtime_context
|
||||
from sglang.srt.arg_groups.arg_utils import namespace_of
|
||||
from sglang.srt.arg_groups.overrides import resolution_result
|
||||
from sglang.srt.runtime_context import publish, reset_context
|
||||
|
||||
mapping = namespace_of(ServerArgs)
|
||||
self.assertGreater(len(mapping), 400, "the namespace mapping collapsed")
|
||||
|
||||
shadowed = _live_topology_leaves()
|
||||
self.assertGreaterEqual(
|
||||
shadowed
|
||||
& {
|
||||
"tp_size",
|
||||
"pp_size",
|
||||
"moe_dp_size",
|
||||
"attn_cp_size",
|
||||
"dcp_size",
|
||||
},
|
||||
{"tp_size", "pp_size", "moe_dp_size", "attn_cp_size", "dcp_size"},
|
||||
"a parallel size stopped being served from the live topology; if it "
|
||||
"is a plain config leaf now, it belongs in the comparison below",
|
||||
)
|
||||
|
||||
compared = 0
|
||||
unreachable, mismatched = [], []
|
||||
for shape in _SHAPES:
|
||||
self.addCleanup(reset_context)
|
||||
server_args = self._resolve(shape)
|
||||
publish(server_args, role="scheduler")
|
||||
for field, path in mapping.items():
|
||||
if field in shadowed:
|
||||
# Served from the process groups by design; `configured_*()`
|
||||
# is what answers with the configured value, and
|
||||
# test_launch_path_reads_configured_sizes pins that.
|
||||
continue
|
||||
groups = path.split(".")
|
||||
accessor = getattr(runtime_context, f"get_{groups[0]}", None)
|
||||
if accessor is None:
|
||||
unreachable.append(f"no get_{groups[0]}() for {path}.{field}")
|
||||
continue
|
||||
node = accessor()
|
||||
try:
|
||||
for group in groups[1:]:
|
||||
node = getattr(node, group)
|
||||
leaf = getattr(node, field)
|
||||
except Exception as exc:
|
||||
unreachable.append(f"{path}.{field}: {type(exc).__name__}: {exc}")
|
||||
continue
|
||||
decided = resolution_result(server_args, field)
|
||||
compared += 1
|
||||
if leaf is not decided and leaf != decided:
|
||||
mismatched.append(
|
||||
f"{shape} -> {path}.{field}: bag={leaf!r} resolution={decided!r}"
|
||||
)
|
||||
reset_context()
|
||||
self.assertEqual(
|
||||
unreachable,
|
||||
[],
|
||||
"these leaves are mapped to a namespace that cannot serve them, so "
|
||||
"a reader following the mapping raises:\n " + "\n ".join(unreachable),
|
||||
)
|
||||
self.assertEqual(
|
||||
mismatched,
|
||||
[],
|
||||
"the published leaf and the resolution result disagree:\n "
|
||||
+ "\n ".join(mismatched),
|
||||
)
|
||||
self.assertGreater(
|
||||
compared, 2000, f"only {compared} leaves were compared; the walk broke"
|
||||
)
|
||||
|
||||
def test_a_child_that_received_the_record_publishes_the_same_bags(self):
|
||||
"""A forked worker gets the record by pickle, and re-projects from it.
|
||||
|
||||
Every process publishes, so a child's bags are only right if the
|
||||
declarations travelled with the object -- and the gate has to hold on
|
||||
the far side, or the child re-runs handlers over their own output. The
|
||||
parent's bags are the reference: this is the multi-process half of the
|
||||
projection, and nothing else exercises it.
|
||||
"""
|
||||
import pickle
|
||||
|
||||
import sglang.srt.runtime_context as runtime_context
|
||||
from sglang.srt.arg_groups.arg_utils import namespace_of
|
||||
from sglang.srt.runtime_context import publish, reset_context
|
||||
|
||||
mapping = namespace_of(ServerArgs)
|
||||
|
||||
def leaves():
|
||||
out = {}
|
||||
for field, path in mapping.items():
|
||||
groups = path.split(".")
|
||||
accessor = getattr(runtime_context, f"get_{groups[0]}", None)
|
||||
if accessor is None:
|
||||
continue
|
||||
node = accessor()
|
||||
try:
|
||||
for group in groups[1:]:
|
||||
node = getattr(node, group)
|
||||
out[f"{path}.{field}"] = repr(getattr(node, field))
|
||||
except Exception:
|
||||
continue
|
||||
return out
|
||||
|
||||
for shape in _SHAPES:
|
||||
self.addCleanup(reset_context)
|
||||
parent = self._resolve(shape)
|
||||
publish(parent, role="scheduler")
|
||||
expected = leaves()
|
||||
|
||||
blob = pickle.dumps(parent)
|
||||
reset_context()
|
||||
child = pickle.loads(blob)
|
||||
entered = []
|
||||
original = ServerArgs._run_resolution_pipeline
|
||||
|
||||
def counted(self, _original=original):
|
||||
entered.append(1)
|
||||
return _original(self)
|
||||
|
||||
with unittest.mock.patch.object(
|
||||
ServerArgs, "_run_resolution_pipeline", counted
|
||||
):
|
||||
publish(child, role="scheduler")
|
||||
self.assertEqual(
|
||||
entered,
|
||||
[],
|
||||
f"{shape}: the child resolved again, so its handlers ran over "
|
||||
"the parent's output",
|
||||
)
|
||||
differences = {
|
||||
key: (expected[key], value)
|
||||
for key, value in leaves().items()
|
||||
if expected.get(key) != value
|
||||
}
|
||||
self.assertEqual(
|
||||
differences,
|
||||
{},
|
||||
f"{shape}: the child published different values than the "
|
||||
f"parent: {differences}",
|
||||
)
|
||||
reset_context()
|
||||
|
||||
def test_late_resolution_reaches_the_projection(self):
|
||||
"""Resolution staged after `__post_init__` is still resolution.
|
||||
|
||||
The parser detection and the LoRA normalization run at launcher stage --
|
||||
they need a tokenizer, a chat template, an adapter directory -- and they
|
||||
write through `declare_late_resolution`. If those writes only reached
|
||||
the fields, the bags would describe the *unresolved* value: a server
|
||||
launched with `--reasoning-parser auto` would advertise and apply
|
||||
`auto` after detection had already replaced it.
|
||||
|
||||
A real model path, not the dummy one: a dummy record never materializes,
|
||||
so its `resolve_once` re-runs and re-snapshots the raw input from
|
||||
already-late-resolved fields, which hides exactly this.
|
||||
"""
|
||||
from sglang.srt.arg_groups.overrides import declare_late_resolution
|
||||
from sglang.srt.runtime_context import get_serving, publish, reset_context
|
||||
|
||||
server_args = self._resolve({"reasoning_parser": "auto"})
|
||||
self.addCleanup(reset_context)
|
||||
declare_late_resolution(
|
||||
server_args, "template-detection", reasoning_parser="qwen3"
|
||||
)
|
||||
self.assertEqual(
|
||||
resolution_result(server_args, "reasoning_parser"),
|
||||
"qwen3",
|
||||
"the projection still reports what the caller asked for, so the "
|
||||
"bags would publish an unresolved parser",
|
||||
)
|
||||
publish(server_args, role="tokenizer")
|
||||
self.assertEqual(get_serving().reasoning_parser, "qwen3")
|
||||
self.assertEqual(server_args.reasoning_parser, get_serving().reasoning_parser)
|
||||
|
||||
def test_the_stash_agrees_with_the_fields_it_declared(self):
|
||||
mismatches = []
|
||||
for shape in _SHAPES:
|
||||
@@ -362,6 +614,90 @@ class TestResolutionDeclarations(CustomTestCase):
|
||||
+ "\n ".join(inversions),
|
||||
)
|
||||
|
||||
def test_a_nested_resolution_decision_reaches_the_bags(self):
|
||||
"""Resolution also decides *inside* a declared object.
|
||||
|
||||
The graph sizing writes `cuda_graph_config.decode.max_bs` through the
|
||||
object the parse step declared -- no field is assigned, so nothing
|
||||
records it. It reaches the bags because the stash holds that same
|
||||
object; a copy taken when it was declared would publish the `None` the
|
||||
parse step declared while the process runs with a real batch size.
|
||||
"""
|
||||
from sglang.srt.runtime_context import get_exec, publish, reset_context
|
||||
|
||||
server_args = self._resolve({"disaggregation_mode": "prefill"})
|
||||
self.addCleanup(reset_context)
|
||||
# Snapshot before publishing: the bag serves the very object the record
|
||||
# holds, so comparing them after the fact compares an object with
|
||||
# itself and passes however the projection behaves.
|
||||
expected = copy.deepcopy(server_args.cuda_graph_config)
|
||||
publish(server_args, role="scheduler")
|
||||
published = get_exec().graph.cuda_graph_config
|
||||
resolved = expected
|
||||
self.assertIsNotNone(
|
||||
published.decode.max_bs,
|
||||
"the published graph config carries the batch size the parse step "
|
||||
"declared, not the one the sizing handler decided",
|
||||
)
|
||||
self.assertEqual(
|
||||
(
|
||||
published.decode.max_bs,
|
||||
published.decode.backend,
|
||||
published.prefill.max_bs,
|
||||
published.prefill.backend,
|
||||
),
|
||||
(
|
||||
resolved.decode.max_bs,
|
||||
resolved.decode.backend,
|
||||
resolved.prefill.max_bs,
|
||||
resolved.prefill.backend,
|
||||
),
|
||||
"the bags and the record disagree about the graph configuration, "
|
||||
"so a decision made inside the declared object was dropped",
|
||||
)
|
||||
|
||||
def test_every_platform_hook_that_takes_the_record_is_captured(self):
|
||||
"""A second out-of-tree config hook must not arrive uncaptured.
|
||||
|
||||
`apply_server_args_defaults` is the one method on the platform
|
||||
interface that is handed the record, and its implementations live in
|
||||
other distributions -- no source scan of this tree can see what they
|
||||
write, so the pipeline diffs the record across the call instead. A new
|
||||
hook of the same shape would be invisible again, and this is what
|
||||
notices. Derived from the interface rather than listed: a rename keeps
|
||||
working, an addition fails.
|
||||
"""
|
||||
interface = _SRT / "platforms" / "interface.py"
|
||||
tree = ast.parse(interface.read_text(encoding="utf-8-sig"))
|
||||
taking_the_record = set()
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
continue
|
||||
arguments = node.args
|
||||
names = [
|
||||
arg.arg
|
||||
for arg in arguments.posonlyargs + arguments.args + arguments.kwonlyargs
|
||||
]
|
||||
if any(name == "server_args" or name.endswith("_args") for name in names):
|
||||
taking_the_record.add(node.name)
|
||||
self.assertEqual(
|
||||
taking_the_record,
|
||||
{"apply_server_args_defaults"},
|
||||
"the platform interface hands the startup record to a method this "
|
||||
"test does not know about; either it only reads, or its writes need "
|
||||
"capturing like apply_server_args_defaults",
|
||||
)
|
||||
|
||||
pipeline = (_SRT / "server_args.py").read_text(encoding="utf-8-sig")
|
||||
for hook in sorted(taking_the_record):
|
||||
self.assertIn(
|
||||
f"current_platform.{hook},",
|
||||
pipeline,
|
||||
f"{hook} is called directly instead of through the write "
|
||||
"capture, so an out-of-tree plugin's defaults would be dropped "
|
||||
"by the projection",
|
||||
)
|
||||
|
||||
def test_the_shapes_reach_the_fields_they_are_meant_to(self):
|
||||
"""A green agreement check over an empty stash would prove nothing."""
|
||||
declared = set()
|
||||
@@ -376,6 +712,41 @@ class TestResolutionDeclarations(CustomTestCase):
|
||||
+ "\n ".join(missing),
|
||||
)
|
||||
|
||||
def test_a_platform_plugin_default_reaches_the_projection(self):
|
||||
"""An out-of-tree platform writes the fields; the diff declares them.
|
||||
|
||||
The plugin interface is not ours to convert -- implementations live in
|
||||
other distributions -- so its writes are captured rather than declared.
|
||||
Without the capture the projection falls through to the raw snapshot,
|
||||
which was taken before the plugin ran, and publishes the value the
|
||||
plugin overrode.
|
||||
"""
|
||||
|
||||
# The pipeline asks the platform other questions on the way through
|
||||
# (whether it is out of tree, whether it supports piecewise capture),
|
||||
# and which of those it reaches depends on the host.
|
||||
class _Plugin(type(server_args_module.current_platform)):
|
||||
device_name = "oot"
|
||||
|
||||
def apply_server_args_defaults(self, server_args):
|
||||
server_args.attention_backend = "triton"
|
||||
server_args.schedule_conservativeness = 0.5
|
||||
|
||||
with unittest.mock.patch.object(
|
||||
server_args_module, "current_platform", _Plugin()
|
||||
):
|
||||
server_args = self._resolve({})
|
||||
self.assertEqual(
|
||||
(
|
||||
resolution_result(server_args, "attention_backend"),
|
||||
resolution_result(server_args, "schedule_conservativeness"),
|
||||
),
|
||||
("triton", 0.5),
|
||||
"the platform plugin's defaults did not reach the resolution "
|
||||
"result, so the projection publishes what the operator passed "
|
||||
"instead of what the platform decided",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -104,6 +104,54 @@ _CONFIGURED_SIZE_CALL_SITES = {
|
||||
("srt/managers/scheduler.py", "configured_attn_cp_size"): (
|
||||
"same pre-distributed-init arithmetic in configure_scheduler_process"
|
||||
),
|
||||
("srt/managers/scheduler.py", "configured_dcp_size"): (
|
||||
"same pre-distributed-init arithmetic in configure_scheduler_process"
|
||||
),
|
||||
("srt/disaggregation/common/conn.py", "configured_pp_size"): (
|
||||
"the bootstrap connection is built by the KV manager on the transfer "
|
||||
"path, which the CPU-only conn tests exercise without ever starting "
|
||||
"torch.distributed"
|
||||
),
|
||||
("srt/elastic_ep/elastic_ep.py", "configured_tp_size"): (
|
||||
"the joiner's rank window is computed against the size the process was "
|
||||
"configured with, not the size of the group it is about to join"
|
||||
),
|
||||
("srt/elastic_ep/expert_backup_manager.py", "configured_tp_size"): (
|
||||
"the backup server counts the clients it expects to report in, which "
|
||||
"is how many the launch configured -- the live group is what they are "
|
||||
"still joining"
|
||||
),
|
||||
(
|
||||
"srt/model_executor/model_runner_components/startup_weight_load.py",
|
||||
"configured_tp_size",
|
||||
): (
|
||||
"the load options are assembled in ModelRunner.__init__ for a runner "
|
||||
"that may be a draft, whose groups are the target's; the configured "
|
||||
"sizes are what the record answered before"
|
||||
),
|
||||
(
|
||||
"srt/model_executor/model_runner_components/startup_weight_load.py",
|
||||
"configured_pp_size",
|
||||
): ("same options object, same reason"),
|
||||
(
|
||||
"srt/model_executor/model_runner_components/startup_weight_load.py",
|
||||
"configured_attn_cp_size",
|
||||
): ("same options object, same reason"),
|
||||
(
|
||||
"srt/model_executor/model_runner_components/startup_weight_load.py",
|
||||
"configured_dcp_size",
|
||||
): ("same options object, same reason"),
|
||||
(
|
||||
"srt/model_executor/model_runner_components/spec_aux_hidden_state.py",
|
||||
"configured_tp_size",
|
||||
): (
|
||||
"the draft KV bytes/token estimate sizes the memory pool before the "
|
||||
"draft runner exists, so its shard count is configuration"
|
||||
),
|
||||
("srt/eplb/expert_location.py", "configured_tp_size"): (
|
||||
"the elastic-EP joiner window, used to size the expert layout: the "
|
||||
"size the process was configured with, not the group it is joining"
|
||||
),
|
||||
("srt/utils/cuda_vmm_transport_utils.py", "configured_tp_size"): (
|
||||
"the consumer count is configured fan-out arithmetic (tp_size // "
|
||||
"dp_size), which is what the record answered before"
|
||||
|
||||
@@ -28,7 +28,7 @@ _LIVE_SHADOWED = {
|
||||
"pp_size": "configured_pp_size()",
|
||||
"moe_dp_size": "configured_moe_dp_size()",
|
||||
"attn_cp_size": "configured_attn_cp_size()",
|
||||
"dcp_size": "a configured accessor (none exists yet; add one beside configured_pp_size)",
|
||||
"dcp_size": "configured_dcp_size()",
|
||||
}
|
||||
|
||||
# Launch paths that decide how many children to spawn are derived below
|
||||
@@ -234,6 +234,119 @@ def _launch_paths():
|
||||
|
||||
|
||||
class TestLaunchPathsReadConfiguredSizes(CustomTestCase):
|
||||
def test_configured_sizes_hold_when_the_live_topology_disagrees(self):
|
||||
"""The other direction: groups exist and answer something else.
|
||||
|
||||
The check above proves nobody reads a live size too early. It says
|
||||
nothing about what `configured_*()` returns once the groups *are* up
|
||||
and answering a different number -- which is not hypothetical: elastic
|
||||
EP scales the live topology away from what the operator configured, and
|
||||
that divergence is the entire reason these five helpers exist. With
|
||||
only the early-read direction covered, a helper that quietly delegated
|
||||
to the live property would look correct.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from unittest.mock import patch
|
||||
|
||||
from sglang.srt.runtime_context import (
|
||||
configured_attn_cp_size,
|
||||
configured_dcp_size,
|
||||
configured_moe_dp_size,
|
||||
configured_pp_size,
|
||||
configured_tp_size,
|
||||
get_parallel,
|
||||
publish,
|
||||
reset_context,
|
||||
)
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
directory = tempfile.mkdtemp(prefix="configured_sizes_")
|
||||
with open(os.path.join(directory, "config.json"), "w") as handle:
|
||||
json.dump(
|
||||
{
|
||||
"architectures": ["LlamaForCausalLM"],
|
||||
"model_type": "llama",
|
||||
"hidden_size": 16,
|
||||
"intermediate_size": 32,
|
||||
"num_attention_heads": 2,
|
||||
"num_key_value_heads": 2,
|
||||
"num_hidden_layers": 2,
|
||||
"vocab_size": 128,
|
||||
"max_position_embeddings": 2048,
|
||||
},
|
||||
handle,
|
||||
)
|
||||
# No resolve_once() here: `tp_size` is raw input, so the configured
|
||||
# value is 2 either way.
|
||||
server_args = ServerArgs(model_path=directory, device="cuda", tp_size=2)
|
||||
self.addCleanup(reset_context)
|
||||
publish(server_args, role="scheduler")
|
||||
|
||||
# The live getter behind each property, read out of ParallelContext
|
||||
# rather than listed here.
|
||||
context_source = ast.parse(
|
||||
(_PACKAGE_ROOT / "srt" / "runtime_context.py").read_text(
|
||||
encoding="utf-8-sig"
|
||||
)
|
||||
)
|
||||
parallel_class = next(
|
||||
node
|
||||
for node in ast.walk(context_source)
|
||||
if isinstance(node, ast.ClassDef) and node.name == "ParallelContext"
|
||||
)
|
||||
live_getter = {}
|
||||
for method in parallel_class.body:
|
||||
if not isinstance(method, ast.FunctionDef):
|
||||
continue
|
||||
for call in ast.walk(method):
|
||||
if not (
|
||||
isinstance(call, ast.Call)
|
||||
and isinstance(call.func, ast.Attribute)
|
||||
and call.func.attr == "_v"
|
||||
and call.args
|
||||
and isinstance(call.args[0], ast.Constant)
|
||||
):
|
||||
continue
|
||||
getter = call.args[1]
|
||||
if isinstance(getter, ast.Attribute):
|
||||
live_getter[call.args[0].value] = getter.attr
|
||||
state = "sglang.srt.distributed.parallel_state"
|
||||
helpers = {
|
||||
"tp_size": configured_tp_size,
|
||||
"pp_size": configured_pp_size,
|
||||
"moe_dp_size": configured_moe_dp_size,
|
||||
"attn_cp_size": configured_attn_cp_size,
|
||||
"dcp_size": configured_dcp_size,
|
||||
}
|
||||
missing = sorted(set(helpers) - set(live_getter))
|
||||
self.assertEqual(
|
||||
missing,
|
||||
[],
|
||||
f"these sizes no longer have a live property to diverge from: {missing}",
|
||||
)
|
||||
cases = tuple(
|
||||
(name, helper, f"{state}.{live_getter[name]}")
|
||||
for name, helper in helpers.items()
|
||||
)
|
||||
for name, helper, target in cases:
|
||||
with self.subTest(size=name):
|
||||
configured = helper()
|
||||
with patch(target, return_value=configured + 41):
|
||||
self.assertEqual(
|
||||
get_parallel().__getattribute__(name),
|
||||
configured + 41,
|
||||
f"{name} no longer follows the live topology",
|
||||
)
|
||||
self.assertEqual(
|
||||
helper(),
|
||||
configured,
|
||||
f"configured_{name}() followed the live topology instead "
|
||||
"of the published configuration",
|
||||
)
|
||||
reset_context()
|
||||
|
||||
def test_no_live_topology_read_before_distributed_init(self):
|
||||
offenders = []
|
||||
for rel, tree in _launch_paths():
|
||||
|
||||
@@ -47,6 +47,156 @@ def _field_names():
|
||||
|
||||
|
||||
class TestServerArgsNamespaces(CustomTestCase):
|
||||
def test_no_module_shadows_a_bag_accessor(self):
|
||||
"""An accessor name bound twice in one module is a silent wrong read.
|
||||
|
||||
This has happened twice. Once a module imported `get_model` from the
|
||||
context and a same-named helper from elsewhere, and once `get_device`
|
||||
-- which names three different things in this tree: the bag accessor,
|
||||
the device-string utility, and a platform method. The second import
|
||||
wins, the converted line calls the wrong callable, and the failure is
|
||||
an AttributeError on whichever branch reaches it, which for a
|
||||
per-pass recorder or a specific accelerator can be none of the ones a
|
||||
CPU suite runs. Nothing else notices; a name scan looks fine.
|
||||
"""
|
||||
import ast
|
||||
import collections
|
||||
import pathlib as _pathlib
|
||||
|
||||
import sglang
|
||||
|
||||
srt = _pathlib.Path(sglang.__file__).resolve().parent / "srt"
|
||||
context_module = ast.parse(
|
||||
(srt / "runtime_context.py").read_text(encoding="utf-8-sig")
|
||||
)
|
||||
accessors = {
|
||||
node.name
|
||||
for node in context_module.body
|
||||
if isinstance(node, ast.FunctionDef)
|
||||
and (node.name.startswith("get_") or node.name.startswith("configured_"))
|
||||
}
|
||||
self.assertGreater(len(accessors), 20, "the accessor derivation broke")
|
||||
|
||||
shadowed = []
|
||||
for path in sorted(srt.rglob("*.py")):
|
||||
if path.name == "runtime_context.py":
|
||||
continue
|
||||
try:
|
||||
tree = ast.parse(path.read_text(encoding="utf-8-sig"))
|
||||
except SyntaxError:
|
||||
self.fail(f"unparsable module in the census: {path}")
|
||||
bindings = collections.defaultdict(set)
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom):
|
||||
origin = node.module or ""
|
||||
kind = (
|
||||
"context"
|
||||
if origin.endswith("runtime_context")
|
||||
else f"{origin or '.'}"
|
||||
)
|
||||
for alias in node.names:
|
||||
bindings[alias.asname or alias.name].add((kind, node.lineno))
|
||||
elif isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
bindings[(alias.asname or alias.name).split(".")[0]].add(
|
||||
("import", node.lineno)
|
||||
)
|
||||
elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
bindings[node.name].add(("def", node.lineno))
|
||||
elif isinstance(node, ast.Assign):
|
||||
for target in node.targets:
|
||||
if isinstance(target, ast.Name):
|
||||
bindings[target.id].add(("assign", node.lineno))
|
||||
for name, where in bindings.items():
|
||||
if name not in accessors:
|
||||
continue
|
||||
kinds = {kind for kind, _ in where}
|
||||
# The same accessor imported from the context more than once
|
||||
# (module level plus a lazy import inside a function) is one
|
||||
# object under one name; a *different* origin is the hazard.
|
||||
if "context" in kinds and kinds - {"context"}:
|
||||
shadowed.append(
|
||||
f"{path.relative_to(srt)}: {name} <- "
|
||||
+ ", ".join(
|
||||
f"{k}@{l}" for k, l in sorted(where, key=lambda w: w[1])
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
shadowed,
|
||||
[],
|
||||
"a bag accessor shares its name with another binding in the same "
|
||||
"module, so the converted reads call whichever import came last; "
|
||||
"alias one of them:\n " + "\n ".join(shadowed),
|
||||
)
|
||||
|
||||
def test_the_readers_agree_with_the_namespace_metadata(self):
|
||||
"""Two independent sources say where a leaf lives; they must match.
|
||||
|
||||
The metadata is one source and the ~2000 hand-written reads
|
||||
(`get_schedule().chunked_prefill_size`) are the other. Checking the
|
||||
projection against the metadata cannot catch a field assigned to the
|
||||
wrong group -- both sides come from the same marker, so the check is
|
||||
true by construction. The readers are written by hand, so a
|
||||
disagreement means one of the two is wrong, and every reader on the
|
||||
losing side raises `has no leaf/subgroup` at runtime on whichever
|
||||
branch reaches it first.
|
||||
"""
|
||||
import ast
|
||||
import pathlib as _pathlib
|
||||
|
||||
import sglang
|
||||
|
||||
srt = _pathlib.Path(sglang.__file__).resolve().parent / "srt"
|
||||
mapping = namespace_of(ServerArgs)
|
||||
accessors = {
|
||||
f"get_{group}" for group in {p.split(".")[0] for p in mapping.values()}
|
||||
}
|
||||
|
||||
sites = 0
|
||||
disagreements = []
|
||||
for path in sorted(srt.rglob("*.py")):
|
||||
try:
|
||||
tree = ast.parse(path.read_text(encoding="utf-8-sig"))
|
||||
except SyntaxError:
|
||||
self.fail(f"unparsable module in the census: {path}")
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Attribute):
|
||||
continue
|
||||
chain, cursor = [], node
|
||||
while isinstance(cursor, ast.Attribute):
|
||||
chain.append(cursor.attr)
|
||||
cursor = cursor.value
|
||||
if not (
|
||||
isinstance(cursor, ast.Call)
|
||||
and isinstance(cursor.func, ast.Name)
|
||||
and cursor.func.id in accessors
|
||||
):
|
||||
continue
|
||||
chain.reverse()
|
||||
field = chain[-1]
|
||||
if field not in mapping:
|
||||
continue
|
||||
sites += 1
|
||||
read = [cursor.func.id[len("get_") :]] + chain[:-1]
|
||||
if mapping[field].split(".") != read:
|
||||
disagreements.append(
|
||||
f"{path.relative_to(srt)}:{node.lineno} reads "
|
||||
f"{'.'.join(read)}.{field}, metadata says "
|
||||
f"{mapping[field]}.{field}"
|
||||
)
|
||||
self.assertEqual(
|
||||
disagreements,
|
||||
[],
|
||||
"a reader and the namespace metadata disagree about where a leaf "
|
||||
"lives; one of them is wrong:\n " + "\n ".join(disagreements),
|
||||
)
|
||||
self.assertGreater(
|
||||
sites,
|
||||
1500,
|
||||
f"only {sites} bag reads were matched; the scan broke and this "
|
||||
"check stopped covering anything",
|
||||
)
|
||||
|
||||
def test_every_field_has_a_namespace(self):
|
||||
nsmap = namespace_of(ServerArgs)
|
||||
missing = sorted(_field_names() - set(nsmap))
|
||||
|
||||
@@ -129,6 +129,15 @@ _ENV_MATRIX = (({}, {"SGLANG_IS_IN_CI": "true"}),)
|
||||
_PASSED = frozenset({"model_path", "device", "random_seed"})
|
||||
|
||||
_EXPOSED = {
|
||||
("dllm/config.py", "max_running_requests"),
|
||||
("dllm/config.py", "model_path"),
|
||||
("multimodal/processors/base_processor.py", "image_processor_backend"),
|
||||
("speculative/spec_registry.py", "disable_overlap_schedule"),
|
||||
("layers/moe/utils.py", "deepep_mode"),
|
||||
("layers/moe/utils.py", "moe_a2a_backend"),
|
||||
("layers/moe/utils.py", "moe_runner_backend"),
|
||||
("layers/moe/utils.py", "quantization"),
|
||||
("layers/moe/utils.py", "speculative_moe_runner_backend"),
|
||||
("entrypoints/sidecar.py", "grpc_port"),
|
||||
("configs/embedding_model_spec.py", "chunked_prefill_size"),
|
||||
("configs/embedding_model_spec.py", "cuda_graph_config"),
|
||||
@@ -144,10 +153,6 @@ _EXPOSED = {
|
||||
("configs/model_config.py", "quantization"),
|
||||
("configs/model_config.py", "speculative_algorithm"),
|
||||
("configs/model_config.py", "speculative_draft_model_quantization"),
|
||||
("disaggregation/common/conn.py", "disaggregation_bootstrap_port"),
|
||||
("disaggregation/common/conn.py", "pp_size"),
|
||||
("disaggregation/decode_kvcache_offload_manager.py", "hicache_io_backend"),
|
||||
("disaggregation/decode_kvcache_offload_manager.py", "served_model_name"),
|
||||
("disaggregation/utils.py", "disaggregation_transfer_backend"),
|
||||
("distributed/bootstrap.py", "disable_custom_all_reduce"),
|
||||
("distributed/bootstrap.py", "enable_symm_mem"),
|
||||
@@ -155,38 +160,6 @@ _EXPOSED = {
|
||||
("distributed/bootstrap.py", "flashinfer_allreduce_fusion_backend"),
|
||||
("distributed/bootstrap.py", "moe_a2a_backend"),
|
||||
("distributed/bootstrap.py", "pre_warm_nccl"),
|
||||
(
|
||||
"distributed/device_communicators/mooncake_transfer_engine.py",
|
||||
"disaggregation_ib_device",
|
||||
),
|
||||
(
|
||||
"distributed/device_communicators/mooncake_transfer_engine.py",
|
||||
"disaggregation_mode",
|
||||
),
|
||||
(
|
||||
"distributed/device_communicators/mooncake_transfer_engine.py",
|
||||
"disaggregation_transfer_backend",
|
||||
),
|
||||
(
|
||||
"distributed/device_communicators/mooncake_transfer_engine.py",
|
||||
"enable_hierarchical_cache",
|
||||
),
|
||||
(
|
||||
"distributed/device_communicators/mooncake_transfer_engine.py",
|
||||
"encoder_transfer_backend",
|
||||
),
|
||||
(
|
||||
"distributed/device_communicators/mooncake_transfer_engine.py",
|
||||
"mooncake_ib_device",
|
||||
),
|
||||
("dllm/config.py", "max_running_requests"),
|
||||
("dllm/config.py", "model_path"),
|
||||
("elastic_ep/elastic_ep.py", "elastic_ep_initial_size"),
|
||||
("elastic_ep/elastic_ep.py", "ep_join_mode"),
|
||||
("elastic_ep/elastic_ep.py", "moe_a2a_backend"),
|
||||
("elastic_ep/expert_backup_manager.py", "disaggregation_ib_device"),
|
||||
("elastic_ep/expert_backup_manager.py", "load_format"),
|
||||
("elastic_ep/expert_backup_manager.py", "mooncake_ib_device"),
|
||||
("entrypoints/engine.py", "attn_cp_size"),
|
||||
("entrypoints/engine.py", "enable_symm_mem"),
|
||||
("entrypoints/engine.py", "moe_dp_size"),
|
||||
@@ -198,17 +171,6 @@ _EXPOSED = {
|
||||
("entrypoints/engine.py", "tool_call_parser"),
|
||||
("eplb/eplb_manager.py", "ep_dispatch_algorithm"),
|
||||
("eplb/eplb_manager.py", "expert_distribution_recorder_buffer_size"),
|
||||
("eplb/expert_distribution.py", "deepep_mode"),
|
||||
("eplb/expert_distribution.py", "device"),
|
||||
("eplb/expert_distribution.py", "expert_distribution_recorder_mode"),
|
||||
("eplb/expert_distribution.py", "moe_a2a_backend"),
|
||||
("eplb/expert_location.py", "device"),
|
||||
("eplb/expert_location.py", "eplb_algorithm"),
|
||||
("kv_canary/api.py", "disaggregation_mode"),
|
||||
("kv_canary/api.py", "speculative_num_steps"),
|
||||
("kv_canary/capacities.py", "chunked_prefill_size"),
|
||||
("kv_canary/capacities.py", "cuda_graph_config"),
|
||||
("kv_canary/capacities.py", "speculative_num_draft_tokens"),
|
||||
("layers/cp/base.py", "attn_cp_size"),
|
||||
("layers/cp/base.py", "cp_strategy"),
|
||||
("layers/cp/base.py", "enable_prefill_cp"),
|
||||
@@ -216,11 +178,6 @@ _EXPOSED = {
|
||||
("layers/cp/bcg.py", "enable_prefill_cp"),
|
||||
("layers/flashinfer_comm_fusion.py", "flashinfer_allreduce_fusion_backend"),
|
||||
("layers/moe/kt_ep_wrapper.py", "chunked_prefill_size"),
|
||||
("layers/moe/utils.py", "deepep_mode"),
|
||||
("layers/moe/utils.py", "moe_a2a_backend"),
|
||||
("layers/moe/utils.py", "moe_runner_backend"),
|
||||
("layers/moe/utils.py", "quantization"),
|
||||
("layers/moe/utils.py", "speculative_moe_runner_backend"),
|
||||
("layers/quantization/unquant.py", "enable_deterministic_inference"),
|
||||
("lora/lora_manager.py", "enable_lora_overlap_loading"),
|
||||
("lora/marlin_lora_temp/policy.py", "enable_lora"),
|
||||
@@ -231,124 +188,18 @@ _EXPOSED = {
|
||||
("managers/data_parallel_controller.py", "moe_dp_size"),
|
||||
("managers/data_parallel_controller.py", "pp_size"),
|
||||
("managers/data_parallel_controller.py", "soft_watchdog_timeout"),
|
||||
("managers/disagg_service.py", "disaggregation_bootstrap_port"),
|
||||
("managers/disagg_service.py", "disaggregation_mode"),
|
||||
("managers/disagg_service.py", "disaggregation_transfer_backend"),
|
||||
("managers/overlap_utils.py", "speculative_algorithm"),
|
||||
("managers/prefill_delayer.py", "disable_overlap_schedule"),
|
||||
("managers/rust_server.py", "mm_process_config"),
|
||||
("managers/schedule_batch.py", "disaggregation_mode"),
|
||||
("managers/scheduler.py", "attn_cp_size"),
|
||||
("managers/scheduler.py", "disable_overlap_schedule"),
|
||||
("managers/scheduler.py", "disaggregation_mode"),
|
||||
("managers/scheduler.py", "enable_hierarchical_cache"),
|
||||
("managers/scheduler.py", "enable_lora"),
|
||||
("managers/scheduler.py", "enable_lora_overlap_loading"),
|
||||
("managers/scheduler.py", "moe_dp_size"),
|
||||
("managers/scheduler.py", "pp_size"),
|
||||
("managers/scheduler.py", "soft_watchdog_timeout"),
|
||||
("managers/scheduler.py", "speculative_algorithm"),
|
||||
("managers/tokenizer_manager.py", "served_model_name"),
|
||||
("managers/tp_worker.py", "disable_overlap_schedule"),
|
||||
("managers/tp_worker.py", "model_path"),
|
||||
("managers/tp_worker.py", "random_seed"),
|
||||
("managers/tp_worker.py", "speculative_algorithm"),
|
||||
("managers/tp_worker.py", "tokenizer_path"),
|
||||
("mem_cache/hiradix_cache.py", "hicache_io_backend"),
|
||||
("mem_cache/hiradix_cache.py", "hicache_mem_layout"),
|
||||
("mem_cache/hiradix_cache.py", "served_model_name"),
|
||||
("mem_cache/hybrid_cache/hybrid_pool_assembler.py", "hicache_io_backend"),
|
||||
("mem_cache/hybrid_cache/hybrid_pool_assembler.py", "hicache_mem_layout"),
|
||||
("mem_cache/hybrid_cache/hybrid_pool_assembler.py", "served_model_name"),
|
||||
("mem_cache/kv_cache_builder.py", "hicache_mem_layout"),
|
||||
("mem_cache/radix_cache_cpp.py", "enable_hierarchical_cache"),
|
||||
("model_executor/model_runner.py", "device"),
|
||||
("model_executor/model_runner.py", "speculative_algorithm"),
|
||||
("model_executor/model_runner.py", "speculative_draft_attention_backend"),
|
||||
("model_executor/model_runner_components/load_model_utils.py", "load_format"),
|
||||
("model_executor/model_runner_components/load_model_utils.py", "quantization"),
|
||||
(
|
||||
"model_executor/model_runner_components/spec_aux_hidden_state.py",
|
||||
"speculative_draft_attention_backend",
|
||||
),
|
||||
(
|
||||
"model_executor/model_runner_components/spec_aux_hidden_state.py",
|
||||
"speculative_draft_model_path",
|
||||
),
|
||||
(
|
||||
"model_executor/model_runner_components/spec_aux_hidden_state.py",
|
||||
"speculative_draft_model_revision",
|
||||
),
|
||||
("model_executor/model_runner_components/startup_weight_load.py", "attn_cp_size"),
|
||||
(
|
||||
"model_executor/model_runner_components/startup_weight_load.py",
|
||||
"cuda_graph_config",
|
||||
),
|
||||
(
|
||||
"model_executor/model_runner_components/startup_weight_load.py",
|
||||
"custom_weight_loader",
|
||||
),
|
||||
("model_executor/model_runner_components/startup_weight_load.py", "device"),
|
||||
("model_executor/model_runner_components/startup_weight_load.py", "enable_lora"),
|
||||
("model_executor/model_runner_components/startup_weight_load.py", "lora_paths"),
|
||||
("model_executor/model_runner_components/startup_weight_load.py", "pp_size"),
|
||||
(
|
||||
"model_executor/model_runner_components/startup_weight_load.py",
|
||||
"speculative_algorithm",
|
||||
),
|
||||
(
|
||||
"model_executor/runner_backend/tc_piecewise_cuda_graph_backend.py",
|
||||
"cuda_graph_config",
|
||||
),
|
||||
("multimodal/processors/base_processor.py", "image_processor_backend"),
|
||||
("observability/metrics_collector.py", "disaggregation_mode"),
|
||||
("observability/metrics_collector.py", "prefill_delayer_max_delay_passes"),
|
||||
("observability/metrics_collector.py", "served_model_name"),
|
||||
("parser/template_detection.py", "model_path"),
|
||||
("speculative/adaptive_spec_params.py", "speculative_algorithm"),
|
||||
("speculative/adaptive_spec_params.py", "speculative_eagle_topk"),
|
||||
("speculative/dflash_worker_v2.py", "speculative_draft_window_size"),
|
||||
("speculative/dflash_worker_v2.py", "speculative_num_draft_tokens"),
|
||||
("speculative/draft_worker_common.py", "speculative_draft_attention_backend"),
|
||||
("speculative/dspark_components/dspark_config.py", "speculative_draft_model_path"),
|
||||
(
|
||||
"speculative/dspark_components/dspark_config.py",
|
||||
"speculative_draft_model_revision",
|
||||
),
|
||||
("speculative/dspark_components/dspark_worker_v2.py", "disaggregation_mode"),
|
||||
(
|
||||
"speculative/dspark_components/dspark_worker_v2.py",
|
||||
"speculative_num_draft_tokens",
|
||||
),
|
||||
("speculative/eagle_worker_v2.py", "device"),
|
||||
("speculative/eagle_worker_v2.py", "speculative_adaptive"),
|
||||
("speculative/eagle_worker_v2.py", "speculative_algorithm"),
|
||||
("speculative/eagle_worker_v2.py", "speculative_eagle_topk"),
|
||||
("speculative/eagle_worker_v2.py", "speculative_num_draft_tokens"),
|
||||
("speculative/eagle_worker_v2.py", "speculative_num_steps"),
|
||||
("speculative/frozen_kv_mtp_worker_v2.py", "device"),
|
||||
("speculative/frozen_kv_mtp_worker_v2.py", "speculative_adaptive"),
|
||||
("speculative/frozen_kv_mtp_worker_v2.py", "speculative_algorithm"),
|
||||
("speculative/frozen_kv_mtp_worker_v2.py", "speculative_eagle_topk"),
|
||||
("speculative/frozen_kv_mtp_worker_v2.py", "speculative_num_draft_tokens"),
|
||||
("speculative/frozen_kv_mtp_worker_v2.py", "speculative_num_steps"),
|
||||
("speculative/multi_layer_eagle_worker_v2.py", "device"),
|
||||
("speculative/multi_layer_eagle_worker_v2.py", "speculative_algorithm"),
|
||||
("speculative/multi_layer_eagle_worker_v2.py", "speculative_eagle_topk"),
|
||||
("speculative/multi_layer_eagle_worker_v2.py", "speculative_num_draft_tokens"),
|
||||
("speculative/multi_layer_eagle_worker_v2.py", "speculative_num_steps"),
|
||||
("speculative/ngram_worker.py", "device"),
|
||||
("speculative/ngram_worker.py", "disable_overlap_schedule"),
|
||||
("speculative/ngram_worker.py", "speculative_eagle_topk"),
|
||||
("speculative/ngram_worker.py", "speculative_num_draft_tokens"),
|
||||
("speculative/ngram_worker.py", "speculative_num_steps"),
|
||||
("speculative/spec_info.py", "enable_multi_layer_eagle"),
|
||||
("speculative/spec_registry.py", "disable_overlap_schedule"),
|
||||
("speculative/standalone_worker_v2.py", "device"),
|
||||
("speculative/standalone_worker_v2.py", "speculative_algorithm"),
|
||||
("speculative/standalone_worker_v2.py", "speculative_eagle_topk"),
|
||||
("speculative/standalone_worker_v2.py", "speculative_num_draft_tokens"),
|
||||
("speculative/standalone_worker_v2.py", "speculative_num_steps"),
|
||||
("utils/common.py", "speculative_num_draft_tokens"),
|
||||
("utils/common.py", "speculative_num_steps"),
|
||||
("utils/cuda_vmm_transport_utils.py", "mm_feature_transport"),
|
||||
@@ -367,56 +218,19 @@ _EXPOSED_CUDA_ONLY: frozenset = frozenset()
|
||||
# some code overrides post-publish. Each needs an ordering judgment, not a blanket
|
||||
# conversion; the list exists so a new one is a decision made when it is written.
|
||||
_OVERRIDDEN_AND_READ = {
|
||||
("dllm/config.py", "model_path"),
|
||||
("entrypoints/engine.py", "reasoning_parser"),
|
||||
("entrypoints/engine.py", "tool_call_parser"),
|
||||
("configs/model_config.py", "dtype"),
|
||||
("configs/model_config.py", "model_path"),
|
||||
("disaggregation/decode_kvcache_offload_manager.py", "hicache_storage_backend"),
|
||||
(
|
||||
"disaggregation/decode_kvcache_offload_manager.py",
|
||||
"hicache_storage_backend_extra_config",
|
||||
),
|
||||
(
|
||||
"distributed/device_communicators/mooncake_transfer_engine.py",
|
||||
"hicache_storage_backend",
|
||||
),
|
||||
("dllm/config.py", "model_path"),
|
||||
("elastic_ep/expert_backup_manager.py", "load_format"),
|
||||
("kv_canary/api.py", "speculative_num_steps"),
|
||||
("kv_canary/capacities.py", "speculative_num_draft_tokens"),
|
||||
("managers/scheduler.py", "hicache_storage_backend"),
|
||||
("managers/tp_worker.py", "model_path"),
|
||||
("mem_cache/hiradix_cache.py", "hicache_storage_backend"),
|
||||
("mem_cache/hiradix_cache.py", "hicache_storage_backend_extra_config"),
|
||||
("mem_cache/hiradix_cache.py", "hicache_storage_prefetch_policy"),
|
||||
("mem_cache/hiradix_cache.py", "hicache_write_policy"),
|
||||
("mem_cache/hybrid_cache/hybrid_pool_assembler.py", "hicache_storage_backend"),
|
||||
("mem_cache/hybrid_cache/hybrid_pool_assembler.py", "hicache_write_policy"),
|
||||
("mem_cache/kv_cache_builder.py", "hicache_storage_backend"),
|
||||
("mem_cache/pool_host/common.py", "hicache_storage_backend"),
|
||||
("mem_cache/pool_host/common.py", "hicache_storage_backend_extra_config"),
|
||||
("mem_cache/radix_cache_cpp.py", "hicache_write_policy"),
|
||||
("mem_cache/unified_radix_cache.py", "hicache_storage_backend"),
|
||||
("mem_cache/unified_radix_cache.py", "hicache_storage_backend_extra_config"),
|
||||
("mem_cache/unified_radix_cache.py", "hicache_storage_prefetch_policy"),
|
||||
("mem_cache/unified_radix_cache.py", "hicache_write_policy"),
|
||||
("model_executor/model_runner_components/load_model_utils.py", "load_format"),
|
||||
("parser/template_detection.py", "model_path"),
|
||||
("speculative/dflash_worker_v2.py", "speculative_num_draft_tokens"),
|
||||
(
|
||||
"speculative/dspark_components/dspark_worker_v2.py",
|
||||
"speculative_num_draft_tokens",
|
||||
),
|
||||
("speculative/eagle_worker_v2.py", "speculative_num_draft_tokens"),
|
||||
("speculative/eagle_worker_v2.py", "speculative_num_steps"),
|
||||
("speculative/frozen_kv_mtp_worker_v2.py", "speculative_num_draft_tokens"),
|
||||
("speculative/frozen_kv_mtp_worker_v2.py", "speculative_num_steps"),
|
||||
("speculative/multi_layer_eagle_worker_v2.py", "speculative_num_draft_tokens"),
|
||||
("speculative/multi_layer_eagle_worker_v2.py", "speculative_num_steps"),
|
||||
("speculative/ngram_worker.py", "speculative_num_draft_tokens"),
|
||||
("speculative/ngram_worker.py", "speculative_num_steps"),
|
||||
("speculative/standalone_worker_v2.py", "speculative_num_draft_tokens"),
|
||||
("speculative/standalone_worker_v2.py", "speculative_num_steps"),
|
||||
("utils/common.py", "speculative_num_draft_tokens"),
|
||||
("utils/common.py", "speculative_num_steps"),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user