minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

config: borrowed-record reads follow the config bags (#35908)

Browse Source 
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Cheng Wan
2026-08-23 01:19:20 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 64aa859da2
commit 362c2ee849
65 changed files with 617 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/attention/test_trtllm_mha_graph_metadata.py
+44 -36
View File
@@ -121,47 +121,55 @@ def test_draft_extend_in_graph_uses_captured_static_q_stride(monkeypatch):
def test_hybrid_wrappers_forward_in_graph_hook():
"""Hybrid wrappers must forward init_forward_metadata_in_graph to the
wrapped backend(s) — the inherited no-op would leave the fused metadata
rebuild out of the captured graph (stale page table on every replay)."""
from sglang.srt.layers.attention.hybrid_attn_backend import HybridAttnBackend
from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
HybridLinearAttnBackend,
)
# The hybrid backend reads the mode from the published configuration.
from sglang.srt.runtime_context import get_context
def make_fake(name, calls):
return SimpleNamespace(
token_to_kv_pool=None,
req_to_token_pool=None,
needs_cpu_seq_lens=False,
init_forward_metadata_in_graph=lambda fb: calls.append(name),
override = get_context().override_server_args(speculative_attention_mode="decode")
override.install()
try:
"""Hybrid wrappers must forward init_forward_metadata_in_graph to the
wrapped backend(s) — the inherited no-op would leave the fused metadata
rebuild out of the captured graph (stale page table on every replay)."""
from sglang.srt.layers.attention.hybrid_attn_backend import HybridAttnBackend
from sglang.srt.layers.attention.hybrid_linear_attn_backend import (
HybridLinearAttnBackend,
)
fb = SimpleNamespace(forward_mode=ForwardMode.DECODE)
def make_fake(name, calls):
return SimpleNamespace(
token_to_kv_pool=None,
req_to_token_pool=None,
needs_cpu_seq_lens=False,
init_forward_metadata_in_graph=lambda fb: calls.append(name),
)
calls = []
hybrid = HybridAttnBackend(
SimpleNamespace(
kv_cache_dtype=torch.bfloat16,
token_to_kv_pool=None,
req_to_token_pool=None,
server_args=SimpleNamespace(speculative_attention_mode="decode"),
model_config=SimpleNamespace(context_len=2048),
),
prefill_backend=make_fake("prefill", calls),
decode_backend=make_fake("decode", calls),
)
hybrid.init_forward_metadata_in_graph(fb)
assert calls == ["decode"]
fb = SimpleNamespace(forward_mode=ForwardMode.DECODE)
calls = []
hybrid_linear = HybridLinearAttnBackend(
full_attn_backend=make_fake("full", calls),
linear_attn_backend=make_fake("linear", calls),
full_attn_layers=[0],
)
hybrid_linear.init_forward_metadata_in_graph(fb)
assert calls == ["full", "linear"]
calls = []
hybrid = HybridAttnBackend(
SimpleNamespace(
kv_cache_dtype=torch.bfloat16,
token_to_kv_pool=None,
req_to_token_pool=None,
server_args=SimpleNamespace(speculative_attention_mode="decode"),
model_config=SimpleNamespace(context_len=2048),
),
prefill_backend=make_fake("prefill", calls),
decode_backend=make_fake("decode", calls),
)
hybrid.init_forward_metadata_in_graph(fb)
assert calls == ["decode"]
calls = []
hybrid_linear = HybridLinearAttnBackend(
full_attn_backend=make_fake("full", calls),
linear_attn_backend=make_fake("linear", calls),
full_attn_layers=[0],
)
hybrid_linear.init_forward_metadata_in_graph(fb)
assert calls == ["full", "linear"]
finally:
override.restore()
def test_metadata_update_records_inside_cuda_graph():
test/registered/unit/entrypoints/test_v1_loads_aggregate.py
+15 -1
View File
@@ -66,13 +66,24 @@ class _FakeHttpTokenizerManager:
pp_size=1,
enable_dp_attention=False,
):
from sglang.srt.runtime_context import get_context
self.loads = loads
self.server_args = SimpleNamespace(
# `tp_size` is raw input and still read off the record; the leaves
# resolution writes come from the bags.
self.server_args = SimpleNamespace(tp_size=tp_size)
# The accelerator arithmetic answers "what will this server do", so it
# reads the resolved topology out of the bags; publish the shape under test.
self._override = get_context().override_server_args(
tp_size=tp_size,
dp_size=dp_size,
pp_size=pp_size,
enable_dp_attention=enable_dp_attention,
)
self._override.install()
def restore(self):
self._override.restore()
async def get_loads(self, include=None, dp_rank=None):
results = []
@@ -95,6 +106,7 @@ class TestLoadsResponse(CustomTestCase):
)
]
)
self.addCleanup(manager.restore)
response = asyncio.run(get_loads(tokenizer_manager=manager))
@@ -111,6 +123,7 @@ class TestLoadsAcceleratorField(CustomTestCase):
"""Guards the response contract: the JSON envelope carries an
accelerator name and the accelerator count for each DP rank."""
manager = _FakeHttpTokenizerManager([LoadSnapshot(dp_rank=0)], tp_size=16)
self.addCleanup(manager.restore)
with mock.patch.object(
v1_loads, "_accelerator_name", return_value="NVIDIA GB300"
@@ -126,6 +139,7 @@ class TestLoadsAcceleratorField(CustomTestCase):
dp_size=8,
enable_dp_attention=True,
)
self.addCleanup(manager.restore)
response = asyncio.run(get_loads(tokenizer_manager=manager))
test/registered/unit/layers/attention/test_linear_attn_config.py
+3 -3
View File
@@ -120,11 +120,11 @@ class TestLinearAttnBackends(CustomTestCase):
from sglang.srt.layers.attention.linear.gdn_backend import GDNAttnBackend
# The draft-token width is a bag leaf read before the stamp.
self._publish(speculative_eagle_topk=0)
runner = SimpleNamespace(
device="cpu",
server_args=SimpleNamespace(
speculative_eagle_topk=0, enable_unified_memory=False
),
server_args=SimpleNamespace(enable_unified_memory=False),
is_draft_worker=False,
req_to_token_pool=SimpleNamespace(
mamba_pool=SimpleNamespace(
test/registered/unit/layers/attention/test_verify_mask.py
+26 -5
View File
@@ -1,3 +1,4 @@
import contextlib
import unittest
from types import SimpleNamespace
@@ -9,6 +10,7 @@ from sglang.srt.layers.attention.verify_mask import (
maybe_create_verify_mask,
tree_mask_numel,
)
from sglang.srt.runtime_context import get_context
from sglang.srt.speculative.eagle_utils import TreeMaskMode, default_tree_mask_mode
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -123,6 +125,24 @@ def _mask(numel, **kwargs):
)
@contextlib.contextmanager
def _published(speculative_attention_mode):
"""The backend reads the mode from the config bags, so publish one.
A stand-in on the model runner stopped being read when the mode became a
published leaf -- the record it would come from is not the one this process
resolved.
"""
override = get_context().override_server_args(
speculative_attention_mode=speculative_attention_mode
)
override.install()
try:
yield
finally:
override.restore()
def _make_hybrid_backend(speculative_attention_mode, prefill_mask, decode_mask):
model_runner = SimpleNamespace(
kv_cache_dtype=None,
@@ -133,11 +153,12 @@ def _make_hybrid_backend(speculative_attention_mode, prefill_mask, decode_mask):
),
model_config=SimpleNamespace(context_len=_MAX_CONTEXT_LEN),
)
return HybridAttnBackend(
model_runner,
prefill_backend=_FakeAttnBackend(prefill_mask),
decode_backend=_FakeAttnBackend(decode_mask),
)
with _published(speculative_attention_mode):
return HybridAttnBackend(
model_runner,
prefill_backend=_FakeAttnBackend(prefill_mask),
decode_backend=_FakeAttnBackend(decode_mask),
)
class TestHybridAttnBackendHandsOutSelectedChildMask(CustomTestCase):
test/registered/unit/lora/test_experimental_sgl_marlin_policy.py
+5 -1
View File
@@ -40,7 +40,11 @@ def _validate_server(**overrides):
server_args.update(overrides)
return validate_experimental_sgl_marlin_server_args(
types.SimpleNamespace(**server_args),
types.SimpleNamespace(ep_size=4, moe_a2a_backend="none"),
types.SimpleNamespace(
ep_size=4,
moe_a2a_backend="none",
enable_lora=server_args["enable_lora"],
),
)
test/registered/unit/mem_cache/test_hybrid_pool_assembler.py
+7 -1
View File
@@ -64,7 +64,13 @@ class TestDraftSidecarPoolDispatch(CustomTestCase):
page_size=512,
)
)
server_args = SimpleNamespace(hicache_mem_layout="page_first")
# The layout comes from the published configuration.
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
server_args = ServerArgs(model_path="dummy", hicache_mem_layout="page_first")
publish(server_args, role="scheduler")
self.addCleanup(reset_context)
with (
patch(
test/registered/unit/model_executor/model_runner_components/test_attention_backend_setup.py
+49 -38
View File
@@ -24,49 +24,60 @@ class _FakeBackend:
def test_split_full_attention_applies_model_wrapper_once():
runner = SimpleNamespace(
server_args=SimpleNamespace(speculative_attention_mode="prefill"),
model_config=SimpleNamespace(context_len=2048),
kv_cache_dtype=None,
token_to_kv_pool=object(),
req_to_token_pool=object(),
init_new_workspace=None,
)
wrapper_inputs = []
wrapped_backend = object()
# The hybrid backend takes the speculative attention mode from the
# published configuration.
from sglang.srt.runtime_context import get_context
def wrap_once(model_runner, backend):
assert model_runner is runner
wrapper_inputs.append(backend)
return wrapped_backend
override = get_context().override_server_args(speculative_attention_mode="prefill")
override.install()
try:
runner = SimpleNamespace(
server_args=SimpleNamespace(speculative_attention_mode="prefill"),
model_config=SimpleNamespace(context_len=2048),
kv_cache_dtype=None,
token_to_kv_pool=object(),
req_to_token_pool=object(),
init_new_workspace=None,
)
wrapper_inputs = []
wrapped_backend = object()
constructors = {
"decode-test": lambda model_runner: _FakeBackend("decode"),
"prefill-test": lambda model_runner: _FakeBackend("prefill"),
}
resolved = ResolvedAttentionBackendStr(decode="decode-test", prefill="prefill-test")
def wrap_once(model_runner, backend):
assert model_runner is runner
wrapper_inputs.append(backend)
return wrapped_backend
with (
patch.dict(attention_backend_setup.ATTENTION_BACKENDS, constructors),
patch.object(
attention_backend_setup,
"attn_backend_wrapper",
side_effect=wrap_once,
),
):
result = attention_backend_setup._build_resolved_backend(
model_runner=runner,
resolved=resolved,
init_new_workspace=True,
constructors = {
"decode-test": lambda model_runner: _FakeBackend("decode"),
"prefill-test": lambda model_runner: _FakeBackend("prefill"),
}
resolved = ResolvedAttentionBackendStr(
decode="decode-test", prefill="prefill-test"
)
assert result is wrapped_backend
assert len(wrapper_inputs) == 1
split_backend = wrapper_inputs[0]
assert isinstance(split_backend, HybridAttnBackend)
assert split_backend.decode_backend.name == "decode"
assert split_backend.prefill_backend.name == "prefill"
assert runner.init_new_workspace is True
with (
patch.dict(attention_backend_setup.ATTENTION_BACKENDS, constructors),
patch.object(
attention_backend_setup,
"attn_backend_wrapper",
side_effect=wrap_once,
),
):
result = attention_backend_setup._build_resolved_backend(
model_runner=runner,
resolved=resolved,
init_new_workspace=True,
)
assert result is wrapped_backend
assert len(wrapper_inputs) == 1
split_backend = wrapper_inputs[0]
assert isinstance(split_backend, HybridAttnBackend)
assert split_backend.decode_backend.name == "decode"
assert split_backend.prefill_backend.name == "prefill"
assert runner.init_new_workspace is True
finally:
override.restore()
if __name__ == "__main__":
test/registered/unit/model_executor/model_runner_components/test_cuda_graph_setup.py
+16 -9
View File
@@ -13,6 +13,15 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu")
def test_model_runner_can_override_decode_graph_runner(monkeypatch):
from sglang.srt.runtime_context import get_context
# The capture decision reads the graph configuration and the MoE backends
# out of the bags.
override = get_context().override_server_args(
cuda_graph_config=SimpleNamespace(decode=SimpleNamespace(backend="default")),
)
override.install()
class CustomGraphRunner:
def __init__(self, model_runner):
self.model_runner = model_runner
@@ -23,12 +32,7 @@ def test_model_runner_can_override_decode_graph_runner(monkeypatch):
gpu_id = 0
is_draft_worker = False
spec_algorithm = SimpleNamespace(is_speculative=lambda: False)
server_args = SimpleNamespace(
model_impl="auto",
cuda_graph_config=SimpleNamespace(
decode=SimpleNamespace(backend="default")
),
)
server_args = SimpleNamespace(model_impl="auto")
def _decode_cuda_graph_runner_cls(self):
return CustomGraphRunner
@@ -43,10 +47,13 @@ def test_model_runner_can_override_decode_graph_runner(monkeypatch):
cuda_graph_setup.current_platform, "is_out_of_tree", lambda: False
)
capture = capture_decode_graph(model_runner=model_runner)
try:
capture = capture_decode_graph(model_runner=model_runner)
assert isinstance(capture.runner, CustomGraphRunner)
assert capture.runner.model_runner is model_runner
assert isinstance(capture.runner, CustomGraphRunner)
assert capture.runner.model_runner is model_runner
finally:
override.restore()
if __name__ == "__main__":
test/registered/unit/model_executor/test_prefill_cuda_graph_runner.py
+29 -30
View File
@@ -65,6 +65,16 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
def test_low_free_memory_still_captures_prefill_graph(self):
eager_runner = object()
prefill_runner = object()
# The capture decision reads the graph configuration and the LoRA flag
# out of the bags.
override = get_context().override_server_args(
enable_lora=False,
cuda_graph_config=SimpleNamespace(
prefill=SimpleNamespace(bs=[1], backend=Backend.BREAKABLE)
),
)
override.install()
self.addCleanup(override.restore)
model_runner = SimpleNamespace(
device="cuda",
gpu_id=0,
@@ -73,11 +83,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
# reads it rather than the process-wide LoRA config.
lora_manager=None,
spec_algorithm=SimpleNamespace(is_eagle=lambda: False),
server_args=SimpleNamespace(
cuda_graph_config=SimpleNamespace(
prefill=SimpleNamespace(bs=[1], backend=Backend.BREAKABLE)
),
),
server_args=SimpleNamespace(),
model=SimpleNamespace(),
model_config=SimpleNamespace(context_len=8192, num_hidden_layers=1),
req_to_token_pool=SimpleNamespace(size=1),
@@ -140,15 +146,18 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
self.assertIs(capture.runner, eager_runner)
def test_prefix_chunk_capacity_is_aggregate_and_can_be_overridden(self):
graph_config = SimpleNamespace(
prefill=SimpleNamespace(full_prefill_prefix_chunk_tokens=None, max_bs=8)
)
# Both leaves come from the bags; the published object is this one, so
# the cases below still drive them by mutating it.
override = get_context().override_server_args(
chunked_prefill_size=16, cuda_graph_config=graph_config
)
published = override.install()
self.addCleanup(override.restore)
model_runner = SimpleNamespace(
server_args=SimpleNamespace(
chunked_prefill_size=16,
cuda_graph_config=SimpleNamespace(
prefill=SimpleNamespace(
full_prefill_prefix_chunk_tokens=None, max_bs=8
)
),
),
server_args=SimpleNamespace(),
# Wider than the token table, so the table is the binding limit.
model_config=SimpleNamespace(context_len=4096),
req_to_token_pool=SimpleNamespace(
@@ -161,24 +170,20 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
(4, 16),
)
model_runner.server_args.chunked_prefill_size = -1
get_context().override("test", chunked_prefill_size=-1)
self.assertEqual(
PrefillCudaGraphRunner._resolve_prefix_chunk_shape(model_runner, 4),
(2, 8),
)
model_runner.server_args.chunked_prefill_size = 16
get_context().override("test", chunked_prefill_size=16)
model_runner.server_args.cuda_graph_config.prefill.full_prefill_prefix_chunk_tokens = (
24
)
graph_config.prefill.full_prefill_prefix_chunk_tokens = 24
self.assertEqual(
PrefillCudaGraphRunner._resolve_prefix_chunk_shape(model_runner, 4),
(6, 24),
)
model_runner.server_args.cuda_graph_config.prefill.full_prefill_prefix_chunk_tokens = (
256
)
graph_config.prefill.full_prefill_prefix_chunk_tokens = 256
self.assertEqual(
PrefillCudaGraphRunner._resolve_prefix_chunk_shape(model_runner, 4),
(32, 128),
@@ -186,9 +191,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
# At least one token is reserved per request lane even if the requested
# aggregate capacity is smaller than the fixed request-slot count.
model_runner.server_args.cuda_graph_config.prefill.full_prefill_prefix_chunk_tokens = (
2
)
graph_config.prefill.full_prefill_prefix_chunk_tokens = 2
self.assertEqual(
PrefillCudaGraphRunner._resolve_prefix_chunk_shape(model_runner, 4),
(1, 4),
@@ -197,17 +200,13 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
# A context shorter than the token table binds instead: a draft runner
# capped at the target's context, or a short --context-length.
model_runner.model_config.context_len = 8
model_runner.server_args.cuda_graph_config.prefill.full_prefill_prefix_chunk_tokens = (
256
)
graph_config.prefill.full_prefill_prefix_chunk_tokens = 256
self.assertEqual(
PrefillCudaGraphRunner._resolve_prefix_chunk_shape(model_runner, 4),
(8, 32),
)
model_runner.server_args.cuda_graph_config.prefill.full_prefill_prefix_chunk_tokens = (
0
)
graph_config.prefill.full_prefill_prefix_chunk_tokens = 0
with self.assertRaisesRegex(ValueError, "must be positive"):
PrefillCudaGraphRunner._resolve_prefix_chunk_shape(model_runner, 4)
test/registered/unit/multimodal/test_gpu_feature_transport.py
+12 -7
View File
@@ -101,7 +101,7 @@ class TestCudaVmmFeatureTransport(unittest.TestCase):
)
# The consumer count comes from the published topology.
override = get_context().override_server_args(
enable_dp_attention=False, tp_size=4
enable_dp_attention=False, tp_size=4, mm_feature_transport="cuda_vmm"
)
override.install()
self.addCleanup(override.restore)
@@ -122,13 +122,16 @@ class TestCudaVmmFeatureTransport(unittest.TestCase):
)
def test_disabled_transport_is_a_noop(self):
from sglang.srt.runtime_context import get_context
from sglang.srt.utils.cuda_vmm_transport_utils import (
CudaVmmFeatureTransport,
)
transport = CudaVmmFeatureTransport(
SimpleNamespace(mm_feature_transport="cpu"), None
)
# The transport choice is a bag leaf.
override = get_context().override_server_args(mm_feature_transport="cpu")
override.install()
self.addCleanup(override.restore)
transport = CudaVmmFeatureTransport(SimpleNamespace(), None)
self.assertEqual(transport.prepare_for_dispatch([None]), [])
transport.cancel_for_dispatch([])
@@ -136,14 +139,16 @@ class TestCudaVmmFeatureTransport(unittest.TestCase):
self.assertIsNone(transport.pool)
def test_vmm_transport_requires_processor(self):
from sglang.srt.runtime_context import get_context
from sglang.srt.utils.cuda_vmm_transport_utils import (
CudaVmmFeatureTransport,
)
override = get_context().override_server_args(mm_feature_transport="cuda_vmm")
override.install()
self.addCleanup(override.restore)
with self.assertRaisesRegex(RuntimeError, "multimodal processor"):
CudaVmmFeatureTransport(
SimpleNamespace(mm_feature_transport="cuda_vmm"), None
)
CudaVmmFeatureTransport(SimpleNamespace(), None)
def test_image_features_are_packed_per_request(self):
from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem
test/registered/unit/server_args/test_server_args.py
+18 -7
View File
@@ -1683,16 +1683,22 @@ class TestCudaGraphConfigDataclassAccess(CustomTestCase):
mock_backend = mock_get_moe_a2a_backend.return_value
mock_backend.is_deepep.return_value = False
mock_backend.is_mooncake.return_value = False
server_args = SimpleNamespace(
from sglang.srt.runtime_context import get_context
# The graph configuration is a bag leaf; the debug switch is raw input
# and stays on the argument.
override = get_context().override_server_args(
cuda_graph_config=CudaGraphConfig(
prefill=PhaseConfig(
backend=Backend.TC_PIECEWISE,
bs=[32, 64],
tc_compiler="eager",
)
),
enable_torch_compile_debug_mode=False,
)
)
override.install()
self.addCleanup(override.restore)
server_args = SimpleNamespace(enable_torch_compile_debug_mode=False)
config = TcPiecewiseCudaGraphBackend.build_compilation_config(server_args)
@@ -2063,15 +2069,15 @@ class TestGrpcServerArgs(CustomTestCase):
def test_sidecar_builds_loopback_grpc_endpoints(self):
self.assertEqual(
build_sidecar_endpoint(SimpleNamespace(host="0.0.0.0", grpc_port=50051)),
build_sidecar_endpoint("0.0.0.0", 50051),
"http://127.0.0.1:50051",
)
self.assertEqual(
build_sidecar_endpoint(SimpleNamespace(host="::", grpc_port=50051)),
build_sidecar_endpoint("::", 50051),
"http://[::1]:50051",
)
self.assertEqual(
build_sidecar_endpoint(SimpleNamespace(host="[::]", grpc_port=50051)),
build_sidecar_endpoint("[::]", 50051),
"http://[::1]:50051",
)
@@ -2083,6 +2089,8 @@ class TestGrpcServerArgs(CustomTestCase):
self.assertEqual(parsed.sidecar_args, argv)
def test_start_sidecar_passes_endpoint_and_provider_argv_separately(self):
from sglang.srt.runtime_context import get_context as get_context_for_config
server_args = SimpleNamespace(
sidecar="example.sidecar",
sidecar_args=[
@@ -2092,8 +2100,11 @@ class TestGrpcServerArgs(CustomTestCase):
"2",
],
host="127.0.0.1",
grpc_port=50051,
)
# The port the sidecar dials is the resolved one, off the bag.
override = get_context_for_config().override_server_args(grpc_port=50051)
override.install()
self.addCleanup(override.restore)
with (
patch("sglang.srt.entrypoints.sidecar.mp.get_context") as get_context,
patch("sglang.srt.entrypoints.sidecar.Sidecar") as sidecar_class,
test/registered/unit/spec/test_dflash_overlap_hostsync.py
+8
View File
@@ -8,6 +8,7 @@ from types import SimpleNamespace
import torch
from sglang.srt.runtime_context import get_context
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
@@ -236,6 +237,13 @@ class TestHybridNeedsCpuSeqLens(CustomTestCase):
req_to_token_pool=None,
model_config=SimpleNamespace(context_len=2048),
)
# The backend takes the mode from the published configuration, not from
# the runner it is handed.
override = get_context().override_server_args(
speculative_attention_mode=spec_mode
)
override.install()
self.addCleanup(override.restore)
return HybridAttnBackend(runner, backend(prefill_flag), backend(decode_flag))
def test_delegation(self):
test/registered/unit/test_global_config_read_ratchet.py
+34
View File
@@ -107,6 +107,40 @@ _CONFIGURED_SIZE_CALL_SITES = {
("srt/managers/scheduler.py", "configured_dcp_size"): (
"same pre-distributed-init arithmetic in configure_scheduler_process"
),
("srt/model_executor/runner/base_runner.py", "configured_pp_size"): (
"the runner's layer window is arithmetic over the configured stage "
"count; a draft runner shares the target's groups, so the live "
"property would answer for the wrong runner"
),
("srt/model_executor/cpu_graph_runner.py", "configured_pp_size"): (
"the same window, on the CPU graph path"
),
(
"srt/managers/scheduler_components/metrics_reporter.py",
"configured_pp_size",
): (
"the reporter labels its metrics with the stage count it was launched "
"with, which is configuration; the live group answers per process"
),
("srt/speculative/eagle_draft_cuda_graph_runner.py", "configured_pp_size"): (
"the draft runner's window over the target's stages: its own groups are "
"the target's, so the configured count is the one that describes it"
),
(
"srt/speculative/eagle_draft_extend_cuda_graph_runner.py",
"configured_pp_size",
): ("the same draft window, on the extend path"),
(
"srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py",
"configured_pp_size",
): ("the same draft window, multi-layer extend"),
("srt/speculative/frozen_kv_mtp_cuda_graph_runner.py", "configured_pp_size"): (
"the same draft window, frozen-KV MTP"
),
("srt/entrypoints/v1_loads.py", "configured_pp_size"): (
"the /v1/loads accelerator count is arithmetic over the launch shape, "
"reported from the tokenizer process, which holds no model groups"
),
("srt/disaggregation/common/conn.py", "configured_pp_size"): (
"the bootstrap connection is built by the KV manager on the transfer "
"path, which the CPU-only conn tests exercise without ever starting "
test/registered/unit/test_supplied_instance_exposure_ratchet.py
-19
View File
@@ -144,7 +144,6 @@ _EXPOSED = {
("layers/moe/utils.py", "moe_runner_backend"),
("layers/moe/utils.py", "quantization"),
("layers/moe/utils.py", "speculative_moe_runner_backend"),
("entrypoints/sidecar.py", "grpc_port"),
("configs/embedding_model_spec.py", "chunked_prefill_size"),
("configs/embedding_model_spec.py", "cuda_graph_config"),
("configs/embedding_model_spec.py", "disable_radix_cache"),
@@ -159,13 +158,6 @@ _EXPOSED = {
("configs/model_config.py", "quantization"),
("configs/model_config.py", "speculative_algorithm"),
("configs/model_config.py", "speculative_draft_model_quantization"),
("disaggregation/utils.py", "disaggregation_transfer_backend"),
("distributed/bootstrap.py", "disable_custom_all_reduce"),
("distributed/bootstrap.py", "enable_symm_mem"),
("distributed/bootstrap.py", "enable_torch_symm_mem"),
("distributed/bootstrap.py", "flashinfer_allreduce_fusion_backend"),
("distributed/bootstrap.py", "moe_a2a_backend"),
("distributed/bootstrap.py", "pre_warm_nccl"),
("entrypoints/engine.py", "attn_cp_size"),
("entrypoints/engine.py", "enable_symm_mem"),
("entrypoints/engine.py", "moe_dp_size"),
@@ -183,10 +175,7 @@ _EXPOSED = {
("layers/cp/bcg.py", "cp_strategy"),
("layers/cp/bcg.py", "enable_prefill_cp"),
("layers/flashinfer_comm_fusion.py", "flashinfer_allreduce_fusion_backend"),
("layers/moe/kt_ep_wrapper.py", "chunked_prefill_size"),
("layers/quantization/unquant.py", "enable_deterministic_inference"),
("lora/lora_manager.py", "enable_lora_overlap_loading"),
("lora/marlin_lora_temp/policy.py", "enable_lora"),
("lora/marlin_lora_temp/policy.py", "lora_paths"),
("managers/data_parallel_controller.py", "attn_cp_size"),
("managers/data_parallel_controller.py", "disaggregation_mode"),
@@ -194,13 +183,6 @@ _EXPOSED = {
("managers/data_parallel_controller.py", "moe_dp_size"),
("managers/data_parallel_controller.py", "pp_size"),
("managers/data_parallel_controller.py", "soft_watchdog_timeout"),
("managers/prefill_delayer.py", "disable_overlap_schedule"),
("managers/rust_server.py", "mm_process_config"),
("mem_cache/kv_cache_builder.py", "hicache_mem_layout"),
(
"model_executor/runner_backend/tc_piecewise_cuda_graph_backend.py",
"cuda_graph_config",
),
("parser/template_detection.py", "model_path"),
("speculative/adaptive_spec_params.py", "speculative_algorithm"),
("speculative/adaptive_spec_params.py", "speculative_eagle_topk"),
@@ -208,7 +190,6 @@ _EXPOSED = {
("speculative/spec_info.py", "enable_multi_layer_eagle"),
("utils/common.py", "speculative_num_draft_tokens"),
("utils/common.py", "speculative_num_steps"),
("utils/cuda_vmm_transport_utils.py", "mm_feature_transport"),
("utils/hf_transformers/processor.py", "image_processor_backend"),
}