config: the runtime readers take the published bags (#36254)
This commit is contained in:
@@ -10,6 +10,7 @@ from sglang.srt.distributed.parallel_state import (
|
||||
from sglang.srt.layers.dp_attention import set_dp_buffer_len
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import FlashinferDispatcher
|
||||
from sglang.srt.layers.moe.utils import initialize_moe_config
|
||||
from sglang.srt.runtime_context import publish
|
||||
from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -22,7 +23,8 @@ class TestFlashinferDispatcher(CustomTestCase):
|
||||
server_args.moe_runner_backend = "flashinfer_cutlass"
|
||||
server_args.moe_a2a_backend = "flashinfer"
|
||||
set_global_server_args_for_scheduler(server_args)
|
||||
initialize_moe_config(server_args)
|
||||
publish(server_args, role="scheduler")
|
||||
initialize_moe_config()
|
||||
|
||||
init_distributed_environment(
|
||||
world_size=-1, # Auto-detect from environment
|
||||
|
||||
@@ -60,7 +60,7 @@ class _FakeCPGroup:
|
||||
|
||||
class TestCPStrategyUnit(CustomTestCase):
|
||||
def tearDown(self):
|
||||
init_cp_strategy(SimpleNamespace(enable_prefill_cp=False))
|
||||
init_cp_strategy(enable_prefill_cp=False, cp_size=1, cp_strategy="zigzag")
|
||||
|
||||
def test_strategy_kind_maps_cli_values(self):
|
||||
self.assertEqual(ContextParallelStrategyKind.NONE.value, 0)
|
||||
@@ -77,11 +77,9 @@ class TestCPStrategyUnit(CustomTestCase):
|
||||
|
||||
def test_init_cp_strategy_binds_zigzag_strategy(self):
|
||||
init_cp_strategy(
|
||||
SimpleNamespace(
|
||||
enable_prefill_cp=True,
|
||||
cp_strategy="zigzag",
|
||||
attn_cp_size=4,
|
||||
)
|
||||
enable_prefill_cp=True,
|
||||
cp_size=4,
|
||||
cp_strategy="zigzag",
|
||||
)
|
||||
|
||||
self.assertTrue(is_cp_enabled())
|
||||
@@ -91,11 +89,9 @@ class TestCPStrategyUnit(CustomTestCase):
|
||||
|
||||
def test_get_cp_strategy_is_initialized_under_cp_v2(self):
|
||||
init_cp_strategy(
|
||||
SimpleNamespace(
|
||||
enable_prefill_cp=True,
|
||||
cp_strategy="interleave",
|
||||
attn_cp_size=4,
|
||||
)
|
||||
enable_prefill_cp=True,
|
||||
cp_size=4,
|
||||
cp_strategy="interleave",
|
||||
)
|
||||
|
||||
with patch(
|
||||
@@ -108,7 +104,7 @@ class TestCPStrategyUnit(CustomTestCase):
|
||||
|
||||
class TestPrefillCPBCGReplay(CustomTestCase):
|
||||
def tearDown(self):
|
||||
init_cp_strategy(SimpleNamespace(enable_prefill_cp=False))
|
||||
init_cp_strategy(enable_prefill_cp=False, cp_size=1, cp_strategy="zigzag")
|
||||
|
||||
def _make_runner(self):
|
||||
runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner)
|
||||
@@ -141,11 +137,9 @@ class TestPrefillCPBCGReplay(CustomTestCase):
|
||||
|
||||
def _enable_zigzag(self):
|
||||
init_cp_strategy(
|
||||
SimpleNamespace(
|
||||
enable_prefill_cp=True,
|
||||
cp_strategy="zigzag",
|
||||
attn_cp_size=4,
|
||||
)
|
||||
enable_prefill_cp=True,
|
||||
cp_size=4,
|
||||
cp_strategy="zigzag",
|
||||
)
|
||||
|
||||
def test_local_capacity_overflow_uses_next_capture_bucket(self):
|
||||
@@ -267,16 +261,13 @@ class TestPrefillCPBCGReplay(CustomTestCase):
|
||||
class TestCPZigzagStrategy(CustomTestCase):
|
||||
def setUp(self):
|
||||
init_cp_strategy(
|
||||
SimpleNamespace(
|
||||
enable_prefill_cp=True,
|
||||
cp_strategy="zigzag",
|
||||
attn_cp_size=4,
|
||||
attention_backend="fa3",
|
||||
)
|
||||
enable_prefill_cp=True,
|
||||
cp_size=4,
|
||||
cp_strategy="zigzag",
|
||||
)
|
||||
|
||||
def tearDown(self):
|
||||
init_cp_strategy(SimpleNamespace(enable_prefill_cp=False))
|
||||
init_cp_strategy(enable_prefill_cp=False, cp_size=1, cp_strategy="zigzag")
|
||||
|
||||
def _metadata_for_rank(self, rank, *, cp_size, seq_lens, extend_seq_lens):
|
||||
strategy = ZigzagCPStrategy(cp_size=cp_size)
|
||||
@@ -809,16 +800,13 @@ class TestCPZigzagStrategy(CustomTestCase):
|
||||
class TestCPInterleaveStrategy(CustomTestCase):
|
||||
def setUp(self):
|
||||
init_cp_strategy(
|
||||
SimpleNamespace(
|
||||
enable_prefill_cp=True,
|
||||
cp_strategy="interleave",
|
||||
attn_cp_size=4,
|
||||
attention_backend="fa3",
|
||||
)
|
||||
enable_prefill_cp=True,
|
||||
cp_size=4,
|
||||
cp_strategy="interleave",
|
||||
)
|
||||
|
||||
def tearDown(self):
|
||||
init_cp_strategy(SimpleNamespace(enable_prefill_cp=False))
|
||||
init_cp_strategy(enable_prefill_cp=False, cp_size=1, cp_strategy="zigzag")
|
||||
|
||||
def _metadata_for_rank(self, rank, *, cp_size, seq_lens, extend_seq_lens):
|
||||
strategy = InterleaveCPStrategy(cp_size=cp_size)
|
||||
|
||||
@@ -71,7 +71,7 @@ class TestEmbeddingModelSpec(unittest.TestCase):
|
||||
)
|
||||
plan = resolved_embedding_plan(
|
||||
spec,
|
||||
server_args=SimpleNamespace(
|
||||
config=SimpleNamespace(
|
||||
is_embedding=True,
|
||||
cuda_graph_config=SimpleNamespace(
|
||||
prefill=SimpleNamespace(
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import contextlib
|
||||
import types
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -90,23 +89,24 @@ def _torch_allreduce_residual_rmsnorm_baseline(
|
||||
|
||||
|
||||
class TestFlashInferCommFusion(CustomTestCase):
|
||||
"""The arch dispatch is `_resolve_backend(backend, is_multi_node)`.
|
||||
|
||||
The public entry above it takes no arguments -- it reads
|
||||
`exec.comm.flashinfer_allreduce_fusion_backend` and `parallel.nnodes` off the
|
||||
published bags -- so the cases here drive the dispatch directly.
|
||||
"""
|
||||
|
||||
def test_auto_backend_resolves_by_arch(self):
|
||||
single_node = types.SimpleNamespace(
|
||||
flashinfer_allreduce_fusion_backend="auto", nnodes=1
|
||||
)
|
||||
multi_node = types.SimpleNamespace(
|
||||
flashinfer_allreduce_fusion_backend="auto", nnodes=2
|
||||
)
|
||||
single_node = ("auto", False)
|
||||
multi_node = ("auto", True)
|
||||
|
||||
# Blackwell: mnnvl on both single-node and multi-node.
|
||||
with patch.object(fusion, "is_sm100_supported", return_value=True):
|
||||
self.assertEqual(
|
||||
fusion.resolve_flashinfer_allreduce_fusion_backend(single_node),
|
||||
fusion._resolve_backend(*single_node),
|
||||
"mnnvl",
|
||||
)
|
||||
self.assertEqual(
|
||||
fusion.resolve_flashinfer_allreduce_fusion_backend(multi_node), "mnnvl"
|
||||
)
|
||||
self.assertEqual(fusion._resolve_backend(*multi_node), "mnnvl")
|
||||
|
||||
# SM90: auto uses trtllm on single-node, multi-node is unsupported.
|
||||
with (
|
||||
@@ -114,11 +114,11 @@ class TestFlashInferCommFusion(CustomTestCase):
|
||||
patch.object(fusion, "is_sm90_supported", return_value=True),
|
||||
):
|
||||
self.assertEqual(
|
||||
fusion.resolve_flashinfer_allreduce_fusion_backend(single_node),
|
||||
fusion._resolve_backend(*single_node),
|
||||
"trtllm",
|
||||
)
|
||||
with self.assertRaises(ValueError):
|
||||
fusion.resolve_flashinfer_allreduce_fusion_backend(multi_node)
|
||||
fusion._resolve_backend(*multi_node)
|
||||
|
||||
# Architectures outside SM90/SM10X are unsupported. Both pre-SM90
|
||||
# and post-SM10X devices (e.g. SM120) must fail closed.
|
||||
@@ -129,48 +129,40 @@ class TestFlashInferCommFusion(CustomTestCase):
|
||||
patch.object(fusion, "is_sm90_supported", return_value=False),
|
||||
):
|
||||
with self.assertRaises(ValueError):
|
||||
fusion.resolve_flashinfer_allreduce_fusion_backend(single_node)
|
||||
fusion._resolve_backend(*single_node)
|
||||
with self.assertRaises(ValueError):
|
||||
fusion.resolve_flashinfer_allreduce_fusion_backend(multi_node)
|
||||
fusion._resolve_backend(*multi_node)
|
||||
|
||||
def test_explicit_backend_validation(self):
|
||||
single_node_mnnvl = types.SimpleNamespace(
|
||||
flashinfer_allreduce_fusion_backend="mnnvl", nnodes=1
|
||||
)
|
||||
multi_node_mnnvl = types.SimpleNamespace(
|
||||
flashinfer_allreduce_fusion_backend="mnnvl", nnodes=2
|
||||
)
|
||||
single_node_trtllm = types.SimpleNamespace(
|
||||
flashinfer_allreduce_fusion_backend="trtllm", nnodes=1
|
||||
)
|
||||
multi_node_trtllm = types.SimpleNamespace(
|
||||
flashinfer_allreduce_fusion_backend="trtllm", nnodes=2
|
||||
)
|
||||
single_node_mnnvl = ("mnnvl", False)
|
||||
multi_node_mnnvl = ("mnnvl", True)
|
||||
single_node_trtllm = ("trtllm", False)
|
||||
multi_node_trtllm = ("trtllm", True)
|
||||
|
||||
with (
|
||||
patch.object(fusion, "is_sm100_supported", return_value=False),
|
||||
patch.object(fusion, "is_sm90_supported", return_value=True),
|
||||
):
|
||||
self.assertEqual(
|
||||
fusion.resolve_flashinfer_allreduce_fusion_backend(single_node_mnnvl),
|
||||
fusion._resolve_backend(*single_node_mnnvl),
|
||||
"mnnvl",
|
||||
)
|
||||
self.assertEqual(
|
||||
fusion.resolve_flashinfer_allreduce_fusion_backend(single_node_trtllm),
|
||||
fusion._resolve_backend(*single_node_trtllm),
|
||||
"trtllm",
|
||||
)
|
||||
with self.assertRaises(ValueError):
|
||||
fusion.resolve_flashinfer_allreduce_fusion_backend(multi_node_mnnvl)
|
||||
fusion._resolve_backend(*multi_node_mnnvl)
|
||||
with self.assertRaises(ValueError):
|
||||
fusion.resolve_flashinfer_allreduce_fusion_backend(multi_node_trtllm)
|
||||
fusion._resolve_backend(*multi_node_trtllm)
|
||||
|
||||
with patch.object(fusion, "is_sm100_supported", return_value=True):
|
||||
self.assertEqual(
|
||||
fusion.resolve_flashinfer_allreduce_fusion_backend(multi_node_mnnvl),
|
||||
fusion._resolve_backend(*multi_node_mnnvl),
|
||||
"mnnvl",
|
||||
)
|
||||
with self.assertRaises(ValueError):
|
||||
fusion.resolve_flashinfer_allreduce_fusion_backend(multi_node_trtllm)
|
||||
fusion._resolve_backend(*multi_node_trtllm)
|
||||
|
||||
for arch in ("pre_sm90", "post_sm10x"):
|
||||
with (
|
||||
@@ -184,9 +176,9 @@ class TestFlashInferCommFusion(CustomTestCase):
|
||||
single_node_trtllm,
|
||||
multi_node_trtllm,
|
||||
):
|
||||
with self.subTest(backend=args.flashinfer_allreduce_fusion_backend):
|
||||
with self.subTest(backend=args[0], multi_node=args[1]):
|
||||
with self.assertRaises(ValueError):
|
||||
fusion.resolve_flashinfer_allreduce_fusion_backend(args)
|
||||
fusion._resolve_backend(*args)
|
||||
|
||||
def test_allreduce_fusion_backends_match_torch_baseline(self):
|
||||
fake_comm = _FakeFlashInferComm()
|
||||
|
||||
@@ -27,6 +27,7 @@ from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Phase,
|
||||
PhaseConfig,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context, get_serving
|
||||
from sglang.srt.server_args import PortArgs, ServerArgs, prepare_server_args
|
||||
from sglang.srt.server_args_config_parser import ConfigArgumentMerger
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
@@ -2356,9 +2357,13 @@ class TestGrpcServerArgs(CustomTestCase):
|
||||
|
||||
fake_core = SimpleNamespace(start_server=MagicMock(return_value="handle"))
|
||||
fake_bridge = SimpleNamespace(RuntimeHandle=MagicMock(return_value="rt"))
|
||||
server_args = SimpleNamespace(
|
||||
host="127.0.0.1", grpc_port=50051, grpc_worker_threads=4
|
||||
)
|
||||
# The host comes from the `serving` bag; `grpc_worker_threads` is not a
|
||||
# field (resolution sets it from the environment), so it stays on the
|
||||
# stand-in the call site is handed.
|
||||
override = get_context().override_server_args(host="127.0.0.1", grpc_port=50051)
|
||||
override.install()
|
||||
self.addCleanup(override.restore)
|
||||
server_args = SimpleNamespace(grpc_worker_threads=4)
|
||||
with (
|
||||
patch(
|
||||
"sglang.srt.rust_extensions.load_rust_extension",
|
||||
@@ -2373,7 +2378,7 @@ class TestGrpcServerArgs(CustomTestCase):
|
||||
tokenizer_manager=MagicMock(),
|
||||
template_manager=MagicMock(),
|
||||
scheduler_info={},
|
||||
grpc_port=resolution_result(server_args, "grpc_port"),
|
||||
grpc_port=get_serving().grpc_port,
|
||||
)
|
||||
|
||||
self.assertEqual(handle, "handle")
|
||||
|
||||
@@ -19,7 +19,7 @@ from sglang.srt.layers.moe.utils import (
|
||||
speculative_moe_backend_context,
|
||||
)
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.runtime_context import get_context, get_flags, get_model
|
||||
from sglang.srt.runtime_context import get_context, get_flags, get_model, publish
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -145,9 +145,11 @@ class TestFusionDecisionFlag(CustomTestCase):
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
self._seed()
|
||||
initialize_moe_config(
|
||||
ServerArgs(model_path="dummy", disable_shared_experts_fusion=True)
|
||||
publish(
|
||||
ServerArgs(model_path="dummy", disable_shared_experts_fusion=True),
|
||||
role="scheduler",
|
||||
)
|
||||
initialize_moe_config()
|
||||
moe = get_flags().moe
|
||||
self.assertTrue(moe.disable_shared_experts_fusion)
|
||||
self.assertTrue(moe.speculative_disable_shared_experts_fusion)
|
||||
|
||||
@@ -53,6 +53,18 @@ _SLOT_OWNERS = ("srt/runtime_context.py", "srt/server_args.py", "srt/arg_groups/
|
||||
# The test below asserts this map is exactly the set of such reads, so the
|
||||
# reasons cannot drift away from the code.
|
||||
_CONFIGURED_SIZE_CALL_SITES = {
|
||||
("srt/layers/cp/base.py", "attn_cp_size"): (
|
||||
"the lazy strategy bind in a worker: the CP group is what the strategy "
|
||||
"is being built for, and the configured width is what describes it"
|
||||
),
|
||||
("benchmark/one_batch.py", "pp_size"): (
|
||||
"CPU affinity for this rank, computed right after the work function "
|
||||
"publishes and before dist init, so the groups do not exist yet"
|
||||
),
|
||||
("benchmark/one_batch.py", "tp_size"): (
|
||||
"the same affinity computation: the layout is the configured one, and "
|
||||
"the live group is not up at this point in the work function"
|
||||
),
|
||||
("srt/entrypoints/engine.py", "pp_size"): (
|
||||
"the launch path decides how many scheduler processes to spawn; it runs "
|
||||
"before any of them exists, so there is no group to ask"
|
||||
@@ -68,6 +80,13 @@ _CONFIGURED_SIZE_CALL_SITES = {
|
||||
"the Ray driver sizes the actor placement group; the actors it is about "
|
||||
"to create are the ones that will hold the process groups"
|
||||
),
|
||||
("srt/ray/engine.py", "tp_size"): (
|
||||
"the same placement arithmetic as the stage count: the driver sizes "
|
||||
"the actors that will hold the process groups"
|
||||
),
|
||||
("srt/ray/data_parallel_controller.py", "tp_size"): (
|
||||
"the same arithmetic on the DP path, also in the driver"
|
||||
),
|
||||
("srt/ray/data_parallel_controller.py", "pp_size"): (
|
||||
"same placement arithmetic on the DP path -- ranks per TP group, "
|
||||
"computed in the driver before the actors start"
|
||||
@@ -115,6 +134,26 @@ _CONFIGURED_SIZE_CALL_SITES = {
|
||||
("srt/managers/scheduler.py", "dcp_size"): (
|
||||
"same pre-distributed-init arithmetic in configure_scheduler_process"
|
||||
),
|
||||
("srt/model_executor/runner/base_runner.py", "tp_size"): (
|
||||
"the same window as the stage count next to it: a draft runner shares "
|
||||
"the target's groups, so the live property would answer for the wrong "
|
||||
"runner"
|
||||
),
|
||||
("srt/model_executor/cpu_graph_runner.py", "tp_size"): (
|
||||
"the same window, on the CPU graph path"
|
||||
),
|
||||
("srt/entrypoints/v1_loads.py", "tp_size"): (
|
||||
"the accelerator count is arithmetic over the launch shape, reported "
|
||||
"from the tokenizer process, which holds no model groups"
|
||||
),
|
||||
("srt/disaggregation/nixl/conn.py", "tp_size"): (
|
||||
"the NIXL rank arithmetic runs on the transfer path, which the CPU-only "
|
||||
"conn tests exercise without starting torch.distributed"
|
||||
),
|
||||
("srt/managers/tokenizer_control_mixin.py", "tp_size"): (
|
||||
"the tokenizer divides its worker count by the launch width; it holds "
|
||||
"no model groups"
|
||||
),
|
||||
("srt/model_executor/runner/base_runner.py", "pp_size"): (
|
||||
"the runner's layer window is arithmetic over the configured stage "
|
||||
"count; a draft runner shares the target's groups, so the live "
|
||||
@@ -212,6 +251,27 @@ _CONFIGURED_SIZE_CALL_SITES = {
|
||||
"the encode server's launch entry sizes its workers before it has "
|
||||
"spawned any of them"
|
||||
),
|
||||
("srt/disaggregation/encoder/grpc_server.py", "tp_size"): (
|
||||
"the same worker-count arithmetic on the gRPC entry: it spawns the TP "
|
||||
"workers, so their groups do not exist yet"
|
||||
),
|
||||
("srt/disaggregation/encoder/server.py", "tp_size"): (
|
||||
"`MMEncoder` builds its own TP group from this size -- "
|
||||
"`initialize_model_parallel` is the call being handed it, so there is "
|
||||
"nothing live to ask"
|
||||
),
|
||||
("srt/disaggregation/encoder/receiver.py", "tp_size"): (
|
||||
"the receiver labels and shards by the launch width; it runs in the "
|
||||
"tokenizer process, which holds no encoder groups"
|
||||
),
|
||||
("srt/managers/rust_server.py", "tp_size"): (
|
||||
"the rust server decides its transport from the launch width, in the "
|
||||
"tokenizer process, which holds no model groups"
|
||||
),
|
||||
("compile_deep_gemm.py", "tp_size"): (
|
||||
"the warm-up request fans bootstrap rooms across the launch's ranks; it "
|
||||
"runs in the tokenizer process, which holds no model groups"
|
||||
),
|
||||
("srt/utils/common.py", "tp_size"): (
|
||||
"the require_*_tp_gather predicates compared the configured tp_size "
|
||||
"when they read the record; the live property answers a different "
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
"""The Ray driver sizes its actors from the published configuration.
|
||||
|
||||
`RayEngine` publishes as part of `Engine._launch_subprocesses` and *then* lays
|
||||
out the actors, so the placement arithmetic reads the `parallel` bag. That is
|
||||
where a resolution decision lives: a launch that leaves `dp_size` to resolution
|
||||
has it in the `parallel` bag, and the override case below is what tells the two
|
||||
apart.
|
||||
|
||||
There is no CI coverage of the Ray path (`test/manual/test_ray_engine.py` boots a
|
||||
real cluster), so these cases drive the two pure helpers directly against a
|
||||
published config -- including the override direction, which is what tells a bag
|
||||
read from a record read.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import unittest
|
||||
|
||||
from sglang.srt.runtime_context import get_context, get_parallel
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=6, suite="base-a-test-cpu")
|
||||
|
||||
# `sglang.srt.ray.engine` imports `ray` at module scope, and the CPU runner has
|
||||
# no ray wheel. The file-scoped source scan below is the part that has to run
|
||||
# everywhere; the three arithmetic cases need the import.
|
||||
_HAS_RAY = importlib.util.find_spec("ray") is not None
|
||||
_needs_ray = unittest.skipUnless(_HAS_RAY, "ray is not installed")
|
||||
|
||||
|
||||
class TestRayDriverReadsTheBags(CustomTestCase):
|
||||
def _publish(self, **fields):
|
||||
override = get_context().override_server_args(**fields)
|
||||
override.install()
|
||||
self.addCleanup(override.restore)
|
||||
|
||||
@_needs_ray
|
||||
def test_world_size_multiplies_the_published_sizes(self):
|
||||
from sglang.srt.ray.engine import _compute_world_size
|
||||
|
||||
self._publish(tp_size=2, pp_size=3, dp_size=4, enable_dp_attention=False)
|
||||
self.assertEqual(_compute_world_size(), 24)
|
||||
|
||||
@_needs_ray
|
||||
def test_dp_attention_folds_dp_into_tp(self):
|
||||
from sglang.srt.ray.engine import _compute_world_size
|
||||
|
||||
self._publish(tp_size=4, pp_size=2, dp_size=4, enable_dp_attention=True)
|
||||
# DP attention folds DP into TP, so dp_size drops out of the product.
|
||||
self.assertEqual(_compute_world_size(), 8)
|
||||
|
||||
@_needs_ray
|
||||
def test_the_world_size_follows_a_post_publish_override(self):
|
||||
"""The direction that separates a bag read from a record read.
|
||||
|
||||
`override` writes the bag and never the record, so a driver still
|
||||
reading `server_args.tp_size` would keep answering with the old size.
|
||||
"""
|
||||
from sglang.srt.ray.engine import _compute_world_size
|
||||
|
||||
self._publish(tp_size=2, pp_size=1, dp_size=1, enable_dp_attention=False)
|
||||
self.assertEqual(_compute_world_size(), 2)
|
||||
get_context().override("test.ray_driver", tp_size=8)
|
||||
self.assertEqual(get_parallel().config.tp_size, 8)
|
||||
self.assertEqual(_compute_world_size(), 8)
|
||||
|
||||
def test_the_driver_modules_read_no_field_off_a_record(self):
|
||||
"""File-scoped: neither Ray driver module reads a config field off an
|
||||
instance any more.
|
||||
|
||||
The Ray path has no CI coverage, so this is what keeps a new
|
||||
`server_args.tp_size` from appearing in it -- the placement arithmetic
|
||||
runs after the publish, and the bags are the surface that carries what
|
||||
resolution decided.
|
||||
"""
|
||||
import ast
|
||||
import dataclasses
|
||||
import pathlib
|
||||
|
||||
import sglang
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
fields = {field.name for field in dataclasses.fields(ServerArgs)}
|
||||
srt = pathlib.Path(sglang.__file__).resolve().parent / "srt"
|
||||
offenders = []
|
||||
for rel in ("ray/engine.py", "ray/data_parallel_controller.py"):
|
||||
tree = ast.parse((srt / rel).read_text(encoding="utf-8-sig"))
|
||||
holders = {"server_args", "sa"}
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
continue
|
||||
for arg in list(node.args.args) + list(node.args.kwonlyargs):
|
||||
if arg.annotation is not None and "ServerArgs" in ast.dump(
|
||||
arg.annotation
|
||||
):
|
||||
holders.add(arg.arg)
|
||||
for node in ast.walk(tree):
|
||||
if (
|
||||
isinstance(node, ast.Attribute)
|
||||
and node.attr in fields
|
||||
and isinstance(node.ctx, ast.Load)
|
||||
and (
|
||||
(isinstance(node.value, ast.Name) and node.value.id in holders)
|
||||
or (
|
||||
isinstance(node.value, ast.Attribute)
|
||||
and node.value.attr == "server_args"
|
||||
)
|
||||
)
|
||||
):
|
||||
offenders.append(f"{rel}:{node.lineno} reads .{node.attr}")
|
||||
self.assertEqual(
|
||||
offenders,
|
||||
[],
|
||||
"the Ray driver reads a config field off a record; the driver runs "
|
||||
"after the publish, so read `get_parallel().config`:\n "
|
||||
+ "\n ".join(offenders),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -519,8 +519,6 @@ class TestMoeFlagsGroup(_IsolatedServerArgs):
|
||||
swap under the speculative contexts and restore on exit."""
|
||||
|
||||
def _init(self, **kw):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.layers.moe.utils import initialize_moe_config
|
||||
|
||||
defaults = dict(
|
||||
@@ -538,7 +536,12 @@ class TestMoeFlagsGroup(_IsolatedServerArgs):
|
||||
disable_shared_experts_fusion=False,
|
||||
)
|
||||
defaults.update(kw)
|
||||
initialize_moe_config(SimpleNamespace(**defaults))
|
||||
# The flags are seeded from the bags, so the test publishes a config
|
||||
# carrying these values.
|
||||
override = get_context().override_server_args(**defaults)
|
||||
override.install()
|
||||
self.addCleanup(override.restore)
|
||||
initialize_moe_config()
|
||||
|
||||
def test_lazy_defaults_before_initialize(self):
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
|
||||
@@ -134,51 +134,12 @@ _ENV_MATRIX = (({}, {"SGLANG_IS_IN_CI": "true"}),)
|
||||
# are step-12 exposure like any other pair.
|
||||
_PASSED = frozenset({"model_path", "device", "random_seed"})
|
||||
|
||||
# The reads that still take a value off the supplied instance. `initialize_moe_config`
|
||||
# is handed the record until the replay goes away; the rest are pre-publish launcher
|
||||
# reads.
|
||||
_EXPOSED = {
|
||||
("disaggregation/encoder/server.py", "model_loader_extra_config"),
|
||||
("layers/moe/utils.py", "deepep_mode"),
|
||||
("layers/moe/utils.py", "disable_shared_experts_fusion"),
|
||||
("layers/moe/utils.py", "moe_a2a_backend"),
|
||||
("layers/moe/utils.py", "moe_runner_backend"),
|
||||
("layers/moe/utils.py", "quantization"),
|
||||
("layers/moe/utils.py", "speculative_moe_runner_backend"),
|
||||
("configs/embedding_model_spec.py", "chunked_prefill_size"),
|
||||
("configs/embedding_model_spec.py", "cuda_graph_config"),
|
||||
("configs/embedding_model_spec.py", "disable_radix_cache"),
|
||||
("configs/embedding_model_spec.py", "is_embedding"),
|
||||
("configs/embedding_model_spec.py", "prefill_only_disable_kv_cache"),
|
||||
("entrypoints/engine.py", "enable_symm_mem"),
|
||||
("entrypoints/engine.py", "reasoning_parser"),
|
||||
("entrypoints/engine.py", "tool_call_parser"),
|
||||
("layers/flashinfer_comm_fusion.py", "flashinfer_allreduce_fusion_backend"),
|
||||
("layers/moe/utils.py", "deepep_mode"),
|
||||
("layers/moe/utils.py", "moe_a2a_backend"),
|
||||
("layers/moe/utils.py", "moe_runner_backend"),
|
||||
("layers/moe/utils.py", "quantization"),
|
||||
("layers/moe/utils.py", "speculative_moe_runner_backend"),
|
||||
("speculative/draft_worker_common.py", "speculative_draft_attention_backend"),
|
||||
("utils/common.py", "speculative_num_draft_tokens"),
|
||||
("utils/common.py", "speculative_num_steps"),
|
||||
("utils/hf_transformers/processor.py", "image_processor_backend"),
|
||||
("weight_cache/daemon.py", "attn_cp_size"),
|
||||
("weight_cache/daemon.py", "deepep_mode"),
|
||||
("weight_cache/daemon.py", "dp_size"),
|
||||
("weight_cache/daemon.py", "dtype"),
|
||||
("weight_cache/daemon.py", "enable_dp_attention"),
|
||||
("weight_cache/daemon.py", "enable_dp_lm_head"),
|
||||
("weight_cache/daemon.py", "ep_size"),
|
||||
("weight_cache/daemon.py", "load_format"),
|
||||
("weight_cache/daemon.py", "model_loader_extra_config"),
|
||||
("weight_cache/daemon.py", "model_path"),
|
||||
("weight_cache/daemon.py", "moe_a2a_backend"),
|
||||
("weight_cache/daemon.py", "moe_dense_tp_size"),
|
||||
("weight_cache/daemon.py", "moe_dp_size"),
|
||||
("weight_cache/daemon.py", "pp_size"),
|
||||
("weight_cache/daemon.py", "quantization"),
|
||||
}
|
||||
# Empty. A pair belongs here when a reader has no bag to read -- it runs before
|
||||
# its process publishes -- and cannot use `resolving_view` either. The launcher's
|
||||
# pre-publish reads (`_set_envs_and_config`, the auto-parser gate) and the
|
||||
# late-resolution detection it calls all read the declarations now, so nothing
|
||||
# qualifies. A new entry needs that kind of reason next to it.
|
||||
_EXPOSED: frozenset = frozenset()
|
||||
|
||||
# Pairs whose resolution write only happens on a CUDA host (capability or
|
||||
# `is_cuda()` gated): asserted on the CUDA registration, invisible to the CPU
|
||||
@@ -191,28 +152,7 @@ _EXPOSED_CUDA_ONLY: frozenset = frozenset()
|
||||
# Axis two: (file, field) pairs where a supplied-instance read names a field that
|
||||
# some code overrides post-publish. Each needs an ordering judgment, not a blanket
|
||||
# conversion; the list exists so a new one is a decision made when it is written.
|
||||
_OVERRIDDEN_AND_READ = {
|
||||
("entrypoints/engine.py", "reasoning_parser"),
|
||||
("entrypoints/engine.py", "tool_call_parser"),
|
||||
("weight_cache/daemon.py", "dp_size"),
|
||||
("weight_cache/daemon.py", "dtype"),
|
||||
("weight_cache/daemon.py", "ep_size"),
|
||||
("weight_cache/daemon.py", "load_format"),
|
||||
("weight_cache/daemon.py", "model_path"),
|
||||
("mem_cache/pool_host/common.py", "hicache_storage_backend"),
|
||||
("mem_cache/pool_host/common.py", "hicache_storage_backend_extra_config"),
|
||||
("mem_cache/unified_radix_cache.py", "hicache_storage_backend"),
|
||||
("mem_cache/unified_radix_cache.py", "hicache_storage_backend_extra_config"),
|
||||
("mem_cache/unified_radix_cache.py", "hicache_storage_prefetch_policy"),
|
||||
("mem_cache/unified_radix_cache.py", "hicache_write_policy"),
|
||||
("utils/common.py", "speculative_num_draft_tokens"),
|
||||
("utils/common.py", "speculative_num_steps"),
|
||||
("weight_cache/daemon.py", "dp_size"),
|
||||
("weight_cache/daemon.py", "dtype"),
|
||||
("weight_cache/daemon.py", "ep_size"),
|
||||
("weight_cache/daemon.py", "load_format"),
|
||||
("weight_cache/daemon.py", "model_path"),
|
||||
}
|
||||
_OVERRIDDEN_AND_READ: frozenset = frozenset()
|
||||
|
||||
|
||||
def _expanded_override_keys(rel, tree, call, kw) -> set:
|
||||
|
||||
Reference in New Issue
Block a user