From 8005df61d32ccbd4d3f3034c7b9af9bc54dcd5dd Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Wed, 26 Aug 2026 03:00:28 -0700 Subject: [PATCH] config: spell the parallel config tier at the call site (#36250) --- .../skills/sglang-runtime-context/SKILL.md | 103 +++--- python/sglang/benchmark/one_batch.py | 2 +- .../srt/batch_overlap/two_batch_overlap.py | 2 +- .../sglang/srt/disaggregation/common/conn.py | 31 +- .../srt/disaggregation/encoder/http_server.py | 2 +- .../srt/disaggregation/encoder/runtime.py | 19 +- python/sglang/srt/disaggregation/prefill.py | 2 +- python/sglang/srt/distributed/bootstrap.py | 4 +- .../triton_symm_mem_ag.py | 2 +- python/sglang/srt/elastic_ep/elastic_ep.py | 14 +- .../srt/elastic_ep/expert_backup_manager.py | 11 +- python/sglang/srt/entrypoints/engine.py | 23 +- python/sglang/srt/entrypoints/http_server.py | 16 +- python/sglang/srt/entrypoints/v1_loads.py | 7 +- python/sglang/srt/eplb/expert_location.py | 16 +- .../srt/layers/attention/dsa/dsa_indexer.py | 3 +- .../sglang/srt/layers/attention/dsa/utils.py | 6 +- .../attention/flashattention_backend.py | 5 +- python/sglang/srt/layers/communicator.py | 8 +- python/sglang/srt/layers/cp/base.py | 2 +- .../sglang/srt/layers/cp/cp_decode_attn_tp.py | 2 +- python/sglang/srt/layers/dp_attention.py | 17 +- python/sglang/srt/layers/logits_processor.py | 6 +- .../srt/layers/moe/fused_moe_triton/layer.py | 4 +- .../moe/moe_runner/flashinfer_cutedsl.py | 2 +- .../srt/layers/moe/token_dispatcher/nixl.py | 4 +- .../srt/layers/moe/token_dispatcher/pplx.py | 2 +- python/sglang/srt/layers/moe/utils.py | 2 +- python/sglang/srt/layers/utils/cp_utils.py | 6 +- python/sglang/srt/lora/lora_manager.py | 2 +- .../srt/managers/data_parallel_controller.py | 61 ++-- python/sglang/srt/managers/disagg_service.py | 3 +- python/sglang/srt/managers/load_snapshot.py | 14 +- python/sglang/srt/managers/prefill_delayer.py | 2 +- python/sglang/srt/managers/scheduler.py | 60 ++-- .../managers/scheduler_components/dp_attn.py | 4 +- .../scheduler_components/metrics_reporter.py | 4 +- .../scheduler_components/recv_skipper.py | 2 +- .../scheduler_components/request_receiver.py | 6 +- .../sglang/srt/managers/scheduler_pp_mixin.py | 18 +- .../srt/managers/tokenizer_control_mixin.py | 36 +- .../sglang/srt/managers/tokenizer_manager.py | 4 +- .../sglang/srt/mem_cache/kv_cache_builder.py | 4 +- .../srt/mem_cache/kv_cache_configurator.py | 3 +- python/sglang/srt/mem_cache/pool_host/base.py | 2 +- .../srt/model_executor/cpu_graph_runner.py | 5 +- .../sglang/srt/model_executor/model_runner.py | 24 +- .../remote_instance_weight_transporter.py | 6 +- .../spec_aux_hidden_state.py | 4 +- .../startup_weight_load.py | 16 +- .../srt/model_executor/runner/base_runner.py | 17 +- .../runner/decode_cuda_graph_runner.py | 2 +- .../srt/model_executor/runner/eager_runner.py | 2 +- .../runner/prefill_cuda_graph_runner.py | 2 +- python/sglang/srt/model_loader/loader.py | 7 +- python/sglang/srt/models/apertus.py | 2 +- python/sglang/srt/models/arcee.py | 2 +- python/sglang/srt/models/bailing_moe.py | 2 +- .../sglang/srt/models/bailing_moe_linear.py | 2 +- python/sglang/srt/models/bailing_moe_nextn.py | 2 +- .../attention_forward_methods/forward_mla.py | 4 +- .../forward_mla_rocm.py | 4 +- python/sglang/srt/models/deepseek_nextn.py | 2 +- python/sglang/srt/models/deepseek_v2.py | 2 +- python/sglang/srt/models/deepseek_v4.py | 2 +- .../sglang/srt/models/deepseek_v4_dspark.py | 2 +- python/sglang/srt/models/deepseek_v4_nextn.py | 2 +- .../srt/models/dots3_common/modeling.py | 2 +- .../sglang/srt/models/dots3_common/nextn.py | 2 +- python/sglang/srt/models/exaone4.py | 2 +- python/sglang/srt/models/exaone_moe.py | 2 +- python/sglang/srt/models/exaone_moe_mtp.py | 2 +- python/sglang/srt/models/falcon_h1.py | 2 +- python/sglang/srt/models/glm4_moe.py | 2 +- python/sglang/srt/models/glm4_moe_lite.py | 2 +- .../sglang/srt/models/glm4_moe_lite_nextn.py | 2 +- python/sglang/srt/models/glm4_moe_nextn.py | 2 +- python/sglang/srt/models/glm4v_moe.py | 2 +- python/sglang/srt/models/glm_ocr_nextn.py | 2 +- python/sglang/srt/models/gpt_oss.py | 4 +- python/sglang/srt/models/kimi_k25.py | 4 +- python/sglang/srt/models/kimi_k3.py | 11 +- python/sglang/srt/models/laguna.py | 2 +- python/sglang/srt/models/llada2.py | 2 +- python/sglang/srt/models/llama.py | 2 +- python/sglang/srt/models/longcat_flash.py | 2 +- python/sglang/srt/models/mellum.py | 2 +- python/sglang/srt/models/mimo_v2.py | 2 +- python/sglang/srt/models/mimo_v2_nextn.py | 2 +- python/sglang/srt/models/minimax_m3.py | 2 +- python/sglang/srt/models/minimax_m3_vl.py | 2 +- python/sglang/srt/models/nemotron_h.py | 2 +- python/sglang/srt/models/nemotron_h_mtp.py | 2 +- python/sglang/srt/models/qwen2_moe.py | 2 +- python/sglang/srt/models/qwen3.py | 2 +- python/sglang/srt/models/qwen3_5_text.py | 2 +- python/sglang/srt/models/qwen3_moe.py | 2 +- python/sglang/srt/models/qwen3_moe_mtp.py | 2 +- python/sglang/srt/models/qwen3_next.py | 2 +- python/sglang/srt/models/qwen3_next_mtp.py | 2 +- python/sglang/srt/models/qwen3_vl.py | 2 +- python/sglang/srt/models/sarvam_moe.py | 2 +- python/sglang/srt/models/sdar.py | 2 +- python/sglang/srt/models/sdar_moe.py | 2 +- python/sglang/srt/models/step3p5.py | 2 +- .../srt/multimodal/transport/__init__.py | 2 +- .../srt/ray/data_parallel_controller.py | 28 +- python/sglang/srt/ray/engine.py | 40 ++- python/sglang/srt/runtime_context.py | 150 ++++---- .../dspark_components/dspark_worker_v2.py | 17 +- .../eagle_draft_cuda_graph_runner.py | 4 +- .../eagle_draft_extend_cuda_graph_runner.py | 4 +- .../sglang/srt/speculative/eagle_worker_v2.py | 6 +- .../frozen_kv_mtp_cuda_graph_runner.py | 4 +- .../speculative/frozen_kv_mtp_worker_v2.py | 4 +- ...er_eagle_draft_extend_cuda_graph_runner.py | 5 +- .../multi_layer_eagle_worker_v2.py | 4 +- .../srt/speculative/standalone_worker_v2.py | 4 +- .../srt/state_capturer/routed_experts.py | 4 +- python/sglang/srt/utils/common.py | 25 +- .../srt/utils/cuda_vmm_transport_utils.py | 7 +- python/sglang/srt/utils/offloader.py | 2 +- python/sglang/srt/weight_cache/ipc_loader.py | 8 +- .../test_disaggregation_wire.py | 10 +- .../model_loader/test_presharded_loader.py | 12 +- test/registered/unit/models/test_kimi_k25.py | 12 +- .../multimodal/test_tensor_transport_mode.py | 8 +- .../test_resolution_declarations.py | 17 +- .../test_resolution_is_reproducible.py | 2 +- .../test_resolution_reads_no_bag.py | 5 +- .../unit/test_global_config_read_ratchet.py | 333 +++++++++++++----- ...test_launch_path_reads_configured_sizes.py | 315 +++++++++-------- test/registered/unit/test_runtime_context.py | 17 +- .../unit/test_runtime_context_override.py | 2 +- .../unit/test_server_args_namespaces.py | 9 +- ...test_supplied_instance_exposure_ratchet.py | 18 +- 136 files changed, 1033 insertions(+), 795 deletions(-) diff --git a/.claude/skills/sglang-runtime-context/SKILL.md b/.claude/skills/sglang-runtime-context/SKILL.md index 4aa0719de..0eea90569 100644 --- a/.claude/skills/sglang-runtime-context/SKILL.md +++ b/.claude/skills/sglang-runtime-context/SKILL.md @@ -15,7 +15,7 @@ One container owns process-static runtime state: `sglang.srt.runtime_context.Run | runtime flags | `get_flags()` | state that is *not* a pure function of config: `capture` (cuda-graph lifecycle), `moe` (ACTIVE backends, swappable), `dp` (DP-attention runtime flags) | materialized at subsystem init; groups offer `override()` for tests | | resources | `get_resources()`, `get_stream(name)`, `get_buffer(name, factory)` | process-level handles: graph pools, EPLB state, EP dispatcher state, named side streams, workspace buffers | lazy; cleared by `reset_context()` | | per-forward | `get_forward()` | forward-scoped flags (multi-stream switch, MoE output buffer, attn-TP inputs, extend-in-batch) | contextvar-backed; `scoped(**kw)` restores on exit; new threads see defaults | -| parallel | `get_parallel()` | **dual**: live topology (tp/pp/moe/attn sizes, ranks, groups — `@property`, read-through) *plus* parallel config-bag leaves via `__getattr__` | live: after dist init; config leaves: after publish | +| parallel | `get_parallel()` | **dual, spelled**: bare names are the live topology (tp/pp/moe/attn sizes, ranks, groups — `@property`, read-through); `get_parallel().config.` is the parallel config bag | live: after dist init; `config`: after publish | `reset_context()` (unit-test teardown) drops the published config and installs fresh flags/resources/forward tiers. @@ -214,23 +214,29 @@ row for a method the Ray actor does not have, and an effective-field set without `load_format` -- and both were invisible because the assertion had slack (`>= len(...) - 1`) or compared key names instead of value sources. -### `get_parallel()`: config leaves vs live topology +### `get_parallel()`: live topology bare, configuration under `config` -Config leaves (`nccl_port`, `enable_dp_attention`, `dp_size`, `ep_size`, -`dwdp_size`, ...) resolve through the parallel bag; live topology (`tp_size`, -`attn_tp_group`, ranks) are `@property` and **win on name collisions**. Five topology -sizes are live-shadowed (`tp/pp/dcp/attn_cp/moe_dp_size`): the live property always -wins on the accessor, so a config-intent read of those goes through -`configured_tp_size()` / `configured_pp_size()` / `configured_moe_dp_size()` / -`configured_attn_cp_size()` (DCP: the live `get_parallel().attn_dcp_size` / -`.dcp_enabled`, which are safe with no group installed — they answer `1` / -`False` — but report the *effective* topology, never the requested size; -a config-intent read would need its own accessor, which no call site -requires today). A process-global seed field-read of one of these -sizes (`get_server_args().tp_size`, or an alias of it) is a read-ratchet failure; the -sites that legitimately go around the live property are the `configured_*_size()` -readers, and those are what the ratchet registers, each with its reason -(`_CONFIGURED_SIZE_CALL_SITES` in `test_global_config_read_ratchet.py`). A +**Bare is the live group, `config` is what was configured.** `get_parallel().tp_size` +and its size / rank / group siblings are `@property` read-through over the canonical +getters; `get_parallel().config.` reads the published `parallel` bag +(`nccl_port`, `enable_dp_attention`, `dp_size`, `ep_size`, `dwdp_size`, ... and the +five sizes that also have a live property). A bare read of a config-only leaf raises +an `AttributeError` naming the `.config` spelling — the tier is never guessed from +whether a property happens to exist. + +The two tiers are **not** two spellings of one number. Live diverges from configured +wherever elastic EP scales the world away from the launch shape, and wherever +`initialize_model_parallel` aliases `_MOE_DP` to `_ATTN_CP` (`attn_cp_size > +moe_dp_size`), which makes a live comparison of that pair degenerate. The five +live-shadowed sizes (`tp/pp/dcp/attn_cp/moe_dp_size`) are where the choice matters, +and every business read of `get_parallel().config.` is registered with +its reason in `_CONFIGURED_SIZE_CALL_SITES` (`test_global_config_read_ratchet.py`). +DCP has a third shape: the live `get_parallel().attn_dcp_size` / `.dcp_enabled` +answer the *effective* topology (`1` / `False` with no group installed), never the +requested size — `.config.dcp_size` is the requested one. + +A process-global seed field-read of one of these sizes +(`get_server_args().tp_size`, or an alias of it) is a read-ratchet failure. A `server_args` the object was *handed* is a different thing and not a ratchet matter — see "Reads that legitimately stay on a ServerArgs instance". Fail-loud is narrower: before dist init, a live size/group read raises — except @@ -238,25 +244,31 @@ the DCP pair, which degrades instead (`dcp_enabled` → `False`, `attn_dcp_size` → `1` when no group is installed; `test_attn_dcp_defaults_when_group_is_uninitialized` pins this). After init, only the DCP group is optional (`_DCP` exists only when `dcp_size > 1`; attn-CP and -moe-DP always install, as size-1 aliases if unused). `ParallelContext.__getattr__` is deliberately dynamo-traceable (no +moe-DP always install, as size-1 aliases if unused). The `config` hop is +deliberately dynamo-traceable (a plain property over a slot, no `object.__getattribute__`); gate helpers like `enable_moe_dense_fully_dp()` run inside compiled model forwards (`test_parallel_config_leaves_trace_under_torch_compile` pins this). +A third surface carries the same names: `ParallelState` (`self.ps` / `mr.ps`), the +frozen per-process snapshot built once in `Scheduler.__init__` from these configured +sizes plus this process's ranks, and handed down (draft runners included). Prefer it +where an object was handed one; it is not a global accessor. + ### Reading config: the seed is off limits `get_server_args().field` in business code is a ratchet failure. Read: - **a resolved leaf** → its namespace bag (`get_exec().moe.moe_runner_backend`, `get_schedule().chunked_prefill_size`, …). Bag-backed reads — a leaf directly, or - a bag-derived accessor below, including `configured_*_size()` which reads the - parallel bag's own leaf — are what see post-publish overrides. Only the + a bag-derived accessor below, including the `get_parallel().config` hop — are + what see post-publish overrides. Only the instance-derived accessors (the ones with no leaf to read) answer from the startup record and therefore do not. - **a leaf the caller names at runtime** (a readback reporting a list of fields) → `get_context().config_leaf(name)`; it resolves the name through `NS` and raises on a non-leaf. A call site that knows its field reads the bag leaf. -- **the live topology** → `get_parallel()`. +- **the live topology** → `get_parallel()` (bare names). - **a value derived from published leaves** → an accessor in `runtime_context` that derives it *from the bags*: `mamba_extra_buffer_enabled()` / `mamba_extra_buffer_lazy_enabled()` read `get_memory()` and `get_exec()`, so @@ -275,24 +287,18 @@ this). property with no bag of its own. A new derived member gets an accessor here rather than call sites reaching for the record, and only when the bag-derived shape above cannot express it. -- **what was *configured*, where `get_parallel()` shadows it with the live value** - → `configured_{tp,pp,moe_dp,attn_cp}_size()` — the full names are - `configured_tp_size`, `configured_pp_size`, `configured_moe_dp_size`, - `configured_attn_cp_size`. They read the parallel bag's own leaf (going - around the live property that shadows those four names), so they answer with - the resolved configuration and follow a post-publish override. DCP has no configured accessor because no - config-intent DCP call site exists today — the live reads go through - `get_parallel().attn_dcp_size` / `.dcp_enabled`, which answer the effective - topology (`1` / `False` when no group is installed). A site - that must know the *requested* DCP size before dist init needs its own - `configured_dcp_size()` (and an entry in `_CONFIGURED_SIZE_CALL_SITES`, which lives - in the ratchet test, not in this skill); note the live pair does not *need* - dist init — with no group it answers `1` / `False` — it just cannot answer - with the requested size. Every (file, accessor) pair is registered +- **what was *configured*, where the bare name is the live value** + → `get_parallel().config.{tp,pp,moe_dp,attn_cp,dcp}_size`. It reads the parallel + bag's own leaf, so it answers with the resolved configuration and follows a + post-publish override. The DCP live pair (`get_parallel().attn_dcp_size` / + `.dcp_enabled`) is a different question again: it answers the effective topology + (`1` / `False` when no group is installed), never the requested size, and it does + not *need* dist init to answer. Every (file, size) pair is registered with its reason in `test_global_config_read_ratchet.py` (`_CONFIGURED_SIZE_CALL_SITES`), and that test fails if the code and the list - disagree — a new file, or a new accessor in a listed file, has to be added — so a new site needs both an answer the live property cannot give and - an entry saying what it is. + disagree — a new file, or a new size in a listed file, has to be added — so a new + site needs both an answer the live property cannot give and an entry saying what + it is. - **this runner's resolved value** → the runner (`prefill_attention_backend_str`, `kv_cache_dtype_str`, `draft_attention_backend`, `num_fused_shared_experts` on the model). @@ -498,13 +504,19 @@ ONE thread — do not design for TBO threads that don't exist. — including local copies of an alias, `cfg = sa` — module-level, or parked on an instance attribute, plus the `getattr(..., "field")` spelling of each; a name computed at runtime or indirection deeper than a local name copy is census-tool - territory, per the test's docstring). The scanners match `get_server_args` and - `configured_*_size` by their literal names, and the same file *bans* - `import ... as` renames of them so that matching stays sound. Exempt by owner + territory, per the test's docstring). The scanner matches `get_server_args` by its + literal name, and the same file *bans* `import ... as` renames of it so that + matching stays sound. Exempt by owner module only (`runtime_context.py`, `server_args.py`, `arg_groups/`). The same file - carries `_CONFIGURED_SIZE_CALL_SITES`, the (file, accessor) map of every - `configured_*_size()` reader with the reason the live property cannot serve it — a new - file or a new accessor in a listed file must be added there. + carries `_CONFIGURED_SIZE_CALL_SITES`, the (file, size) map of every + `get_parallel().config.` reader with the reason the live property + cannot serve it — a new file or a new size in a listed file must be added there. Its + subject set is *derived* (property names ∩ `parallel` NS leaves), and it resolves + every spelling of the call itself — an aliased import, a module-qualified receiver + (including the whole dotted path an unaliased `import` binds), a local bound to either + hop — so neither a rename nor a new shadowed size escapes it. + `TestParallelConfigReadSpellings` in that file runs each spelling, because a spelling + the scanner cannot resolve drops the read instead of failing anything. 6. **Module-state ratchet** (`test_module_state_ratchet.py`): `global` statements in the flag-owning layers are pinned by name. A new module-level runtime global belongs on a flags group / resources slot instead; migrating a pinned survivor must shrink the pin. @@ -538,8 +550,9 @@ Never module-skip a test "until the migration settles" — seed the context inst form** (attribute-source ints get automatic-dynamic after the first size change). Bools (≤2 values) are tolerable in any form — see `ForwardFlags._GRAPH_VISIBLE`. Config-bag leaves are real instance attributes for - exactly this reason, and `ParallelContext.__getattr__` must stay free of - `object.__getattribute__` (dynamo graph-breaks on it). Before moving such state, + exactly this reason, and the parallel config tier is read through the plain + `ParallelContext.config` property for the same reason (`__getattr__` is + error-only, and `object.__getattribute__` graph-breaks). Before moving such state, prove its readers sit outside compile coverage; a piecewise-prefill boot of a small model is the fast check (recompile storms show as `torch._dynamo hit config.recompile_limit` during the compile pass). diff --git a/python/sglang/benchmark/one_batch.py b/python/sglang/benchmark/one_batch.py index b133297d0..38addce4b 100644 --- a/python/sglang/benchmark/one_batch.py +++ b/python/sglang/benchmark/one_batch.py @@ -542,7 +542,7 @@ def _maybe_prepare_mlp_sync_batch(batch: ScheduleBatch, model_runner): prepare_mlp_sync_batch_raw( batch, model_runner=model_runner, - dp_size=get_parallel().dp_size, + dp_size=get_parallel().config.dp_size, attn_tp_size=get_parallel().attn_tp_size, attn_cp_size=model_runner.ps.attn_cp_size, tp_group=model_runner.tp_group, diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index 7ca3d0dbd..55ed74897 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -769,7 +769,7 @@ class TboForwardBatchPreparer: # TODO improve, e.g. unify w/ `init_raw` if ( - get_parallel().moe_dense_tp_size == 1 + get_parallel().config.moe_dense_tp_size == 1 and batch.global_dp_buffer_len is not None ): sum_len = end_token_index - start_token_index diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index f59bc181e..036c317de 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -37,7 +37,6 @@ from sglang.srt.layers.dp_attention import ( get_attention_dp_size, ) from sglang.srt.runtime_context import ( - configured_pp_size, get_disagg, get_parallel, get_serving, @@ -171,7 +170,7 @@ class CommonKVManager(BaseKVManager): # for p/d multi node infer self.bootstrap_host = get_serving().host self.bootstrap_port = get_disagg().disaggregation_bootstrap_port - self.dist_init_addr = get_parallel().dist_init_addr + self.dist_init_addr = get_parallel().config.dist_init_addr parallel = get_parallel() self.attn_tp_size = parallel.attn_tp_size self.attn_tp_rank = parallel.attn_tp_rank @@ -182,16 +181,19 @@ class CommonKVManager(BaseKVManager): self.attn_dp_size = get_attention_dp_size() self.attn_dp_rank = get_attention_dp_rank() self.system_dp_size = ( - 1 if get_parallel().enable_dp_attention else get_parallel().dp_size + 1 + if get_parallel().config.enable_dp_attention + else get_parallel().config.dp_size ) self.system_dp_rank = ( self.kv_args.system_dp_rank if self.kv_args.system_dp_rank else 0 ) - self.pp_size = configured_pp_size() + self.pp_size = get_parallel().config.pp_size self.pp_rank = self.kv_args.pp_rank self.local_ip = get_local_ip_auto() cp_sharded_prefill = self.attn_cp_size > 1 and ( - self.is_hybrid_mla_backend or get_parallel().enable_dsa_cache_layer_split + self.is_hybrid_mla_backend + or get_parallel().config.enable_dsa_cache_layer_split ) hybrid_decode_pulls_all_ranks = ( @@ -304,7 +306,7 @@ class CommonKVManager(BaseKVManager): return ( self.attn_cp_size > 1 and self.attn_cp_rank != 0 - and not get_parallel().enable_dsa_cache_layer_split + and not get_parallel().config.enable_dsa_cache_layer_split ) def requires_dcp_relayout(self, dst_dcp_size: int, dst_dcp_rank: int) -> bool: @@ -749,7 +751,7 @@ class CommonKVManager(BaseKVManager): `Connection refused`, and the leader's `prefill_port_table` ends up missing rows. """ - if not self.dist_init_addr or get_parallel().nnodes == 1: + if not self.dist_init_addr or get_parallel().config.nnodes == 1: return local_port if not (dist.is_available() and dist.is_initialized()): @@ -801,8 +803,8 @@ class CommonKVManager(BaseKVManager): "rank_port": self.rank_port, "page_size": self.kv_args.page_size, "kv_cache_dtype": self.kv_cache_dtype_str, - "load_balance_method": get_parallel().load_balance_method, - "enable_dsa_cache_layer_split": get_parallel().enable_dsa_cache_layer_split, + "load_balance_method": get_parallel().config.load_balance_method, + "enable_dsa_cache_layer_split": get_parallel().config.enable_dsa_cache_layer_split, # Self-register the HTTP API port so the decode can derive the PD # retract rebootstrap /generate URL from bootstrap info instead of a # router-injected pd_rebootstrap_prefill_url. @@ -1184,11 +1186,12 @@ class CommonKVSender(BaseKVSender): return self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Bootstrapping) - if get_parallel().dp_size > 1 and not req_has_disagg_prefill_dp_rank: - if get_parallel().load_balance_method != "follow_bootstrap_room": + if get_parallel().config.dp_size > 1 and not req_has_disagg_prefill_dp_rank: + if get_parallel().config.load_balance_method != "follow_bootstrap_room": self._register_prefill_dp_rank() elif ( - self.kv_mgr.attn_dp_rank != self.bootstrap_room % get_parallel().dp_size + self.kv_mgr.attn_dp_rank + != self.bootstrap_room % get_parallel().config.dp_size ): # follow_bootstrap_room was overridden by external routed_dp_rank if envs.SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK.get(): @@ -1199,7 +1202,7 @@ class CommonKVSender(BaseKVSender): f"follow_bootstrap_room conflict: dispatched to dp_rank " f"{self.kv_mgr.attn_dp_rank} but bootstrap_room " f"{self.bootstrap_room} implies dp_rank " - f"{self.bootstrap_room % get_parallel().dp_size}. " + f"{self.bootstrap_room % get_parallel().config.dp_size}. " f"Set SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK=1 " f"to allow mixed routing.", ) @@ -1273,7 +1276,7 @@ class CommonKVSender(BaseKVSender): if ( self.kv_mgr.enable_all_cp_ranks_for_transfer - and not get_parallel().enable_dsa_cache_layer_split + and not get_parallel().config.enable_dsa_cache_layer_split ): kv_indices, index_slice = filter_kv_indices_for_cp_rank( self.kv_mgr, diff --git a/python/sglang/srt/disaggregation/encoder/http_server.py b/python/sglang/srt/disaggregation/encoder/http_server.py index 1e952c2a3..e204048c6 100644 --- a/python/sglang/srt/disaggregation/encoder/http_server.py +++ b/python/sglang/srt/disaggregation/encoder/http_server.py @@ -207,7 +207,7 @@ def launch_server(server_args: ServerArgs): configure_logger(server_args, prefix=" encode_server") publish(server_args, role="encoder") - if get_parallel().dp_size > 1: + if get_parallel().config.dp_size > 1: dp_dispatcher = launch_dp_runtime(server_args) # runtime initializes multiprocess metrics before spawning; # HTTP only exposes their endpoint. diff --git a/python/sglang/srt/disaggregation/encoder/runtime.py b/python/sglang/srt/disaggregation/encoder/runtime.py index ba47db066..330a8b933 100644 --- a/python/sglang/srt/disaggregation/encoder/runtime.py +++ b/python/sglang/srt/disaggregation/encoder/runtime.py @@ -50,7 +50,6 @@ from sglang.srt.observability.trace import ( trace_set_thread_info, ) from sglang.srt.runtime_context import ( - configured_tp_size, get_observability, get_parallel, get_serving, @@ -1491,10 +1490,10 @@ def launch_local_runtime(server_args: ServerArgs) -> EncoderRuntime: This function owns backend construction only. HTTP/gRPC middleware, service registration, and network serving remain Transport concerns. """ - if get_parallel().dp_size > 1: + if get_parallel().config.dp_size > 1: raise ValueError( "launch_local_runtime requires --dp-size 1; got " - f"dp_size={get_parallel().dp_size}." + f"dp_size={get_parallel().config.dp_size}." ) # Set up prometheus metrics. @@ -1512,8 +1511,10 @@ def launch_local_runtime(server_args: ServerArgs) -> EncoderRuntime: zmq_context = zmq.Context(10) ipc_path_prefix = random_uuid() port_args = PortArgs.init_new(server_args) - if get_parallel().dist_init_addr: - dist_init_method = NetworkAddress.parse(get_parallel().dist_init_addr).to_tcp() + if get_parallel().config.dist_init_addr: + dist_init_method = NetworkAddress.parse( + get_parallel().config.dist_init_addr + ).to_tcp() else: dist_init_method = NetworkAddress( get_serving().host or "127.0.0.1", port_args.nccl_port @@ -1529,7 +1530,7 @@ def launch_local_runtime(server_args: ServerArgs) -> EncoderRuntime: send_sockets: List[zmq.Socket] = [] tp_processes: List[mp.Process] = [] - for rank in range(1, configured_tp_size()): + for rank in range(1, get_parallel().config.tp_size): schedule_path = f"ipc:///tmp/{ipc_path_prefix}_schedule_{rank}" send_sockets.append( get_zmq_socket(zmq_context, zmq.PUSH, schedule_path, bind=False) @@ -1569,12 +1570,12 @@ def launch_dp_runtime(server_args: ServerArgs) -> DPDispatcher: HTTP uses this entry point today. gRPC can reuse it later without importing HTTP application state or Uvicorn. """ - if get_parallel().dp_size <= 1 or server_args.tp_size != 1: + if get_parallel().config.dp_size <= 1 or server_args.tp_size != 1: raise ValueError( "Encoder DP mode requires --dp-size > 1 and --tp-size 1; got " - f"dp_size={get_parallel().dp_size}, tp_size={server_args.tp_size}." + f"dp_size={get_parallel().config.dp_size}, tp_size={server_args.tp_size}." ) - dp_size = get_parallel().dp_size + dp_size = get_parallel().config.dp_size logger.info(f"Launching encoder in DP mode: dp_size={dp_size}") # DP mode: workers (subprocesses) write metrics to the shared multiproc dir; diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 20f0e8bbe..a4e18be31 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -180,7 +180,7 @@ class PrefillBootstrapQueue: "SGLANG_DISAGG_STAGING_BUFFER with pp_size > 1 is only " "supported by Mooncake." ) - if get_parallel().enable_prefill_context_parallel: + if get_parallel().config.enable_prefill_context_parallel: # CP rewrites index_slice per rank, breaking the chunk grid. raise RuntimeError( "SGLANG_DISAGG_STAGING_BUFFER does not support " diff --git a/python/sglang/srt/distributed/bootstrap.py b/python/sglang/srt/distributed/bootstrap.py index fc6cd4adc..623b9aa63 100644 --- a/python/sglang/srt/distributed/bootstrap.py +++ b/python/sglang/srt/distributed/bootstrap.py @@ -128,7 +128,7 @@ def init_torch_distributed( # included in later KV-cache sizing instead of appearing during capture. if ( device == "cuda" - and get_parallel().enable_tp_lm_head_all_to_all + and get_parallel().config.enable_tp_lm_head_all_to_all and ps.tp_size > 1 ): _prewarm_tp_lm_head_all_to_all() @@ -266,7 +266,7 @@ def _init_parallel_groups( duplicate_attn_cp_group=( is_hip() and server_args.enable_two_batch_overlap - and get_parallel().enable_dsa_prefill_context_parallel + and get_parallel().config.enable_dsa_prefill_context_parallel ), enable_symm_mem=get_exec().comm.enable_symm_mem, recovered_rank=is_ep_joiner, diff --git a/python/sglang/srt/distributed/device_communicators/triton_symm_mem_ag.py b/python/sglang/srt/distributed/device_communicators/triton_symm_mem_ag.py index 8d80f2ab8..74e82d73a 100644 --- a/python/sglang/srt/distributed/device_communicators/triton_symm_mem_ag.py +++ b/python/sglang/srt/distributed/device_communicators/triton_symm_mem_ag.py @@ -477,7 +477,7 @@ class MultimemAllGatherer: # EP/mooncake setups, and keep multimem enabled. if ( tp_group.world_size > 1 - and get_parallel().nnodes > 1 + and get_parallel().config.nnodes > 1 and not all(in_the_same_node_as(tp_group.cpu_group, source_rank=0)) ): logger.warning( diff --git a/python/sglang/srt/elastic_ep/elastic_ep.py b/python/sglang/srt/elastic_ep/elastic_ep.py index 26bba8641..1dc43078a 100644 --- a/python/sglang/srt/elastic_ep/elastic_ep.py +++ b/python/sglang/srt/elastic_ep/elastic_ep.py @@ -12,7 +12,6 @@ from sglang.srt.distributed.utils import get_global_tcp_store from sglang.srt.eplb.expert_location import broadcast_global_expert_location_metadata from sglang.srt.managers.schedule_batch import ServerArgs from sglang.srt.runtime_context import ( - configured_tp_size, get_exec, get_parallel, ) @@ -93,7 +92,7 @@ class ElasticEPStateManager: if get_exec().moe.elastic_ep_backend is not None: world_size = torch.distributed.get_world_size() - active_rank_capacity = get_parallel().max_ep_size or world_size + active_rank_capacity = get_parallel().config.max_ep_size or world_size assert active_rank_capacity >= world_size, ( f"--max-ep-size ({active_rank_capacity}) must be >= " f"world_size ({world_size})." @@ -110,7 +109,7 @@ class ElasticEPStateManager: if get_exec().moe.moe_a2a_backend == "nixl": cls._on_scale = cls._on_scale_nixl - inst.ep_join_rank_offset = get_parallel().ep_join_rank_offset + inst.ep_join_rank_offset = get_parallel().config.ep_join_rank_offset if server_args.is_ep_joiner: cls._init_joiner_state(inst, server_args) @@ -128,11 +127,12 @@ class ElasticEPStateManager: if get_exec().moe.ep_join_mode == "scale": inst.effective_ep_size = ( - get_parallel().ep_join_rank_offset + configured_tp_size() + get_parallel().config.ep_join_rank_offset + + get_parallel().config.tp_size ) inst.original_ep_size = ( - get_parallel().elastic_ep_initial_size - or get_parallel().ep_join_rank_offset + get_parallel().config.elastic_ep_initial_size + or get_parallel().config.ep_join_rank_offset ) inst.has_scaled = True else: @@ -317,7 +317,7 @@ def elastic_expanded_world_enabled() -> bool: inst = ElasticEPStateManager.instance() if inst is None: return False - if get_parallel().max_ep_size is None: + if get_parallel().config.max_ep_size is None: return False active_target_size = inst.effective_ep_size if inst.pending_ep_size is not None and inst.scale_phase in ( diff --git a/python/sglang/srt/elastic_ep/expert_backup_manager.py b/python/sglang/srt/elastic_ep/expert_backup_manager.py index 7fdd2423a..0a7083c91 100644 --- a/python/sglang/srt/elastic_ep/expert_backup_manager.py +++ b/python/sglang/srt/elastic_ep/expert_backup_manager.py @@ -18,7 +18,6 @@ from sglang.srt.managers.io_struct import ( from sglang.srt.model_loader.loader import DefaultModelLoader, get_model_loader from sglang.srt.model_loader.utils import set_default_torch_dtype from sglang.srt.runtime_context import ( - configured_tp_size, get_disagg, get_exec, get_model, @@ -51,8 +50,8 @@ class ExpertBackupManager: self.weight_pointer_map = {} self.transfer_engine = None self.session_id = None - self.engine_num = get_parallel().nnodes - self.engine_rank = get_parallel().node_rank + self.engine_num = get_parallel().config.nnodes + self.engine_rank = get_parallel().config.node_rank self.expert_num = self.model_config.hf_config.n_routed_experts self.idmn = (self.expert_num // self.engine_num) * self.engine_rank self.idmx = (self.expert_num // self.engine_num) * (self.engine_rank + 1) @@ -60,11 +59,11 @@ class ExpertBackupManager: # Synchronization socket to avoid PUB/SUB slow joiner issues. self.recv_from_expert_backup_client = context.socket(zmq.PULL) self.recv_from_expert_backup_client.bind( - f"tcp://{get_local_ip_auto()}:{PORT_BASE + get_parallel().node_rank * 2}" + f"tcp://{get_local_ip_auto()}:{PORT_BASE + get_parallel().config.node_rank * 2}" ) self.send_to_expert_backup_client = context.socket(zmq.PUB) self.send_to_expert_backup_client.bind( - f"tcp://{get_local_ip_auto()}:{PORT_BASE + get_parallel().node_rank * 2 + 1}" + f"tcp://{get_local_ip_auto()}:{PORT_BASE + get_parallel().config.node_rank * 2 + 1}" ) self.backup_weights_from_disk() self.start_transfer_server() @@ -73,7 +72,7 @@ class ExpertBackupManager: # losing the initial PUB message due to slow joiners. num_ready_clients = 0 - while num_ready_clients < configured_tp_size(): + while num_ready_clients < get_parallel().config.tp_size: sock_recv(self.recv_from_expert_backup_client) num_ready_clients += 1 diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index 46cfa4dfe..118b05f85 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -98,9 +98,6 @@ from sglang.srt.parser.template_detection import resolve_auto_parsers from sglang.srt.parser.template_manager import TemplateManager from sglang.srt.plugins import load_plugins from sglang.srt.runtime_context import ( - configured_attn_cp_size, - configured_moe_dp_size, - configured_pp_size, get_exec, get_model, get_parallel, @@ -345,7 +342,7 @@ class Engine(EngineScoreMixin, EngineBase): routed_dp_rank = data_parallel_rank if routed_dp_rank is not None: - dp_size = get_parallel().dp_size + dp_size = get_parallel().config.dp_size if dp_size <= 1 and routed_dp_rank == 0: logger.debug( f"routed_dp_rank={routed_dp_rank} is ignored because dp_size={dp_size}" @@ -684,7 +681,7 @@ class Engine(EngineScoreMixin, EngineBase): pp_rank_range, tp_rank_range, pp_size_per_node, tp_size_per_node = ( _calculate_rank_ranges( server_args.nnodes, - configured_pp_size(), + get_parallel().config.pp_size, tp_size, server_args.node_rank, ) @@ -832,7 +829,7 @@ class Engine(EngineScoreMixin, EngineBase): """ scheduler_procs = [] use_dp_controller = ( - get_parallel().dp_size > 1 or get_exec().moe.ep_join_mode == "scale" + get_parallel().config.dp_size > 1 or get_exec().moe.ep_join_mode == "scale" ) if not use_dp_controller: @@ -845,7 +842,7 @@ class Engine(EngineScoreMixin, EngineBase): pp_rank_range, tp_rank_range, pp_size_per_node, tp_size_per_node = ( _calculate_rank_ranges( server_args.nnodes, - configured_pp_size(), + get_parallel().config.pp_size, server_args.tp_size, server_args.node_rank, ) @@ -1845,10 +1842,14 @@ def _compute_parallelism_ranks( Called while the launcher is deciding what to spawn, so the sizes are the configured ones -- the groups this is laying out do not exist yet. """ - attn_dp_size = get_parallel().dp_size if get_parallel().enable_dp_attention else 1 + attn_dp_size = ( + get_parallel().config.dp_size + if get_parallel().config.enable_dp_attention + else 1 + ) tp_size = server_args.tp_size - attn_cp_size = configured_attn_cp_size() - moe_dp_size = configured_moe_dp_size() + attn_cp_size = get_parallel().config.attn_cp_size + moe_dp_size = get_parallel().config.moe_dp_size # Parallelism hierarchy (outermost to innermost): # - Attention: Global(TP) -> DP -> ATTN_CP -> ATTN_TP (innermost) @@ -1859,6 +1860,6 @@ def _compute_parallelism_ranks( moe_ep_rank = ( tp_rank % (tp_size // moe_dp_size) - // (tp_size // moe_dp_size // get_parallel().ep_size) + // (tp_size // moe_dp_size // get_parallel().config.ep_size) ) return attn_cp_rank, moe_dp_rank, moe_ep_rank diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index cb363cca7..df6d09fc3 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -2177,7 +2177,7 @@ async def _send_disaggregation_warmup_requests( return await asyncio.gather( *( send_request(session, dp_rank) - for dp_rank in range(get_parallel().dp_size) + for dp_rank in range(get_parallel().config.dp_size) ) ) @@ -2236,9 +2236,11 @@ def _execute_server_warmup(server_args: ServerArgs): }, } if server_args.skip_tokenizer_init: - json_data["input_ids"] = [[10, 11, 12] for _ in range(get_parallel().dp_size)] + json_data["input_ids"] = [ + [10, 11, 12] for _ in range(get_parallel().config.dp_size) + ] # TODO Workaround the bug that embedding errors for list of size 1 - if get_parallel().dp_size == 1: + if get_parallel().config.dp_size == 1: json_data["input_ids"] = json_data["input_ids"][0] elif ( is_vlm @@ -2282,9 +2284,11 @@ def _execute_server_warmup(server_args: ServerArgs): "temperature": 0.0, } else: - json_data["text"] = ["The capital city of France is"] * get_parallel().dp_size + json_data["text"] = [ + "The capital city of France is" + ] * get_parallel().config.dp_size # TODO Workaround the bug that embedding errors for list of size 1 - if get_parallel().dp_size == 1: + if get_parallel().config.dp_size == 1: json_data["text"] = json_data["text"][0] # Config debug dumping @@ -2326,7 +2330,7 @@ def _execute_server_warmup(server_args: ServerArgs): if not failed_status_codes: logger.info( "Disaggregation warmup requests completed for all %s DP ranks", - get_parallel().dp_size, + get_parallel().config.dp_size, ) logger.info("End of disaggregation warmup") else: diff --git a/python/sglang/srt/entrypoints/v1_loads.py b/python/sglang/srt/entrypoints/v1_loads.py index 7d87cbb5f..93f77bd3c 100644 --- a/python/sglang/srt/entrypoints/v1_loads.py +++ b/python/sglang/srt/entrypoints/v1_loads.py @@ -27,7 +27,6 @@ from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import Response from sglang.srt.runtime_context import ( - configured_pp_size, get_parallel, ) from sglang.srt.utils import get_device_name @@ -148,9 +147,9 @@ async def get_loads( "accelerator": _accelerator_name(), "num_accelerators": _num_accelerators_per_dp_rank( tokenizer_manager.server_args.tp_size, - configured_pp_size(), - get_parallel().dp_size, - get_parallel().enable_dp_attention, + get_parallel().config.pp_size, + get_parallel().config.dp_size, + get_parallel().config.enable_dp_attention, ), "loads": loads, } diff --git a/python/sglang/srt/eplb/expert_location.py b/python/sglang/srt/eplb/expert_location.py index afeea11d5..19624be81 100644 --- a/python/sglang/srt/eplb/expert_location.py +++ b/python/sglang/srt/eplb/expert_location.py @@ -26,7 +26,6 @@ import torch.distributed import torch.nn.functional as F from sglang.srt.runtime_context import ( - configured_tp_size, get_device, get_exec, ) @@ -202,7 +201,7 @@ class ExpertLocationMetadata: model_config_for_expert_location = common["model_config_for_expert_location"] num_physical_experts = common["num_physical_experts"] num_groups = model_config_for_expert_location.num_groups - num_nodes = 1 if use_flat_topology else get_parallel().nnodes + num_nodes = 1 if use_flat_topology else get_parallel().config.nnodes from sglang.srt.eplb import eplb_algorithms @@ -245,14 +244,15 @@ class ExpertLocationMetadata: + get_exec().moe.ep_num_redundant_experts ) # elastic-EP scale-up rewrites ep_size on the published config - ep_size = get_parallel().ep_size + ep_size = get_parallel().config.ep_size num_physical_experts = base_num_physical_experts - initial_ep_size = get_parallel().elastic_ep_initial_size + initial_ep_size = get_parallel().config.elastic_ep_initial_size if initial_ep_size is not None: if get_exec().moe.ep_join_mode == "scale": ep_size = max( ep_size, - get_parallel().ep_join_rank_offset + configured_tp_size(), + get_parallel().config.ep_join_rank_offset + + get_parallel().config.tp_size, ) num_physical_experts, num_local_physical_experts = ( _compute_elastic_expert_layout( @@ -580,7 +580,7 @@ def _compute_logical_to_all_physical_map( num_local_gpu_physical_experts = num_physical_experts // ep_size prefer_same_node = _prefer_same_node_experts() num_gpus_per_node = ( - get_parallel().ep_size // get_parallel().nnodes + get_parallel().config.ep_size // get_parallel().config.nnodes if prefer_same_node else None ) @@ -644,7 +644,9 @@ def compute_logical_to_rank_dispatch_physical_map( num_local_gpu_physical_experts = num_physical_experts // ep_size prefer_same_node = _prefer_same_node_experts() num_gpus_per_node = ( - get_parallel().ep_size // get_parallel().nnodes if prefer_same_node else None + get_parallel().config.ep_size // get_parallel().config.nnodes + if prefer_same_node + else None ) num_local_node_physical_experts = ( num_local_gpu_physical_experts * num_gpus_per_node diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py index 42a0c2208..2dd615d00 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py @@ -40,7 +40,6 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo is_in_tc_piecewise_cuda_graph, ) from sglang.srt.runtime_context import ( - configured_pp_size, get_device, get_exec, get_parallel, @@ -249,7 +248,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp): if _is_cuda: self.sm_count = deep_gemm.get_num_sms() self.half_device_sm_count = ceil_align(self.sm_count // 2, 8) - pp_size = configured_pp_size() + pp_size = get_parallel().config.pp_size self.logits_with_pp_recv = pp_size > 1 and not get_pp_group().is_last_rank else: self.logits_with_pp_recv = False diff --git a/python/sglang/srt/layers/attention/dsa/utils.py b/python/sglang/srt/layers/attention/dsa/utils.py index 1b1b34df6..cce2e5252 100644 --- a/python/sglang/srt/layers/attention/dsa/utils.py +++ b/python/sglang/srt/layers/attention/dsa/utils.py @@ -106,7 +106,7 @@ def should_use_dsa_fused_topk(seed_dsa_topk_from_draft_extend: bool) -> bool: def is_dsa_enable_prefill_cp(): if not envs.SGLANG_ENABLE_CP_V2.get(): - return get_parallel().enable_dsa_prefill_context_parallel + return get_parallel().config.enable_dsa_prefill_context_parallel # Derive from the runtime CP topology + model arch rather than the legacy # flag under CP-v2: DSA prefill CP is active when the CP group is on for a @@ -122,14 +122,14 @@ def is_dsa_enable_prefill_cp(): def is_dsa_prefill_cp_in_seq_split(): return ( is_dsa_enable_prefill_cp() - and get_parallel().dsa_prefill_cp_mode == "in-seq-split" + and get_parallel().config.dsa_prefill_cp_mode == "in-seq-split" ) def is_dsa_prefill_cp_round_robin_split(): return ( is_dsa_enable_prefill_cp() - and get_parallel().dsa_prefill_cp_mode == "round-robin-split" + and get_parallel().config.dsa_prefill_cp_mode == "round-robin-split" ) diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index 2f7d0d5d6..48bd00522 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -57,7 +57,10 @@ from sglang.kernels.ops.attention.flash_attention import ( def _should_disable_scheduler_metadata_precompute() -> bool: - return bool(get_parallel().enable_prefill_cp or get_parallel().enable_dp_attention) + return bool( + get_parallel().config.enable_prefill_cp + or get_parallel().config.enable_dp_attention + ) @dataclass diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index e92806fd4..b69b1a08a 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -269,7 +269,7 @@ class AttnTpContext: def init_context(self, q_lora_rank, is_dsa): self.is_dsa = is_dsa self.allow_input_scattered = ( - get_parallel().enable_attn_tp_input_scattered + get_parallel().config.enable_attn_tp_input_scattered and (_is_cuda or _is_npu) and q_lora_rank is not None and not is_dsa @@ -280,7 +280,7 @@ class AttnTpContext: and not check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE) and get_spec().speculative_algorithm != "EAGLE3" ) - if get_parallel().enable_attn_tp_input_scattered: + if get_parallel().config.enable_attn_tp_input_scattered: if not self.allow_input_scattered: logging.info( "attn_tp_input_scattered is not enabled while other conditions are not met" @@ -438,11 +438,11 @@ class LayerScatterModes: def enable_moe_dense_fully_dp(): - return get_parallel().moe_dense_tp_size == 1 + return get_parallel().config.moe_dense_tp_size == 1 def enable_dwdp(): - return get_parallel().dwdp_size > 1 + return get_parallel().config.dwdp_size > 1 class LayerCommunicator: diff --git a/python/sglang/srt/layers/cp/base.py b/python/sglang/srt/layers/cp/base.py index 65d01693b..4191089ed 100644 --- a/python/sglang/srt/layers/cp/base.py +++ b/python/sglang/srt/layers/cp/base.py @@ -283,7 +283,7 @@ def get_cp_strategy() -> Optional[ContextParallelStrategy]: server_args = get_server_args() except ValueError: return None - if server_args is not None and get_parallel().enable_prefill_cp: + if server_args is not None and get_parallel().config.enable_prefill_cp: init_cp_strategy(server_args) return _STRATEGY diff --git a/python/sglang/srt/layers/cp/cp_decode_attn_tp.py b/python/sglang/srt/layers/cp/cp_decode_attn_tp.py index 0abfcf764..7ad40c137 100644 --- a/python/sglang/srt/layers/cp/cp_decode_attn_tp.py +++ b/python/sglang/srt/layers/cp/cp_decode_attn_tp.py @@ -51,7 +51,7 @@ class CpDecodeAttnTpContext: """Slices replicated attention weights across CP ranks during decode.""" def __init__(self): - enable_attn_tp = get_parallel().enable_cp_decode_attn_tp + enable_attn_tp = get_parallel().config.enable_cp_decode_attn_tp if enable_attn_tp and get_parallel().attn_cp_size > 1: self.decode_tp_rank = get_parallel().attn_cp_rank diff --git a/python/sglang/srt/layers/dp_attention.py b/python/sglang/srt/layers/dp_attention.py index ea7810c0f..bbbe36508 100644 --- a/python/sglang/srt/layers/dp_attention.py +++ b/python/sglang/srt/layers/dp_attention.py @@ -29,8 +29,6 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) from sglang.srt.runtime_context import ( - configured_attn_cp_size, - configured_moe_dp_size, get_device, get_exec, get_flags, @@ -349,9 +347,9 @@ def initialize_dp_attention( dp.max_len_with_idle = ( getattr(model_config.hf_config, "hybrid_override_pattern", None) is not None ) - enable_dp_attention = get_parallel().enable_dp_attention - dp_size = get_parallel().dp_size - attn_cp_size = configured_attn_cp_size() + enable_dp_attention = get_parallel().config.enable_dp_attention + dp_size = get_parallel().config.dp_size + attn_cp_size = get_parallel().config.attn_cp_size dp.enabled = enable_dp_attention @@ -363,8 +361,11 @@ def initialize_dp_attention( ) _ATTN_DP_SIZE = dp_size if enable_dp_attention else 1 - if get_exec().moe.elastic_ep_backend is not None and get_parallel().max_ep_size: - _ATTN_DP_RANK = tp_rank + get_parallel().ep_join_rank_offset + if ( + get_exec().moe.elastic_ep_backend is not None + and get_parallel().config.max_ep_size + ): + _ATTN_DP_RANK = tp_rank + get_parallel().config.ep_join_rank_offset if server_args.is_ep_scale_joiner: dp.joiner_skip_all_gather = True @@ -1033,7 +1034,7 @@ def is_enable_moe_cp_allgather() -> bool: (``parallel_state.py``), so the live sizes are equal and the comparison would always be false. """ - return configured_attn_cp_size() > configured_moe_dp_size() + return get_parallel().config.attn_cp_size > get_parallel().config.moe_dp_size def moe_cp_all_gather_into_tensor(output: torch.Tensor, input: torch.Tensor): diff --git a/python/sglang/srt/layers/logits_processor.py b/python/sglang/srt/layers/logits_processor.py index 7a678e944..70ca6de11 100644 --- a/python/sglang/srt/layers/logits_processor.py +++ b/python/sglang/srt/layers/logits_processor.py @@ -296,8 +296,10 @@ class LogitsProcessor(nn.Module): self.config = config self.vocab_size = config.vocab_size self.logit_scale = logit_scale - self.use_attn_tp_group = get_parallel().enable_dp_lm_head - self.use_tp_lm_head_all_to_all = get_parallel().enable_tp_lm_head_all_to_all + self.use_attn_tp_group = get_parallel().config.enable_dp_lm_head + self.use_tp_lm_head_all_to_all = ( + get_parallel().config.enable_tp_lm_head_all_to_all + ) self.use_fp32_lm_head = get_exec().features.enable_fp32_lm_head if self.use_attn_tp_group: self.attn_tp_size = get_parallel().attn_tp_size diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index a8feead4a..bd799ef22 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -293,10 +293,10 @@ class FusedMoE(torch.nn.Module): self._num_global_routed = num_experts - num_shared_slots if get_exec().moe.ep_join_mode == "scale": - storage_ep_size = get_parallel().elastic_ep_initial_size + storage_ep_size = get_parallel().config.elastic_ep_initial_size assert storage_ep_size is not None self._expert_storage_rank = ( - get_parallel().ep_join_rank_offset + self.moe_ep_rank + get_parallel().config.ep_join_rank_offset + self.moe_ep_rank ) else: storage_ep_size = self.moe_ep_size diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py index 82b635db6..bf8eabd4a 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py @@ -337,7 +337,7 @@ def ensure_cutedsl_wrapper(layer: torch.nn.Module) -> None: else: # Standard allgather path: the MoE sees up to dp_size local forwards # gathered together, so scale the per-rank forward bound by dp_size. - max_num_tokens = get_parallel().dp_size * cutedsl_moe_max_num_tokens() + max_num_tokens = get_parallel().config.dp_size * cutedsl_moe_max_num_tokens() top_k = layer.top_k if layer.top_k is not None else layer.moe_runner_config.top_k # inference_mode(False) ensures the wrapper's pre-allocated CUDA-graph # buffers are normal tensors. This call typically happens inside diff --git a/python/sglang/srt/layers/moe/token_dispatcher/nixl.py b/python/sglang/srt/layers/moe/token_dispatcher/nixl.py index 2f2e3e849..e40ed1ad5 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/nixl.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/nixl.py @@ -135,7 +135,7 @@ class NixlEPBuffer: offset = ElasticEPStateManager.get_ep_join_rank_offset() global_rank = rank + offset - max_ep_size = get_parallel().max_ep_size or world_size + max_ep_size = get_parallel().config.max_ep_size or world_size nixl_max_ranks = max_ep_size num_rdma_bytes = 0 @@ -233,7 +233,7 @@ class _NixlEPDispatcherImplBase: ) self._active_world_size = dist.get_world_size(group) - _max_ep = get_parallel().max_ep_size or self._active_world_size + _max_ep = get_parallel().config.max_ep_size or self._active_world_size self._mask_buffer = ( torch.zeros(_max_ep, dtype=torch.int32, device="cuda") if self.active_ranks is not None diff --git a/python/sglang/srt/layers/moe/token_dispatcher/pplx.py b/python/sglang/srt/layers/moe/token_dispatcher/pplx.py index 6337bb944..05c7ceffb 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/pplx.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/pplx.py @@ -155,7 +155,7 @@ class PplxAllToAllManager: # pplx forces ep_size == world_size # with pp_size == 1 (enforced in _ensure_nvshmem), so the EP group spans # a single node iff the whole job runs on one node. - is_internode = get_parallel().nnodes > 1 + is_internode = get_parallel().config.nnodes > 1 if is_internode: cls._all_to_all = AllToAll.internode( diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py index 593c65372..7f4e51024 100644 --- a/python/sglang/srt/layers/moe/utils.py +++ b/python/sglang/srt/layers/moe/utils.py @@ -607,7 +607,7 @@ def should_skip_post_experts_all_reduce(*, is_tp_path: bool) -> bool: """ if should_skip_mlp_all_reduce(): return True - if get_parallel().dwdp_size > 1: + if get_parallel().config.dwdp_size > 1: return True if should_use_dp_reduce_scatterv(): return True diff --git a/python/sglang/srt/layers/utils/cp_utils.py b/python/sglang/srt/layers/utils/cp_utils.py index 2a692eeb4..03500c7ad 100644 --- a/python/sglang/srt/layers/utils/cp_utils.py +++ b/python/sglang/srt/layers/utils/cp_utils.py @@ -63,18 +63,18 @@ class ContextParallelMetadata: def is_prefill_context_parallel_enabled(): - return get_parallel().enable_prefill_context_parallel + return get_parallel().config.enable_prefill_context_parallel def is_prefill_cp_in_seq_split(): return ( is_prefill_context_parallel_enabled() - and get_parallel().prefill_cp_mode == "in-seq-split" + and get_parallel().config.prefill_cp_mode == "in-seq-split" ) def is_mla_prefill_cp_enabled() -> bool: - return get_parallel().enable_prefill_context_parallel and uses_mla_backend() + return get_parallel().config.enable_prefill_context_parallel and uses_mla_backend() def mla_use_prefill_cp(forward_batch, mla_enable_prefill_cp=None): diff --git a/python/sglang/srt/lora/lora_manager.py b/python/sglang/srt/lora/lora_manager.py index d19f3a6c7..ba07718a0 100644 --- a/python/sglang/srt/lora/lora_manager.py +++ b/python/sglang/srt/lora/lora_manager.py @@ -99,7 +99,7 @@ class LoRAManager: self.pending_lora_load_events = {} self.eviction_policy = server_args.lora_eviction_policy - self.enable_dp_attention: bool = get_parallel().enable_dp_attention + self.enable_dp_attention: bool = get_parallel().config.enable_dp_attention self._experts_shared_outer_override: Optional[bool] = ( server_args.experts_shared_outer_loras ) diff --git a/python/sglang/srt/managers/data_parallel_controller.py b/python/sglang/srt/managers/data_parallel_controller.py index 0b3c4c904..d59aab3a0 100644 --- a/python/sglang/srt/managers/data_parallel_controller.py +++ b/python/sglang/srt/managers/data_parallel_controller.py @@ -50,9 +50,6 @@ from sglang.srt.observability.req_time_stats import DPControllerReqTimeStats from sglang.srt.observability.startup_time import aggregate_scheduler_startup_times from sglang.srt.observability.trace import process_tracing_init, trace_set_thread_info from sglang.srt.runtime_context import ( - configured_attn_cp_size, - configured_moe_dp_size, - configured_pp_size, get_device, get_disagg, get_exec, @@ -151,12 +148,12 @@ class DataParallelController: self.server_args = server_args self.port_args = port_args self.load_balance_method = LoadBalanceMethod.from_str( - get_parallel().load_balance_method + get_parallel().config.load_balance_method ) self.run_scheduler_process_func = run_scheduler_process_func # Init inter-process communication - self.context = zmq.Context(1 + get_parallel().dp_size) + self.context = zmq.Context(1 + get_parallel().config.dp_size) if server_args.node_rank == 0: self.recv_from_tokenizer = get_zmq_socket( self.context, zmq.PULL, port_args.scheduler_input_ipc_name, False @@ -176,8 +173,8 @@ class DataParallelController: LoadBalanceMethod.TOTAL_TOKENS, ) - self.launch_dp_size: int = get_parallel().dp_size - self.max_dp_size: int = server_args.max_ep_size or get_parallel().dp_size + self.launch_dp_size: int = get_parallel().config.dp_size + self.max_dp_size: int = server_args.max_ep_size or get_parallel().config.dp_size assert self.max_dp_size >= self.launch_dp_size, ( f"--max-ep-size ({self.max_dp_size}) must be >= " f"--dp ({self.launch_dp_size})." @@ -187,7 +184,7 @@ class DataParallelController: self.max_dp_size - self.launch_dp_size ) - self.dp_budget = DPBudget(get_parallel().dp_size) + self.dp_budget = DPBudget(get_parallel().config.dp_size) self.load_snapshot_reader = create_load_snapshot_reader( port_args, caller="DataParallelController", @@ -204,14 +201,16 @@ class DataParallelController: self._active_workers: List[int] = list(range(self.launch_dp_size)) self._active_count_cache: int = self.launch_dp_size - if get_parallel().enable_dp_attention: + if get_parallel().config.enable_dp_attention: self.launch_dp_attention_schedulers(server_args, port_args) # When local control broadcast is enabled, send control messages to # every DP group leader (attn_tp_rank=0) so each leader broadcasts # within its own attn_tp_group instead of the full tp_group. # Otherwise fall back to the original behaviour: send to only the # first leader, which then broadcasts over the full tp_group. - local_ctrl = get_parallel().enable_dp_attention_local_control_broadcast + local_ctrl = ( + get_parallel().config.enable_dp_attention_local_control_broadcast + ) self.control_message_step = 1 if local_ctrl else server_args.tp_size else: self.launch_dp_schedulers(server_args, port_args) @@ -375,7 +374,7 @@ class DataParallelController: threads = [] sockets = [] ready_events = [] - for dp_rank in range(get_parallel().dp_size): + for dp_rank in range(get_parallel().config.dp_size): tmp_port_args = PortArgs.init_new(server_args) tmp_port_args.tokenizer_ipc_name = port_args.tokenizer_ipc_name tmp_port_args.detokenizer_ipc_name = port_args.detokenizer_ipc_name @@ -395,7 +394,9 @@ class DataParallelController: ) threads.append(thread) base_gpu_id += ( - server_args.tp_size * configured_pp_size() * server_args.gpu_id_step + server_args.tp_size + * get_parallel().config.pp_size + * server_args.gpu_id_step ) if server_args.node_rank == 0: @@ -577,7 +578,7 @@ class DataParallelController: bind_count = ( self.max_dp_size if server_args.elastic_ep_backend is not None - else get_parallel().dp_size + else get_parallel().config.dp_size ) for slot in range(bind_count): worker_port, worker_socket = get_zmq_socket_on_host( @@ -607,7 +608,7 @@ class DataParallelController: dp_rank: Optional[int], worker_ports: Optional[List[int]] = None, ): - if not get_parallel().enable_dp_attention: + if not get_parallel().config.enable_dp_attention: logger.info(f"Launch DP{dp_rank} starting at GPU #{base_gpu_id}.") memory_saver_adapter = TorchMemorySaverAdapter.create( @@ -616,8 +617,8 @@ class DataParallelController: scheduler_pipe_readers = [] - pp_size_per_node = max(configured_pp_size() // server_args.nnodes, 1) - nnodes_per_pp_rank = max(server_args.nnodes // configured_pp_size(), 1) + pp_size_per_node = max(get_parallel().config.pp_size // server_args.nnodes, 1) + nnodes_per_pp_rank = max(server_args.nnodes // get_parallel().config.pp_size, 1) pp_rank_range = range( pp_size_per_node * (server_args.node_rank // nnodes_per_pp_rank), pp_size_per_node * (server_args.node_rank // nnodes_per_pp_rank + 1), @@ -641,14 +642,14 @@ class DataParallelController: for tp_rank in tp_rank_range: rank_port_args = port_args - if get_parallel().enable_dp_attention: + if get_parallel().config.enable_dp_attention: # dp attention has different sharding logic _, _, dp_rank, _ = compute_dp_attention_world_info( - get_parallel().enable_dp_attention, + get_parallel().config.enable_dp_attention, tp_rank, server_args.tp_size, - get_parallel().dp_size, - configured_attn_cp_size(), + get_parallel().config.dp_size, + get_parallel().config.attn_cp_size, ) # compute zmq ports for this dp rank rank_port_args = PortArgs.init_new( @@ -677,26 +678,32 @@ class DataParallelController: + (tp_rank % tp_size_per_node) * server_args.gpu_id_step ) attn_dp_size = ( - get_parallel().dp_size if get_parallel().enable_dp_attention else 1 + get_parallel().config.dp_size + if get_parallel().config.enable_dp_attention + else 1 ) # Parallelism hierarchy (outermost to innermost): # - Attention: Global(TP) -> DP -> ATTN_CP -> ATTN_TP (innermost) # - MoE: Global(TP) -> MOE_DP -> EP -> MOE_TP (innermost) attn_tp_size = ( - server_args.tp_size // attn_dp_size // configured_attn_cp_size() + server_args.tp_size + // attn_dp_size + // get_parallel().config.attn_cp_size ) - attn_cp_rank = (tp_rank // attn_tp_size) % configured_attn_cp_size() + attn_cp_rank = ( + tp_rank // attn_tp_size + ) % get_parallel().config.attn_cp_size moe_dp_rank = tp_rank // ( - server_args.tp_size // configured_moe_dp_size() + server_args.tp_size // get_parallel().config.moe_dp_size ) moe_ep_rank = ( tp_rank - % (server_args.tp_size // configured_moe_dp_size()) + % (server_args.tp_size // get_parallel().config.moe_dp_size) // ( server_args.tp_size - // configured_moe_dp_size() - // get_parallel().ep_size + // get_parallel().config.moe_dp_size + // get_parallel().config.ep_size ) ) diff --git a/python/sglang/srt/managers/disagg_service.py b/python/sglang/srt/managers/disagg_service.py index 66fcf09e2..be82b892b 100644 --- a/python/sglang/srt/managers/disagg_service.py +++ b/python/sglang/srt/managers/disagg_service.py @@ -49,7 +49,8 @@ def maybe_create_ascend_config_store( which the rust registry ports verbatim), leaving this store as the only ``start_disagg_service`` duty left to perform.""" if not ( - get_parallel().node_rank == 0 and transfer_backend == TransferBackend.ASCEND + get_parallel().config.node_rank == 0 + and transfer_backend == TransferBackend.ASCEND ): return try: diff --git a/python/sglang/srt/managers/load_snapshot.py b/python/sglang/srt/managers/load_snapshot.py index 26e306e4d..140691a72 100644 --- a/python/sglang/srt/managers/load_snapshot.py +++ b/python/sglang/srt/managers/load_snapshot.py @@ -71,7 +71,7 @@ def should_use_zmq() -> bool: ``SGLANG_LOAD_SNAPSHOT_USE_ZMQ`` forces zmq mode for testing. """ return ( - get_parallel().enable_dp_attention and get_parallel().nnodes > 1 + get_parallel().config.enable_dp_attention and get_parallel().config.nnodes > 1 ) or envs.SGLANG_LOAD_SNAPSHOT_USE_ZMQ.get() @@ -116,15 +116,15 @@ def zmq_reader_owner(caller: str) -> bool: """ if not should_use_zmq(): return False - if get_parallel().node_rank != 0: + if get_parallel().config.node_rank != 0: return False if caller == "DataParallelController": return ( - get_parallel().dp_size > 1 - and get_parallel().load_balance_method.lower() in _LOAD_AWARE_METHODS + get_parallel().config.dp_size > 1 + and get_parallel().config.load_balance_method.lower() in _LOAD_AWARE_METHODS ) - if get_parallel().dp_size > 1 and ( - get_parallel().load_balance_method.lower() in _LOAD_AWARE_METHODS + if get_parallel().config.dp_size > 1 and ( + get_parallel().config.load_balance_method.lower() in _LOAD_AWARE_METHODS ): return False return caller == _tokenizer_load_snapshot_owner_caller() @@ -658,7 +658,7 @@ def create_load_snapshot_reader(port_args, caller: str): ``"MultiTokenizerRouter"`` -- determines who binds the zmq PULL socket when zmq mode is active. """ - dp_size = get_parallel().dp_size + dp_size = get_parallel().config.dp_size if zmq_reader_owner(caller): return ZmqShmLoadSnapshotReader( _zmq_addr_for(port_args), shm_path_for(port_args.instance_id), dp_size diff --git a/python/sglang/srt/managers/prefill_delayer.py b/python/sglang/srt/managers/prefill_delayer.py index 198be9a0d..308fbd9af 100644 --- a/python/sglang/srt/managers/prefill_delayer.py +++ b/python/sglang/srt/managers/prefill_delayer.py @@ -108,7 +108,7 @@ class PrefillDelayer: f"queue_trigger_enabled={self._queue_trigger_enabled}" ) self.dp_size = dp_size - self.enable_dp_attention = get_parallel().enable_dp_attention + self.enable_dp_attention = get_parallel().config.enable_dp_attention dp_size_dim = dp_size if self.enable_dp_attention else 1 # Mirror scheduler_dp_attn_mixin's NCCL all-gather path: when the diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index bb2fb4941..56b2200ca 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -29,11 +29,6 @@ from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Set, Tuple, from sglang.srt.runtime_context import ( attention_backends, - configured_attn_cp_size, - configured_dcp_size, - configured_moe_dp_size, - configured_pp_size, - configured_tp_size, get_device, get_disagg, get_exec, @@ -463,38 +458,38 @@ class Scheduler( self.max_recv_per_poll = envs.SGLANG_SCHEDULER_MAX_RECV_PER_POLL.get() self.max_new_tokens_limit = envs.SGLANG_MAX_NEW_TOKENS_LIMIT.get() self.enable_hisparse = get_memory().enable_hisparse - self.enable_dp_attention = get_parallel().enable_dp_attention + self.enable_dp_attention = get_parallel().config.enable_dp_attention self.enable_unified_memory = get_memory().enable_unified_memory # Distributed rank info attn_tp_rank, attn_tp_size, attn_dp_rank, attn_dp_size = ( compute_dp_attention_world_info( - get_parallel().enable_dp_attention, + get_parallel().config.enable_dp_attention, tp_rank, - configured_tp_size(), - get_parallel().dp_size, - configured_attn_cp_size(), + get_parallel().config.tp_size, + get_parallel().config.dp_size, + get_parallel().config.attn_cp_size, ) ) self.ps = ParallelState( tp_rank=tp_rank, - tp_size=configured_tp_size(), + tp_size=get_parallel().config.tp_size, pp_rank=pp_rank, - pp_size=configured_pp_size(), + pp_size=get_parallel().config.pp_size, dp_rank=dp_rank, - dp_size=get_parallel().dp_size, + dp_size=get_parallel().config.dp_size, attn_tp_rank=attn_tp_rank, attn_tp_size=attn_tp_size, attn_cp_rank=attn_cp_rank, - attn_cp_size=configured_attn_cp_size(), - attn_dcp_rank=tp_rank % configured_dcp_size(), - attn_dcp_size=configured_dcp_size(), + attn_cp_size=get_parallel().config.attn_cp_size, + attn_dcp_rank=tp_rank % get_parallel().config.dcp_size, + attn_dcp_size=get_parallel().config.dcp_size, attn_dp_rank=attn_dp_rank, attn_dp_size=attn_dp_size, moe_ep_rank=moe_ep_rank, - moe_ep_size=get_parallel().ep_size, + moe_ep_size=get_parallel().config.ep_size, moe_dp_rank=moe_dp_rank, - moe_dp_size=configured_moe_dp_size(), + moe_dp_size=get_parallel().config.moe_dp_size, gpu_id=gpu_id, ) @@ -1069,7 +1064,7 @@ class Scheduler( self.min_free_slots_delayer = MinFreeSlotsDelayer( min_free_slots=min_free_slots ) - if not get_parallel().pp_max_micro_batch_size: + if not get_parallel().config.pp_max_micro_batch_size: get_context().override( "scheduler.pp_max_micro_batch_size_default", pp_max_micro_batch_size=max( @@ -1414,7 +1409,7 @@ class Scheduler( gloo_group=self.attn_tp_cpu_group, tp_rank=self.ps.tp_rank, tp_size=self.ps.tp_size, - dp_size=get_parallel().dp_size, + dp_size=get_parallel().config.dp_size, gpu_id=self.ps.gpu_id, bootstrap_port=get_disagg().disaggregation_bootstrap_port, max_total_num_tokens=self.max_total_num_tokens, @@ -3233,7 +3228,7 @@ class Scheduler( return NextBatchPlan(batch_to_run=ret, running_batch=running_batch) def get_num_allocatable_reqs(self, running_bs): - res = get_parallel().pp_max_micro_batch_size - running_bs + res = get_parallel().config.pp_max_micro_batch_size - running_bs res = min(res, self.req_to_token_pool.available_size()) return res @@ -4890,7 +4885,7 @@ class Scheduler( old_ep_size = ElasticEPStateManager.get_effective_ep_size() new_ep_size = recv_req.new_ep_size - max_ep_size = get_parallel().max_ep_size or old_ep_size + max_ep_size = get_parallel().config.max_ep_size or old_ep_size logger.debug( "[Elastic EP][scale] request received: new_ep_size=%d " @@ -5095,7 +5090,7 @@ def dispatch_event_loop(scheduler: Scheduler): if disaggregation_mode == DisaggregationMode.NULL: if scheduler.enable_pdmux: scheduler.event_loop_pdmux() - elif configured_pp_size() > 1: + elif get_parallel().config.pp_size > 1: scheduler.event_loop_pp() elif scheduler.enable_overlap_mlx: scheduler.event_loop_overlap_mlx() @@ -5104,14 +5099,14 @@ def dispatch_event_loop(scheduler: Scheduler): else: scheduler.event_loop_normal() elif disaggregation_mode == DisaggregationMode.PREFILL: - if configured_pp_size() > 1: + if get_parallel().config.pp_size > 1: scheduler.event_loop_pp_disagg_prefill() elif scheduler.enable_overlap: scheduler.event_loop_overlap_disagg_prefill() else: scheduler.event_loop_normal_disagg_prefill() elif disaggregation_mode == DisaggregationMode.DECODE: - if configured_pp_size() > 1: + if get_parallel().config.pp_size > 1: scheduler.event_loop_pp_disagg_decode() elif scheduler.enable_overlap: scheduler.event_loop_overlap_disagg_decode() @@ -5152,15 +5147,15 @@ def configure_scheduler_process( prefix = "" if shown_dp is not None: prefix += f" DP{shown_dp}" - if configured_pp_size() > 1: + if get_parallel().config.pp_size > 1: prefix += f" PP{pp_rank}" - if configured_attn_cp_size() > 1: + if get_parallel().config.attn_cp_size > 1: prefix += f" ATTN_CP{attn_cp_rank}" - if configured_moe_dp_size() > 1: + if get_parallel().config.moe_dp_size > 1: prefix += f" MOE_DP{moe_dp_rank}" - if configured_tp_size() > 1: + if get_parallel().config.tp_size > 1: prefix += f" TP{shown_tp}" - if get_parallel().ep_size > 1: + if get_parallel().config.ep_size > 1: prefix += f" EP{shown_moe_ep}" # Config the process @@ -5174,7 +5169,10 @@ def configure_scheduler_process( # Set cpu affinity to this gpu process if envs.SGLANG_SET_CPU_AFFINITY.get(): set_gpu_proc_affinity( - configured_pp_size(), configured_tp_size(), get_parallel().nnodes, gpu_id + get_parallel().config.pp_size, + get_parallel().config.tp_size, + get_parallel().config.nnodes, + gpu_id, ) if not envs.SGLANG_NUMA_BIND_V2.get(): numa_node = get_numa_node_if_available(server_args, gpu_id) diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index 2b1b0e28a..b443da627 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -411,7 +411,7 @@ class SchedulerDPAttnAdapter: return prepare_mlp_sync_batch_raw( local_batch, model_runner=self.model_runner, - dp_size=get_parallel().dp_size, + dp_size=get_parallel().config.dp_size, attn_tp_size=self.ps.attn_tp_size, attn_cp_size=self.ps.attn_cp_size, tp_group=self.tp_group, @@ -420,7 +420,7 @@ class SchedulerDPAttnAdapter: require_mlp_tp_gather=require_mlp_tp_gather(self.server_args), disable_overlap_schedule=get_schedule().disable_overlap_schedule, offload_tags=self.offload_tags, - dwdp=get_parallel().dwdp_size > 1, + dwdp=get_parallel().config.dwdp_size > 1, ) def maybe_prepare_mlp_sync_batch( diff --git a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py index 2ce0a489b..1e5357162 100644 --- a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py +++ b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py @@ -23,10 +23,10 @@ from sglang.srt.observability.metrics_collector import ( compute_routing_key_stats, ) from sglang.srt.runtime_context import ( - configured_pp_size, get_context, get_disagg, get_observability, + get_parallel, get_spec, ) from sglang.srt.utils.device_timer import DeviceTimer @@ -1117,7 +1117,7 @@ class SchedulerMetricsReporter: active_lora_ids = set() # For PP mode, check all running micro batches - if configured_pp_size() > 1: + if get_parallel().config.pp_size > 1: for batch in self.scheduler.running_mbs: if batch and hasattr(batch, "reqs"): for req in batch.reqs: diff --git a/python/sglang/srt/managers/scheduler_components/recv_skipper.py b/python/sglang/srt/managers/scheduler_components/recv_skipper.py index 364723b6c..5fd65ff35 100644 --- a/python/sglang/srt/managers/scheduler_components/recv_skipper.py +++ b/python/sglang/srt/managers/scheduler_components/recv_skipper.py @@ -34,7 +34,7 @@ class SchedulerRecvSkipper: return ForwardMode.DECODE def __init__(self): - self._use_synced_mode = get_parallel().enable_dp_attention + self._use_synced_mode = get_parallel().config.enable_dp_attention self._counter = 0 self._threshold = get_schedule().scheduler_recv_interval # All can be tuned if needed diff --git a/python/sglang/srt/managers/scheduler_components/request_receiver.py b/python/sglang/srt/managers/scheduler_components/request_receiver.py index bcdfff5a3..9741a16c5 100644 --- a/python/sglang/srt/managers/scheduler_components/request_receiver.py +++ b/python/sglang/srt/managers/scheduler_components/request_receiver.py @@ -151,7 +151,7 @@ class SchedulerRequestReceiver: return recv_reqs def _broadcast_reqs_across_ranks(self, recv_reqs: Optional[List]) -> List: - if get_parallel().enable_dp_attention: + if get_parallel().config.enable_dp_attention: if self.ps.attn_tp_rank == 0 and self.ps.attn_cp_rank == 0: work_reqs, control_reqs = self._split_work_and_control_reqs(recv_reqs) else: @@ -180,7 +180,7 @@ class SchedulerRequestReceiver: # instead of the full tp_group. This avoids an expensive # all-ranks gloo sync. _local_ctrl = ( - get_parallel().enable_dp_attention_local_control_broadcast + get_parallel().config.enable_dp_attention_local_control_broadcast or is_ep_scale_joiner() ) if _local_ctrl: @@ -258,7 +258,7 @@ class SchedulerRequestReceiver: # peer ranks may still be unpickling ShmPointerMMData # (-> shm_open). Synchronize the same CPU groups that carried # SHM-backed work requests before materialize() unlinks them. - if get_parallel().enable_dp_attention: + if get_parallel().config.enable_dp_attention: if self.ps.attn_tp_size > 1: barrier(group=self.attn_tp_cpu_group) if self.ps.attn_cp_size > 1: diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 4ac4eef1c..333780384 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -128,7 +128,7 @@ class SchedulerPPMixin: next_pp_outputs = None next_batch_result = None d2h_event = None - if get_parallel().pp_async_batch_depth > 0: + if get_parallel().config.pp_async_batch_depth > 0: next_pp_outputs, next_batch_result, d2h_event = ( self._pp_commit_send_output_work_and_preprocess_output_tensors( next_first_rank_mb_id, @@ -144,7 +144,7 @@ class SchedulerPPMixin: self.mb_metadata, self.last_rank_comm_queue, ) - if get_parallel().pp_async_batch_depth == 0: + if get_parallel().config.pp_async_batch_depth == 0: next_pp_outputs, next_batch_result, d2h_event = ( self._pp_commit_send_output_work_and_preprocess_output_tensors( next_first_rank_mb_id, @@ -274,7 +274,7 @@ class SchedulerPPMixin: server_is_idle = False pp_proxy_tensors = self._pp_recv_proxy_tensors() - if get_parallel().pp_async_batch_depth > 0: + if get_parallel().config.pp_async_batch_depth > 0: next_pp_outputs, next_batch_result, d2h_event = ( self._pp_commit_send_output_work_and_preprocess_output_tensors( next_first_rank_mb_id, @@ -292,7 +292,7 @@ class SchedulerPPMixin: self.mb_metadata, self.last_rank_comm_queue, ) - if get_parallel().pp_async_batch_depth == 0: + if get_parallel().config.pp_async_batch_depth == 0: next_pp_outputs, next_batch_result, d2h_event = ( self._pp_commit_send_output_work_and_preprocess_output_tensors( next_first_rank_mb_id, @@ -435,7 +435,7 @@ class SchedulerPPMixin: pp_proxy_tensors = self._pp_recv_proxy_tensors() # early send output if possible - if get_parallel().pp_async_batch_depth > 0: + if get_parallel().config.pp_async_batch_depth > 0: next_pp_outputs, next_batch_result, d2h_event = ( self._pp_commit_send_output_work_and_preprocess_output_tensors( next_first_rank_mb_id, @@ -453,7 +453,7 @@ class SchedulerPPMixin: self.last_rank_comm_queue, ) - if get_parallel().pp_async_batch_depth == 0: + if get_parallel().config.pp_async_batch_depth == 0: next_pp_outputs, next_batch_result, d2h_event = ( self._pp_commit_send_output_work_and_preprocess_output_tensors( next_first_rank_mb_id, @@ -564,10 +564,12 @@ class SchedulerPPMixin: self.on_idle() def init_pp_loop_state(self: Scheduler): - self.pp_loop_size: int = self.ps.pp_size + get_parallel().pp_async_batch_depth + self.pp_loop_size: int = ( + self.ps.pp_size + get_parallel().config.pp_async_batch_depth + ) # In CP mode, attention weights are duplicated, eliminating the need for the attention TP all-gather operation. self.require_attn_tp_allgather = ( - not get_parallel().enable_dsa_prefill_context_parallel + not get_parallel().config.enable_dsa_prefill_context_parallel ) self.mbs = [None] * self.pp_loop_size self.last_mbs = [None] * self.pp_loop_size diff --git a/python/sglang/srt/managers/tokenizer_control_mixin.py b/python/sglang/srt/managers/tokenizer_control_mixin.py index 994c4b614..c7713a0e4 100644 --- a/python/sglang/srt/managers/tokenizer_control_mixin.py +++ b/python/sglang/srt/managers/tokenizer_control_mixin.py @@ -165,7 +165,7 @@ class TokenizerControlMixin: mode = spec[2] if len(spec) > 2 else "queueing" comm = FanOutCommunicator( self._dispatch_to_scheduler, - get_parallel().dp_size, + get_parallel().config.dp_size, mode, ) setattr(self, f"{name}_communicator", comm) @@ -174,8 +174,8 @@ class TokenizerControlMixin: def update_control_communicator_fan_out(self: TokenizerManager, worker_count: int): primary_group_control = ( - get_parallel().enable_dp_attention - and not get_parallel().enable_dp_attention_local_control_broadcast + get_parallel().config.enable_dp_attention + and not get_parallel().config.enable_dp_attention_local_control_broadcast ) if primary_group_control: control_fan_out = ( @@ -428,7 +428,8 @@ class TokenizerControlMixin: ) -> Tuple[bool, str]: self.auto_create_handle_loop() assert ( - get_parallel().dp_size == 1 or get_parallel().enable_dp_attention + get_parallel().config.dp_size == 1 + or get_parallel().config.enable_dp_attention ), "dp_size must be 1 or dp attention must be enabled for update weights from distributed" results = await self.init_weights_update_group_communicator(obj) @@ -441,7 +442,8 @@ class TokenizerControlMixin: ) -> Tuple[bool, str]: self.auto_create_handle_loop() assert ( - get_parallel().dp_size == 1 or get_parallel().enable_dp_attention + get_parallel().config.dp_size == 1 + or get_parallel().config.enable_dp_attention ), "dp_size must be 1 or dp attention must be enabled for destroy parameter update group" results = await self.destroy_weights_update_group_communicator(obj) @@ -454,7 +456,8 @@ class TokenizerControlMixin: ) -> Tuple[bool, str]: self.auto_create_handle_loop() assert ( - get_parallel().dp_size == 1 or get_parallel().enable_dp_attention + get_parallel().config.dp_size == 1 + or get_parallel().config.enable_dp_attention ), "dp_size must be 1 or dp attention must be enabled for update weights from distributed" if obj.abort_all_requests: @@ -487,7 +490,7 @@ class TokenizerControlMixin: self.auto_create_handle_loop() # TODO: support DP assert ( - get_parallel().dp_size == 1 + get_parallel().config.dp_size == 1 ), "dp_size must be 1 for init_weights_send_group_for_remote_instance" result = ( await self.init_weights_send_group_for_remote_instance_communicator(obj) @@ -502,7 +505,7 @@ class TokenizerControlMixin: self.auto_create_handle_loop() # TODO: support DP assert ( - get_parallel().dp_size == 1 + get_parallel().config.dp_size == 1 ), "dp_size must be 1 for send_weights_to_remote_instance" result = (await self.send_weights_to_remote_instance_communicator(obj))[0] return result.success, result.message @@ -514,7 +517,8 @@ class TokenizerControlMixin: ) -> Tuple[bool, str]: self.auto_create_handle_loop() assert ( - get_parallel().dp_size == 1 or get_parallel().enable_dp_attention + get_parallel().config.dp_size == 1 + or get_parallel().config.enable_dp_attention ), "dp_size must be 1 or dp attention must be enabled for update weights from tensor" if obj.abort_all_requests: @@ -552,7 +556,8 @@ class TokenizerControlMixin: try: # For now, we only support single data parallel instance assert ( - get_parallel().dp_size == 1 or get_parallel().enable_dp_attention + get_parallel().config.dp_size == 1 + or get_parallel().config.enable_dp_attention ), "dp_size must be 1 or dp attention must be enabled for update weights from IPC" logger.info("Starting IPC weight update") @@ -615,7 +620,8 @@ class TokenizerControlMixin: ) assert ( - get_parallel().dp_size == 1 or get_parallel().enable_dp_attention + get_parallel().config.dp_size == 1 + or get_parallel().config.enable_dp_attention ), "dp_size must be 1 or dp attention must be enabled for dynamic lora loading" logger.info( "Start load Lora adapter. Lora name=%s, path=%s", @@ -693,7 +699,8 @@ class TokenizerControlMixin: ) assert ( - get_parallel().dp_size == 1 or get_parallel().enable_dp_attention + get_parallel().config.dp_size == 1 + or get_parallel().config.enable_dp_attention ), "dp_size must be 1 or dp attention must be enabled for dynamic lora loading" logger.info( "Start load Lora adapter from tensors. Lora name=%s", @@ -773,7 +780,8 @@ class TokenizerControlMixin: ), "lora_name must be provided to unload LoRA adapter" assert ( - get_parallel().dp_size == 1 or get_parallel().enable_dp_attention + get_parallel().config.dp_size == 1 + or get_parallel().config.enable_dp_attention ), "dp_size must be 1 or dp attention must be enabled for dynamic lora loading" logger.info( "Start unload Lora adapter. Lora name=%s", @@ -793,7 +801,7 @@ class TokenizerControlMixin: self.auto_create_handle_loop() results = await self.get_weights_by_name_communicator(obj) all_parameters = [r.parameter for r in results] - if get_parallel().dp_size == 1: + if get_parallel().config.dp_size == 1: return all_parameters[0] else: return all_parameters diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index b9a102087..c21fe1754 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -411,7 +411,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): self.server_args = server_args ensure_published(server_args, role="tokenizer") self.startup_time: Optional[Dict[str, Any]] = None - self.elastic_worker_count = get_parallel().dp_size + self.elastic_worker_count = get_parallel().config.dp_size self.elastic_pending_ep_size = None self.elastic_scale_phase = "idle" self.elastic_last_error = None @@ -1548,7 +1548,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): return batch_size > 0 and ( get_serving().enable_tokenizer_batch_encode or ( - (not get_parallel().enable_dp_attention) + (not get_parallel().config.enable_dp_attention) and (not self._batch_has_text(batch_size, requests)) ) ) diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index 08ee50f9f..52b700bc1 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -275,7 +275,9 @@ def build_kv_cache( ), is_eagle=spec_algorithm.is_eagle(), tp_cache_group=( - attn_tp_cpu_group if get_parallel().enable_dp_attention else tp_cpu_group + attn_tp_cpu_group + if get_parallel().config.enable_dp_attention + else tp_cpu_group ), attn_cp_cache_group=attn_cp_cpu_group, attn_tp_cache_group=attn_tp_cpu_group, diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 47df501ec..068fda733 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -65,7 +65,6 @@ from sglang.srt.mem_cache.memory_pool import ( from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.platforms import current_platform from sglang.srt.runtime_context import ( - configured_pp_size, get_context, get_disagg, get_exec, @@ -1922,7 +1921,7 @@ class KVCacheConfigurator: token_capacity = min(token_capacity, user_limit) # Sync across PP ranks (each may have different layer counts) - if configured_pp_size() > 1: + if get_parallel().config.pp_size > 1: tensor = torch.tensor(token_capacity, dtype=torch.int64) torch.distributed.all_reduce( tensor, diff --git a/python/sglang/srt/mem_cache/pool_host/base.py b/python/sglang/srt/mem_cache/pool_host/base.py index 1c258a771..067a4cd0a 100644 --- a/python/sglang/srt/mem_cache/pool_host/base.py +++ b/python/sglang/srt/mem_cache/pool_host/base.py @@ -45,7 +45,7 @@ def ranks_per_host() -> int: return 1 if world_group.world_size == 1: return 1 - return max(world_group.world_size // get_parallel().nnodes, 1) + return max(world_group.world_size // get_parallel().config.nnodes, 1) def host_memory_budget_bytes() -> int: diff --git a/python/sglang/srt/model_executor/cpu_graph_runner.py b/python/sglang/srt/model_executor/cpu_graph_runner.py index 102828d7a..be0a80da3 100644 --- a/python/sglang/srt/model_executor/cpu_graph_runner.py +++ b/python/sglang/srt/model_executor/cpu_graph_runner.py @@ -40,7 +40,6 @@ from sglang.srt.model_executor.forward_batch_info import ( from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.model_executor.runner_utils.capture_mode import model_capture_mode from sglang.srt.runtime_context import ( - configured_pp_size, get_exec, get_flags, get_lora, @@ -611,8 +610,8 @@ class CPUGraphRunner: model_runner.server_args.enable_profile_cuda_graph ) self.tp_size = model_runner.server_args.tp_size - self.dp_size = get_parallel().dp_size - self.pp_size = configured_pp_size() + self.dp_size = get_parallel().config.dp_size + self.pp_size = get_parallel().config.pp_size self.capture_forward_mode = ForwardMode.DECODE self.capture_hidden_mode = self.return_hidden_states_mode diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 8c4d18f61..b990b4914 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -489,17 +489,19 @@ class ModelRunner: if not (get_exec().moe.elastic_ep_backend is not None and is_ep_scale_joiner()): return - join_effective_ep_size = get_parallel().ep_join_rank_offset + self.ps.tp_size + join_effective_ep_size = ( + get_parallel().config.ep_join_rank_offset + self.ps.tp_size + ) dist.barrier(group=self.tp_group.cpu_group) if self.ps.tp_rank == 0: register_scale_cohort( - get_parallel().ep_join_rank_offset, + get_parallel().config.ep_join_rank_offset, join_effective_ep_size, ) join_scale_process_group() get_context().override("elastic_ep.scale_join", ep_size=join_effective_ep_size) - global_ep_rank = self.ps.tp_rank + get_parallel().ep_join_rank_offset + global_ep_rank = self.ps.tp_rank + get_parallel().config.ep_join_rank_offset broadcast_global_expert_location_metadata( model_config=self.model_config, moe_ep_rank=global_ep_rank, @@ -699,7 +701,7 @@ class ModelRunner: if self.is_draft_worker: return expert_rank = self.ps.moe_ep_rank + ( - get_parallel().ep_join_rank_offset if is_ep_scale_joiner() else 0 + get_parallel().config.ep_join_rank_offset if is_ep_scale_joiner() else 0 ) set_global_expert_location_metadata( compute_initial_expert_location_metadata( @@ -897,7 +899,7 @@ class ModelRunner: device=self.device, tp_group=( self.attention_tp_group.cpu_group - if get_parallel().enable_dp_attention + if get_parallel().config.enable_dp_attention else self.tp_group.cpu_group ), host_to_device_ratio=hisparse_cfg.host_to_device_ratio, @@ -933,7 +935,7 @@ class ModelRunner: def post_capture_elastic_ep_recover(self): join_process_groups() - global_ep_rank = self.ps.tp_rank + get_parallel().ep_join_rank_offset + global_ep_rank = self.ps.tp_rank + get_parallel().config.ep_join_rank_offset broadcast_global_expert_location_metadata( model_config=self.model_config, moe_ep_rank=global_ep_rank, @@ -973,7 +975,7 @@ class ModelRunner: self.decode_attn_backend = backends.decode_attn_backend self.decode_attn_backend_group = backends.decode_attn_backend_group - if get_parallel().dcp_enabled and get_parallel().dcp_replicate_q_proj: + if get_parallel().dcp_enabled and get_parallel().config.dcp_replicate_q_proj: self._prepare_replicated_q_proj() def _prepare_replicated_q_proj(self) -> None: @@ -1257,7 +1259,7 @@ class ModelRunner: def maybe_init_dwdp(self): if self.is_draft_worker: return - if get_parallel().dwdp_size <= 1: + if get_parallel().config.dwdp_size <= 1: return from sglang.srt.layers.moe.dwdp import DwdpManager @@ -1431,7 +1433,7 @@ class ModelRunner: # rather than spawning additional processes, so dp_size must not be # multiplied into the process count here (unlike regular DP, where # dp_size * tp_size * pp_size is the true worker count). - dp_size = 1 if get_parallel().enable_dp_attention else self.ps.dp_size + dp_size = 1 if get_parallel().config.enable_dp_attention else self.ps.dp_size self.local_omp_cpuid = numa_utils.init_threads_binding( numa_index=self.gpu_id, world_size=dp_size * self.ps.tp_size * self.ps.pp_size, @@ -1916,7 +1918,7 @@ class ModelRunner: if added <= 0: return - initial_ep_size = get_parallel().elastic_ep_initial_size + initial_ep_size = get_parallel().config.elastic_ep_initial_size assert initial_ep_size is not None get_context().override("elastic_ep.scale", ep_size=effective_size) @@ -1935,7 +1937,7 @@ class ModelRunner: set_global_expert_location_metadata(new_metadata, allow_overwrite=True) def _elastic_global_rank(self) -> int: - return self.ps.tp_rank + get_parallel().ep_join_rank_offset + return self.ps.tp_rank + get_parallel().config.ep_join_rank_offset def _rearm_eplb_after_elastic_scale(self) -> None: if self.eplb_manager is None: diff --git a/python/sglang/srt/model_executor/model_runner_components/remote_instance_weight_transporter.py b/python/sglang/srt/model_executor/model_runner_components/remote_instance_weight_transporter.py index 14690ab21..56213b229 100644 --- a/python/sglang/srt/model_executor/model_runner_components/remote_instance_weight_transporter.py +++ b/python/sglang/srt/model_executor/model_runner_components/remote_instance_weight_transporter.py @@ -78,11 +78,13 @@ class RemoteInstanceWeightTransporter: """ import requests as http_requests - if get_parallel().dist_init_addr: + if get_parallel().config.dist_init_addr: # Multi-node: bootstrap server is on the head node (node_rank==0). # Derive host from dist_init_addr (shared across all nodes). bootstrap_host = ( - NetworkAddress.parse(get_parallel().dist_init_addr).resolved().host + NetworkAddress.parse(get_parallel().config.dist_init_addr) + .resolved() + .host ) else: bootstrap_host = "127.0.0.1" diff --git a/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py b/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py index f3429e077..3e26fd8ad 100644 --- a/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py +++ b/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py @@ -7,8 +7,8 @@ import msgspec from sglang.srt.configs.model_config import ModelConfig from sglang.srt.runtime_context import ( - configured_tp_size, get_model, + get_parallel, get_spec, ) @@ -242,7 +242,7 @@ def _resolve_dflash_draft_cell_size( draft_model_config=draft_model_config, draft_num_layers=draft_num_layers, draft_kv_cache_dtype=draft_kv_cache_dtype, - tp_size=configured_tp_size(), + tp_size=get_parallel().config.tp_size, ) except Exception as e: # noqa: BLE001 logger.warning( diff --git a/python/sglang/srt/model_executor/model_runner_components/startup_weight_load.py b/python/sglang/srt/model_executor/model_runner_components/startup_weight_load.py index df6d87069..202e602c5 100644 --- a/python/sglang/srt/model_executor/model_runner_components/startup_weight_load.py +++ b/python/sglang/srt/model_executor/model_runner_components/startup_weight_load.py @@ -21,10 +21,6 @@ from sglang.srt.model_loader.weight_utils import ( ) from sglang.srt.platforms import current_platform from sglang.srt.runtime_context import ( - configured_attn_cp_size, - configured_dcp_size, - configured_pp_size, - configured_tp_size, get_device, get_exec, get_lora, @@ -120,12 +116,12 @@ class StartupWeightLoadOptions: prefill_cuda_graph_backend=cuda_graph_config.prefill.backend, is_draft_worker=is_draft_worker, speculative_algorithm=get_spec().speculative_algorithm, - tp_size=configured_tp_size(), - attn_cp_size=configured_attn_cp_size(), - dcp_size=configured_dcp_size(), - pp_size=configured_pp_size(), - dp_size=get_parallel().dp_size, - ep_size=get_parallel().ep_size, + tp_size=get_parallel().config.tp_size, + attn_cp_size=get_parallel().config.attn_cp_size, + dcp_size=get_parallel().config.dcp_size, + pp_size=get_parallel().config.pp_size, + dp_size=get_parallel().config.dp_size, + ep_size=get_parallel().config.ep_size, cpu_offload_gb=get_exec().offload.cpu_offload_gb, offload_group_size=get_exec().offload.offload_group_size, enable_memory_saver=get_exec().features.enable_memory_saver, diff --git a/python/sglang/srt/model_executor/runner/base_runner.py b/python/sglang/srt/model_executor/runner/base_runner.py index ca2030a16..20a024aad 100644 --- a/python/sglang/srt/model_executor/runner/base_runner.py +++ b/python/sglang/srt/model_executor/runner/base_runner.py @@ -47,7 +47,6 @@ from sglang.srt.model_executor.runner.flashinfer_autotune import ( should_run_flashinfer_autotune, ) from sglang.srt.runtime_context import ( - configured_pp_size, get_disagg, get_exec, get_flags, @@ -219,8 +218,8 @@ class BaseRunner(ABC): self.device_module = torch.get_device_module(self.device) self.tp_size = model_runner.server_args.tp_size # elastic-EP scale-up rewrites dp_size on the published config - self.dp_size = get_parallel().dp_size - self.pp_size = configured_pp_size() + self.dp_size = get_parallel().config.dp_size + self.pp_size = get_parallel().config.pp_size self.enable_pdmux = model_runner.server_args.enable_pdmux self.return_hidden_states_mode = ( CaptureHiddenMode.NULL @@ -290,7 +289,7 @@ class BaseRunner(ABC): """ if ( not get_parallel().dcp_enabled - or get_parallel().dcp_comm_backend != "fi_a2a" + or get_parallel().config.dcp_comm_backend != "fi_a2a" ): return @@ -349,8 +348,8 @@ class BaseRunner(ABC): hidden_size=mr.model_config.hidden_size, vocab_size=mr.model_config.vocab_size, dtype=mr.model_config.dtype, - dp_size=get_parallel().dp_size, - pp_size=configured_pp_size(), + dp_size=get_parallel().config.dp_size, + pp_size=get_parallel().config.pp_size, is_encoder_decoder=mr.model_config.is_encoder_decoder, require_mlp_tp_gather=require_mlp_tp_gather(mr.server_args), seq_len_fill_value=mr.attn_backend.get_cuda_graph_seq_len_fill_value(), @@ -522,7 +521,7 @@ class BaseRunner(ABC): extend_prefix_lens = None extend_start_loc = None - if configured_pp_size() > 1: + if get_parallel().config.pp_size > 1: # PP0 already cp-split hidden_states before send. pp_hidden_tokens = num_tokens if ( @@ -542,7 +541,7 @@ class BaseRunner(ABC): assert require_mlp_tp_gather_ or require_attn_tp_gather_ if require_mlp_tp_gather_: - global_num_tokens_cpu = [num_tokens] * get_parallel().dp_size + global_num_tokens_cpu = [num_tokens] * get_parallel().config.dp_size elif require_attn_tp_gather_: global_num_tokens_cpu = [num_tokens] else: @@ -646,7 +645,7 @@ class BaseRunner(ABC): kwargs = {} if ( - configured_pp_size() > 1 + get_parallel().config.pp_size > 1 and "pp_proxy_tensors" in inspect.signature(mr.model.forward).parameters ): kwargs["pp_proxy_tensors"] = PPProxyTensors( diff --git a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py index 934be8494..65323e63b 100644 --- a/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/decode_cuda_graph_runner.py @@ -238,7 +238,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner): self.require_mlp_tp_gather or self.require_attn_tp_gather ) self.require_mlp_sync = ( - get_parallel().enable_dp_attention or self.require_gathered_buffer + get_parallel().config.enable_dp_attention or self.require_gathered_buffer ) self.enable_two_batch_overlap = ( model_runner.server_args.enable_two_batch_overlap diff --git a/python/sglang/srt/model_executor/runner/eager_runner.py b/python/sglang/srt/model_executor/runner/eager_runner.py index cf6d3703c..6476c3a87 100644 --- a/python/sglang/srt/model_executor/runner/eager_runner.py +++ b/python/sglang/srt/model_executor/runner/eager_runner.py @@ -141,7 +141,7 @@ class EagerRunner(BaseRunner): encoder_lens_dtype=( torch.int64 if torch.device(mr.device).type == "cpu" else torch.int32 ), - dp_size=get_parallel().dp_size, + dp_size=get_parallel().config.dp_size, ) # Eager has no capture step, so warm up here (run-once via mr._kernel_warmed_up). self.warmup() diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index e703d4e1e..96d57bfca 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -348,7 +348,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): self.moe_fusions = self.model_runner.moe_fusions self.dsa_indexers = getattr(self.model_runner, "dsa_indexers", None) - self.dp_size = get_parallel().dp_size + self.dp_size = get_parallel().config.dp_size self.require_mlp_tp_gather = require_mlp_tp_gather(model_runner.server_args) self.require_attn_tp_gather = require_attn_tp_gather(model_runner.server_args) diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index 85be7f2b5..23481b31f 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -49,7 +49,6 @@ from sglang.srt.model_loader.remote_instance_weight_loader_utils import ( register_memory_region, ) from sglang.srt.runtime_context import ( - configured_moe_dp_size, get_exec, get_model, get_parallel, @@ -1891,9 +1890,9 @@ class PreshardedModelLoader(DefaultModelLoader): "dp": _safe(lambda: parallel.moe_dp_size), "ep": _safe(lambda: parallel.moe_ep_size), "pp": _safe(lambda: parallel.pp_size), - "moe_dense_tp_size": parallel.moe_dense_tp_size, - "moe_dp_size": configured_moe_dp_size(), - "enable_dp_lm_head": parallel.enable_dp_lm_head, + "moe_dense_tp_size": parallel.config.moe_dense_tp_size, + "moe_dp_size": get_parallel().config.moe_dp_size, + "enable_dp_lm_head": parallel.config.enable_dp_lm_head, "enable_fp32_lm_head": get_exec().features.enable_fp32_lm_head, "quantization": model_config.quantization, "model_dtype": str(model_config.dtype), diff --git a/python/sglang/srt/models/apertus.py b/python/sglang/srt/models/apertus.py index a016bfdde..de4aaa7db 100644 --- a/python/sglang/srt/models/apertus.py +++ b/python/sglang/srt/models/apertus.py @@ -442,7 +442,7 @@ class ApertusForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True) diff --git a/python/sglang/srt/models/arcee.py b/python/sglang/srt/models/arcee.py index 79a6beff4..e4fd0d0ce 100644 --- a/python/sglang/srt/models/arcee.py +++ b/python/sglang/srt/models/arcee.py @@ -405,7 +405,7 @@ class ArceeForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True) diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index 3258d849d..fd38159b7 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -817,7 +817,7 @@ class BailingMoEForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index 66de4b208..981ade376 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -1084,7 +1084,7 @@ class BailingMoELinearForCausalLM(nn.Module): config.hidden_size, params_dtype=torch.float32, quant_config=quant_config, - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/bailing_moe_nextn.py b/python/sglang/srt/models/bailing_moe_nextn.py index dab75ef02..fabf6cad6 100644 --- a/python/sglang/srt/models/bailing_moe_nextn.py +++ b/python/sglang/srt/models/bailing_moe_nextn.py @@ -208,7 +208,7 @@ class BailingMoeForCausalLMNextN(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) if hasattr(self.config, "model_type") and config.model_type == "bailing_hybrid": diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py index 7dad2fb6b..3e02ed9de 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py @@ -299,7 +299,7 @@ class DeepseekMLAForwardMixin: # --dcp-replicate-q-proj: project full-head Q locally from pre-gathered # weights and skip the per-layer Q all-gather (bf16 decode absorb only). q_replicate_active = ( - get_parallel().dcp_replicate_q_proj + get_parallel().config.dcp_replicate_q_proj and is_dcp_mla_decode_phase(forward_batch) and not self.use_deep_gemm_bmm and self.w_kc_qrep is not None @@ -778,7 +778,7 @@ class DeepseekMLAForwardMixin: attn_output, self.num_local_heads ) else: - dcp_comm_backend = get_parallel().dcp_comm_backend + dcp_comm_backend = get_parallel().config.dcp_comm_backend is_lse_base_on_e = is_mla_dcp_lse_base_on_e( self.current_attention_backend ) diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py index 378f3cbf6..e6373bc5b 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_rocm.py @@ -331,7 +331,7 @@ class DeepseekMLARocmForwardMixin: from sglang.srt.model_executor.runner import get_is_capture_mode q_replicate_active = ( - get_parallel().dcp_replicate_q_proj + get_parallel().config.dcp_replicate_q_proj and is_dcp_mla_decode_phase(forward_batch) and not self.use_deep_gemm_bmm and self.w_kc_qrep is not None @@ -778,7 +778,7 @@ class DeepseekMLARocmForwardMixin: attn_output, self.num_local_heads ) else: - dcp_comm_backend = get_parallel().dcp_comm_backend + dcp_comm_backend = get_parallel().config.dcp_comm_backend is_lse_base_on_e = is_mla_dcp_lse_base_on_e( self.current_attention_backend ) diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py index bb03b82a6..517b37e86 100644 --- a/python/sglang/srt/models/deepseek_nextn.py +++ b/python/sglang/srt/models/deepseek_nextn.py @@ -366,7 +366,7 @@ class DeepseekV3ForCausalLMNextN(DeepseekV3ForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 1259dcd2f..8bafc13e0 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -3003,7 +3003,7 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) else: # ranks other than the last rank will have a placeholder layer diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 247c05908..30ee337f9 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -3230,7 +3230,7 @@ class DeepseekV4ForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) else: self.lm_head = PPMissingLayer() diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py index b826635b8..8e55b79ee 100644 --- a/python/sglang/srt/models/deepseek_v4_dspark.py +++ b/python/sglang/srt/models/deepseek_v4_dspark.py @@ -756,7 +756,7 @@ class DeepseekV4ForCausalLMDSpark(nn.Module): config.vocab_size, config.hidden_size, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) else: self.embed_tokens: Optional[nn.Module] = None diff --git a/python/sglang/srt/models/deepseek_v4_nextn.py b/python/sglang/srt/models/deepseek_v4_nextn.py index 39c064d9c..0335e674c 100644 --- a/python/sglang/srt/models/deepseek_v4_nextn.py +++ b/python/sglang/srt/models/deepseek_v4_nextn.py @@ -250,7 +250,7 @@ class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 9a64c59a6..3aa48e600 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -1876,7 +1876,7 @@ class Dots3LanguageModelForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/dots3_common/nextn.py b/python/sglang/srt/models/dots3_common/nextn.py index 2ae34a728..0fd2ce579 100644 --- a/python/sglang/srt/models/dots3_common/nextn.py +++ b/python/sglang/srt/models/dots3_common/nextn.py @@ -167,7 +167,7 @@ class Dots3NoteForCausalLMNextN(Dots3LanguageModelForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self._mtp_loaded_embed = False diff --git a/python/sglang/srt/models/exaone4.py b/python/sglang/srt/models/exaone4.py index d01dad06c..99703c10d 100644 --- a/python/sglang/srt/models/exaone4.py +++ b/python/sglang/srt/models/exaone4.py @@ -439,7 +439,7 @@ class Exaone4ForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/exaone_moe.py b/python/sglang/srt/models/exaone_moe.py index 93e8de9fb..237a93169 100755 --- a/python/sglang/srt/models/exaone_moe.py +++ b/python/sglang/srt/models/exaone_moe.py @@ -643,7 +643,7 @@ class ExaoneMoEForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) # For EAGLE3 support diff --git a/python/sglang/srt/models/exaone_moe_mtp.py b/python/sglang/srt/models/exaone_moe_mtp.py index 10dea5461..110aa4921 100644 --- a/python/sglang/srt/models/exaone_moe_mtp.py +++ b/python/sglang/srt/models/exaone_moe_mtp.py @@ -63,7 +63,7 @@ class ExaoneMoEForCausalLMMTP(ExaoneMoEForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py index 8e3060cfb..d5d1c793d 100644 --- a/python/sglang/srt/models/falcon_h1.py +++ b/python/sglang/srt/models/falcon_h1.py @@ -472,7 +472,7 @@ class FalconH1ForCausalLM(nn.Module): quant_config=quant_config, org_num_embeddings=config.vocab_size, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.lm_head = self.lm_head.float() self.lm_head_multiplier = config.lm_head_multiplier diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index a4aa494c7..037ee098f 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -1163,7 +1163,7 @@ class Glm4MoeForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 0388d2180..5f9440c85 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -905,7 +905,7 @@ class Glm4MoeLiteForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/glm4_moe_lite_nextn.py b/python/sglang/srt/models/glm4_moe_lite_nextn.py index bcd3870cb..607c5e765 100644 --- a/python/sglang/srt/models/glm4_moe_lite_nextn.py +++ b/python/sglang/srt/models/glm4_moe_lite_nextn.py @@ -159,7 +159,7 @@ class Glm4MoeLiteForCausalLMNextN(Glm4MoeLiteForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/glm4_moe_nextn.py b/python/sglang/srt/models/glm4_moe_nextn.py index 72fde2c97..b8efbd353 100644 --- a/python/sglang/srt/models/glm4_moe_nextn.py +++ b/python/sglang/srt/models/glm4_moe_nextn.py @@ -144,7 +144,7 @@ class Glm4MoeForCausalLMNextN(Glm4MoeForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/glm4v_moe.py b/python/sglang/srt/models/glm4v_moe.py index 9c976fa80..68dabe217 100644 --- a/python/sglang/srt/models/glm4v_moe.py +++ b/python/sglang/srt/models/glm4v_moe.py @@ -70,7 +70,7 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) else: # ranks other than the last rank will have a placeholder layer diff --git a/python/sglang/srt/models/glm_ocr_nextn.py b/python/sglang/srt/models/glm_ocr_nextn.py index a29de15cb..726dc97d1 100644 --- a/python/sglang/srt/models/glm_ocr_nextn.py +++ b/python/sglang/srt/models/glm_ocr_nextn.py @@ -135,7 +135,7 @@ class GlmOcrForConditionalGenerationNextN(GlmOcrForConditionalGeneration): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index 1ec1e1962..216187049 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -258,7 +258,7 @@ class GptOssSparseMoeBlock(nn.Module): hidden_states: torch.Tensor, forward_batch: Optional[ForwardBatch] = None, ) -> torch.Tensor: - if get_parallel().dwdp_size > 1: + if get_parallel().config.dwdp_size > 1: return self.forward_dwdp(hidden_states) if not get_moe_a2a_backend().is_deepep(): @@ -786,7 +786,7 @@ class GptOssForCausalLM(nn.Module): config.hidden_size, # quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.capture_aux_hidden_states = False diff --git a/python/sglang/srt/models/kimi_k25.py b/python/sglang/srt/models/kimi_k25.py index cd81cf5d4..a98a1fd7c 100644 --- a/python/sglang/srt/models/kimi_k25.py +++ b/python/sglang/srt/models/kimi_k25.py @@ -46,9 +46,9 @@ from sglang.srt.multimodal.mm_utils import ( run_dp_sharded_mrope_vision_model, ) from sglang.srt.runtime_context import ( - configured_tp_size, get_exec, get_mm, + get_parallel, ) from sglang.srt.utils import add_prefix, is_cuda, is_npu @@ -733,7 +733,7 @@ class KimiK25ForConditionalGeneration(nn.Module): # Match the configured TP consumer count captured when the # tokenizer creates MmItemMemoryPool. A live attention subgroup # size could leave acknowledgements missing and strand the lease. - ipc_consumer_count = max(configured_tp_size(), 1) + ipc_consumer_count = max(get_parallel().config.tp_size, 1) device_index = device.index if device.type == "cuda" and device_index is None: device_index = torch.cuda.current_device() diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 3e0d30aba..a7a59d87a 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -119,7 +119,6 @@ from sglang.srt.multimodal.kimi_k3_image_processing import ( ) from sglang.srt.multimodal.mm_utils import materialize_multimodal_features from sglang.srt.runtime_context import ( - configured_tp_size, get_exec, get_parallel, get_server_args, @@ -298,7 +297,7 @@ class KimiK3MLP(nn.Module): # but allow the NPU launcher to retain the proven attention-TP layout # without a device-type branch in shared model code. self._dense_attn_tp = ( - get_parallel().enable_dense_mlp_attn_tp + get_parallel().config.enable_dense_mlp_attn_tp and is_dp_attention_enabled() and tp_rank is None and tp_size is None @@ -556,13 +555,13 @@ class KimiK3MoE(nn.Module): # a TP-sharded partial sum could never be reduced across ranks that # hold different tokens. self._shared_experts_tp1 = ( - self._ep_a2a and not get_parallel().enable_shared_experts_attn_tp + self._ep_a2a and not get_parallel().config.enable_shared_experts_attn_tp ) # NPU compatibility mode keeps DeepEP's DP-local token dispatch but # uses the original TP-sharded shared MLP. Gather only that branch's # inputs, then reduce-scatter its output back to the DP-local rows. self._shared_experts_attn_tp_comm = ( - get_parallel().enable_shared_experts_attn_tp + get_parallel().config.enable_shared_experts_attn_tp and self._ep_a2a and self._dp_attention and get_parallel().attn_tp_size > 1 @@ -2872,7 +2871,7 @@ class KimiK3LinearForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=maybe_prefix(prefix, "lm_head"), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) else: self.lm_head = PPMissingLayer() @@ -3384,7 +3383,7 @@ class KimiK3ForConditionalGeneration(nn.Module): # Match the configured TP consumer count captured when the # tokenizer creates MmItemMemoryPool. A live attention subgroup # size could leave acknowledgements missing and strand the lease. - ipc_consumer_count = max(configured_tp_size(), 1) + ipc_consumer_count = max(get_parallel().config.tp_size, 1) device_index = device.index if device.type == "cuda" and device_index is None: device_index = torch.cuda.current_device() diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 9aa8d4591..b951dc55d 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -667,7 +667,7 @@ class LagunaForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) else: self.lm_head = PPMissingLayer() diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index 626914dfe..6616c662f 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -824,7 +824,7 @@ class LLaDA2MoeModelLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config, return_full_logits=True) diff --git a/python/sglang/srt/models/llama.py b/python/sglang/srt/models/llama.py index c3349251d..0f560b75d 100644 --- a/python/sglang/srt/models/llama.py +++ b/python/sglang/srt/models/llama.py @@ -536,7 +536,7 @@ class LlamaForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True) diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index 4bed1df35..020c559e6 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -721,7 +721,7 @@ class LongcatFlashForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.capture_aux_hidden_states = False diff --git a/python/sglang/srt/models/mellum.py b/python/sglang/srt/models/mellum.py index 625bef1cf..a11de0d52 100644 --- a/python/sglang/srt/models/mellum.py +++ b/python/sglang/srt/models/mellum.py @@ -520,7 +520,7 @@ class MellumForCausalLM(Qwen3MoeForCausalLM): cfg.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(cfg) self.capture_aux_hidden_states = False diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 315adf08a..b2e188ade 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -1187,7 +1187,7 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) else: self.lm_head = PPMissingLayer() diff --git a/python/sglang/srt/models/mimo_v2_nextn.py b/python/sglang/srt/models/mimo_v2_nextn.py index 1eaa1f453..9ccc20e90 100644 --- a/python/sglang/srt/models/mimo_v2_nextn.py +++ b/python/sglang/srt/models/mimo_v2_nextn.py @@ -259,7 +259,7 @@ class MiMoV2MTP(MiMoV2ForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 5e9c0f42f..2993fa023 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -1574,7 +1574,7 @@ class MiniMaxM3SparseForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/minimax_m3_vl.py b/python/sglang/srt/models/minimax_m3_vl.py index 2fdff0370..ac2a437f3 100644 --- a/python/sglang/srt/models/minimax_m3_vl.py +++ b/python/sglang/srt/models/minimax_m3_vl.py @@ -123,7 +123,7 @@ class MiniMaxM3SparseForConditionalGeneration(nn.Module): text_config.hidden_size, quant_config=quant_config, prefix=add_prefix("language_model.lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) else: self.lm_head = PPMissingLayer() diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index 803589f33..210544b1f 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -964,7 +964,7 @@ class NemotronHForCausalLM(nn.Module): else lora_config.lora_vocab_padding_size ), quant_config=quant_config, - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/nemotron_h_mtp.py b/python/sglang/srt/models/nemotron_h_mtp.py index 3c375e90c..f569baf1c 100644 --- a/python/sglang/srt/models/nemotron_h_mtp.py +++ b/python/sglang/srt/models/nemotron_h_mtp.py @@ -339,7 +339,7 @@ class NemotronHForCausalLMMTP(NemotronHForCausalLM): self.config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index a259053ad..adc84684d 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -1119,7 +1119,7 @@ class Qwen2MoeForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) # For EAGLE3 support diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index 9fbd04670..c670129a4 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -492,7 +492,7 @@ class Qwen3ForCausalLM(nn.Module): config.vocab_size, config.hidden_size, quant_config=quant_config, - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/qwen3_5_text.py b/python/sglang/srt/models/qwen3_5_text.py index f045c2ce0..0963e766c 100644 --- a/python/sglang/srt/models/qwen3_5_text.py +++ b/python/sglang/srt/models/qwen3_5_text.py @@ -80,7 +80,7 @@ class Qwen3_5ForCausalLM(nn.Module): quant_config=quant_config, org_num_embeddings=config.vocab_size, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) else: self.lm_head = PPMissingLayer() diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index dac64b987..d5f5bd76f 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -961,7 +961,7 @@ class Qwen3MoeForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.capture_aux_hidden_states = False diff --git a/python/sglang/srt/models/qwen3_moe_mtp.py b/python/sglang/srt/models/qwen3_moe_mtp.py index e351fb4d7..fad314fe0 100644 --- a/python/sglang/srt/models/qwen3_moe_mtp.py +++ b/python/sglang/srt/models/qwen3_moe_mtp.py @@ -63,7 +63,7 @@ class Qwen3MoeForCausalLMMTP(Qwen3MoeForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index 3aca4ec7c..f429e9062 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -1027,7 +1027,7 @@ class Qwen3NextForCausalLM(nn.Module): quant_config=quant_config, org_num_embeddings=config.vocab_size, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) # For EAGLE3 support diff --git a/python/sglang/srt/models/qwen3_next_mtp.py b/python/sglang/srt/models/qwen3_next_mtp.py index dd36afb51..8c0cc2e95 100644 --- a/python/sglang/srt/models/qwen3_next_mtp.py +++ b/python/sglang/srt/models/qwen3_next_mtp.py @@ -80,7 +80,7 @@ class Qwen3NextForCausalLMMTP(Qwen3NextForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) # Mirror Qwen3NextForCausalLM.__init__'s shared-expert fusion setup so diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 176bb40e4..39f7aa8df 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -1316,7 +1316,7 @@ class Qwen3VLForConditionalGeneration(nn.Module): self.config.vocab_size, self.config.hidden_size, quant_config=quant_config, - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index 63373a9a2..13bc7a090 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -1220,7 +1220,7 @@ class SarvamMLAForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py index 3d5912d97..c75b7e9ed 100644 --- a/python/sglang/srt/models/sdar.py +++ b/python/sglang/srt/models/sdar.py @@ -469,7 +469,7 @@ class SDARForCausalLM(nn.Module): config.vocab_size, config.hidden_size, quant_config=quant_config, - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index 9ac15f3fe..b01bd7498 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -556,7 +556,7 @@ class SDARMoeForCausalLM(nn.Module): config.vocab_size, config.hidden_size, quant_config=quant_config, - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index c9220f2f3..85dc5aa4f 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -816,7 +816,7 @@ class Step3p5ForCausalLM(nn.Module): config.vocab_size, config.hidden_size, quant_config=quant_config, - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_parallel().config.enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/multimodal/transport/__init__.py b/python/sglang/srt/multimodal/transport/__init__.py index 3509b7400..1622a33f2 100644 --- a/python/sglang/srt/multimodal/transport/__init__.py +++ b/python/sglang/srt/multimodal/transport/__init__.py @@ -14,7 +14,7 @@ def determine_tensor_transport_mode() -> TensorTransportMode: inject the address only into scheduler actors for a multi-node deployment, and external launchers may use an environment-based rendezvous instead. """ - if get_parallel().nnodes > 1: + if get_parallel().config.nnodes > 1: # CUDA IPC and POSIX shared memory are local to one node. return "default" return "cuda_ipc" diff --git a/python/sglang/srt/ray/data_parallel_controller.py b/python/sglang/srt/ray/data_parallel_controller.py index 1d3d87b27..4bbd15f68 100644 --- a/python/sglang/srt/ray/data_parallel_controller.py +++ b/python/sglang/srt/ray/data_parallel_controller.py @@ -32,8 +32,6 @@ from sglang.srt.ray.engine import ( _resolve_bundle_indices, ) from sglang.srt.runtime_context import ( - configured_attn_cp_size, - configured_pp_size, get_parallel, ) from sglang.srt.server_args import PortArgs, ServerArgs @@ -79,7 +77,7 @@ class RayDataParallelController(DataParallelController): sockets = [] dp_port_args_list = [] - for dp_rank in range(get_parallel().dp_size): + for dp_rank in range(get_parallel().config.dp_size): tmp_port_args = PortArgs.init_new(server_args) tmp_port_args.tokenizer_ipc_name = port_args.tokenizer_ipc_name tmp_port_args.detokenizer_ipc_name = port_args.detokenizer_ipc_name @@ -103,7 +101,7 @@ class RayDataParallelController(DataParallelController): sock.close() # Create actors for each DP rank sequentially - for dp_rank in range(get_parallel().dp_size): + for dp_rank in range(get_parallel().config.dp_size): self._launch_ray_tp_group(server_args, dp_port_args_list[dp_rank], dp_rank) def launch_dp_attention_schedulers( @@ -114,7 +112,7 @@ class RayDataParallelController(DataParallelController): # rank-0 node IP instead of tcp://* to avoid exposing unauthenticated # ZMQ sockets (CVE-2026-3060). worker_ports = [] - for dp_rank in range(get_parallel().dp_size): + for dp_rank in range(get_parallel().config.dp_size): worker_port, worker_socket = get_zmq_socket_on_host( self.context, zmq.PUSH, host=self.rank0_node_ip ) @@ -149,7 +147,7 @@ class RayDataParallelController(DataParallelController): bundle_idx = self.bundle_for_node[node_idx] pp_range, tp_range, pp_per_node, tp_per_node = _calculate_rank_ranges( nnodes, - configured_pp_size(), + get_parallel().config.pp_size, server_args.tp_size, node_rank=node_idx, ) @@ -162,13 +160,13 @@ class RayDataParallelController(DataParallelController): tp_rank % tp_per_node ) - if get_parallel().enable_dp_attention: + if get_parallel().config.enable_dp_attention: _, _, actual_dp_rank, _ = compute_dp_attention_world_info( - get_parallel().enable_dp_attention, + get_parallel().config.enable_dp_attention, tp_rank, server_args.tp_size, - get_parallel().dp_size, - configured_attn_cp_size(), + get_parallel().config.dp_size, + get_parallel().config.attn_cp_size, ) rank_port_args = PortArgs.init_new( server_args, actual_dp_rank, worker_ports @@ -209,7 +207,7 @@ class RayDataParallelController(DataParallelController): world_size = _compute_world_size(server_args) bundle_indices = _resolve_bundle_indices(self.pg, world_size) - ranks_per_tp_group = server_args.tp_size * configured_pp_size() + ranks_per_tp_group = server_args.tp_size * get_parallel().config.pp_size if dp_rank is not None: start_rank = dp_rank * ranks_per_tp_group end_rank = start_rank + ranks_per_tp_group @@ -233,13 +231,13 @@ class RayDataParallelController(DataParallelController): bundle_idx = bundle_indices[global_rank] - if get_parallel().enable_dp_attention: + if get_parallel().config.enable_dp_attention: _, _, actual_dp_rank, _ = compute_dp_attention_world_info( - get_parallel().enable_dp_attention, + get_parallel().config.enable_dp_attention, tp_rank, server_args.tp_size, - get_parallel().dp_size, - configured_attn_cp_size(), + get_parallel().config.dp_size, + get_parallel().config.attn_cp_size, ) rank_port_args = PortArgs.init_new( server_args, actual_dp_rank, worker_ports diff --git a/python/sglang/srt/ray/engine.py b/python/sglang/srt/ray/engine.py index 4c807e88c..3dbf3e57b 100644 --- a/python/sglang/srt/ray/engine.py +++ b/python/sglang/srt/ray/engine.py @@ -32,7 +32,9 @@ from sglang.srt.entrypoints.engine import ( ) from sglang.srt.environ import envs from sglang.srt.ray.scheduler_actor import SchedulerActor -from sglang.srt.runtime_context import configured_pp_size, get_parallel +from sglang.srt.runtime_context import ( + get_parallel, +) from sglang.srt.server_args import PortArgs, ServerArgs logger = logging.getLogger(__name__) @@ -108,9 +110,13 @@ def _compute_world_size(server_args: ServerArgs) -> int: Normal: dp_size * tp_size * pp_size; DP attention: tp_size * pp_size. """ - if get_parallel().enable_dp_attention: - return server_args.tp_size * configured_pp_size() - return get_parallel().dp_size * server_args.tp_size * configured_pp_size() + if get_parallel().config.enable_dp_attention: + return server_args.tp_size * get_parallel().config.pp_size + return ( + get_parallel().config.dp_size + * server_args.tp_size + * get_parallel().config.pp_size + ) def _resolve_bundle_indices(pg: PlacementGroup, world_size: int) -> List[int]: @@ -268,11 +274,13 @@ class RayEngine(Engine): placement_group as create_placement_group, ) - if get_parallel().enable_dp_attention: - total_gpus = server_args.tp_size * configured_pp_size() + if get_parallel().config.enable_dp_attention: + total_gpus = server_args.tp_size * get_parallel().config.pp_size else: total_gpus = ( - get_parallel().dp_size * server_args.tp_size * configured_pp_size() + get_parallel().config.dp_size + * server_args.tp_size + * get_parallel().config.pp_size ) nnodes = server_args.nnodes @@ -314,7 +322,7 @@ class RayEngine(Engine): rank0_bundle_idx = int(indices_str.split(",")[0]) if indices_str else 0 rank0_node_ip = _get_bundle_node_ip(pg, rank0_bundle_idx) - if get_parallel().dp_size == 1: + if get_parallel().config.dp_size == 1: dist_init_addr = f"{rank0_node_ip}:{port_args.nccl_port}" logger.info(f"dist_init_addr: {dist_init_addr}") @@ -332,7 +340,7 @@ class RayEngine(Engine): pp_range, tp_range, pp_per_node, tp_per_node = ( _calculate_rank_ranges( nnodes, - configured_pp_size(), + get_parallel().config.pp_size, server_args.tp_size, node_rank=node_idx, ) @@ -447,19 +455,21 @@ class RayEngine(Engine): RayDataParallelController, ) - if get_parallel().enable_dp_attention: + if get_parallel().config.enable_dp_attention: # DP attention folds DP into TP — total GPUs = tp_size * pp_size - total_gpus = server_args.tp_size * configured_pp_size() + total_gpus = server_args.tp_size * get_parallel().config.pp_size else: total_gpus = ( - get_parallel().dp_size * server_args.tp_size * configured_pp_size() + get_parallel().config.dp_size + * server_args.tp_size + * get_parallel().config.pp_size ) gpus_per_node = total_gpus // server_args.nnodes logger.info( f"Ray DP cluster: {server_args.nnodes} nodes, " - f"{gpus_per_node} GPUs/node, dp_size={get_parallel().dp_size}, " - f"tp_size={server_args.tp_size}, pp_size={configured_pp_size()}, " - f"enable_dp_attention={get_parallel().enable_dp_attention}" + f"{gpus_per_node} GPUs/node, dp_size={get_parallel().config.dp_size}, " + f"tp_size={server_args.tp_size}, pp_size={get_parallel().config.pp_size}, " + f"enable_dp_attention={get_parallel().config.enable_dp_attention}" ) # Set dist_init_addr on server_args so PortArgs.init_new() can compute diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index eeed570c7..20fd37120 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -13,13 +13,16 @@ # ============================================================================== """A single structured accessor for process-static runtime state. -``get_parallel()`` returns a ``ParallelContext`` whose attributes — tp / dcp / pp / -moe / attn size and rank, plus the process-group handles — each delegate live to -the canonical getter in ``distributed.parallel_state`` / ``layers.dp_attention``. -Returned values are exactly what those getters return; this is a read-through -wrapper, not a cache. It gives call-sites one import and one naming scheme in -place of a dozen free functions, plus a test-only ``override()`` hook to force a -topology without monkeypatching the underlying getters. +``get_parallel()`` returns a ``ParallelContext`` whose bare attributes — tp / dcp +/ pp / moe / attn size and rank, plus the process-group handles — each delegate +live to the canonical getter in ``distributed.parallel_state`` / +``layers.dp_attention``. Returned values are exactly what those getters return; +this is a read-through wrapper, not a cache. It gives call-sites one import and +one naming scheme in place of a dozen free functions, plus a test-only +``override()`` hook to force a topology without monkeypatching the underlying +getters. The resolved parallel **configuration** is the same object's ``config`` +hop (``get_parallel().config.tp_size``), which reads the published ``parallel`` +bag: bare is the live group, ``config`` is what was configured. ``get_server_args()`` returns the process-wide ``ServerArgs``. This is the pristine / resolved-at-startup **read-only** record kept for debug and reproduction; business code reads resolved config from the @@ -114,14 +117,21 @@ _PARALLEL_FIELDS = frozenset( class ParallelContext: - """Parallel-topology namespace. + """Parallel-topology namespace: the live groups bare, configuration under + ``config``. - Live topology (size / rank / group) is read-through via ``@property`` (the - canonical getters). Parallel **config** leaves (``nccl_port``, - ``pp_max_micro_batch_size``, ``enable_dp_attention``, …) come from the - published ``parallel`` config bag via ``__getattr__``. Where a config leaf - shares a name with a live property (``tp_size`` …), the property (the live - fact) wins; the same-name==same-value invariant holds once dist is up. + ``get_parallel().tp_size`` and its size / rank / group siblings are + read-through ``@property`` over the canonical getters, so they answer with + the **live** process groups and raise before distributed init. The resolved + parallel **configuration** is one hop away, on the published bag: + ``get_parallel().config.tp_size``, ``.config.nccl_port``. It answers in any + process at any point after publish, and follows a post-publish ``override``. + + The two disagree by design, so which one a call site wants is spelled at the + call site — no ``config`` means live. Elastic EP scales the live world away + from the configured one, and ``initialize_model_parallel`` aliases + ``_MOE_DP`` to ``_ATTN_CP`` when ``attn_cp_size > moe_dp_size``, which makes + a live comparison of that pair degenerate. """ __slots__ = ("_overrides", "_config") @@ -130,28 +140,37 @@ class ParallelContext: self._overrides = {} self._config = None # parallel config bag, wired at publish + @property + def config(self) -> _ConfigBag: + """The published ``parallel`` config bag. + + Reads the slot directly: ``parallel`` sits outside the per-role + namespace table (every process reads topology config), so no role check + applies here. The body stays + dynamo-traceable — ``get_parallel().config.moe_dense_tp_size`` and the + gate helpers over it run inside compiled model forwards. + """ + config = self._config + if config is None: + raise ValueError("config namespace 'parallel' not published") + return config + def __getattr__(self, name): - # Reached only for names that are neither a live @property nor a slot: - # serve parallel config leaves from the published bag. The body must - # stay dynamo-traceable — config-leaf reads such as - # ``get_parallel().moe_dense_tp_size`` run inside compiled model - # forwards, and ``object.__getattribute__`` graph-breaks. + # Reached only for names that are neither a live @property nor a slot. + # Config leaves are read under ``config``, so naming one here is a + # call-site mistake and this only builds the error that says so. if name.startswith("_"): - # No config leaf is underscored; this also breaks the recursion - # when the ``_config`` slot itself is still unset (pickle/copy - # protocols probe attributes before __init__ runs). + # This also breaks the recursion when the ``_config`` slot itself is + # still unset (pickle/copy protocols probe attributes before + # __init__ runs). raise AttributeError(name) config = self._config - # ``_fields`` is a plain ``__dict__`` entry on the bag; ``in`` on the - # dict avoids ``_ConfigBag.__contains__`` (not traceable). if config is not None and name in config._fields: - return getattr(config, name) - detail = ( - "not a published parallel config leaf" - if config is not None - else "config not published" - ) - raise AttributeError(f"ParallelContext has no {name!r} ({detail})") + raise AttributeError( + f"{name!r} is a parallel config leaf, not live topology; read it " + f"as get_parallel().config.{name}" + ) + raise AttributeError(f"ParallelContext has no {name!r}") def _v(self, name, getter): overrides = self._overrides @@ -817,9 +836,9 @@ class RuntimeContext: # truth for config reads). Driven by NS(...) metadata; a mock/partial # config with no NS markers yields an empty tree (no bags projected). self._config_bags = _build_config_bags(server_args) - # Wire the parallel config leaves onto the live wrapper (config-only - # leaves like pp_max_micro_batch_size are served via ParallelContext - # __getattr__; live topology properties still win by name). + # Wire the published `parallel` bag onto the live wrapper: it is the slot + # the `config` property reads, which is how config-only leaves like + # pp_max_micro_batch_size are spelled. self.parallel._config = self._config_bags.get("parallel") # A direct install is roleless; ``publish`` assigns the role afterwards. self._overrides_log = [] @@ -1121,8 +1140,8 @@ def get_forward() -> ForwardFlags: # --- Resolved config namespaces ------------------------- # Each returns the top-level snapshot bag; reads are `get_exec().moe.field` etc. # All fail with ValueError("... not published") until publish has projected them. -# ``parallel`` config leaves are served by ``get_parallel()`` (live wrapper); -# their config-bag wiring is a scoped follow-up. +# ``parallel`` has no getter of its own: its bag is reached as +# ``get_parallel().config``, alongside the live topology it belongs to. def get_device() -> _ConfigBag: return _CONTEXT.config_bag("device") @@ -1173,8 +1192,8 @@ def get_observability() -> _ConfigBag: # table declares which top-level config namespaces each role reads. ``None`` # means the full tree — either the role genuinely needs everything (scheduler) # or its deployment shape has not been audited yet (restrict only what smoke -# coverage can verify). ``parallel`` is served by ``get_parallel()`` and every -# process legitimately reads topology config, so it is not part of this table. +# coverage can verify). ``parallel`` is served by ``get_parallel().config`` and +# every process legitimately reads topology config, so it is not in this table. # # ``SGLANG_ROLE_NAMESPACES`` selects the mode (read once at import): # off (default) no bookkeeping, zero overhead; @@ -1559,7 +1578,7 @@ def max_prefill_buffer_tokens() -> int: tokens = chunked if ( schedule.enable_dynamic_chunking - and _configured_parallel("pp_size") > 1 + and get_parallel().config.pp_size > 1 and chunked ): tokens = max( @@ -1591,10 +1610,9 @@ def pre_capture_activation_reserve_mb(gpu_mem: float | None) -> float: activation_tokens = max(schedule.chunked_prefill_size, 2048) else: activation_tokens = max(schedule.max_prefill_tokens, 2048) + parallel = get_parallel().config reserved_mem = ( - 512 - + activation_tokens * 1.5 - + _configured_parallel("tp_size") * _configured_parallel("pp_size") / 8 * 1024 + 512 + activation_tokens * 1.5 + parallel.tp_size * parallel.pp_size / 8 * 1024 ) if gpu_mem is not None and gpu_mem > 60 * 1024: reserved_mem = max(reserved_mem, 10 * 1024) @@ -1714,54 +1732,6 @@ def cutedsl_moe_max_num_tokens() -> int: return max(prefill_tokens, decode_max_bs * num_tokens_per_req) -# --- Configured (not live) parallel sizes ------------------------------------ -# -# ``get_parallel()`` shadows these names with the LIVE topology, which is the -# right answer almost everywhere. A handful of call sites need what was -# *configured* instead — before the groups exist, in a process that has none, -# or where the live value is deliberately aliased to another dimension. Each -# accessor below names that intent so no business call site has to reach for -# the startup record; the per-site reasons live in the read ratchet. -# -# They read the published leaf rather than the record: the bag is what -# ``override`` writes, and once the instance holds only the user's raw input -# the record would answer with what was *typed* instead of what resolution -# produced. Going through the bag directly is what gets past the live property -# that shadows these four names on ``get_parallel()``. - - -def _configured_parallel(name: str): - # The bag itself, not ParallelContext, whose live property shadows these - # four names. Read through the parallel slot the way the leaf accessor - # does — ``parallel`` is deliberately outside the per-role namespace table - # (every process reads topology config), so this must not route through - # ``config_bag()``'s role check, which would record or reject the read. - config = _CONTEXT.parallel._config - if config is None: - raise ValueError("config namespace 'parallel' not published") - return getattr(config, name) - - -def configured_tp_size() -> int: - return _configured_parallel("tp_size") - - -def configured_pp_size() -> int: - return _configured_parallel("pp_size") - - -def configured_moe_dp_size() -> int: - return _configured_parallel("moe_dp_size") - - -def configured_attn_cp_size() -> int: - return _configured_parallel("attn_cp_size") - - -def configured_dcp_size() -> int: - return _configured_parallel("dcp_size") - - def is_ep_joiner() -> bool: """True in a process launched as an elastic-EP joiner (scale or recover). diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index f200c34dc..53fe887c2 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -105,14 +105,14 @@ class DSparkWorkerV2(BaseSpecWorker): self._draft_is_moe = draft_is_deepseek_v4(server_args=server_args) self._draft_dp_context_enabled = ( - get_parallel().enable_dp_attention and not self._draft_is_moe + get_parallel().config.enable_dp_attention and not self._draft_is_moe ) self._is_pd_prefill = get_disagg().disaggregation_mode == "prefill" self._decode_graph_allowed = ( not get_exec().graph.disable_cuda_graph and not self._is_pd_prefill ) if ( - get_parallel().enable_dp_attention + get_parallel().config.enable_dp_attention and self._draft_is_moe and ps.attn_tp_size > 1 ): @@ -218,7 +218,7 @@ class DSparkWorkerV2(BaseSpecWorker): verify_num_draft_tokens=self.verify_num_draft_tokens, ) if ( - get_parallel().enable_dp_attention + get_parallel().config.enable_dp_attention and not self._draft_is_moe and self._verify_planner.is_compact_mode and self._decode_graph_allowed @@ -244,7 +244,8 @@ class DSparkWorkerV2(BaseSpecWorker): gamma=self.gamma, mask_token_id=self._mask_token_id, draft_block_spec_info=self._draft_block_spec_info, - dp_moe_sync=self._draft_is_moe and get_parallel().enable_dp_attention, + dp_moe_sync=self._draft_is_moe + and get_parallel().config.enable_dp_attention, ) self._verify_epilogue = None if ( @@ -446,7 +447,7 @@ class DSparkWorkerV2(BaseSpecWorker): self, batch: ScheduleBatch, on_publish ) -> GenerationBatchResult: if batch.forward_mode.is_idle(): - if get_parallel().enable_dp_attention: + if get_parallel().config.enable_dp_attention: self.target_worker.forward_batch_generation( batch, capture_hidden_mode=CaptureHiddenMode.FULL ) @@ -532,7 +533,7 @@ class DSparkWorkerV2(BaseSpecWorker): def _dp_verify_tier_num_tokens(self, batch: ScheduleBatch) -> Optional[int]: if not ( self._draft_is_moe - and get_parallel().enable_dp_attention + and get_parallel().config.enable_dp_attention and batch.global_num_tokens is not None and self._verify_planner.is_compact_mode ): @@ -576,7 +577,7 @@ class DSparkWorkerV2(BaseSpecWorker): if batch.forward_mode.is_idle(): self._observers.note_idle_decode_step() - if get_parallel().enable_dp_attention: + if get_parallel().config.enable_dp_attention: if self._draft_is_moe: self._proposer.run_idle_participation(batch) self._verify_executor.run_idle_participation( @@ -637,7 +638,7 @@ class DSparkWorkerV2(BaseSpecWorker): global_num_reqs = ( max(batch.global_num_tokens) if self._draft_is_moe - and get_parallel().enable_dp_attention + and get_parallel().config.enable_dp_attention and batch.global_num_tokens is not None else None ) diff --git a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py index 5ac5189ed..dc21efc40 100644 --- a/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py @@ -35,8 +35,8 @@ from sglang.srt.model_executor.runner_backend_utils import ( CUDA_GRAPH_CAPTURE_FAILED_MSG, ) from sglang.srt.runtime_context import ( - configured_pp_size, get_flags, + get_parallel, get_spec, ) from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo @@ -113,7 +113,7 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner): self.device_module = torch.get_device_module(self.device) self.tp_size = model_runner.ps.tp_size self.attn_dp_size = model_runner.ps.attn_dp_size - self.pp_size = configured_pp_size() + self.pp_size = get_parallel().config.pp_size self.enable_torch_compile = get_flags().capture.enable_torch_compile self.disable_padding = model_runner.server_args.disable_cuda_graph_padding self.require_gathered_buffer = require_gathered_buffer(model_runner.server_args) diff --git a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py index deede532a..3c4c63d7f 100644 --- a/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py @@ -36,8 +36,8 @@ from sglang.srt.model_executor.runner_backend_utils import ( CUDA_GRAPH_CAPTURE_FAILED_MSG, ) from sglang.srt.runtime_context import ( - configured_pp_size, get_flags, + get_parallel, get_spec, ) from sglang.srt.speculative.eagle_info import EagleDraftExtendInput @@ -99,7 +99,7 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): self.device_module = torch.get_device_module(self.device) self.tp_size = model_runner.ps.tp_size self.attn_dp_size = model_runner.ps.attn_dp_size - self.pp_size = configured_pp_size() + self.pp_size = get_parallel().config.pp_size self.enable_torch_compile = get_flags().capture.enable_torch_compile self.disable_padding = model_runner.server_args.disable_cuda_graph_padding self.require_gathered_buffer = require_gathered_buffer(model_runner.server_args) diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 9ad5e5275..17233ede8 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -160,7 +160,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): # Load draft model weights only. if ( - get_parallel().enable_dp_attention + get_parallel().config.enable_dp_attention and self.speculative_algorithm.is_eagle3() ): ctx = draft_tp_context(get_parallel().attn_tp_group) @@ -186,7 +186,9 @@ class EagleDraftWorker(EagleDraftWorkerBase): # Eager draft-extend seed buffer (graph paths use their own static ones). self.dsa_extend_topk_buf: Optional[torch.Tensor] = None self.draft_tp_context = ( - draft_tp_context if get_parallel().enable_dp_attention else empty_context + draft_tp_context + if get_parallel().config.enable_dp_attention + else empty_context ) self.tree_mask_mode = default_tree_mask_mode() diff --git a/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py b/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py index b86b8737d..3c7352035 100644 --- a/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/frozen_kv_mtp_cuda_graph_runner.py @@ -33,8 +33,8 @@ from sglang.srt.model_executor.runner_backend_utils import ( CUDA_GRAPH_CAPTURE_FAILED_MSG, ) from sglang.srt.runtime_context import ( - configured_pp_size, get_flags, + get_parallel, get_spec, ) from sglang.srt.speculative.frozen_kv_mtp_info import FrozenKVMTPDraftInput @@ -99,7 +99,7 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner): self.require_attn_tp_gather = require_attn_tp_gather(model_runner.server_args) self.tp_size = self.model_runner.ps.tp_size self.attn_dp_size = self.model_runner.ps.attn_dp_size - self.pp_size = configured_pp_size() + self.pp_size = get_parallel().config.pp_size self.speculative_num_steps = get_spec().speculative_num_steps self.topk = get_spec().speculative_eagle_topk self.draft_attn_backend = frozen_kv_mtp_worker.draft_attn_backend diff --git a/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py b/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py index 443ed9ed6..0d007507d 100644 --- a/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py +++ b/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py @@ -163,7 +163,9 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker): self.kv_context: Optional[FrozenKVMTPContext] = None self.draft_tp_context = ( - draft_tp_context if get_parallel().enable_dp_attention else empty_context + draft_tp_context + if get_parallel().config.enable_dp_attention + else empty_context ) self.draft_attn_backend = None diff --git a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py index 359bdfa4f..aa15c97da 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py @@ -60,7 +60,6 @@ from sglang.srt.model_executor.runner_backend_utils import ( CUDA_GRAPH_CAPTURE_FAILED_MSG, ) from sglang.srt.runtime_context import ( - configured_pp_size, get_flags, get_parallel, get_spec, @@ -155,8 +154,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner): self.device = model_runner.device self.device_module = torch.get_device_module(self.device) self.tp_size = model_runner.ps.tp_size - self.dp_size = get_parallel().dp_size - self.pp_size = configured_pp_size() + self.dp_size = get_parallel().config.dp_size + self.pp_size = get_parallel().config.pp_size self.enable_torch_compile = get_flags().capture.enable_torch_compile self.disable_padding = model_runner.server_args.disable_cuda_graph_padding self.require_gathered_buffer = require_gathered_buffer(model_runner.server_args) diff --git a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py index 057e7310c..ea75a3fd2 100644 --- a/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py +++ b/python/sglang/srt/speculative/multi_layer_eagle_worker_v2.py @@ -185,7 +185,9 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase): "InklingForConditionalGenerationMTP", ] self.draft_tp_context = ( - draft_tp_context if get_parallel().enable_dp_attention else empty_context + draft_tp_context + if get_parallel().config.enable_dp_attention + else empty_context ) self.tree_mask_mode = default_tree_mask_mode() self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device) diff --git a/python/sglang/srt/speculative/standalone_worker_v2.py b/python/sglang/srt/speculative/standalone_worker_v2.py index e8b3a11cd..82dd49fc5 100644 --- a/python/sglang/srt/speculative/standalone_worker_v2.py +++ b/python/sglang/srt/speculative/standalone_worker_v2.py @@ -94,7 +94,9 @@ class StandaloneDraftWorker(EagleDraftWorker): # Alias for better readability self.draft_runner = self.draft_worker.model_runner self.draft_tp_context = ( - draft_tp_context if get_parallel().enable_dp_attention else empty_context + draft_tp_context + if get_parallel().config.enable_dp_attention + else empty_context ) self.tree_mask_mode = default_tree_mask_mode() self.plan_stream, self.plan_stream_ctx = get_plan_stream(self.device) diff --git a/python/sglang/srt/state_capturer/routed_experts.py b/python/sglang/srt/state_capturer/routed_experts.py index 6d1accde1..15a200cd0 100644 --- a/python/sglang/srt/state_capturer/routed_experts.py +++ b/python/sglang/srt/state_capturer/routed_experts.py @@ -70,8 +70,8 @@ class RoutedExpertsCapturer(BaseTopkCapturer): # chunked_prefill_size. # FIXME: spec decoding's num_verify_tokens is still not accounted for. max_batch_size = max( - get_schedule().chunked_prefill_size * get_parallel().dp_size, - max_running_requests * get_parallel().dp_size, + get_schedule().chunked_prefill_size * get_parallel().config.dp_size, + max_running_requests * get_parallel().config.dp_size, ) super().__init__( diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 053f2cdb9..d1fd11a02 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -99,7 +99,6 @@ from sglang.srt.environ import envs from sglang.srt.observability.func_timer import enable_func_timer from sglang.srt.platforms import current_platform from sglang.srt.runtime_context import ( - configured_tp_size, get_exec, get_parallel, ) @@ -3728,8 +3727,8 @@ def require_mlp_tp_gather(server_args: ServerArgs): from sglang.srt.runtime_context import get_exec, get_parallel # elastic-EP scale-up rewrites dp_size on the published config - if get_parallel().enable_dp_attention: - assert get_parallel().dp_size > 1, "dp_size must be greater than 1" + if get_parallel().config.enable_dp_attention: + assert get_parallel().config.dp_size > 1, "dp_size must be greater than 1" if get_exec().moe.elastic_ep_backend is not None: from sglang.srt.elastic_ep.elastic_ep import ( elastic_expanded_world_enabled, @@ -3738,10 +3737,10 @@ def require_mlp_tp_gather(server_args: ServerArgs): if elastic_expanded_world_enabled(): return True if ( - get_parallel().moe_dense_tp_size is None + get_parallel().config.moe_dense_tp_size is None ): # TODO(ch-wan): some MoE models do not have dense layers return True - elif not get_parallel().enable_dp_lm_head: + elif not get_parallel().config.enable_dp_lm_head: return True elif get_moe_a2a_backend().is_none(): return True @@ -3757,8 +3756,8 @@ def require_mlp_tp_gather(server_args: ServerArgs): return True else: return ( - get_parallel().moe_dense_tp_size - > configured_tp_size() // get_parallel().dp_size + get_parallel().config.moe_dense_tp_size + > get_parallel().config.tp_size // get_parallel().config.dp_size ) else: return False @@ -3774,17 +3773,17 @@ def require_attn_tp_gather(server_args: ServerArgs): # autotuners to pick suboptimal kernel variants at small batches. from sglang.srt.runtime_context import get_parallel - if get_parallel().disable_attn_tp_gather: + if get_parallel().config.disable_attn_tp_gather: return False from sglang.srt.layers.moe.utils import get_moe_a2a_backend if ( not get_moe_a2a_backend().is_none() - or get_parallel().moe_dense_tp_size is not None + or get_parallel().config.moe_dense_tp_size is not None ): - if get_parallel().enable_dp_attention: - return get_parallel().dp_size < configured_tp_size() + if get_parallel().config.enable_dp_attention: + return get_parallel().config.dp_size < get_parallel().config.tp_size else: return True else: @@ -3798,7 +3797,9 @@ def require_gathered_buffer(server_args: ServerArgs): def require_mlp_sync(server_args: ServerArgs): from sglang.srt.runtime_context import get_parallel - return get_parallel().enable_dp_attention or require_gathered_buffer(server_args) + return get_parallel().config.enable_dp_attention or require_gathered_buffer( + server_args + ) def get_cuda_graph_batch_size_alignment(server_args: ServerArgs) -> int: diff --git a/python/sglang/srt/utils/cuda_vmm_transport_utils.py b/python/sglang/srt/utils/cuda_vmm_transport_utils.py index 3430fa711..8525e5812 100644 --- a/python/sglang/srt/utils/cuda_vmm_transport_utils.py +++ b/python/sglang/srt/utils/cuda_vmm_transport_utils.py @@ -16,7 +16,6 @@ from sglang.srt.managers.schedule_batch import ( MultimodalProcessorOutput, ) from sglang.srt.runtime_context import ( - configured_tp_size, get_mm, get_parallel, ) @@ -163,9 +162,9 @@ def _contains_tensor_container(value) -> bool: def get_vmm_feature_consumer_count(server_args) -> int: - if get_parallel().enable_dp_attention: - return configured_tp_size() // get_parallel().dp_size - return configured_tp_size() + if get_parallel().config.enable_dp_attention: + return get_parallel().config.tp_size // get_parallel().config.dp_size + return get_parallel().config.tp_size class CudaVmmMemoryPool: diff --git a/python/sglang/srt/utils/offloader.py b/python/sglang/srt/utils/offloader.py index 56b006a52..63c4a8954 100644 --- a/python/sglang/srt/utils/offloader.py +++ b/python/sglang/srt/utils/offloader.py @@ -81,7 +81,7 @@ def create_offloader_from_server_args(server_args: ServerArgs, dp_rank: int): prefetch_step=get_exec().offload.offload_prefetch_step, mode=get_exec().offload.offload_mode, dp_rank=dp_rank, - dp_size=get_parallel().dp_size, + dp_size=get_parallel().config.dp_size, ) return NoopOffloader() diff --git a/python/sglang/srt/weight_cache/ipc_loader.py b/python/sglang/srt/weight_cache/ipc_loader.py index 5e6c942bf..ddc54bed7 100644 --- a/python/sglang/srt/weight_cache/ipc_loader.py +++ b/python/sglang/srt/weight_cache/ipc_loader.py @@ -492,7 +492,7 @@ class IpcModelLoader(BaseModelLoader): moe_dp_rank = ps.moe_dp_rank moe_ep_rank = ps.moe_ep_rank - dp_size = ps.dp_size + dp_size = get_parallel().config.dp_size quant_method, quant_config = self._resolve_engine_quant(model_config) @@ -512,10 +512,10 @@ class IpcModelLoader(BaseModelLoader): moe_dp_size=moe_dp_size, moe_dp_rank=moe_dp_rank, moe_ep_rank=moe_ep_rank, - enable_dp_attention=ps.enable_dp_attention, - enable_dp_lm_head=ps.enable_dp_lm_head, + enable_dp_attention=ps.config.enable_dp_attention, + enable_dp_lm_head=ps.config.enable_dp_lm_head, attn_cp_size=ps.attn_cp_size, - moe_dense_tp_size=ps.moe_dense_tp_size, + moe_dense_tp_size=ps.config.moe_dense_tp_size, moe_a2a_backend=get_exec().moe.moe_a2a_backend, quant_method=quant_method, quant_config_hash=hash_quant_config(quant_config), diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index e65364289..229fad519 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -158,8 +158,12 @@ class TestCPReplicatedStateTransfer(unittest.TestCase): manager = object.__new__(CommonKVManager) manager.attn_cp_size = cp_size manager.attn_cp_rank = cp_rank + # The policy reads the configured tier, so the stand-in + # carries the leaf under `config`, where the bag serves it. parallel = SimpleNamespace( - enable_dsa_cache_layer_split=layer_split, + config=SimpleNamespace( + enable_dsa_cache_layer_split=layer_split, + ), ) with patch( "sglang.srt.disaggregation.common.conn.get_parallel", @@ -178,7 +182,9 @@ class TestCPReplicatedStateTransfer(unittest.TestCase): with patch( "sglang.srt.disaggregation.common.conn.get_parallel", - return_value=SimpleNamespace(enable_dsa_cache_layer_split=False), + return_value=SimpleNamespace( + config=SimpleNamespace(enable_dsa_cache_layer_split=False) + ), ): self.assertEqual( manager._get_dsa_cache_transfer_skip_flags(None), diff --git a/test/registered/unit/model_loader/test_presharded_loader.py b/test/registered/unit/model_loader/test_presharded_loader.py index d1ff489e5..d293a2a90 100644 --- a/test/registered/unit/model_loader/test_presharded_loader.py +++ b/test/registered/unit/model_loader/test_presharded_loader.py @@ -803,18 +803,20 @@ class TestShardConfig(unittest.TestCase): "init_expert_location", "structural_signature", } + # Both tiers on one stand-in: bare names are the live groups, `config` + # is the published parallel bag. parallel = SimpleNamespace( tp_size=8, moe_dp_size=2, moe_ep_size=4, pp_size=1, - moe_dense_tp_size=1, - enable_dp_lm_head=True, + config=SimpleNamespace( + moe_dp_size=2, + moe_dense_tp_size=1, + enable_dp_lm_head=True, + ), ) with mock.patch( - "sglang.srt.model_loader.loader.configured_moe_dp_size", - return_value=2, - ), mock.patch( "sglang.srt.model_loader.loader.get_parallel", return_value=parallel, ), mock.patch( diff --git a/test/registered/unit/models/test_kimi_k25.py b/test/registered/unit/models/test_kimi_k25.py index 88a7e19a6..dbb2046b2 100644 --- a/test/registered/unit/models/test_kimi_k25.py +++ b/test/registered/unit/models/test_kimi_k25.py @@ -66,7 +66,13 @@ from sglang.srt.multimodal.transport.cuda_ipc import ( DEFER_CUDA_IPC_FEATURE_RECONSTRUCTION_KEY, CudaIpcTensorTransportProxy, ) -from sglang.srt.runtime_context import get_context, get_parallel, publish, reset_context +from sglang.srt.runtime_context import ( + ParallelContext, + get_context, + get_parallel, + publish, + reset_context, +) from sglang.srt.server_args import ServerArgs from sglang.srt.utils import ImageData from sglang.test.ci.ci_register import register_cpu_ci @@ -903,7 +909,7 @@ def test_kimi_k3_normal_cache_path_connects_real_producer_to_model_consumer(): hot_items = pickle.loads(pickle.dumps(hot.mm_items)) with ( - patch("sglang.srt.models.kimi_k3.configured_tp_size", return_value=1), + patch.object(ParallelContext, "config", SimpleNamespace(tp_size=1)), patch( "sglang.srt.multimodal.processors.kimi_k25._gpu_preprocess_images", return_value=( @@ -967,7 +973,7 @@ def test_kimi_k3_model_accepts_mixed_cached_eager_and_deferred_artifacts(): ) with ( - patch("sglang.srt.models.kimi_k3.configured_tp_size", return_value=1), + patch.object(ParallelContext, "config", SimpleNamespace(tp_size=1)), patch( "sglang.srt.multimodal.processors.kimi_k25._gpu_preprocess_images", return_value=(torch.full((1, 3), 2.0), torch.tensor([[1, 1, 1]])), diff --git a/test/registered/unit/multimodal/test_tensor_transport_mode.py b/test/registered/unit/multimodal/test_tensor_transport_mode.py index efa2a32bf..d7e1169c0 100644 --- a/test/registered/unit/multimodal/test_tensor_transport_mode.py +++ b/test/registered/unit/multimodal/test_tensor_transport_mode.py @@ -22,9 +22,13 @@ class TestTensorTransportMode(CustomTestCase): for nnodes, dist_init_addr, expected in cases: with self.subTest(nnodes=nnodes, dist_init_addr=dist_init_addr): + # `nnodes` is a config-only leaf, so the stand-in carries it + # under `config`, where the published bag serves it. parallel = SimpleNamespace( - nnodes=nnodes, - dist_init_addr=dist_init_addr, + config=SimpleNamespace( + nnodes=nnodes, + dist_init_addr=dist_init_addr, + ), ) with patch( "sglang.srt.multimodal.transport.get_parallel", diff --git a/test/registered/unit/server_args/test_resolution_declarations.py b/test/registered/unit/server_args/test_resolution_declarations.py index 6bc6eef12..292581161 100644 --- a/test/registered/unit/server_args/test_resolution_declarations.py +++ b/test/registered/unit/server_args/test_resolution_declarations.py @@ -396,9 +396,10 @@ class TestResolutionDeclarations(CustomTestCase): mapping = namespace_of(ServerArgs) self.assertGreater(len(mapping), 400, "the namespace mapping collapsed") - shadowed = _live_topology_leaves() + # The five sizes keep a live property shadowing the bare name; the + # comparison below reaches them anyway, through `get_parallel().config`. self.assertGreaterEqual( - shadowed + _live_topology_leaves() & { "tp_size", "pp_size", @@ -407,8 +408,7 @@ class TestResolutionDeclarations(CustomTestCase): "dcp_size", }, {"tp_size", "pp_size", "moe_dp_size", "attn_cp_size", "dcp_size"}, - "a parallel size stopped being served from the live topology; if it " - "is a plain config leaf now, it belongs in the comparison below", + "a parallel size stopped being served from the live topology", ) compared = 0 @@ -418,17 +418,16 @@ class TestResolutionDeclarations(CustomTestCase): server_args = self._resolve(shape) publish(server_args, role="scheduler") for field, path in mapping.items(): - if field in shadowed: - # Served from the process groups by design; `configured_*()` - # is what answers with the configured value, and - # test_launch_path_reads_configured_sizes pins that. - continue groups = path.split(".") accessor = getattr(runtime_context, f"get_{groups[0]}", None) if accessor is None: unreachable.append(f"no get_{groups[0]}() for {path}.{field}") continue node = accessor() + if groups[0] == "parallel": + # Bare names there are the live topology; the published + # leaves are one hop down, so the reader takes that hop. + node = node.config try: for group in groups[1:]: node = getattr(node, group) diff --git a/test/registered/unit/server_args/test_resolution_is_reproducible.py b/test/registered/unit/server_args/test_resolution_is_reproducible.py index f5decb963..0badd493b 100644 --- a/test/registered/unit/server_args/test_resolution_is_reproducible.py +++ b/test/registered/unit/server_args/test_resolution_is_reproducible.py @@ -844,7 +844,7 @@ class TestACopyStaysResolved(_RestoresProcessState, CustomTestCase): self.addCleanup(reset_context) reset_context() publish(copy_, role="scheduler") - self.assertEqual(get_parallel().dist_init_addr, "1.2.3.4:5000") + self.assertEqual(get_parallel().config.dist_init_addr, "1.2.3.4:5000") self.assertEqual( get_schedule().chunked_prefill_size, parent.chunked_prefill_size, diff --git a/test/registered/unit/server_args/test_resolution_reads_no_bag.py b/test/registered/unit/server_args/test_resolution_reads_no_bag.py index 2d4faf5d2..63bed4ecf 100644 --- a/test/registered/unit/server_args/test_resolution_reads_no_bag.py +++ b/test/registered/unit/server_args/test_resolution_reads_no_bag.py @@ -53,8 +53,7 @@ def _accessor_names(): names = { node.name for node in tree.body - if isinstance(node, ast.FunctionDef) - and (node.name.startswith("get_") or node.name.startswith("configured_")) + if isinstance(node, ast.FunctionDef) and node.name.startswith("get_") } # The context object itself is not a bag: it exists before anything is # published, and `declare_late_resolution` calls it deliberately to find @@ -232,7 +231,7 @@ class TestResolutionReadsNoBag(CustomTestCase): """A shrunken accessor set would make every other check pass quietly.""" self.assertGreaterEqual( len(_BAG_ACCESSORS), - 20, + 15, f"only {len(_BAG_ACCESSORS)} accessors were derived from " "runtime_context; the derivation broke", ) diff --git a/test/registered/unit/test_global_config_read_ratchet.py b/test/registered/unit/test_global_config_read_ratchet.py index a155b972c..6b840f970 100644 --- a/test/registered/unit/test_global_config_read_ratchet.py +++ b/test/registered/unit/test_global_config_read_ratchet.py @@ -12,10 +12,10 @@ slot. The reads that remain live in ``runtime_context.py`` (exempt by module): the ``@property`` / method members computed from several fields plus the HF config, -which are not namespace leaves and have no home but ``ServerArgs``, and the -``configured_*_size()`` accessors for the sizes ``get_parallel()`` shadows with -the live topology. ``_CONFIGURED_SIZE_CALL_SITES`` registers every one of the -latter with the reason the live property cannot serve it. +which are not namespace leaves and have no home but ``ServerArgs``. +Separately, ``_CONFIGURED_SIZE_CALL_SITES`` registers every business read of +``get_parallel().config.`` — the config tier of a size whose bare name is +the live topology — with the reason the live property cannot serve it. What the scan sees: ``get_server_args().field``, an alias (``sa = get_server_args()`` then ``sa.field`` -- function-local, module-level, or parked @@ -48,133 +48,134 @@ _PACKAGE_ROOT = Path(next(iter(sglang.__path__))) # resolution pipeline. _SLOT_OWNERS = ("srt/runtime_context.py", "srt/server_args.py", "srt/arg_groups/") -# Every call site of a ``configured_*_size()`` accessor, with the reason the -# live topology cannot answer there. The test below asserts this map is exactly -# the set of call sites, so the reasons cannot drift away from the code. +# Every configured read of a live-shadowed size (``get_parallel().config.pp_size`` +# and its four siblings), with the reason the live topology cannot answer there. +# The test below asserts this map is exactly the set of such reads, so the +# reasons cannot drift away from the code. _CONFIGURED_SIZE_CALL_SITES = { - ("srt/entrypoints/engine.py", "configured_pp_size"): ( + ("srt/entrypoints/engine.py", "pp_size"): ( "the launch path decides how many scheduler processes to spawn; it runs " "before any of them exists, so there is no group to ask" ), - ("srt/entrypoints/engine.py", "configured_attn_cp_size"): ( + ("srt/entrypoints/engine.py", "attn_cp_size"): ( "the launcher's per-TP-rank layout, computed while deciding what to " "spawn -- the groups it is laying out do not exist yet" ), - ("srt/entrypoints/engine.py", "configured_moe_dp_size"): ( + ("srt/entrypoints/engine.py", "moe_dp_size"): ( "the MoE factor of that same pre-spawn layout" ), - ("srt/ray/engine.py", "configured_pp_size"): ( + ("srt/ray/engine.py", "pp_size"): ( "the Ray driver sizes the actor placement group; the actors it is about " "to create are the ones that will hold the process groups" ), - ("srt/ray/data_parallel_controller.py", "configured_pp_size"): ( + ("srt/ray/data_parallel_controller.py", "pp_size"): ( "same placement arithmetic on the DP path -- ranks per TP group, " "computed in the driver before the actors start" ), - ("srt/ray/data_parallel_controller.py", "configured_attn_cp_size"): ( + ("srt/ray/data_parallel_controller.py", "attn_cp_size"): ( "the attention-CP factor of that same placement arithmetic, and the one " "size whose live value cannot express the configured intent when " "attn_cp_size > moe_dp_size aliases the groups" ), - ("srt/layers/attention/dsa/dsa_indexer.py", "configured_pp_size"): ( + ("srt/layers/attention/dsa/dsa_indexer.py", "pp_size"): ( "gates `pp_size > 1 and not get_pp_group()...`; the short circuit is the " "point, since with PP off the group is never touched, which is what lets " "the Indexer be constructed before distributed init" ), - ("srt/managers/scheduler.py", "configured_pp_size"): ( + ("srt/managers/scheduler.py", "pp_size"): ( "dispatch_event_loop picks the PP event loop; the MLX runner stub never " "initializes torch.distributed, so the live property asserts before the " "MLX loop can start -- the configured leaf answers the same value " "wherever the live groups exist" ), - ("srt/mem_cache/kv_cache_configurator.py", "configured_pp_size"): ( + ("srt/mem_cache/kv_cache_configurator.py", "pp_size"): ( "decides whether the token capacity needs a cross-PP all-reduce at all; " "asking the configured size keeps that decision independent of whether a " "PP group is installed in this process" ), - ("srt/layers/dp_attention.py", "configured_attn_cp_size"): ( + ("srt/layers/dp_attention.py", "attn_cp_size"): ( "compared against the configured moe_dp_size below" ), - ("srt/layers/dp_attention.py", "configured_moe_dp_size"): ( + ("srt/layers/dp_attention.py", "moe_dp_size"): ( "the configuration this predicate detects (attn_cp_size > moe_dp_size) is " "the one where initialize_model_parallel aliases _MOE_DP to _ATTN_CP, so " "the live sizes are equal there and a live comparison is always false" ), - ("srt/managers/scheduler.py", "configured_tp_size"): ( + ("srt/managers/scheduler.py", "tp_size"): ( "configure_scheduler_process runs before the scheduler's own process " "groups exist -- configuring the process is what it is for -- so there " "is nothing live to ask yet" ), - ("srt/managers/scheduler.py", "configured_moe_dp_size"): ( + ("srt/managers/scheduler.py", "moe_dp_size"): ( "same pre-distributed-init arithmetic in configure_scheduler_process" ), - ("srt/managers/scheduler.py", "configured_attn_cp_size"): ( + ("srt/managers/scheduler.py", "attn_cp_size"): ( "same pre-distributed-init arithmetic in configure_scheduler_process" ), - ("srt/managers/scheduler.py", "configured_dcp_size"): ( + ("srt/managers/scheduler.py", "dcp_size"): ( "same pre-distributed-init arithmetic in configure_scheduler_process" ), - ("srt/model_executor/runner/base_runner.py", "configured_pp_size"): ( + ("srt/model_executor/runner/base_runner.py", "pp_size"): ( "the runner's layer window is arithmetic over the configured stage " "count; a draft runner shares the target's groups, so the live " "property would answer for the wrong runner" ), - ("srt/model_executor/cpu_graph_runner.py", "configured_pp_size"): ( + ("srt/model_executor/cpu_graph_runner.py", "pp_size"): ( "the same window, on the CPU graph path" ), ( "srt/managers/scheduler_components/metrics_reporter.py", - "configured_pp_size", + "pp_size", ): ( "the reporter labels its metrics with the stage count it was launched " "with, which is configuration; the live group answers per process" ), - ("srt/speculative/eagle_draft_cuda_graph_runner.py", "configured_pp_size"): ( + ("srt/speculative/eagle_draft_cuda_graph_runner.py", "pp_size"): ( "the draft runner's window over the target's stages: its own groups are " "the target's, so the configured count is the one that describes it" ), ( "srt/speculative/eagle_draft_extend_cuda_graph_runner.py", - "configured_pp_size", + "pp_size", ): ("the same draft window, on the extend path"), ( "srt/speculative/multi_layer_eagle_draft_extend_cuda_graph_runner.py", - "configured_pp_size", + "pp_size", ): ("the same draft window, multi-layer extend"), - ("srt/speculative/frozen_kv_mtp_cuda_graph_runner.py", "configured_pp_size"): ( + ("srt/speculative/frozen_kv_mtp_cuda_graph_runner.py", "pp_size"): ( "the same draft window, frozen-KV MTP" ), - ("srt/managers/data_parallel_controller.py", "configured_pp_size"): ( + ("srt/managers/data_parallel_controller.py", "pp_size"): ( "the controller lays out its schedulers' ranks before spawning them, so " "the groups it is sizing for do not exist yet" ), - ("srt/managers/data_parallel_controller.py", "configured_attn_cp_size"): ( + ("srt/managers/data_parallel_controller.py", "attn_cp_size"): ( "the same pre-spawn rank arithmetic" ), - ("srt/managers/data_parallel_controller.py", "configured_moe_dp_size"): ( + ("srt/managers/data_parallel_controller.py", "moe_dp_size"): ( "the same pre-spawn rank arithmetic" ), - ("srt/entrypoints/v1_loads.py", "configured_pp_size"): ( + ("srt/entrypoints/v1_loads.py", "pp_size"): ( "the /v1/loads accelerator count is arithmetic over the launch shape, " "reported from the tokenizer process, which holds no model groups" ), - ("srt/disaggregation/common/conn.py", "configured_pp_size"): ( + ("srt/disaggregation/common/conn.py", "pp_size"): ( "the bootstrap connection is built by the KV manager on the transfer " "path, which the CPU-only conn tests exercise without ever starting " "torch.distributed" ), - ("srt/elastic_ep/elastic_ep.py", "configured_tp_size"): ( + ("srt/elastic_ep/elastic_ep.py", "tp_size"): ( "the joiner's rank window is computed against the size the process was " "configured with, not the size of the group it is about to join" ), - ("srt/elastic_ep/expert_backup_manager.py", "configured_tp_size"): ( + ("srt/elastic_ep/expert_backup_manager.py", "tp_size"): ( "the backup server counts the clients it expects to report in, which " "is how many the launch configured -- the live group is what they are " "still joining" ), ( "srt/model_executor/model_runner_components/startup_weight_load.py", - "configured_tp_size", + "tp_size", ): ( "the load options are assembled in ModelRunner.__init__ for a runner " "that may be a draft, whose groups are the target's; the configured " @@ -182,51 +183,51 @@ _CONFIGURED_SIZE_CALL_SITES = { ), ( "srt/model_executor/model_runner_components/startup_weight_load.py", - "configured_pp_size", + "pp_size", ): ("same options object, same reason"), ( "srt/model_executor/model_runner_components/startup_weight_load.py", - "configured_attn_cp_size", + "attn_cp_size", ): ("same options object, same reason"), ( "srt/model_executor/model_runner_components/startup_weight_load.py", - "configured_dcp_size", + "dcp_size", ): ("same options object, same reason"), ( "srt/model_executor/model_runner_components/spec_aux_hidden_state.py", - "configured_tp_size", + "tp_size", ): ( "the draft KV bytes/token estimate sizes the memory pool before the " "draft runner exists, so its shard count is configuration" ), - ("srt/eplb/expert_location.py", "configured_tp_size"): ( + ("srt/eplb/expert_location.py", "tp_size"): ( "the elastic-EP joiner window, used to size the expert layout: the " "size the process was configured with, not the group it is joining" ), - ("srt/utils/cuda_vmm_transport_utils.py", "configured_tp_size"): ( + ("srt/utils/cuda_vmm_transport_utils.py", "tp_size"): ( "the consumer count is configured fan-out arithmetic (tp_size // " "dp_size), which is what the record answered before" ), - ("srt/disaggregation/encoder/runtime.py", "configured_tp_size"): ( + ("srt/disaggregation/encoder/runtime.py", "tp_size"): ( "the encode server's launch entry sizes its workers before it has " "spawned any of them" ), - ("srt/utils/common.py", "configured_tp_size"): ( + ("srt/utils/common.py", "tp_size"): ( "the require_*_tp_gather predicates compared the configured tp_size " "when they read the record; the live property answers a different " "question wherever the groups alias, so the configured accessor is the " "mechanical substitution and the live one would be a semantic change" ), - ("srt/model_loader/loader.py", "configured_moe_dp_size"): ( + ("srt/model_loader/loader.py", "moe_dp_size"): ( "the same dict already carries the live moe_dp_size under 'dp'; this entry " "is the configured intent" ), - ("srt/models/kimi_k25.py", "configured_tp_size"): ( + ("srt/models/kimi_k25.py", "tp_size"): ( "the IPC refcount must match the configured TP consumer count captured " "when the tokenizer creates MmItemMemoryPool; a live attention subgroup " "size could strand leases in the bounded pool" ), - ("srt/models/kimi_k3.py", "configured_tp_size"): ( + ("srt/models/kimi_k3.py", "tp_size"): ( "same as kimi_k25: the IPC refcount must agree with the recycler's waiter" ), } @@ -533,69 +534,236 @@ class TestGlobalConfigReadRatchet(CustomTestCase): self._check("alias-form", alias, _ALIAS_BASELINE) +def _live_shadowed_sizes() -> frozenset: + """Names that are BOTH a live ``ParallelContext`` property and a ``parallel`` + config leaf. + + Derived from the two sides themselves: a size that gains a live property, or + a live property that gains a leaf, joins the registry's subject set without a + list here. + """ + from sglang.srt.arg_groups.arg_utils import namespace_of + from sglang.srt.runtime_context import ParallelContext + from sglang.srt.server_args import ServerArgs + + live = { + name + for name, value in vars(ParallelContext).items() + if isinstance(value, property) + } + leaves = { + field for field, path in namespace_of(ServerArgs).items() if path == "parallel" + } + shadowed = frozenset(live & leaves) + assert shadowed, "no live-shadowed size found; the derivation is broken" + return shadowed + + +def _parallel_config_reads(tree, subjects): + """Names in ``subjects`` read through the parallel bag's ``config`` hop. + + Sees ``get_parallel().config.pp_size``, the module-qualified spelling, a + local bound to either hop (``p = get_parallel()`` / ``cfg = p.config``), and + the ``getattr`` form of each. + """ + fns, modules = set(), set() + for node in ast.walk(tree): + if ( + isinstance(node, ast.ImportFrom) + and node.module + and node.module.endswith("runtime_context") + ): + fns |= {a.asname or a.name for a in node.names if a.name == "get_parallel"} + elif isinstance(node, ast.ImportFrom) and node.module: + # `from sglang.srt import runtime_context as rc` binds the module. + for a in node.names: + if f"{node.module}.{a.name}".endswith("runtime_context"): + modules.add(a.asname or a.name) + elif isinstance(node, ast.Import): + for a in node.names: + if a.name.endswith("runtime_context"): + # Unaliased, the call site spells the whole dotted path. + modules.add(a.asname or a.name) + + def dotted(node): + parts = [] + while isinstance(node, ast.Attribute): + parts.append(node.attr) + node = node.value + if not isinstance(node, ast.Name): + return None + parts.append(node.id) + return ".".join(reversed(parts)) + + def is_bag_call(node): + if not isinstance(node, ast.Call): + return False + func = node.func + if isinstance(func, ast.Name): + return func.id in fns + return ( + isinstance(func, ast.Attribute) + and func.attr == "get_parallel" + and dotted(func.value) in modules + ) + + bag_aliases, config_aliases = set(), set() + for _ in range(2): # a local copy of a local is still the same object + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + value = node.value + if is_bag_call(value) or ( + isinstance(value, ast.Name) and value.id in bag_aliases + ): + bucket = bag_aliases + elif ( + isinstance(value, ast.Attribute) + and value.attr == "config" + and ( + is_bag_call(value.value) + or ( + isinstance(value.value, ast.Name) + and value.value.id in bag_aliases + ) + ) + ) or (isinstance(value, ast.Name) and value.id in config_aliases): + bucket = config_aliases + else: + continue + bucket |= {t.id for t in node.targets if isinstance(t, ast.Name)} + + def is_config_hop(node): + return ( + isinstance(node, ast.Attribute) + and node.attr == "config" + and ( + is_bag_call(node.value) + or (isinstance(node.value, ast.Name) and node.value.id in bag_aliases) + ) + ) or (isinstance(node, ast.Name) and node.id in config_aliases) + + found = set() + for node in ast.walk(tree): + if isinstance(node, ast.Attribute) and node.attr in subjects: + base, name = node.value, node.attr + elif ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "getattr" + and len(node.args) >= 2 + and isinstance(node.args[1], ast.Constant) + and node.args[1].value in subjects + ): + base, name = node.args[0], node.args[1].value + else: + continue + if is_config_hop(base): + found.add(name) + return found + + +_READ_SPELLINGS = ( + "from sglang.srt.runtime_context import get_parallel\nx = get_parallel().config.tp_size", + "from sglang.srt.runtime_context import get_parallel as gp\nx = gp().config.tp_size", + "from sglang.srt import runtime_context as rc\nx = rc.get_parallel().config.tp_size", + "import sglang.srt.runtime_context\nx = sglang.srt.runtime_context.get_parallel().config.tp_size", + "from sglang.srt.runtime_context import get_parallel\np = get_parallel()\nx = p.config.tp_size", + "from sglang.srt.runtime_context import get_parallel\nc = get_parallel().config\nx = c.tp_size", + 'from sglang.srt.runtime_context import get_parallel\nx = getattr(get_parallel().config, "tp_size")', +) + + +class TestParallelConfigReadSpellings(CustomTestCase): + """``_parallel_config_reads`` resolves every spelling it claims to. + + The scan below decides what the documented set is compared against, so a + spelling it cannot resolve does not fail anything -- it drops the read. + """ + + def test_every_documented_spelling_resolves(self): + for source in _READ_SPELLINGS: + with self.subTest(source=source): + found = _parallel_config_reads(ast.parse(source), {"tp_size"}) + self.assertEqual({"tp_size"}, set(found)) + + def test_the_live_property_is_not_a_config_read(self): + source = ( + "from sglang.srt.runtime_context import get_parallel\n" + "x = get_parallel().tp_size" + ) + self.assertEqual( + set(), set(_parallel_config_reads(ast.parse(source), {"tp_size"})) + ) + + class TestConfiguredSizeCallSites(CustomTestCase): """The configured-vs-live exceptions are enumerated, with reasons. - ``configured_*_size()`` answers what the user asked for where - ``get_parallel()`` would answer what the process ended up with. Each such - exception is listed above with why the live property cannot serve it, and - this case fails if the code and that list disagree. + ``get_parallel().config.tp_size`` answers what the process was configured + with where the bare ``get_parallel().tp_size`` answers what the process ended + up with. Each site that needs the former is listed above with why the live + property cannot serve it, and this case fails if the code and that list + disagree. - The unit is **(file, accessor)**, not the individual call: a second - `configured_pp_size()` in a file already registered for it collapses into - the same entry, so the reason has to cover the file's use of that accessor - rather than one line. A new file, or a new accessor in a listed file, is - what this catches -- in either call form (bare or module-qualified). + The unit is **(file, size)**, not the individual read: a second + ``.config.pp_size`` in a file already registered for it collapses into the + same entry, so the reason has to cover the file's use of that size rather + than one line. A new file, or a new size in a listed file, is what this + catches -- through any spelling of the hop. """ def test_the_call_sites_match_the_documented_set(self): + subjects = _live_shadowed_sizes() found = set() + scanned = 0 for path in sorted(_PACKAGE_ROOT.rglob("*.py")): rel = path.relative_to(_PACKAGE_ROOT).as_posix() if rel.startswith(_SLOT_OWNERS): continue source = path.read_text() - if "configured_" not in source: + # Every spelling `_parallel_config_reads` resolves -- the direct + # call, an aliased import, a module-qualified call, a local bound to + # either hop -- needs the name in the source, so skipping the rest is + # free. Filtering on anything narrower silently empties the scan. + if "get_parallel" not in source: continue + scanned += 1 try: tree = ast.parse(source) except SyntaxError: continue - for node in ast.walk(tree): - if not isinstance(node, ast.Call): - continue - func = node.func - name = ( - func.id - if isinstance(func, ast.Name) - else (func.attr if isinstance(func, ast.Attribute) else None) - ) - if name and name.startswith("configured_") and name.endswith("_size"): - found.add((rel, name)) + found |= {(rel, name) for name in _parallel_config_reads(tree, subjects)} + self.assertGreater( + scanned, + 50, + f"the pre-filter left only {scanned} files to scan; the derivation " + "is broken, not the tree", + ) documented = set(_CONFIGURED_SIZE_CALL_SITES) self.assertEqual( documented, found, - "configured-size call sites drifted from their documented reasons.\n" + "configured-size reads drifted from their documented reasons.\n" f" undocumented: {sorted(found - documented)}\n" f" stale entries: {sorted(documented - found)}", ) class TestNoRenamedAccessorImports(CustomTestCase): - """The scanners above match ``get_server_args`` and ``configured_*_size`` - by their literal names, so an ``import ... as`` rename would walk a read - straight past both the zero baseline and the call-site registry. Renaming - these accessors buys nothing (the names are already short and unambiguous), - so it is banned outright — which is exactly what makes literal-name - matching sound.""" + """The baseline scanner matches ``get_server_args`` by its literal name, so + an ``import ... as`` rename would walk a read straight past the zero + baseline. Renaming the accessor buys nothing (the name is already short and + unambiguous), so it is banned outright — which is exactly what makes + literal-name matching sound. (The configured-size registry resolves + ``get_parallel`` aliases itself, so it needs no such ban.)""" def test_the_scanned_accessors_are_never_import_renamed(self): offenders = [] for path in sorted(_PACKAGE_ROOT.rglob("*.py")): rel = path.relative_to(_PACKAGE_ROOT).as_posix() source = path.read_text() - if "get_server_args" not in source and "configured_" not in source: + if "get_server_args" not in source: continue try: tree = ast.parse(source) @@ -608,19 +776,16 @@ class TestNoRenamedAccessorImports(CustomTestCase): if imported.asname is None or imported.asname == imported.name: continue base = imported.name.rsplit(".", 1)[-1] - if base == "get_server_args" or ( - base.startswith("configured_") and base.endswith("_size") - ): + if base == "get_server_args": offenders.append( f"{rel}:{node.lineno}: {imported.name} as " f"{imported.asname}" ) self.assertFalse( offenders, - "get_server_args / configured_*_size imported under another name; " - "the read ratchet and the configured-size registry match these " - "accessors by their literal names, so a rename silently escapes " - "both:\n" + "\n".join(offenders), + "get_server_args imported under another name; the read ratchet " + "matches it by its literal name, so a rename silently escapes the " + "baseline:\n" + "\n".join(offenders), ) diff --git a/test/registered/unit/test_launch_path_reads_configured_sizes.py b/test/registered/unit/test_launch_path_reads_configured_sizes.py index ca4e6cc9a..fa754bc78 100644 --- a/test/registered/unit/test_launch_path_reads_configured_sizes.py +++ b/test/registered/unit/test_launch_path_reads_configured_sizes.py @@ -4,11 +4,12 @@ the process groups, so they answer only after distributed init. The launcher decides how many processes to spawn *before* that, and a live read there raises `Distributed environment is not initialized` -- a startup crash no unit test -reaches, because nothing short of booting a server runs the launcher. +reaches, because nothing short of booting a server runs the launcher. The +configured answer is one hop away on the same object, +`get_parallel().config.pp_size`, which reads the published `parallel` bag. """ import ast -import functools import pathlib import unittest @@ -20,17 +21,39 @@ register_cpu_ci(est_time=9, suite="base-a-test-cpu") _PACKAGE_ROOT = pathlib.Path(sglang.__file__).resolve().parent -# Live-shadowed sizes a launch path is known to have read. ParallelContext -# shadows more properties than these (every `_v(name, ...)` one raises the same -# "Distributed environment is not initialized"); this dict carries the ones a -# `configured_*` accessor answers, so it is a remedy map, not a census. -_LIVE_SHADOWED = { - "tp_size": "configured_tp_size()", - "pp_size": "configured_pp_size()", - "moe_dp_size": "configured_moe_dp_size()", - "attn_cp_size": "configured_attn_cp_size()", - "dcp_size": "configured_dcp_size()", -} + +def _live_shadowed() -> dict: + """{name: remedy} for every name that is BOTH a live ParallelContext + property and a `parallel` config leaf. + + Derived from the two sides themselves, so a new size that gains a live + property (or a live property that gains a leaf) is watched without a second + list here. ParallelContext shadows more properties than these -- every + `_v(name, ...)` one raises the same "Distributed environment is not + initialized" -- but only a shadowed name has a configured answer to point a + launcher at. + """ + from sglang.srt.arg_groups.arg_utils import namespace_of + from sglang.srt.runtime_context import ParallelContext + from sglang.srt.server_args import ServerArgs + + live = { + name + for name, value in vars(ParallelContext).items() + if isinstance(value, property) + } + leaves = { + field for field, path in namespace_of(ServerArgs).items() if path == "parallel" + } + shadowed = live & leaves + assert shadowed, ( + "no live-shadowed parallel size found; the derivation is broken, not " + "the tree" + ) + return {name: f"get_parallel().config.{name}" for name in sorted(shadowed)} + + +_LIVE_SHADOWED = _live_shadowed() # Launch paths that decide how many children to spawn are derived below # from the spawn itself. These launch without a size-driven spawn, so no @@ -83,81 +106,41 @@ def _multiprocessing_names(tree): return modules, constructors -@functools.lru_cache(maxsize=None) -def _configured_accessors() -> frozenset: - """The `configured_*_size()` names `runtime_context` exports. - - Derived from that module, so a new accessor keeps its launcher watched - without a second list here. - """ - tree = ast.parse( - (_PACKAGE_ROOT / "srt/runtime_context.py").read_text(encoding="utf-8-sig") - ) - names = frozenset( - node.name - for node in tree.body - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) - and node.name.startswith("configured_") - and node.name.endswith("_size") - ) - assert names, ( - "no configured_*_size accessors found in runtime_context; the " - "derivation is broken, not the tree" - ) - return names - - def _spawns_from_a_size(tree) -> bool: - """Does any function here construct a child process *and* read one of the - five sizes -- live off the parallel bag, or through its `configured_*_size()` - answer? That is a spawn count decided from the topology. + """Does a function here spawn a child *and* read a live-shadowed size? - Counting the configured read too is what keeps a launcher watched after it - is converted. Deriving on the live read alone means the file drops out of - the scan the moment it stops offending, so the guard would only ever watch - the launchers that already fail it. + Both tiers count: deriving on the live read alone drops a launcher from the + scan the moment it is converted, so the guard would only watch the ones + that already fail it. """ - configured = _configured_accessors() modules, constructors = _multiprocessing_names(tree) - names, qualified = _parallel_bag_names(tree) - aliases = _bag_aliases(tree, names, qualified) for fn in ast.walk(tree): if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)): continue - spawns = reads = False + spawns = False for node in ast.walk(fn): - if isinstance(node, ast.Call): - func = node.func - if isinstance(func, ast.Attribute) and func.attr in ( - "Process", - "ProcessPoolExecutor", - "Popen", - "spawn", - ): - # `mp.Process(`, `mp.get_context("spawn").Process(` and - # `subprocess.Popen(` all reach a child process; the - # receiver of a chained call is itself a call, so this - # cannot require a bare Name. - spawns = True - elif isinstance(func, ast.Name) and func.id in constructors: - spawns = True - if (isinstance(func, ast.Name) and func.id in configured) or ( - isinstance(func, ast.Attribute) and func.attr in configured - ): - reads = True - elif ( - isinstance(node, ast.Attribute) - and node.attr in _LIVE_SHADOWED - and ( - _is_parallel_bag_call(node.value, names, qualified) - or (isinstance(node.value, ast.Name) and node.value.id in aliases) - ) + if not isinstance(node, ast.Call): + continue + func = node.func + if isinstance(func, ast.Attribute) and func.attr in ( + "Process", + "ProcessPoolExecutor", + "Popen", + "spawn", ): - # A record read (`server_args.tp_size`) sizes a spawn too, but - # it cannot raise pre-dist; only the bag read is this guard's - # subject, so only it forces a module into _PRE_DIST. - reads = True - if spawns and reads: + # `mp.Process(`, `mp.get_context("spawn").Process(` and + # `subprocess.Popen(` all reach a child process; the receiver of + # a chained call is itself a call, so this cannot require a bare + # Name. + spawns = True + elif isinstance(func, ast.Name) and func.id in constructors: + spawns = True + if not spawns: + continue + # A record read (`server_args.tp_size`) sizes a spawn too, but it cannot + # raise pre-dist; only a bag read is this guard's subject. + live, configured = _shadowed_size_reads(tree, scope=fn) + if live or configured: return True return False @@ -178,6 +161,12 @@ def _parallel_bag_names(tree): names |= { a.asname or a.name for a in node.names if a.name == "get_parallel" } + elif isinstance(node, ast.ImportFrom) and node.module: + # `from sglang.srt import runtime_context as rc` binds the module, + # so `rc.get_parallel()` is the same call under another spelling. + for a in node.names: + if f"{node.module}.{a.name}".endswith("runtime_context"): + modules.add(a.asname or a.name) elif isinstance(node, ast.Import): for a in node.names: if a.name.endswith("runtime_context"): @@ -199,16 +188,75 @@ def _is_parallel_bag_call(node, names, modules) -> bool: def _bag_aliases(tree, names, qualified): - """Locals bound to the parallel bag: `p = get_parallel()` then `p.pp_size` - is the same read one line later.""" - return { - target.id - for node in ast.walk(tree) - if isinstance(node, ast.Assign) - and _is_parallel_bag_call(node.value, names, qualified) - for target in node.targets - if isinstance(target, ast.Name) - } + """Locals bound to either tier: `p = get_parallel()` then `p.pp_size` is the + same live read one line later, and `cfg = get_parallel().config` then + `cfg.pp_size` is the same configured read.""" + live, config = set(), set() + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + value = node.value + if _is_parallel_bag_call(value, names, qualified): + bucket = live + elif ( + isinstance(value, ast.Attribute) + and value.attr == "config" + and _is_parallel_bag_call(value.value, names, qualified) + ): + bucket = config + else: + continue + bucket |= {t.id for t in node.targets if isinstance(t, ast.Name)} + return live, config + + +def _shadowed_size_reads(module_tree, scope=None): + """(live, configured) reads of a live-shadowed size in `scope`. + + `.tp_size` is the live group; `.config.tp_size` + is the published leaf. Both spellings are reported so a caller can tell a + launcher that reads the topology at all from one that reads it live. + + What binds the bag -- the import, a module-level alias -- lives at module + scope, so those names always come from `module_tree` even when only one + function is being walked. Deriving them from the function alone finds no + import, reports no reads, and quietly answers "this launcher reads nothing". + """ + names, qualified = _parallel_bag_names(module_tree) + live_aliases, config_aliases = _bag_aliases(module_tree, names, qualified) + + def is_live_bag(node): + return _is_parallel_bag_call(node, names, qualified) or ( + isinstance(node, ast.Name) and node.id in live_aliases + ) + + def is_config_bag(node): + return ( + isinstance(node, ast.Attribute) + and node.attr == "config" + and is_live_bag(node.value) + ) or (isinstance(node, ast.Name) and node.id in config_aliases) + + live, configured = [], [] + for node in ast.walk(scope if scope is not None else module_tree): + if isinstance(node, ast.Attribute) and node.attr in _LIVE_SHADOWED: + base, name, spelling = node.value, node.attr, "attribute" + elif ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "getattr" + and len(node.args) >= 2 + and isinstance(node.args[1], ast.Constant) + and node.args[1].value in _LIVE_SHADOWED + ): + base, name, spelling = node.args[0], node.args[1].value, "getattr" + else: + continue + if is_config_bag(base): + configured.append((node.lineno, name, spelling)) + elif is_live_bag(base): + live.append((node.lineno, name, spelling)) + return live, configured def _launch_paths(): @@ -219,7 +267,7 @@ def _launch_paths(): nothing, which no derivation can reach. """ seen = {} - sizes = frozenset(_LIVE_SHADOWED) | _configured_accessors() + sizes = frozenset(_LIVE_SHADOWED) for path in sorted(_PACKAGE_ROOT.rglob("*.py")): source = path.read_text() # Every spawn shape below names Process, ProcessPoolExecutor or Popen. @@ -243,12 +291,12 @@ class TestLaunchPathsReadConfiguredSizes(CustomTestCase): """The other direction: groups exist and answer something else. The check above proves nobody reads a live size too early. It says - nothing about what `configured_*()` returns once the groups *are* up - and answering a different number -- which is not hypothetical: elastic - EP scales the live topology away from what the operator configured, and - that divergence is the entire reason these five helpers exist. With - only the early-read direction covered, a helper that quietly delegated - to the live property would look correct. + nothing about what `.config.` returns once the groups *are* up and + answering a different number -- which is not hypothetical: elastic EP + scales the live topology away from what the operator configured, and + that divergence is the entire reason the two tiers are separate. With + only the early-read direction covered, a `config` hop that quietly + delegated to the live property would look correct. """ import json import os @@ -256,11 +304,6 @@ class TestLaunchPathsReadConfiguredSizes(CustomTestCase): from unittest.mock import patch from sglang.srt.runtime_context import ( - configured_attn_cp_size, - configured_dcp_size, - configured_moe_dp_size, - configured_pp_size, - configured_tp_size, get_parallel, publish, reset_context, @@ -318,26 +361,16 @@ class TestLaunchPathsReadConfiguredSizes(CustomTestCase): if isinstance(getter, ast.Attribute): live_getter[call.args[0].value] = getter.attr state = "sglang.srt.distributed.parallel_state" - helpers = { - "tp_size": configured_tp_size, - "pp_size": configured_pp_size, - "moe_dp_size": configured_moe_dp_size, - "attn_cp_size": configured_attn_cp_size, - "dcp_size": configured_dcp_size, - } - missing = sorted(set(helpers) - set(live_getter)) + missing = sorted(set(_LIVE_SHADOWED) - set(live_getter)) self.assertEqual( missing, [], f"these sizes no longer have a live property to diverge from: {missing}", ) - cases = tuple( - (name, helper, f"{state}.{live_getter[name]}") - for name, helper in helpers.items() - ) - for name, helper, target in cases: + for name in sorted(_LIVE_SHADOWED): with self.subTest(size=name): - configured = helper() + target = f"{state}.{live_getter[name]}" + configured = getattr(get_parallel().config, name) with patch(target, return_value=configured + 41): self.assertEqual( get_parallel().__getattribute__(name), @@ -345,48 +378,30 @@ class TestLaunchPathsReadConfiguredSizes(CustomTestCase): f"{name} no longer follows the live topology", ) self.assertEqual( - helper(), + getattr(get_parallel().config, name), configured, - f"configured_{name}() followed the live topology instead " - "of the published configuration", + f"get_parallel().config.{name} followed the live topology " + "instead of the published configuration", ) + # A bare read of a leaf with no live property is not a config read any + # more, and the error says where it went. Spelled through `getattr` so a + # mechanical `.config` sweep cannot "fix" the very read under test. + with self.assertRaisesRegex( + AttributeError, r"read it as get_parallel\(\)\.config\.nccl_port" + ): + getattr(get_parallel(), "nccl_port") reset_context() def test_no_live_topology_read_before_distributed_init(self): offenders = [] for rel, tree in _launch_paths(): - names, modules = _parallel_bag_names(tree) - aliases = _bag_aliases(tree, names, modules) - for node in ast.walk(tree): - if isinstance(node, ast.Attribute) and node.attr in _LIVE_SHADOWED: - base = node.value - if _is_parallel_bag_call(base, names, modules) or ( - isinstance(base, ast.Name) and base.id in aliases - ): - offenders.append( - f"{rel}:{node.lineno} reads the live {node.attr}; " - f"use {_LIVE_SHADOWED[node.attr]}" - ) - elif ( - isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id == "getattr" - and len(node.args) >= 2 - and isinstance(node.args[1], ast.Constant) - and node.args[1].value in _LIVE_SHADOWED - and ( - _is_parallel_bag_call(node.args[0], names, modules) - or ( - isinstance(node.args[0], ast.Name) - and node.args[0].id in aliases - ) - ) - ): - offenders.append( - f"{rel}:{node.lineno} reads the live " - f"{node.args[1].value} through getattr; " - f"use {_LIVE_SHADOWED[node.args[1].value]}" - ) + live, _ = _shadowed_size_reads(tree) + for lineno, name, spelling in live: + through = " through getattr" if spelling == "getattr" else "" + offenders.append( + f"{rel}:{lineno} reads the live {name}{through}; " + f"use {_LIVE_SHADOWED[name]}" + ) self.assertEqual( offenders, [], diff --git a/test/registered/unit/test_runtime_context.py b/test/registered/unit/test_runtime_context.py index 4b1d12a8d..b655102d1 100644 --- a/test/registered/unit/test_runtime_context.py +++ b/test/registered/unit/test_runtime_context.py @@ -907,12 +907,11 @@ class TestForwardFlags(_IsolatedServerArgs): self.assertEqual(probe(torch.zeros(())).item(), 0) def test_parallel_config_leaves_trace_under_torch_compile(self): - # Regression: parallel config leaves resolve through - # ``ParallelContext.__getattr__`` (the bag fallback), and gate helpers - # such as ``enable_moe_dense_fully_dp()`` read them inside compiled - # model forwards — the fallback body must stay dynamo-traceable - # (``object.__getattribute__`` graph-breaks). fullgraph=True turns any - # graph break back into a failure. + # Regression: gate helpers such as ``enable_moe_dense_fully_dp()`` read + # parallel config leaves inside compiled model forwards through the + # `config` property, which must stay dynamo-traceable + # (``object.__getattribute__`` graph-breaks). + # fullgraph=True turns any graph break back into a failure. import torch from sglang.srt.runtime_context import get_parallel @@ -923,11 +922,11 @@ class TestForwardFlags(_IsolatedServerArgs): @torch.compile(fullgraph=True, backend="eager", dynamic=False) def probe(x): par = get_parallel() - if par.enable_prefill_context_parallel: + if par.config.enable_prefill_context_parallel: x = x + 1 - if par.moe_dense_tp_size == 1: + if par.config.moe_dense_tp_size == 1: x = x + 2 - if par.dwdp_size > 1: + if par.config.dwdp_size > 1: x = x + 4 return x diff --git a/test/registered/unit/test_runtime_context_override.py b/test/registered/unit/test_runtime_context_override.py index 1216363a3..e7dfd144e 100644 --- a/test/registered/unit/test_runtime_context_override.py +++ b/test/registered/unit/test_runtime_context_override.py @@ -87,7 +87,7 @@ class TestContextOverride(CustomTestCase): speculative_accept_threshold_single=0.5, speculative_accept_threshold_acc=0.9, ) - self.assertEqual(rc.get_parallel().pp_max_micro_batch_size, 8) + self.assertEqual(rc.get_parallel().config.pp_max_micro_batch_size, 8) self.assertEqual(rc.get_spec().speculative_accept_threshold_single, 0.5) self.assertEqual(rc.get_spec().speculative_accept_threshold_acc, 0.9) diff --git a/test/registered/unit/test_server_args_namespaces.py b/test/registered/unit/test_server_args_namespaces.py index d136ec0d6..601758447 100644 --- a/test/registered/unit/test_server_args_namespaces.py +++ b/test/registered/unit/test_server_args_namespaces.py @@ -72,10 +72,9 @@ class TestServerArgsNamespaces(CustomTestCase): accessors = { node.name for node in context_module.body - if isinstance(node, ast.FunctionDef) - and (node.name.startswith("get_") or node.name.startswith("configured_")) + if isinstance(node, ast.FunctionDef) and node.name.startswith("get_") } - self.assertGreater(len(accessors), 20, "the accessor derivation broke") + self.assertGreater(len(accessors), 15, "the accessor derivation broke") shadowed = [] for path in sorted(srt.rglob("*.py")): @@ -184,6 +183,10 @@ class TestServerArgsNamespaces(CustomTestCase): continue sites += 1 read = [cursor.func.id[len("get_") :]] + chain[:-1] + if read[:2] == ["parallel", "config"]: + # `config` on `get_parallel()` is the tier hop, not a + # sub-namespace: bare names there are the live topology. + del read[1] if mapping[field].split(".") != read: disagreements.append( f"{path.relative_to(srt)}:{node.lineno} reads " diff --git a/test/registered/unit/test_supplied_instance_exposure_ratchet.py b/test/registered/unit/test_supplied_instance_exposure_ratchet.py index a52354d79..aa9af73ce 100644 --- a/test/registered/unit/test_supplied_instance_exposure_ratchet.py +++ b/test/registered/unit/test_supplied_instance_exposure_ratchet.py @@ -160,6 +160,8 @@ _EXPOSED = { ("configs/model_config.py", "quantization"), ("configs/model_config.py", "speculative_algorithm"), ("configs/model_config.py", "speculative_draft_model_quantization"), + ("dllm/config.py", "max_running_requests"), + ("dllm/config.py", "model_path"), ("entrypoints/engine.py", "enable_symm_mem"), ("entrypoints/engine.py", "reasoning_parser"), ("entrypoints/engine.py", "tool_call_parser"), @@ -171,20 +173,25 @@ _EXPOSED = { ("layers/cp/bcg.py", "cp_strategy"), ("layers/cp/bcg.py", "enable_prefill_cp"), ("layers/flashinfer_comm_fusion.py", "flashinfer_allreduce_fusion_backend"), + ("layers/moe/utils.py", "deepep_mode"), + ("layers/moe/utils.py", "moe_a2a_backend"), + ("layers/moe/utils.py", "moe_runner_backend"), + ("layers/moe/utils.py", "quantization"), + ("layers/moe/utils.py", "speculative_moe_runner_backend"), ("lora/lora_manager.py", "enable_lora_overlap_loading"), ("lora/marlin_lora_temp/policy.py", "lora_paths"), ("model_loader/expert_pack_runtime.py", "model_path"), ("model_loader/expert_pack_runtime.py", "tokenizer_path"), + ("multimodal/processors/base_processor.py", "image_processor_backend"), ("parser/template_detection.py", "model_path"), ("speculative/adaptive_spec_params.py", "speculative_algorithm"), ("speculative/adaptive_spec_params.py", "speculative_eagle_topk"), ("speculative/draft_worker_common.py", "speculative_draft_attention_backend"), ("speculative/spec_info.py", "enable_multi_layer_eagle"), + ("speculative/spec_registry.py", "disable_overlap_schedule"), ("utils/common.py", "speculative_num_draft_tokens"), ("utils/common.py", "speculative_num_steps"), ("utils/hf_transformers/processor.py", "image_processor_backend"), - # The daemon command and constructor snapshot the resolved startup layout - # before the daemon's loading lifecycle can apply any runtime overrides. ("weight_cache/daemon.py", "attn_cp_size"), ("weight_cache/daemon.py", "deepep_mode"), ("weight_cache/daemon.py", "dp_size"), @@ -214,6 +221,8 @@ _EXPOSED_CUDA_ONLY: frozenset = frozenset() # some code overrides post-publish. Each needs an ordering judgment, not a blanket # conversion; the list exists so a new one is a decision made when it is written. _OVERRIDDEN_AND_READ = { + ("configs/model_config.py", "dtype"), + ("configs/model_config.py", "model_path"), ("dllm/config.py", "model_path"), ("entrypoints/engine.py", "reasoning_parser"), ("entrypoints/engine.py", "tool_call_parser"), @@ -234,6 +243,11 @@ _OVERRIDDEN_AND_READ = { ("parser/template_detection.py", "model_path"), ("utils/common.py", "speculative_num_draft_tokens"), ("utils/common.py", "speculative_num_steps"), + ("weight_cache/daemon.py", "dp_size"), + ("weight_cache/daemon.py", "dtype"), + ("weight_cache/daemon.py", "ep_size"), + ("weight_cache/daemon.py", "load_format"), + ("weight_cache/daemon.py", "model_path"), }