Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
2146 lines
81 KiB
Python
2146 lines
81 KiB
Python
# Copyright 2023-2026 SGLang Team
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
# ==============================================================================
|
|
"""Process-wide configuration, parallel state, flags, and resources.
|
|
|
|
``get_server_args()`` retains the raw input for diagnostics. Namespace
|
|
accessors such as ``get_exec()`` return resolved configuration published
|
|
from declarations; ``RuntimeContext.override`` changes those bags.
|
|
``get_parallel()`` also exposes ranks and process groups, with scoped and
|
|
permanent overrides taking precedence over published values.
|
|
|
|
``get_flags()`` holds mutable runtime state; ``get_resources()`` owns process
|
|
handles, and ``get_forward()`` holds per-forward state.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import functools
|
|
import logging
|
|
import math
|
|
import os
|
|
import sys
|
|
from contextlib import contextmanager
|
|
from enum import Enum, auto
|
|
from typing import TYPE_CHECKING, Any, Dict, Optional
|
|
|
|
import msgspec
|
|
|
|
from sglang.srt.arg_groups.prefill_buffer_ceiling import prefill_buffer_ceiling_of
|
|
|
|
if TYPE_CHECKING:
|
|
from sglang.srt.model_executor.runner_utils.pool import GraphPoolBorrowState
|
|
from sglang.srt.server_args import ServerArgs
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@functools.lru_cache(maxsize=1)
|
|
def _parallel_config_leaves() -> frozenset:
|
|
"""Return configured parallel field names, including before publication."""
|
|
from sglang.srt.arg_groups.arg_utils import namespace_of
|
|
from sglang.srt.server_args import ServerArgs
|
|
|
|
return frozenset(
|
|
field
|
|
for field, path in namespace_of(ServerArgs).items()
|
|
if path.split(".")[0] == "parallel"
|
|
)
|
|
|
|
|
|
@functools.lru_cache(maxsize=1)
|
|
def _parallel_fields() -> frozenset:
|
|
"""Return configured and derived parallel names accepted by overrides."""
|
|
from sglang.srt.arg_groups.arg_utils import Derived
|
|
from sglang.srt.arg_groups.fields.parallel import Parallel
|
|
|
|
derived = {
|
|
name for name, decl in vars(Parallel).items() if isinstance(decl, Derived)
|
|
}
|
|
return frozenset(_parallel_config_leaves() | derived)
|
|
|
|
|
|
def derive_attn_tp_size(*, tp_size: int, attn_cp_size: int, attn_dp_size: int) -> int:
|
|
"""Return the attention-TP width: what is left of the TP group after the
|
|
attention-DP and attention-CP dimensions."""
|
|
return tp_size // attn_dp_size // attn_cp_size
|
|
|
|
|
|
def derive_attention_ranks(
|
|
*, tp_rank: int, attn_tp_size: int, attn_cp_size: int
|
|
) -> tuple:
|
|
"""Return (attn_tp_rank, attn_dp_rank) for a process at ``tp_rank``.
|
|
|
|
The rank layout is (dp, cp, tp), with tp changing fastest::
|
|
|
|
tp_rank = (attn_dp_rank * attn_cp_size + attn_cp_rank) * attn_tp_size
|
|
+ attn_tp_rank
|
|
|
|
Without attention DP the TP group is one attention-DP group, so
|
|
``attn_dp_rank`` is zero.
|
|
"""
|
|
return tp_rank % attn_tp_size, tp_rank // (attn_tp_size * attn_cp_size)
|
|
|
|
|
|
def spawn_world_rank(server_args, *, tp_rank: int, pp_rank: int) -> int:
|
|
"""Compute the WORLD rank from launcher TP and PP ranks.
|
|
|
|
Uses a resolving view because callers may run before ``publish``.
|
|
"""
|
|
from sglang.srt.arg_groups.model_override_base import resolving_view
|
|
|
|
cfg = resolving_view(server_args)
|
|
return cfg.ep_join_rank_offset + cfg.tp_size * pp_rank + tp_rank
|
|
|
|
|
|
def derive_spawn_ranks(
|
|
*,
|
|
world_rank: int,
|
|
tp_size: int,
|
|
ep_join_rank_offset: int,
|
|
attn_cp_size: int,
|
|
attn_tp_size: int,
|
|
moe_dp_size: int,
|
|
moe_ep_size: int,
|
|
) -> dict:
|
|
"""Derive TP, PP, attention, and MoE ranks without process groups.
|
|
|
|
WORLD uses ``rank = ep_join_rank_offset + tp_size * pp_rank + tp_rank``.
|
|
TP groups are contiguous blocks; PP groups are strided by ``tp_size``.
|
|
"""
|
|
local = world_rank - ep_join_rank_offset
|
|
tp_rank = local % tp_size
|
|
return {
|
|
"tp_rank": tp_rank,
|
|
"pp_rank": local // tp_size,
|
|
"attn_cp_rank": (tp_rank // attn_tp_size) % attn_cp_size,
|
|
"moe_dp_rank": tp_rank // (tp_size // moe_dp_size),
|
|
"moe_ep_rank": (
|
|
tp_rank
|
|
% (tp_size // moe_dp_size)
|
|
// (tp_size // moe_dp_size // moe_ep_size)
|
|
),
|
|
"moe_tp_rank": tp_rank % (tp_size // moe_dp_size // moe_ep_size),
|
|
}
|
|
|
|
|
|
def derive_parallel_widths(
|
|
*,
|
|
tp_size: int,
|
|
attn_cp_size: int,
|
|
attn_dp_size: int,
|
|
moe_ep_size: int,
|
|
moe_dp_size: int,
|
|
dcp_size: int,
|
|
dcp_enabled: bool,
|
|
) -> dict:
|
|
"""Derive attention and MoE widths and DCP settings from configuration."""
|
|
return {
|
|
"attn_dp_size": attn_dp_size,
|
|
"attn_tp_size": derive_attn_tp_size(
|
|
tp_size=tp_size, attn_cp_size=attn_cp_size, attn_dp_size=attn_dp_size
|
|
),
|
|
"moe_ep_size": moe_ep_size,
|
|
"moe_tp_size": tp_size // moe_ep_size // moe_dp_size,
|
|
"dcp_enabled": dcp_enabled,
|
|
"attn_dcp_size": dcp_size if dcp_enabled else 1,
|
|
}
|
|
|
|
|
|
def parallel_widths_of(cfg: Any) -> dict:
|
|
"""Return derived parallel settings from resolved configuration."""
|
|
return derive_parallel_widths(
|
|
tp_size=cfg.tp_size,
|
|
attn_cp_size=cfg.attn_cp_size,
|
|
attn_dp_size=cfg.attn_dp_size,
|
|
moe_ep_size=cfg.ep_size,
|
|
moe_dp_size=cfg.moe_dp_size,
|
|
dcp_size=cfg.dcp_size,
|
|
dcp_enabled=cfg.dcp_size > 1,
|
|
)
|
|
|
|
|
|
def launch_world_size_of(cfg: Any):
|
|
"""Return the initial WORLD width, including existing ranks below a joiner.
|
|
|
|
This value remains fixed after elastic scale-up.
|
|
"""
|
|
return cfg.ep_join_rank_offset + cfg.tp_size * cfg.pp_size
|
|
|
|
|
|
def max_world_size_of(cfg: Any):
|
|
"""Return the WORLD capacity: ``max_ep_size`` or the launch width."""
|
|
return cfg.max_ep_size or launch_world_size_of(cfg)
|
|
|
|
|
|
def attn_tp_size_of(cfg: Any):
|
|
"""`attn_tp_size`, computed at publish. See `parallel_widths_of`."""
|
|
return parallel_widths_of(cfg)["attn_tp_size"]
|
|
|
|
|
|
def attn_dp_enabled_of(cfg: Any) -> bool:
|
|
"""`attn_dp_enabled`, computed at publish.
|
|
|
|
A scale joiner's own group can be one rank wide, but it joins an
|
|
attention-DP deployment and runs its DP-attention paths.
|
|
"""
|
|
from sglang.srt.arg_groups.model_override_base import ep_scale_joiner_of
|
|
|
|
return cfg.attn_dp_size > 1 or ep_scale_joiner_of(cfg)
|
|
|
|
|
|
def num_dp_ranks_of(cfg: Any) -> int:
|
|
"""`num_dp_ranks`, computed at publish: replicas times attention-DP groups.
|
|
|
|
DP attention does not run inside replicas, so one of the two is one.
|
|
"""
|
|
return cfg.dp_size * cfg.attn_dp_size
|
|
|
|
|
|
def attn_dcp_size_of(cfg: Any):
|
|
"""`attn_dcp_size`, computed at publish. See `parallel_widths_of`."""
|
|
return parallel_widths_of(cfg)["attn_dcp_size"]
|
|
|
|
|
|
def moe_ep_size_of(cfg: Any):
|
|
"""`moe_ep_size`, computed at publish. See `parallel_widths_of`."""
|
|
return parallel_widths_of(cfg)["moe_ep_size"]
|
|
|
|
|
|
def moe_tp_size_of(cfg: Any):
|
|
"""`moe_tp_size`, computed at publish. See `parallel_widths_of`."""
|
|
return parallel_widths_of(cfg)["moe_tp_size"]
|
|
|
|
|
|
def dcp_enabled_of(cfg: Any):
|
|
"""`dcp_enabled`, computed at publish. See `parallel_widths_of`."""
|
|
return parallel_widths_of(cfg)["dcp_enabled"]
|
|
|
|
|
|
class SpawnRanks(msgspec.Struct, frozen=True):
|
|
"""Process placement supplied by the launcher.
|
|
|
|
``world_rank`` determines TP, PP, attention, and MoE ranks from the
|
|
configured widths. ``dp_rank`` identifies the replica across separate
|
|
WORLD groups; ``None`` means no data-parallel controller. ``gpu_id`` is
|
|
the assigned device index, or ``None`` for a process without a device.
|
|
"""
|
|
|
|
world_rank: int
|
|
dp_rank: Optional[int] = None
|
|
gpu_id: Optional[int] = None
|
|
|
|
|
|
_RANK_AND_WIDTH = (
|
|
("tp_rank", "tp_size"),
|
|
("pp_rank", "pp_size"),
|
|
("attn_tp_rank", "attn_tp_size"),
|
|
("attn_dp_rank", "attn_dp_size"),
|
|
("attn_cp_rank", "attn_cp_size"),
|
|
("moe_ep_rank", "moe_ep_size"),
|
|
)
|
|
|
|
# MoE-DP may alias a wider attention-CP group, so its configured width
|
|
# need not match the group width.
|
|
_WIDTH_AND_GROUP = (
|
|
("tp_size", "tp_group"),
|
|
("pp_size", "pp_group"),
|
|
("attn_tp_size", "attn_tp_group"),
|
|
("attn_cp_size", "attn_cp_group"),
|
|
("moe_ep_size", "moe_ep_group"),
|
|
)
|
|
|
|
_UNREADABLE = object()
|
|
|
|
|
|
def _validate_parallel(parallel, source: str) -> None:
|
|
"""Check rank bounds, topology factorizations, and group widths.
|
|
|
|
Skip unavailable or non-integer values so partially initialized contexts
|
|
can be validated.
|
|
"""
|
|
|
|
def read(name):
|
|
"""Return an integer topology value, or ``_UNREADABLE``; exclude booleans."""
|
|
try:
|
|
value = getattr(parallel, name)
|
|
except Exception:
|
|
return _UNREADABLE
|
|
if isinstance(value, bool) or not isinstance(value, int):
|
|
return _UNREADABLE
|
|
return value
|
|
|
|
problems = []
|
|
|
|
for rank_name, size_name in _RANK_AND_WIDTH:
|
|
rank, size = read(rank_name), read(size_name)
|
|
if _UNREADABLE in (rank, size):
|
|
continue
|
|
if not 0 <= rank < size:
|
|
problems.append(
|
|
f"0 <= {rank_name} < {size_name}\n {rank} is not a rank of {size}"
|
|
)
|
|
|
|
terms = ("tp_size", "attn_tp_size", "attn_dp_size", "attn_cp_size")
|
|
tp_size, a_tp, a_dp, a_cp = (read(n) for n in terms)
|
|
if _UNREADABLE not in (tp_size, a_tp, a_dp, a_cp):
|
|
if tp_size != a_tp * a_dp * a_cp:
|
|
problems.append(
|
|
"tp_size == attn_tp_size * attn_dp_size * attn_cp_size\n"
|
|
f" {tp_size} != {a_tp} * {a_dp} * {a_cp} (= {a_tp * a_dp * a_cp})"
|
|
)
|
|
|
|
moe_terms = ("tp_size", "moe_ep_size", "moe_dp_size", "moe_tp_size")
|
|
tp_size, m_ep, m_dp, m_tp = (read(n) for n in moe_terms)
|
|
if _UNREADABLE not in (tp_size, m_ep, m_dp, m_tp):
|
|
if tp_size != m_ep * m_dp * m_tp:
|
|
problems.append(
|
|
"tp_size == moe_ep_size * moe_dp_size * moe_tp_size\n"
|
|
f" {tp_size} != {m_ep} * {m_dp} * {m_tp} (= {m_ep * m_dp * m_tp})"
|
|
)
|
|
|
|
layout_terms = (
|
|
"tp_rank",
|
|
"attn_dp_rank",
|
|
"attn_cp_rank",
|
|
"attn_tp_rank",
|
|
"attn_cp_size",
|
|
"attn_tp_size",
|
|
)
|
|
tp_rank, r_dp, r_cp, r_tp, w_cp, w_tp = (read(n) for n in layout_terms)
|
|
if _UNREADABLE not in (tp_rank, r_dp, r_cp, r_tp, w_cp, w_tp):
|
|
laid_out = (r_dp * w_cp + r_cp) * w_tp + r_tp
|
|
if tp_rank != laid_out:
|
|
problems.append(
|
|
"tp_rank == (attn_dp_rank * attn_cp_size + attn_cp_rank)"
|
|
" * attn_tp_size + attn_tp_rank\n"
|
|
f" {tp_rank} != ({r_dp} * {w_cp} + {r_cp})"
|
|
f" * {w_tp} + {r_tp} (= {laid_out})"
|
|
)
|
|
|
|
for size_name, group_name in _WIDTH_AND_GROUP:
|
|
size = read(size_name)
|
|
if size is _UNREADABLE:
|
|
continue
|
|
try:
|
|
group = getattr(parallel, group_name)
|
|
except Exception:
|
|
continue
|
|
built = getattr(group, "world_size", _UNREADABLE)
|
|
if isinstance(built, int) and not isinstance(built, bool) and built != size:
|
|
problems.append(
|
|
f"{group_name}.world_size == {size_name}\n"
|
|
f" built {built}, configured {size}"
|
|
)
|
|
|
|
if problems:
|
|
raise ValueError(
|
|
f"parallel topology is inconsistent (set by {source}):\n"
|
|
+ "\n".join(problems)
|
|
)
|
|
|
|
|
|
class ParallelContext:
|
|
"""Parallel configuration, process ranks, and group handles.
|
|
|
|
Configured and derived widths come from the published configuration.
|
|
``publish`` records ranks from the launcher; distributed initialization
|
|
records group handles. Scoped overrides take precedence over permanent
|
|
overrides and configuration. Uninitialized runtime fields raise on read.
|
|
"""
|
|
|
|
__slots__ = ("_overrides", "_stamp", "_config")
|
|
|
|
def __init__(self):
|
|
self._overrides = {} # scoped, restored when the `with` block exits
|
|
self._stamp = {} # permanent for the process, dropped by clear_stamp
|
|
self._config = None # parallel config bag, wired at publish
|
|
|
|
def __getattr__(self, name):
|
|
if name.startswith("_"):
|
|
# This also breaks the recursion when the ``_config`` slot itself is
|
|
# still unset (pickle/copy protocols probe attributes before
|
|
# __init__ runs).
|
|
raise AttributeError(name)
|
|
return self._read(name)
|
|
|
|
def _read(self, name):
|
|
"""Read a scoped override, permanent override, or published value, in order."""
|
|
overrides = self._overrides
|
|
if name in overrides:
|
|
return overrides[name]
|
|
stamp = self._stamp
|
|
if name in stamp:
|
|
return stamp[name]
|
|
config = self._config
|
|
if config is not None and name in config._fields:
|
|
return getattr(config, name)
|
|
if config is None and name in _parallel_config_leaves():
|
|
raise ValueError("config namespace 'parallel' not published")
|
|
declared = _derived_widths().get(name)
|
|
if declared is not None and not declared.fn:
|
|
raise RuntimeError(
|
|
f"parallel name {name!r} has not been written in this process. "
|
|
+ declared.doc
|
|
+ f" Write it by publishing a rank bundle or building the groups, "
|
|
f"or state it with get_parallel().override({name}=...)"
|
|
)
|
|
if declared is not None:
|
|
raise RuntimeError(
|
|
f"derived parallel width {name!r} is not available: it is computed "
|
|
"from the configured leaves at publish, and permanently corrected "
|
|
"when the process groups are built. Nothing is published and "
|
|
"nothing has been set with override_permanently -- publish a "
|
|
f"parallel config, or state the width with get_parallel().override({name}=...)"
|
|
)
|
|
raise AttributeError(f"ParallelContext has no {name!r}")
|
|
|
|
def override_permanently(self, **values) -> None:
|
|
"""Set parallel values until ``clear_stamp`` or ``reset_context``.
|
|
|
|
Works without published configuration. Validate the combined topology
|
|
and restore the previous values if validation fails.
|
|
"""
|
|
unknown = set(values) - _parallel_fields()
|
|
if unknown:
|
|
raise ValueError(f"unknown parallel field(s): {sorted(unknown)}")
|
|
saved = dict(self._stamp)
|
|
self._stamp.update(values)
|
|
try:
|
|
_validate_parallel(self, "override_permanently")
|
|
except Exception:
|
|
self._stamp = saved
|
|
raise
|
|
|
|
def clear_stamp(self) -> None:
|
|
"""Drop every stamped name, ranks included."""
|
|
self._stamp.clear()
|
|
|
|
@contextmanager
|
|
def override(self, **kwargs):
|
|
"""Temporarily force parallel values, restoring on exit. Validates keys and
|
|
supports nesting."""
|
|
unknown = set(kwargs) - _parallel_fields()
|
|
if unknown:
|
|
raise ValueError(f"unknown parallel field(s): {sorted(unknown)}")
|
|
saved = dict(self._overrides)
|
|
self._overrides.update(kwargs)
|
|
try:
|
|
_validate_parallel(self, "override")
|
|
except Exception:
|
|
self._overrides = saved
|
|
raise
|
|
try:
|
|
yield self
|
|
finally:
|
|
self._overrides = saved
|
|
|
|
|
|
def _derived_widths() -> dict:
|
|
"""Return parallel ``Derived`` declarations, including ranks and groups."""
|
|
from sglang.srt.arg_groups.arg_utils import Derived
|
|
from sglang.srt.arg_groups.fields.parallel import Parallel
|
|
|
|
return {
|
|
name: decl for name, decl in vars(Parallel).items() if isinstance(decl, Derived)
|
|
}
|
|
|
|
|
|
def _install_parallel_properties() -> None:
|
|
"""Expose declared parallel fields as documented properties.
|
|
|
|
Properties support class-level introspection; all reads use ``_read``.
|
|
"""
|
|
for name, decl in _derived_widths().items():
|
|
|
|
def getter(self, _name=name):
|
|
return self._read(_name)
|
|
|
|
getter.__name__ = name
|
|
getter.__doc__ = decl.doc
|
|
setattr(ParallelContext, name, property(getter))
|
|
|
|
|
|
_install_parallel_properties()
|
|
|
|
|
|
class _FlagGroupBase(msgspec.Struct):
|
|
"""Shared flag-group behavior: typo-safe writes + transactional ``override()``.
|
|
|
|
``__struct_fields__`` is the single source of truth for which leaves exist,
|
|
so a mistyped name fails loudly instead of creating a stray attribute. The
|
|
write goes through ``super().__setattr__``: a ``Struct`` keeps its fields in
|
|
its own layout, so ``object.__setattr__`` does not reach them.
|
|
"""
|
|
|
|
def __setattr__(self, name: str, value: Any) -> None:
|
|
if name not in type(self).__struct_fields__:
|
|
raise AttributeError(
|
|
f"{type(self).__name__} has no flag '{name}' (leaves are "
|
|
"declared as struct fields; check for typos)"
|
|
)
|
|
super().__setattr__(name, value)
|
|
|
|
@contextmanager
|
|
def override(self, **kwargs):
|
|
"""Temporarily force flag values, restoring on exit. Transactional
|
|
(keys validated before any write) — the test-only injection
|
|
primitive."""
|
|
fields = type(self).__struct_fields__
|
|
unknown = set(kwargs) - set(fields)
|
|
if unknown:
|
|
raise ValueError(
|
|
f"unknown flag(s) for {type(self).__name__}: {sorted(unknown)}"
|
|
)
|
|
saved = {name: getattr(self, name) for name in kwargs}
|
|
for name, value in kwargs.items():
|
|
setattr(self, name, value)
|
|
try:
|
|
yield self
|
|
finally:
|
|
for name, value in saved.items():
|
|
setattr(self, name, value)
|
|
|
|
|
|
class CaptureFlags(_FlagGroupBase):
|
|
"""Capture-time flags; never frozen (written during cuda-graph capture)."""
|
|
|
|
# Seeded from server_args at publish; a model whose _can_torch_compile is
|
|
# False clears it during warmup (the only post-publish writer).
|
|
enable_torch_compile: bool = False
|
|
|
|
# Set for the duration of decode/spec graph capture (model_capture_mode).
|
|
# While set, dispose_tensor() is a no-op so deep_gemm's pre-permute does not
|
|
# free hidden_states that the dual-stream MoE shared expert reads afterward.
|
|
disable_dispose_tensor: bool = False
|
|
|
|
|
|
class MoeFlags(_FlagGroupBase):
|
|
"""MoE runtime flags, materialized by ``initialize_moe_config`` (scheduler
|
|
init, after distributed setup). ``a2a_backend`` / ``runner_backend`` /
|
|
``disable_fp4_allgather`` are the ACTIVE values: the speculative contexts
|
|
in ``layers.moe.utils`` swap them around draft-model forwards. Values are
|
|
the parsed enums from ``layers.moe.utils``; ``None`` means "not
|
|
initialized yet" and the accessors fall back lazily.
|
|
"""
|
|
|
|
a2a_backend: Any = None
|
|
runner_backend: Any = None
|
|
speculative_runner_backend: Any = None
|
|
speculative_a2a_backend: Any = None
|
|
deepep_mode: Any = None
|
|
deepep_config: str | None = None
|
|
tbo_enabled: bool | None = None
|
|
sbo_enabled: bool | None = None
|
|
tbo_token_distribution_threshold: float | None = None
|
|
disable_fp4_allgather: bool | None = None
|
|
quantization: str | None = None
|
|
# The shared-experts-fusion decision, per runner — the runner_backend /
|
|
# speculative_runner_backend shape. Both leaves are seeded from the config
|
|
# intent by ``initialize_moe_config``; each MoE model's gate
|
|
# (determine_num_fused_shared_experts) refines the ACTIVE leaf, both ways,
|
|
# before its layers build and read it. ``speculative_moe_backend_context``
|
|
# brackets a draft's build: on exit the draft's effective decision is
|
|
# persisted onto the speculative leaf (inspectable afterwards) and the
|
|
# target's ACTIVE value returns.
|
|
disable_shared_experts_fusion: bool | None = None
|
|
speculative_disable_shared_experts_fusion: bool | None = None
|
|
# Lifecycle marker (the capture.disable_dispose_tensor family): set while
|
|
# speculative_moe_backend_context is active, so a draft gate's write also
|
|
# lands on the speculative leaf.
|
|
in_speculative_scope: bool = False
|
|
# Draft construction/execution uses a separate one-sided A2A workspace from
|
|
# the target model's concurrently live CUDA graphs.
|
|
speculative_context: bool = False
|
|
|
|
|
|
class DpFlags(_FlagGroupBase):
|
|
"""DP-attention runtime flags set by ``initialize_dp_attention``.
|
|
|
|
Attention-DP width and rank are stored on ``get_parallel()``.
|
|
"""
|
|
|
|
enabled: bool = False
|
|
use_world_group_for_gather: bool = False
|
|
joiner_skip_all_gather: bool = False
|
|
# Hybrid-SSM models materialize idle ranks via the MAX_LEN fabricated-row
|
|
# conversion (set when hf_config has hybrid_override_pattern).
|
|
max_len_with_idle: bool = False
|
|
# Set while the prefill CUDA graph runner captures; latched by the DP
|
|
# gather/scatter helpers, whose captured geometry needs one shared bucket.
|
|
capturing_prefill_graph: bool = False
|
|
prefill_graph_has_dp_gather: bool = False
|
|
# This process's slot in the target's DP sync while a draft scope narrows
|
|
# attention; dp_gather_width() also reads it as the in-scope marker.
|
|
scoped_gather_slot: Optional[int] = None
|
|
# DP gathered-buffer allocation metadata (model hidden size / dtype /
|
|
# device), set by initialize_dp_attention alongside the flags above.
|
|
buffer_hidden_size: Any = None
|
|
buffer_dtype: Any = None
|
|
buffer_device: Any = None
|
|
|
|
|
|
class SpFlags(_FlagGroupBase):
|
|
"""LayerNorm sequence-parallelism flags, materialized by
|
|
``initialize_layernorm_sp`` (after distributed setup; reads the model
|
|
config). See ``layers.layernorm_sp``."""
|
|
|
|
enabled: bool = False
|
|
|
|
|
|
class Flags(_FlagGroupBase):
|
|
"""Mutable runtime state; resolved configuration lives in the config bags."""
|
|
|
|
capture: CaptureFlags = msgspec.field(default_factory=CaptureFlags)
|
|
moe: MoeFlags = msgspec.field(default_factory=MoeFlags)
|
|
dp: DpFlags = msgspec.field(default_factory=DpFlags)
|
|
sp: SpFlags = msgspec.field(default_factory=SpFlags)
|
|
|
|
|
|
class Resources(_FlagGroupBase):
|
|
"""Process-level resource handles: named slots with one reset lifecycle,
|
|
scoped test injection via ``override()``, and the creation/publish
|
|
semantics kept in the owning modules' accessors (which are thin shims
|
|
over these slots)."""
|
|
|
|
# CUDA graph memory pool shared across the prefill and decode graph
|
|
# backends (created lazily by model_executor.runner_utils.pool).
|
|
graph_memory_pool: Any = None
|
|
graph_pool_borrow: GraphPoolBorrowState | None = None
|
|
# EPLB: per-process recorder and the publish-once location metadata
|
|
# (owning accessors live in sglang.srt.eplb).
|
|
expert_distribution_recorder: Any = None
|
|
expert_location_metadata: Any = None
|
|
# LPLB: layer_id -> solver.
|
|
lplb_solvers: dict = msgspec.field(default_factory=dict)
|
|
# Named side streams (see RuntimeContext.get_stream): name -> stream.
|
|
streams: dict = msgspec.field(default_factory=dict)
|
|
# Named persistent buffers (see RuntimeContext.get_buffer): name -> tensor.
|
|
# Accessors with bespoke semantics (grow-only, per-device keys) manage
|
|
# their entries directly.
|
|
buffers: dict = msgspec.field(default_factory=dict)
|
|
# Persistent reusable CUDA events for non-EP DP TBO, keyed by
|
|
# (kind, subbatch) — see dp_attention._tbo_event for why reuse matters.
|
|
tbo_event_pool: dict = msgspec.field(default_factory=dict)
|
|
flashinfer_megamoe_workspaces: dict = msgspec.field(default_factory=dict)
|
|
# State capturers (installed by their subsystems when capture is on).
|
|
indexer_capturer: Any = None
|
|
experts_capturer: Any = None
|
|
# The shared TCPStore created during distributed initialization.
|
|
tcp_store: Any = None
|
|
# Trace verbosity; the accessor seeds it lazily from SGLANG_TRACE_LEVEL.
|
|
trace_level: Any = None
|
|
|
|
|
|
class LoRABatchLayout(Enum):
|
|
"""Token layout used by LoRA kernels in the current model section."""
|
|
|
|
DP_LOCAL = auto()
|
|
TP_GLOBAL = auto()
|
|
|
|
|
|
class ForwardFlags:
|
|
"""Scoped per-forward flags.
|
|
|
|
Eager-only flags use ContextVars for thread and nested-scope isolation.
|
|
``_GRAPH_VISIBLE`` flags use plain dict slots because Dynamo cannot trace
|
|
ContextVar access. Their readers and writers run on one thread per process.
|
|
``scoped`` restores both backings on exit; ``set`` supports unscoped setters.
|
|
"""
|
|
|
|
_DEFAULTS = {
|
|
"multi_stream": False,
|
|
"moe_output_buffer": None,
|
|
# Attention-TP input-scattering (set per forward by
|
|
# AttnTpContext.maybe_input_scattered / set_attn_inputs).
|
|
"attn_input_scattered": False,
|
|
"attn_inputs": None,
|
|
# Sticky across forwards: every ForwardBatch construction writes it;
|
|
# graph runners force False around capture.
|
|
"is_extend_in_batch": False,
|
|
# Per-layer MLP collective control (set by decoder via scoped()
|
|
# around the MLP / MoE / hybrid mixer call).
|
|
# mlp_reduce_scatter: postprocess will reduce-scatter (skip MLP AR).
|
|
# flashinfer_trtllm_bypass: deepseek dual-stream graph topk bypass.
|
|
# defer_moe_finalize: next layer's fused collective absorbs the finalize.
|
|
"mlp_reduce_scatter": False,
|
|
"flashinfer_trtllm_bypass": False,
|
|
"defer_moe_finalize": False,
|
|
# Attention runs on a DP-local token batch. The MLP runs after the DP
|
|
# gather and therefore uses the TP-global token batch.
|
|
"lora_batch_layout": LoRABatchLayout.DP_LOCAL,
|
|
# LayerNorm sequence parallelism region; see layers/layernorm_sp.py.
|
|
"sp_active": False,
|
|
}
|
|
|
|
# Read/written inside compiled graphs (vocab embedding, layer boundaries,
|
|
# EP dispatch, DP gather/scatter, MLP/MoE skip-AR): plain-slot backed.
|
|
# Before moving a flag out of this set, prove no read/write site sits
|
|
# under torch.compile.
|
|
_GRAPH_VISIBLE = frozenset(
|
|
{
|
|
"attn_input_scattered",
|
|
"attn_inputs",
|
|
"is_extend_in_batch",
|
|
"mlp_reduce_scatter",
|
|
"flashinfer_trtllm_bypass",
|
|
"defer_moe_finalize",
|
|
"lora_batch_layout",
|
|
"sp_active",
|
|
}
|
|
)
|
|
|
|
__slots__ = ("_vars", "_plain")
|
|
|
|
def __init__(self):
|
|
import contextvars
|
|
|
|
object.__setattr__(
|
|
self,
|
|
"_plain",
|
|
{
|
|
name: default
|
|
for name, default in self._DEFAULTS.items()
|
|
if name in self._GRAPH_VISIBLE
|
|
},
|
|
)
|
|
object.__setattr__(
|
|
self,
|
|
"_vars",
|
|
{
|
|
name: contextvars.ContextVar(f"forward.{name}", default=default)
|
|
for name, default in self._DEFAULTS.items()
|
|
if name not in self._GRAPH_VISIBLE
|
|
},
|
|
)
|
|
|
|
def __getattr__(self, name: str) -> Any:
|
|
plain = self._plain
|
|
if name in plain:
|
|
return plain[name]
|
|
try:
|
|
return self._vars[name].get()
|
|
except KeyError:
|
|
raise AttributeError(
|
|
f"ForwardFlags has no flag '{name}' (flags are declared in "
|
|
"ForwardFlags._DEFAULTS; check for typos)"
|
|
) from None
|
|
|
|
def __setattr__(self, name: str, value: Any) -> None:
|
|
raise AttributeError(
|
|
"ForwardFlags is written through scoped(**kw) (or the legacy "
|
|
"set() shim), never by attribute assignment"
|
|
)
|
|
|
|
def set(self, name: str, value: Any) -> None:
|
|
"""Unscoped write for legacy setter shims; persists until the next
|
|
write (current context only, for contextvar-backed flags)."""
|
|
if name in self._plain:
|
|
self._plain[name] = value
|
|
else:
|
|
self._vars[name].set(value)
|
|
|
|
@contextmanager
|
|
def scoped(self, **kwargs):
|
|
"""Set flags for the current scope, restoring on exit. Transactional
|
|
(keys validated before any write) and exception-safe."""
|
|
unknown = set(kwargs) - set(self._DEFAULTS)
|
|
if unknown:
|
|
raise ValueError(f"unknown forward flag(s): {sorted(unknown)}")
|
|
plain_saved = [
|
|
(name, self._plain[name]) for name in kwargs if name in self._plain
|
|
]
|
|
tokens = []
|
|
for name, value in kwargs.items():
|
|
if name in self._plain:
|
|
self._plain[name] = value
|
|
else:
|
|
tokens.append((self._vars[name], self._vars[name].set(value)))
|
|
try:
|
|
yield self
|
|
finally:
|
|
for var, token in reversed(tokens):
|
|
var.reset(token)
|
|
for name, value in reversed(plain_saved):
|
|
self._plain[name] = value
|
|
|
|
|
|
class _ConfigBag:
|
|
"""Resolved namespace with nested bags and guarded writes.
|
|
|
|
Publication snapshots resolved values; ``RuntimeContext.override`` updates
|
|
them permanently and ``override`` restores scoped changes on exit.
|
|
Leaves and sub-bags are real instance attributes so Dynamo can trace reads.
|
|
``_fields`` and ``_subs`` track names for routing and restoration.
|
|
"""
|
|
|
|
def __init__(self, path: str):
|
|
object.__setattr__(self, "_path", path)
|
|
object.__setattr__(self, "_fields", {}) # {leaf: value}
|
|
object.__setattr__(self, "_subs", {}) # {subname: _ConfigBag}
|
|
|
|
def __getattr__(self, name: str) -> Any:
|
|
# Fallback only: real leaves/sub-bags resolve via __dict__ before this
|
|
# runs. Uses object.__getattribute__ (not self._fields) to stay safe if
|
|
# invoked before __init__ populates the bookkeeping dicts.
|
|
fields = object.__getattribute__(self, "_fields")
|
|
if name in fields:
|
|
return fields[name]
|
|
subs = object.__getattribute__(self, "_subs")
|
|
if name in subs:
|
|
return subs[name]
|
|
path = object.__getattribute__(self, "_path")
|
|
raise AttributeError(f"config namespace {path!r} has no leaf/subgroup {name!r}")
|
|
|
|
def __setattr__(self, name: str, value: Any) -> None:
|
|
raise AttributeError(
|
|
f"config namespace {self._path!r} is read-only; write via "
|
|
"get_context().override(source, ...) or the scoped .override(**kw)"
|
|
)
|
|
|
|
def _set(self, name: str, value: Any) -> None:
|
|
"""Update the leaf map and traceable attribute, bypassing the write guard."""
|
|
object.__getattribute__(self, "_fields")[name] = value
|
|
object.__setattr__(self, name, value)
|
|
|
|
def _set_sub(self, name: str, sub: _ConfigBag) -> None:
|
|
"""Register a nested bag in the lookup map and as a traceable attribute."""
|
|
object.__getattribute__(self, "_subs")[name] = sub
|
|
object.__setattr__(self, name, sub)
|
|
|
|
def __contains__(self, name: str) -> bool:
|
|
return name in object.__getattribute__(self, "_fields")
|
|
|
|
@contextmanager
|
|
def override(self, **kwargs):
|
|
"""Temporarily override this bag's leaves, validating all keys before writing.
|
|
|
|
Restores on exit; use ``RuntimeContext.override`` for permanent changes.
|
|
"""
|
|
fields = object.__getattribute__(self, "_fields")
|
|
unknown = set(kwargs) - set(fields)
|
|
if unknown:
|
|
path = object.__getattribute__(self, "_path")
|
|
raise ValueError(f"unknown config leaf for {path!r}: {sorted(unknown)}")
|
|
saved = {name: fields[name] for name in kwargs}
|
|
for name, value in kwargs.items():
|
|
self._set(name, value)
|
|
try:
|
|
yield self
|
|
finally:
|
|
for name, value in saved.items():
|
|
self._set(name, value)
|
|
|
|
|
|
def _build_config_bags(server_args: Any) -> dict:
|
|
"""Project resolved fields by namespace, rejecting leaf/subgroup name collisions."""
|
|
from sglang.srt.arg_groups.arg_utils import namespace_of
|
|
from sglang.srt.arg_groups.overrides import resolution_result
|
|
|
|
_MISSING = object()
|
|
tops: dict = {}
|
|
for field, path in namespace_of(type(server_args)).items():
|
|
value = resolution_result(server_args, field, _MISSING)
|
|
if value is _MISSING:
|
|
raise AttributeError(
|
|
f"config field {field!r} belongs to namespace {path!r} but is absent from "
|
|
f"the published {type(server_args).__name__}; cannot project its bag leaf"
|
|
)
|
|
parts = path.split(".")
|
|
bag = tops.get(parts[0])
|
|
if bag is None:
|
|
bag = tops[parts[0]] = _ConfigBag(parts[0])
|
|
for depth in range(1, len(parts)):
|
|
name = parts[depth]
|
|
if name in object.__getattribute__(bag, "_fields"):
|
|
raise ValueError(
|
|
f"config namespace collision: {'.'.join(parts[: depth + 1])!r} "
|
|
"is declared as both a leaf and a subgroup"
|
|
)
|
|
subs = object.__getattribute__(bag, "_subs")
|
|
child = subs.get(name)
|
|
if child is None:
|
|
child = _ConfigBag(".".join(parts[: depth + 1]))
|
|
bag._set_sub(name, child)
|
|
bag = child
|
|
if field in object.__getattribute__(bag, "_subs"):
|
|
raise ValueError(
|
|
f"config namespace collision: leaf {field!r} under {path!r} "
|
|
"clashes with a subgroup of the same name"
|
|
)
|
|
bag._set(field, value)
|
|
_install_derived_leaves(tops, server_args)
|
|
return tops
|
|
|
|
|
|
def _install_derived_leaves(tops: dict, server_args: Any) -> None:
|
|
"""Compute declared derived leaves once at publication.
|
|
|
|
Derivations receive the whole resolved config because they may span namespaces.
|
|
"""
|
|
import importlib
|
|
|
|
from sglang.srt.arg_groups.arg_utils import _NO_DEFAULT, Derived
|
|
from sglang.srt.arg_groups.overrides import resolved_view
|
|
|
|
namespaces = getattr(type(server_args), "_NAMESPACES", None)
|
|
if not namespaces:
|
|
return
|
|
view = resolved_view(server_args)
|
|
for source in namespaces:
|
|
path = getattr(source, "_NS_PATH", None)
|
|
if path is None:
|
|
continue
|
|
for name, decl in vars(source).items():
|
|
if not isinstance(decl, Derived):
|
|
continue
|
|
if not decl.fn:
|
|
# Initialize runtime-only fields that declare a default.
|
|
if decl.default is not _NO_DEFAULT:
|
|
bag = tops.get(path.split(".")[0])
|
|
for segment in path.split(".")[1:]:
|
|
bag = bag and getattr(bag, segment, None)
|
|
if bag is not None:
|
|
bag._set(name, decl.default)
|
|
continue
|
|
module, _, attr = decl.fn.rpartition(".")
|
|
bag = tops.get(path.split(".")[0])
|
|
for segment in path.split(".")[1:]:
|
|
bag = bag and getattr(bag, segment, None)
|
|
if bag is None:
|
|
continue
|
|
bag._set(name, getattr(importlib.import_module(module), attr)(view))
|
|
|
|
|
|
def _resolved_or_field(server_args: Any, name: str, default: Any) -> Any:
|
|
"""What resolution decided for `name`, falling back to the field.
|
|
|
|
Publishes that carry no config at all (sentinels, mocks) have neither, and
|
|
answer with `default`.
|
|
"""
|
|
if server_args is None:
|
|
return default
|
|
from sglang.srt.arg_groups.overrides import resolution_result
|
|
|
|
decided = resolution_result(server_args, name)
|
|
if decided is not None:
|
|
return decided
|
|
return getattr(server_args, name, default)
|
|
|
|
|
|
class RuntimeContext:
|
|
"""Container for the structured runtime accessors; exposes ``parallel``,
|
|
``server_args``, the resolved config namespace bags, ``flags``,
|
|
``resources``, and ``forward``."""
|
|
|
|
__slots__ = (
|
|
"parallel",
|
|
"_server_args",
|
|
"_config_bags",
|
|
"_overrides_log",
|
|
"_publish_role",
|
|
"flags",
|
|
"resources",
|
|
"forward",
|
|
)
|
|
|
|
def __init__(self, parallel: ParallelContext):
|
|
self.parallel = parallel
|
|
self._server_args: ServerArgs | None = None
|
|
self._config_bags: dict | None = None
|
|
self._overrides_log: list = []
|
|
self._publish_role: str | None = None
|
|
self.flags = Flags()
|
|
self.resources = Resources()
|
|
self.forward = ForwardFlags()
|
|
|
|
def get_stream(self, name: str) -> Any:
|
|
"""Get or create a named process stream. Call before CUDA graph capture."""
|
|
from sglang.srt.arg_groups.overrides import resolution_result
|
|
|
|
stream = self.resources.streams.get(name)
|
|
if stream is None:
|
|
import torch
|
|
|
|
device = (
|
|
resolution_result(self._server_args, "device")
|
|
if self._server_args
|
|
else "cuda"
|
|
)
|
|
stream = torch.get_device_module(device).Stream()
|
|
self.resources.streams[name] = stream
|
|
return stream
|
|
|
|
def set_stream(self, name: str, stream: Any) -> Any:
|
|
"""Install (or replace) the named stream — explicit injection for
|
|
tests and backends that bring their own stream."""
|
|
self.resources.streams[name] = stream
|
|
return stream
|
|
|
|
def get_buffer(self, name: str, factory: Any) -> Any:
|
|
"""Get or create a named persistent buffer using ``factory()``."""
|
|
buf = self.resources.buffers.get(name)
|
|
if buf is None:
|
|
buf = factory()
|
|
self.resources.buffers[name] = buf
|
|
return buf
|
|
|
|
@property
|
|
def server_args(self) -> ServerArgs:
|
|
"""The process-wide ``ServerArgs`` (context-owned slot)."""
|
|
server_args = self._server_args
|
|
if server_args is None:
|
|
# Verbatim legacy message: tests and user scripts may match on it.
|
|
raise ValueError("Global server args is not set yet!")
|
|
return server_args
|
|
|
|
def set_server_args(self, server_args: ServerArgs) -> None:
|
|
"""Replace the raw input and rebuild resolved config bags, clearing override provenance."""
|
|
self.flags.capture.enable_torch_compile = bool(
|
|
_resolved_or_field(server_args, "enable_torch_compile", False)
|
|
)
|
|
self._server_args = server_args
|
|
# Preserve the launcher-assigned device when rebuilding config bags.
|
|
stated = {}
|
|
if self._config_bags is not None:
|
|
device = self._config_bags.get("device")
|
|
fields = object.__getattribute__(device, "_fields") if device else {}
|
|
if "gpu_id" in fields:
|
|
stated["gpu_id"] = fields["gpu_id"]
|
|
self._config_bags = _build_config_bags(server_args)
|
|
device = self._config_bags.get("device")
|
|
if stated and device is not None:
|
|
device._set("gpu_id", stated["gpu_id"])
|
|
spec = self._config_bags.get("spec")
|
|
if spec is not None:
|
|
from sglang.srt.arg_groups.overrides import (
|
|
max_speculative_num_draft_tokens as max_draft_tokens_of,
|
|
)
|
|
|
|
# Keep the launch-time capacity stable while adaptive algorithms
|
|
# change the active width in this bag.
|
|
spec._set(
|
|
"max_speculative_num_draft_tokens",
|
|
max_draft_tokens_of(server_args),
|
|
)
|
|
self.parallel._config = self._config_bags.get("parallel")
|
|
# A direct install is roleless; ``publish`` assigns the role afterwards.
|
|
self._overrides_log = []
|
|
self._publish_role = None
|
|
|
|
def config_bag(self, name: str) -> _ConfigBag:
|
|
"""Return a top-level namespace bag; raise if it has not been published."""
|
|
bags = self._config_bags
|
|
if not bags or name not in bags:
|
|
raise ValueError(f"config namespace {name!r} not published")
|
|
if _ROLE_NS_MODE != "off":
|
|
self._check_role_namespace(name)
|
|
return bags[name]
|
|
|
|
def is_config_namespace_published(self, name: str) -> bool:
|
|
"""Return whether a config namespace exists in the current context."""
|
|
bags = self._config_bags
|
|
return bags is not None and name in bags
|
|
|
|
def _check_role_namespace(self, name: str) -> None:
|
|
# Out of line so the mode gate above stays one dead-branch-prunable
|
|
# check under dynamo in the default "off" mode (config_bag runs inside
|
|
# compiled model forwards).
|
|
role = self._publish_role
|
|
if _ROLE_NS_MODE == "record":
|
|
if not _is_compiling():
|
|
_record_namespace_read(role, name)
|
|
elif _ROLE_NS_MODE == "enforce" and role is not None:
|
|
if role not in ROLE_NAMESPACE_SETS:
|
|
raise ValueError(
|
|
f"publish role {role!r} has no ROLE_NAMESPACE_SETS entry; "
|
|
"declare its namespace set (None for the full tree)."
|
|
)
|
|
allowed = ROLE_NAMESPACE_SETS[role]
|
|
if allowed is not None and name not in allowed:
|
|
raise ValueError(
|
|
f"config namespace {name!r} is outside the declared set "
|
|
f"for publish role {role!r} ({sorted(allowed)}). If this "
|
|
"read is legitimate for the process type, extend "
|
|
"ROLE_NAMESPACE_SETS; if not, the read belongs in a "
|
|
"different process or behind a per-instance boundary."
|
|
)
|
|
|
|
def override(self, source: str, **fields) -> None:
|
|
"""Update resolved config leaves without changing the raw input.
|
|
|
|
Validate all field names and target bags before writing. Record ``source``
|
|
and the changed fields for provenance.
|
|
"""
|
|
if not fields:
|
|
return
|
|
bags = self._config_bags
|
|
if bags is None:
|
|
raise ValueError("config not published; cannot override")
|
|
from sglang.srt.arg_groups.arg_utils import namespace_of
|
|
|
|
nsmap = namespace_of(type(self._server_args))
|
|
targets = [] # (bag, leaf, value) — resolved before any write
|
|
for name, value in fields.items():
|
|
path = nsmap.get(name)
|
|
if path is None:
|
|
raise ValueError(
|
|
f"override: unknown config field {name!r} (no namespace) — "
|
|
"not a resolved config leaf"
|
|
)
|
|
parts = path.split(".")
|
|
bag = bags.get(parts[0])
|
|
if bag is None:
|
|
raise ValueError(f"override: namespace {parts[0]!r} not published")
|
|
for seg in parts[1:]:
|
|
bag = object.__getattribute__(bag, "_subs").get(seg)
|
|
if bag is None:
|
|
raise ValueError(
|
|
f"override: subgroup {seg!r} missing under {path!r}"
|
|
)
|
|
if name not in bag:
|
|
raise ValueError(f"override: field {name!r} not projected on {path!r}")
|
|
targets.append((bag, name, value))
|
|
for bag, name, value in targets:
|
|
bag._set(name, value)
|
|
self._overrides_log.append((source, dict(fields)))
|
|
|
|
def config_leaf(self, name: str):
|
|
"""Read a resolved config leaf by its flat field name."""
|
|
bags = self._config_bags
|
|
if bags is None:
|
|
raise ValueError("config not published; cannot read a config leaf")
|
|
from sglang.srt.arg_groups.arg_utils import namespace_of
|
|
|
|
path = namespace_of(type(self._server_args)).get(name)
|
|
if path is None:
|
|
raise ValueError(f"{name!r} is not a config leaf (no namespace)")
|
|
parts = path.split(".")
|
|
bag = self.config_bag(parts[0])
|
|
for seg in parts[1:]:
|
|
bag = object.__getattribute__(bag, "_subs").get(seg)
|
|
if bag is None:
|
|
raise ValueError(f"subgroup {seg!r} missing under {path!r}")
|
|
return getattr(bag, name)
|
|
|
|
def overrides_log(self) -> list:
|
|
"""Return ``(source, fields)`` entries with shallow copies of the field dicts."""
|
|
return [(source, dict(fields)) for source, fields in self._overrides_log]
|
|
|
|
def resolved_server_args_dict(self, base: dict | None = None) -> dict:
|
|
"""Overlay this process's runtime overrides on the resolved startup config.
|
|
|
|
``base`` defaults to ``server_args.resolved_dict()``. Scheduler readbacks
|
|
include these overrides in ``/server_info``'s ``internal_states`` block;
|
|
the top-level fields describe startup config. TokenizerManager separately
|
|
overlays its per-instance model identity.
|
|
"""
|
|
d = self.server_args.resolved_dict() if base is None else dict(base)
|
|
for _source, fields in self._overrides_log:
|
|
d.update(fields)
|
|
return d
|
|
|
|
def override_server_args(self, **fields) -> _ServerArgsOverride:
|
|
"""Publish a temporary dummy config for tests and restore the previous context.
|
|
|
|
Use as a context manager or call ``install`` / ``restore`` explicitly.
|
|
Prefer a bag's scoped ``override`` when only existing leaves need changing.
|
|
"""
|
|
return _ServerArgsOverride(self, fields)
|
|
|
|
|
|
class _ServerArgsOverride:
|
|
"""Scoped config override (see ``RuntimeContext.override_server_args``).
|
|
|
|
Deliberately a plain class rather than a generator context manager:
|
|
fixtures that live for a whole test case install the override without a
|
|
``with`` block, and a suspended generator would run its restore whenever
|
|
the garbage collector closes it — un-publishing the active config at a
|
|
nondeterministic point.
|
|
"""
|
|
|
|
__slots__ = (
|
|
"_context",
|
|
"_fields",
|
|
"_prev_server_args",
|
|
"_prev_bags",
|
|
"_prev_overrides_log",
|
|
"_prev_publish_role",
|
|
"_prev_parallel_config",
|
|
"_prev_capture",
|
|
"_installed",
|
|
)
|
|
|
|
def __init__(self, context: RuntimeContext, fields: dict):
|
|
self._context = context
|
|
self._fields = fields
|
|
self._installed = False
|
|
|
|
def install(self) -> ServerArgs:
|
|
"""Publish a fresh dummy-boundary ``ServerArgs`` carrying the
|
|
overrides; returns the published instance."""
|
|
from sglang.srt.server_args import ServerArgs
|
|
|
|
assert not self._installed, "override_server_args already installed"
|
|
ctx = self._context
|
|
self._prev_server_args = ctx._server_args
|
|
self._prev_bags = ctx._config_bags
|
|
self._prev_overrides_log = ctx._overrides_log
|
|
self._prev_publish_role = ctx._publish_role
|
|
self._prev_parallel_config = ctx.parallel._config
|
|
self._prev_capture = ctx.flags.capture.enable_torch_compile
|
|
from sglang.srt.arg_groups.overrides import (
|
|
declare_resolution,
|
|
)
|
|
|
|
server_args = ServerArgs(model_path="dummy")
|
|
server_args.resolve_once()
|
|
# Underscore names seed private property caches (the strict guard
|
|
# exempts them); everything else must be a real config field.
|
|
unknown = {name for name in self._fields if not name.startswith("_")} - set(
|
|
type(server_args).__struct_fields__
|
|
)
|
|
if unknown:
|
|
raise ValueError(
|
|
f"override_server_args: unknown ServerArgs field(s): {sorted(unknown)}"
|
|
)
|
|
# Declare config fields even when underscore-prefixed; only non-fields seed caches.
|
|
fields = set(type(server_args).__struct_fields__)
|
|
declared = {n: v for n, v in self._fields.items() if n in fields}
|
|
if declared:
|
|
declare_resolution(server_args, "override_server_args", **declared)
|
|
seeds = {n: v for n, v in self._fields.items() if n not in fields}
|
|
for name, value in seeds.items():
|
|
msgspec.Struct.__setattr__(server_args, name, value)
|
|
ctx.set_server_args(server_args)
|
|
self._installed = True
|
|
return server_args
|
|
|
|
def restore(self) -> None:
|
|
"""Reinstate the exact pre-install lifecycle state (or the empty slot)."""
|
|
if not self._installed:
|
|
return
|
|
self._installed = False
|
|
ctx = self._context
|
|
ctx._server_args = self._prev_server_args
|
|
ctx._config_bags = self._prev_bags
|
|
ctx._overrides_log = self._prev_overrides_log
|
|
ctx._publish_role = self._prev_publish_role
|
|
ctx.parallel._config = self._prev_parallel_config
|
|
ctx.flags.capture.enable_torch_compile = self._prev_capture
|
|
self._prev_server_args = None
|
|
self._prev_bags = None
|
|
self._prev_overrides_log = None
|
|
self._prev_parallel_config = None
|
|
|
|
def __enter__(self) -> ServerArgs:
|
|
return self.install()
|
|
|
|
def __exit__(self, *exc) -> None:
|
|
self.restore()
|
|
|
|
|
|
_PARALLEL = ParallelContext()
|
|
_CONTEXT = RuntimeContext(parallel=_PARALLEL)
|
|
|
|
|
|
def get_context() -> RuntimeContext:
|
|
return _CONTEXT
|
|
|
|
|
|
def get_parallel() -> ParallelContext:
|
|
return _PARALLEL
|
|
|
|
|
|
def get_server_args() -> ServerArgs:
|
|
return _CONTEXT.server_args
|
|
|
|
|
|
def get_flags() -> Flags:
|
|
return _CONTEXT.flags
|
|
|
|
|
|
def get_resources() -> Resources:
|
|
return _CONTEXT.resources
|
|
|
|
|
|
def get_forward() -> ForwardFlags:
|
|
return _CONTEXT.forward
|
|
|
|
|
|
# --- Resolved config namespaces -------------------------
|
|
# Each returns the top-level snapshot bag; reads are `get_exec().moe.field` etc.
|
|
# All fail with ValueError("... not published") until publish has projected them.
|
|
# ``parallel`` has no bag getter: ``get_parallel()`` answers its leaves
|
|
# directly, alongside the live topology they belong to.
|
|
def get_device() -> _ConfigBag:
|
|
return _CONTEXT.config_bag("device")
|
|
|
|
|
|
def get_model() -> _ConfigBag:
|
|
return _CONTEXT.config_bag("model")
|
|
|
|
|
|
def get_exec() -> _ConfigBag:
|
|
return _CONTEXT.config_bag("exec")
|
|
|
|
|
|
def get_schedule() -> _ConfigBag:
|
|
return _CONTEXT.config_bag("schedule")
|
|
|
|
|
|
def get_memory() -> _ConfigBag:
|
|
return _CONTEXT.config_bag("memory")
|
|
|
|
|
|
def get_spec() -> _ConfigBag:
|
|
return _CONTEXT.config_bag("spec")
|
|
|
|
|
|
def get_lora() -> _ConfigBag:
|
|
return _CONTEXT.config_bag("lora")
|
|
|
|
|
|
def get_mm() -> _ConfigBag:
|
|
return _CONTEXT.config_bag("mm")
|
|
|
|
|
|
def get_disagg() -> _ConfigBag:
|
|
return _CONTEXT.config_bag("disagg")
|
|
|
|
|
|
def get_serving() -> _ConfigBag:
|
|
return _CONTEXT.config_bag("serving")
|
|
|
|
|
|
def get_observability() -> _ConfigBag:
|
|
return _CONTEXT.config_bag("observability")
|
|
|
|
|
|
# --- Per-role namespace sets -------------------------------------------
|
|
#
|
|
# ``publish(role=...)`` records which process type installed the config; this
|
|
# table declares which top-level config namespaces each role reads. ``None``
|
|
# means the full tree — either the role genuinely needs everything (scheduler)
|
|
# or its deployment shape has not been audited yet (restrict only what smoke
|
|
# coverage can verify). ``parallel`` is served by ``get_parallel()`` and
|
|
# every process legitimately reads topology config, so it is not in this table.
|
|
#
|
|
# ``SGLANG_ROLE_NAMESPACES`` selects the mode (read once at import):
|
|
# off (default) no bookkeeping, zero overhead;
|
|
# record audit mode — collect (role, namespace) reads per process and dump
|
|
# them at exit (the data that seeds this table). Reads made inside
|
|
# torch.compile-traced code are NOT observed (recording is pruned
|
|
# under tracing to keep capture legal) — run audits with
|
|
# compilation disabled before restricting a role.
|
|
# enforce fail closed — a bag read outside the role's declared set raises.
|
|
ROLE_NAMESPACE_SETS: dict[str, frozenset[str] | None] = {
|
|
# Reads (almost) everything by design — the model-executing process.
|
|
"scheduler": None,
|
|
"test": None,
|
|
"dp_controller": frozenset(
|
|
{"exec", "parallel", "device", "disagg", "observability", "serving"}
|
|
),
|
|
# Keep unrestricted until multimodal, LoRA, disaggregation, and gRPC reads are audited.
|
|
"tokenizer": None,
|
|
# Deployment shapes not exercised locally; audit before restricting.
|
|
"detokenizer": None,
|
|
"encoder": None,
|
|
"expert_backup": None,
|
|
"weight_cache_daemon": None,
|
|
# The diffusion GPU worker runs a model and publishes a placeholder so
|
|
# shared SRT reads do not fail closed; declared full for that reason.
|
|
"diffusion_gpu_worker": None,
|
|
}
|
|
|
|
|
|
def _validated_role_ns_mode(value: str) -> str:
|
|
mode = value.strip().lower()
|
|
if mode not in ("off", "record", "enforce"):
|
|
raise ValueError(
|
|
f"SGLANG_ROLE_NAMESPACES={value!r} is not one of off / record / "
|
|
"enforce — refusing to guess (a typo here would silently disable "
|
|
"enforcement)."
|
|
)
|
|
return mode
|
|
|
|
|
|
def _role_ns_mode_from_env() -> str:
|
|
# Resolved once at import so the config_bag gate stays a dynamo-prunable
|
|
# constant; validated fail-loud here (EnvField's warn-and-default parse
|
|
# would silently turn a typo into "off").
|
|
from sglang.srt.environ import envs
|
|
|
|
return _validated_role_ns_mode(envs.SGLANG_ROLE_NAMESPACES.get())
|
|
|
|
|
|
_ROLE_NS_MODE = _role_ns_mode_from_env()
|
|
_RECORDED_NS_READS: set[tuple[str | None, str]] = set()
|
|
_RECORD_DUMP_REGISTERED = False
|
|
|
|
|
|
def _is_compiling() -> bool:
|
|
# Recording has Python side effects (set mutation, file I/O, atexit) that
|
|
# must never run under tracing; torch.compiler.is_compiling() is dynamo's
|
|
# sanctioned probe. The function-level import keeps this module
|
|
# import-light; a sys.modules lookup here breaks fullgraph tracing (dynamo
|
|
# enumerates the dict, which other imports mutate mid-trace).
|
|
import torch
|
|
|
|
return torch.compiler.is_compiling()
|
|
|
|
|
|
def _ensure_record_dump_registered() -> None:
|
|
global _RECORD_DUMP_REGISTERED
|
|
if not _RECORD_DUMP_REGISTERED:
|
|
_RECORD_DUMP_REGISTERED = True
|
|
import atexit
|
|
|
|
atexit.register(_dump_recorded_namespace_reads)
|
|
|
|
|
|
def _append_role_ns_out(role: str | None, name: str) -> None:
|
|
# Persist immediately: worker processes are routinely torn down with
|
|
# signals that skip atexit, and the audit must survive that.
|
|
from sglang.srt.environ import envs
|
|
|
|
out = envs.SGLANG_ROLE_NAMESPACES_OUT.get()
|
|
if not out:
|
|
return
|
|
try:
|
|
with open(out, "a") as f:
|
|
f.write(f"{role} {name}\n")
|
|
except OSError as e:
|
|
# The entry stays in the in-memory set; the exit summary still covers it.
|
|
print(
|
|
f"[role-namespaces] pid={os.getpid()} failed to append "
|
|
f"({role}, {name}) to {out!r}: {e}",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
|
|
|
|
def _record_namespace_read(role: str | None, name: str) -> None:
|
|
if (role, name) in _RECORDED_NS_READS:
|
|
return
|
|
_RECORDED_NS_READS.add((role, name))
|
|
_append_role_ns_out(role, name)
|
|
_ensure_record_dump_registered()
|
|
|
|
|
|
def _dump_recorded_namespace_reads() -> None:
|
|
"""Emit the record-mode audit: one line per role with the namespaces its
|
|
process actually read (multi-process runs dump once per process). The
|
|
process's own publish role is always included, so a zero-read role emits
|
|
an (empty) line rather than being indistinguishable from a process where
|
|
recording never ran."""
|
|
by_role: dict = {}
|
|
own_role = _CONTEXT._publish_role
|
|
if own_role is not None:
|
|
by_role.setdefault(own_role, set())
|
|
for role, name in _RECORDED_NS_READS:
|
|
if name == "-": # publish-time marker, not a namespace read
|
|
by_role.setdefault(role, set())
|
|
continue
|
|
by_role.setdefault(role, set()).add(name)
|
|
for role in sorted(by_role, key=str):
|
|
print(
|
|
f"[role-namespaces] pid={os.getpid()} role={role} "
|
|
f"read={','.join(sorted(by_role[role]))}",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
|
|
|
|
def publish(
|
|
server_args,
|
|
*,
|
|
role: str,
|
|
hf_config: Any = None,
|
|
ranks: SpawnRanks | None = None,
|
|
) -> RuntimeContext:
|
|
"""Install process-wide config for this OS process.
|
|
|
|
Records the process ``role`` — one of the ``ROLE_NAMESPACE_SETS`` keys,
|
|
which is the one place the roles are enumerated — and
|
|
projects the config bags. Draft workers skip publish (they must not clobber
|
|
the target). ``role`` is provenance, and — when ``SGLANG_ROLE_NAMESPACES``
|
|
is ``enforce`` — the key into ``ROLE_NAMESPACE_SETS`` for fail-closed
|
|
namespace-read enforcement (``record`` audits the reads instead).
|
|
``hf_config`` is accepted for forward-compat and currently unused.
|
|
|
|
``ranks`` supplies launcher placement for roles that participate in the
|
|
parallel topology. Without it, rank reads require an explicit override,
|
|
except for ``attn_dcp_rank=0`` when DCP is disabled.
|
|
|
|
A process holds at most one live config: the bags always describe the
|
|
engine running now. Re-publish is allowed and is **last-publish-wins**
|
|
(bags re-projected, provenance reset, role overwritten), which is what
|
|
lets one process rebuild an engine after shutting the previous one down.
|
|
"""
|
|
if _ROLE_NS_MODE == "enforce" and role not in ROLE_NAMESPACE_SETS:
|
|
# Fail closed at publish time, not at the first stray read.
|
|
raise ValueError(
|
|
f"publish role {role!r} has no ROLE_NAMESPACE_SETS entry; declare "
|
|
"its namespace set (None for the full tree)."
|
|
)
|
|
server_args.resolve_once()
|
|
discarded = _CONTEXT.overrides_log()
|
|
_CONTEXT.set_server_args(server_args)
|
|
if discarded:
|
|
logger.warning(
|
|
"publish(role=%s) re-projected the config bags and dropped %d "
|
|
"override(s) taken since the last publish: %s",
|
|
role,
|
|
len(discarded),
|
|
", ".join(
|
|
f"{source}({', '.join(sorted(fields))})" for source, fields in discarded
|
|
),
|
|
)
|
|
_CONTEXT._publish_role = role
|
|
# Disabled DCP has rank zero even in processes without a rank bundle.
|
|
if not _CONTEXT.parallel.dcp_enabled:
|
|
_CONTEXT.parallel.override_permanently(attn_dcp_rank=0)
|
|
# The device is assigned by the launcher; it is not a config field.
|
|
_CONTEXT.config_bag("device")._set(
|
|
"gpu_id", ranks.gpu_id if ranks is not None else None
|
|
)
|
|
if ranks is not None:
|
|
parallel = _CONTEXT.parallel
|
|
placement = derive_spawn_ranks(
|
|
world_rank=ranks.world_rank,
|
|
tp_size=parallel.tp_size,
|
|
ep_join_rank_offset=parallel.ep_join_rank_offset,
|
|
attn_cp_size=parallel.attn_cp_size,
|
|
attn_tp_size=parallel.attn_tp_size,
|
|
moe_dp_size=parallel.moe_dp_size,
|
|
moe_ep_size=parallel.moe_ep_size,
|
|
)
|
|
# MoE-DP aliases the attention-CP group when CP is wider.
|
|
if parallel.moe_dp_size < parallel.attn_cp_size:
|
|
placement["moe_dp_rank"] = placement["attn_cp_rank"]
|
|
# `None` means no data-parallel controller.
|
|
placement["dp_rank"] = ranks.dp_rank
|
|
placement["launch_world_rank"] = ranks.world_rank
|
|
placement.update(_attention_ranks(parallel, placement["tp_rank"]))
|
|
# DCP groups are contiguous slices of a TP group.
|
|
if parallel.dcp_enabled:
|
|
placement["dcp_rank"] = placement["tp_rank"] % parallel.dcp_size
|
|
placement["attn_dcp_rank"] = placement.get("dcp_rank", 0)
|
|
parallel.override_permanently(**placement)
|
|
_validate_parallel(parallel, "publish")
|
|
if _ROLE_NS_MODE == "record":
|
|
# The '-' marker distinguishes a zero-read role from a process where
|
|
# recording never ran (signal teardown skips atexit).
|
|
_record_namespace_read(role, "-")
|
|
print(
|
|
f"[role-namespaces] pid={os.getpid()} role={role} recording; note: "
|
|
"reads inside torch.compile-traced code are not observed — audit "
|
|
"with compilation disabled before restricting a role.",
|
|
file=sys.stderr,
|
|
flush=True,
|
|
)
|
|
return _CONTEXT
|
|
|
|
|
|
def _attention_ranks(parallel, tp_rank: int) -> dict:
|
|
"""Derive attention ranks from the configured widths and the TP rank."""
|
|
attn_tp_rank, attn_dp_rank = derive_attention_ranks(
|
|
tp_rank=tp_rank,
|
|
attn_tp_size=parallel.attn_tp_size,
|
|
attn_cp_size=parallel.attn_cp_size,
|
|
)
|
|
return {"attn_tp_rank": attn_tp_rank, "attn_dp_rank": attn_dp_rank}
|
|
|
|
|
|
def assert_published(server_args, *, role: str) -> RuntimeContext:
|
|
"""Require this record and role to have been published by the process entry.
|
|
|
|
Re-publishing here would discard runtime overrides and their provenance.
|
|
"""
|
|
if _CONTEXT._server_args is server_args and _CONTEXT._publish_role == role:
|
|
return _CONTEXT
|
|
if _CONTEXT._server_args is None:
|
|
detail = "nothing is published in this process"
|
|
elif _CONTEXT._server_args is not server_args:
|
|
detail = (
|
|
"a different record is published "
|
|
f"(role={_CONTEXT._publish_role!r}); this constructor was handed "
|
|
"one the process never published"
|
|
)
|
|
else:
|
|
detail = (
|
|
f"this record is published under role "
|
|
f"{_CONTEXT._publish_role!r}, not {role!r}"
|
|
)
|
|
raise RuntimeError(
|
|
f"config not published for role {role!r}: {detail}. The process entry "
|
|
"publishes -- add publish(server_args, role=...) there rather than "
|
|
"publishing from a constructor."
|
|
)
|
|
|
|
|
|
def publish_role() -> str | None:
|
|
"""The role recorded by the last ``publish`` (None for a legacy set)."""
|
|
return _CONTEXT._publish_role
|
|
|
|
|
|
def get_stream(name: str) -> Any:
|
|
return _CONTEXT.get_stream(name)
|
|
|
|
|
|
def set_stream(name: str, stream: Any) -> Any:
|
|
return _CONTEXT.set_stream(name, stream)
|
|
|
|
|
|
def get_buffer(name: str, factory: Any) -> Any:
|
|
return _CONTEXT.get_buffer(name, factory)
|
|
|
|
|
|
_GLOBAL_DWDP_MANAGER: Any = None
|
|
|
|
|
|
def get_global_dwdp_manager() -> Any:
|
|
return _GLOBAL_DWDP_MANAGER
|
|
|
|
|
|
def set_global_dwdp_manager(manager: Any) -> None:
|
|
global _GLOBAL_DWDP_MANAGER
|
|
_GLOBAL_DWDP_MANAGER = manager
|
|
|
|
|
|
def _group_leaves(group: _FlagGroupBase) -> dict[str, Any]:
|
|
"""The leaf values of a flag group, recursively."""
|
|
leaves: dict[str, Any] = {}
|
|
for name in type(group).__struct_fields__:
|
|
value = getattr(group, name)
|
|
if isinstance(value, _FlagGroupBase):
|
|
leaves[name] = _group_leaves(value)
|
|
elif isinstance(value, (dict, list)):
|
|
leaves[name] = type(value)(value)
|
|
else:
|
|
leaves[name] = value
|
|
return leaves
|
|
|
|
|
|
def _restore_leaves(group: _FlagGroupBase, leaves: dict[str, Any]) -> None:
|
|
for name, value in leaves.items():
|
|
current = getattr(group, name)
|
|
if isinstance(current, _FlagGroupBase):
|
|
_restore_leaves(current, value)
|
|
elif isinstance(current, dict):
|
|
current.clear()
|
|
current.update(value)
|
|
elif isinstance(current, list):
|
|
current[:] = value
|
|
else:
|
|
setattr(group, name, value)
|
|
|
|
|
|
def snapshot_context() -> dict[str, Any]:
|
|
"""Everything a publish replaces, so a failed launch can put it back.
|
|
|
|
Enumerated from ``__slots__`` rather than listed by hand: a hand-picked copy
|
|
of context state is one field behind the day a slot is added, and the copy
|
|
that silently drops one is worse than none. Flag groups are snapshotted by
|
|
leaf, not by reference: publish writes *into* the same ``Flags`` object
|
|
(``capture.enable_torch_compile``), so a reference held here would already
|
|
carry the failed launch's value by the time it is put back.
|
|
"""
|
|
state: dict[str, Any] = {}
|
|
for name in RuntimeContext.__slots__:
|
|
if name == "parallel":
|
|
continue
|
|
value = getattr(_CONTEXT, name)
|
|
if isinstance(value, _FlagGroupBase):
|
|
state[name] = (value, _group_leaves(value))
|
|
elif isinstance(value, list):
|
|
state[name] = list(value)
|
|
else:
|
|
state[name] = value
|
|
state["__parallel__"] = {
|
|
name: getattr(_CONTEXT.parallel, name)
|
|
for name in type(_CONTEXT.parallel).__slots__
|
|
}
|
|
state["__dwdp__"] = get_global_dwdp_manager()
|
|
return state
|
|
|
|
|
|
def restore_context(state: dict[str, Any]) -> None:
|
|
"""Put back what ``snapshot_context`` captured."""
|
|
for name in RuntimeContext.__slots__:
|
|
if name == "parallel":
|
|
continue
|
|
value = state[name]
|
|
if isinstance(value, tuple) and isinstance(value[0], _FlagGroupBase):
|
|
group, leaves = value
|
|
setattr(_CONTEXT, name, group)
|
|
_restore_leaves(group, leaves)
|
|
else:
|
|
setattr(_CONTEXT, name, value)
|
|
for name, value in state["__parallel__"].items():
|
|
setattr(_CONTEXT.parallel, name, value)
|
|
set_global_dwdp_manager(state["__dwdp__"])
|
|
|
|
|
|
def reset_context() -> None:
|
|
"""Clear published configuration, parallel overrides, flags, and resources.
|
|
|
|
Used for test teardown and runtime lifecycle reset.
|
|
"""
|
|
_CONTEXT._server_args = None
|
|
_CONTEXT._config_bags = None
|
|
_CONTEXT._overrides_log = []
|
|
_CONTEXT._publish_role = None
|
|
_CONTEXT.parallel._config = None
|
|
_CONTEXT.parallel.clear_stamp()
|
|
_CONTEXT.flags = Flags()
|
|
_CONTEXT.resources = Resources()
|
|
_CONTEXT.forward = ForwardFlags()
|
|
set_global_dwdp_manager(None)
|
|
|
|
|
|
def remote_instance_transfer_engine_enabled(load_format: str | None = None) -> bool:
|
|
"""Check transfer-engine loading against current model config.
|
|
|
|
``load_format`` can override the process value for a draft runner.
|
|
"""
|
|
from sglang.srt.arg_groups.overrides import remote_instance_transfer_engine_of
|
|
|
|
return remote_instance_transfer_engine_of(get_model(), load_format)
|
|
|
|
|
|
def max_prefill_buffer_tokens() -> int:
|
|
"""Return the registered prefill ceiling, or derive it from current config.
|
|
|
|
PP dynamic chunking can grow toward ``max_prefill_tokens`` and probe at 1.25x.
|
|
"""
|
|
schedule = get_schedule()
|
|
chunked = (
|
|
schedule.chunked_prefill_size
|
|
if schedule.chunked_prefill_size and schedule.chunked_prefill_size > 0
|
|
else 0
|
|
)
|
|
tokens = chunked
|
|
if schedule.enable_dynamic_chunking and get_parallel().pp_size > 1 and chunked:
|
|
tokens = max(
|
|
tokens, schedule.max_prefill_tokens or 0, math.ceil(chunked * 1.25)
|
|
)
|
|
return prefill_buffer_ceiling_of(get_server_args(), tokens)
|
|
|
|
|
|
def pre_capture_activation_reserve_mb(gpu_mem: float | None) -> float:
|
|
"""Return the activation reserve in MB before CUDA graph capture, using current config."""
|
|
schedule = get_schedule()
|
|
if get_disagg().disaggregation_mode == "decode":
|
|
running_requests = (
|
|
schedule.max_running_requests
|
|
or get_exec().graph.cuda_graph_config.decode.max_bs
|
|
or 1
|
|
)
|
|
activation_tokens = max(
|
|
running_requests * (get_spec().speculative_num_draft_tokens or 1), 2048
|
|
)
|
|
elif schedule.chunked_prefill_size > 0:
|
|
activation_tokens = max(schedule.chunked_prefill_size, 2048)
|
|
else:
|
|
activation_tokens = max(schedule.max_prefill_tokens, 2048)
|
|
parallel = get_parallel()
|
|
reserved_mem = (
|
|
512 + activation_tokens * 1.5 + parallel.tp_size * parallel.pp_size / 8 * 1024
|
|
)
|
|
if gpu_mem is not None and gpu_mem > 60 * 1024:
|
|
reserved_mem = max(reserved_mem, 10 * 1024)
|
|
return reserved_mem
|
|
|
|
|
|
# Platform probes work before publication and support scoped overrides.
|
|
|
|
_PLATFORM_PROBES: Dict[str, str] = {
|
|
"is_cuda": "is_cuda",
|
|
"is_hip": "is_hip",
|
|
"is_npu": "is_npu",
|
|
"is_xpu": "is_xpu",
|
|
"is_musa": "is_musa",
|
|
"is_mps": "is_mps",
|
|
"is_sm90": "is_sm90_supported",
|
|
"is_sm100": "is_sm100_supported",
|
|
"is_sm100_or_sm110": "is_sm100_or_sm110_supported",
|
|
"is_sm110": "is_sm110_supported",
|
|
"is_sm120": "is_sm120_supported",
|
|
"is_blackwell": "is_blackwell_supported",
|
|
"is_hopper_with_cuda_12_3": "is_hopper_with_cuda_12_3",
|
|
"has_amx": "cpu_has_amx_support",
|
|
"has_flashinfer": "is_flashinfer_available",
|
|
}
|
|
|
|
# Not yes/no facts, same address.
|
|
_PLATFORM_VALUES: Dict[str, str] = {
|
|
"device_sm": "get_device_sm",
|
|
"device_capability": "get_device_capability",
|
|
}
|
|
|
|
|
|
class PlatformContext:
|
|
"""The machine's own facts, with one place to override them.
|
|
|
|
Every name maps to a probe in `utils.common`; the probes are
|
|
`lru_cache`-d, so reading through here costs a call and a dict lookup
|
|
(~26 ns) rather than a device query.
|
|
"""
|
|
|
|
__slots__ = ("_overrides",)
|
|
|
|
def __init__(self) -> None:
|
|
object.__setattr__(self, "_overrides", {})
|
|
|
|
def __getattr__(self, name: str) -> Any:
|
|
probe = _PLATFORM_PROBES.get(name) or _PLATFORM_VALUES.get(name)
|
|
if probe is None:
|
|
known = sorted(set(_PLATFORM_PROBES) | set(_PLATFORM_VALUES))
|
|
raise AttributeError(
|
|
f"unknown platform fact {name!r}; known: {', '.join(known)}"
|
|
)
|
|
overrides = object.__getattribute__(self, "_overrides")
|
|
if name in overrides:
|
|
return overrides[name]
|
|
from sglang.srt.utils import common as _common
|
|
|
|
return getattr(_common, probe)()
|
|
|
|
def __setattr__(self, name: str, value: Any) -> None:
|
|
raise AttributeError(
|
|
"platform facts are not assigned; use "
|
|
"`sglang.srt.runtime_context.override_platform(...)` so every "
|
|
"reader agrees"
|
|
)
|
|
|
|
def _install(self, **facts: Any) -> Dict[str, Any]:
|
|
unknown = set(facts) - set(_PLATFORM_PROBES) - set(_PLATFORM_VALUES)
|
|
if unknown:
|
|
raise ValueError(f"unknown platform fact(s): {sorted(unknown)}")
|
|
overrides = object.__getattribute__(self, "_overrides")
|
|
previous = {k: overrides[k] for k in facts if k in overrides}
|
|
missing = [k for k in facts if k not in overrides]
|
|
overrides.update(facts)
|
|
return {"previous": previous, "missing": missing}
|
|
|
|
def _restore(self, saved: Dict[str, Any]) -> None:
|
|
overrides = object.__getattribute__(self, "_overrides")
|
|
overrides.update(saved["previous"])
|
|
for k in saved["missing"]:
|
|
overrides.pop(k, None)
|
|
|
|
|
|
_PLATFORM = PlatformContext()
|
|
|
|
|
|
def get_platform() -> PlatformContext:
|
|
"""The machine's facts. Answers before publish, unlike a config bag."""
|
|
return _PLATFORM
|
|
|
|
|
|
class _PlatformOverride:
|
|
"""Scoped platform override: `with override_platform(is_sm100=True): ...`"""
|
|
|
|
__slots__ = ("_facts", "_saved")
|
|
|
|
def __init__(self, **facts: Any) -> None:
|
|
self._facts = facts
|
|
self._saved = None
|
|
|
|
def install(self) -> PlatformContext:
|
|
self._saved = _PLATFORM._install(**self._facts)
|
|
return _PLATFORM
|
|
|
|
def restore(self) -> None:
|
|
if self._saved is not None:
|
|
_PLATFORM._restore(self._saved)
|
|
self._saved = None
|
|
|
|
def __enter__(self) -> PlatformContext:
|
|
return self.install()
|
|
|
|
def __exit__(self, *exc: Any) -> None:
|
|
self.restore()
|
|
|
|
def __call__(self, fn: Any) -> Any:
|
|
"""Also usable as a decorator, like the `patch` it replaces.
|
|
|
|
A fresh scope per call: the same object decorating two tests must not
|
|
share one saved state.
|
|
"""
|
|
import functools
|
|
|
|
facts = dict(self._facts)
|
|
|
|
@functools.wraps(fn)
|
|
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
with _PlatformOverride(**facts):
|
|
return fn(*args, **kwargs)
|
|
|
|
return wrapper
|
|
|
|
|
|
def override_platform(**facts: Any) -> _PlatformOverride:
|
|
"""Say what kind of machine this is, once, for every reader."""
|
|
return _PlatformOverride(**facts)
|
|
|
|
|
|
# Derived config accessors use the process model (the target during speculation).
|
|
|
|
|
|
def mamba_cache_chunk_size() -> int:
|
|
"""The caching point granularity for mamba state: ``max(the model's mamba
|
|
chunk size, page_size)``. Cached on the config after the first call."""
|
|
from sglang.srt.arg_groups.overrides import mamba_cache_chunk_size as _of
|
|
|
|
return _of(get_server_args())
|
|
|
|
|
|
def mamba_checkpoint_grid(tree_page: int) -> int:
|
|
"""The granularity a donated mamba checkpoint's depth must land on so the
|
|
radix tree can name it. Pass the page the tree actually allocates on: DCP
|
|
widens it past ``mamba_cache_chunk_size``, and deriving that here would be a
|
|
second copy of a predicate that already lives in the cache builder."""
|
|
return math.lcm(mamba_cache_chunk_size(), tree_page)
|
|
|
|
|
|
def mamba_track_grid(tree_page: int) -> int:
|
|
"""The same granularity for a decode-donated checkpoint, which additionally
|
|
has to land on the requested ``mamba_track_interval``."""
|
|
return math.lcm(
|
|
mamba_checkpoint_grid(tree_page), get_exec().mamba.mamba_track_interval
|
|
)
|
|
|
|
|
|
def max_speculative_num_draft_tokens() -> int | None:
|
|
"""The largest draft-token count speculative decoding may use.
|
|
|
|
Adaptive algorithms may switch to a longer state after the scheduler
|
|
reserves KV for the next decode batch, so include the capacity captured
|
|
when the resolved configuration was published.
|
|
"""
|
|
spec = get_spec()
|
|
return max(
|
|
(
|
|
bound
|
|
for bound in (
|
|
spec.speculative_num_draft_tokens,
|
|
spec.max_speculative_num_draft_tokens,
|
|
)
|
|
if bound is not None
|
|
),
|
|
default=None,
|
|
)
|
|
|
|
|
|
def uses_mla_backend() -> bool:
|
|
"""Whether this process's model runs the MLA attention path."""
|
|
from sglang.srt.arg_groups.overrides import use_mla_backend
|
|
|
|
return use_mla_backend(get_server_args())
|
|
|
|
|
|
def attention_backends() -> tuple:
|
|
"""The configured ``(prefill, decode)`` backend pair, split fields falling
|
|
back to ``attention_backend``.
|
|
|
|
All three inputs are ``exec.kernel`` leaves, so this derives from the bags
|
|
and follows a post-publish override; ``overrides.attention_backends_of``
|
|
is the pre-publish equivalent the resolution pipeline uses. A built runner
|
|
stamps its own resolved pair (``ModelRunner.prefill_attention_backend_str``);
|
|
read that when there is a runner in hand.
|
|
"""
|
|
from sglang.srt.arg_groups.overrides import attention_backends_of
|
|
|
|
return attention_backends_of(get_exec().kernel)
|
|
|
|
|
|
def process_model_config():
|
|
"""The process's ``ModelConfig`` (built once from the published config)."""
|
|
from sglang.srt.arg_groups.overrides import model_config_of
|
|
|
|
return model_config_of(get_server_args())
|
|
|
|
|
|
def reports_expert_balancedness() -> bool:
|
|
"""Whether the expert-balancedness report is on at all.
|
|
|
|
`overrides.should_report_expert_balancedness` is the pre-publish equivalent.
|
|
"""
|
|
return get_exec().moe.expert_balancedness_report_mode != "off"
|
|
|
|
|
|
def logs_expert_balancedness_to_server_log() -> bool:
|
|
"""Whether the balancedness report goes to the server log."""
|
|
return get_exec().moe.expert_balancedness_report_mode in ("server_log", "both")
|
|
|
|
|
|
def exports_expert_balancedness_to_prometheus() -> bool:
|
|
"""Whether the balancedness report goes to Prometheus."""
|
|
return get_exec().moe.expert_balancedness_report_mode in ("prometheus", "both")
|
|
|
|
|
|
def cutedsl_moe_max_num_tokens() -> int:
|
|
"""Largest token count one forward routes through a CuteDSL MoE layer on one
|
|
DP rank; sizes the standard-allgather wrapper, MegaMoE, and AR fusion buffers.
|
|
|
|
Every input is a published leaf (``spec``, ``schedule``, ``exec.graph``), so
|
|
this follows a post-publish override. Max over the prefill bound, the
|
|
piecewise-prefill capture, and the decode/verify bound.
|
|
"""
|
|
from sglang.srt.model_executor.cuda_graph_config import Backend
|
|
|
|
spec = get_spec()
|
|
num_tokens_per_req = (
|
|
(spec.speculative_num_draft_tokens or 1) if spec.speculative_algorithm else 1
|
|
)
|
|
prefill_tokens = get_schedule().max_prefill_tokens
|
|
cg_config = get_exec().graph.cuda_graph_config
|
|
if cg_config is not None and cg_config.prefill.backend == Backend.TC_PIECEWISE:
|
|
prefill_tokens = max(prefill_tokens, cg_config.prefill.max_bs or 0)
|
|
decode_max_bs = (cg_config.decode.max_bs if cg_config is not None else 0) or 0
|
|
return max(prefill_tokens, decode_max_bs * num_tokens_per_req)
|
|
|
|
|
|
def describe_kv_events_publisher(server_args: Any) -> Optional[dict]:
|
|
"""Return a structured description of this server's KV-event
|
|
publisher, or `None` if publishing is disabled / misconfigured.
|
|
|
|
This is the wire contract surfaced under the `kv_events` key on
|
|
`/server_info` so KV-aware routers (e.g. the SGLang model
|
|
gateway) can subscribe per-worker without operator-supplied port
|
|
coordination. The router constructs the per-DP-rank SUB endpoint
|
|
as tcp://<worker_host>:<endpoint_port_base + dp_rank> for
|
|
every rank reported in dp_size.
|
|
|
|
Returned descriptor shape:
|
|
|
|
{
|
|
"publisher": "zmq",
|
|
"endpoint_host": "*", # may be a ZMQ wildcard
|
|
# ("*", "0.0.0.0", "::");
|
|
# subscribers MUST substitute
|
|
# the worker URL's host when
|
|
# dialing
|
|
"endpoint_port_base": 5557, # base TCP port; per-rank
|
|
# port = base + dp_rank
|
|
"topic": "", # ZMQ topic prefix on the
|
|
# SUB filter (empty =
|
|
# subscribe-all)
|
|
"block_size": <kv_event_block_size>, # subscribers MUST
|
|
# hash prompts at this size
|
|
"dp_size": <dp_size>, # number of SUB sockets to
|
|
# open; not DCP-scaled, as
|
|
# DCP shards within a rank
|
|
# rather than adding
|
|
# publishers
|
|
"load_endpoint_port_base": <resolved>,
|
|
# base TCP port of the load
|
|
# range (load rank r = base
|
|
# + r). Consumers MUST read
|
|
# this key, not re-derive
|
|
# it; present only when
|
|
# --load-publish-endpoint
|
|
# opted in and a range
|
|
# resolved
|
|
"load_topic": "load", # SUB filter for the load
|
|
# socket; present iff
|
|
# load_endpoint_port_base
|
|
# is present
|
|
"replay_endpoint_port_base": 5558,
|
|
# ROUTER replay port; rank r
|
|
# = base + r, same host rule
|
|
# as the SUB endpoints;
|
|
# present only when
|
|
# replay_endpoint is tcp
|
|
}
|
|
|
|
Returns None (i.e. "no publisher to describe") when any of:
|
|
|
|
* --kv-events-config is unset / empty / malformed JSON,
|
|
* the configured publisher is "null",
|
|
* page_size is missing or non-positive (a placeholder
|
|
block_size would cause silent KV-cache misses by hashing
|
|
prompts at the wrong granularity on the router side),
|
|
* the endpoint is not a routable TCP address (inproc:// /
|
|
ipc://, missing port, non-integer port, port outside
|
|
1..65535, or a bare unbracketed IPv6 host, which is
|
|
ambiguous).
|
|
|
|
NOTE for load-socket consumers: pair the load port with the worker's
|
|
own URL host, as with the KV SUB endpoints — endpoint_host is a
|
|
wildcard ("*", "0.0.0.0", "::") whenever the default packing applies,
|
|
so splicing it yields tcp://*:PORT and connects to nothing.
|
|
|
|
Reuses parse_advertisable_tcp and resolve_load_pub_range — the same
|
|
helpers the scheduler binds through — so the advertisement cannot
|
|
drift from the sockets.
|
|
"""
|
|
from sglang.srt.arg_groups.overrides import kv_event_block_size_of, resolving_view
|
|
|
|
# Lazy import so loading server_args doesn't pull in
|
|
# disaggregation / msgspec / zmq at module top level.
|
|
from sglang.srt.disaggregation.kv_events import (
|
|
LOAD_TOPIC,
|
|
KVEventsConfig,
|
|
parse_advertisable_tcp,
|
|
resolve_load_pub_range,
|
|
)
|
|
|
|
resolved = resolving_view(server_args)
|
|
raw = resolved.kv_events_config
|
|
page_size = resolved.page_size
|
|
if not raw or page_size is None or page_size <= 0:
|
|
return None
|
|
try:
|
|
cfg = KVEventsConfig.from_cli(raw)
|
|
except Exception:
|
|
# Malformed JSON / schema mismatch. The publisher would
|
|
# have failed at server startup; /server_info must
|
|
# keep working, so just report "no publisher" to consumers.
|
|
return None
|
|
if cfg.publisher == "null" or not cfg.endpoint:
|
|
return None
|
|
resolved_kv = parse_advertisable_tcp(cfg.endpoint)
|
|
if resolved_kv is None:
|
|
return None
|
|
host, port = resolved_kv
|
|
|
|
descriptor = {
|
|
"publisher": cfg.publisher,
|
|
"endpoint_host": host,
|
|
"endpoint_port_base": port,
|
|
"topic": cfg.topic,
|
|
"block_size": kv_event_block_size_of(resolved),
|
|
# The wire key predates attention DP; it counts DP ranks.
|
|
"dp_size": num_dp_ranks_of(resolved),
|
|
}
|
|
# Load range, from the same resolver SchedulerLoadPublisher binds
|
|
# with (so the two can't drift). The decline reason is logged once at
|
|
# startup, not here — this runs per /server_info request.
|
|
resolved_range, _reason = resolve_load_pub_range(
|
|
kv_endpoint=cfg.endpoint,
|
|
replay_endpoint=cfg.replay_endpoint,
|
|
dp_size=num_dp_ranks_of(resolved),
|
|
load_publish_endpoint=resolved.load_publish_endpoint,
|
|
)
|
|
if resolved_range is not None:
|
|
descriptor["load_endpoint_port_base"] = resolved_range[1]
|
|
descriptor["load_topic"] = LOAD_TOPIC
|
|
resolved_replay = parse_advertisable_tcp(cfg.replay_endpoint)
|
|
if resolved_replay is not None:
|
|
descriptor["replay_endpoint_port_base"] = resolved_replay[1]
|
|
return descriptor
|