Take the model config out of the parallel group build, and finish retiring the parallel getters (#40707)

This commit is contained in:
Cheng Wan
2026-09-22 10:55:21 -07:00
committed by GitHub
parent 66454d704b
commit 91c329cc86
17 changed files with 278 additions and 128 deletions
+1 -1
View File
@@ -329,10 +329,10 @@ def load_model(server_args, port_args, gpu_id, tp_rank):
bootstrap.init_parallel_runtime(
server_args=server_args,
model_config=model_config,
device=get_device().device,
dist_port=port_args.nccl_port,
)
bootstrap.init_layer_runtime(model_config=model_config)
_use_mlx = use_mlx()
if _use_mlx:
@@ -39,7 +39,10 @@ from sglang.srt.distributed.parallel_state import (
initialize_model_parallel,
)
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import initialize_dp_attention
from sglang.srt.layers.dp_attention import (
init_dp_gathered_buffer,
initialize_dp_attention_flags,
)
from sglang.srt.managers.io_struct import (
ProfileReq,
ProfileReqType,
@@ -610,7 +613,8 @@ class MMEncoder:
moe_tp_size=parallel.tp_size,
)
initialize_model_parallel()
initialize_dp_attention(server_args, self.model_config)
initialize_dp_attention_flags(server_args)
init_dp_gathered_buffer(self.model_config)
self.model = load_model(
model_config=self.model_config,
+11 -9
View File
@@ -24,7 +24,10 @@ from sglang.srt.distributed.parallel_state import (
set_torch_symm_mem_all_reduce,
)
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import initialize_dp_attention
from sglang.srt.layers.dp_attention import (
init_dp_gathered_buffer,
initialize_dp_attention,
)
from sglang.srt.layers.layernorm_sp import initialize_layernorm_sp
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import (
@@ -90,7 +93,6 @@ def _bind_threads_if_cpu(*, device: str) -> "Optional[List[int]]":
def init_parallel_runtime(
*,
server_args: ServerArgs,
model_config: ModelConfig,
device: str,
dist_port: int,
) -> None:
@@ -150,7 +152,6 @@ def init_parallel_runtime(
backend=backend,
dist_init_method=dist_init_method,
server_args=server_args,
model_config=model_config,
gpu_id=get_device().gpu_id,
)
@@ -292,7 +293,6 @@ def _init_parallel_groups(
backend: str,
dist_init_method: str,
server_args: ServerArgs,
model_config: ModelConfig,
gpu_id: int,
) -> None:
parallel = get_parallel()
@@ -327,15 +327,17 @@ def _init_parallel_groups(
max_world_size=None if is_scale_joiner else get_parallel().max_ep_size,
)
_tag_groups_for_flashinfer_allreduce_only()
initialize_dp_attention(
server_args=server_args,
model_config=model_config,
)
initialize_layernorm_sp(model_config=model_config)
initialize_dp_attention(server_args=server_args)
if is_npu():
register_sgl_tp_rank(gpu_id)
def init_layer_runtime(*, model_config: ModelConfig) -> None:
"""Size the DP gathered buffer and resolve layernorm SP from the model."""
init_dp_gathered_buffer(model_config)
initialize_layernorm_sp(model_config=model_config)
def _prewarm_nccl(*, tp_size: int, pp_size: int, moe_ep_size: int) -> None:
warmup_start = time.perf_counter()
tp_group_handle = get_tp_group().device_group
@@ -3412,7 +3412,9 @@ _EXEMPT_CALLERS = ("sglang.srt.distributed.",)
_CONTEXT_NAME_OF = {
"get_world_group": "world_group",
"get_tp_group": "tp_group",
"get_tensor_model_parallel_group": "tp_group",
"get_pp_group": "pp_group",
"get_pipeline_model_parallel_group": "pp_group",
"get_moe_ep_group": "moe_ep_group",
"get_moe_dp_group": "moe_dp_group",
"get_moe_tp_group": "moe_tp_group",
+22 -27
View File
@@ -394,40 +394,35 @@ def compute_dp_attention_world_info(
return attn_tp_rank, attn_tp_size, attn_dp_rank, attn_dp_size
def initialize_dp_attention(
server_args: ServerArgs,
model_config: ModelConfig,
):
def initialize_dp_attention_flags(server_args: ServerArgs):
"""Initialize DP runtime flags without changing the worker's placement."""
dp = get_flags().dp
dp.max_len_with_idle = (
getattr(model_config.hf_config, "hybrid_override_pattern", None) is not None
)
enable_dp_attention = get_parallel().enable_dp_attention
dp_size = get_parallel().dp_size
attn_cp_size = get_parallel().attn_cp_size
dp.enabled = enable_dp_attention
tp_rank = get_parallel().tp_rank
tp_size = get_parallel().tp_size
_, _, attn_dp_rank, attn_dp_size = compute_dp_attention_world_info(
enable_dp_attention, tp_rank, tp_size, dp_size, attn_cp_size
)
dp.enabled = get_parallel().enable_dp_attention
if get_exec().moe.elastic_ep_backend is not None and get_parallel().max_ep_size:
# Reads the resolution, not a bag: this runs under
# `initialize_dp_attention`, which the weight-cache daemon calls from
# `_init_distributed` -- and other callers reach it from processes
# whose publish is not guaranteed to have happened yet. (The daemon
# itself publishes first, at `daemon.py:284`, before `:320`.)
if ep_scale_joiner_of(resolving_view(server_args)):
dp.joiner_skip_all_gather = True
get_parallel().override_permanently(
attn_dp_size=attn_dp_size, attn_dp_rank=attn_dp_rank
)
def initialize_dp_attention(server_args: ServerArgs):
"""Initialize DP flags and state placement from the published topology."""
initialize_dp_attention_flags(server_args)
parallel = get_parallel()
_, _, attn_dp_rank, attn_dp_size = compute_dp_attention_world_info(
parallel.enable_dp_attention,
parallel.tp_rank,
parallel.tp_size,
parallel.dp_size,
parallel.attn_cp_size,
)
parallel.override_permanently(attn_dp_size=attn_dp_size, attn_dp_rank=attn_dp_rank)
def init_dp_gathered_buffer(model_config: ModelConfig):
"""Size the gathered buffer from the model this worker is about to run."""
get_flags().dp.max_len_with_idle = (
getattr(model_config.hf_config, "hybrid_override_pattern", None) is not None
)
_DpGatheredBufferWrapper.set_metadata(
hidden_size=model_config.hidden_size,
dtype=model_config.dtype,
+1 -1
View File
@@ -519,10 +519,10 @@ class Scheduler(
bootstrap.init_parallel_runtime(
server_args=server_args,
model_config=self.model_config,
device=get_device().device,
dist_port=self.nccl_port,
)
bootstrap.init_layer_runtime(model_config=self.model_config)
# Init metrics stats
self.init_metrics_collector(tp_rank, pp_rank, dp_rank)
+13 -16
View File
@@ -14,9 +14,6 @@ from sglang.srt.configs.inkling import (
InklingModelConfig,
InklingVisionConfig,
)
from sglang.srt.distributed import (
get_tensor_model_parallel_group,
)
from sglang.srt.environ import envs
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
@@ -308,7 +305,7 @@ class InklingDecoderLayer(nn.Module):
hs,
prev_mlp_sconv,
forward_batch,
get_tensor_model_parallel_group(),
get_parallel().tp_group,
norm=self.attn_norm,
norm_residual=res,
)
@@ -317,7 +314,7 @@ class InklingDecoderLayer(nn.Module):
# MoE's unreduced partials; the kernel returns the gathered
# post-conv [T, H], and the norm runs unfused below.
hs = ar_scattered_sconv_fused(
hs, prev_mlp_sconv, forward_batch, get_tensor_model_parallel_group()
hs, prev_mlp_sconv, forward_batch, get_parallel().tp_group
)
hs, res = self.attn_norm(hs, res)
elif prev_mlp_partial:
@@ -331,13 +328,13 @@ class InklingDecoderLayer(nn.Module):
prev_mlp_sconv,
self.attn_norm,
forward_batch,
get_tensor_model_parallel_group(),
get_parallel().tp_group,
)
else:
# Fused extend {AR + full-width sconv + cache update}
# (non-scattered); norm runs unfused on the gathered [T, H].
hs = ar_fullwidth_sconv_fused(
hs, prev_mlp_sconv, forward_batch, get_tensor_model_parallel_group()
hs, prev_mlp_sconv, forward_batch, get_parallel().tp_group
)
hs, res = self.attn_norm(hs, res)
else:
@@ -618,7 +615,7 @@ class InklingCausalLLM(nn.Module):
# the prefill graph disabled and --skip-server-warmup there is no eager
# forward to build them lazily; decode capture would bake the fallback).
if envs.SGLANG_OPT_USE_INKLING_CUSTOM_AR.get():
ensure_inkling_ar_resources(get_tensor_model_parallel_group())
ensure_inkling_ar_resources(get_parallel().tp_group)
ensure_inkling_ar_resources(get_parallel().attn_tp_group)
# Warm the fused decode {AR -> mlp_sconv -> norm} JIT module (both
@@ -763,7 +760,7 @@ class InklingCausalLLM(nn.Module):
fuse_ar_sconv = (
not forward_batch.forward_mode.is_idle()
and ar_sconv_norm_fusable(
get_tensor_model_parallel_group(),
get_parallel().tp_group,
forward_batch,
hidden_states.shape[0],
hidden_states.shape[-1],
@@ -788,7 +785,7 @@ class InklingCausalLLM(nn.Module):
# kernel. Mutually exclusive with ar_sconv_norm_fusable by mode
# (extend vs decode/verify) and by the scattered gate inside it.
if not forward_batch.forward_mode.is_idle() and scattered_ar_sconv_fusable(
get_tensor_model_parallel_group(),
get_parallel().tp_group,
forward_batch,
hidden_states.shape[0],
hidden_states.shape[-1],
@@ -803,7 +800,7 @@ class InklingCausalLLM(nn.Module):
# (mode for ar_sconv_norm_fusable, the scattered flag for
# scattered_ar_sconv_fusable).
if not forward_batch.forward_mode.is_idle() and fullwidth_ar_sconv_fusable(
get_tensor_model_parallel_group(),
get_parallel().tp_group,
forward_batch,
hidden_states.shape[0],
hidden_states.shape[-1],
@@ -856,7 +853,7 @@ class InklingCausalLLM(nn.Module):
hidden_states,
prev_mlp_sconv,
forward_batch,
get_tensor_model_parallel_group(),
get_parallel().tp_group,
norm=self.norm,
norm_residual=residual,
)
@@ -871,7 +868,7 @@ class InklingCausalLLM(nn.Module):
hidden_states,
prev_mlp_sconv,
forward_batch,
get_tensor_model_parallel_group(),
get_parallel().tp_group,
)
hidden_states, _ = self.norm(hidden_states, residual)
return (
@@ -890,7 +887,7 @@ class InklingCausalLLM(nn.Module):
prev_mlp_sconv,
self.norm,
forward_batch,
get_tensor_model_parallel_group(),
get_parallel().tp_group,
)
return (
(hidden_states, aux_hidden_states)
@@ -903,7 +900,7 @@ class InklingCausalLLM(nn.Module):
hidden_states,
prev_mlp_sconv,
forward_batch,
get_tensor_model_parallel_group(),
get_parallel().tp_group,
)
hidden_states, _ = self.norm(hidden_states, residual)
return (
@@ -1787,7 +1784,7 @@ class InklingMTPLayer(nn.Module):
h_inproj.dtype,
)
if not fm_idle and scattered_ar_sconv_fusable(
get_tensor_model_parallel_group(),
get_parallel().tp_group,
forward_batch,
h_inproj.shape[0],
h_inproj.shape[-1],
@@ -31,9 +31,6 @@ from sglang.kernels.ops.moe.sigmoid_gate_topk_renorm import (
sigmoid_gate_topk_renorm,
)
from sglang.srt.configs.inkling import InklingModelConfig
from sglang.srt.distributed import (
get_tensor_model_parallel_group,
)
from sglang.srt.environ import GateGemvMode, envs
from sglang.srt.layers.moe import get_moe_runner_backend
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -753,7 +750,7 @@ def _build_inkling_shared_experts(
inference_moe_w13_interleaved=inference_moe_w13_interleaved,
tp_rank=moe_tp_rank,
tp_size=moe_tp_size,
tp_group=get_tensor_model_parallel_group(),
tp_group=get_parallel().tp_group,
)
return InklingBatchDenseMLP(
**dense_kwargs,
@@ -772,7 +769,7 @@ def _build_inkling_shared_experts(
quant_config=quant_config,
tp_rank=moe_tp_rank,
tp_size=moe_tp_size,
tp_group=get_tensor_model_parallel_group(),
tp_group=get_parallel().tp_group,
)
@@ -1071,7 +1068,7 @@ class InklingMoE(nn.Module):
# {routed + shared} add on the fold paths.
stash_ar_shared(shared_out)
return out
tp = get_tensor_model_parallel_group()
tp = get_parallel().tp_group
buf = get_ar_buffer(tp, out.shape[0], out.shape[1], out.dtype)
if buf is not None:
torch.add(out, shared_out, out=buf)
@@ -1079,7 +1076,7 @@ class InklingMoE(nn.Module):
return out + shared_out
return out
tp = get_tensor_model_parallel_group()
tp = get_parallel().tp_group
if shared_out is not None:
if self._fused_ar_shared and not self.scattered_sconv:
# The AR dispatch folds in-kernel on the fold paths and pre-adds
-24
View File
@@ -63,27 +63,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
# Imported lazily so this module has no import-time dependencies: any module can
# import get_parallel at module level without risking an import cycle.
_PARALLEL_STATE = None
def _ps():
"""Lazily import and cache the parallel-state module."""
global _PARALLEL_STATE
if _PARALLEL_STATE is None:
from sglang.srt.distributed import parallel_state
_PARALLEL_STATE = parallel_state
return _PARALLEL_STATE
def _dp():
from sglang.srt.layers import dp_attention
return dp_attention
@functools.lru_cache(maxsize=1)
def _parallel_config_leaves() -> frozenset:
"""Return configured parallel field names, including before publication."""
@@ -97,9 +76,6 @@ def _parallel_config_leaves() -> frozenset:
)
_MISSING_READ = object()
@functools.lru_cache(maxsize=1)
def _parallel_fields() -> frozenset:
"""Return configured and derived parallel names accepted by overrides."""
+6 -2
View File
@@ -235,9 +235,13 @@ class WeightCacheDaemon:
initialize_model_parallel()
# Initialize DP attention state (required by some models like Qwen3 MoE)
from sglang.srt.layers.dp_attention import initialize_dp_attention
from sglang.srt.layers.dp_attention import (
init_dp_gathered_buffer,
initialize_dp_attention,
)
initialize_dp_attention(server_args, model_config)
initialize_dp_attention(server_args)
init_dp_gathered_buffer(model_config)
logger.info(
f"[WeightCacheDaemon gpu={self.gpu_id} tp_rank={self.tp_rank}] "