Take the model config out of the parallel group build, and finish retiring the parallel getters (#40707)
This commit is contained in:
@@ -329,10 +329,10 @@ def load_model(server_args, port_args, gpu_id, tp_rank):
|
||||
|
||||
bootstrap.init_parallel_runtime(
|
||||
server_args=server_args,
|
||||
model_config=model_config,
|
||||
device=get_device().device,
|
||||
dist_port=port_args.nccl_port,
|
||||
)
|
||||
bootstrap.init_layer_runtime(model_config=model_config)
|
||||
|
||||
_use_mlx = use_mlx()
|
||||
if _use_mlx:
|
||||
|
||||
@@ -39,7 +39,10 @@ from sglang.srt.distributed.parallel_state import (
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import initialize_dp_attention
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
init_dp_gathered_buffer,
|
||||
initialize_dp_attention_flags,
|
||||
)
|
||||
from sglang.srt.managers.io_struct import (
|
||||
ProfileReq,
|
||||
ProfileReqType,
|
||||
@@ -610,7 +613,8 @@ class MMEncoder:
|
||||
moe_tp_size=parallel.tp_size,
|
||||
)
|
||||
initialize_model_parallel()
|
||||
initialize_dp_attention(server_args, self.model_config)
|
||||
initialize_dp_attention_flags(server_args)
|
||||
init_dp_gathered_buffer(self.model_config)
|
||||
|
||||
self.model = load_model(
|
||||
model_config=self.model_config,
|
||||
|
||||
@@ -24,7 +24,10 @@ from sglang.srt.distributed.parallel_state import (
|
||||
set_torch_symm_mem_all_reduce,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import initialize_dp_attention
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
init_dp_gathered_buffer,
|
||||
initialize_dp_attention,
|
||||
)
|
||||
from sglang.srt.layers.layernorm_sp import initialize_layernorm_sp
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import (
|
||||
@@ -90,7 +93,6 @@ def _bind_threads_if_cpu(*, device: str) -> "Optional[List[int]]":
|
||||
def init_parallel_runtime(
|
||||
*,
|
||||
server_args: ServerArgs,
|
||||
model_config: ModelConfig,
|
||||
device: str,
|
||||
dist_port: int,
|
||||
) -> None:
|
||||
@@ -150,7 +152,6 @@ def init_parallel_runtime(
|
||||
backend=backend,
|
||||
dist_init_method=dist_init_method,
|
||||
server_args=server_args,
|
||||
model_config=model_config,
|
||||
gpu_id=get_device().gpu_id,
|
||||
)
|
||||
|
||||
@@ -292,7 +293,6 @@ def _init_parallel_groups(
|
||||
backend: str,
|
||||
dist_init_method: str,
|
||||
server_args: ServerArgs,
|
||||
model_config: ModelConfig,
|
||||
gpu_id: int,
|
||||
) -> None:
|
||||
parallel = get_parallel()
|
||||
@@ -327,15 +327,17 @@ def _init_parallel_groups(
|
||||
max_world_size=None if is_scale_joiner else get_parallel().max_ep_size,
|
||||
)
|
||||
_tag_groups_for_flashinfer_allreduce_only()
|
||||
initialize_dp_attention(
|
||||
server_args=server_args,
|
||||
model_config=model_config,
|
||||
)
|
||||
initialize_layernorm_sp(model_config=model_config)
|
||||
initialize_dp_attention(server_args=server_args)
|
||||
if is_npu():
|
||||
register_sgl_tp_rank(gpu_id)
|
||||
|
||||
|
||||
def init_layer_runtime(*, model_config: ModelConfig) -> None:
|
||||
"""Size the DP gathered buffer and resolve layernorm SP from the model."""
|
||||
init_dp_gathered_buffer(model_config)
|
||||
initialize_layernorm_sp(model_config=model_config)
|
||||
|
||||
|
||||
def _prewarm_nccl(*, tp_size: int, pp_size: int, moe_ep_size: int) -> None:
|
||||
warmup_start = time.perf_counter()
|
||||
tp_group_handle = get_tp_group().device_group
|
||||
|
||||
@@ -3412,7 +3412,9 @@ _EXEMPT_CALLERS = ("sglang.srt.distributed.",)
|
||||
_CONTEXT_NAME_OF = {
|
||||
"get_world_group": "world_group",
|
||||
"get_tp_group": "tp_group",
|
||||
"get_tensor_model_parallel_group": "tp_group",
|
||||
"get_pp_group": "pp_group",
|
||||
"get_pipeline_model_parallel_group": "pp_group",
|
||||
"get_moe_ep_group": "moe_ep_group",
|
||||
"get_moe_dp_group": "moe_dp_group",
|
||||
"get_moe_tp_group": "moe_tp_group",
|
||||
|
||||
@@ -394,40 +394,35 @@ def compute_dp_attention_world_info(
|
||||
return attn_tp_rank, attn_tp_size, attn_dp_rank, attn_dp_size
|
||||
|
||||
|
||||
def initialize_dp_attention(
|
||||
server_args: ServerArgs,
|
||||
model_config: ModelConfig,
|
||||
):
|
||||
def initialize_dp_attention_flags(server_args: ServerArgs):
|
||||
"""Initialize DP runtime flags without changing the worker's placement."""
|
||||
dp = get_flags().dp
|
||||
dp.max_len_with_idle = (
|
||||
getattr(model_config.hf_config, "hybrid_override_pattern", None) is not None
|
||||
)
|
||||
enable_dp_attention = get_parallel().enable_dp_attention
|
||||
dp_size = get_parallel().dp_size
|
||||
attn_cp_size = get_parallel().attn_cp_size
|
||||
|
||||
dp.enabled = enable_dp_attention
|
||||
|
||||
tp_rank = get_parallel().tp_rank
|
||||
tp_size = get_parallel().tp_size
|
||||
|
||||
_, _, attn_dp_rank, attn_dp_size = compute_dp_attention_world_info(
|
||||
enable_dp_attention, tp_rank, tp_size, dp_size, attn_cp_size
|
||||
)
|
||||
dp.enabled = get_parallel().enable_dp_attention
|
||||
|
||||
if get_exec().moe.elastic_ep_backend is not None and get_parallel().max_ep_size:
|
||||
# Reads the resolution, not a bag: this runs under
|
||||
# `initialize_dp_attention`, which the weight-cache daemon calls from
|
||||
# `_init_distributed` -- and other callers reach it from processes
|
||||
# whose publish is not guaranteed to have happened yet. (The daemon
|
||||
# itself publishes first, at `daemon.py:284`, before `:320`.)
|
||||
if ep_scale_joiner_of(resolving_view(server_args)):
|
||||
dp.joiner_skip_all_gather = True
|
||||
|
||||
get_parallel().override_permanently(
|
||||
attn_dp_size=attn_dp_size, attn_dp_rank=attn_dp_rank
|
||||
)
|
||||
|
||||
def initialize_dp_attention(server_args: ServerArgs):
|
||||
"""Initialize DP flags and state placement from the published topology."""
|
||||
initialize_dp_attention_flags(server_args)
|
||||
parallel = get_parallel()
|
||||
_, _, attn_dp_rank, attn_dp_size = compute_dp_attention_world_info(
|
||||
parallel.enable_dp_attention,
|
||||
parallel.tp_rank,
|
||||
parallel.tp_size,
|
||||
parallel.dp_size,
|
||||
parallel.attn_cp_size,
|
||||
)
|
||||
parallel.override_permanently(attn_dp_size=attn_dp_size, attn_dp_rank=attn_dp_rank)
|
||||
|
||||
|
||||
def init_dp_gathered_buffer(model_config: ModelConfig):
|
||||
"""Size the gathered buffer from the model this worker is about to run."""
|
||||
get_flags().dp.max_len_with_idle = (
|
||||
getattr(model_config.hf_config, "hybrid_override_pattern", None) is not None
|
||||
)
|
||||
_DpGatheredBufferWrapper.set_metadata(
|
||||
hidden_size=model_config.hidden_size,
|
||||
dtype=model_config.dtype,
|
||||
|
||||
@@ -519,10 +519,10 @@ class Scheduler(
|
||||
|
||||
bootstrap.init_parallel_runtime(
|
||||
server_args=server_args,
|
||||
model_config=self.model_config,
|
||||
device=get_device().device,
|
||||
dist_port=self.nccl_port,
|
||||
)
|
||||
bootstrap.init_layer_runtime(model_config=self.model_config)
|
||||
|
||||
# Init metrics stats
|
||||
self.init_metrics_collector(tp_rank, pp_rank, dp_rank)
|
||||
|
||||
@@ -14,9 +14,6 @@ from sglang.srt.configs.inkling import (
|
||||
InklingModelConfig,
|
||||
InklingVisionConfig,
|
||||
)
|
||||
from sglang.srt.distributed import (
|
||||
get_tensor_model_parallel_group,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -308,7 +305,7 @@ class InklingDecoderLayer(nn.Module):
|
||||
hs,
|
||||
prev_mlp_sconv,
|
||||
forward_batch,
|
||||
get_tensor_model_parallel_group(),
|
||||
get_parallel().tp_group,
|
||||
norm=self.attn_norm,
|
||||
norm_residual=res,
|
||||
)
|
||||
@@ -317,7 +314,7 @@ class InklingDecoderLayer(nn.Module):
|
||||
# MoE's unreduced partials; the kernel returns the gathered
|
||||
# post-conv [T, H], and the norm runs unfused below.
|
||||
hs = ar_scattered_sconv_fused(
|
||||
hs, prev_mlp_sconv, forward_batch, get_tensor_model_parallel_group()
|
||||
hs, prev_mlp_sconv, forward_batch, get_parallel().tp_group
|
||||
)
|
||||
hs, res = self.attn_norm(hs, res)
|
||||
elif prev_mlp_partial:
|
||||
@@ -331,13 +328,13 @@ class InklingDecoderLayer(nn.Module):
|
||||
prev_mlp_sconv,
|
||||
self.attn_norm,
|
||||
forward_batch,
|
||||
get_tensor_model_parallel_group(),
|
||||
get_parallel().tp_group,
|
||||
)
|
||||
else:
|
||||
# Fused extend {AR + full-width sconv + cache update}
|
||||
# (non-scattered); norm runs unfused on the gathered [T, H].
|
||||
hs = ar_fullwidth_sconv_fused(
|
||||
hs, prev_mlp_sconv, forward_batch, get_tensor_model_parallel_group()
|
||||
hs, prev_mlp_sconv, forward_batch, get_parallel().tp_group
|
||||
)
|
||||
hs, res = self.attn_norm(hs, res)
|
||||
else:
|
||||
@@ -618,7 +615,7 @@ class InklingCausalLLM(nn.Module):
|
||||
# the prefill graph disabled and --skip-server-warmup there is no eager
|
||||
# forward to build them lazily; decode capture would bake the fallback).
|
||||
if envs.SGLANG_OPT_USE_INKLING_CUSTOM_AR.get():
|
||||
ensure_inkling_ar_resources(get_tensor_model_parallel_group())
|
||||
ensure_inkling_ar_resources(get_parallel().tp_group)
|
||||
ensure_inkling_ar_resources(get_parallel().attn_tp_group)
|
||||
|
||||
# Warm the fused decode {AR -> mlp_sconv -> norm} JIT module (both
|
||||
@@ -763,7 +760,7 @@ class InklingCausalLLM(nn.Module):
|
||||
fuse_ar_sconv = (
|
||||
not forward_batch.forward_mode.is_idle()
|
||||
and ar_sconv_norm_fusable(
|
||||
get_tensor_model_parallel_group(),
|
||||
get_parallel().tp_group,
|
||||
forward_batch,
|
||||
hidden_states.shape[0],
|
||||
hidden_states.shape[-1],
|
||||
@@ -788,7 +785,7 @@ class InklingCausalLLM(nn.Module):
|
||||
# kernel. Mutually exclusive with ar_sconv_norm_fusable by mode
|
||||
# (extend vs decode/verify) and by the scattered gate inside it.
|
||||
if not forward_batch.forward_mode.is_idle() and scattered_ar_sconv_fusable(
|
||||
get_tensor_model_parallel_group(),
|
||||
get_parallel().tp_group,
|
||||
forward_batch,
|
||||
hidden_states.shape[0],
|
||||
hidden_states.shape[-1],
|
||||
@@ -803,7 +800,7 @@ class InklingCausalLLM(nn.Module):
|
||||
# (mode for ar_sconv_norm_fusable, the scattered flag for
|
||||
# scattered_ar_sconv_fusable).
|
||||
if not forward_batch.forward_mode.is_idle() and fullwidth_ar_sconv_fusable(
|
||||
get_tensor_model_parallel_group(),
|
||||
get_parallel().tp_group,
|
||||
forward_batch,
|
||||
hidden_states.shape[0],
|
||||
hidden_states.shape[-1],
|
||||
@@ -856,7 +853,7 @@ class InklingCausalLLM(nn.Module):
|
||||
hidden_states,
|
||||
prev_mlp_sconv,
|
||||
forward_batch,
|
||||
get_tensor_model_parallel_group(),
|
||||
get_parallel().tp_group,
|
||||
norm=self.norm,
|
||||
norm_residual=residual,
|
||||
)
|
||||
@@ -871,7 +868,7 @@ class InklingCausalLLM(nn.Module):
|
||||
hidden_states,
|
||||
prev_mlp_sconv,
|
||||
forward_batch,
|
||||
get_tensor_model_parallel_group(),
|
||||
get_parallel().tp_group,
|
||||
)
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return (
|
||||
@@ -890,7 +887,7 @@ class InklingCausalLLM(nn.Module):
|
||||
prev_mlp_sconv,
|
||||
self.norm,
|
||||
forward_batch,
|
||||
get_tensor_model_parallel_group(),
|
||||
get_parallel().tp_group,
|
||||
)
|
||||
return (
|
||||
(hidden_states, aux_hidden_states)
|
||||
@@ -903,7 +900,7 @@ class InklingCausalLLM(nn.Module):
|
||||
hidden_states,
|
||||
prev_mlp_sconv,
|
||||
forward_batch,
|
||||
get_tensor_model_parallel_group(),
|
||||
get_parallel().tp_group,
|
||||
)
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return (
|
||||
@@ -1787,7 +1784,7 @@ class InklingMTPLayer(nn.Module):
|
||||
h_inproj.dtype,
|
||||
)
|
||||
if not fm_idle and scattered_ar_sconv_fusable(
|
||||
get_tensor_model_parallel_group(),
|
||||
get_parallel().tp_group,
|
||||
forward_batch,
|
||||
h_inproj.shape[0],
|
||||
h_inproj.shape[-1],
|
||||
|
||||
@@ -31,9 +31,6 @@ from sglang.kernels.ops.moe.sigmoid_gate_topk_renorm import (
|
||||
sigmoid_gate_topk_renorm,
|
||||
)
|
||||
from sglang.srt.configs.inkling import InklingModelConfig
|
||||
from sglang.srt.distributed import (
|
||||
get_tensor_model_parallel_group,
|
||||
)
|
||||
from sglang.srt.environ import GateGemvMode, envs
|
||||
from sglang.srt.layers.moe import get_moe_runner_backend
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
@@ -753,7 +750,7 @@ def _build_inkling_shared_experts(
|
||||
inference_moe_w13_interleaved=inference_moe_w13_interleaved,
|
||||
tp_rank=moe_tp_rank,
|
||||
tp_size=moe_tp_size,
|
||||
tp_group=get_tensor_model_parallel_group(),
|
||||
tp_group=get_parallel().tp_group,
|
||||
)
|
||||
return InklingBatchDenseMLP(
|
||||
**dense_kwargs,
|
||||
@@ -772,7 +769,7 @@ def _build_inkling_shared_experts(
|
||||
quant_config=quant_config,
|
||||
tp_rank=moe_tp_rank,
|
||||
tp_size=moe_tp_size,
|
||||
tp_group=get_tensor_model_parallel_group(),
|
||||
tp_group=get_parallel().tp_group,
|
||||
)
|
||||
|
||||
|
||||
@@ -1071,7 +1068,7 @@ class InklingMoE(nn.Module):
|
||||
# {routed + shared} add on the fold paths.
|
||||
stash_ar_shared(shared_out)
|
||||
return out
|
||||
tp = get_tensor_model_parallel_group()
|
||||
tp = get_parallel().tp_group
|
||||
buf = get_ar_buffer(tp, out.shape[0], out.shape[1], out.dtype)
|
||||
if buf is not None:
|
||||
torch.add(out, shared_out, out=buf)
|
||||
@@ -1079,7 +1076,7 @@ class InklingMoE(nn.Module):
|
||||
return out + shared_out
|
||||
return out
|
||||
|
||||
tp = get_tensor_model_parallel_group()
|
||||
tp = get_parallel().tp_group
|
||||
if shared_out is not None:
|
||||
if self._fused_ar_shared and not self.scattered_sconv:
|
||||
# The AR dispatch folds in-kernel on the fold paths and pre-adds
|
||||
|
||||
@@ -63,27 +63,6 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Imported lazily so this module has no import-time dependencies: any module can
|
||||
# import get_parallel at module level without risking an import cycle.
|
||||
_PARALLEL_STATE = None
|
||||
|
||||
|
||||
def _ps():
|
||||
"""Lazily import and cache the parallel-state module."""
|
||||
global _PARALLEL_STATE
|
||||
if _PARALLEL_STATE is None:
|
||||
from sglang.srt.distributed import parallel_state
|
||||
|
||||
_PARALLEL_STATE = parallel_state
|
||||
return _PARALLEL_STATE
|
||||
|
||||
|
||||
def _dp():
|
||||
from sglang.srt.layers import dp_attention
|
||||
|
||||
return dp_attention
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _parallel_config_leaves() -> frozenset:
|
||||
"""Return configured parallel field names, including before publication."""
|
||||
@@ -97,9 +76,6 @@ def _parallel_config_leaves() -> frozenset:
|
||||
)
|
||||
|
||||
|
||||
_MISSING_READ = object()
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _parallel_fields() -> frozenset:
|
||||
"""Return configured and derived parallel names accepted by overrides."""
|
||||
|
||||
@@ -235,9 +235,13 @@ class WeightCacheDaemon:
|
||||
initialize_model_parallel()
|
||||
|
||||
# Initialize DP attention state (required by some models like Qwen3 MoE)
|
||||
from sglang.srt.layers.dp_attention import initialize_dp_attention
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
init_dp_gathered_buffer,
|
||||
initialize_dp_attention,
|
||||
)
|
||||
|
||||
initialize_dp_attention(server_args, model_config)
|
||||
initialize_dp_attention(server_args)
|
||||
init_dp_gathered_buffer(model_config)
|
||||
|
||||
logger.info(
|
||||
f"[WeightCacheDaemon gpu={self.gpu_id} tp_rank={self.tp_rank}] "
|
||||
|
||||
Reference in New Issue
Block a user