[Perf] Lazy-load built-in model definitions and nixl_ep at startup (#41061)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: Yongji Wu <30348494+libertyeagle@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-09-24 20:29:33 -07:00
committed by GitHub
co-authored by metamergebot Yongji Wu
parent 7325b38412
commit e0b4d66733
4 changed files with 145 additions and 25 deletions
@@ -2,6 +2,7 @@ from __future__ import annotations
import logging
from enum import Enum, auto
from functools import cache
import torch
import torch.distributed as dist
@@ -28,26 +29,31 @@ from sglang.srt.runtime_context import (
get_resources,
)
try:
from nixl_ep import Buffer
use_nixl = True
except ImportError:
use_nixl = False
try:
from nixl_ep import topk_idx_t as NIXL_EP_TOPK_INDICES_DTYPE
except ImportError:
NIXL_EP_TOPK_INDICES_DTYPE = torch.int64
assert isinstance(NIXL_EP_TOPK_INDICES_DTYPE, torch.dtype)
logger = logging.getLogger(__name__)
NixlEPDispatchOutput = DeepEPLLDispatchOutput
NixlEPCombineInput = DeepEPLLCombineInput
@cache
def _load_nixl_ep() -> tuple[type, torch.dtype]:
try:
from nixl_ep import Buffer
except ImportError as exc:
raise ImportError(
"NixlEP is not installed. Please install NixlEP package from "
"https://github.com/ai-dynamo/nixl."
) from exc
try:
from nixl_ep import topk_idx_t
except ImportError:
topk_idx_t = torch.int64
assert isinstance(topk_idx_t, torch.dtype)
return Buffer, topk_idx_t
class NixlEPBuffer:
"""Managing facade for the process-wide NIXL EP buffer; the state itself
lives on ``ctx.resources``."""
@@ -125,6 +131,7 @@ class NixlEPBuffer:
cls._update_connections(state, state.scale_to)
return state.buffer
Buffer, _ = _load_nixl_ep()
state.hidden_size = hidden_size
state.num_max_dispatch_tokens_per_rank = num_max_dispatch_tokens_per_rank
state.num_experts = num_experts
@@ -207,11 +214,7 @@ class _NixlEPDispatcherImplBase:
params_dtype: torch.dtype,
deepep_mode: DeepEPMode,
):
if not use_nixl:
raise ImportError(
"NixlEP is not installed. Please install NixlEP package from "
"https://github.com/ai-dynamo/nixl."
)
_, self.topk_indices_dtype = _load_nixl_ep()
self.group = group
self.router_topk = router_topk
@@ -296,7 +299,7 @@ class _NixlEPDispatcherImpl(_NixlEPDispatcherImplBase):
):
buffer = self._get_buffer()
topk_weights, topk_ids = topk_output.topk_weights, topk_output.topk_ids
topk_ids = topk_ids.to(NIXL_EP_TOPK_INDICES_DTYPE)
topk_ids = topk_ids.to(self.topk_indices_dtype)
state = NixlEPBuffer._state()
dispatch_ep_size = state.dispatch_ep_size
num_local_experts = state.num_local_experts
+3 -2
View File
@@ -221,8 +221,9 @@ def get_model_architecture(model_config: ModelConfig) -> Tuple[Type[nn.Module],
):
architectures = ["QuantMixtralForCausalLM"]
supported_archs = ModelRegistry.get_supported_archs()
is_native_supported = any(arch in supported_archs for arch in architectures)
is_native_supported = any(
ModelRegistry.is_arch_supported(arch) for arch in architectures
)
if model_config.model_impl == ModelImpl.MINDSPORE:
architectures = ["MindSporeForCausalLM"]
+28 -3
View File
@@ -20,10 +20,19 @@ logger = logging.getLogger(__name__)
class _ModelRegistry:
# Keyed by model_arch
models: Dict[str, Union[Type[nn.Module], str]] = field(default_factory=dict)
_default_models_loaded: bool = True
def _load_default_models(self):
if self._default_models_loaded:
return
self.models = import_model_classes("sglang.srt.models") | self.models
self._default_models_loaded = True
def register(
self, package_name: str, overwrite: bool = False, strict: bool = False
):
self._load_default_models()
new_models = import_model_classes(package_name, strict=strict)
if overwrite:
self.models.update(new_models)
@@ -36,8 +45,12 @@ class _ModelRegistry:
self.models[arch] = cls
def get_supported_archs(self) -> AbstractSet[str]:
self._load_default_models()
return self.models.keys()
def is_arch_supported(self, architecture: str) -> bool:
return architecture in self.models or architecture in self.get_supported_archs()
def _raise_for_unsupported(self, architectures: List[str]):
all_supported_archs = self.get_supported_archs()
@@ -81,6 +94,16 @@ class _ModelRegistry:
self,
architectures: Union[str, List[str]],
) -> Tuple[Type[nn.Module], str]:
# Fast startup path: resolve registered models without importing defaults.
if architectures:
first_arch = (
architectures if isinstance(architectures, str) else architectures[0]
)
model_cls = self.models.get(first_arch)
if model_cls is not None:
return model_cls, first_arch
self._load_default_models()
architectures = self._normalize_archs(architectures)
for arch in architectures:
@@ -128,7 +151,9 @@ def import_model_classes(package_name: str, strict: bool = False):
ModelRegistry = _ModelRegistry()
ModelRegistry.register("sglang.srt.models")
if external_pkg := envs.SGLANG_EXTERNAL_MODEL_PACKAGE.get():
ModelRegistry.register(external_pkg, overwrite=True)
# Defer default model imports to speed up startup when using external models.
ModelRegistry._default_models_loaded = False
ModelRegistry.models.update(import_model_classes(external_pkg))
else:
ModelRegistry.register("sglang.srt.models")