[Perf] Lazy-load built-in model definitions and nixl_ep at startup (#41061)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: Yongji Wu <30348494+libertyeagle@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
Yongji Wu
parent
7325b38412
commit
e0b4d66733
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from enum import Enum, auto
|
||||
from functools import cache
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
@@ -28,26 +29,31 @@ from sglang.srt.runtime_context import (
|
||||
get_resources,
|
||||
)
|
||||
|
||||
try:
|
||||
from nixl_ep import Buffer
|
||||
|
||||
use_nixl = True
|
||||
except ImportError:
|
||||
use_nixl = False
|
||||
|
||||
try:
|
||||
from nixl_ep import topk_idx_t as NIXL_EP_TOPK_INDICES_DTYPE
|
||||
except ImportError:
|
||||
NIXL_EP_TOPK_INDICES_DTYPE = torch.int64
|
||||
|
||||
assert isinstance(NIXL_EP_TOPK_INDICES_DTYPE, torch.dtype)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NixlEPDispatchOutput = DeepEPLLDispatchOutput
|
||||
NixlEPCombineInput = DeepEPLLCombineInput
|
||||
|
||||
|
||||
@cache
|
||||
def _load_nixl_ep() -> tuple[type, torch.dtype]:
|
||||
try:
|
||||
from nixl_ep import Buffer
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"NixlEP is not installed. Please install NixlEP package from "
|
||||
"https://github.com/ai-dynamo/nixl."
|
||||
) from exc
|
||||
|
||||
try:
|
||||
from nixl_ep import topk_idx_t
|
||||
except ImportError:
|
||||
topk_idx_t = torch.int64
|
||||
|
||||
assert isinstance(topk_idx_t, torch.dtype)
|
||||
return Buffer, topk_idx_t
|
||||
|
||||
|
||||
class NixlEPBuffer:
|
||||
"""Managing facade for the process-wide NIXL EP buffer; the state itself
|
||||
lives on ``ctx.resources``."""
|
||||
@@ -125,6 +131,7 @@ class NixlEPBuffer:
|
||||
cls._update_connections(state, state.scale_to)
|
||||
return state.buffer
|
||||
|
||||
Buffer, _ = _load_nixl_ep()
|
||||
state.hidden_size = hidden_size
|
||||
state.num_max_dispatch_tokens_per_rank = num_max_dispatch_tokens_per_rank
|
||||
state.num_experts = num_experts
|
||||
@@ -207,11 +214,7 @@ class _NixlEPDispatcherImplBase:
|
||||
params_dtype: torch.dtype,
|
||||
deepep_mode: DeepEPMode,
|
||||
):
|
||||
if not use_nixl:
|
||||
raise ImportError(
|
||||
"NixlEP is not installed. Please install NixlEP package from "
|
||||
"https://github.com/ai-dynamo/nixl."
|
||||
)
|
||||
_, self.topk_indices_dtype = _load_nixl_ep()
|
||||
|
||||
self.group = group
|
||||
self.router_topk = router_topk
|
||||
@@ -296,7 +299,7 @@ class _NixlEPDispatcherImpl(_NixlEPDispatcherImplBase):
|
||||
):
|
||||
buffer = self._get_buffer()
|
||||
topk_weights, topk_ids = topk_output.topk_weights, topk_output.topk_ids
|
||||
topk_ids = topk_ids.to(NIXL_EP_TOPK_INDICES_DTYPE)
|
||||
topk_ids = topk_ids.to(self.topk_indices_dtype)
|
||||
state = NixlEPBuffer._state()
|
||||
dispatch_ep_size = state.dispatch_ep_size
|
||||
num_local_experts = state.num_local_experts
|
||||
|
||||
@@ -221,8 +221,9 @@ def get_model_architecture(model_config: ModelConfig) -> Tuple[Type[nn.Module],
|
||||
):
|
||||
architectures = ["QuantMixtralForCausalLM"]
|
||||
|
||||
supported_archs = ModelRegistry.get_supported_archs()
|
||||
is_native_supported = any(arch in supported_archs for arch in architectures)
|
||||
is_native_supported = any(
|
||||
ModelRegistry.is_arch_supported(arch) for arch in architectures
|
||||
)
|
||||
|
||||
if model_config.model_impl == ModelImpl.MINDSPORE:
|
||||
architectures = ["MindSporeForCausalLM"]
|
||||
|
||||
@@ -20,10 +20,19 @@ logger = logging.getLogger(__name__)
|
||||
class _ModelRegistry:
|
||||
# Keyed by model_arch
|
||||
models: Dict[str, Union[Type[nn.Module], str]] = field(default_factory=dict)
|
||||
_default_models_loaded: bool = True
|
||||
|
||||
def _load_default_models(self):
|
||||
if self._default_models_loaded:
|
||||
return
|
||||
|
||||
self.models = import_model_classes("sglang.srt.models") | self.models
|
||||
self._default_models_loaded = True
|
||||
|
||||
def register(
|
||||
self, package_name: str, overwrite: bool = False, strict: bool = False
|
||||
):
|
||||
self._load_default_models()
|
||||
new_models = import_model_classes(package_name, strict=strict)
|
||||
if overwrite:
|
||||
self.models.update(new_models)
|
||||
@@ -36,8 +45,12 @@ class _ModelRegistry:
|
||||
self.models[arch] = cls
|
||||
|
||||
def get_supported_archs(self) -> AbstractSet[str]:
|
||||
self._load_default_models()
|
||||
return self.models.keys()
|
||||
|
||||
def is_arch_supported(self, architecture: str) -> bool:
|
||||
return architecture in self.models or architecture in self.get_supported_archs()
|
||||
|
||||
def _raise_for_unsupported(self, architectures: List[str]):
|
||||
all_supported_archs = self.get_supported_archs()
|
||||
|
||||
@@ -81,6 +94,16 @@ class _ModelRegistry:
|
||||
self,
|
||||
architectures: Union[str, List[str]],
|
||||
) -> Tuple[Type[nn.Module], str]:
|
||||
# Fast startup path: resolve registered models without importing defaults.
|
||||
if architectures:
|
||||
first_arch = (
|
||||
architectures if isinstance(architectures, str) else architectures[0]
|
||||
)
|
||||
model_cls = self.models.get(first_arch)
|
||||
if model_cls is not None:
|
||||
return model_cls, first_arch
|
||||
|
||||
self._load_default_models()
|
||||
architectures = self._normalize_archs(architectures)
|
||||
|
||||
for arch in architectures:
|
||||
@@ -128,7 +151,9 @@ def import_model_classes(package_name: str, strict: bool = False):
|
||||
|
||||
|
||||
ModelRegistry = _ModelRegistry()
|
||||
ModelRegistry.register("sglang.srt.models")
|
||||
|
||||
if external_pkg := envs.SGLANG_EXTERNAL_MODEL_PACKAGE.get():
|
||||
ModelRegistry.register(external_pkg, overwrite=True)
|
||||
# Defer default model imports to speed up startup when using external models.
|
||||
ModelRegistry._default_models_loaded = False
|
||||
ModelRegistry.models.update(import_model_classes(external_pkg))
|
||||
else:
|
||||
ModelRegistry.register("sglang.srt.models")
|
||||
|
||||
Reference in New Issue
Block a user