[Apple Silicon] Add an Apple Silicon (MPS) platform to the standard Torch model runner (#36780)
This commit is contained in:
@@ -162,7 +162,10 @@ class Stream:
|
||||
"""
|
||||
|
||||
def __init__(self, device: Any = None, priority: int = 0) -> None:
|
||||
pass
|
||||
import torch
|
||||
|
||||
self.device = torch.device(device or "mps")
|
||||
self.priority = priority
|
||||
|
||||
def synchronize(self) -> None:
|
||||
pass
|
||||
@@ -276,13 +279,24 @@ _cached_props: _MPSDeviceProperties | None = None
|
||||
|
||||
|
||||
def get_device_properties(device: Any = 0) -> _MPSDeviceProperties: # noqa: ARG001
|
||||
"""Return the properties of the MPS device. Results are cached after first call."""
|
||||
"""Implement ``torch.mps.get_device_properties``, not other device backends.
|
||||
|
||||
Metal's recommended working-set limit bounds usable GPU memory, even though
|
||||
Apple Silicon shares physical RAM with the CPU.
|
||||
"""
|
||||
global _cached_props
|
||||
if _cached_props is None:
|
||||
import psutil
|
||||
import torch
|
||||
|
||||
total_memory = int(torch.mps.recommended_max_memory())
|
||||
if total_memory <= 0:
|
||||
raise RuntimeError(
|
||||
"torch.mps.recommended_max_memory() returned a non-positive "
|
||||
"Metal working-set limit"
|
||||
)
|
||||
|
||||
_cached_props = _MPSDeviceProperties(
|
||||
total_memory=psutil.virtual_memory().total,
|
||||
total_memory=total_memory,
|
||||
)
|
||||
return _cached_props
|
||||
|
||||
|
||||
@@ -485,6 +485,85 @@ def validate_experimental_sgl_marlin(server_args: Any):
|
||||
validate_experimental_sgl_marlin_server_args(server_args, view)
|
||||
|
||||
|
||||
def validate_mps_model_config(
|
||||
model_config: Any,
|
||||
*,
|
||||
lora_enabled: bool = False,
|
||||
) -> None:
|
||||
"""Validate checkpoint-derived MPS constraints."""
|
||||
quantization = getattr(model_config, "quantization", None)
|
||||
if quantization not in (None, "unquant"):
|
||||
raise ValueError(
|
||||
"Torch MPS currently supports only unquantized model weights; "
|
||||
"the resolved model configuration detected "
|
||||
f"quantization={quantization!r}"
|
||||
)
|
||||
if bool(getattr(model_config, "is_multimodal", False)):
|
||||
raise ValueError(
|
||||
"Torch MPS multimodal serving does not yet have a model-specific "
|
||||
"end-to-end contract; use a text-only model until its encoder, "
|
||||
"processor, and decoder paths are validated on MPS"
|
||||
)
|
||||
if lora_enabled:
|
||||
for config in (
|
||||
getattr(model_config, "hf_text_config", None),
|
||||
getattr(model_config, "hf_config", None),
|
||||
):
|
||||
if config is None:
|
||||
continue
|
||||
for field_name in (
|
||||
"num_experts",
|
||||
"num_local_experts",
|
||||
"n_routed_experts",
|
||||
):
|
||||
value = getattr(config, field_name, None)
|
||||
if value is not None and int(value) > 0:
|
||||
raise ValueError(
|
||||
"Torch MPS LoRA currently supports dense models only; "
|
||||
f"the model config declares {field_name}={value!r}"
|
||||
)
|
||||
|
||||
|
||||
def validate_standard_mps_server_args(server_args: Any):
|
||||
"""Validate execution modes supported by the Torch MPS path."""
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
supported_attention_backends = {None, "torch_native"}
|
||||
for field in (
|
||||
"attention_backend",
|
||||
"prefill_attention_backend",
|
||||
"decode_attention_backend",
|
||||
):
|
||||
value = getattr(cfg, field, None)
|
||||
normalized = getattr(value, "value", value)
|
||||
normalized = None if normalized is None else str(normalized).lower()
|
||||
if normalized not in supported_attention_backends:
|
||||
raise ValueError(
|
||||
"The standard Torch MPS path currently supports only the "
|
||||
f"torch_native attention backend; got {field}={value!r}"
|
||||
)
|
||||
|
||||
sampling_backend = getattr(cfg, "sampling_backend", None)
|
||||
normalized_sampling = getattr(sampling_backend, "value", sampling_backend)
|
||||
normalized_sampling = (
|
||||
None if normalized_sampling is None else str(normalized_sampling).lower()
|
||||
)
|
||||
if normalized_sampling not in {None, "pytorch"}:
|
||||
raise ValueError(
|
||||
"The standard Torch MPS path currently supports only the "
|
||||
f"pytorch sampling backend; got sampling_backend={sampling_backend!r}"
|
||||
)
|
||||
|
||||
if (
|
||||
getattr(cfg, "tp_size", 1) != 1
|
||||
or getattr(cfg, "pp_size", 1) != 1
|
||||
or getattr(cfg, "dp_size", 1) != 1
|
||||
):
|
||||
raise ValueError(
|
||||
"The standard Torch MPS path requires tp_size=1, pp_size=1, and dp_size=1"
|
||||
)
|
||||
|
||||
|
||||
def validate_prefill_decode_interval(server_args: Any):
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.prefill_decode_interval is not None and cfg.prefill_decode_interval < 0:
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Apple MPS SRT Providers"""
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Runtime validation for SGLang's Apple MPS backend."""
|
||||
|
||||
import torch
|
||||
from packaging.version import Version
|
||||
|
||||
|
||||
def validate_mps_runtime() -> None:
|
||||
"""Validate the Torch runtime required by the standard MPS model path."""
|
||||
if Version(torch.__version__) < Version("2.13.0"):
|
||||
raise RuntimeError(
|
||||
"The standard SGLang MPS model path requires Torch >= 2.13.0; "
|
||||
f"found Torch {torch.__version__}; reinstall with the "
|
||||
"srt_mps extra"
|
||||
)
|
||||
|
||||
if not torch.backends.mps.is_available():
|
||||
raise RuntimeError(
|
||||
"The SGLang MPS backend requires an available PyTorch MPS device"
|
||||
)
|
||||
for memory_api in ("recommended_max_memory", "driver_allocated_memory"):
|
||||
if not callable(getattr(torch.mps, memory_api, None)):
|
||||
raise RuntimeError(
|
||||
f"The SGLang MPS backend requires torch.mps.{memory_api} from "
|
||||
"Torch >= 2.13.0"
|
||||
)
|
||||
@@ -411,10 +411,21 @@ class ModelRunner:
|
||||
if get_exec().features.enable_tf32_matmul:
|
||||
torch.set_float32_matmul_precision("high")
|
||||
|
||||
# Set device early so that TransferEngine init (e.g. Ascend NPU)
|
||||
# can access the device context.
|
||||
# Set the device before TransferEngine init. MPS has one implicit device.
|
||||
is_mps_device = str(self.device).split(":", 1)[0] == "mps"
|
||||
if is_mps_device:
|
||||
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
|
||||
|
||||
if not use_mlx():
|
||||
# Direct ModelRunner construction bypasses the ServerArgs gate.
|
||||
from sglang.srt.hardware_backend.mps.runtime import (
|
||||
validate_mps_runtime,
|
||||
)
|
||||
|
||||
validate_mps_runtime()
|
||||
try:
|
||||
torch.get_device_module(self.device).set_device(get_device().gpu_id)
|
||||
if not is_mps_device:
|
||||
torch.get_device_module(self.device).set_device(get_device().gpu_id)
|
||||
except Exception:
|
||||
import os
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.platforms.cpu import CpuSRTPlatform
|
||||
from sglang.srt.platforms.cuda import CudaSRTPlatform
|
||||
from sglang.srt.platforms.interface import SRTPlatform
|
||||
from sglang.srt.platforms.mps import MpsSRTPlatform
|
||||
from sglang.srt.platforms.npu import NPUSRTPlatform
|
||||
from sglang.srt.platforms.rocm import RocmSRTPlatform
|
||||
from sglang.srt.platforms.xpu import XpuSRTPlatform
|
||||
@@ -51,6 +52,10 @@ def _is_xpu_available() -> bool:
|
||||
return torch.xpu.is_available()
|
||||
|
||||
|
||||
def _is_mps_available() -> bool:
|
||||
return bool(getattr(torch, "mps", None) is not None and torch.mps.is_available())
|
||||
|
||||
|
||||
def _resolve_platform() -> SRTPlatform:
|
||||
"""
|
||||
Discover and instantiate the active platform.
|
||||
@@ -73,6 +78,7 @@ def _resolve_platform() -> SRTPlatform:
|
||||
- 0 activated + CUDA available → fallback CudaSRTPlatform
|
||||
- 0 activated + ROCm available → fallback RocmSRTPlatform
|
||||
- 0 activated + XPU available → fallback XpuSRTPlatform
|
||||
- 0 activated + MPS available → fallback MpsSRTPlatform
|
||||
- 0 activated + none of the above → fallback base SRTPlatform
|
||||
- 1 activated → use it
|
||||
- N activated → RuntimeError (must set SGLANG_PLATFORM)
|
||||
@@ -143,6 +149,9 @@ def _resolve_platform() -> SRTPlatform:
|
||||
if _is_xpu_available():
|
||||
logger.debug("No platform plugin detected. Using XPU SRTPlatform defaults.")
|
||||
return XpuSRTPlatform()
|
||||
if _is_mps_available():
|
||||
logger.debug("No platform plugin detected. Using MPS SRTPlatform defaults.")
|
||||
return MpsSRTPlatform()
|
||||
logger.debug("No platform detected. Using base SRTPlatform.")
|
||||
return SRTPlatform()
|
||||
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Apple MPS support for SRT's in-tree platform layer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.platforms.device_mixin import (
|
||||
DeviceMixin,
|
||||
PlatformEnum,
|
||||
)
|
||||
from sglang.srt.platforms.interface import SRTPlatform
|
||||
|
||||
|
||||
def _recommended_working_set_size() -> int:
|
||||
getter = getattr(torch.mps, "recommended_max_memory", None)
|
||||
if not callable(getter):
|
||||
raise RuntimeError(
|
||||
"The SGLang MPS backend requires "
|
||||
"torch.mps.recommended_max_memory() from Torch >= 2.13.0"
|
||||
)
|
||||
total = int(getter())
|
||||
if total <= 0:
|
||||
raise RuntimeError(
|
||||
"torch.mps.recommended_max_memory() returned a non-positive Metal "
|
||||
"working-set limit; refusing to size the model and KV pools from "
|
||||
"raw system RAM"
|
||||
)
|
||||
return total
|
||||
|
||||
|
||||
class MpsDeviceMixin(DeviceMixin):
|
||||
"""Device operations backed by ``torch.mps``."""
|
||||
|
||||
_enum: PlatformEnum = PlatformEnum.MPS
|
||||
device_name: str = "mps"
|
||||
device_type: str = "mps"
|
||||
|
||||
def get_device_total_memory(self, device_id: int = 0) -> int:
|
||||
# Metal's working-set limit, not total system RAM, bounds model memory.
|
||||
return _recommended_working_set_size()
|
||||
|
||||
def get_current_memory_usage(self, device: Optional[torch.device] = None) -> float:
|
||||
# Driver allocation includes resident cached MPS buffers.
|
||||
return float(torch.mps.driver_allocated_memory())
|
||||
|
||||
def get_device(self, local_rank: int = 0) -> torch.device:
|
||||
if local_rank not in (-1, 0):
|
||||
raise ValueError(f"MPS exposes one device; got local_rank={local_rank}")
|
||||
return torch.device("mps")
|
||||
|
||||
def set_device(self, device: torch.device) -> None:
|
||||
if str(device).split(":", 1)[0] != "mps":
|
||||
raise ValueError(f"MPS platform cannot select device {device}")
|
||||
# MPS has no device-selection API.
|
||||
|
||||
def get_device_name(self, device_id: int = 0) -> str:
|
||||
return str(torch.backends.mps.get_name())
|
||||
|
||||
def get_device_uuid(self, device_id: int = 0) -> str:
|
||||
# PyTorch exposes no stable MPS UUID.
|
||||
return "mps:0"
|
||||
|
||||
def get_device_capability(self, device_id: int = 0):
|
||||
return None
|
||||
|
||||
def empty_cache(self) -> None:
|
||||
empty_cache = getattr(torch.mps, "empty_cache", None)
|
||||
if callable(empty_cache):
|
||||
empty_cache()
|
||||
|
||||
def synchronize(self) -> None:
|
||||
synchronize = getattr(torch.mps, "synchronize", None)
|
||||
if callable(synchronize):
|
||||
synchronize()
|
||||
|
||||
|
||||
class MpsSRTPlatform(MpsDeviceMixin, SRTPlatform):
|
||||
"""Built-in SRT platform for Apple Silicon MPS."""
|
||||
|
||||
def get_default_attention_backend(self) -> str:
|
||||
return "torch_native"
|
||||
@@ -1772,6 +1772,7 @@ _PLATFORM_PROBES: Dict[str, str] = {
|
||||
"is_npu": "is_npu",
|
||||
"is_xpu": "is_xpu",
|
||||
"is_musa": "is_musa",
|
||||
"is_mps": "is_mps",
|
||||
"is_sm90": "is_sm90_supported",
|
||||
"is_sm100": "is_sm100_supported",
|
||||
"is_sm100_or_sm110": "is_sm100_or_sm110_supported",
|
||||
|
||||
@@ -38,11 +38,18 @@ from sglang.srt.arg_groups.argparse_actions import (
|
||||
)
|
||||
from sglang.srt.arg_groups.model_override_base import ep_joiner_of, ep_scale_joiner_of
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
model_config_of,
|
||||
remote_instance_transfer_engine_of,
|
||||
resolution_result,
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.arg_groups.validation_hook import (
|
||||
validate_mps_model_config,
|
||||
validate_standard_mps_server_args,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
|
||||
from sglang.srt.hardware_backend.mps.runtime import validate_mps_runtime
|
||||
from sglang.srt.runtime_context import (
|
||||
attn_dp_enabled_of,
|
||||
get_platform,
|
||||
@@ -251,8 +258,38 @@ class ServerArgs:
|
||||
from sglang.srt.arg_groups.pipeline import run_resolution_pipeline
|
||||
|
||||
self._input_frozen = True
|
||||
|
||||
try:
|
||||
# Validate Torch MPS before model resolution; MLX has its own gate.
|
||||
# Platform detection covers the usual omitted --device on macOS.
|
||||
if not use_mlx():
|
||||
requested_device = getattr(self, "device", None)
|
||||
explicitly_mps = (
|
||||
requested_device is not None
|
||||
and str(requested_device).split(":", 1)[0] == "mps"
|
||||
)
|
||||
if explicitly_mps or (
|
||||
requested_device is None and get_platform().is_mps
|
||||
):
|
||||
validate_mps_runtime()
|
||||
|
||||
run_resolution_pipeline(self)
|
||||
|
||||
# Validate resolved arguments and checkpoint-derived constraints.
|
||||
cfg = resolving_view(self)
|
||||
if (
|
||||
cfg.device == "mps"
|
||||
and not use_mlx()
|
||||
and str(cfg.model_path).lower() not in ("none", "dummy")
|
||||
):
|
||||
validate_standard_mps_server_args(self)
|
||||
lora_enabled = bool(cfg.enable_lora) or (
|
||||
cfg.enable_lora is None and bool(cfg.lora_paths)
|
||||
)
|
||||
validate_mps_model_config(
|
||||
model_config_of(self),
|
||||
lora_enabled=lora_enabled,
|
||||
)
|
||||
except BaseException:
|
||||
self._resolution_failed = True
|
||||
raise
|
||||
|
||||
@@ -233,7 +233,7 @@ def is_musa() -> bool:
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def is_mps() -> bool:
|
||||
return torch.backends.mps.is_available()
|
||||
return hasattr(torch, "mps") and torch.mps.is_available()
|
||||
|
||||
|
||||
def is_float4_e2m1fn_x2(dtype) -> bool:
|
||||
@@ -535,7 +535,23 @@ def get_available_gpu_memory(
|
||||
free_gpu_memory = psutil.virtual_memory().available
|
||||
free_gpu_memory, total_gpu_memory = torch.musa.mem_get_info()
|
||||
elif device == "mps":
|
||||
free_gpu_memory = psutil.virtual_memory().available
|
||||
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
|
||||
|
||||
num_gpus = torch.mps.device_count()
|
||||
assert gpu_id < num_gpus
|
||||
|
||||
if use_mlx():
|
||||
# Torch's allocator does not track MLX Metal allocations.
|
||||
free_gpu_memory = psutil.virtual_memory().available
|
||||
else:
|
||||
if empty_cache:
|
||||
empty_device_cache(torch.mps)
|
||||
# Bound free memory by host RAM and Metal's working-set limit.
|
||||
total_gpu_memory = torch.mps.recommended_max_memory()
|
||||
metal_headroom = max(
|
||||
0, total_gpu_memory - torch.mps.driver_allocated_memory()
|
||||
)
|
||||
free_gpu_memory = min(psutil.virtual_memory().available, metal_headroom)
|
||||
else:
|
||||
if not current_platform.is_out_of_tree():
|
||||
raise ValueError(
|
||||
@@ -867,6 +883,16 @@ def get_xpu_memory_capacity():
|
||||
raise RuntimeError("torch.xpu is not available.")
|
||||
|
||||
|
||||
def get_mps_memory_capacity():
|
||||
# Metal's working-set limit is smaller than unified system RAM.
|
||||
try:
|
||||
if torch.mps.is_available():
|
||||
return torch.mps.recommended_max_memory() // 1024 // 1024 # unit: MB
|
||||
raise ValueError("No GPU memory values found.")
|
||||
except AttributeError:
|
||||
raise RuntimeError("torch.mps is not available.")
|
||||
|
||||
|
||||
def get_mtgpu_memory_capacity():
|
||||
try:
|
||||
# Run mthreads-gmi and capture the output
|
||||
@@ -919,6 +945,8 @@ def get_device_memory_capacity(device: str = None):
|
||||
gpu_mem = get_nvgpu_memory_capacity()
|
||||
elif is_hip():
|
||||
gpu_mem = get_amdgpu_memory_capacity()
|
||||
elif device == "mps":
|
||||
gpu_mem = get_mps_memory_capacity()
|
||||
elif device == "hpu":
|
||||
gpu_mem = get_hpu_memory_capacity()
|
||||
elif device == "npu":
|
||||
@@ -937,6 +965,9 @@ def get_device_memory_capacity(device: str = None):
|
||||
|
||||
|
||||
def get_device_name(device_id: int = 0) -> str:
|
||||
if hasattr(torch, "mps") and torch.mps.is_available():
|
||||
return torch.backends.mps.get_name()
|
||||
|
||||
if (hasattr(torch, "cuda") and torch.cuda.is_available()) or is_musa():
|
||||
return torch.cuda.get_device_name(device_id)
|
||||
|
||||
@@ -1037,6 +1068,12 @@ def get_device(device_id: Optional[int] = None) -> str:
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_device_count() -> int:
|
||||
if hasattr(torch, "mps") and torch.mps.is_available():
|
||||
try:
|
||||
return torch.mps.device_count()
|
||||
except RuntimeError:
|
||||
return 0
|
||||
|
||||
if (hasattr(torch, "cuda") and torch.cuda.is_available()) or is_musa():
|
||||
try:
|
||||
return torch.cuda.device_count()
|
||||
@@ -1101,6 +1138,10 @@ def get_compiler_backend(mode=None) -> str:
|
||||
if current_platform.is_out_of_tree():
|
||||
return current_platform.get_compile_backend(mode)
|
||||
|
||||
if hasattr(torch, "mps") and torch.mps.is_available():
|
||||
# MPS has no SGLang graph runner.
|
||||
return "eager"
|
||||
|
||||
if hasattr(torch, "hpu") and torch.hpu.is_available():
|
||||
return "hpu_backend"
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ __all__ = [
|
||||
"register_xpu_ci",
|
||||
"register_musa_ci",
|
||||
"register_mlx_ci",
|
||||
"register_mps_ci",
|
||||
"register_ppu_ci",
|
||||
"ut_parse_one_file",
|
||||
]
|
||||
@@ -38,6 +39,7 @@ class HWBackend(Enum):
|
||||
XPU = auto()
|
||||
MUSA = auto()
|
||||
MLX = auto()
|
||||
MPS = auto()
|
||||
PPU = auto()
|
||||
|
||||
|
||||
@@ -165,6 +167,19 @@ def register_ppu_ci(
|
||||
return None
|
||||
|
||||
|
||||
def register_mps_ci(
|
||||
est_time: float,
|
||||
suite: Optional[str] = None,
|
||||
nightly: bool = False,
|
||||
disabled: Optional[str] = None,
|
||||
*,
|
||||
stage: Optional[str] = None,
|
||||
runner_config: Optional[str] = None,
|
||||
):
|
||||
"""Marker for Apple MPS CI registration (parsed via AST; runtime no-op)."""
|
||||
return None
|
||||
|
||||
|
||||
REGISTER_MAPPING = {
|
||||
"register_cpu_ci": HWBackend.CPU,
|
||||
"register_cuda_ci": HWBackend.CUDA,
|
||||
@@ -174,6 +189,7 @@ REGISTER_MAPPING = {
|
||||
"register_xpu_ci": HWBackend.XPU,
|
||||
"register_musa_ci": HWBackend.MUSA,
|
||||
"register_mlx_ci": HWBackend.MLX,
|
||||
"register_mps_ci": HWBackend.MPS,
|
||||
"register_ppu_ci": HWBackend.PPU,
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user