[Apple Silicon] Add an Apple Silicon (MPS) platform to the standard Torch model runner (#36780)

This commit is contained in:
Aditya Sharma
2026-10-04 15:03:40 -07:00
committed by GitHub
parent bab04cd791
commit caf96324fc
20 changed files with 821 additions and 12 deletions
+18 -4
View File
@@ -162,7 +162,10 @@ class Stream:
"""
def __init__(self, device: Any = None, priority: int = 0) -> None:
pass
import torch
self.device = torch.device(device or "mps")
self.priority = priority
def synchronize(self) -> None:
pass
@@ -276,13 +279,24 @@ _cached_props: _MPSDeviceProperties | None = None
def get_device_properties(device: Any = 0) -> _MPSDeviceProperties: # noqa: ARG001
"""Return the properties of the MPS device. Results are cached after first call."""
"""Implement ``torch.mps.get_device_properties``, not other device backends.
Metal's recommended working-set limit bounds usable GPU memory, even though
Apple Silicon shares physical RAM with the CPU.
"""
global _cached_props
if _cached_props is None:
import psutil
import torch
total_memory = int(torch.mps.recommended_max_memory())
if total_memory <= 0:
raise RuntimeError(
"torch.mps.recommended_max_memory() returned a non-positive "
"Metal working-set limit"
)
_cached_props = _MPSDeviceProperties(
total_memory=psutil.virtual_memory().total,
total_memory=total_memory,
)
return _cached_props
@@ -485,6 +485,85 @@ def validate_experimental_sgl_marlin(server_args: Any):
validate_experimental_sgl_marlin_server_args(server_args, view)
def validate_mps_model_config(
model_config: Any,
*,
lora_enabled: bool = False,
) -> None:
"""Validate checkpoint-derived MPS constraints."""
quantization = getattr(model_config, "quantization", None)
if quantization not in (None, "unquant"):
raise ValueError(
"Torch MPS currently supports only unquantized model weights; "
"the resolved model configuration detected "
f"quantization={quantization!r}"
)
if bool(getattr(model_config, "is_multimodal", False)):
raise ValueError(
"Torch MPS multimodal serving does not yet have a model-specific "
"end-to-end contract; use a text-only model until its encoder, "
"processor, and decoder paths are validated on MPS"
)
if lora_enabled:
for config in (
getattr(model_config, "hf_text_config", None),
getattr(model_config, "hf_config", None),
):
if config is None:
continue
for field_name in (
"num_experts",
"num_local_experts",
"n_routed_experts",
):
value = getattr(config, field_name, None)
if value is not None and int(value) > 0:
raise ValueError(
"Torch MPS LoRA currently supports dense models only; "
f"the model config declares {field_name}={value!r}"
)
def validate_standard_mps_server_args(server_args: Any):
"""Validate execution modes supported by the Torch MPS path."""
cfg = resolving_view(server_args)
supported_attention_backends = {None, "torch_native"}
for field in (
"attention_backend",
"prefill_attention_backend",
"decode_attention_backend",
):
value = getattr(cfg, field, None)
normalized = getattr(value, "value", value)
normalized = None if normalized is None else str(normalized).lower()
if normalized not in supported_attention_backends:
raise ValueError(
"The standard Torch MPS path currently supports only the "
f"torch_native attention backend; got {field}={value!r}"
)
sampling_backend = getattr(cfg, "sampling_backend", None)
normalized_sampling = getattr(sampling_backend, "value", sampling_backend)
normalized_sampling = (
None if normalized_sampling is None else str(normalized_sampling).lower()
)
if normalized_sampling not in {None, "pytorch"}:
raise ValueError(
"The standard Torch MPS path currently supports only the "
f"pytorch sampling backend; got sampling_backend={sampling_backend!r}"
)
if (
getattr(cfg, "tp_size", 1) != 1
or getattr(cfg, "pp_size", 1) != 1
or getattr(cfg, "dp_size", 1) != 1
):
raise ValueError(
"The standard Torch MPS path requires tp_size=1, pp_size=1, and dp_size=1"
)
def validate_prefill_decode_interval(server_args: Any):
cfg = resolving_view(server_args)
if cfg.prefill_decode_interval is not None and cfg.prefill_decode_interval < 0:
@@ -0,0 +1 @@
"""Apple MPS SRT Providers"""
@@ -0,0 +1,25 @@
"""Runtime validation for SGLang's Apple MPS backend."""
import torch
from packaging.version import Version
def validate_mps_runtime() -> None:
"""Validate the Torch runtime required by the standard MPS model path."""
if Version(torch.__version__) < Version("2.13.0"):
raise RuntimeError(
"The standard SGLang MPS model path requires Torch >= 2.13.0; "
f"found Torch {torch.__version__}; reinstall with the "
"srt_mps extra"
)
if not torch.backends.mps.is_available():
raise RuntimeError(
"The SGLang MPS backend requires an available PyTorch MPS device"
)
for memory_api in ("recommended_max_memory", "driver_allocated_memory"):
if not callable(getattr(torch.mps, memory_api, None)):
raise RuntimeError(
f"The SGLang MPS backend requires torch.mps.{memory_api} from "
"Torch >= 2.13.0"
)
@@ -411,10 +411,21 @@ class ModelRunner:
if get_exec().features.enable_tf32_matmul:
torch.set_float32_matmul_precision("high")
# Set device early so that TransferEngine init (e.g. Ascend NPU)
# can access the device context.
# Set the device before TransferEngine init. MPS has one implicit device.
is_mps_device = str(self.device).split(":", 1)[0] == "mps"
if is_mps_device:
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
if not use_mlx():
# Direct ModelRunner construction bypasses the ServerArgs gate.
from sglang.srt.hardware_backend.mps.runtime import (
validate_mps_runtime,
)
validate_mps_runtime()
try:
torch.get_device_module(self.device).set_device(get_device().gpu_id)
if not is_mps_device:
torch.get_device_module(self.device).set_device(get_device().gpu_id)
except Exception:
import os
+9
View File
@@ -21,6 +21,7 @@ from sglang.srt.environ import envs
from sglang.srt.platforms.cpu import CpuSRTPlatform
from sglang.srt.platforms.cuda import CudaSRTPlatform
from sglang.srt.platforms.interface import SRTPlatform
from sglang.srt.platforms.mps import MpsSRTPlatform
from sglang.srt.platforms.npu import NPUSRTPlatform
from sglang.srt.platforms.rocm import RocmSRTPlatform
from sglang.srt.platforms.xpu import XpuSRTPlatform
@@ -51,6 +52,10 @@ def _is_xpu_available() -> bool:
return torch.xpu.is_available()
def _is_mps_available() -> bool:
return bool(getattr(torch, "mps", None) is not None and torch.mps.is_available())
def _resolve_platform() -> SRTPlatform:
"""
Discover and instantiate the active platform.
@@ -73,6 +78,7 @@ def _resolve_platform() -> SRTPlatform:
- 0 activated + CUDA available → fallback CudaSRTPlatform
- 0 activated + ROCm available → fallback RocmSRTPlatform
- 0 activated + XPU available → fallback XpuSRTPlatform
- 0 activated + MPS available → fallback MpsSRTPlatform
- 0 activated + none of the above → fallback base SRTPlatform
- 1 activated → use it
- N activated → RuntimeError (must set SGLANG_PLATFORM)
@@ -143,6 +149,9 @@ def _resolve_platform() -> SRTPlatform:
if _is_xpu_available():
logger.debug("No platform plugin detected. Using XPU SRTPlatform defaults.")
return XpuSRTPlatform()
if _is_mps_available():
logger.debug("No platform plugin detected. Using MPS SRTPlatform defaults.")
return MpsSRTPlatform()
logger.debug("No platform detected. Using base SRTPlatform.")
return SRTPlatform()
+83
View File
@@ -0,0 +1,83 @@
"""Apple MPS support for SRT's in-tree platform layer."""
from __future__ import annotations
from typing import Optional
import torch
from sglang.srt.platforms.device_mixin import (
DeviceMixin,
PlatformEnum,
)
from sglang.srt.platforms.interface import SRTPlatform
def _recommended_working_set_size() -> int:
getter = getattr(torch.mps, "recommended_max_memory", None)
if not callable(getter):
raise RuntimeError(
"The SGLang MPS backend requires "
"torch.mps.recommended_max_memory() from Torch >= 2.13.0"
)
total = int(getter())
if total <= 0:
raise RuntimeError(
"torch.mps.recommended_max_memory() returned a non-positive Metal "
"working-set limit; refusing to size the model and KV pools from "
"raw system RAM"
)
return total
class MpsDeviceMixin(DeviceMixin):
"""Device operations backed by ``torch.mps``."""
_enum: PlatformEnum = PlatformEnum.MPS
device_name: str = "mps"
device_type: str = "mps"
def get_device_total_memory(self, device_id: int = 0) -> int:
# Metal's working-set limit, not total system RAM, bounds model memory.
return _recommended_working_set_size()
def get_current_memory_usage(self, device: Optional[torch.device] = None) -> float:
# Driver allocation includes resident cached MPS buffers.
return float(torch.mps.driver_allocated_memory())
def get_device(self, local_rank: int = 0) -> torch.device:
if local_rank not in (-1, 0):
raise ValueError(f"MPS exposes one device; got local_rank={local_rank}")
return torch.device("mps")
def set_device(self, device: torch.device) -> None:
if str(device).split(":", 1)[0] != "mps":
raise ValueError(f"MPS platform cannot select device {device}")
# MPS has no device-selection API.
def get_device_name(self, device_id: int = 0) -> str:
return str(torch.backends.mps.get_name())
def get_device_uuid(self, device_id: int = 0) -> str:
# PyTorch exposes no stable MPS UUID.
return "mps:0"
def get_device_capability(self, device_id: int = 0):
return None
def empty_cache(self) -> None:
empty_cache = getattr(torch.mps, "empty_cache", None)
if callable(empty_cache):
empty_cache()
def synchronize(self) -> None:
synchronize = getattr(torch.mps, "synchronize", None)
if callable(synchronize):
synchronize()
class MpsSRTPlatform(MpsDeviceMixin, SRTPlatform):
"""Built-in SRT platform for Apple Silicon MPS."""
def get_default_attention_backend(self) -> str:
return "torch_native"
+1
View File
@@ -1772,6 +1772,7 @@ _PLATFORM_PROBES: Dict[str, str] = {
"is_npu": "is_npu",
"is_xpu": "is_xpu",
"is_musa": "is_musa",
"is_mps": "is_mps",
"is_sm90": "is_sm90_supported",
"is_sm100": "is_sm100_supported",
"is_sm100_or_sm110": "is_sm100_or_sm110_supported",
+37
View File
@@ -38,11 +38,18 @@ from sglang.srt.arg_groups.argparse_actions import (
)
from sglang.srt.arg_groups.model_override_base import ep_joiner_of, ep_scale_joiner_of
from sglang.srt.arg_groups.overrides import (
model_config_of,
remote_instance_transfer_engine_of,
resolution_result,
resolving_view,
)
from sglang.srt.arg_groups.validation_hook import (
validate_mps_model_config,
validate_standard_mps_server_args,
)
from sglang.srt.environ import envs
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
from sglang.srt.hardware_backend.mps.runtime import validate_mps_runtime
from sglang.srt.runtime_context import (
attn_dp_enabled_of,
get_platform,
@@ -251,8 +258,38 @@ class ServerArgs:
from sglang.srt.arg_groups.pipeline import run_resolution_pipeline
self._input_frozen = True
try:
# Validate Torch MPS before model resolution; MLX has its own gate.
# Platform detection covers the usual omitted --device on macOS.
if not use_mlx():
requested_device = getattr(self, "device", None)
explicitly_mps = (
requested_device is not None
and str(requested_device).split(":", 1)[0] == "mps"
)
if explicitly_mps or (
requested_device is None and get_platform().is_mps
):
validate_mps_runtime()
run_resolution_pipeline(self)
# Validate resolved arguments and checkpoint-derived constraints.
cfg = resolving_view(self)
if (
cfg.device == "mps"
and not use_mlx()
and str(cfg.model_path).lower() not in ("none", "dummy")
):
validate_standard_mps_server_args(self)
lora_enabled = bool(cfg.enable_lora) or (
cfg.enable_lora is None and bool(cfg.lora_paths)
)
validate_mps_model_config(
model_config_of(self),
lora_enabled=lora_enabled,
)
except BaseException:
self._resolution_failed = True
raise
+43 -2
View File
@@ -233,7 +233,7 @@ def is_musa() -> bool:
@lru_cache(maxsize=1)
def is_mps() -> bool:
return torch.backends.mps.is_available()
return hasattr(torch, "mps") and torch.mps.is_available()
def is_float4_e2m1fn_x2(dtype) -> bool:
@@ -535,7 +535,23 @@ def get_available_gpu_memory(
free_gpu_memory = psutil.virtual_memory().available
free_gpu_memory, total_gpu_memory = torch.musa.mem_get_info()
elif device == "mps":
free_gpu_memory = psutil.virtual_memory().available
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
num_gpus = torch.mps.device_count()
assert gpu_id < num_gpus
if use_mlx():
# Torch's allocator does not track MLX Metal allocations.
free_gpu_memory = psutil.virtual_memory().available
else:
if empty_cache:
empty_device_cache(torch.mps)
# Bound free memory by host RAM and Metal's working-set limit.
total_gpu_memory = torch.mps.recommended_max_memory()
metal_headroom = max(
0, total_gpu_memory - torch.mps.driver_allocated_memory()
)
free_gpu_memory = min(psutil.virtual_memory().available, metal_headroom)
else:
if not current_platform.is_out_of_tree():
raise ValueError(
@@ -867,6 +883,16 @@ def get_xpu_memory_capacity():
raise RuntimeError("torch.xpu is not available.")
def get_mps_memory_capacity():
# Metal's working-set limit is smaller than unified system RAM.
try:
if torch.mps.is_available():
return torch.mps.recommended_max_memory() // 1024 // 1024 # unit: MB
raise ValueError("No GPU memory values found.")
except AttributeError:
raise RuntimeError("torch.mps is not available.")
def get_mtgpu_memory_capacity():
try:
# Run mthreads-gmi and capture the output
@@ -919,6 +945,8 @@ def get_device_memory_capacity(device: str = None):
gpu_mem = get_nvgpu_memory_capacity()
elif is_hip():
gpu_mem = get_amdgpu_memory_capacity()
elif device == "mps":
gpu_mem = get_mps_memory_capacity()
elif device == "hpu":
gpu_mem = get_hpu_memory_capacity()
elif device == "npu":
@@ -937,6 +965,9 @@ def get_device_memory_capacity(device: str = None):
def get_device_name(device_id: int = 0) -> str:
if hasattr(torch, "mps") and torch.mps.is_available():
return torch.backends.mps.get_name()
if (hasattr(torch, "cuda") and torch.cuda.is_available()) or is_musa():
return torch.cuda.get_device_name(device_id)
@@ -1037,6 +1068,12 @@ def get_device(device_id: Optional[int] = None) -> str:
@lru_cache(maxsize=1)
def get_device_count() -> int:
if hasattr(torch, "mps") and torch.mps.is_available():
try:
return torch.mps.device_count()
except RuntimeError:
return 0
if (hasattr(torch, "cuda") and torch.cuda.is_available()) or is_musa():
try:
return torch.cuda.device_count()
@@ -1101,6 +1138,10 @@ def get_compiler_backend(mode=None) -> str:
if current_platform.is_out_of_tree():
return current_platform.get_compile_backend(mode)
if hasattr(torch, "mps") and torch.mps.is_available():
# MPS has no SGLang graph runner.
return "eager"
if hasattr(torch, "hpu") and torch.hpu.is_available():
return "hpu_backend"
+16
View File
@@ -17,6 +17,7 @@ __all__ = [
"register_xpu_ci",
"register_musa_ci",
"register_mlx_ci",
"register_mps_ci",
"register_ppu_ci",
"ut_parse_one_file",
]
@@ -38,6 +39,7 @@ class HWBackend(Enum):
XPU = auto()
MUSA = auto()
MLX = auto()
MPS = auto()
PPU = auto()
@@ -165,6 +167,19 @@ def register_ppu_ci(
return None
def register_mps_ci(
est_time: float,
suite: Optional[str] = None,
nightly: bool = False,
disabled: Optional[str] = None,
*,
stage: Optional[str] = None,
runner_config: Optional[str] = None,
):
"""Marker for Apple MPS CI registration (parsed via AST; runtime no-op)."""
return None
REGISTER_MAPPING = {
"register_cpu_ci": HWBackend.CPU,
"register_cuda_ci": HWBackend.CUDA,
@@ -174,6 +189,7 @@ REGISTER_MAPPING = {
"register_xpu_ci": HWBackend.XPU,
"register_musa_ci": HWBackend.MUSA,
"register_mlx_ci": HWBackend.MLX,
"register_mps_ci": HWBackend.MPS,
"register_ppu_ci": HWBackend.PPU,
}