diff --git a/.github/workflows/pr-test-mlx.yml b/.github/workflows/pr-test-mlx.yml index 96d863f93a..569258460a 100644 --- a/.github/workflows/pr-test-mlx.yml +++ b/.github/workflows/pr-test-mlx.yml @@ -16,6 +16,8 @@ on: - '' - stage-a-unit-test-mlx - stage-b-e2e-test-mlx + - stage-a-unit-test-mps + - stage-b-e2e-test-mps concurrency: group: pr-test-mlx-${{ github.ref }} @@ -160,6 +162,100 @@ jobs: run: | uv run python test/run_suite.py --hw mlx --suite stage-b-e2e-mlx + # Run the Torch MPS path without SGLANG_USE_MLX. + stage-a-unit-test-mps: + needs: [check-changes, pr-gate] + if: | + !cancelled() && ( + (needs.pr-gate.result == 'success' && + (needs.check-changes.outputs.main_package == 'true' || + needs.check-changes.outputs.sgl_kernel == 'true')) || + (github.event_name == 'workflow_dispatch' && + inputs.target_stage == 'stage-a-unit-test-mps') + ) + runs-on: macos-26 + timeout-minutes: 60 + env: + SGLANG_IS_IN_CI: true + # Forbid HF downloads to enforce the model-free guarantee. + HF_HUB_OFFLINE: 1 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: 'python/pyproject_other.toml' + + - name: Verify Apple Silicon runner + run: | + echo "uname -m: $(uname -m)" + python3 -c "import platform; assert platform.machine()=='arm64', platform.machine(); print('machine:', platform.machine(), 'system:', platform.system())" + + - name: Install dependencies (srt_mps extra) + timeout-minutes: 30 + run: | + test -f python/pyproject_other.toml || { echo "alt pyproject_other.toml missing"; exit 1; } + rm -f python/pyproject.toml + mv python/pyproject_other.toml python/pyproject.toml + uv venv + uv pip install -e "python[srt_mps,test]" + + - name: Report torch version and MPS availability + run: | + uv run python -c "import torch; print('torch', torch.__version__, 'mps', torch.mps.is_available())" + + - name: Run model-free MPS unit tests + timeout-minutes: 15 + run: | + uv run python test/run_suite.py --hw mps --suite stage-a-unit-test-mps + + stage-b-e2e-test-mps: + needs: [check-changes, pr-gate] + # Manual-only, same rationale as the MLX e2e stage above. + if: | + !cancelled() && inputs.target_stage == 'stage-b-e2e-test-mps' + runs-on: [self-hosted, macOS, ARM64] + timeout-minutes: 120 + env: + SGLANG_IS_IN_CI: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: 'python/pyproject_other.toml' + + - name: Verify Apple Silicon runner + run: | + echo "uname -m: $(uname -m)" + python3 -c "import platform; assert platform.machine()=='arm64', platform.machine(); print('machine:', platform.machine(), 'system:', platform.system())" + + - name: Install dependencies (srt_mps extra) + timeout-minutes: 30 + run: | + test -f python/pyproject_other.toml || { echo "alt pyproject_other.toml missing"; exit 1; } + rm -f python/pyproject.toml + mv python/pyproject_other.toml python/pyproject.toml + uv venv --python 3.11 + uv pip install -e "python[srt_mps,test]" + + # Needs a warm HF cache or network and roughly 24 GB of unified memory. + - name: Run MPS e2e smoke tests + timeout-minutes: 90 + run: | + uv run python test/run_suite.py --hw mps --suite stage-b-e2e-mps + pr-test-mlx-finish: needs: [ @@ -167,6 +263,8 @@ jobs: check-changes, stage-a-unit-test-mlx, stage-b-e2e-test-mlx, + stage-a-unit-test-mps, + stage-b-e2e-test-mps, ] if: always() runs-on: ubuntu-latest diff --git a/python/sglang/_platform_stubs.py b/python/sglang/_platform_stubs.py index b62ba73955..90d8c1d3e4 100644 --- a/python/sglang/_platform_stubs.py +++ b/python/sglang/_platform_stubs.py @@ -162,7 +162,10 @@ class Stream: """ def __init__(self, device: Any = None, priority: int = 0) -> None: - pass + import torch + + self.device = torch.device(device or "mps") + self.priority = priority def synchronize(self) -> None: pass @@ -276,13 +279,24 @@ _cached_props: _MPSDeviceProperties | None = None def get_device_properties(device: Any = 0) -> _MPSDeviceProperties: # noqa: ARG001 - """Return the properties of the MPS device. Results are cached after first call.""" + """Implement ``torch.mps.get_device_properties``, not other device backends. + + Metal's recommended working-set limit bounds usable GPU memory, even though + Apple Silicon shares physical RAM with the CPU. + """ global _cached_props if _cached_props is None: - import psutil + import torch + + total_memory = int(torch.mps.recommended_max_memory()) + if total_memory <= 0: + raise RuntimeError( + "torch.mps.recommended_max_memory() returned a non-positive " + "Metal working-set limit" + ) _cached_props = _MPSDeviceProperties( - total_memory=psutil.virtual_memory().total, + total_memory=total_memory, ) return _cached_props diff --git a/python/sglang/srt/arg_groups/validation_hook.py b/python/sglang/srt/arg_groups/validation_hook.py index 582329a877..b5785fa8e6 100644 --- a/python/sglang/srt/arg_groups/validation_hook.py +++ b/python/sglang/srt/arg_groups/validation_hook.py @@ -485,6 +485,85 @@ def validate_experimental_sgl_marlin(server_args: Any): validate_experimental_sgl_marlin_server_args(server_args, view) +def validate_mps_model_config( + model_config: Any, + *, + lora_enabled: bool = False, +) -> None: + """Validate checkpoint-derived MPS constraints.""" + quantization = getattr(model_config, "quantization", None) + if quantization not in (None, "unquant"): + raise ValueError( + "Torch MPS currently supports only unquantized model weights; " + "the resolved model configuration detected " + f"quantization={quantization!r}" + ) + if bool(getattr(model_config, "is_multimodal", False)): + raise ValueError( + "Torch MPS multimodal serving does not yet have a model-specific " + "end-to-end contract; use a text-only model until its encoder, " + "processor, and decoder paths are validated on MPS" + ) + if lora_enabled: + for config in ( + getattr(model_config, "hf_text_config", None), + getattr(model_config, "hf_config", None), + ): + if config is None: + continue + for field_name in ( + "num_experts", + "num_local_experts", + "n_routed_experts", + ): + value = getattr(config, field_name, None) + if value is not None and int(value) > 0: + raise ValueError( + "Torch MPS LoRA currently supports dense models only; " + f"the model config declares {field_name}={value!r}" + ) + + +def validate_standard_mps_server_args(server_args: Any): + """Validate execution modes supported by the Torch MPS path.""" + + cfg = resolving_view(server_args) + supported_attention_backends = {None, "torch_native"} + for field in ( + "attention_backend", + "prefill_attention_backend", + "decode_attention_backend", + ): + value = getattr(cfg, field, None) + normalized = getattr(value, "value", value) + normalized = None if normalized is None else str(normalized).lower() + if normalized not in supported_attention_backends: + raise ValueError( + "The standard Torch MPS path currently supports only the " + f"torch_native attention backend; got {field}={value!r}" + ) + + sampling_backend = getattr(cfg, "sampling_backend", None) + normalized_sampling = getattr(sampling_backend, "value", sampling_backend) + normalized_sampling = ( + None if normalized_sampling is None else str(normalized_sampling).lower() + ) + if normalized_sampling not in {None, "pytorch"}: + raise ValueError( + "The standard Torch MPS path currently supports only the " + f"pytorch sampling backend; got sampling_backend={sampling_backend!r}" + ) + + if ( + getattr(cfg, "tp_size", 1) != 1 + or getattr(cfg, "pp_size", 1) != 1 + or getattr(cfg, "dp_size", 1) != 1 + ): + raise ValueError( + "The standard Torch MPS path requires tp_size=1, pp_size=1, and dp_size=1" + ) + + def validate_prefill_decode_interval(server_args: Any): cfg = resolving_view(server_args) if cfg.prefill_decode_interval is not None and cfg.prefill_decode_interval < 0: diff --git a/python/sglang/srt/hardware_backend/mps/__init__.py b/python/sglang/srt/hardware_backend/mps/__init__.py new file mode 100644 index 0000000000..4081a387aa --- /dev/null +++ b/python/sglang/srt/hardware_backend/mps/__init__.py @@ -0,0 +1 @@ +"""Apple MPS SRT Providers""" diff --git a/python/sglang/srt/hardware_backend/mps/runtime.py b/python/sglang/srt/hardware_backend/mps/runtime.py new file mode 100644 index 0000000000..fa54c4cccc --- /dev/null +++ b/python/sglang/srt/hardware_backend/mps/runtime.py @@ -0,0 +1,25 @@ +"""Runtime validation for SGLang's Apple MPS backend.""" + +import torch +from packaging.version import Version + + +def validate_mps_runtime() -> None: + """Validate the Torch runtime required by the standard MPS model path.""" + if Version(torch.__version__) < Version("2.13.0"): + raise RuntimeError( + "The standard SGLang MPS model path requires Torch >= 2.13.0; " + f"found Torch {torch.__version__}; reinstall with the " + "srt_mps extra" + ) + + if not torch.backends.mps.is_available(): + raise RuntimeError( + "The SGLang MPS backend requires an available PyTorch MPS device" + ) + for memory_api in ("recommended_max_memory", "driver_allocated_memory"): + if not callable(getattr(torch.mps, memory_api, None)): + raise RuntimeError( + f"The SGLang MPS backend requires torch.mps.{memory_api} from " + "Torch >= 2.13.0" + ) diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 614a26ad2f..d20c68f8e0 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -411,10 +411,21 @@ class ModelRunner: if get_exec().features.enable_tf32_matmul: torch.set_float32_matmul_precision("high") - # Set device early so that TransferEngine init (e.g. Ascend NPU) - # can access the device context. + # Set the device before TransferEngine init. MPS has one implicit device. + is_mps_device = str(self.device).split(":", 1)[0] == "mps" + if is_mps_device: + from sglang.srt.hardware_backend.mlx.runtime import use_mlx + + if not use_mlx(): + # Direct ModelRunner construction bypasses the ServerArgs gate. + from sglang.srt.hardware_backend.mps.runtime import ( + validate_mps_runtime, + ) + + validate_mps_runtime() try: - torch.get_device_module(self.device).set_device(get_device().gpu_id) + if not is_mps_device: + torch.get_device_module(self.device).set_device(get_device().gpu_id) except Exception: import os diff --git a/python/sglang/srt/platforms/__init__.py b/python/sglang/srt/platforms/__init__.py index b5b411ff5a..f6655300d6 100644 --- a/python/sglang/srt/platforms/__init__.py +++ b/python/sglang/srt/platforms/__init__.py @@ -21,6 +21,7 @@ from sglang.srt.environ import envs from sglang.srt.platforms.cpu import CpuSRTPlatform from sglang.srt.platforms.cuda import CudaSRTPlatform from sglang.srt.platforms.interface import SRTPlatform +from sglang.srt.platforms.mps import MpsSRTPlatform from sglang.srt.platforms.npu import NPUSRTPlatform from sglang.srt.platforms.rocm import RocmSRTPlatform from sglang.srt.platforms.xpu import XpuSRTPlatform @@ -51,6 +52,10 @@ def _is_xpu_available() -> bool: return torch.xpu.is_available() +def _is_mps_available() -> bool: + return bool(getattr(torch, "mps", None) is not None and torch.mps.is_available()) + + def _resolve_platform() -> SRTPlatform: """ Discover and instantiate the active platform. @@ -73,6 +78,7 @@ def _resolve_platform() -> SRTPlatform: - 0 activated + CUDA available → fallback CudaSRTPlatform - 0 activated + ROCm available → fallback RocmSRTPlatform - 0 activated + XPU available → fallback XpuSRTPlatform + - 0 activated + MPS available → fallback MpsSRTPlatform - 0 activated + none of the above → fallback base SRTPlatform - 1 activated → use it - N activated → RuntimeError (must set SGLANG_PLATFORM) @@ -143,6 +149,9 @@ def _resolve_platform() -> SRTPlatform: if _is_xpu_available(): logger.debug("No platform plugin detected. Using XPU SRTPlatform defaults.") return XpuSRTPlatform() + if _is_mps_available(): + logger.debug("No platform plugin detected. Using MPS SRTPlatform defaults.") + return MpsSRTPlatform() logger.debug("No platform detected. Using base SRTPlatform.") return SRTPlatform() diff --git a/python/sglang/srt/platforms/mps.py b/python/sglang/srt/platforms/mps.py new file mode 100644 index 0000000000..086e183adb --- /dev/null +++ b/python/sglang/srt/platforms/mps.py @@ -0,0 +1,83 @@ +"""Apple MPS support for SRT's in-tree platform layer.""" + +from __future__ import annotations + +from typing import Optional + +import torch + +from sglang.srt.platforms.device_mixin import ( + DeviceMixin, + PlatformEnum, +) +from sglang.srt.platforms.interface import SRTPlatform + + +def _recommended_working_set_size() -> int: + getter = getattr(torch.mps, "recommended_max_memory", None) + if not callable(getter): + raise RuntimeError( + "The SGLang MPS backend requires " + "torch.mps.recommended_max_memory() from Torch >= 2.13.0" + ) + total = int(getter()) + if total <= 0: + raise RuntimeError( + "torch.mps.recommended_max_memory() returned a non-positive Metal " + "working-set limit; refusing to size the model and KV pools from " + "raw system RAM" + ) + return total + + +class MpsDeviceMixin(DeviceMixin): + """Device operations backed by ``torch.mps``.""" + + _enum: PlatformEnum = PlatformEnum.MPS + device_name: str = "mps" + device_type: str = "mps" + + def get_device_total_memory(self, device_id: int = 0) -> int: + # Metal's working-set limit, not total system RAM, bounds model memory. + return _recommended_working_set_size() + + def get_current_memory_usage(self, device: Optional[torch.device] = None) -> float: + # Driver allocation includes resident cached MPS buffers. + return float(torch.mps.driver_allocated_memory()) + + def get_device(self, local_rank: int = 0) -> torch.device: + if local_rank not in (-1, 0): + raise ValueError(f"MPS exposes one device; got local_rank={local_rank}") + return torch.device("mps") + + def set_device(self, device: torch.device) -> None: + if str(device).split(":", 1)[0] != "mps": + raise ValueError(f"MPS platform cannot select device {device}") + # MPS has no device-selection API. + + def get_device_name(self, device_id: int = 0) -> str: + return str(torch.backends.mps.get_name()) + + def get_device_uuid(self, device_id: int = 0) -> str: + # PyTorch exposes no stable MPS UUID. + return "mps:0" + + def get_device_capability(self, device_id: int = 0): + return None + + def empty_cache(self) -> None: + empty_cache = getattr(torch.mps, "empty_cache", None) + if callable(empty_cache): + empty_cache() + + def synchronize(self) -> None: + synchronize = getattr(torch.mps, "synchronize", None) + if callable(synchronize): + synchronize() + + +class MpsSRTPlatform(MpsDeviceMixin, SRTPlatform): + """Built-in SRT platform for Apple Silicon MPS.""" + + def get_default_attention_backend(self) -> str: + return "torch_native" diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index 00ca5e8910..a361146e76 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -1772,6 +1772,7 @@ _PLATFORM_PROBES: Dict[str, str] = { "is_npu": "is_npu", "is_xpu": "is_xpu", "is_musa": "is_musa", + "is_mps": "is_mps", "is_sm90": "is_sm90_supported", "is_sm100": "is_sm100_supported", "is_sm100_or_sm110": "is_sm100_or_sm110_supported", diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 4c6e03734f..cfe97d5500 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -38,11 +38,18 @@ from sglang.srt.arg_groups.argparse_actions import ( ) from sglang.srt.arg_groups.model_override_base import ep_joiner_of, ep_scale_joiner_of from sglang.srt.arg_groups.overrides import ( + model_config_of, remote_instance_transfer_engine_of, resolution_result, resolving_view, ) +from sglang.srt.arg_groups.validation_hook import ( + validate_mps_model_config, + validate_standard_mps_server_args, +) from sglang.srt.environ import envs +from sglang.srt.hardware_backend.mlx.runtime import use_mlx +from sglang.srt.hardware_backend.mps.runtime import validate_mps_runtime from sglang.srt.runtime_context import ( attn_dp_enabled_of, get_platform, @@ -251,8 +258,38 @@ class ServerArgs: from sglang.srt.arg_groups.pipeline import run_resolution_pipeline self._input_frozen = True + try: + # Validate Torch MPS before model resolution; MLX has its own gate. + # Platform detection covers the usual omitted --device on macOS. + if not use_mlx(): + requested_device = getattr(self, "device", None) + explicitly_mps = ( + requested_device is not None + and str(requested_device).split(":", 1)[0] == "mps" + ) + if explicitly_mps or ( + requested_device is None and get_platform().is_mps + ): + validate_mps_runtime() + run_resolution_pipeline(self) + + # Validate resolved arguments and checkpoint-derived constraints. + cfg = resolving_view(self) + if ( + cfg.device == "mps" + and not use_mlx() + and str(cfg.model_path).lower() not in ("none", "dummy") + ): + validate_standard_mps_server_args(self) + lora_enabled = bool(cfg.enable_lora) or ( + cfg.enable_lora is None and bool(cfg.lora_paths) + ) + validate_mps_model_config( + model_config_of(self), + lora_enabled=lora_enabled, + ) except BaseException: self._resolution_failed = True raise diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 3f17f45226..9791ca5f2f 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -233,7 +233,7 @@ def is_musa() -> bool: @lru_cache(maxsize=1) def is_mps() -> bool: - return torch.backends.mps.is_available() + return hasattr(torch, "mps") and torch.mps.is_available() def is_float4_e2m1fn_x2(dtype) -> bool: @@ -535,7 +535,23 @@ def get_available_gpu_memory( free_gpu_memory = psutil.virtual_memory().available free_gpu_memory, total_gpu_memory = torch.musa.mem_get_info() elif device == "mps": - free_gpu_memory = psutil.virtual_memory().available + from sglang.srt.hardware_backend.mlx.runtime import use_mlx + + num_gpus = torch.mps.device_count() + assert gpu_id < num_gpus + + if use_mlx(): + # Torch's allocator does not track MLX Metal allocations. + free_gpu_memory = psutil.virtual_memory().available + else: + if empty_cache: + empty_device_cache(torch.mps) + # Bound free memory by host RAM and Metal's working-set limit. + total_gpu_memory = torch.mps.recommended_max_memory() + metal_headroom = max( + 0, total_gpu_memory - torch.mps.driver_allocated_memory() + ) + free_gpu_memory = min(psutil.virtual_memory().available, metal_headroom) else: if not current_platform.is_out_of_tree(): raise ValueError( @@ -867,6 +883,16 @@ def get_xpu_memory_capacity(): raise RuntimeError("torch.xpu is not available.") +def get_mps_memory_capacity(): + # Metal's working-set limit is smaller than unified system RAM. + try: + if torch.mps.is_available(): + return torch.mps.recommended_max_memory() // 1024 // 1024 # unit: MB + raise ValueError("No GPU memory values found.") + except AttributeError: + raise RuntimeError("torch.mps is not available.") + + def get_mtgpu_memory_capacity(): try: # Run mthreads-gmi and capture the output @@ -919,6 +945,8 @@ def get_device_memory_capacity(device: str = None): gpu_mem = get_nvgpu_memory_capacity() elif is_hip(): gpu_mem = get_amdgpu_memory_capacity() + elif device == "mps": + gpu_mem = get_mps_memory_capacity() elif device == "hpu": gpu_mem = get_hpu_memory_capacity() elif device == "npu": @@ -937,6 +965,9 @@ def get_device_memory_capacity(device: str = None): def get_device_name(device_id: int = 0) -> str: + if hasattr(torch, "mps") and torch.mps.is_available(): + return torch.backends.mps.get_name() + if (hasattr(torch, "cuda") and torch.cuda.is_available()) or is_musa(): return torch.cuda.get_device_name(device_id) @@ -1037,6 +1068,12 @@ def get_device(device_id: Optional[int] = None) -> str: @lru_cache(maxsize=1) def get_device_count() -> int: + if hasattr(torch, "mps") and torch.mps.is_available(): + try: + return torch.mps.device_count() + except RuntimeError: + return 0 + if (hasattr(torch, "cuda") and torch.cuda.is_available()) or is_musa(): try: return torch.cuda.device_count() @@ -1101,6 +1138,10 @@ def get_compiler_backend(mode=None) -> str: if current_platform.is_out_of_tree(): return current_platform.get_compile_backend(mode) + if hasattr(torch, "mps") and torch.mps.is_available(): + # MPS has no SGLang graph runner. + return "eager" + if hasattr(torch, "hpu") and torch.hpu.is_available(): return "hpu_backend" diff --git a/python/sglang/test/ci/ci_register.py b/python/sglang/test/ci/ci_register.py index 9d946f5aae..0df77a63d1 100644 --- a/python/sglang/test/ci/ci_register.py +++ b/python/sglang/test/ci/ci_register.py @@ -17,6 +17,7 @@ __all__ = [ "register_xpu_ci", "register_musa_ci", "register_mlx_ci", + "register_mps_ci", "register_ppu_ci", "ut_parse_one_file", ] @@ -38,6 +39,7 @@ class HWBackend(Enum): XPU = auto() MUSA = auto() MLX = auto() + MPS = auto() PPU = auto() @@ -165,6 +167,19 @@ def register_ppu_ci( return None +def register_mps_ci( + est_time: float, + suite: Optional[str] = None, + nightly: bool = False, + disabled: Optional[str] = None, + *, + stage: Optional[str] = None, + runner_config: Optional[str] = None, +): + """Marker for Apple MPS CI registration (parsed via AST; runtime no-op).""" + return None + + REGISTER_MAPPING = { "register_cpu_ci": HWBackend.CPU, "register_cuda_ci": HWBackend.CUDA, @@ -174,6 +189,7 @@ REGISTER_MAPPING = { "register_xpu_ci": HWBackend.XPU, "register_musa_ci": HWBackend.MUSA, "register_mlx_ci": HWBackend.MLX, + "register_mps_ci": HWBackend.MPS, "register_ppu_ci": HWBackend.PPU, } diff --git a/scripts/ci/utils/ci_coverage_report.py b/scripts/ci/utils/ci_coverage_report.py index 7bda046ae9..a1acbb64bb 100755 --- a/scripts/ci/utils/ci_coverage_report.py +++ b/scripts/ci/utils/ci_coverage_report.py @@ -35,7 +35,17 @@ from ci_register import CIRegistry, HWBackend, ut_parse_one_file # the right display slot. Order isn't alphabetical: CUDA/AMD/NPU/CPU lead # (highest test volume historically), then accelerators that have been # wired into the registry more recently (XPU, MUSA, MLX, PPU). -BACKEND_DISPLAY_ORDER = ("CUDA", "AMD", "NPU", "CPU", "XPU", "MUSA", "MLX", "PPU") +BACKEND_DISPLAY_ORDER = ( + "CUDA", + "AMD", + "NPU", + "CPU", + "XPU", + "MUSA", + "MLX", + "MPS", + "PPU", +) assert set(BACKEND_DISPLAY_ORDER) == {b.name for b in HWBackend}, ( "BACKEND_DISPLAY_ORDER is out of sync with HWBackend" ) diff --git a/scripts/ci/utils/slash_command_handler.py b/scripts/ci/utils/slash_command_handler.py index 570661c2e4..67df8aa293 100644 --- a/scripts/ci/utils/slash_command_handler.py +++ b/scripts/ci/utils/slash_command_handler.py @@ -946,6 +946,7 @@ _OTHER_BACKEND_REGISTERS = { "register_xpu_ci": "XPU", "register_musa_ci": "MUSA", "register_mlx_ci": "MLX", + "register_mps_ci": "MPS", "register_ppu_ci": "PPU", } diff --git a/scripts/lint/check_no_registered_tests_in_package.py b/scripts/lint/check_no_registered_tests_in_package.py index 799b0cc102..621033c1ca 100755 --- a/scripts/lint/check_no_registered_tests_in_package.py +++ b/scripts/lint/check_no_registered_tests_in_package.py @@ -30,6 +30,7 @@ _MARKERS = ( "register_npu_ci", "register_xpu_ci", "register_musa_ci", + "register_mps_ci", "register_ppu_ci", ) diff --git a/test/registered/mps/models_e2e/test_qwen3_torch_mps.py b/test/registered/mps/models_e2e/test_qwen3_torch_mps.py new file mode 100644 index 0000000000..5f605c9026 --- /dev/null +++ b/test/registered/mps/models_e2e/test_qwen3_torch_mps.py @@ -0,0 +1,90 @@ +"""End-to-end smoke test for the standard Torch ModelRunner on Apple MPS.""" + +import os +import unittest + +import requests +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_mps_ci +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, + try_cached_model, +) + +register_mps_ci(est_time=240, suite="stage-b-e2e-mps") + +MODEL_PATH = os.environ.get("SGLANG_MPS_TEST_MODEL", "Qwen/Qwen3-0.6B") + + +@unittest.skipUnless( + hasattr(torch, "mps") and torch.mps.is_available(), "requires Apple MPS" +) +class TestQwen3TorchMps(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = try_cached_model(MODEL_PATH) + cls.base_url = DEFAULT_URL_FOR_TEST + + def _generate(self, prompt: str) -> dict: + response = requests.post( + f"{self.base_url}/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0, + "ignore_eos": True, + "max_new_tokens": 4, + }, + }, + timeout=120, + ) + response.raise_for_status() + return response.json() + + def test_standard_runner_reuses_radix_cache(self): + # Ensure the child uses Torch rather than the MLX runner. + env = os.environ.copy() + env["SGLANG_USE_MLX"] = "0" + process = popen_launch_server( + self.model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + device="mps", + env=env, + other_args=[ + "--disable-overlap-schedule", + "--attention-backend", + "torch_native", + "--sampling-backend", + "pytorch", + "--mem-fraction-static", + "0.6", + "--max-total-tokens", + "4096", + "--context-length", + "2048", + ], + ) + try: + prompt = ( + "This is a deterministic prefix about science and geography. " * 24 + + "The capital of France is" + ) + cold = self._generate(prompt) + warm = self._generate(prompt) + + self.assertEqual(cold["meta_info"]["cached_tokens"], 0) + self.assertGreater(warm["meta_info"]["cached_tokens"], 0) + self.assertEqual(cold["output_ids"], warm["output_ids"]) + finally: + kill_process_tree(process.pid, wait_timeout=30) + process.wait(timeout=5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/mps/unit/test_mps_runtime.py b/test/registered/mps/unit/test_mps_runtime.py new file mode 100644 index 0000000000..2415d14f7d --- /dev/null +++ b/test/registered/mps/unit/test_mps_runtime.py @@ -0,0 +1,147 @@ +"""Tests for the standard Torch-owned MPS runtime contract.""" + +import types +import unittest +from unittest import mock + +import torch + +from sglang.srt.arg_groups.validation_hook import validate_mps_model_config +from sglang.srt.hardware_backend.mps import runtime +from sglang.srt.runtime_context import override_platform +from sglang.test.ci.ci_register import register_mps_ci +from sglang.test.test_utils import CustomTestCase + +register_mps_ci(est_time=1, suite="stage-a-unit-test-mps") + + +class TestMpsRuntime(CustomTestCase): + def test_runtime_accepts_torch_213_and_newer(self): + for version in ("2.13.0", "2.13.7", "2.14.0", "3.0.0"): + with ( + self.subTest(version=version), + mock.patch.object(torch, "__version__", version), + mock.patch.object( + torch.backends.mps, "is_available", return_value=True + ), + mock.patch.object(torch.mps, "recommended_max_memory", create=True), + mock.patch.object(torch.mps, "driver_allocated_memory", create=True), + ): + self.assertIsNone(runtime.validate_mps_runtime()) + + def test_runtime_does_not_require_mlx_or_metal_kernel_apis(self): + with ( + mock.patch.object(torch, "__version__", "2.13.4"), + mock.patch.object(torch.backends.mps, "is_available", return_value=True), + mock.patch.object( + torch.mps, "recommended_max_memory", return_value=8 << 30, create=True + ), + mock.patch.object( + torch.mps, "driver_allocated_memory", return_value=0, create=True + ), + mock.patch.object(torch.mps, "compile_shader", None, create=True), + mock.patch.object(torch.mps, "load_metallib", None, create=True), + mock.patch.dict("sys.modules", {"mlx": None, "mlx.core": None}), + ): + self.assertIsNone(runtime.validate_mps_runtime()) + + def test_runtime_rejects_unsupported_torch_and_missing_memory_apis(self): + for version in ("2.12.1", "2.13.0rc1"): + with ( + self.subTest(version=version), + mock.patch.object(torch, "__version__", version), + self.assertRaisesRegex(RuntimeError, "Torch >= 2.13"), + ): + runtime.validate_mps_runtime() + + with ( + mock.patch.object(torch, "__version__", "2.13.0"), + mock.patch.object(torch.backends.mps, "is_available", return_value=True), + mock.patch.object(torch.mps, "recommended_max_memory", None, create=True), + self.assertRaisesRegex(RuntimeError, "recommended_max_memory"), + ): + runtime.validate_mps_runtime() + + def test_runtime_rejects_unavailable_mps(self): + with ( + mock.patch.object(torch, "__version__", "2.13.0"), + mock.patch.object(torch.backends.mps, "is_available", return_value=False), + self.assertRaisesRegex(RuntimeError, "available PyTorch MPS device"), + ): + runtime.validate_mps_runtime() + + @staticmethod + def _resolve_with_gate(device, *, mlx, detected_mps=False): + """Resolve dummy arguments and return the patched runtime validator.""" + from sglang.srt import server_args + from sglang.srt.server_args import ServerArgs + + with ( + mock.patch.object(server_args, "use_mlx", return_value=mlx), + override_platform(is_mps=detected_mps), + mock.patch.object(server_args, "validate_mps_runtime") as validate, + ): + ServerArgs(model_path="dummy", device=device).resolve_once() + return validate + + def test_server_args_selects_runtime_gate_by_execution_path(self): + self._resolve_with_gate("mps", mlx=False).assert_called_once_with() + self._resolve_with_gate("mps", mlx=True).assert_not_called() + + def test_runtime_gate_follows_autodetected_platform_when_device_is_unset(self): + """Use platform detection when --device is omitted.""" + self._resolve_with_gate( + None, mlx=False, detected_mps=True + ).assert_called_once_with() + self._resolve_with_gate(None, mlx=False, detected_mps=False).assert_not_called() + + def test_checkpoint_derived_execution_modes_are_rejected(self): + self.assertIsNone( + validate_mps_model_config( + types.SimpleNamespace(quantization=None, is_multimodal=False) + ) + ) + with self.assertRaisesRegex(ValueError, "quantization='awq'"): + validate_mps_model_config( + types.SimpleNamespace(quantization="awq", is_multimodal=False) + ) + with self.assertRaisesRegex(ValueError, "multimodal serving"): + validate_mps_model_config( + types.SimpleNamespace(quantization=None, is_multimodal=True) + ) + + def test_standard_path_rejects_unsupported_execution_modes(self): + from sglang.srt.arg_groups.validation_hook import ( + validate_standard_mps_server_args, + ) + + def make(**overrides): + base = dict( + attention_backend="torch_native", + prefill_attention_backend=None, + decode_attention_backend=None, + sampling_backend="pytorch", + tp_size=1, + pp_size=1, + dp_size=1, + quantization=None, + ) + base.update(overrides) + return types.SimpleNamespace(**base) + + self.assertIsNone(validate_standard_mps_server_args(make())) + + for overrides, expected in ( + ({"attention_backend": "fa3"}, "torch_native attention backend"), + ({"sampling_backend": "flashinfer"}, "pytorch sampling backend"), + ({"tp_size": 2}, "tp_size=1"), + ({"pp_size": 2}, "tp_size=1"), + ({"dp_size": 2}, "tp_size=1"), + ): + with self.subTest(**overrides): + with self.assertRaisesRegex(ValueError, expected): + validate_standard_mps_server_args(make(**overrides)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/observability/test_customized_info_streaming.py b/test/registered/observability/test_customized_info_streaming.py index 0a81043765..a89dfe76d4 100644 --- a/test/registered/observability/test_customized_info_streaming.py +++ b/test/registered/observability/test_customized_info_streaming.py @@ -8,6 +8,7 @@ import torch from sglang.srt.entrypoints.engine import Engine from sglang.srt.layers.sampler import Sampler, register_sampler_backend from sglang.srt.managers.scheduler import run_scheduler_process +from sglang.srt.platforms import current_platform from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.mock_model.utils import MOCK_MODEL_PATH from sglang.test.test_utils import CustomTestCase @@ -83,6 +84,11 @@ class _CustomizedInfoEngine(Engine): ) +@unittest.skipIf( + current_platform.is_mps(), + "The standard Torch MPS path only supports the pytorch sampling backend, " + "so the customized-info probe backend cannot be installed there.", +) class TestCustomizedInfoStreaming(CustomTestCase): @classmethod def setUpClass(cls): diff --git a/test/registered/unit/platforms/test_platform_interface.py b/test/registered/unit/platforms/test_platform_interface.py index 17d6eca01b..0edc411926 100644 --- a/test/registered/unit/platforms/test_platform_interface.py +++ b/test/registered/unit/platforms/test_platform_interface.py @@ -9,6 +9,7 @@ from unittest.mock import MagicMock, patch import torch +from sglang import _platform_stubs from sglang.srt.platforms import _load_platform_class, _resolve_platform from sglang.srt.platforms.cpu import CpuSRTPlatform from sglang.srt.platforms.cuda import CudaSRTPlatform @@ -19,6 +20,7 @@ from sglang.srt.platforms.device_mixin import ( PlatformEnum, ) from sglang.srt.platforms.interface import SRTPlatform +from sglang.srt.platforms.mps import MpsSRTPlatform from sglang.srt.platforms.npu import NPUSRTPlatform from sglang.srt.platforms.rocm import RocmSRTPlatform from sglang.srt.platforms.xpu import XpuSRTPlatform @@ -372,6 +374,114 @@ class TestCpuDeviceMixin(CustomTestCase): self.assertIn("x86_64", name) +class TestMpsDeviceMixin(CustomTestCase): + """Tests for the built-in Apple MPS platform contract.""" + + def test_identity_and_backend_defaults(self): + base = MpsSRTPlatform() + self.assertTrue(base.is_mps()) + self.assertEqual(base.get_device(), torch.device("mps")) + self.assertEqual(base.get_default_attention_backend(), "torch_native") + self.assertEqual(base.get_torch_distributed_backend_str(), "gloo") + self.assertFalse(base.is_pin_memory_available()) + + def test_single_device_contract(self): + base = MpsSRTPlatform() + base.set_device(torch.device("mps")) + with self.assertRaisesRegex(ValueError, "one device"): + base.get_device(1) + with self.assertRaisesRegex(ValueError, "cannot select"): + base.set_device(torch.device("cpu")) + + @patch("sglang.srt.platforms.mps.torch.mps.driver_allocated_memory") + @patch("sglang.srt.platforms.mps.torch.mps.recommended_max_memory") + @patch( + "sglang.srt.platforms.mps.torch.mps.current_allocated_memory", + side_effect=AssertionError("device budget must use driver allocation"), + ) + def test_unified_memory_contract(self, mock_current, mock_recommended, mock_driver): + mock_recommended.return_value = 800 + mock_driver.return_value = 350 + base = MpsSRTPlatform() + # Use Metal capacity and driver allocation for memory accounting. + self.assertEqual(base.get_device_total_memory(), 800) + self.assertEqual(base.get_current_memory_usage(), 350.0) + mock_current.assert_not_called() + + mock_recommended.return_value = 0 + with self.assertRaisesRegex(RuntimeError, "non-positive Metal working-set"): + base.get_device_total_memory() + + @patch("torch.mps.recommended_max_memory") + def test_stub_device_properties_use_metal_working_set(self, mock_recommended): + mock_recommended.return_value = 800 + with patch.object(_platform_stubs, "_cached_props", None): + self.assertEqual(_platform_stubs.get_device_properties().total_memory, 800) + + @patch("torch.backends.mps.get_name", return_value="Apple M-series") + def test_device_metadata_contract(self, _mock_name): + base = MpsSRTPlatform() + self.assertIsNone(base.get_device_capability()) + self.assertEqual(base.get_device_name(), "Apple M-series") + + @patch("torch.mps.driver_allocated_memory") + @patch("torch.mps.recommended_max_memory") + def test_common_device_helpers_handle_mps_inline( + self, mock_recommended, mock_driver + ): + """Handle MPS directly instead of using OOT dispatch.""" + from sglang.srt.utils import common + + mock_recommended.return_value = 8 << 30 + mock_driver.return_value = 5 << 30 + + with ( + patch.object(common, "is_mps", return_value=True), + patch.object(common, "is_cuda", return_value=False), + patch.object(common, "is_hip", return_value=False), + patch.object(common, "is_cpu", return_value=False), + patch.object(common, "is_npu", return_value=False), + patch.object(common, "is_musa", return_value=False), + patch.object(common, "is_habana_available", return_value=False), + patch.object(torch.cuda, "is_available", return_value=False), + patch.object(torch.xpu, "is_available", return_value=False), + patch.object(torch.mps, "is_available", return_value=True), + patch.object(torch.mps, "device_count", return_value=1), + patch.object(common, "empty_device_cache"), + # The Linux CPU suite has no MPS device name. + patch("torch.backends.mps.get_name", return_value="Apple M-series"), + patch( + "sglang.srt.hardware_backend.mlx.runtime.use_mlx", + return_value=False, + ), + ): + common.get_device_count.cache_clear() + common.get_device.cache_clear() + try: + self.assertEqual(common.get_device_memory_capacity("mps"), 8192) + self.assertEqual(common.get_device_name(), "Apple M-series") + self.assertEqual(common.get_device_count(), 1) + self.assertEqual(common.get_device(), "mps") + self.assertEqual(common.get_device(0), "mps:0") + self.assertEqual(common.get_compiler_backend(), "eager") + + # Metal headroom is the tighter limit. + with patch.object(common.psutil, "virtual_memory") as mock_vm: + mock_vm.return_value.available = 6 << 30 + self.assertEqual(common.get_available_gpu_memory("mps", 0), 3.0) + + # Host memory can be the tighter limit. + mock_vm.return_value.available = 1 << 30 + self.assertEqual(common.get_available_gpu_memory("mps", 0), 1.0) + + # Clamp negative Metal headroom. + mock_driver.return_value = 9 << 30 + self.assertEqual(common.get_available_gpu_memory("mps", 0), 0.0) + finally: + common.get_device_count.cache_clear() + common.get_device.cache_clear() + + class TestPinMemoryAvailability(CustomTestCase): """Tests for common pin-memory helper dispatch through platforms.""" @@ -484,7 +594,7 @@ class TestResolvePlatformWithEnv(CustomTestCase): mock_envs.SGLANG_PLATFORM.get.return_value = "nonexistent" mock_ep.return_value = [] with self.assertRaises(RuntimeError): - _resolve_platform() + result = _resolve_platform() @patch("sglang.srt.platforms.entry_points") @patch("sglang.srt.platforms.envs") @@ -564,10 +674,11 @@ class TestResolvePlatformAutoDiscover(CustomTestCase): self.assertIsInstance(result, CudaSRTPlatform) @patch("sglang.srt.platforms.load_plugins_by_group") + @patch("sglang.srt.platforms._is_mps_available", return_value=False) @patch("sglang.srt.platforms._is_cuda_available") @patch("sglang.srt.platforms.envs") def test_no_plugin_no_cuda_activates_base_fallback( - self, mock_envs, mock_is_cuda_available, mock_load + self, mock_envs, mock_is_cuda_available, _mock_is_mps_available, mock_load ): """When no plugin or CUDA is available, return the abstract base platform.""" mock_envs.SGLANG_PLATFORM.get.return_value = "" @@ -577,6 +688,27 @@ class TestResolvePlatformAutoDiscover(CustomTestCase): self.assertIsInstance(result, SRTPlatform) self.assertNotIsInstance(result, CudaSRTPlatform) + @patch("sglang.srt.platforms.load_plugins_by_group") + @patch("sglang.srt.platforms._is_mps_available", return_value=True) + @patch("sglang.srt.platforms._is_xpu_available", return_value=False) + @patch("sglang.srt.platforms._is_rocm_available", return_value=False) + @patch("sglang.srt.platforms._is_cuda_available", return_value=False) + @patch("sglang.srt.platforms._is_cpu_available", return_value=False) + @patch("sglang.srt.platforms.envs") + def test_no_plugin_activates_mps_fallback( + self, + mock_envs, + _mock_is_cpu, + _mock_is_cuda, + _mock_is_rocm, + _mock_is_xpu, + _mock_is_mps, + mock_load, + ): + mock_envs.SGLANG_PLATFORM.get.return_value = "" + mock_load.return_value = {} + self.assertIsInstance(_resolve_platform(), MpsSRTPlatform) + @patch("sglang.srt.platforms.load_plugins_by_group") @patch("sglang.srt.platforms.torch") @patch("sglang.srt.platforms.envs") diff --git a/test/run_suite.py b/test/run_suite.py index c234be8c66..d5ac68da11 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -23,6 +23,7 @@ HW_MAPPING = { "npu": HWBackend.NPU, "xpu": HWBackend.XPU, "mlx": HWBackend.MLX, + "mps": HWBackend.MPS, "ppu": HWBackend.PPU, } @@ -125,6 +126,11 @@ PER_COMMIT_SUITES = { "stage-a-unit-test-mlx", "stage-b-e2e-mlx", ], + HWBackend.MPS: [ + "stage-a-unit-test-mps", + # stage-b needs a self-hosted Apple Silicon runner; dispatched manually. + "stage-b-e2e-mps", + ], # PPU has no suite in any of the three dicts yet: pr-test-ppu.yml only runs # the runner preflight until the PPU SRT platform and AOT kernels land. # Declaring a name here before a workflow dispatches it would let a test @@ -241,6 +247,7 @@ _SUITE_CHECKED_BACKENDS = { HWBackend.MUSA, HWBackend.XPU, HWBackend.MLX, + HWBackend.MPS, HWBackend.PPU, }