diff --git a/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py b/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py index 80793e875d..27d7c8a7d4 100644 --- a/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py +++ b/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py @@ -10,6 +10,7 @@ from typing import Tuple import torch from sglang.srt.configs.hybrid_arch import mambaish_config +from sglang.srt.configs.shared_experts import SharedExpertsConfig from sglang.srt.hardware_backend.mlx.kv_cache.auxiliary_state import ( MlxAuxiliaryStateReqToTokenPool, ) @@ -84,6 +85,10 @@ class _DummyModel: """Minimal stand-in so that `inspect.signature(model.forward)` and `getattr(model, ...)` calls in ModelRunner.__init__ don't crash.""" + # The model loader sets this on every Torch model, and the graph runners + # read it during warmup. No Torch MoE layers exist here, so nothing is fused. + shared_experts_config = SharedExpertsConfig() + @staticmethod def forward(): pass diff --git a/test/registered/unit/hardware_backend/mlx/test_mlx_runner_pool_contract.py b/test/registered/unit/hardware_backend/mlx/test_mlx_runner_pool_contract.py index 6de3fac850..8e9950ae50 100644 --- a/test/registered/unit/hardware_backend/mlx/test_mlx_runner_pool_contract.py +++ b/test/registered/unit/hardware_backend/mlx/test_mlx_runner_pool_contract.py @@ -11,6 +11,9 @@ attention backend named by ``server_args.attention_backend``; MLX never uses one, and some backends read real KV buffers in ``__init__``, which crashes on ``_DummyKVCache``. +The graph runners select ``model.shared_experts_config`` during warmup. The +Torch model loader sets it, so the stub's placeholder model must carry one. + The base ``preloaded_weights_bytes`` property reads the Torch model loader. The MLX stub never creates one because its native runner owns model loading and KV sizing, so it must report zero for the Torch accounting hook. @@ -39,6 +42,7 @@ if _HAS_MLX: from sglang.srt.managers.tp_worker import TpModelWorker from sglang.srt.mem_cache.memory_pool import ReqToTokenPool from sglang.srt.model_executor.model_runner import ModelRunner + from sglang.srt.model_executor.runner.base_runner import BaseRunner @unittest.skipUnless(_HAS_MLX, _SKIP_REASON) @@ -85,6 +89,26 @@ class TestMlxRunnerPoolContract(CustomTestCase): runner.account_preloaded_weights(runner.preloaded_weights_bytes) self.assertEqual(runner.pre_model_load_memory, 1.0) + def test_stub_model_supports_runner_model_context(self): + """Runner warmup must find a shared-expert layout on the stub model.""" + runner = object.__new__(MlxModelRunnerStub) + runner.model_config = SimpleNamespace( + is_hybrid_swa=False, + sliding_window_size=None, + attention_chunk_size=None, + dtype=None, + ) + runner.load_model() + graph_runner = SimpleNamespace(model_runner=runner) + + with ( + get_context().override_server_args(), + BaseRunner.model_context(graph_runner), + ): + pass + + self.assertEqual(runner.model.shared_experts_config.num_fused_shared_experts, 0) + def test_stub_overrides_base_alloc_memory_pool(self): self.assertIn( "alloc_memory_pool",