diff --git a/python/sglang/srt/model_executor/runner/flashinfer_autotune.py b/python/sglang/srt/model_executor/runner/flashinfer_autotune.py index c8d2f35519..71885e9a2e 100644 --- a/python/sglang/srt/model_executor/runner/flashinfer_autotune.py +++ b/python/sglang/srt/model_executor/runner/flashinfer_autotune.py @@ -263,6 +263,8 @@ def _drop_diverged_autotune_cache( @contextlib.contextmanager def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool): + from flashinfer import autotune_v2 + # The gate below decides on the same inputs load_configs does. from flashinfer.autotuner import AutoTuner, _collect_metadata, autotune @@ -331,6 +333,7 @@ def flashinfer_autotune_context(model_runner: ModelRunner, *, run_lm_head: bool) cache=None if reuse_cache else str(autotune_cache), skip_ops=skip_ops, ), + autotune_v2(mode="tune", persistent_cache=False), mega_context, autotune_dummy_run_mode(run_lm_head=run_lm_head), ): diff --git a/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py b/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py index 14ed53a32a..2611c769a1 100644 --- a/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py +++ b/test/registered/unit/model_executor/runner/test_flashinfer_autotune_sync.py @@ -80,6 +80,7 @@ class TestMegaMoEAutotuneStartup(CustomTestCase): ) general = Mock(side_effect=lambda *_args, **_kwargs: nullcontext()) modules = { + "flashinfer": SimpleNamespace(autotune_v2=lambda **_: nullcontext()), "flashinfer.autotuner": SimpleNamespace( AutoTuner=SimpleNamespace( get=lambda: SimpleNamespace(