diff --git a/python/sglang/multimodal_gen/runtime/utils/torch_compile.py b/python/sglang/multimodal_gen/runtime/utils/torch_compile.py index 094fa2de8c..84d7777a66 100644 --- a/python/sglang/multimodal_gen/runtime/utils/torch_compile.py +++ b/python/sglang/multimodal_gen/runtime/utils/torch_compile.py @@ -31,6 +31,13 @@ def apply_inductor_config(overrides: dict[str, Any]) -> None: logger.warning("Unknown torch._inductor.config option %r, ignoring", name) continue setattr(_inductor_cfg, name, value) + if overrides.get("multi_kernel_hints"): + # torch reads its saved multi-kernel pick through sub-kernels an + # FX-graph cache hit has not restored yet (triton _hash_lock is None), + # so every warm restart fails the compile -- under TP on one rank only, + # which deadlocks the others. Without the saved pick, each process + # re-picks during warmup. + os.environ.setdefault("TORCHINDUCTOR_DISABLE_MULTI_KERNEL_CACHE", "1") def build_torch_compile_kwargs( diff --git a/python/sglang/multimodal_gen/test/unit/test_torch_compile_inductor_config.py b/python/sglang/multimodal_gen/test/unit/test_torch_compile_inductor_config.py new file mode 100644 index 0000000000..8f5db67e10 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_torch_compile_inductor_config.py @@ -0,0 +1,46 @@ +"""Per-model inductor overrides, and what they must switch off alongside.""" + +import os +import unittest +from unittest.mock import patch + +import torch._inductor.config as inductor_config + +from sglang.multimodal_gen.runtime.utils.torch_compile import apply_inductor_config +from sglang.test.test_utils import CustomTestCase + +_NO_PICK_CACHE = "TORCHINDUCTOR_DISABLE_MULTI_KERNEL_CACHE" + + +class TestApplyInductorConfig(CustomTestCase): + def test_multi_kernel_hints_drop_the_saved_pick(self): + with ( + inductor_config.patch(multi_kernel_hints=[]), + patch.dict(os.environ, clear=True), + ): + apply_inductor_config({"multi_kernel_hints": [64, 4096]}) + + self.assertEqual(inductor_config.multi_kernel_hints, [64, 4096]) + self.assertEqual(os.environ.get(_NO_PICK_CACHE), "1") + + def test_other_overrides_keep_the_saved_pick(self): + with ( + inductor_config.patch(max_autotune=False), + patch.dict(os.environ, clear=True), + ): + apply_inductor_config({"max_autotune": True}) + + self.assertNotIn(_NO_PICK_CACHE, os.environ) + + def test_an_explicit_setting_wins(self): + with ( + inductor_config.patch(multi_kernel_hints=[]), + patch.dict(os.environ, {_NO_PICK_CACHE: "0"}, clear=True), + ): + apply_inductor_config({"multi_kernel_hints": [64, 4096]}) + + self.assertEqual(os.environ[_NO_PICK_CACHE], "0") + + +if __name__ == "__main__": + unittest.main()