diff --git a/python/sglang/srt/layers/attention/vision.py b/python/sglang/srt/layers/attention/vision.py index 29c5bd02b9..76032eab86 100644 --- a/python/sglang/srt/layers/attention/vision.py +++ b/python/sglang/srt/layers/attention/vision.py @@ -40,6 +40,7 @@ from sglang.srt.utils.multi_stream_utils import ( maybe_execute_in_parallel, with_multi_stream, ) +from sglang.utils import LazyImport _is_cpu = is_cpu() _is_cuda = is_cuda() @@ -50,10 +51,14 @@ _is_cpu_amx_available = cpu_has_amx_support() _is_xpu = is_xpu() if _is_cuda: - from flashinfer.prefill import cudnn_batch_prefill_with_kv_cache - from sglang.kernels.ops.attention.flash_attention import flash_attn_varlen_func + # Resolved on first call: CUDA builds without FlashInfer must still import + # this module. + cudnn_batch_prefill_with_kv_cache = LazyImport( + "flashinfer.prefill", "cudnn_batch_prefill_with_kv_cache" + ) + def flash_attn_func(*args, ver: int = 3, **kwargs): if ver == 4: from sglang.kernels.ops.attention.flash_attention_v4 import ( diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 6fa480b3b8..cfcd6e1088 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -54,6 +54,7 @@ from sglang.srt.utils import ( ) from sglang.srt.utils.common import torch_release from sglang.srt.utils.custom_op import register_custom_op +from sglang.utils import LazyImport if TYPE_CHECKING: from sglang.srt.layers.linear import LinearBase @@ -252,7 +253,9 @@ if _is_cuda: N = mat_b.shape[-1] return mat_a.new_empty((M, N), dtype=out_dtype) - from flashinfer import bmm_fp8 as _raw_bmm_fp8_batched + # Resolved on first call: CUDA builds without FlashInfer must still import + # this module. + _raw_bmm_fp8_batched = LazyImport("flashinfer", "bmm_fp8") @register_custom_op(op_name="flashinfer_bmm_fp8_batched", mutates_args=["out"]) def _bmm_fp8_batched_op( diff --git a/python/sglang/srt/layers/sampler.py b/python/sglang/srt/layers/sampler.py index 73c89ce9ef..7d3a34ab1a 100644 --- a/python/sglang/srt/layers/sampler.py +++ b/python/sglang/srt/layers/sampler.py @@ -34,17 +34,23 @@ from sglang.srt.utils.common import ( is_npu, is_xpu, ) +from sglang.utils import LazyImport if is_cuda(): - from flashinfer.sampling import ( - min_p_sampling_from_probs, - top_k_top_p_sampling_from_probs, - ) from sgl_kernel import ( top_k_renorm_prob, top_p_renorm_prob, ) + # Resolved on first call: CUDA builds without FlashInfer must still import + # this module; they default to the pytorch sampling backend. + min_p_sampling_from_probs = LazyImport( + "flashinfer.sampling", "min_p_sampling_from_probs" + ) + top_k_top_p_sampling_from_probs = LazyImport( + "flashinfer.sampling", "top_k_top_p_sampling_from_probs" + ) + if is_musa() or is_xpu(): from sgl_kernel import ( min_p_sampling_from_probs, diff --git a/python/sglang/srt/speculative/dflash_utils.py b/python/sglang/srt/speculative/dflash_utils.py index d0f8efb8e4..4eab4f3df8 100644 --- a/python/sglang/srt/speculative/dflash_utils.py +++ b/python/sglang/srt/speculative/dflash_utils.py @@ -21,6 +21,7 @@ from sglang.srt.model_executor.runner_utils.pool import borrow_graph_pool from sglang.srt.runtime_context import get_spec from sglang.srt.speculative.spec_utils import sample_simulated_acc_len from sglang.srt.utils import is_cuda, is_hip, is_musa, is_npu +from sglang.utils import LazyImport DEFAULT_DFLASH_MASK_TOKEN = "<|MASK|>" @@ -42,13 +43,15 @@ _DFLASH_VERIFY_SKIP_CUSTOM_MASK_BACKENDS = frozenset( if is_cuda(): - from flashinfer.sampling import top_k_renorm_probs as top_k_renorm_prob - from flashinfer.sampling import top_p_renorm_probs as top_p_renorm_prob - from sglang.kernels.ops.speculative.sampling import ( tree_speculative_sampling_target_only, ) + # Resolved on first call: CUDA builds without FlashInfer must still import + # this module. + top_k_renorm_prob = LazyImport("flashinfer.sampling", "top_k_renorm_probs") + top_p_renorm_prob = LazyImport("flashinfer.sampling", "top_p_renorm_probs") + _DFLASH_SAMPLING_VERIFY_AVAILABLE = True elif is_musa(): try: diff --git a/test/registered/unit/test_import_without_flashinfer.py b/test/registered/unit/test_import_without_flashinfer.py new file mode 100644 index 0000000000..169a84393b --- /dev/null +++ b/test/registered/unit/test_import_without_flashinfer.py @@ -0,0 +1,35 @@ +"""Server start must not need FlashInfer. + +Importing the HTTP entrypoint reaches every module that bound FlashInfer at +module level behind ``is_cuda()`` alone (sampler, dflash_utils, fp8_utils, +attention/vision); a FlashInfer-less CUDA image failed there at import time. +""" + +import subprocess +import sys + +import pytest +import torch + +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=25, stage="base-b", runner_config="1-gpu-small") + + +@pytest.mark.skipif( + not torch.cuda.is_available(), reason="is_cuda() gates the imports under test" +) +def test_http_server_imports_without_flashinfer() -> None: + # None in sys.modules makes `import flashinfer` raise and find_spec return None. + code = ( + "import sys; sys.modules['flashinfer'] = None; " + "import sglang.srt.entrypoints.http_server" + ) + proc = subprocess.run( + [sys.executable, "-c", code], capture_output=True, text=True, timeout=600 + ) + assert proc.returncode == 0, proc.stderr[-3000:] + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__]))