[Fix] imports: Bind FlashInfer lazily at four import sites so the server starts without it (#44430)

Co-authored-by: fei-xx <fei-xx@users.noreply.github.com>
Co-authored-by: yeqcharlotte <yeqcharlotte@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-15 18:52:33 -07:00
committed by GitHub
co-authored by fei-xx yeqcharlotte
parent 2957ff3165
commit 4e85e8ed37
5 changed files with 62 additions and 10 deletions
+7 -2
View File
@@ -40,6 +40,7 @@ from sglang.srt.utils.multi_stream_utils import (
maybe_execute_in_parallel,
with_multi_stream,
)
from sglang.utils import LazyImport
_is_cpu = is_cpu()
_is_cuda = is_cuda()
@@ -50,10 +51,14 @@ _is_cpu_amx_available = cpu_has_amx_support()
_is_xpu = is_xpu()
if _is_cuda:
from flashinfer.prefill import cudnn_batch_prefill_with_kv_cache
from sglang.kernels.ops.attention.flash_attention import flash_attn_varlen_func
# Resolved on first call: CUDA builds without FlashInfer must still import
# this module.
cudnn_batch_prefill_with_kv_cache = LazyImport(
"flashinfer.prefill", "cudnn_batch_prefill_with_kv_cache"
)
def flash_attn_func(*args, ver: int = 3, **kwargs):
if ver == 4:
from sglang.kernels.ops.attention.flash_attention_v4 import (
@@ -54,6 +54,7 @@ from sglang.srt.utils import (
)
from sglang.srt.utils.common import torch_release
from sglang.srt.utils.custom_op import register_custom_op
from sglang.utils import LazyImport
if TYPE_CHECKING:
from sglang.srt.layers.linear import LinearBase
@@ -252,7 +253,9 @@ if _is_cuda:
N = mat_b.shape[-1]
return mat_a.new_empty((M, N), dtype=out_dtype)
from flashinfer import bmm_fp8 as _raw_bmm_fp8_batched
# Resolved on first call: CUDA builds without FlashInfer must still import
# this module.
_raw_bmm_fp8_batched = LazyImport("flashinfer", "bmm_fp8")
@register_custom_op(op_name="flashinfer_bmm_fp8_batched", mutates_args=["out"])
def _bmm_fp8_batched_op(
+10 -4
View File
@@ -34,17 +34,23 @@ from sglang.srt.utils.common import (
is_npu,
is_xpu,
)
from sglang.utils import LazyImport
if is_cuda():
from flashinfer.sampling import (
min_p_sampling_from_probs,
top_k_top_p_sampling_from_probs,
)
from sgl_kernel import (
top_k_renorm_prob,
top_p_renorm_prob,
)
# Resolved on first call: CUDA builds without FlashInfer must still import
# this module; they default to the pytorch sampling backend.
min_p_sampling_from_probs = LazyImport(
"flashinfer.sampling", "min_p_sampling_from_probs"
)
top_k_top_p_sampling_from_probs = LazyImport(
"flashinfer.sampling", "top_k_top_p_sampling_from_probs"
)
if is_musa() or is_xpu():
from sgl_kernel import (
min_p_sampling_from_probs,
@@ -21,6 +21,7 @@ from sglang.srt.model_executor.runner_utils.pool import borrow_graph_pool
from sglang.srt.runtime_context import get_spec
from sglang.srt.speculative.spec_utils import sample_simulated_acc_len
from sglang.srt.utils import is_cuda, is_hip, is_musa, is_npu
from sglang.utils import LazyImport
DEFAULT_DFLASH_MASK_TOKEN = "<|MASK|>"
@@ -42,13 +43,15 @@ _DFLASH_VERIFY_SKIP_CUSTOM_MASK_BACKENDS = frozenset(
if is_cuda():
from flashinfer.sampling import top_k_renorm_probs as top_k_renorm_prob
from flashinfer.sampling import top_p_renorm_probs as top_p_renorm_prob
from sglang.kernels.ops.speculative.sampling import (
tree_speculative_sampling_target_only,
)
# Resolved on first call: CUDA builds without FlashInfer must still import
# this module.
top_k_renorm_prob = LazyImport("flashinfer.sampling", "top_k_renorm_probs")
top_p_renorm_prob = LazyImport("flashinfer.sampling", "top_p_renorm_probs")
_DFLASH_SAMPLING_VERIFY_AVAILABLE = True
elif is_musa():
try: