[Fix] imports: Bind FlashInfer lazily at four import sites so the server starts without it (#44430)
Co-authored-by: fei-xx <fei-xx@users.noreply.github.com> Co-authored-by: yeqcharlotte <yeqcharlotte@users.noreply.github.com>
This commit is contained in:
co-authored by
fei-xx
yeqcharlotte
parent
2957ff3165
commit
4e85e8ed37
@@ -40,6 +40,7 @@ from sglang.srt.utils.multi_stream_utils import (
|
||||
maybe_execute_in_parallel,
|
||||
with_multi_stream,
|
||||
)
|
||||
from sglang.utils import LazyImport
|
||||
|
||||
_is_cpu = is_cpu()
|
||||
_is_cuda = is_cuda()
|
||||
@@ -50,10 +51,14 @@ _is_cpu_amx_available = cpu_has_amx_support()
|
||||
_is_xpu = is_xpu()
|
||||
|
||||
if _is_cuda:
|
||||
from flashinfer.prefill import cudnn_batch_prefill_with_kv_cache
|
||||
|
||||
from sglang.kernels.ops.attention.flash_attention import flash_attn_varlen_func
|
||||
|
||||
# Resolved on first call: CUDA builds without FlashInfer must still import
|
||||
# this module.
|
||||
cudnn_batch_prefill_with_kv_cache = LazyImport(
|
||||
"flashinfer.prefill", "cudnn_batch_prefill_with_kv_cache"
|
||||
)
|
||||
|
||||
def flash_attn_func(*args, ver: int = 3, **kwargs):
|
||||
if ver == 4:
|
||||
from sglang.kernels.ops.attention.flash_attention_v4 import (
|
||||
|
||||
@@ -54,6 +54,7 @@ from sglang.srt.utils import (
|
||||
)
|
||||
from sglang.srt.utils.common import torch_release
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
from sglang.utils import LazyImport
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.linear import LinearBase
|
||||
@@ -252,7 +253,9 @@ if _is_cuda:
|
||||
N = mat_b.shape[-1]
|
||||
return mat_a.new_empty((M, N), dtype=out_dtype)
|
||||
|
||||
from flashinfer import bmm_fp8 as _raw_bmm_fp8_batched
|
||||
# Resolved on first call: CUDA builds without FlashInfer must still import
|
||||
# this module.
|
||||
_raw_bmm_fp8_batched = LazyImport("flashinfer", "bmm_fp8")
|
||||
|
||||
@register_custom_op(op_name="flashinfer_bmm_fp8_batched", mutates_args=["out"])
|
||||
def _bmm_fp8_batched_op(
|
||||
|
||||
@@ -34,17 +34,23 @@ from sglang.srt.utils.common import (
|
||||
is_npu,
|
||||
is_xpu,
|
||||
)
|
||||
from sglang.utils import LazyImport
|
||||
|
||||
if is_cuda():
|
||||
from flashinfer.sampling import (
|
||||
min_p_sampling_from_probs,
|
||||
top_k_top_p_sampling_from_probs,
|
||||
)
|
||||
from sgl_kernel import (
|
||||
top_k_renorm_prob,
|
||||
top_p_renorm_prob,
|
||||
)
|
||||
|
||||
# Resolved on first call: CUDA builds without FlashInfer must still import
|
||||
# this module; they default to the pytorch sampling backend.
|
||||
min_p_sampling_from_probs = LazyImport(
|
||||
"flashinfer.sampling", "min_p_sampling_from_probs"
|
||||
)
|
||||
top_k_top_p_sampling_from_probs = LazyImport(
|
||||
"flashinfer.sampling", "top_k_top_p_sampling_from_probs"
|
||||
)
|
||||
|
||||
if is_musa() or is_xpu():
|
||||
from sgl_kernel import (
|
||||
min_p_sampling_from_probs,
|
||||
|
||||
@@ -21,6 +21,7 @@ from sglang.srt.model_executor.runner_utils.pool import borrow_graph_pool
|
||||
from sglang.srt.runtime_context import get_spec
|
||||
from sglang.srt.speculative.spec_utils import sample_simulated_acc_len
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_musa, is_npu
|
||||
from sglang.utils import LazyImport
|
||||
|
||||
DEFAULT_DFLASH_MASK_TOKEN = "<|MASK|>"
|
||||
|
||||
@@ -42,13 +43,15 @@ _DFLASH_VERIFY_SKIP_CUSTOM_MASK_BACKENDS = frozenset(
|
||||
|
||||
|
||||
if is_cuda():
|
||||
from flashinfer.sampling import top_k_renorm_probs as top_k_renorm_prob
|
||||
from flashinfer.sampling import top_p_renorm_probs as top_p_renorm_prob
|
||||
|
||||
from sglang.kernels.ops.speculative.sampling import (
|
||||
tree_speculative_sampling_target_only,
|
||||
)
|
||||
|
||||
# Resolved on first call: CUDA builds without FlashInfer must still import
|
||||
# this module.
|
||||
top_k_renorm_prob = LazyImport("flashinfer.sampling", "top_k_renorm_probs")
|
||||
top_p_renorm_prob = LazyImport("flashinfer.sampling", "top_p_renorm_probs")
|
||||
|
||||
_DFLASH_SAMPLING_VERIFY_AVAILABLE = True
|
||||
elif is_musa():
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user