minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

Use sgl fp4 quant kernel by default (#12482)

Browse Source
This commit is contained in:
Qiaolin Yu
2025-10-31 19:51:28 -07:00
committed by GitHub
parent 795e98f8a6
commit 59cce5941a
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/layers/quantization/modelopt_quant.py
+5 -1
View File
@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional
import torch import torch
from torch.nn.parameter import Parameter from torch.nn.parameter import Parameter
from python.sglang.srt.utils.common import is_sm120_supported
from sglang.srt.distributed import get_tp_group from sglang.srt.distributed import get_tp_group
from sglang.srt.layers.dp_attention import get_dp_global_num_tokens, get_local_dp_buffer from sglang.srt.layers.dp_attention import get_dp_global_num_tokens, get_local_dp_buffer
from sglang.srt.layers.moe import ( from sglang.srt.layers.moe import (
@@ -51,7 +52,10 @@ if TYPE_CHECKING:
from sglang.srt.single_batch_overlap import DownGemmOverlapArgs from sglang.srt.single_batch_overlap import DownGemmOverlapArgs
try: try:
from flashinfer import fp4_quantize if is_sm120_supported():
from flashinfer import fp4_quantize
else:
from sgl_kernel import scaled_fp4_quant as fp4_quantize
except ImportError: except ImportError:
fp4_quantize = None fp4_quantize = None