Use Cute-DSL MXFP8 quantize kernels (#25486)
Co-authored-by: b8zhong <b8zhong@users.noreply.github.com>
This commit is contained in:
@@ -21,7 +21,9 @@ from sglang.srt.utils import (
|
||||
log_info_on_rank0,
|
||||
set_weight_attrs,
|
||||
)
|
||||
from sglang.srt.utils.common import next_power_of_2
|
||||
from sglang.srt.utils.common import is_sm100_supported, next_power_of_2
|
||||
|
||||
_MXFP8_QUANTIZE_BACKEND = "cute-dsl" if is_sm100_supported() else "cuda"
|
||||
|
||||
if is_flashinfer_available():
|
||||
from flashinfer import mxfp8_quantize, shuffle_matrix_a, shuffle_matrix_sf_a
|
||||
@@ -379,7 +381,10 @@ class Mxfp4FlashinferTrtllmMoEMethod:
|
||||
)
|
||||
elif precision == "default":
|
||||
x_quant, x_scale = mxfp8_quantize(
|
||||
hidden_states, False, alignment=hidden_size
|
||||
hidden_states,
|
||||
False,
|
||||
alignment=hidden_size,
|
||||
backend=_MXFP8_QUANTIZE_BACKEND,
|
||||
)
|
||||
x_scale = x_scale.view(torch.float8_e4m3fn).reshape(
|
||||
*hidden_states.shape[:-1], -1
|
||||
|
||||
Reference in New Issue
Block a user