minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[FlashInfer v0.6.13] Use CuTe DSL backend for FlashInfer per-token NVFP4 quantization (#28220)

Browse Source
This commit is contained in:
Ziang Li
2026-07-13 22:37:46 +08:00
committed by GitHub
parent f391c71758
commit 2cf2920d07
3 changed files with 3 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py
+1
View File
@@ -964,6 +964,7 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
1.0 / (e4m3_max * 6.0), 1.0 / (e4m3_max * 6.0),
sfLayout=SfLayout.layout_linear, sfLayout=SfLayout.layout_linear,
per_token_activation=True, per_token_activation=True,
backend="cute-dsl",
) )
seq_len, hidden_size = hidden_states.shape seq_len, hidden_size = hidden_states.shape
python/sglang/srt/layers/quantization/nvfp4_online.py
+1 -1
View File
@@ -225,7 +225,7 @@ class ModelOptNvFp4OnlineFusedMoEMethod(ModelOptNvFp4FusedMoEMethod):
weight.contiguous(), weight.contiguous(),
1.0 / weight_scale_2, 1.0 / weight_scale_2,
sfLayout=SfLayout.layout_linear, sfLayout=SfLayout.layout_linear,
backend="cuda", backend="cute-dsl",
) )
rows, cols = weight.shape rows, cols = weight.shape
weight_sf = weight_sf.view(torch.float8_e4m3fn).reshape(rows, cols // 16) weight_sf = weight_sf.view(torch.float8_e4m3fn).reshape(rows, cols // 16)
python/sglang/srt/model_loader/loader.py
+1 -4
View File
@@ -817,10 +817,7 @@ class DefaultModelLoader(BaseModelLoader):
if is_nvfp4_online: if is_nvfp4_online:
# Scope exact FP4 quantization math to load-time conversion only; # Scope exact FP4 quantization math to load-time conversion only;
# restore the original environment before serving starts. # restore the original environment before serving starts.
with temp_set_env( with temp_set_env(FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1"):
TRTLLM_DISABLE_FP4_QUANT_FAST_MATH="1",
FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH="1",
):
model.load_weights(weights) model.load_weights(weights)
if target_device.type == "cuda": if target_device.type == "cuda":
torch.cuda.synchronize() torch.cuda.synchronize()