minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[bugfix][quark] Fixed an issue where per_token could not be properly recognized when the token count was 1. (#14415)

Browse Source
This commit is contained in:
haoyangli-amd
2025-12-15 22:54:31 -08:00
committed by GitHub
parent 36fcf71fff
commit 9327482baa
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/layers/quantization/fp8_utils.py
+3 -1
View File
@@ -992,7 +992,9 @@ def apply_fp8_linear(
# torch.scaled_mm supports per tensor weights + activations only # torch.scaled_mm supports per tensor weights + activations only
# so fallback to naive if per channel or per token # so fallback to naive if per channel or per token
per_tensor_weights = weight_scale.numel() == 1 per_tensor_weights = weight_scale.numel() == 1
per_tensor_activations = x_scale.numel() == 1 # When the number of token is 1,
# per-token scale has shape (1, 1), per-tensor scale has shape (1) or ().
per_tensor_activations = (x_scale.numel() == 1) and x_scale.dim() < 2
if ( if (
use_per_token_if_dynamic use_per_token_if_dynamic