[Perf] [NVIDIA] muse-spark-open: Fuse pre-expert RMSNorm with static-scale NVFP4 quantization (#44554)

This commit is contained in:
Wookje Han
2026-10-22 20:00:37 -07:00
committed by GitHub
parent 698651acd0
commit f2873f1613
3 changed files with 109 additions and 32 deletions
@@ -6,11 +6,14 @@
* normed_out: [M, K] bf16, the normed rows; null unless requested.
* fp4_out: [M, K/2] u8, packed e2m1 pairs (low nibble first).
* sf_out: [M, K/16] u8, e4m3 block scales in linear (row-major) layout.
* per_token_scale: [M] fp32.
* per_token_scale: [M] fp32; per-token mode only.
* global_scale: [1] fp32 static encode scale (w13_input_scale_quant); static mode only.
*
* The quantizer reproduces FlashInfer's `nvfp4_quantize(per_token_activation=True,
* backend="cute-dsl", sfLayout=layout_linear)` byte for byte: the same
* rcp.approx / cvt.rn.satfinite sequence applied to the bf16-rounded normed row.
* Static mode reproduces FlashInfer's CUDA `fp4_quantize(global_sf, sf_vec_size=16,
* is_sf_swizzled_layout=False)` the same way.
*/
#include <sgl_kernel/tensor.h>
@@ -38,6 +41,7 @@ struct NormFp4QuantParams {
void* __restrict__ fp4_out;
void* __restrict__ sf_out;
void* __restrict__ per_token_scale;
const float* __restrict__ global_scale;
float eps;
float global_scale_inv;
int num_rows;
@@ -111,9 +115,23 @@ SGL_DEVICE float mul_f32(float a, float b) {
return r;
}
SGL_DEVICE float output_scale_static(uint32_t fp8, float block_max, float global_scale) {
if (block_max == 0.0f) return 0.0f;
fp8_e4m3_t sf;
sf.__x = static_cast<uint8_t>(fp8);
return rcp_approx_ftz(mul_f32(static_cast<float>(sf), rcp_approx_ftz(global_scale)));
}
} // namespace device::norm_fp4_quant
template <int64_t kHidden, int kNumThreads, bool kUsePDL, bool kQuant, bool kNormedOut, bool kHasWeight>
template <
int64_t kHidden,
int kNumThreads,
bool kUsePDL,
bool kQuant,
bool kNormedOut,
bool kHasWeight,
bool kStaticScale>
__global__ void __launch_bounds__(kNumThreads)
muse_spark_norm_fp4_quant_kernel(const __grid_constant__ NormFp4QuantParams params) {
using namespace device;
@@ -123,6 +141,7 @@ __global__ void __launch_bounds__(kNumThreads)
constexpr int kNumWarps = kNumThreads / kWarpThreads;
static_assert(kHidden % (kNumThreads * kBlock) == 0);
static_assert(kQuant || kNormedOut);
static_assert(kQuant || !kStaticScale);
using BF16Vec = AlignedVector<bf16x2_t, 4>; // 8 bf16 = 16B; two per block
@@ -180,7 +199,7 @@ __global__ void __launch_bounds__(kNumThreads)
v.y *= wf.y;
}
normed[p][q] = cast<bf16x2_t>(v);
if constexpr (kQuant) {
if constexpr (kQuant && !kStaticScale) {
const auto r = cast<fp32x2_t>(normed[p][q]);
local_amax = fmaxf(local_amax, fmaxf(fabsf(r.x), fabsf(r.y)));
}
@@ -199,23 +218,27 @@ __global__ void __launch_bounds__(kNumThreads)
}
if constexpr (kQuant) {
float* max_smem = kNumWarps == 1 ? smem : smem + kNumWarps;
float row_amax = warp::reduce_max(local_amax);
if constexpr (kNumWarps > 1) {
const int lane = threadIdx.x % kWarpThreads;
if (lane == 0) max_smem[threadIdx.x / kWarpThreads] = row_amax;
__syncthreads();
row_amax = warp::reduce_max(lane < kNumWarps ? max_smem[lane] : 0.0f);
}
float per_token_scale = 0.0f;
float encode_scale = kFloat32Max;
if (row_amax != 0.0f) {
per_token_scale = mul_f32(row_amax, params.global_scale_inv);
encode_scale = rcp_approx_ftz(per_token_scale);
}
if (threadIdx.x == 0) {
static_cast<float*>(params.per_token_scale)[row] = per_token_scale;
if constexpr (kStaticScale) {
encode_scale = params.global_scale[0];
} else {
float* max_smem = kNumWarps == 1 ? smem : smem + kNumWarps;
float row_amax = warp::reduce_max(local_amax);
if constexpr (kNumWarps > 1) {
const int lane = threadIdx.x % kWarpThreads;
if (lane == 0) max_smem[threadIdx.x / kWarpThreads] = row_amax;
__syncthreads();
row_amax = warp::reduce_max(lane < kNumWarps ? max_smem[lane] : 0.0f);
}
float per_token_scale = 0.0f;
if (row_amax != 0.0f) {
per_token_scale = mul_f32(row_amax, params.global_scale_inv);
encode_scale = rcp_approx_ftz(per_token_scale);
}
if (threadIdx.x == 0) {
static_cast<float*>(params.per_token_scale)[row] = per_token_scale;
}
}
const float fp4_max_rcp = rcp_approx_ftz(6.0f);
@@ -235,7 +258,8 @@ __global__ void __launch_bounds__(kNumThreads)
}
const float scale_f = mul_f32(encode_scale, mul_f32(block_max, fp4_max_rcp));
const uint32_t fp8 = cvt_f32_to_e4m3(scale_f) & 0xFFu;
const float out_scale = output_scale(fp8, encode_scale);
const float out_scale =
kStaticScale ? output_scale_static(fp8, block_max, encode_scale) : output_scale(fp8, encode_scale);
#pragma unroll
for (int i = 0; i < kBlock; ++i)
f[i] = mul_f32(f[i], out_scale);
@@ -248,7 +272,7 @@ __global__ void __launch_bounds__(kNumThreads)
PDLTriggerSecondary<kUsePDL>();
}
template <int64_t kHidden, int kNumThreads, bool kUsePDL, bool kQuant, bool kNormedOut>
template <int64_t kHidden, int kNumThreads, bool kUsePDL, bool kQuant, bool kNormedOut, bool kStaticScale>
struct MuseSparkNormFp4Quant {
static void
run(const tvm::ffi::TensorView x,
@@ -257,6 +281,7 @@ struct MuseSparkNormFp4Quant {
const tvm::ffi::Optional<tvm::ffi::TensorView> fp4_out,
const tvm::ffi::Optional<tvm::ffi::TensorView> sf_out,
const tvm::ffi::Optional<tvm::ffi::TensorView> per_token_scale,
const tvm::ffi::Optional<tvm::ffi::TensorView> global_scale,
double eps,
double global_scale_inv) {
using namespace host;
@@ -283,8 +308,15 @@ struct MuseSparkNormFp4Quant {
if constexpr (kQuant) {
TensorMatcher({M, K2}).with_dtype<uint8_t>().with_device(device_).verify(fp4_out.value());
TensorMatcher({M, K16}).with_dtype<uint8_t>().with_device(device_).verify(sf_out.value());
}
RuntimeCheck(per_token_scale.has_value() == (kQuant && !kStaticScale), "per_token_scale is per-token mode only");
if (per_token_scale.has_value()) {
TensorMatcher({M}).with_dtype<fp32_t>().with_device(device_).verify(per_token_scale.value());
}
RuntimeCheck(global_scale.has_value() == kStaticScale, "global_scale is static mode only");
if constexpr (kStaticScale) {
TensorMatcher({1}).with_dtype<fp32_t>().with_device(device_).verify(global_scale.value());
}
const auto num_rows = static_cast<int>(M.unwrap());
if (num_rows == 0) return;
@@ -295,7 +327,8 @@ struct MuseSparkNormFp4Quant {
.normed_out = kNormedOut ? normed_out.value().data_ptr() : nullptr,
.fp4_out = kQuant ? fp4_out.value().data_ptr() : nullptr,
.sf_out = kQuant ? sf_out.value().data_ptr() : nullptr,
.per_token_scale = kQuant ? per_token_scale.value().data_ptr() : nullptr,
.per_token_scale = per_token_scale.has_value() ? per_token_scale.value().data_ptr() : nullptr,
.global_scale = kStaticScale ? static_cast<const float*>(global_scale.value().data_ptr()) : nullptr,
.eps = static_cast<float>(eps),
.global_scale_inv = static_cast<float>(global_scale_inv),
.num_rows = num_rows,
@@ -306,11 +339,13 @@ struct MuseSparkNormFp4Quant {
if (weight.has_value()) {
LaunchKernel(grid, kNumThreads, device)
.enable_pdl(kUsePDL)(
muse_spark_norm_fp4_quant_kernel<kHidden, kNumThreads, kUsePDL, kQuant, kNormedOut, true>, params);
muse_spark_norm_fp4_quant_kernel<kHidden, kNumThreads, kUsePDL, kQuant, kNormedOut, true, kStaticScale>,
params);
} else {
LaunchKernel(grid, kNumThreads, device)
.enable_pdl(kUsePDL)(
muse_spark_norm_fp4_quant_kernel<kHidden, kNumThreads, kUsePDL, kQuant, kNormedOut, false>, params);
muse_spark_norm_fp4_quant_kernel<kHidden, kNumThreads, kUsePDL, kQuant, kNormedOut, false, kStaticScale>,
params);
}
}
};
@@ -1,7 +1,7 @@
"""Fused RMS norm + per-token NVFP4 quantization for MuseSpark v1.2.
"""Fused RMS norm + (per-token) NVFP4 quantization for MuseSpark v1.2.
Byte-exact with FlashInfer's per-token ``layout_linear`` CuTe-DSL quantizer on
the same bf16 row.
the same bf16 row. The static-scale mode matches FlashInfer's CUDA ``fp4_quantize``.
"""
from __future__ import annotations
@@ -48,9 +48,16 @@ def _jit_module(hidden_size: int, num_threads: int) -> Module:
*args,
cuda_files=["muse_spark/norm_fp4_quant.cuh"],
cuda_wrappers=[
("run_quant", f"MuseSparkNormFp4Quant<{args}, true, false>::run"),
("run_quant_normed", f"MuseSparkNormFp4Quant<{args}, true, true>::run"),
("run_normed", f"MuseSparkNormFp4Quant<{args}, false, true>::run"),
("run_quant", f"MuseSparkNormFp4Quant<{args}, true, false, false>::run"),
(
"run_quant_normed",
f"MuseSparkNormFp4Quant<{args}, true, true, false>::run",
),
("run_normed", f"MuseSparkNormFp4Quant<{args}, false, true, false>::run"),
(
"run_quant_static",
f"MuseSparkNormFp4Quant<{args}, true, false, true>::run",
),
],
)
@@ -99,7 +106,7 @@ def _fused_norm_fp4_quant_jit(
fn = module.run_quant
else:
fn = module.run_normed
fn(x, weight, normed, fp4, sf, pts, float(eps), float(global_scale_inv))
fn(x, weight, normed, fp4, sf, pts, None, float(eps), float(global_scale_inv))
return fp4, sf, pts, normed
@@ -304,3 +311,31 @@ def fused_norm_fp4_quant(
quantize=quantize,
want_normed=want_normed,
)
def fused_norm_fp4_quant_static(
x: torch.Tensor,
weight: Optional[torch.Tensor],
eps: float,
global_scale: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Return ``(fp4, sf)``: RMS norm, then NVFP4 with the static fp32 ``global_scale``.
``fp4`` is ``[M, K/2]`` uint8 and ``sf`` is ``[M, K/16]`` uint8 (e4m3 bits,
row-major).
"""
assert x.dtype == torch.bfloat16 and x.dim() == 2 and x.is_contiguous()
assert global_scale.dtype == torch.float32 and global_scale.numel() == 1
m, hidden_size = x.shape
if weight is not None:
assert weight.dtype == torch.bfloat16 and weight.shape == (hidden_size,)
weight = weight.contiguous()
fp4 = torch.empty(m, hidden_size // 2, dtype=torch.uint8, device=x.device)
sf = torch.empty(m, hidden_size // _BLOCK, dtype=torch.uint8, device=x.device)
if m == 0:
return fp4, sf
module = _jit_module(hidden_size, _num_threads(hidden_size))
module.run_quant_static(
x, weight, None, fp4, sf, None, global_scale.reshape(1), float(eps), 0.0
)
return fp4, sf
@@ -24,6 +24,7 @@ from sglang.kernels.ops.communication.muse_spark_moe_finalize_all_reduce import
)
from sglang.kernels.ops.layernorm.muse_spark_rmsnorm_nvfp4_quant import (
fused_norm_fp4_quant,
fused_norm_fp4_quant_static,
)
from sglang.kernels.ops.moe.muse_spark_post_expert_finalize_triton import (
fused_post_expert_norm_weighted_sum,
@@ -584,9 +585,15 @@ class MuseSpark1xFp4Experts(nn.Module):
hs_scale_linear = hs_sf_bytes.view(torch.float8_e4m3fn).reshape(
seq_len, hidden_size // 16
)
elif pre_expert_norm is not None:
per_token_scale = None
hs_fp4, hs_scale_linear = fused_norm_fp4_quant_static(
hidden_states,
self._pre_norm_weight(pre_expert_norm),
pre_expert_norm.eps,
self.w13_input_scale_quant,
)
else:
if pre_expert_norm is not None:
hidden_states = pre_expert_norm(hidden_states)
# Match the known-good public vLLM activation path. SGLang's
# helper selects FlashInfer's experimental CuTeDSL quantizer on
# SM100, while vLLM's scaled_fp4_quant uses the stable CUDA