From f2873f16136a44b10275301a8b70804fe26b9652 Mon Sep 17 00:00:00 2001 From: Wookje Han <52435620+wookjeHan@users.noreply.github.com> Date: Thu, 22 Oct 2026 20:00:37 -0700 Subject: [PATCH] [Perf] [NVIDIA] muse-spark-open: Fuse pre-expert RMSNorm with static-scale NVFP4 quantization (#44554) --- .../jit/csrc/muse_spark/norm_fp4_quant.cuh | 83 +++++++++++++------ .../muse_spark_rmsnorm_nvfp4_quant.py | 47 +++++++++-- .../layers/muse_spark/muse_spark_fp4_moe.py | 11 ++- 3 files changed, 109 insertions(+), 32 deletions(-) diff --git a/python/sglang/kernels/jit/csrc/muse_spark/norm_fp4_quant.cuh b/python/sglang/kernels/jit/csrc/muse_spark/norm_fp4_quant.cuh index 6e4394631d..eabe7084ff 100644 --- a/python/sglang/kernels/jit/csrc/muse_spark/norm_fp4_quant.cuh +++ b/python/sglang/kernels/jit/csrc/muse_spark/norm_fp4_quant.cuh @@ -6,11 +6,14 @@ * normed_out: [M, K] bf16, the normed rows; null unless requested. * fp4_out: [M, K/2] u8, packed e2m1 pairs (low nibble first). * sf_out: [M, K/16] u8, e4m3 block scales in linear (row-major) layout. - * per_token_scale: [M] fp32. + * per_token_scale: [M] fp32; per-token mode only. + * global_scale: [1] fp32 static encode scale (w13_input_scale_quant); static mode only. * * The quantizer reproduces FlashInfer's `nvfp4_quantize(per_token_activation=True, * backend="cute-dsl", sfLayout=layout_linear)` byte for byte: the same * rcp.approx / cvt.rn.satfinite sequence applied to the bf16-rounded normed row. + * Static mode reproduces FlashInfer's CUDA `fp4_quantize(global_sf, sf_vec_size=16, + * is_sf_swizzled_layout=False)` the same way. */ #include @@ -38,6 +41,7 @@ struct NormFp4QuantParams { void* __restrict__ fp4_out; void* __restrict__ sf_out; void* __restrict__ per_token_scale; + const float* __restrict__ global_scale; float eps; float global_scale_inv; int num_rows; @@ -111,9 +115,23 @@ SGL_DEVICE float mul_f32(float a, float b) { return r; } +SGL_DEVICE float output_scale_static(uint32_t fp8, float block_max, float global_scale) { + if (block_max == 0.0f) return 0.0f; + fp8_e4m3_t sf; + sf.__x = static_cast(fp8); + return rcp_approx_ftz(mul_f32(static_cast(sf), rcp_approx_ftz(global_scale))); +} + } // namespace device::norm_fp4_quant -template +template < + int64_t kHidden, + int kNumThreads, + bool kUsePDL, + bool kQuant, + bool kNormedOut, + bool kHasWeight, + bool kStaticScale> __global__ void __launch_bounds__(kNumThreads) muse_spark_norm_fp4_quant_kernel(const __grid_constant__ NormFp4QuantParams params) { using namespace device; @@ -123,6 +141,7 @@ __global__ void __launch_bounds__(kNumThreads) constexpr int kNumWarps = kNumThreads / kWarpThreads; static_assert(kHidden % (kNumThreads * kBlock) == 0); static_assert(kQuant || kNormedOut); + static_assert(kQuant || !kStaticScale); using BF16Vec = AlignedVector; // 8 bf16 = 16B; two per block @@ -180,7 +199,7 @@ __global__ void __launch_bounds__(kNumThreads) v.y *= wf.y; } normed[p][q] = cast(v); - if constexpr (kQuant) { + if constexpr (kQuant && !kStaticScale) { const auto r = cast(normed[p][q]); local_amax = fmaxf(local_amax, fmaxf(fabsf(r.x), fabsf(r.y))); } @@ -199,23 +218,27 @@ __global__ void __launch_bounds__(kNumThreads) } if constexpr (kQuant) { - float* max_smem = kNumWarps == 1 ? smem : smem + kNumWarps; - float row_amax = warp::reduce_max(local_amax); - if constexpr (kNumWarps > 1) { - const int lane = threadIdx.x % kWarpThreads; - if (lane == 0) max_smem[threadIdx.x / kWarpThreads] = row_amax; - __syncthreads(); - row_amax = warp::reduce_max(lane < kNumWarps ? max_smem[lane] : 0.0f); - } - - float per_token_scale = 0.0f; float encode_scale = kFloat32Max; - if (row_amax != 0.0f) { - per_token_scale = mul_f32(row_amax, params.global_scale_inv); - encode_scale = rcp_approx_ftz(per_token_scale); - } - if (threadIdx.x == 0) { - static_cast(params.per_token_scale)[row] = per_token_scale; + if constexpr (kStaticScale) { + encode_scale = params.global_scale[0]; + } else { + float* max_smem = kNumWarps == 1 ? smem : smem + kNumWarps; + float row_amax = warp::reduce_max(local_amax); + if constexpr (kNumWarps > 1) { + const int lane = threadIdx.x % kWarpThreads; + if (lane == 0) max_smem[threadIdx.x / kWarpThreads] = row_amax; + __syncthreads(); + row_amax = warp::reduce_max(lane < kNumWarps ? max_smem[lane] : 0.0f); + } + + float per_token_scale = 0.0f; + if (row_amax != 0.0f) { + per_token_scale = mul_f32(row_amax, params.global_scale_inv); + encode_scale = rcp_approx_ftz(per_token_scale); + } + if (threadIdx.x == 0) { + static_cast(params.per_token_scale)[row] = per_token_scale; + } } const float fp4_max_rcp = rcp_approx_ftz(6.0f); @@ -235,7 +258,8 @@ __global__ void __launch_bounds__(kNumThreads) } const float scale_f = mul_f32(encode_scale, mul_f32(block_max, fp4_max_rcp)); const uint32_t fp8 = cvt_f32_to_e4m3(scale_f) & 0xFFu; - const float out_scale = output_scale(fp8, encode_scale); + const float out_scale = + kStaticScale ? output_scale_static(fp8, block_max, encode_scale) : output_scale(fp8, encode_scale); #pragma unroll for (int i = 0; i < kBlock; ++i) f[i] = mul_f32(f[i], out_scale); @@ -248,7 +272,7 @@ __global__ void __launch_bounds__(kNumThreads) PDLTriggerSecondary(); } -template +template struct MuseSparkNormFp4Quant { static void run(const tvm::ffi::TensorView x, @@ -257,6 +281,7 @@ struct MuseSparkNormFp4Quant { const tvm::ffi::Optional fp4_out, const tvm::ffi::Optional sf_out, const tvm::ffi::Optional per_token_scale, + const tvm::ffi::Optional global_scale, double eps, double global_scale_inv) { using namespace host; @@ -283,8 +308,15 @@ struct MuseSparkNormFp4Quant { if constexpr (kQuant) { TensorMatcher({M, K2}).with_dtype().with_device(device_).verify(fp4_out.value()); TensorMatcher({M, K16}).with_dtype().with_device(device_).verify(sf_out.value()); + } + RuntimeCheck(per_token_scale.has_value() == (kQuant && !kStaticScale), "per_token_scale is per-token mode only"); + if (per_token_scale.has_value()) { TensorMatcher({M}).with_dtype().with_device(device_).verify(per_token_scale.value()); } + RuntimeCheck(global_scale.has_value() == kStaticScale, "global_scale is static mode only"); + if constexpr (kStaticScale) { + TensorMatcher({1}).with_dtype().with_device(device_).verify(global_scale.value()); + } const auto num_rows = static_cast(M.unwrap()); if (num_rows == 0) return; @@ -295,7 +327,8 @@ struct MuseSparkNormFp4Quant { .normed_out = kNormedOut ? normed_out.value().data_ptr() : nullptr, .fp4_out = kQuant ? fp4_out.value().data_ptr() : nullptr, .sf_out = kQuant ? sf_out.value().data_ptr() : nullptr, - .per_token_scale = kQuant ? per_token_scale.value().data_ptr() : nullptr, + .per_token_scale = per_token_scale.has_value() ? per_token_scale.value().data_ptr() : nullptr, + .global_scale = kStaticScale ? static_cast(global_scale.value().data_ptr()) : nullptr, .eps = static_cast(eps), .global_scale_inv = static_cast(global_scale_inv), .num_rows = num_rows, @@ -306,11 +339,13 @@ struct MuseSparkNormFp4Quant { if (weight.has_value()) { LaunchKernel(grid, kNumThreads, device) .enable_pdl(kUsePDL)( - muse_spark_norm_fp4_quant_kernel, params); + muse_spark_norm_fp4_quant_kernel, + params); } else { LaunchKernel(grid, kNumThreads, device) .enable_pdl(kUsePDL)( - muse_spark_norm_fp4_quant_kernel, params); + muse_spark_norm_fp4_quant_kernel, + params); } } }; diff --git a/python/sglang/kernels/ops/layernorm/muse_spark_rmsnorm_nvfp4_quant.py b/python/sglang/kernels/ops/layernorm/muse_spark_rmsnorm_nvfp4_quant.py index 8f7e5e4a46..459138208f 100644 --- a/python/sglang/kernels/ops/layernorm/muse_spark_rmsnorm_nvfp4_quant.py +++ b/python/sglang/kernels/ops/layernorm/muse_spark_rmsnorm_nvfp4_quant.py @@ -1,7 +1,7 @@ -"""Fused RMS norm + per-token NVFP4 quantization for MuseSpark v1.2. +"""Fused RMS norm + (per-token) NVFP4 quantization for MuseSpark v1.2. Byte-exact with FlashInfer's per-token ``layout_linear`` CuTe-DSL quantizer on -the same bf16 row. +the same bf16 row. The static-scale mode matches FlashInfer's CUDA ``fp4_quantize``. """ from __future__ import annotations @@ -48,9 +48,16 @@ def _jit_module(hidden_size: int, num_threads: int) -> Module: *args, cuda_files=["muse_spark/norm_fp4_quant.cuh"], cuda_wrappers=[ - ("run_quant", f"MuseSparkNormFp4Quant<{args}, true, false>::run"), - ("run_quant_normed", f"MuseSparkNormFp4Quant<{args}, true, true>::run"), - ("run_normed", f"MuseSparkNormFp4Quant<{args}, false, true>::run"), + ("run_quant", f"MuseSparkNormFp4Quant<{args}, true, false, false>::run"), + ( + "run_quant_normed", + f"MuseSparkNormFp4Quant<{args}, true, true, false>::run", + ), + ("run_normed", f"MuseSparkNormFp4Quant<{args}, false, true, false>::run"), + ( + "run_quant_static", + f"MuseSparkNormFp4Quant<{args}, true, false, true>::run", + ), ], ) @@ -99,7 +106,7 @@ def _fused_norm_fp4_quant_jit( fn = module.run_quant else: fn = module.run_normed - fn(x, weight, normed, fp4, sf, pts, float(eps), float(global_scale_inv)) + fn(x, weight, normed, fp4, sf, pts, None, float(eps), float(global_scale_inv)) return fp4, sf, pts, normed @@ -304,3 +311,31 @@ def fused_norm_fp4_quant( quantize=quantize, want_normed=want_normed, ) + + +def fused_norm_fp4_quant_static( + x: torch.Tensor, + weight: Optional[torch.Tensor], + eps: float, + global_scale: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Return ``(fp4, sf)``: RMS norm, then NVFP4 with the static fp32 ``global_scale``. + + ``fp4`` is ``[M, K/2]`` uint8 and ``sf`` is ``[M, K/16]`` uint8 (e4m3 bits, + row-major). + """ + assert x.dtype == torch.bfloat16 and x.dim() == 2 and x.is_contiguous() + assert global_scale.dtype == torch.float32 and global_scale.numel() == 1 + m, hidden_size = x.shape + if weight is not None: + assert weight.dtype == torch.bfloat16 and weight.shape == (hidden_size,) + weight = weight.contiguous() + fp4 = torch.empty(m, hidden_size // 2, dtype=torch.uint8, device=x.device) + sf = torch.empty(m, hidden_size // _BLOCK, dtype=torch.uint8, device=x.device) + if m == 0: + return fp4, sf + module = _jit_module(hidden_size, _num_threads(hidden_size)) + module.run_quant_static( + x, weight, None, fp4, sf, None, global_scale.reshape(1), float(eps), 0.0 + ) + return fp4, sf diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py b/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py index 61313d2ac9..e697d07c9d 100644 --- a/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py +++ b/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py @@ -24,6 +24,7 @@ from sglang.kernels.ops.communication.muse_spark_moe_finalize_all_reduce import ) from sglang.kernels.ops.layernorm.muse_spark_rmsnorm_nvfp4_quant import ( fused_norm_fp4_quant, + fused_norm_fp4_quant_static, ) from sglang.kernels.ops.moe.muse_spark_post_expert_finalize_triton import ( fused_post_expert_norm_weighted_sum, @@ -584,9 +585,15 @@ class MuseSpark1xFp4Experts(nn.Module): hs_scale_linear = hs_sf_bytes.view(torch.float8_e4m3fn).reshape( seq_len, hidden_size // 16 ) + elif pre_expert_norm is not None: + per_token_scale = None + hs_fp4, hs_scale_linear = fused_norm_fp4_quant_static( + hidden_states, + self._pre_norm_weight(pre_expert_norm), + pre_expert_norm.eps, + self.w13_input_scale_quant, + ) else: - if pre_expert_norm is not None: - hidden_states = pre_expert_norm(hidden_states) # Match the known-good public vLLM activation path. SGLang's # helper selects FlashInfer's experimental CuTeDSL quantizer on # SM100, while vLLM's scaled_fp4_quant uses the stable CUDA