From 7c9a78efbecf7288e43baf0de9e6a4a6dd5a628f Mon Sep 17 00:00:00 2001 From: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> Date: Sun, 18 Oct 2026 22:27:22 -0700 Subject: [PATCH] [Feat] nemotron: Port Nemotron 3.5 Super serving changes (#44518) Co-authored-by: BBuf <1182563586@qq.com> Co-authored-by: elvischenv <219235043+elvischenv@users.noreply.github.com> Co-authored-by: Jimmy Shong Co-authored-by: Claude Opus 5.5 --- .../jit/csrc/gemm/hopper_bf16_gemv.cuh | 21 +- .../kernels/jit/csrc/gemm/sm120_fp8_gemv.cuh | 23 +- .../jit/csrc/gemm/sm120_nvfp4_moe_mtp_mma.cuh | 410 ++++++++++++++++++ .../gemm/sm120_nvfp4_weight_transpose.cuh | 49 +++ .../kernels/ops/gemm/hopper_bf16_gemv.py | 36 +- .../sglang/kernels/ops/gemm/sm120_fp8_gemv.py | 18 +- .../kernels/ops/gemm/sm120_nvfp4_moe.py | 167 +++++++ .../sglang/kernels/ops/moe/moe_fused_gate.py | 119 ++++- .../ops/speculative/pack_verify_mask.py | 41 ++ python/sglang/srt/environ.py | 7 + .../layers/attention/flashinfer_backend.py | 133 +++++- .../moe/moe_runner/flashinfer_cutlass.py | 52 ++- .../srt/layers/quantization/modelopt_quant.py | 3 + .../sglang/srt/layers/quantization/unquant.py | 6 +- .../sglang/srt/model_executor/model_runner.py | 3 + .../cuda_graph_setup.py | 40 +- .../weight_exporter.py | 11 + .../runner/prefill_cuda_graph_runner.py | 4 + python/sglang/srt/models/nano_nemotron_vl.py | 3 + python/sglang/srt/models/nemotron_h.py | 171 +++++++- .../attention/test_flashinfer_host_plan.py | 392 +++++++++++++++++ .../ops/moe/test_sm120_topk_gate_small_m.py | 122 ++++++ .../moe/test_moe_runner_config_placement.py | 52 +++ 23 files changed, 1825 insertions(+), 58 deletions(-) create mode 100644 python/sglang/kernels/jit/csrc/gemm/sm120_nvfp4_moe_mtp_mma.cuh create mode 100644 python/sglang/kernels/jit/csrc/gemm/sm120_nvfp4_weight_transpose.cuh create mode 100644 python/sglang/kernels/ops/gemm/sm120_nvfp4_moe.py create mode 100644 python/sglang/kernels/ops/speculative/pack_verify_mask.py create mode 100644 test/registered/kernels/ops/attention/test_flashinfer_host_plan.py create mode 100644 test/registered/kernels/ops/moe/test_sm120_topk_gate_small_m.py diff --git a/python/sglang/kernels/jit/csrc/gemm/hopper_bf16_gemv.cuh b/python/sglang/kernels/jit/csrc/gemm/hopper_bf16_gemv.cuh index 5efde8ceba..f123a03109 100644 --- a/python/sglang/kernels/jit/csrc/gemm/hopper_bf16_gemv.cuh +++ b/python/sglang/kernels/jit/csrc/gemm/hopper_bf16_gemv.cuh @@ -37,12 +37,14 @@ __device__ __forceinline__ float dot8_f32(const float4 wv, const float4 xv) { return acc; } -template +template __global__ void __launch_bounds__(kNumWarps * 32) hopper_bf16_gemv_kernel(bf16_t* __restrict__ out, const bf16_t* __restrict__ x, const bf16_t* __restrict__ w) { __shared__ bf16_t sx[K]; + PDLTriggerSecondary(); const uint32_t tid = threadIdx.x; + PDLWaitPrimary(); for (uint32_t i = tid * kGemvVecSize; i < K; i += kNumWarps * 32 * kGemvVecSize) { *reinterpret_cast(sx + i) = *reinterpret_cast(x + i); } @@ -118,9 +120,11 @@ __global__ void __launch_bounds__(kNumWarps * 32) } } -template +template struct HopperBf16GemvKernel { - static_assert(K % (32 * kGemvVecSize * kUnroll) == 0, "K must cover full unrolled warp strides"); + static_assert( + kUnroll == 1 ? (K % kGemvVecSize == 0) : (K % (32 * kGemvVecSize * kUnroll) == 0), + "K must be 16B-aligned (kUnroll==1) or cover full unrolled warp strides"); static_assert(K * sizeof(bf16_t) <= 48 * 1024, "activation row must fit static shared memory"); static void run(const tvm::ffi::TensorView x, const tvm::ffi::TensorView w, const tvm::ffi::TensorView out) { @@ -134,11 +138,12 @@ struct HopperBf16GemvKernel { constexpr uint32_t kRowsPerBlock = kRows * kNumWarps; constexpr uint32_t kNumBlocks = (N + kRowsPerBlock - 1) / kRowsPerBlock; - LaunchKernel(kNumBlocks, kNumWarps * 32, device.unwrap())( - hopper_bf16_gemv_kernel, - static_cast(out.data_ptr()), - static_cast(x.data_ptr()), - static_cast(w.data_ptr())); + LaunchKernel(kNumBlocks, kNumWarps * 32, device.unwrap()) + .enable_pdl(kUsePDL)( + hopper_bf16_gemv_kernel, + static_cast(out.data_ptr()), + static_cast(x.data_ptr()), + static_cast(w.data_ptr())); } }; diff --git a/python/sglang/kernels/jit/csrc/gemm/sm120_fp8_gemv.cuh b/python/sglang/kernels/jit/csrc/gemm/sm120_fp8_gemv.cuh index 4947d62263..158138ebc2 100644 --- a/python/sglang/kernels/jit/csrc/gemm/sm120_fp8_gemv.cuh +++ b/python/sglang/kernels/jit/csrc/gemm/sm120_fp8_gemv.cuh @@ -39,7 +39,7 @@ __device__ __forceinline__ float dot16_fp8_f32(const float4 wv, const float4 xv) return acc; } -template +template __global__ void __launch_bounds__(kNumWarps * 32) sm120_fp8_gemv_kernel( bf16_t* __restrict__ out, const uint8_t* __restrict__ x, @@ -47,7 +47,9 @@ __global__ void __launch_bounds__(kNumWarps * 32) sm120_fp8_gemv_kernel( const float* __restrict__ alpha) { __shared__ uint8_t sx[K]; + PDLTriggerSecondary(); const uint32_t tid = threadIdx.x; + PDLWaitPrimary(); for (uint32_t i = tid * kFp8VecSize; i < K; i += kNumWarps * 32 * kFp8VecSize) { *reinterpret_cast(sx + i) = *reinterpret_cast(x + i); } @@ -123,9 +125,11 @@ __global__ void __launch_bounds__(kNumWarps * 32) sm120_fp8_gemv_kernel( } } -template +template struct Sm120Fp8GemvKernel { - static_assert(K % (32 * kFp8VecSize * kUnroll) == 0, "K must cover full unrolled warp strides"); + static_assert( + kUnroll == 1 ? (K % kFp8VecSize == 0) : (K % (32 * kFp8VecSize * kUnroll) == 0), + "K must be 16B-aligned (kUnroll==1) or cover full unrolled warp strides"); static_assert(K <= 48 * 1024, "activation row must fit static shared memory"); static void @@ -144,12 +148,13 @@ struct Sm120Fp8GemvKernel { constexpr uint32_t kRowsPerBlock = kRows * kNumWarps; constexpr uint32_t kNumBlocks = (N + kRowsPerBlock - 1) / kRowsPerBlock; - LaunchKernel(kNumBlocks, kNumWarps * 32, device.unwrap())( - sm120_fp8_gemv_kernel, - static_cast(out.data_ptr()), - static_cast(x.data_ptr()), - static_cast(w.data_ptr()), - static_cast(alpha.data_ptr())); + LaunchKernel(kNumBlocks, kNumWarps * 32, device.unwrap()) + .enable_pdl(kUsePDL)( + sm120_fp8_gemv_kernel, + static_cast(out.data_ptr()), + static_cast(x.data_ptr()), + static_cast(w.data_ptr()), + static_cast(alpha.data_ptr())); } }; diff --git a/python/sglang/kernels/jit/csrc/gemm/sm120_nvfp4_moe_mtp_mma.cuh b/python/sglang/kernels/jit/csrc/gemm/sm120_nvfp4_moe_mtp_mma.cuh new file mode 100644 index 0000000000..781e74609b --- /dev/null +++ b/python/sglang/kernels/jit/csrc/gemm/sm120_nvfp4_moe_mtp_mma.cuh @@ -0,0 +1,410 @@ +/*************************************************************************************************** + * Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + +// Native SM120 NVFP4 MMA operand layout follows NVIDIA CUTLASS mma_sm120.hpp +// and mma_traits_sm120.hpp, distributed under BSD-3-Clause. Exact installed-source hashes are recorded in the research +// task history/moe-mtp-native-upstream.json. +#include + +#include +#include + +#include +#include + +namespace sglang { +namespace nvfp4_mtp_mma { +using namespace device; + +__device__ __forceinline__ float rcp(float x) { + float y; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} +__device__ __forceinline__ uint32_t fp4x2(float hi, float lo) { + uint32_t d; + asm volatile("{ .reg .b8 t; cvt.rn.satfinite.e2m1x2.f32 t, %1, %2; cvt.u32.u8 %0, t; }" : "=r"(d) : "f"(hi), "f"(lo)); + return d; +} +__device__ __forceinline__ void quantize16(const float* v, uint32_t& q0, uint32_t& q1, uint8_t& sf, float global) { + float amax = 0.f; +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) + amax = fmaxf(amax, fabsf(v[i])); + __nv_fp8_e4m3 f(global * (amax * rcp(6.f))); + sf = f.__x; + float scale = amax != 0.f ? rcp(static_cast(f) * rcp(global)) : 0.f; + q0 = 0; + q1 = 0; +#pragma unroll + for (uint32_t i = 0; i < 4; ++i) { + q0 |= fp4x2(v[2 * i + 1] * scale, v[2 * i] * scale) << (8 * i); + q1 |= fp4x2(v[8 + 2 * i + 1] * scale, v[8 + 2 * i] * scale) << (8 * i); + } +} +template +__device__ __forceinline__ uint32_t sf_index(uint32_t row, uint32_t group) { + return (row / 128) * (128 * G) + (row % 32) * 16 + ((row % 128) / 32) * 4 + (group / 4) * 512 + group % 4; +} +__device__ __forceinline__ void +mma(float& d0, + float& d1, + float& d2, + float& d3, + uint32_t a0, + uint32_t a1, + uint32_t a2, + uint32_t a3, + uint32_t b0, + uint32_t b1, + uint32_t sfa, + uint32_t sfb) { + asm volatile( + "mma.sync.aligned.kind::mxf4nvf4.block_scale.scale_vec::4X.m16n8k64.row.col.f32.e2m1.e2m1.f32.ue4m3 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3}, {%10}, {0,0}, {%11}, {0,0};" + : "+f"(d0), "+f"(d1), "+f"(d2), "+f"(d3) + : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1), "r"(sfa), "r"(sfb)); +} + +// Stage 256 K elements in contiguous 128B row segments. Padding each shared +// row by 16B avoids the eight-way ldmatrix bank conflict at a 128B stride. +template +__device__ __forceinline__ void +prefetch(uint8_t* shared, const uint8_t* weights, uint32_t tile, uint32_t step, uint32_t lane) { +#pragma unroll + for (uint32_t j = 0; j < 4; ++j) { + uint32_t row = lane / 8 + 4 * j, col = (lane % 8) * 16; + uint32_t dst = static_cast(__cvta_generic_to_shared(shared + row * 144 + col)); + uint32_t source_col = step * 128 + col; + uint32_t bytes = source_col < K / 2 ? 16 : 0; + const uint8_t* src = weights + (tile + row) * (K / 2) + (bytes ? source_col : 0); + asm volatile("cp.async.cg.shared.global [%0], [%1], 16, %2;" ::"r"(dst), "l"(src), "r"(bytes) : "memory"); + } + asm volatile("cp.async.commit_group;" ::: "memory"); +} +__device__ __forceinline__ void +load_a(uint32_t& a0, uint32_t& a1, uint32_t& a2, uint32_t& a3, const uint8_t* shared, uint32_t lane) { + uint32_t addr = static_cast(__cvta_generic_to_shared(shared + (lane % 16) * 144 + (lane / 16) * 16)); + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];" + : "=r"(a0), "=r"(a1), "=r"(a2), "=r"(a3) + : "r"(addr) + : "memory"); +} + +template +__global__ __launch_bounds__(128) void fc1( + const bf16_t* __restrict__ x, + const uint8_t* __restrict__ w, + const uint8_t* __restrict__ sf, + const float* __restrict__ alpha, + const float* __restrict__ global1, + const float* __restrict__ global2, + const int32_t* __restrict__ ids, + uint32_t* __restrict__ z, + uint8_t* __restrict__ zsf) { + __shared__ uint32_t xq[K / 8]; + __shared__ uint8_t xs[K / 16]; + __shared__ float act[4][16]; + __shared__ __align__(16) uint8_t ws[4][2][2304]; + uint32_t tid = threadIdx.x, warp = tid / 32, lane = tid % 32; + uint32_t token = blockIdx.z, slot = blockIdx.y; + PDLTriggerSecondary(); + PDLWaitPrimary(); + if (tid < K / 16) { + float v[16]; +#pragma unroll + for (uint32_t i = 0; i < 16; ++i) + v[i] = static_cast(x[token * K + tid * 16 + i]); + quantize16(v, xq[2 * tid], xq[2 * tid + 1], xs[tid], *global1); + } + __syncthreads(); + uint32_t expert = ids[token * TopK + slot]; + uint32_t tile = (blockIdx.x * 4 + warp) * 16; + uint32_t row = tile + lane / 4, col = (lane % 4) * 4; + const uint8_t* weights = w + static_cast(expert) * N * K / 2; + const uint8_t* scales = sf + static_cast(expert) * N * (K / 16); + float d0 = 0, d1 = 0, d2 = 0, d3 = 0; +#pragma unroll + for (uint32_t stage = 0; stage < 2; ++stage) + prefetch(ws[warp][stage], weights, tile, stage, lane); +#pragma unroll 1 + for (uint32_t base = 0; base < K; base += 256) { + asm volatile("cp.async.wait_group 1;" ::: "memory"); + __syncwarp(); +#pragma unroll + for (uint32_t inner = 0; inner < 256; inner += 64) { + if (base + inner >= K) break; + uint32_t k = base + inner; + uint32_t a0, a1, a2, a3; + load_a(a0, a1, a2, a3, ws[warp][(base / 256) % 2] + inner / 2, lane); + uint32_t sa = + *reinterpret_cast(scales + sf_index(tile + lane / 4 + (lane % 2) * 8, k / 16)); + uint32_t sb = *reinterpret_cast(xs + k / 16); + mma(d0, d1, d2, d3, a0, a1, a2, a3, xq[k / 8 + lane % 4], xq[k / 8 + lane % 4 + 4], sa, sb); + } + __syncwarp(); + prefetch(ws[warp][(base / 256) % 2], weights, tile, base / 256 + 2, lane); + } + asm volatile("cp.async.wait_group 0;" ::: "memory"); + if (lane % 4 == 0) { + float y0 = static_cast(cast(d0 * alpha[expert])); + float y1 = static_cast(cast(d2 * alpha[expert])); + y0 = fmaxf(y0, 0.f); + y1 = fmaxf(y1, 0.f); + act[warp][lane / 4] = static_cast(cast(y0 * y0)); + act[warp][lane / 4 + 8] = static_cast(cast(y1 * y1)); + } + __syncwarp(); + if (lane == 0) { + size_t qoff = (token * TopK + slot) * (N / 8) + tile / 8; + size_t soff = (token * TopK + slot) * (N / 16) + tile / 16; + quantize16(act[warp], z[qoff], z[qoff + 1], zsf[soff], *global2); + } +} + +template +__global__ __launch_bounds__(128) void fc2( + const uint32_t* __restrict__ z, + const uint8_t* __restrict__ zs, + const uint8_t* __restrict__ w, + const uint8_t* __restrict__ sf, + const float* __restrict__ alpha, + const int32_t* __restrict__ ids, + const float* __restrict__ routing, + bf16_t* __restrict__ partial) { + __shared__ __align__(16) uint8_t ws[4][2][2304]; + uint32_t warp = threadIdx.x / 32, lane = threadIdx.x % 32; + uint32_t token = blockIdx.z, slot = blockIdx.y, route = token * TopK + slot; + PDLTriggerSecondary(); + PDLWaitPrimary(); + uint32_t expert = ids[route], tile = (blockIdx.x * 4 + warp) * 16; + uint32_t row = tile + lane / 4, col = (lane % 4) * 4; + const uint8_t* weights = w + static_cast(expert) * N * K / 2; + const uint8_t* scales = sf + static_cast(expert) * N * (K / 16); + z += route * (K / 8); + zs += route * (K / 16); + float d0 = 0, d1 = 0, d2 = 0, d3 = 0; +#pragma unroll + for (uint32_t stage = 0; stage < 2; ++stage) + prefetch(ws[warp][stage], weights, tile, stage, lane); +#pragma unroll 1 + for (uint32_t base = 0; base < K; base += 256) { + asm volatile("cp.async.wait_group 1;" ::: "memory"); + __syncwarp(); +#pragma unroll + for (uint32_t inner = 0; inner < 256; inner += 64) { + if (base + inner >= K) break; + uint32_t k = base + inner; + uint32_t a0, a1, a2, a3; + load_a(a0, a1, a2, a3, ws[warp][(base / 256) % 2] + inner / 2, lane); + uint32_t sa = + *reinterpret_cast(scales + sf_index(tile + lane / 4 + (lane % 2) * 8, k / 16)); + uint32_t sb = *reinterpret_cast(zs + k / 16); + mma(d0, d1, d2, d3, a0, a1, a2, a3, z[k / 8 + lane % 4], z[k / 8 + lane % 4 + 4], sa, sb); + } + __syncwarp(); + prefetch(ws[warp][(base / 256) % 2], weights, tile, base / 256 + 2, lane); + } + asm volatile("cp.async.wait_group 0;" ::: "memory"); + if (lane % 4 == 0) { + partial[route * N + row] = cast((d0 * alpha[expert]) * routing[route]); + partial[route * N + row + 8] = cast((d2 * alpha[expert]) * routing[route]); + } +} +template +__global__ void reduce(const bf16_t* __restrict__ partial, bf16_t* __restrict__ out, uint32_t rows) { + PDLTriggerSecondary(); + uint32_t i = blockIdx.x * blockDim.x + threadIdx.x; + PDLWaitPrimary(); + if (i < rows * N) { + uint32_t token = i / N, col = i % N; + float value = 0.f; +#pragma unroll + for (uint32_t slot = 0; slot < TopK; ++slot) + value += static_cast(partial[(token * TopK + slot) * N + col]); + out[i] = cast(value); + } +} +} // namespace nvfp4_mtp_mma + +namespace nvfp4_pair_mma { +using namespace nvfp4_mtp_mma; +// Store each original pair of FP4 weight bits across contiguous output channels. +// One 16-byte fetch then corresponds to exactly two activation values, enabling +// zero-filled asynchronous loads without discarding any nonzero multiplication. +template +__device__ __forceinline__ void +pair_prefetch(uint8_t* shared, const uint8_t* weights, unsigned tile, unsigned step, unsigned lane, const uint32_t* z) { +#pragma unroll + for (unsigned j = 0; j < 2; ++j) { + unsigned pair = lane + 32 * j, source_pair = step * 64 + pair; + unsigned bytes = source_pair < K / 2 ? 16 : 0; + if (bytes && (reinterpret_cast(z)[source_pair] & 0x77u) == 0) bytes = 0; + const uint8_t* src = weights + (bytes ? source_pair : 0) * N + tile; + unsigned dst = static_cast(__cvta_generic_to_shared(shared + pair * 16)); + asm volatile("cp.async.cg.shared.global [%0], [%1], 16, %2;" ::"r"(dst), "l"(src), "r"(bytes) : "memory"); + } + asm volatile("cp.async.commit_group;" ::: "memory"); +} +__device__ __forceinline__ void +pair_load_a(unsigned& a0, unsigned& a1, unsigned& a2, unsigned& a3, const uint8_t* s, unsigned lane) { + unsigned addr = static_cast(__cvta_generic_to_shared(s + lane * 16)); + asm volatile("ldmatrix.sync.aligned.m16n16.x2.trans.shared.b8 {%0,%1,%2,%3}, [%4];" + : "=r"(a0), "=r"(a1), "=r"(a2), "=r"(a3) + : "r"(addr) + : "memory"); +} +template +__global__ __launch_bounds__(128) void fc2( + const uint32_t* __restrict__ z, + const uint8_t* __restrict__ zs, + const uint8_t* __restrict__ w, + const uint8_t* __restrict__ sf, + const float* __restrict__ alpha, + const int32_t* __restrict__ ids, + const float* __restrict__ routing, + bf16_t* __restrict__ partial) { + __shared__ __align__(16) uint8_t ws[4][2][1024]; + uint32_t warp = threadIdx.x / 32, lane = threadIdx.x % 32; + uint32_t token = blockIdx.z, slot = blockIdx.y, route = token * TopK + slot; + PDLTriggerSecondary(); + PDLWaitPrimary(); + uint32_t expert = ids[route], tile = (blockIdx.x * 4 + warp) * 16; + uint32_t row = tile + lane / 4, col = (lane % 4) * 4; + const uint8_t* weights = w + static_cast(expert) * N * K / 2; + const uint8_t* scales = sf + static_cast(expert) * N * (K / 16); + z += route * (K / 8); + zs += route * (K / 16); + float d0 = 0, d1 = 0, d2 = 0, d3 = 0; +#pragma unroll + for (uint32_t stage = 0; stage < 2; ++stage) + pair_prefetch(ws[warp][stage], weights, tile, stage, lane, z); +#pragma unroll 1 + for (uint32_t base = 0; base < K; base += 128) { + asm volatile("cp.async.wait_group 1;" ::: "memory"); + __syncwarp(); +#pragma unroll + for (uint32_t inner = 0; inner < 128; inner += 64) { + if (base + inner >= K) break; + uint32_t k = base + inner; + uint32_t a0, a1, a2, a3; + pair_load_a(a0, a1, a2, a3, ws[warp][(base / 128) % 2] + (inner / 2) * 16, lane); + uint32_t sa = + *reinterpret_cast(scales + sf_index(tile + lane / 4 + (lane % 2) * 8, k / 16)); + uint32_t sb = *reinterpret_cast(zs + k / 16); + mma(d0, d1, d2, d3, a0, a1, a2, a3, z[k / 8 + lane % 4], z[k / 8 + lane % 4 + 4], sa, sb); + } + __syncwarp(); + pair_prefetch(ws[warp][(base / 128) % 2], weights, tile, base / 128 + 2, lane, z); + } + asm volatile("cp.async.wait_group 0;" ::: "memory"); + if (lane % 4 == 0) { + partial[route * N + row] = cast((d0 * alpha[expert]) * routing[route]); + partial[route * N + row + 8] = cast((d2 * alpha[expert]) * routing[route]); + } +} +} // namespace nvfp4_pair_mma + +template +struct Sm120Nvfp4MoeMtpMma { + static_assert(N1 == 2688 && K1 == 1024 && N2 == 1024 && K2 == 2688 && TopK == 22); + /// Run 1 to 4 token rows with BF16 epilogues and deterministic reduction. + static void + run(tvm::ffi::TensorView x, + tvm::ffi::TensorView w1, + tvm::ffi::TensorView s1, + tvm::ffi::TensorView w2, + tvm::ffi::TensorView s2, + tvm::ffi::TensorView a1, + tvm::ffi::TensorView a2, + tvm::ffi::TensorView g1, + tvm::ffi::TensorView g2, + tvm::ffi::TensorView ids, + tvm::ffi::TensorView routing, + tvm::ffi::TensorView z, + tvm::ffi::TensorView zs, + tvm::ffi::TensorView partial, + tvm::ffi::TensorView out) { + using namespace host; + using namespace device; + auto dev = SymbolicDevice{}; + dev.set_options(); + auto rows = SymbolicSize{"token_rows"}; + TensorMatcher({rows, K1}).with_dtype().with_device(dev).verify(x); + CHECK_HOST(rows.unwrap() >= 1 && rows.unwrap() <= 4) << "Expected 1 <= M <= 4"; + const uint32_t M = static_cast(rows.unwrap()); + TensorMatcher({512, N1, K1 / 2}).with_dtype().with_device(dev).verify(w1); + if constexpr (PairLayout) + TensorMatcher({512, K2 / 2, N2}).with_dtype().with_device(dev).verify(w2); + else + TensorMatcher({512, N2, K2 / 2}).with_dtype().with_device(dev).verify(w2); + TensorMatcher({512 * N1 * K1 / 16}).with_dtype().with_device(dev).verify(s1); + TensorMatcher({512 * N2 * K2 / 16}).with_dtype().with_device(dev).verify(s2); + TensorMatcher({512}).with_dtype().with_device(dev).verify(a1).verify(a2); + TensorMatcher({1}).with_dtype().with_device(dev).verify(g1).verify(g2); + TensorMatcher({M, TopK}).with_dtype().with_device(dev).verify(ids); + TensorMatcher({M, TopK}).with_dtype().with_device(dev).verify(routing); + TensorMatcher({M * TopK * N1 / 8}).with_dtype().with_device(dev).verify(z); + TensorMatcher({M * TopK * N1 / 16}).with_dtype().with_device(dev).verify(zs); + TensorMatcher({M * TopK * N2}).with_dtype().with_device(dev).verify(partial); + TensorMatcher({M, N2}).with_dtype().with_device(dev).verify(out); + LaunchKernel({N1 / 64, TopK, M}, 128, dev.unwrap()) + .enable_pdl(PDL)( + nvfp4_mtp_mma::fc1, + static_cast(x.data_ptr()), + static_cast(w1.data_ptr()), + static_cast(s1.data_ptr()), + static_cast(a1.data_ptr()), + static_cast(g1.data_ptr()), + static_cast(g2.data_ptr()), + static_cast(ids.data_ptr()), + static_cast(z.data_ptr()), + static_cast(zs.data_ptr())); + LaunchKernel({N2 / 64, TopK, M}, 128, dev.unwrap()) + .enable_pdl(PDL)( + PairLayout ? nvfp4_pair_mma::fc2 : nvfp4_mtp_mma::fc2, + static_cast(z.data_ptr()), + static_cast(zs.data_ptr()), + static_cast(w2.data_ptr()), + static_cast(s2.data_ptr()), + static_cast(a2.data_ptr()), + static_cast(ids.data_ptr()), + static_cast(routing.data_ptr()), + static_cast(partial.data_ptr())); + LaunchKernel({(M * N2 + 255) / 256}, 256, dev.unwrap()) + .enable_pdl(PDL)( + nvfp4_mtp_mma::reduce, + static_cast(partial.data_ptr()), + static_cast(out.data_ptr()), + M); + } +}; +} // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/gemm/sm120_nvfp4_weight_transpose.cuh b/python/sglang/kernels/jit/csrc/gemm/sm120_nvfp4_weight_transpose.cuh new file mode 100644 index 0000000000..577b460762 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/gemm/sm120_nvfp4_weight_transpose.cuh @@ -0,0 +1,49 @@ +// Exact byte transpose for changing weight layout once per request phase. +#include + +#include +#include +namespace sglang { +namespace nvfp4_weight_transpose { +template +__global__ __launch_bounds__(256) void kernel(const uint8_t* __restrict__ x, uint8_t* __restrict__ y) { + __shared__ uint8_t data[Tile][Tile + 4]; + unsigned tx = threadIdx.x % 32, ty = threadIdx.x / 32; + unsigned rbase = blockIdx.y * Tile, cbase = blockIdx.x * Tile; + size_t expert = static_cast(blockIdx.z) * Rows * Cols; +#pragma unroll + for (unsigned j = 0; j < Tile / 8; ++j) { + unsigned row = rbase + ty + j * 8, col = cbase + tx * 4; + unsigned v = 0; + if (row < Rows && col < Cols) v = *reinterpret_cast(x + expert + row * Cols + col); + *reinterpret_cast(&data[ty + j * 8][tx * 4]) = v; + } + __syncthreads(); +#pragma unroll + for (unsigned j = 0; j < Tile / 8; ++j) { + unsigned row = cbase + ty + j * 8, col = rbase + tx * 4; + if (row < Cols && col < Rows) { + unsigned v = static_cast(data[tx * 4 + 0][ty + j * 8]) | + (static_cast(data[tx * 4 + 1][ty + j * 8]) << 8) | + (static_cast(data[tx * 4 + 2][ty + j * 8]) << 16) | + (static_cast(data[tx * 4 + 3][ty + j * 8]) << 24); + *reinterpret_cast(y + expert + row * Rows + col) = v; + } + } +} +} // namespace nvfp4_weight_transpose +template +struct Nvfp4WeightTranspose { + static void run(tvm::ffi::TensorView x, tvm::ffi::TensorView y) { + using namespace host; + auto dev = SymbolicDevice{}; + dev.set_options(); + TensorMatcher({512, Rows, Cols}).with_dtype().with_device(dev).verify(x); + TensorMatcher({512, Cols, Rows}).with_dtype().with_device(dev).verify(y); + LaunchKernel({(Cols + 127) / 128, (Rows + 127) / 128, 512}, 256, dev.unwrap())( + nvfp4_weight_transpose::kernel, + static_cast(x.data_ptr()), + static_cast(y.data_ptr())); + } +}; +} // namespace sglang diff --git a/python/sglang/kernels/ops/gemm/hopper_bf16_gemv.py b/python/sglang/kernels/ops/gemm/hopper_bf16_gemv.py index 5115720094..81af6d6e7c 100644 --- a/python/sglang/kernels/ops/gemm/hopper_bf16_gemv.py +++ b/python/sglang/kernels/ops/gemm/hopper_bf16_gemv.py @@ -4,7 +4,12 @@ from typing import TYPE_CHECKING import torch -from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args +from sglang.kernels.jit.utils import ( + cache_once, + is_arch_support_pdl, + load_jit, + make_cpp_args, +) if TYPE_CHECKING: from tvm_ffi.module import Module @@ -20,8 +25,18 @@ _MAX_K = 17408 # static smem limit (K * 2 bytes <= 48KB) with margin _MAX_N = 65536 # very large N (lm_head) is already at the bandwidth ceiling +def _is_sm12x() -> bool: + from sglang.kernels.jit.utils import get_jit_cuda_arch + + return get_jit_cuda_arch().major >= 12 + + def _config(n: int) -> tuple[int, int, int]: - """(rows_per_warp, k_unroll, num_warps) tuned on H200.""" + """(rows_per_warp, k_unroll, num_warps) tuned on H200 (SM90) and + DRAM-cold on RTX PRO 6000 (SM120), where kUnroll must stay 1 so the + in-loop k < K guard also serves K % 512 != 0 shapes.""" + if _is_sm12x(): + return (1, 1, 8) if n >= 8192: return (2, 2, 8) return (1, 2, 8) @@ -30,7 +45,7 @@ def _config(n: int) -> tuple[int, int, int]: @cache_once def _jit_hopper_bf16_gemv_module(n: int, k: int) -> Module: rows, unroll, warps = _config(n) - args = make_cpp_args(n, k, rows, unroll, warps) + args = make_cpp_args(n, k, rows, unroll, warps, is_arch_support_pdl()) return load_jit( "hopper_bf16_gemv", *args, @@ -41,6 +56,21 @@ def _jit_hopper_bf16_gemv_module(n: int, k: int) -> Module: def use_hopper_bf16_gemv(m: int, n: int, k: int) -> bool: + if _is_sm12x(): + # SM120 (GDDR7): kUnroll==1 needs only 16B-aligned K (128 kept for + # vector-alignment margin). cuBLAS M=1 bf16 trails the streaming GEMV + # on small/odd N and very wide N; the 12288..32768 window (e.g. the + # 18560-row bf16 in_proj) already runs at the DRAM ceiling under + # cuBLAS, so leave it there. + if not ( + m == 1 + and k % 128 == 0 + and 512 <= k <= _MAX_K + and n % 8 == 0 + and 64 <= n <= 140000 + ): + return False + return n < 12288 or n >= 32768 if not ( m == 1 and k % 512 == 0 diff --git a/python/sglang/kernels/ops/gemm/sm120_fp8_gemv.py b/python/sglang/kernels/ops/gemm/sm120_fp8_gemv.py index 718fb466d2..821b3c9e2c 100644 --- a/python/sglang/kernels/ops/gemm/sm120_fp8_gemv.py +++ b/python/sglang/kernels/ops/gemm/sm120_fp8_gemv.py @@ -4,7 +4,12 @@ from typing import TYPE_CHECKING import torch -from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args +from sglang.kernels.jit.utils import ( + cache_once, + is_arch_support_pdl, + load_jit, + make_cpp_args, +) if TYPE_CHECKING: from tvm_ffi.module import Module @@ -22,16 +27,16 @@ _MAX_N = 32768 # very large N is served well by cuBLAS already def _config(n: int) -> tuple[int, int, int]: - """(rows_per_warp, k_unroll, num_warps).""" + """(rows_per_warp, k_unroll, num_warps); tuned DRAM-cold on RTX PRO 6000.""" if n >= 8192: - return (2, 1, 8) + return (4, 1, 8) return (1, 1, 8) @cache_once def _jit_sm120_fp8_gemv_module(n: int, k: int) -> Module: rows, unroll, warps = _config(n) - args = make_cpp_args(n, k, rows, unroll, warps) + args = make_cpp_args(n, k, rows, unroll, warps, is_arch_support_pdl()) return load_jit( "sm120_fp8_gemv", *args, @@ -42,9 +47,12 @@ def _jit_sm120_fp8_gemv_module(n: int, k: int) -> Module: def use_sm120_fp8_gemv(m: int, n: int, k: int) -> bool: + # kUnroll == 1 streams 16B per lane-step with an in-loop k < K guard, so + # K only needs 16B alignment; keep 128 for conservative vector alignment + # (covers the K=5376 shared-expert down_proj). return ( m == 1 - and k % 512 == 0 + and k % 128 == 0 and 512 <= k <= _MAX_K and n % 16 == 0 and 256 <= n <= _MAX_N diff --git a/python/sglang/kernels/ops/gemm/sm120_nvfp4_moe.py b/python/sglang/kernels/ops/gemm/sm120_nvfp4_moe.py new file mode 100644 index 0000000000..e9ab06f3e3 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/sm120_nvfp4_moe.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import torch + +from sglang.kernels.jit.utils import ( + cache_once, + load_jit, + make_cpp_args, +) + + +@cache_once +def _mtp_module(pair_layout: bool = False): + args = make_cpp_args(2688, 1024, 1024, 2688, 22, False, pair_layout) + return load_jit( + "sm120_nvfp4_moe_mtp_mma", + *args, + cuda_files=["gemm/sm120_nvfp4_moe_mtp_mma.cuh"], + cuda_wrappers=[("run", f"sglang::Sm120Nvfp4MoeMtpMma<{args}>::run")], + extra_cuda_cflags=["-O3"], + ) + + +def nvfp4_moe_mtp_mma( + x, w1, s1, w2, s2, a1, a2, g1, g2, ids, routing, out=None, *, pair_layout=False +): + m = x.shape[0] + module = _mtp_module(pair_layout) + if out is None: + out = torch.empty((m, 1024), dtype=torch.bfloat16, device=x.device) + z = torch.empty(m * 22 * 2688 // 8, dtype=torch.int32, device=x.device) + zs = torch.empty(m * 22 * 2688 // 16, dtype=torch.uint8, device=x.device) + partial = torch.empty(m * 22 * 1024, dtype=torch.bfloat16, device=x.device) + module.run( + x, + w1.view(torch.uint8), + s1.view(torch.uint8).view(-1), + w2.view(torch.uint8), + s2.view(torch.uint8).view(-1), + a1.view(-1), + a2.view(-1), + g1.view(-1), + g2.view(-1), + ids.to(torch.int32), + routing.float(), + z, + zs, + partial, + out, + ) + return out + + +def try_nvfp4_moe( + x, + w13, + w2, + scales, + ids, + routing, + *, + output_dtype, + out=None, + pair_w2_weight=None, + use_mtp=True, +): + """Select native small-M kernels; unsupported shapes retain CUTLASS.""" + if ( + use_mtp + and x.ndim == 2 + and 1 <= x.shape[0] <= 4 + and x.shape[1] == 1024 + and x.dtype == output_dtype == torch.bfloat16 + and w13.shape == (512, 2688, 512) + and w2.shape == (512, 1024, 1344) + and ids.shape == (x.shape[0], 22) + ): + return nvfp4_moe_mtp_mma( + x, + w13, + scales[1], + pair_w2_weight if pair_w2_weight is not None else w2, + scales[4], + scales[2], + scales[5], + scales[0], + scales[3], + ids, + routing, + out=out, + pair_layout=pair_w2_weight is not None, + ) + if pair_w2_weight is not None: + raise RuntimeError("A paired NVFP4 weight reached an unsupported MoE path") + return None + + +@cache_once +def _transpose_module(rows: int, cols: int): + return load_jit( + "sm120_nvfp4_weight_transpose", + rows, + cols, + cuda_files=["gemm/sm120_nvfp4_weight_transpose.cuh"], + cuda_wrappers=[("run", f"sglang::Nvfp4WeightTranspose<{rows},{cols}>::run")], + extra_cuda_cflags=["-O3"], + ) + + +class Nvfp4LayoutCycle: + """Own the two views of a fixed, serial set of [512,1024,1344] weights.""" + + def __init__(self, weights): + if not weights: + raise ValueError("A layout cycle requires at least one expert weight") + first = weights[0] + if first.device.type != "cuda": + raise ValueError("The layout cycle requires CUDA weights") + for weight in weights: + if ( + weight.shape != (512, 1024, 1344) + or weight.dtype != torch.uint8 + or not weight.is_contiguous() + or weight.device != first.device + ): + raise ValueError("Incompatible NVFP4 FC2 weight layout") + if len({weight.data_ptr() for weight in weights}) != len(weights): + raise ValueError("Expert weights in a layout cycle must not alias") + self.original = tuple(weights) + self.workspace_bytes = first.numel() + self.scratch = None + self.pair = () + self.is_pair = False + self.switches = 0 + # Compile before any graph capture can reach the state transition. + self._forward = _transpose_module(1024, 1344) + self._inverse = _transpose_module(1344, 1024) + + def prepare_workspace(self): + if self.scratch is not None: + return + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError("NVFP4 workspace must be prepared before graph capture") + # Use runtime workspace headroom, like graph/kernel scratch allocations. + # Allocating this before Mamba/KV pool sizing incorrectly charges it as + # persistent checkpoint storage and can round the state pool below one + # request. The normal graph warmup prepares stable pointers before use. + self.scratch = torch.empty_like(self.original[0]) + self.pair = tuple( + weight.view(512, 1344, 1024) + for weight in (self.scratch, *self.original[:-1]) + ) + + def ensure(self, pair: bool): + if pair == self.is_pair: + return + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError("NVFP4 layout transitions must run before graph capture") + if pair: + self.prepare_workspace() + for original, paired in zip(self.original, self.pair): + self._forward.run(original, paired) + else: + for original, paired in reversed(tuple(zip(self.original, self.pair))): + self._inverse.run(paired, original) + self.is_pair = pair + self.switches += 1 diff --git a/python/sglang/kernels/ops/moe/moe_fused_gate.py b/python/sglang/kernels/ops/moe/moe_fused_gate.py index 8d554ed6fd..9cd0f6b864 100644 --- a/python/sglang/kernels/ops/moe/moe_fused_gate.py +++ b/python/sglang/kernels/ops/moe/moe_fused_gate.py @@ -18,6 +18,85 @@ _SCORING_FUNC_MAP = { } +@triton.jit +def _sm120_router_top22_kernel( + scores_ptr, + bias_ptr, + weights_ptr, + ids_ptr, + stride_sm, + stride_sn, + scale, + RENORMALIZE: tl.constexpr, + APPLY_SCALE: tl.constexpr, + USE_PDL: tl.constexpr, +): + row = tl.program_id(0) + n = tl.arange(0, 512) + if USE_PDL: + tl.extra.cuda.gdc_wait() + scores = tl.load(scores_ptr + row * stride_sm + n * stride_sn).to(tl.float32) + bias = tl.load(bias_ptr + n).to(tl.float32) + activated = tl.sigmoid(scores) + biased = activated + bias + biased = tl.where(biased == biased, biased, -1e30) + biased = tl.where(biased == 0.0, 0.0, biased) + bits = biased.to(tl.uint32, bitcast=True) + ordered = tl.where((bits & 0x80000000) != 0, ~bits, bits ^ 0x80000000) + keys = (ordered.to(tl.uint64) << 32) | (0xFFFFFFFF - n.to(tl.uint32)).to(tl.uint64) + winners = tl.topk(keys, 32) + ids = (0xFFFFFFFF - (winners & 0xFFFFFFFF).to(tl.uint32)).to(tl.int32) + k = tl.arange(0, 32) + values = tl.gather(activated, ids, axis=0) + values = tl.where(k < 22, values, 0.0) + total = tl.sum(values, 0) + if USE_PDL: + tl.extra.cuda.gdc_launch_dependents() + if RENORMALIZE: + values = values / tl.where(total > 0.0, total, 1.0) + if APPLY_SCALE: + values = values * scale + tl.store(weights_ptr + row * 22 + k, values, k < 22) + tl.store(ids_ptr + row * 22 + k, ids, k < 22) + + +def _topk_gate_small_m( + scores, + bias, + topk=22, + scoring_func="sigmoid", + num_fused_shared_experts=0, + renormalize=True, + routed_scaling_factor=1.0, + apply_routed_scaling_factor_on_output=False, +): + assert scores.is_cuda and bias.device == scores.device + assert scores.ndim == 2 and scores.shape[1] == 512 + assert scores.dtype == torch.float32 + assert 1 <= scores.shape[0] <= 16 + assert bias.shape == (512,) and bias.dtype == torch.float32 and bias.is_contiguous() + assert topk == 22 and scoring_func == "sigmoid" and num_fused_shared_experts == 0 + m = scores.shape[0] + weights = torch.empty((m, 22), device=scores.device, dtype=torch.float32) + ids = torch.empty((m, 22), device=scores.device, dtype=torch.int32) + pdl = is_arch_support_pdl() + _sm120_router_top22_kernel[(m,)]( + scores, + bias, + weights, + ids, + scores.stride(0), + scores.stride(1), + float(routed_scaling_factor), + RENORMALIZE=bool(renormalize), + APPLY_SCALE=bool(apply_routed_scaling_factor_on_output), + USE_PDL=pdl, + num_warps=1, + **({"launch_pdl": True} if pdl else {}), + ) + return weights, ids + + @triton.jit def _router_triton_kernel( scores_ptr, # [M, N] raw logits, fp32/fp16/bf16 (upcast to fp32 on load) @@ -328,12 +407,46 @@ def moe_fused_gate( assert input_ids.shape == (scores.size(0),) if packed_out is not None: assert packed_out.dtype == torch.int32, "packed_out must be int32" - assert packed_out.shape == (scores.size(0), topk), ( - "packed_out must be [M, topk]" - ) + assert packed_out.shape == ( + scores.size(0), + topk, + ), "packed_out must be [M, topk]" if routed_scaling_factor is None: routed_scaling_factor = 1.0 + from sglang.srt.environ import envs + + if ( + envs.SGLANG_SM120_MTP_ROUTER_TOP22.get() + and scores.is_cuda + and torch.cuda.get_device_capability(scores.device) == (12, 0) + and 2 <= scores.shape[0] <= 16 + and scores.shape[1] == 512 + and scores.dtype == torch.float32 + and scores.stride(1) == 1 + and bias is not None + and bias.dtype == torch.float32 + and bias.is_contiguous() + and topk == 22 + and scoring_func.lower() == "sigmoid" + and num_fused_shared_experts == 0 + and num_expert_group <= 1 + and moe_softcapping == 0.0 + and input_ids is None + and num_token_non_padded is None + and renormalize_epsilon == 0.0 + and packed_out is None + and not sqrtsoftplus_log1p + ): + return _topk_gate_small_m( + scores, + bias, + topk=topk, + renormalize=renormalize, + routed_scaling_factor=routed_scaling_factor, + apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, + ) + # K3 radix-select fast path: native-CUDA radix-select replaces the 16 # dependent argmax rounds (single CTA per token; ids bit-identical to this # triton kernel incl. ties). diff --git a/python/sglang/kernels/ops/speculative/pack_verify_mask.py b/python/sglang/kernels/ops/speculative/pack_verify_mask.py new file mode 100644 index 0000000000..0552d1dd87 --- /dev/null +++ b/python/sglang/kernels/ops/speculative/pack_verify_mask.py @@ -0,0 +1,41 @@ +"""Pack a uniform-width verify tree mask into existing FlashInfer buffers.""" + +import triton +import triton.language as tl + + +@triton.jit +def _pack_verify_mask_kernel( + mask, + kv_indptr, + packed_indptr, + output, + Q: tl.constexpr, + BLOCK: tl.constexpr, +): + row = tl.program_id(0) + byte = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK) + bit = tl.arange(0, 8) + kv_start = tl.load(kv_indptr + row) + kv_end = tl.load(kv_indptr + row + 1) + count = (kv_end - kv_start) * Q + offsets = byte[:, None] * 8 + bit[None, :] + values = tl.load(mask + kv_start * Q + offsets, offsets < count, other=0) + packed = tl.sum(values.to(tl.uint32) << bit[None, :], axis=1) + start = tl.load(packed_indptr + row) + tl.store(output + start + byte, packed.to(tl.uint8), byte < tl.cdiv(count, 8)) + + +def pack_verify_mask_into( + mask, kv_indptr, packed_indptr, output, *, batch_size, query_width, max_kv_len +): + _pack_verify_mask_kernel[ + (batch_size, triton.cdiv(query_width * max_kv_len, 8 * 256)) + ]( + mask.contiguous().view(-1), + kv_indptr, + packed_indptr, + output, + Q=query_width, + BLOCK=256, + ) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 8b73503ecf..7037952167 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1157,6 +1157,9 @@ class Envs: SGLANG_TRTLLM_MOE_PDL_MAX_TOKENS = EnvInt(8192) # Use FlashInfer's fused atomic CUTLASS/CuTe DSL MoE finalize. SGLANG_FLASHINFER_MOE_FUSED_FINALIZE = EnvBool(False) + # Experimental Nemotron small-M NVFP4 path; validated only on SM120. + SGLANG_SM120_NVFP4_PHASE_LAYOUT = EnvBool(False) + SGLANG_SM120_MTP_ROUTER_TOP22 = EnvBool(False) # Master switch for the experimental TRT-LLM LoRA fast path; when OFF (default) every # fine-grained opt switch reads False, keeping non-experimental paths byte-identical. SGLANG_EXPERIMENTAL_LORA_OPTI = EnvBool(False) @@ -1453,6 +1456,10 @@ class Envs: # into a small CUDA graph, collapsing its host dispatch cost to one launch. # Experimental; auto-falls back to eager if the backend's prep is not capturable. SGLANG_ENABLE_METADATA_GLUE_GRAPH = EnvBool(False) + # Reuse the resolved CPU sequence lengths for FlashInfer draft planning, + # avoiding a second blocking readback after generating the same indptr. + SGLANG_FLASHINFER_DRAFT_HOST_INDPTR = EnvBool(False) + SGLANG_FLASHINFER_VERIFY_HOST_PLAN = EnvBool(False) SGLANG_OPT_FUSED_KDA_VERIFY = EnvBool(False) # A/B: keep the DFLASH draft greedy head eager (not folded in-graph). SGLANG_DFLASH_EAGER_DRAFT_SAMPLER = EnvBool(False) diff --git a/python/sglang/srt/layers/attention/flashinfer_backend.py b/python/sglang/srt/layers/attention/flashinfer_backend.py index ce541a9efd..04c0fd1abd 100644 --- a/python/sglang/srt/layers/attention/flashinfer_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_backend.py @@ -210,9 +210,10 @@ def fast_prefill_plan( kv_lens_host: torch.Tensor, max_q_len: int, max_kv_len: int, + uniform_verify_width: Optional[int] = None, ) -> None: """Sync-free ``BatchPrefillWithPagedKVCacheWrapper.plan`` for the EAGLE - draft-extend CUDA graph (FlashInfer fa2, cuda-graph mode only). + draft-extend or uniform-width verify CUDA graph (FlashInfer fa2 only). Upstream plan() always does qo/paged_kv/last_page_len ``.to("cpu")`` to build its host scheduling metadata, a blocking D2H that drains the GPU queue every @@ -259,6 +260,30 @@ def fast_prefill_plan( non_blocking=(paged_kv_indices.device == self.device) and non_blocking, ) + if custom_mask is not None: + # EAGLE's mask remains data-dependent: pack every actual tree-mask + # bit, including rejected branches. Only its row lengths are known + # on the host, so no device scalar read is needed for allocation. + assert uniform_verify_width is not None and page_size == 1 + from sglang.kernels.ops.speculative.pack_verify_mask import ( + pack_verify_mask_into, + ) + + packed_lengths = (kv_lens_host * uniform_verify_width + 7) // 8 + packed_indptr = torch.zeros(batch_size + 1, dtype=torch.int32, device="cpu") + packed_indptr[1:] = torch.cumsum(packed_lengths, dim=0) + assert int(packed_indptr[-1]) <= self._custom_mask_buf.numel() + self._mask_indptr_buf.copy_(packed_indptr, non_blocking=non_blocking) + pack_verify_mask_into( + custom_mask, + paged_kv_indptr, + self._mask_indptr_buf, + self._custom_mask_buf, + batch_size=batch_size, + query_width=uniform_verify_width, + max_kv_len=max_kv_len, + ) + self._cached_q_data_type = q_data_type self._cached_kv_data_type = ( kv_data_type if kv_data_type is not None else q_data_type @@ -855,9 +880,24 @@ class FlashInferAttnBackend(AttentionBackend): # call_begin_forward therefore applies verbatim; installing the # sync-free plan removes three blocking .to("cpu") reads per # replay that otherwise stall the CPU behind the in-flight graph. - # EAGLE target-verify keeps the plain plan(): its spec input is - # not DFLASH_VERIFY, and this branch keys off the capture-time - # spec_info of these per-bs wrappers. + # EAGLE uses the separate opt-in branch below because it must + # also pack its actual tree mask on every replay. + for w in self.prefill_cuda_graph_metadata[bs]: + w.begin_forward = partial(fast_prefill_plan, w) + + if ( + in_capture + and envs.SGLANG_FLASHINFER_VERIFY_HOST_PLAN.get() + and forward_mode.is_target_verify() + and spec_info is not None + and spec_info.spec_input_type == SpecInputType.EAGLE_VERIFY + and spec_info.topk == 1 + and seq_lens_cpu is not None + and self.prefill_backend == "fa2" + and self.dispatch_reason is None + and self.flashinfer_kv_cache_dtype + in (torch.bfloat16, torch.float16, torch.float8_e4m3fn, torch.float8_e5m2) + ): for w in self.prefill_cuda_graph_metadata[bs]: w.begin_forward = partial(fast_prefill_plan, w) @@ -2322,10 +2362,13 @@ class FlashInferIndicesUpdaterPrefill: assert num_tokens_per_req is not None and num_tokens_per_req > 0, ( f"fast_prefill_plan replay requires num_tokens_per_req > 0 (got {num_tokens_per_req})" ) - assert use_custom_mask is None, ( - "fast_prefill_plan does not support custom_mask; keep the plain plan()" - ) seq_lens_cpu_i32 = seq_lens_cpu.to(torch.int32) + eagle_verify = spec_info.spec_input_type == SpecInputType.EAGLE_VERIFY + assert use_custom_mask is None or eagle_verify + if eagle_verify: + # EAGLE carries the committed prefix length; its generated KV + # layout includes the complete candidate window after it. + seq_lens_cpu_i32 = seq_lens_cpu_i32 + num_tokens_per_req qo_indptr_host = torch.arange( 0, (bs + 1) * num_tokens_per_req, @@ -2342,6 +2385,8 @@ class FlashInferIndicesUpdaterPrefill: max_q_len=num_tokens_per_req, max_kv_len=int(seq_lens_cpu_i32.max()), ) + if eagle_verify: + paged_plan_kwargs["uniform_verify_width"] = num_tokens_per_req if window_left >= 0: # selects the module with the per-element window mask compiled in @@ -2369,6 +2414,30 @@ class FlashInferIndicesUpdaterPrefill: ) +def _build_draft_indptr_host( + seq_lens_cpu: torch.Tensor, topk: int, num_steps: int +) -> torch.Tensor: + """Mirror the draft kernel's prefix sums from resolved host lengths. + + At draft entry, positions repeat each committed sequence length ``topk`` + times. Step i adds i+1 tokens per branch. This also covers the graph + runner's padded rows, whose CPU lengths have already been filled. + """ + assert seq_lens_cpu.device.type == "cpu" and seq_lens_cpu.ndim == 1 + offsets = [0] + for length in seq_lens_cpu.tolist(): + for _ in range(topk): + offsets.append(offsets[-1] + length) + return torch.tensor( + [ + [offset + branch * (step + 1) for branch, offset in enumerate(offsets)] + for step in range(num_steps) + ], + dtype=torch.int32, + device="cpu", + ) + + class FlashInferMultiStepDraftBackend: """ Wrap multiple flashinfer attention backends as one for multiple consecutive @@ -2447,8 +2516,23 @@ class FlashInferMultiStepDraftBackend: req_pool_indices=forward_batch.req_pool_indices, bs=num_seqs, ) + host_lengths = forward_batch.seq_lens_cpu + use_host_indptr = ( + envs.SGLANG_FLASHINFER_DRAFT_HOST_INDPTR.get() + and self.draft_window_size == 0 + and host_lengths is not None + and host_lengths.device.type == "cpu" + and host_lengths.numel() == num_seqs + ) + # The existing token-block kernel copies every index, but distributes + # long contexts across CTAs instead of serially walking them in 3 CTAs. + num_blocks = ( + min(32, max(1, (max(host_lengths.tolist(), default=0) + 511) // 512)) + if use_host_indptr + else 1 + ) self.generate_draft_decode_kv_indices[ - (self.speculative_num_steps, num_seqs, self.topk) + (self.speculative_num_steps * num_blocks, num_seqs, self.topk) ]( src.row_ids, src.ids, @@ -2468,24 +2552,33 @@ class FlashInferMultiStepDraftBackend: ENTRY_PAGE_SIZE=src.entry_page_size, v2p=src.v2p, TRANSLATE=src.v2p is not None, + NUM_STEPS=self.speculative_num_steps if use_host_indptr else 0, ) assert forward_batch.spec_info is not None assert forward_batch.spec_info.is_draft_input() - # Copy the kv_indptr once to avoid multiple device-to-host copies in flashinfer's plan. - indptr_cpu_whole = self.kv_indptr[:, : bs + 1].cpu() + # resolve_seq_lens_cpu already supplies the committed lengths after + # acceptance/rollback. Do not drain the GPU queue to read them again. + indptr_cpu_whole = ( + _build_draft_indptr_host( + host_lengths, self.topk, self.speculative_num_steps + ) + if use_host_indptr + else self.kv_indptr[:, : bs + 1].cpu() + ) global global_override_indptr_cpu - - for i in range(self.speculative_num_steps - 1): - forward_batch.spec_info.kv_indptr = self.kv_indptr[i, : bs + 1] - forward_batch.spec_info.kv_indices = kv_indices_buffer[i][ - : draft_kv_indices_used_len(seq_lens_sum, self.topk, bs, i + 1) - ] - global_override_indptr_cpu = indptr_cpu_whole[i] - call_fn(i, forward_batch) - - global_override_indptr_cpu = None + previous_override = global_override_indptr_cpu + try: + for i in range(self.speculative_num_steps - 1): + forward_batch.spec_info.kv_indptr = self.kv_indptr[i, : bs + 1] + forward_batch.spec_info.kv_indices = kv_indices_buffer[i][ + : draft_kv_indices_used_len(seq_lens_sum, self.topk, bs, i + 1) + ] + global_override_indptr_cpu = indptr_cpu_whole[i] + call_fn(i, forward_batch) + finally: + global_override_indptr_cpu = previous_override def init_forward_metadata(self, forward_batch: ForwardBatch): kv_indices_width = draft_kv_indices_buffer_width( diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py index fd9dfae582..453247fb21 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py @@ -26,7 +26,7 @@ from sglang.srt.layers.moe.moe_runner.base import ( ) from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import is_flashinfer_available -from sglang.srt.utils.common import next_power_of_2 +from sglang.srt.utils.common import is_sm120, next_power_of_2 if TYPE_CHECKING: from sglang.srt.layers.moe.token_dispatcher.flashinfer import ( @@ -59,6 +59,7 @@ class FlashInferCutlassMoeQuantInfo(MoeQuantInfo): swiglu_beta: Optional[torch.Tensor] = None swiglu_limit: Optional[torch.Tensor] = None apply_routed_scaling_factor: bool = True + pair_w2_weight: Optional[torch.Tensor] = None def materialize_swiglu_params_for_cutlass( @@ -250,6 +251,26 @@ def _prepare_input( return x, x_sf, output_dtype, output_col +def _sm120_mtp_mma_supported() -> bool: + import os + + # This implementation preserves FlashInfer's default FP4 quantizer and + # fused FC2 epilogue. Other quantization recipes retain the stock runner. + return ( + is_sm120() + and envs.SGLANG_FLASHINFER_MOE_FUSED_FINALIZE.get() + and not envs.FLASHINFER_NVFP4_4OVER6.get() + and os.environ.get("FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH", "0") == "0" + and os.environ.get("TRTLLM_DISABLE_FP4_QUANT_FAST_MATH", "0") == "0" + ) + + +def _flashinfer_autotune_active() -> bool: + from flashinfer.autotuner import AutoTuner + + return AutoTuner.get().is_tuning_mode + + def _run_flashinfer_cutlass( *, dispatch_output, @@ -268,6 +289,35 @@ def _run_flashinfer_cutlass( dispatch_output, quant_info, runner_config ) + if ( + quant_info.quant_type == "fp4" + and not runner_config.is_gated + and runner_config.activation == "relu2" + and runner_config.moe_ep_size == runner_config.moe_tp_size == 1 + and x_sf is None + and not enable_alltoall + and is_sm120() + ): + from sglang.kernels.ops.gemm.sm120_nvfp4_moe import try_nvfp4_moe + + result = try_nvfp4_moe( + x, + quant_info.w13_weight, + quant_info.w2_weight, + quant_info.quant_scales, + topk_ids, + topk_weights, + output_dtype=output_dtype, + out=output, + pair_w2_weight=quant_info.pair_w2_weight, + # Preserve FlashInfer target/draft tactic discovery during autotuning. + use_mtp=_sm120_mtp_mma_supported() and not _flashinfer_autotune_active(), + ) + if result is not None: + return _maybe_apply_routed_scaling_factor(result, quant_info, runner_config) + if quant_info.pair_w2_weight is not None: + raise RuntimeError("A paired NVFP4 weight reached an unsupported MoE path") + if output is None: with use_symmetric_memory( get_parallel().tp_group, disabled=not is_allocation_symmetric() diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index 3a30f3beef..0a84ce8fe0 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -3272,7 +3272,10 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): "apply_router_weight_on_input is not supported for Flashinfer" ) swiglu_alpha, swiglu_beta, swiglu_limit = layer._cutlass_swiglu_params + slot = getattr(layer, "_nvfp4_layout_slot", None) + pair_weight = slot[0].pair[slot[1]] if slot and slot[0].is_pair else None quant_info = FlashInferCutlassMoeQuantInfo( + pair_w2_weight=pair_weight, quant_type="fp4", w13_weight=layer.w13_weight, w2_weight=layer.w2_weight, diff --git a/python/sglang/srt/layers/quantization/unquant.py b/python/sglang/srt/layers/quantization/unquant.py index 4222073c40..f46c88ea48 100644 --- a/python/sglang/srt/layers/quantization/unquant.py +++ b/python/sglang/srt/layers/quantization/unquant.py @@ -317,8 +317,8 @@ def initialize_bf16_gemm_config() -> None: backend = Bf16GemmBackend(backend_str) if backend.is_gemv(): - if torch.cuda.get_device_capability()[0] != 9: - raise ValueError("--bf16-gemm-backend gemv requires SM90 (Hopper)") + if torch.cuda.get_device_capability()[0] not in (9, 12): + raise ValueError("--bf16-gemm-backend gemv requires SM90 (Hopper) or SM12x") global _hopper_bf16_gemv, _use_hopper_bf16_gemv from sglang.kernels.ops.gemm.hopper_bf16_gemv import ( @@ -589,7 +589,7 @@ class UnquantizedLinearMethod(LinearMethodBase): return tgemm.mm(x, layer.weight, bias, otype=x.dtype) elif ( - get_bf16_gemm_backend().is_cutedsl() + (get_bf16_gemm_backend().is_cutedsl() or get_bf16_gemm_backend().is_gemv()) and x.is_cuda and x.dtype == torch.bfloat16 and layer.weight.dtype == torch.bfloat16 diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 59b6eaf29a..ec85d24a66 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -1953,6 +1953,9 @@ class ModelRunner: reinit_attn_backend: bool = False, split_forward_count: int = 1, ) -> ModelRunnerOutput: + prepare = getattr(self.model, "prepare_for_forward", None) + if prepare is not None: + prepare(forward_batch) if has_forward_context(): ctx_mgr = contextlib.nullcontext() else: diff --git a/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py b/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py index 4bfd28f59e..49c8191fc3 100644 --- a/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py +++ b/python/sglang/srt/model_executor/model_runner_components/cuda_graph_setup.py @@ -96,6 +96,25 @@ def has_standard_gqa_for_all_local_layers( return attention_layer_count >= end_layer - start_layer +def has_hybrid_graph_handlers( + *, + attention_layers: list[Any], + moe_layers: list[Any], + start_layer: int, + end_layer: int, +) -> bool: + """Accept pure MoE layers while rejecting gaps in the local decoder range.""" + return ( + 0 <= start_layer < end_layer + and end_layer <= len(attention_layers) + and end_layer <= len(moe_layers) + and all( + attention_layers[i] is not None or moe_layers[i] is not None + for i in range(start_layer, end_layer) + ) + ) + + def index_attention_layers_by_global_id( attention_layers: list[Any], mha_companion_layers: list[Any], @@ -118,8 +137,13 @@ def index_attention_layers_by_global_id( ) return attention_layers, mha_companion_layers max_layer_id = max(int(layer.layer_id) for layer in populated) - indexed_attention = [None] * (max_layer_id + 1) - indexed_companions = [None] * (max_layer_id + 1) + # Keep trailing pure MoE layers represented as well. Their graph handler + # lives in moe_layers, and shrinking this list loses decoder alignment. + indexed_size = max( + max_layer_id + 1, len(layer_model.layers) if layer_model is not None else 0 + ) + indexed_attention = [None] * indexed_size + indexed_companions = [None] * indexed_size has_reused_layers = False for attention, companion in zip(attention_layers, mha_companion_layers): if attention is None: @@ -621,7 +645,17 @@ def capture_prefill_graph( layer_model, ) - if not has_standard_gqa_for_all_local_layers( + supports_hybrid_graph = ( + getattr(layer_model, "supports_hybrid_prefill_graph", False) + and prefill_backend == Backend.BREAKABLE + and has_hybrid_graph_handlers( + attention_layers=model_runner.attention_layers, + moe_layers=model_runner.moe_layers, + start_layer=model_runner.layer_info.start_layer, + end_layer=model_runner.layer_info.end_layer, + ) + ) + if not supports_hybrid_graph and not has_standard_gqa_for_all_local_layers( attention_layer_count=sum( layer is not None for layer in model_runner.attention_layers ), diff --git a/python/sglang/srt/model_executor/model_runner_components/weight_exporter.py b/python/sglang/srt/model_executor/model_runner_components/weight_exporter.py index 0d3983731e..f327e6d219 100644 --- a/python/sglang/srt/model_executor/model_runner_components/weight_exporter.py +++ b/python/sglang/srt/model_executor/model_runner_components/weight_exporter.py @@ -24,6 +24,12 @@ class WeightExporter: get_model: Callable[[], Any] _weights_send_group: dict = field(default_factory=dict) + def _check_weight_export_supported(self): + for module in self.get_model().modules(): + reason = getattr(module, "_weight_export_error", None) + if reason is not None: + raise RuntimeError(reason) + def init_weights_send_group_for_remote_instance( self, master_address, @@ -33,6 +39,7 @@ class WeightExporter: group_name, backend="nccl", ): + self._check_weight_export_supported() assert torch.distributed.is_initialized(), ( "Default torch process group must be initialized" ) @@ -81,6 +88,7 @@ class WeightExporter: ports, group_name, ): + self._check_weight_export_supported() assert torch.distributed.is_initialized(), ( "Default torch process group must be initialized" ) @@ -124,6 +132,7 @@ class WeightExporter: return success, message def save_remote_model(self: WeightExporter, url: str): + self._check_weight_export_supported() from sglang.srt.model_loader.loader import RemoteModelLoader logger.info(f"Saving model to {url}") @@ -135,6 +144,7 @@ class WeightExporter: pattern: Optional[str] = None, max_size: Optional[int] = None, ): + self._check_weight_export_supported() from sglang.srt.model_loader.loader import ShardedStateLoader logger.info( @@ -150,6 +160,7 @@ class WeightExporter: Only used for unit test with an unoptimized performance. For optimized performance, please use torch.save and torch.load. """ + self._check_weight_export_supported() # TODO: (chenyang) Add support for Qwen models. try: return self.get_model().get_weights_by_name( diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index c093f6e621..89c5f5816c 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -1709,6 +1709,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner): """ num_tokens = size forward_batch, attn_backend = self.capture_prepare(num_tokens) + # Recapture can follow decode replay, which may change model-owned storage. + prepare = getattr(self.model_runner.model, "prepare_for_forward", None) + if prepare is not None: + prepare(forward_batch) if self.enable_cp_bcg_capture: assert self.prefill_cp_bcg_input is not None self.prefill_cp_bcg_input.prepare( diff --git a/python/sglang/srt/models/nano_nemotron_vl.py b/python/sglang/srt/models/nano_nemotron_vl.py index cbedda5b06..c5a5b86495 100644 --- a/python/sglang/srt/models/nano_nemotron_vl.py +++ b/python/sglang/srt/models/nano_nemotron_vl.py @@ -447,6 +447,9 @@ class NemotronH_Omni_Reasoning_V3(NemotronH_Nano_VL_V2): else None ) + def prepare_for_forward(self, forward_batch: ForwardBatch) -> None: + self.language_model.prepare_for_forward(forward_batch) + @property def lm_head(self): return self.language_model.lm_head diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index c32b608f0a..d03994f4e7 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -18,7 +18,7 @@ """Inference-only NemotronH model.""" from collections.abc import Iterable -from functools import partial +from functools import lru_cache, partial import torch from torch import nn @@ -26,6 +26,7 @@ from torch import nn from sglang.srt.compilation.compilation_config import register_split_op from sglang.srt.configs import NemotronHConfig from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MLP, MOE +from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo @@ -65,7 +66,11 @@ from sglang.srt.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors +from sglang.srt.model_executor.forward_batch_info import ( + ForwardBatch, + ForwardMode, + PPProxyTensors, +) from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( @@ -84,12 +89,13 @@ from sglang.srt.model_loader.weight_utils import ( ) from sglang.srt.models.nemotron_h_utils import make_stage_boundary, stage_facts from sglang.srt.models.utils import WeightsMapper -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_exec, get_parallel, get_schedule from sglang.srt.utils import ( LazyValue, add_prefix, get_current_device_stream_fast, is_cuda, + is_sm120, make_pp_layers, ) from sglang.srt.utils.custom_op import register_custom_op @@ -157,6 +163,58 @@ def _get_or_create_alt_stream(device_module): return _alt_stream +@lru_cache(maxsize=None) +def _prefill_shared_expert_pool(device_index: int): + # Shared experts overlap the main graph, so their intermediates must use + # a separate pool. Calls on the shared stream are ordered across layers. + with torch.cuda.device(device_index): + return torch.cuda.graph_pool_handle() + + +class _PrefillSharedExpertCall: + def __init__(self, stream): + self.graph = None + self.stream = stream + self.ready = torch.cuda.Event(external=True) + self.done = torch.cuda.Event(external=True) + self.output = None + + def launch(self, module, hidden_states): + main_stream = torch.cuda.current_stream() + with torch.cuda.stream(self.stream): + if self.graph is None: + # BCG warmup follows the same dependencies, without capture. + self.ready.wait() + self.output = module(hidden_states) + self.done.record() + self.output.record_stream(main_stream) + else: + self.graph.replay() + return self.output + + +def _capture_prefill_shared_expert(module, hidden_states, call): + call.graph = torch.cuda.CUDAGraph() + with torch.cuda.graph( + call.graph, + pool=_prefill_shared_expert_pool(hidden_states.device.index), + stream=call.stream, + ): + call.ready.wait() + call.output = module(hidden_states) + call.done.record() + # The ready record belongs to the preceding main-stream graph. Do not + # replay here: that graph has only been captured, not launched yet. + return call.output + + +@eager_on_graph(True, capture_stub=_capture_prefill_shared_expert) +def _launch_prefill_shared_expert(module, hidden_states, call): + # BCG retains this call, its graph, events and output until graph cleanup. + # A replay launches on the same explicit stream for every layer. + return call.launch(module, hidden_states) + + def _latent_proj_fuses_shared_add(projection: nn.Module) -> bool: return ( # LoRA swaps a wrapper module over this attribute after init, @@ -288,7 +346,23 @@ class NemotronHMoE(nn.Module): self, hidden_states: torch.Tensor, num_token_non_padded: torch.Tensor | None, + forward_batch: ForwardBatch | None = None, ) -> tuple[torch.Tensor, torch.Tensor | None]: + if ( + forward_batch is not None + and forward_batch.forward_mode == ForwardMode.EXTEND + and hidden_states.shape == (1024, 4096) + and hidden_states.dtype == torch.bfloat16 + and get_parallel().tp_size == self.ep_size == 1 + and self.n_routed_experts == 512 + and self.moe_hidden_size == 1024 + and self.shared_experts is not None + and is_in_breakable_cuda_graph() + and is_sm120() + ): + return self._forward_core_prefill_overlap( + hidden_states, num_token_non_padded + ) if _is_cuda and ( not get_moe_a2a_backend().is_flashinfer() or get_is_capture_mode() ): @@ -363,6 +437,38 @@ class NemotronHMoE(nn.Module): return final_hidden_states, shared_output + def _forward_core_prefill_overlap( + self, + hidden_states: torch.Tensor, + num_token_non_padded: torch.Tensor | None, + ) -> tuple[torch.Tensor, torch.Tensor]: + # A multi-stream graph per hybrid segment creates one internal stream + # per executable. Keep each graph linear and launch shared experts on + # one explicit stream, preserving overlap with the routed experts. + call = _PrefillSharedExpertCall(_get_or_create_alt_stream(self.device_module)) + call.ready.record() + shared_output = _launch_prefill_shared_expert( + self.shared_experts, hidden_states, call + ) + router_logits = torch.mm( + hidden_states, self.gate.weight.t(), out_dtype=torch.float32 + ) + topk_output = self.topk( + hidden_states, + router_logits, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=( + ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id) + if not self.is_nextn + else None + ), + ) + if self.use_latent_moe: + hidden_states = self._apply_fc1_latent_proj(hidden_states) + final_hidden_states = self.experts(hidden_states, topk_output) + call.done.wait() + return final_hidden_states, shared_output + def get_moe_weights(self) -> list[torch.Tensor]: return [ x.data @@ -404,6 +510,7 @@ class NemotronHMoE(nn.Module): if forward_batch is not None else None ), + forward_batch=forward_batch, ) if self.use_latent_moe: @@ -760,6 +867,10 @@ class NemotronHModel(nn.Module): self.norm_f = PPMissingLayer(return_tuple=True) self.layers_to_capture: set[int] = set() + @property + def supports_hybrid_prefill_graph(self) -> bool: + return is_sm120() + def forward( self, input_ids: torch.Tensor, @@ -856,6 +967,7 @@ class NemotronHForCausalLM(nn.Module): ): super().__init__() lora_config = None + self._use_nvfp4_phase_layout = envs.SGLANG_SM120_NVFP4_PHASE_LAYOUT.get() self.config = config self.quant_config = quant_config self.model = self._init_model( @@ -1027,6 +1139,58 @@ class NemotronHForCausalLM(nn.Module): f"get_hidden_dim not implemented for {module_name}" ) + def prepare_for_forward(self, forward_batch: ForwardBatch) -> None: + if not getattr(self, "_use_nvfp4_phase_layout", False): + return + from sglang.kernels.ops.gemm.sm120_nvfp4_moe import Nvfp4LayoutCycle + from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import ( + _flashinfer_autotune_active, + _sm120_mtp_mma_supported, + ) + + cycle = getattr(self, "_nvfp4_layout", None) + if cycle is None: + if not ( + get_parallel().tp_size + == get_parallel().ep_size + == get_parallel().pp_size + == 1 + and get_schedule().max_running_requests == 1 + and _sm120_mtp_mma_supported() + ): + raise ValueError( + "NVFP4 phase layout requires serial SM120 CUTLASS MoE with TP/EP/PP=1" + ) + experts = [ + m.experts for m in self.model.modules() if isinstance(m, NemotronHMoE) + ] + if not experts or not all( + getattr(m.quant_method, "enable_flashinfer_cutlass_moe", False) + and m.w13_weight.shape == (512, 2688, 512) + for m in experts + ): + raise ValueError( + "NVFP4 phase layout requires compatible native NVFP4 experts" + ) + cycle = self._nvfp4_layout = Nvfp4LayoutCycle( + [m.w2_weight for m in experts] + ) + for index, module in enumerate(experts): + module._nvfp4_layout_slot = (cycle, index) + self._derived_weight_cache_error = ( + "Online weight updates require disabling NVFP4 phase layout" + ) + self._weight_export_error = ( + "Weight export requires disabling NVFP4 phase layout" + ) + + mode = forward_batch.forward_mode + cycle.ensure( + (mode.is_target_verify() or mode.is_decode_or_idle()) + and forward_batch.input_ids.numel() <= 4 + and not _flashinfer_autotune_active() + ) + @torch.no_grad() def forward( self, @@ -1036,6 +1200,7 @@ class NemotronHForCausalLM(nn.Module): input_embeds: torch.Tensor | None = None, pp_proxy_tensors: PPProxyTensors | None = None, ): + self.prepare_for_forward(forward_batch) hidden_states = self.model.forward( input_ids, positions, forward_batch, pp_proxy_tensors, input_embeds ) diff --git a/test/registered/kernels/ops/attention/test_flashinfer_host_plan.py b/test/registered/kernels/ops/attention/test_flashinfer_host_plan.py new file mode 100644 index 0000000000..b8d9064312 --- /dev/null +++ b/test/registered/kernels/ops/attention/test_flashinfer_host_plan.py @@ -0,0 +1,392 @@ +"""Host planning preserves live draft indices, verify masks and graph attention.""" + +import unittest +from functools import partial +from types import SimpleNamespace + +import flashinfer +import torch +from flashinfer.quantization import segment_packbits + +from sglang.kernels.ops.speculative.cache_locs import generate_draft_decode_kv_indices +from sglang.kernels.ops.speculative.pack_verify_mask import pack_verify_mask_into +from sglang.srt.environ import envs +from sglang.srt.layers.attention import flashinfer_backend as backend +from sglang.srt.layers.attention.flashinfer_backend import ( + FlashInferIndicesUpdaterPrefill, + fast_prefill_plan, +) +from sglang.srt.speculative.eagle_info import EagleVerifyInput +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=70, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +@unittest.skipUnless(torch.cuda.is_available(), "CUDA required") +class TestFlashInferDraftHostIndptr(CustomTestCase): + def _build(self, topk, steps, page_size, lengths): + n = len(lengths) + pool_len = 16640 + pool = torch.randperm((n + 1) * pool_len, device="cuda", dtype=torch.int32) + pool = pool.reshape(n + 1, pool_len) + obj = backend.FlashInferMultiStepDraftBackend.__new__( + backend.FlashInferMultiStepDraftBackend + ) + obj.topk = topk + obj.speculative_num_steps = steps + obj.generate_draft_decode_kv_indices = generate_draft_decode_kv_indices + obj.draft_window_size = obj.draft_sink_size = 0 + obj.kv_index_translator = SimpleNamespace( + read_source=lambda plan, **kw: SimpleNamespace( + row_ids=kw["req_pool_indices"], + ids=pool, + row_stride=pool_len, + entry_page_size=1, + v2p=None, + ) + ) + obj.page_size = page_size + obj.pool_len = pool_len + obj.req_to_token_pool = SimpleNamespace(req_to_token=pool) + obj.kv_indptr = torch.zeros( + (steps, n * topk + 1), device="cuda", dtype=torch.int32 + ) + indices = torch.full( + (steps, n * topk * pool_len), -17, device="cuda", dtype=torch.int32 + ) + host = torch.tensor(lengths, dtype=torch.int64) + seq = host.cuda() + fb = SimpleNamespace( + kv_loc_plan=None, + batch_size=n, + seq_lens=seq, + seq_lens_cpu=host, + seq_lens_sum=sum(lengths), + req_pool_indices=torch.arange(n, device="cuda", dtype=torch.int32), + positions=seq.repeat_interleave(topk), + spec_info=SimpleNamespace(is_draft_input=lambda: True), + ) + return obj, indices, fb + + def test_production_metadata_and_live_graph_after_length_rollback(self): + for topk, steps, page_size, initial in [ + (1, 3, 1, [8192]), + (1, 3, 64, [16300, 1, 32]), + (2, 5, 64, [17, 4097, 1]), + (4, 3, 1, [0, 8192]), + (1, 1, 1, [1]), + ]: + with self.subTest(topk=topk, steps=steps, page_size=page_size): + obj, indices, fb = self._build(topk, steps, page_size, initial) + observed = [] + + def plan(i, batch): + observed.append(backend.global_override_indptr_cpu.clone()) + + with envs.SGLANG_FLASHINFER_DRAFT_HOST_INDPTR.override(True): + obj.common_template(fb, indices, plan) + graph = torch.cuda.CUDAGraph() + consumer = torch.empty_like(indices) + consumer_indptr = torch.empty_like(obj.kv_indptr) + with torch.cuda.graph(graph): + consumer.copy_(indices) + consumer_indptr.copy_(obj.kv_indptr) + + for case in range(4): + # Shrink, grow, then remap cache slots; padded length-one + # rows and zero prefixes are mixed with long contexts. + live = [max(0, x - 7 * case) for x in initial] + if case == 2: + live = [min(16300, x + 13) for x in live] + fb.seq_lens_cpu.copy_(torch.tensor(live)) + fb.seq_lens.copy_(fb.seq_lens_cpu) + fb.seq_lens_sum = sum(live) + fb.positions.copy_(fb.seq_lens.repeat_interleave(topk)) + fb.req_pool_indices.add_(1).remainder_(len(initial) + 1) + indices.fill_(-17) + with envs.SGLANG_FLASHINFER_DRAFT_HOST_INDPTR.override(False): + obj.common_template(fb, indices, lambda i, batch: None) + original = indices.clone() + original_indptr = obj.kv_indptr.clone() + indices.fill_(-17) + observed.clear() + with envs.SGLANG_FLASHINFER_DRAFT_HOST_INDPTR.override(True): + obj.common_template(fb, indices, plan) + torch.testing.assert_close(indices, original, rtol=0, atol=0) + torch.testing.assert_close( + obj.kv_indptr, original_indptr, rtol=0, atol=0 + ) + for i, host in enumerate(observed): + torch.testing.assert_close( + host, original_indptr[i].cpu(), rtol=0, atol=0 + ) + graph.replay() + torch.testing.assert_close(consumer, original, rtol=0, atol=0) + torch.testing.assert_close( + consumer_indptr, original_indptr, rtol=0, atol=0 + ) + + def test_sliding_window_uses_bounded_device_metadata(self): + obj, indices, fb = self._build(1, 3, 1, [8192, 1024]) + obj.draft_window_size = 256 + obj.draft_sink_size = 16 + with envs.SGLANG_FLASHINFER_DRAFT_HOST_INDPTR.override(False): + obj.common_template(fb, indices, lambda i, batch: None) + expected = indices.clone() + expected_indptr = obj.kv_indptr.clone() + observed = [] + indices.fill_(-17) + with envs.SGLANG_FLASHINFER_DRAFT_HOST_INDPTR.override(True): + obj.common_template( + fb, + indices, + lambda i, batch: observed.append( + backend.global_override_indptr_cpu.clone() + ), + ) + torch.testing.assert_close(indices, expected, rtol=0, atol=0) + torch.testing.assert_close(obj.kv_indptr, expected_indptr, rtol=0, atol=0) + for i, host in enumerate(observed): + torch.testing.assert_close(host, expected_indptr[i].cpu(), rtol=0, atol=0) + + def test_missing_host_mirror_and_failed_plan_restore_override(self): + obj, indices, fb = self._build(1, 3, 1, [8192]) + with envs.SGLANG_FLASHINFER_DRAFT_HOST_INDPTR.override(False): + obj.common_template(fb, indices, lambda i, batch: None) + expected = indices.clone() + fb.seq_lens_cpu = None + indices.fill_(-17) + sentinel = torch.tensor([123], dtype=torch.int32) + previous = backend.global_override_indptr_cpu + backend.global_override_indptr_cpu = sentinel + try: + with envs.SGLANG_FLASHINFER_DRAFT_HOST_INDPTR.override(True): + obj.common_template(fb, indices, lambda i, batch: None) + self.assertIs(backend.global_override_indptr_cpu, sentinel) + torch.testing.assert_close(indices, expected, rtol=0, atol=0) + + def fail(i, batch): + raise RuntimeError("test plan failure") + + with self.assertRaisesRegex(RuntimeError, "test plan failure"): + obj.common_template(fb, indices, fail) + self.assertIs(backend.global_override_indptr_cpu, sentinel) + finally: + backend.global_override_indptr_cpu = previous + + +@unittest.skipUnless(torch.cuda.is_available(), "CUDA required") +class TestFlashInferVerifyHostPlan(CustomTestCase): + def test_packed_bits_and_padding(self): + for width, lengths in [(4, [8196]), (3, [1, 7, 8191]), (5, [13, 17, 16379])]: + with self.subTest(width=width, lengths=lengths): + indptr = ( + torch.tensor([0] + lengths, dtype=torch.int32) + .cumsum(0) + .int() + .cuda() + ) + mask = torch.randint( + 0, 2, (sum(lengths) * width,), device="cuda" + ).bool() + expected, packed_indptr = segment_packbits( + mask, indptr * width, bitorder="little" + ) + result = torch.full( + (expected.numel() + 256,), 173, dtype=torch.uint8, device="cuda" + ) + pack_verify_mask_into( + mask, + indptr, + packed_indptr, + result, + batch_size=len(lengths), + query_width=width, + max_kv_len=max(lengths), + ) + torch.testing.assert_close( + result[: expected.numel()], expected, rtol=0, atol=0 + ) + self.assertTrue(torch.all(result[expected.numel() :] == 173).item()) + + def test_real_wrapper_live_lengths_and_masks(self): + torch.manual_seed(20261008) + for dtype, batch in [(torch.bfloat16, 1), (torch.float8_e4m3fn, 3)]: + with self.subTest(dtype=dtype, batch=batch): + width, capacity, heads, kv_heads, dim = 4, 9216, 32, 2, 128 + q = torch.randn( + batch * width, heads, dim, device="cuda", dtype=torch.bfloat16 + ) + kv = torch.randn( + capacity, 2, 1, kv_heads, dim, device="cuda", dtype=torch.bfloat16 + ).to(dtype) + wrappers = [] + for _ in range(2): + wrappers.append( + flashinfer.BatchPrefillWithPagedKVCacheWrapper( + torch.empty( + 512 * 1024 * 1024, dtype=torch.uint8, device="cuda" + ), + use_cuda_graph=True, + qo_indptr_buf=torch.empty( + batch + 1, dtype=torch.int32, device="cuda" + ), + paged_kv_indptr_buf=torch.empty( + batch + 1, dtype=torch.int32, device="cuda" + ), + paged_kv_indices_buf=torch.empty( + capacity * batch, dtype=torch.int32, device="cuda" + ), + paged_kv_last_page_len_buf=torch.ones( + batch, dtype=torch.int32, device="cuda" + ), + custom_mask_buf=torch.full( + (capacity * batch * width,), + 173, + dtype=torch.uint8, + device="cuda", + ), + mask_indptr_buf=torch.empty( + batch + 1, dtype=torch.int32, device="cuda" + ), + backend="fa2", + ) + ) + graphs, outputs = [], [] + for step, first in enumerate([8196, 8189, 8213, 13, 9213]): + lengths = [max(width, first - i * 13) for i in range(batch)] + host_lens = torch.tensor(lengths, dtype=torch.int32) + host_kv = torch.cat( + [torch.zeros(1, dtype=torch.int32), host_lens.cumsum(0).int()] + ) + host_q = torch.arange(batch + 1, dtype=torch.int32) * width + gpu_kv, gpu_q = host_kv.cuda(), host_q.cuda() + indices = torch.cat( + [ + torch.randperm(capacity, device="cuda", dtype=torch.int32)[ + :n + ] + for n in lengths + ] + ) + masks = [] + for n in lengths: + # Actual live masks may differ after rejection. Random + # branch visibility exercises more than a causal mask. + row = torch.rand(width, n, device="cuda") > 0.15 + row[:, 0] = True + row[:, -width:] = torch.ones( + width, width, device="cuda", dtype=torch.bool + ).tril() + masks.append(row.flatten()) + mask = torch.cat(masks) + last = torch.ones(batch, dtype=torch.int32, device="cuda") + args = (gpu_q, gpu_kv, indices, last, heads, kv_heads, dim, 1) + kwargs = { + "q_data_type": q.dtype, + "kv_data_type": dtype, + "custom_mask": mask, + } + wrappers[0].plan(*args, **kwargs) + if step == 0: + wrappers[1].plan(*args, **kwargs) + wrappers[1].begin_forward = partial( + fast_prefill_plan, wrappers[1] + ) + else: + # Exercise the SGLang entry point, including the + # committed-prefix + candidate-window host layout. + pool = torch.empty( + batch, capacity, dtype=torch.int32, device="cuda" + ) + for i, n in enumerate(lengths): + pool[i, :n] = indices[int(host_kv[i]) : int(host_kv[i + 1])] + spec = EagleVerifyInput.__new__(EagleVerifyInput) + from sglang.srt.speculative.spec_info import SpecInputType + + spec.spec_input_type = SpecInputType.EAGLE_VERIFY + spec.custom_mask = mask + spec.draft_token_num = spec.num_tokens_per_req = width + updater = FlashInferIndicesUpdaterPrefill.__new__( + FlashInferIndicesUpdaterPrefill + ) + + def pack_read_stream(plan, **kw): + lengths_cpu = kw["seq_lens"].cpu().tolist() + rows_cpu = kw["req_pool_indices"].cpu().tolist() + kw["out"].copy_( + torch.cat( + [ + pool[row, :n] + for row, n in zip(rows_cpu, lengths_cpu) + ] + ) + ) + + updater.attn_backend = SimpleNamespace( + kv_index_translator=SimpleNamespace( + reads_are_translated=False, + pack_read_stream=pack_read_stream, + ) + ) + updater.req_to_token = pool + updater.kv_last_page_len = last + updater.num_qo_heads, updater.num_kv_heads = heads, kv_heads + updater.head_dim, updater.q_data_type, updater.data_type = ( + dim, + q.dtype, + dtype, + ) + committed = host_lens - width + updater.call_begin_forward( + None, + wrappers[1], + torch.arange(batch, dtype=torch.int32, device="cuda"), + committed.cuda(), + int(committed.sum()), + committed.cuda(), + None, + None, + gpu_kv, + gpu_q, + False, + spec, + seq_lens_cpu=committed, + plan=None, + ) + expected_bits, expected_ptr = segment_packbits( + mask, gpu_kv * width, bitorder="little" + ) + self.assertEqual(wrappers[0]._plan_info, wrappers[1]._plan_info) + for wrapper in wrappers: + torch.testing.assert_close( + wrapper._mask_indptr_buf, expected_ptr, rtol=0, atol=0 + ) + torch.testing.assert_close( + wrapper._custom_mask_buf[: expected_bits.numel()], + expected_bits, + rtol=0, + atol=0, + ) + torch.testing.assert_close( + wrappers[0].run(q, kv), wrappers[1].run(q, kv), rtol=0, atol=0 + ) + if step == 0: + for wrapper in wrappers: + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = wrapper.run(q, kv) + graphs.append(graph) + outputs.append(out) + q.normal_() + for graph in graphs: + graph.replay() + torch.testing.assert_close(outputs[0], outputs[1], rtol=0, atol=0) + torch.testing.assert_close( + outputs[1], wrappers[0].run(q, kv), rtol=0, atol=0 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/moe/test_sm120_topk_gate_small_m.py b/test/registered/kernels/ops/moe/test_sm120_topk_gate_small_m.py new file mode 100644 index 0000000000..d1d89755d9 --- /dev/null +++ b/test/registered/kernels/ops/moe/test_sm120_topk_gate_small_m.py @@ -0,0 +1,122 @@ +import os +import unittest +from unittest.mock import patch + +import torch + +from sglang.kernels.ops.moe import moe_fused_gate as gate +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="1-gpu-small") + + +@unittest.skipUnless( + torch.cuda.is_available() and torch.cuda.get_device_capability() == (12, 0), + "SM120 required", +) +class TestSmallMTop22(CustomTestCase): + def route(self, scores, bias, enabled, **kwargs): + with patch.dict( + os.environ, {"SGLANG_SM120_MTP_ROUTER_TOP22": str(int(enabled))} + ): + return gate.moe_fused_gate(scores, bias, topk=22, **kwargs) + + def assert_bits(self, actual, expected): + for a, b in zip(actual, expected): + self.assertTrue(torch.equal(a.view(torch.int32), b.view(torch.int32))) + + def test_sorted_ties_weights_and_live_graph(self): + torch.manual_seed(73) + for m in range(2, 17): + scores = torch.randn(m, 512, device="cuda") + bias = torch.randn(512, device="cuda") + for case in range(5): + with self.subTest(m=m, case=case): + if case == 1: + scores.zero_() + bias.zero_() + elif case == 2: + scores[:, ::2] = 200 + scores[:, 1::2] = -200 + bias.mul_(1e-8) + elif case == 3: + scores.normal_() + scores[:, :25] = float("nan") + bias[25:50] = -float("inf") + bias[50:75] = float("inf") + elif case == 4: + scores.zero_() + bias.fill_(-float("inf")) + bias[5:10] = 0 + reference = self.route(scores, bias, False) + # Prove that the production wrapper takes the new kernel. + with patch.object(gate, "_router_triton_kernel", None): + self.assert_bits(self.route(scores, bias, True), reference) + scores.normal_() + bias.normal_() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + result = self.route(scores, bias, True) + for _ in range(3): + scores.normal_() + bias.normal_() + graph.replay() + self.assert_bits(result, self.route(scores, bias, False)) + + def test_normalization_scaling_and_fallback(self): + scores = torch.randn(4, 512, device="cuda") + bias = torch.randn(512, device="cuda") + for norm in (False, True): + for scale in (False, True): + kwargs = dict( + renormalize=norm, + routed_scaling_factor=5.0, + apply_routed_scaling_factor_on_output=scale, + ) + self.assert_bits( + self.route(scores, bias, True, **kwargs), + self.route(scores, bias, False, **kwargs), + ) + for m in (1, 17): + x = torch.randn(m, 512, device="cuda") + with patch.object(gate, "_topk_gate_small_m", side_effect=AssertionError): + self.assert_bits(self.route(x, bias, True), self.route(x, bias, False)) + with patch.object(gate, "_topk_gate_small_m", side_effect=AssertionError): + self.assert_bits( + self.route(scores.bfloat16(), bias, True), + self.route(scores.bfloat16(), bias, False), + ) + + def test_new_router_options_fall_back(self): + scores = torch.randn(4, 512, device="cuda") + bias = torch.randn(512, device="cuda") + cases = [ + dict(renormalize_epsilon=0.1), + dict( + num_token_non_padded=torch.tensor(2, device="cuda", dtype=torch.int32) + ), + dict( + input_ids=torch.tensor([0, 1, 0, 1], device="cuda"), + bias_alt=-bias, + bias_alt_token_id=1, + ), + dict(packed_out=torch.empty((4, 22), device="cuda", dtype=torch.int32)), + ] + for kwargs in cases: + with ( + self.subTest(options=list(kwargs)), + patch.object(gate, "_topk_gate_small_m", side_effect=AssertionError), + ): + expected = self.route(scores, bias, False, **kwargs) + packed = kwargs.get("packed_out") + expected_packed = packed.clone() if packed is not None else None + if packed is not None: + packed.zero_() + self.assert_bits(self.route(scores, bias, True, **kwargs), expected) + if packed is not None: + self.assertTrue(torch.equal(packed, expected_packed)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/moe/test_moe_runner_config_placement.py b/test/registered/unit/layers/moe/test_moe_runner_config_placement.py index 1384ac833e..4ad03fd52e 100644 --- a/test/registered/unit/layers/moe/test_moe_runner_config_placement.py +++ b/test/registered/unit/layers/moe/test_moe_runner_config_placement.py @@ -143,5 +143,57 @@ def test_cutlass_passes_the_runner_config_placement(monkeypatch) -> None: assert placement == (2, 1, 4, 3) +def test_sm120_nvfp4_dispatch_uses_live_layer_placement(monkeypatch) -> None: + from sglang.kernels.ops.gemm import sm120_nvfp4_moe + + calls = [] + monkeypatch.setattr(flashinfer_cutlass, "is_sm120", lambda: True) + monkeypatch.setattr(flashinfer_cutlass, "_sm120_mtp_mma_supported", lambda: True) + monkeypatch.setattr( + flashinfer_cutlass, "_flashinfer_autotune_active", lambda: False + ) + + def native(*args, **kwargs): + calls.append("native") + return kwargs["out"] + + def fallback(**kwargs): + calls.append((kwargs["tp_size"], kwargs["ep_size"])) + return [kwargs["output"]] + + monkeypatch.setattr(sm120_nvfp4_moe, "nvfp4_moe_mtp_mma", native) + monkeypatch.setattr( + flashinfer_cutlass, "_flashinfer_cutlass_fused_moe", lambda: (fallback, None) + ) + monkeypatch.setattr(flashinfer_cutlass, "_activation_type", lambda config: None) + layer = SimpleNamespace(moe_tp_size=1, moe_tp_rank=0, moe_ep_size=1, moe_ep_rank=0) + config = MoeRunnerConfig(activation="relu2", is_gated=False, layer=layer) + quant = FlashInferCutlassMoeQuantInfo( + "fp4", + torch.empty((512, 2688, 512), device="meta", dtype=torch.uint8), + torch.empty((512, 1024, 1344), device="meta", dtype=torch.uint8), + quant_scales=[torch.ones(1)] * 6, + output_dtype=torch.bfloat16, + ) + x = torch.zeros((1, 1024), dtype=torch.bfloat16) + dispatch = SimpleNamespace( + hidden_states=x, + hidden_states_scale=None, + topk_output=SimpleNamespace( + topk_ids=torch.zeros((1, 22), dtype=torch.int32), + topk_weights=torch.ones((1, 22)), + ), + ) + for tp, ep in [(1, 1), (2, 1), (1, 2)]: + layer.moe_tp_size, layer.moe_ep_size = tp, ep + flashinfer_cutlass._run_flashinfer_cutlass( + dispatch_output=dispatch, + quant_info=quant, + runner_config=config, + output=torch.empty_like(x), + ) + assert calls == ["native", (2, 1), (1, 2)] + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"]))