[Feat] nemotron: Port Nemotron 3.5 Super serving changes (#44518)

Co-authored-by: BBuf <1182563586@qq.com>
Co-authored-by: elvischenv <219235043+elvischenv@users.noreply.github.com>
Co-authored-by: Jimmy Shong <jimmy.shong@mac.attlocal.net>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jimmy Shong
2026-10-19 13:27:22 +08:00
committed by GitHub
co-authored by BBuf elvischenv Jimmy Shong Claude Opus 5.5
parent f726461287
commit 7c9a78efbe
23 changed files with 1825 additions and 58 deletions
@@ -37,12 +37,14 @@ __device__ __forceinline__ float dot8_f32(const float4 wv, const float4 xv) {
return acc;
}
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps>
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps, bool kUsePDL>
__global__ void __launch_bounds__(kNumWarps * 32)
hopper_bf16_gemv_kernel(bf16_t* __restrict__ out, const bf16_t* __restrict__ x, const bf16_t* __restrict__ w) {
__shared__ bf16_t sx[K];
PDLTriggerSecondary<kUsePDL>();
const uint32_t tid = threadIdx.x;
PDLWaitPrimary<kUsePDL>();
for (uint32_t i = tid * kGemvVecSize; i < K; i += kNumWarps * 32 * kGemvVecSize) {
*reinterpret_cast<float4*>(sx + i) = *reinterpret_cast<const float4*>(x + i);
}
@@ -118,9 +120,11 @@ __global__ void __launch_bounds__(kNumWarps * 32)
}
}
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps>
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps, bool kUsePDL>
struct HopperBf16GemvKernel {
static_assert(K % (32 * kGemvVecSize * kUnroll) == 0, "K must cover full unrolled warp strides");
static_assert(
kUnroll == 1 ? (K % kGemvVecSize == 0) : (K % (32 * kGemvVecSize * kUnroll) == 0),
"K must be 16B-aligned (kUnroll==1) or cover full unrolled warp strides");
static_assert(K * sizeof(bf16_t) <= 48 * 1024, "activation row must fit static shared memory");
static void run(const tvm::ffi::TensorView x, const tvm::ffi::TensorView w, const tvm::ffi::TensorView out) {
@@ -134,11 +138,12 @@ struct HopperBf16GemvKernel {
constexpr uint32_t kRowsPerBlock = kRows * kNumWarps;
constexpr uint32_t kNumBlocks = (N + kRowsPerBlock - 1) / kRowsPerBlock;
LaunchKernel(kNumBlocks, kNumWarps * 32, device.unwrap())(
hopper_bf16_gemv_kernel<N, K, kRows, kUnroll, kNumWarps>,
static_cast<bf16_t*>(out.data_ptr()),
static_cast<const bf16_t*>(x.data_ptr()),
static_cast<const bf16_t*>(w.data_ptr()));
LaunchKernel(kNumBlocks, kNumWarps * 32, device.unwrap())
.enable_pdl(kUsePDL)(
hopper_bf16_gemv_kernel<N, K, kRows, kUnroll, kNumWarps, kUsePDL>,
static_cast<bf16_t*>(out.data_ptr()),
static_cast<const bf16_t*>(x.data_ptr()),
static_cast<const bf16_t*>(w.data_ptr()));
}
};
@@ -39,7 +39,7 @@ __device__ __forceinline__ float dot16_fp8_f32(const float4 wv, const float4 xv)
return acc;
}
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps>
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps, bool kUsePDL>
__global__ void __launch_bounds__(kNumWarps * 32) sm120_fp8_gemv_kernel(
bf16_t* __restrict__ out,
const uint8_t* __restrict__ x,
@@ -47,7 +47,9 @@ __global__ void __launch_bounds__(kNumWarps * 32) sm120_fp8_gemv_kernel(
const float* __restrict__ alpha) {
__shared__ uint8_t sx[K];
PDLTriggerSecondary<kUsePDL>();
const uint32_t tid = threadIdx.x;
PDLWaitPrimary<kUsePDL>();
for (uint32_t i = tid * kFp8VecSize; i < K; i += kNumWarps * 32 * kFp8VecSize) {
*reinterpret_cast<float4*>(sx + i) = *reinterpret_cast<const float4*>(x + i);
}
@@ -123,9 +125,11 @@ __global__ void __launch_bounds__(kNumWarps * 32) sm120_fp8_gemv_kernel(
}
}
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps>
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps, bool kUsePDL>
struct Sm120Fp8GemvKernel {
static_assert(K % (32 * kFp8VecSize * kUnroll) == 0, "K must cover full unrolled warp strides");
static_assert(
kUnroll == 1 ? (K % kFp8VecSize == 0) : (K % (32 * kFp8VecSize * kUnroll) == 0),
"K must be 16B-aligned (kUnroll==1) or cover full unrolled warp strides");
static_assert(K <= 48 * 1024, "activation row must fit static shared memory");
static void
@@ -144,12 +148,13 @@ struct Sm120Fp8GemvKernel {
constexpr uint32_t kRowsPerBlock = kRows * kNumWarps;
constexpr uint32_t kNumBlocks = (N + kRowsPerBlock - 1) / kRowsPerBlock;
LaunchKernel(kNumBlocks, kNumWarps * 32, device.unwrap())(
sm120_fp8_gemv_kernel<N, K, kRows, kUnroll, kNumWarps>,
static_cast<bf16_t*>(out.data_ptr()),
static_cast<const uint8_t*>(x.data_ptr()),
static_cast<const uint8_t*>(w.data_ptr()),
static_cast<const float*>(alpha.data_ptr()));
LaunchKernel(kNumBlocks, kNumWarps * 32, device.unwrap())
.enable_pdl(kUsePDL)(
sm120_fp8_gemv_kernel<N, K, kRows, kUnroll, kNumWarps, kUsePDL>,
static_cast<bf16_t*>(out.data_ptr()),
static_cast<const uint8_t*>(x.data_ptr()),
static_cast<const uint8_t*>(w.data_ptr()),
static_cast<const float*>(alpha.data_ptr()));
}
};
@@ -0,0 +1,410 @@
/***************************************************************************************************
* Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
// Native SM120 NVFP4 MMA operand layout follows NVIDIA CUTLASS mma_sm120.hpp
// and mma_traits_sm120.hpp, distributed under BSD-3-Clause. Exact installed-source hashes are recorded in the research
// task history/moe-mtp-native-upstream.json.
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
#include <cstdint>
#include <cuda_fp8.h>
namespace sglang {
namespace nvfp4_mtp_mma {
using namespace device;
__device__ __forceinline__ float rcp(float x) {
float y;
asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x));
return y;
}
__device__ __forceinline__ uint32_t fp4x2(float hi, float lo) {
uint32_t d;
asm volatile("{ .reg .b8 t; cvt.rn.satfinite.e2m1x2.f32 t, %1, %2; cvt.u32.u8 %0, t; }" : "=r"(d) : "f"(hi), "f"(lo));
return d;
}
__device__ __forceinline__ void quantize16(const float* v, uint32_t& q0, uint32_t& q1, uint8_t& sf, float global) {
float amax = 0.f;
#pragma unroll
for (uint32_t i = 0; i < 16; ++i)
amax = fmaxf(amax, fabsf(v[i]));
__nv_fp8_e4m3 f(global * (amax * rcp(6.f)));
sf = f.__x;
float scale = amax != 0.f ? rcp(static_cast<float>(f) * rcp(global)) : 0.f;
q0 = 0;
q1 = 0;
#pragma unroll
for (uint32_t i = 0; i < 4; ++i) {
q0 |= fp4x2(v[2 * i + 1] * scale, v[2 * i] * scale) << (8 * i);
q1 |= fp4x2(v[8 + 2 * i + 1] * scale, v[8 + 2 * i] * scale) << (8 * i);
}
}
template <uint32_t G>
__device__ __forceinline__ uint32_t sf_index(uint32_t row, uint32_t group) {
return (row / 128) * (128 * G) + (row % 32) * 16 + ((row % 128) / 32) * 4 + (group / 4) * 512 + group % 4;
}
__device__ __forceinline__ void
mma(float& d0,
float& d1,
float& d2,
float& d3,
uint32_t a0,
uint32_t a1,
uint32_t a2,
uint32_t a3,
uint32_t b0,
uint32_t b1,
uint32_t sfa,
uint32_t sfb) {
asm volatile(
"mma.sync.aligned.kind::mxf4nvf4.block_scale.scale_vec::4X.m16n8k64.row.col.f32.e2m1.e2m1.f32.ue4m3 "
"{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3}, {%10}, {0,0}, {%11}, {0,0};"
: "+f"(d0), "+f"(d1), "+f"(d2), "+f"(d3)
: "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1), "r"(sfa), "r"(sfb));
}
// Stage 256 K elements in contiguous 128B row segments. Padding each shared
// row by 16B avoids the eight-way ldmatrix bank conflict at a 128B stride.
template <uint32_t K>
__device__ __forceinline__ void
prefetch(uint8_t* shared, const uint8_t* weights, uint32_t tile, uint32_t step, uint32_t lane) {
#pragma unroll
for (uint32_t j = 0; j < 4; ++j) {
uint32_t row = lane / 8 + 4 * j, col = (lane % 8) * 16;
uint32_t dst = static_cast<uint32_t>(__cvta_generic_to_shared(shared + row * 144 + col));
uint32_t source_col = step * 128 + col;
uint32_t bytes = source_col < K / 2 ? 16 : 0;
const uint8_t* src = weights + (tile + row) * (K / 2) + (bytes ? source_col : 0);
asm volatile("cp.async.cg.shared.global [%0], [%1], 16, %2;" ::"r"(dst), "l"(src), "r"(bytes) : "memory");
}
asm volatile("cp.async.commit_group;" ::: "memory");
}
__device__ __forceinline__ void
load_a(uint32_t& a0, uint32_t& a1, uint32_t& a2, uint32_t& a3, const uint8_t* shared, uint32_t lane) {
uint32_t addr = static_cast<uint32_t>(__cvta_generic_to_shared(shared + (lane % 16) * 144 + (lane / 16) * 16));
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];"
: "=r"(a0), "=r"(a1), "=r"(a2), "=r"(a3)
: "r"(addr)
: "memory");
}
template <uint32_t N, uint32_t K, uint32_t TopK, bool PDL>
__global__ __launch_bounds__(128) void fc1(
const bf16_t* __restrict__ x,
const uint8_t* __restrict__ w,
const uint8_t* __restrict__ sf,
const float* __restrict__ alpha,
const float* __restrict__ global1,
const float* __restrict__ global2,
const int32_t* __restrict__ ids,
uint32_t* __restrict__ z,
uint8_t* __restrict__ zsf) {
__shared__ uint32_t xq[K / 8];
__shared__ uint8_t xs[K / 16];
__shared__ float act[4][16];
__shared__ __align__(16) uint8_t ws[4][2][2304];
uint32_t tid = threadIdx.x, warp = tid / 32, lane = tid % 32;
uint32_t token = blockIdx.z, slot = blockIdx.y;
PDLTriggerSecondary<PDL>();
PDLWaitPrimary<PDL>();
if (tid < K / 16) {
float v[16];
#pragma unroll
for (uint32_t i = 0; i < 16; ++i)
v[i] = static_cast<float>(x[token * K + tid * 16 + i]);
quantize16(v, xq[2 * tid], xq[2 * tid + 1], xs[tid], *global1);
}
__syncthreads();
uint32_t expert = ids[token * TopK + slot];
uint32_t tile = (blockIdx.x * 4 + warp) * 16;
uint32_t row = tile + lane / 4, col = (lane % 4) * 4;
const uint8_t* weights = w + static_cast<size_t>(expert) * N * K / 2;
const uint8_t* scales = sf + static_cast<size_t>(expert) * N * (K / 16);
float d0 = 0, d1 = 0, d2 = 0, d3 = 0;
#pragma unroll
for (uint32_t stage = 0; stage < 2; ++stage)
prefetch<K>(ws[warp][stage], weights, tile, stage, lane);
#pragma unroll 1
for (uint32_t base = 0; base < K; base += 256) {
asm volatile("cp.async.wait_group 1;" ::: "memory");
__syncwarp();
#pragma unroll
for (uint32_t inner = 0; inner < 256; inner += 64) {
if (base + inner >= K) break;
uint32_t k = base + inner;
uint32_t a0, a1, a2, a3;
load_a(a0, a1, a2, a3, ws[warp][(base / 256) % 2] + inner / 2, lane);
uint32_t sa =
*reinterpret_cast<const uint32_t*>(scales + sf_index<K / 16>(tile + lane / 4 + (lane % 2) * 8, k / 16));
uint32_t sb = *reinterpret_cast<const uint32_t*>(xs + k / 16);
mma(d0, d1, d2, d3, a0, a1, a2, a3, xq[k / 8 + lane % 4], xq[k / 8 + lane % 4 + 4], sa, sb);
}
__syncwarp();
prefetch<K>(ws[warp][(base / 256) % 2], weights, tile, base / 256 + 2, lane);
}
asm volatile("cp.async.wait_group 0;" ::: "memory");
if (lane % 4 == 0) {
float y0 = static_cast<float>(cast<bf16_t>(d0 * alpha[expert]));
float y1 = static_cast<float>(cast<bf16_t>(d2 * alpha[expert]));
y0 = fmaxf(y0, 0.f);
y1 = fmaxf(y1, 0.f);
act[warp][lane / 4] = static_cast<float>(cast<bf16_t>(y0 * y0));
act[warp][lane / 4 + 8] = static_cast<float>(cast<bf16_t>(y1 * y1));
}
__syncwarp();
if (lane == 0) {
size_t qoff = (token * TopK + slot) * (N / 8) + tile / 8;
size_t soff = (token * TopK + slot) * (N / 16) + tile / 16;
quantize16(act[warp], z[qoff], z[qoff + 1], zsf[soff], *global2);
}
}
template <uint32_t N, uint32_t K, uint32_t TopK, bool PDL>
__global__ __launch_bounds__(128) void fc2(
const uint32_t* __restrict__ z,
const uint8_t* __restrict__ zs,
const uint8_t* __restrict__ w,
const uint8_t* __restrict__ sf,
const float* __restrict__ alpha,
const int32_t* __restrict__ ids,
const float* __restrict__ routing,
bf16_t* __restrict__ partial) {
__shared__ __align__(16) uint8_t ws[4][2][2304];
uint32_t warp = threadIdx.x / 32, lane = threadIdx.x % 32;
uint32_t token = blockIdx.z, slot = blockIdx.y, route = token * TopK + slot;
PDLTriggerSecondary<PDL>();
PDLWaitPrimary<PDL>();
uint32_t expert = ids[route], tile = (blockIdx.x * 4 + warp) * 16;
uint32_t row = tile + lane / 4, col = (lane % 4) * 4;
const uint8_t* weights = w + static_cast<size_t>(expert) * N * K / 2;
const uint8_t* scales = sf + static_cast<size_t>(expert) * N * (K / 16);
z += route * (K / 8);
zs += route * (K / 16);
float d0 = 0, d1 = 0, d2 = 0, d3 = 0;
#pragma unroll
for (uint32_t stage = 0; stage < 2; ++stage)
prefetch<K>(ws[warp][stage], weights, tile, stage, lane);
#pragma unroll 1
for (uint32_t base = 0; base < K; base += 256) {
asm volatile("cp.async.wait_group 1;" ::: "memory");
__syncwarp();
#pragma unroll
for (uint32_t inner = 0; inner < 256; inner += 64) {
if (base + inner >= K) break;
uint32_t k = base + inner;
uint32_t a0, a1, a2, a3;
load_a(a0, a1, a2, a3, ws[warp][(base / 256) % 2] + inner / 2, lane);
uint32_t sa =
*reinterpret_cast<const uint32_t*>(scales + sf_index<K / 16>(tile + lane / 4 + (lane % 2) * 8, k / 16));
uint32_t sb = *reinterpret_cast<const uint32_t*>(zs + k / 16);
mma(d0, d1, d2, d3, a0, a1, a2, a3, z[k / 8 + lane % 4], z[k / 8 + lane % 4 + 4], sa, sb);
}
__syncwarp();
prefetch<K>(ws[warp][(base / 256) % 2], weights, tile, base / 256 + 2, lane);
}
asm volatile("cp.async.wait_group 0;" ::: "memory");
if (lane % 4 == 0) {
partial[route * N + row] = cast<bf16_t>((d0 * alpha[expert]) * routing[route]);
partial[route * N + row + 8] = cast<bf16_t>((d2 * alpha[expert]) * routing[route]);
}
}
template <uint32_t N, uint32_t TopK, bool PDL>
__global__ void reduce(const bf16_t* __restrict__ partial, bf16_t* __restrict__ out, uint32_t rows) {
PDLTriggerSecondary<PDL>();
uint32_t i = blockIdx.x * blockDim.x + threadIdx.x;
PDLWaitPrimary<PDL>();
if (i < rows * N) {
uint32_t token = i / N, col = i % N;
float value = 0.f;
#pragma unroll
for (uint32_t slot = 0; slot < TopK; ++slot)
value += static_cast<float>(partial[(token * TopK + slot) * N + col]);
out[i] = cast<bf16_t>(value);
}
}
} // namespace nvfp4_mtp_mma
namespace nvfp4_pair_mma {
using namespace nvfp4_mtp_mma;
// Store each original pair of FP4 weight bits across contiguous output channels.
// One 16-byte fetch then corresponds to exactly two activation values, enabling
// zero-filled asynchronous loads without discarding any nonzero multiplication.
template <unsigned N, unsigned K>
__device__ __forceinline__ void
pair_prefetch(uint8_t* shared, const uint8_t* weights, unsigned tile, unsigned step, unsigned lane, const uint32_t* z) {
#pragma unroll
for (unsigned j = 0; j < 2; ++j) {
unsigned pair = lane + 32 * j, source_pair = step * 64 + pair;
unsigned bytes = source_pair < K / 2 ? 16 : 0;
if (bytes && (reinterpret_cast<const uint8_t*>(z)[source_pair] & 0x77u) == 0) bytes = 0;
const uint8_t* src = weights + (bytes ? source_pair : 0) * N + tile;
unsigned dst = static_cast<unsigned>(__cvta_generic_to_shared(shared + pair * 16));
asm volatile("cp.async.cg.shared.global [%0], [%1], 16, %2;" ::"r"(dst), "l"(src), "r"(bytes) : "memory");
}
asm volatile("cp.async.commit_group;" ::: "memory");
}
__device__ __forceinline__ void
pair_load_a(unsigned& a0, unsigned& a1, unsigned& a2, unsigned& a3, const uint8_t* s, unsigned lane) {
unsigned addr = static_cast<unsigned>(__cvta_generic_to_shared(s + lane * 16));
asm volatile("ldmatrix.sync.aligned.m16n16.x2.trans.shared.b8 {%0,%1,%2,%3}, [%4];"
: "=r"(a0), "=r"(a1), "=r"(a2), "=r"(a3)
: "r"(addr)
: "memory");
}
template <uint32_t N, uint32_t K, uint32_t TopK, bool PDL>
__global__ __launch_bounds__(128) void fc2(
const uint32_t* __restrict__ z,
const uint8_t* __restrict__ zs,
const uint8_t* __restrict__ w,
const uint8_t* __restrict__ sf,
const float* __restrict__ alpha,
const int32_t* __restrict__ ids,
const float* __restrict__ routing,
bf16_t* __restrict__ partial) {
__shared__ __align__(16) uint8_t ws[4][2][1024];
uint32_t warp = threadIdx.x / 32, lane = threadIdx.x % 32;
uint32_t token = blockIdx.z, slot = blockIdx.y, route = token * TopK + slot;
PDLTriggerSecondary<PDL>();
PDLWaitPrimary<PDL>();
uint32_t expert = ids[route], tile = (blockIdx.x * 4 + warp) * 16;
uint32_t row = tile + lane / 4, col = (lane % 4) * 4;
const uint8_t* weights = w + static_cast<size_t>(expert) * N * K / 2;
const uint8_t* scales = sf + static_cast<size_t>(expert) * N * (K / 16);
z += route * (K / 8);
zs += route * (K / 16);
float d0 = 0, d1 = 0, d2 = 0, d3 = 0;
#pragma unroll
for (uint32_t stage = 0; stage < 2; ++stage)
pair_prefetch<N, K>(ws[warp][stage], weights, tile, stage, lane, z);
#pragma unroll 1
for (uint32_t base = 0; base < K; base += 128) {
asm volatile("cp.async.wait_group 1;" ::: "memory");
__syncwarp();
#pragma unroll
for (uint32_t inner = 0; inner < 128; inner += 64) {
if (base + inner >= K) break;
uint32_t k = base + inner;
uint32_t a0, a1, a2, a3;
pair_load_a(a0, a1, a2, a3, ws[warp][(base / 128) % 2] + (inner / 2) * 16, lane);
uint32_t sa =
*reinterpret_cast<const uint32_t*>(scales + sf_index<K / 16>(tile + lane / 4 + (lane % 2) * 8, k / 16));
uint32_t sb = *reinterpret_cast<const uint32_t*>(zs + k / 16);
mma(d0, d1, d2, d3, a0, a1, a2, a3, z[k / 8 + lane % 4], z[k / 8 + lane % 4 + 4], sa, sb);
}
__syncwarp();
pair_prefetch<N, K>(ws[warp][(base / 128) % 2], weights, tile, base / 128 + 2, lane, z);
}
asm volatile("cp.async.wait_group 0;" ::: "memory");
if (lane % 4 == 0) {
partial[route * N + row] = cast<bf16_t>((d0 * alpha[expert]) * routing[route]);
partial[route * N + row + 8] = cast<bf16_t>((d2 * alpha[expert]) * routing[route]);
}
}
} // namespace nvfp4_pair_mma
template <uint32_t N1, uint32_t K1, uint32_t N2, uint32_t K2, uint32_t TopK, bool PDL, bool PairLayout>
struct Sm120Nvfp4MoeMtpMma {
static_assert(N1 == 2688 && K1 == 1024 && N2 == 1024 && K2 == 2688 && TopK == 22);
/// Run 1 to 4 token rows with BF16 epilogues and deterministic reduction.
static void
run(tvm::ffi::TensorView x,
tvm::ffi::TensorView w1,
tvm::ffi::TensorView s1,
tvm::ffi::TensorView w2,
tvm::ffi::TensorView s2,
tvm::ffi::TensorView a1,
tvm::ffi::TensorView a2,
tvm::ffi::TensorView g1,
tvm::ffi::TensorView g2,
tvm::ffi::TensorView ids,
tvm::ffi::TensorView routing,
tvm::ffi::TensorView z,
tvm::ffi::TensorView zs,
tvm::ffi::TensorView partial,
tvm::ffi::TensorView out) {
using namespace host;
using namespace device;
auto dev = SymbolicDevice{};
dev.set_options<kDLCUDA>();
auto rows = SymbolicSize{"token_rows"};
TensorMatcher({rows, K1}).with_dtype<bf16_t>().with_device(dev).verify(x);
CHECK_HOST(rows.unwrap() >= 1 && rows.unwrap() <= 4) << "Expected 1 <= M <= 4";
const uint32_t M = static_cast<uint32_t>(rows.unwrap());
TensorMatcher({512, N1, K1 / 2}).with_dtype<uint8_t>().with_device(dev).verify(w1);
if constexpr (PairLayout)
TensorMatcher({512, K2 / 2, N2}).with_dtype<uint8_t>().with_device(dev).verify(w2);
else
TensorMatcher({512, N2, K2 / 2}).with_dtype<uint8_t>().with_device(dev).verify(w2);
TensorMatcher({512 * N1 * K1 / 16}).with_dtype<uint8_t>().with_device(dev).verify(s1);
TensorMatcher({512 * N2 * K2 / 16}).with_dtype<uint8_t>().with_device(dev).verify(s2);
TensorMatcher({512}).with_dtype<float>().with_device(dev).verify(a1).verify(a2);
TensorMatcher({1}).with_dtype<float>().with_device(dev).verify(g1).verify(g2);
TensorMatcher({M, TopK}).with_dtype<int32_t>().with_device(dev).verify(ids);
TensorMatcher({M, TopK}).with_dtype<float>().with_device(dev).verify(routing);
TensorMatcher({M * TopK * N1 / 8}).with_dtype<int32_t>().with_device(dev).verify(z);
TensorMatcher({M * TopK * N1 / 16}).with_dtype<uint8_t>().with_device(dev).verify(zs);
TensorMatcher({M * TopK * N2}).with_dtype<bf16_t>().with_device(dev).verify(partial);
TensorMatcher({M, N2}).with_dtype<bf16_t>().with_device(dev).verify(out);
LaunchKernel({N1 / 64, TopK, M}, 128, dev.unwrap())
.enable_pdl(PDL)(
nvfp4_mtp_mma::fc1<N1, K1, TopK, PDL>,
static_cast<const bf16_t*>(x.data_ptr()),
static_cast<const uint8_t*>(w1.data_ptr()),
static_cast<const uint8_t*>(s1.data_ptr()),
static_cast<const float*>(a1.data_ptr()),
static_cast<const float*>(g1.data_ptr()),
static_cast<const float*>(g2.data_ptr()),
static_cast<const int32_t*>(ids.data_ptr()),
static_cast<uint32_t*>(z.data_ptr()),
static_cast<uint8_t*>(zs.data_ptr()));
LaunchKernel({N2 / 64, TopK, M}, 128, dev.unwrap())
.enable_pdl(PDL)(
PairLayout ? nvfp4_pair_mma::fc2<N2, K2, TopK, PDL> : nvfp4_mtp_mma::fc2<N2, K2, TopK, PDL>,
static_cast<const uint32_t*>(z.data_ptr()),
static_cast<const uint8_t*>(zs.data_ptr()),
static_cast<const uint8_t*>(w2.data_ptr()),
static_cast<const uint8_t*>(s2.data_ptr()),
static_cast<const float*>(a2.data_ptr()),
static_cast<const int32_t*>(ids.data_ptr()),
static_cast<const float*>(routing.data_ptr()),
static_cast<bf16_t*>(partial.data_ptr()));
LaunchKernel({(M * N2 + 255) / 256}, 256, dev.unwrap())
.enable_pdl(PDL)(
nvfp4_mtp_mma::reduce<N2, TopK, PDL>,
static_cast<const bf16_t*>(partial.data_ptr()),
static_cast<bf16_t*>(out.data_ptr()),
M);
}
};
} // namespace sglang
@@ -0,0 +1,49 @@
// Exact byte transpose for changing weight layout once per request phase.
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/type.cuh>
#include <sgl_kernel/utils.cuh>
namespace sglang {
namespace nvfp4_weight_transpose {
template <unsigned Rows, unsigned Cols, unsigned Tile>
__global__ __launch_bounds__(256) void kernel(const uint8_t* __restrict__ x, uint8_t* __restrict__ y) {
__shared__ uint8_t data[Tile][Tile + 4];
unsigned tx = threadIdx.x % 32, ty = threadIdx.x / 32;
unsigned rbase = blockIdx.y * Tile, cbase = blockIdx.x * Tile;
size_t expert = static_cast<size_t>(blockIdx.z) * Rows * Cols;
#pragma unroll
for (unsigned j = 0; j < Tile / 8; ++j) {
unsigned row = rbase + ty + j * 8, col = cbase + tx * 4;
unsigned v = 0;
if (row < Rows && col < Cols) v = *reinterpret_cast<const unsigned*>(x + expert + row * Cols + col);
*reinterpret_cast<unsigned*>(&data[ty + j * 8][tx * 4]) = v;
}
__syncthreads();
#pragma unroll
for (unsigned j = 0; j < Tile / 8; ++j) {
unsigned row = cbase + ty + j * 8, col = rbase + tx * 4;
if (row < Cols && col < Rows) {
unsigned v = static_cast<unsigned>(data[tx * 4 + 0][ty + j * 8]) |
(static_cast<unsigned>(data[tx * 4 + 1][ty + j * 8]) << 8) |
(static_cast<unsigned>(data[tx * 4 + 2][ty + j * 8]) << 16) |
(static_cast<unsigned>(data[tx * 4 + 3][ty + j * 8]) << 24);
*reinterpret_cast<unsigned*>(y + expert + row * Rows + col) = v;
}
}
}
} // namespace nvfp4_weight_transpose
template <unsigned Rows, unsigned Cols>
struct Nvfp4WeightTranspose {
static void run(tvm::ffi::TensorView x, tvm::ffi::TensorView y) {
using namespace host;
auto dev = SymbolicDevice{};
dev.set_options<kDLCUDA>();
TensorMatcher({512, Rows, Cols}).with_dtype<uint8_t>().with_device(dev).verify(x);
TensorMatcher({512, Cols, Rows}).with_dtype<uint8_t>().with_device(dev).verify(y);
LaunchKernel({(Cols + 127) / 128, (Rows + 127) / 128, 512}, 256, dev.unwrap())(
nvfp4_weight_transpose::kernel<Rows, Cols, 128>,
static_cast<const uint8_t*>(x.data_ptr()),
static_cast<uint8_t*>(y.data_ptr()));
}
};
} // namespace sglang
@@ -4,7 +4,12 @@ from typing import TYPE_CHECKING
import torch
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
from sglang.kernels.jit.utils import (
cache_once,
is_arch_support_pdl,
load_jit,
make_cpp_args,
)
if TYPE_CHECKING:
from tvm_ffi.module import Module
@@ -20,8 +25,18 @@ _MAX_K = 17408 # static smem limit (K * 2 bytes <= 48KB) with margin
_MAX_N = 65536 # very large N (lm_head) is already at the bandwidth ceiling
def _is_sm12x() -> bool:
from sglang.kernels.jit.utils import get_jit_cuda_arch
return get_jit_cuda_arch().major >= 12
def _config(n: int) -> tuple[int, int, int]:
"""(rows_per_warp, k_unroll, num_warps) tuned on H200."""
"""(rows_per_warp, k_unroll, num_warps) tuned on H200 (SM90) and
DRAM-cold on RTX PRO 6000 (SM120), where kUnroll must stay 1 so the
in-loop k < K guard also serves K % 512 != 0 shapes."""
if _is_sm12x():
return (1, 1, 8)
if n >= 8192:
return (2, 2, 8)
return (1, 2, 8)
@@ -30,7 +45,7 @@ def _config(n: int) -> tuple[int, int, int]:
@cache_once
def _jit_hopper_bf16_gemv_module(n: int, k: int) -> Module:
rows, unroll, warps = _config(n)
args = make_cpp_args(n, k, rows, unroll, warps)
args = make_cpp_args(n, k, rows, unroll, warps, is_arch_support_pdl())
return load_jit(
"hopper_bf16_gemv",
*args,
@@ -41,6 +56,21 @@ def _jit_hopper_bf16_gemv_module(n: int, k: int) -> Module:
def use_hopper_bf16_gemv(m: int, n: int, k: int) -> bool:
if _is_sm12x():
# SM120 (GDDR7): kUnroll==1 needs only 16B-aligned K (128 kept for
# vector-alignment margin). cuBLAS M=1 bf16 trails the streaming GEMV
# on small/odd N and very wide N; the 12288..32768 window (e.g. the
# 18560-row bf16 in_proj) already runs at the DRAM ceiling under
# cuBLAS, so leave it there.
if not (
m == 1
and k % 128 == 0
and 512 <= k <= _MAX_K
and n % 8 == 0
and 64 <= n <= 140000
):
return False
return n < 12288 or n >= 32768
if not (
m == 1
and k % 512 == 0
@@ -4,7 +4,12 @@ from typing import TYPE_CHECKING
import torch
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
from sglang.kernels.jit.utils import (
cache_once,
is_arch_support_pdl,
load_jit,
make_cpp_args,
)
if TYPE_CHECKING:
from tvm_ffi.module import Module
@@ -22,16 +27,16 @@ _MAX_N = 32768 # very large N is served well by cuBLAS already
def _config(n: int) -> tuple[int, int, int]:
"""(rows_per_warp, k_unroll, num_warps)."""
"""(rows_per_warp, k_unroll, num_warps); tuned DRAM-cold on RTX PRO 6000."""
if n >= 8192:
return (2, 1, 8)
return (4, 1, 8)
return (1, 1, 8)
@cache_once
def _jit_sm120_fp8_gemv_module(n: int, k: int) -> Module:
rows, unroll, warps = _config(n)
args = make_cpp_args(n, k, rows, unroll, warps)
args = make_cpp_args(n, k, rows, unroll, warps, is_arch_support_pdl())
return load_jit(
"sm120_fp8_gemv",
*args,
@@ -42,9 +47,12 @@ def _jit_sm120_fp8_gemv_module(n: int, k: int) -> Module:
def use_sm120_fp8_gemv(m: int, n: int, k: int) -> bool:
# kUnroll == 1 streams 16B per lane-step with an in-loop k < K guard, so
# K only needs 16B alignment; keep 128 for conservative vector alignment
# (covers the K=5376 shared-expert down_proj).
return (
m == 1
and k % 512 == 0
and k % 128 == 0
and 512 <= k <= _MAX_K
and n % 16 == 0
and 256 <= n <= _MAX_N
@@ -0,0 +1,167 @@
from __future__ import annotations
import torch
from sglang.kernels.jit.utils import (
cache_once,
load_jit,
make_cpp_args,
)
@cache_once
def _mtp_module(pair_layout: bool = False):
args = make_cpp_args(2688, 1024, 1024, 2688, 22, False, pair_layout)
return load_jit(
"sm120_nvfp4_moe_mtp_mma",
*args,
cuda_files=["gemm/sm120_nvfp4_moe_mtp_mma.cuh"],
cuda_wrappers=[("run", f"sglang::Sm120Nvfp4MoeMtpMma<{args}>::run")],
extra_cuda_cflags=["-O3"],
)
def nvfp4_moe_mtp_mma(
x, w1, s1, w2, s2, a1, a2, g1, g2, ids, routing, out=None, *, pair_layout=False
):
m = x.shape[0]
module = _mtp_module(pair_layout)
if out is None:
out = torch.empty((m, 1024), dtype=torch.bfloat16, device=x.device)
z = torch.empty(m * 22 * 2688 // 8, dtype=torch.int32, device=x.device)
zs = torch.empty(m * 22 * 2688 // 16, dtype=torch.uint8, device=x.device)
partial = torch.empty(m * 22 * 1024, dtype=torch.bfloat16, device=x.device)
module.run(
x,
w1.view(torch.uint8),
s1.view(torch.uint8).view(-1),
w2.view(torch.uint8),
s2.view(torch.uint8).view(-1),
a1.view(-1),
a2.view(-1),
g1.view(-1),
g2.view(-1),
ids.to(torch.int32),
routing.float(),
z,
zs,
partial,
out,
)
return out
def try_nvfp4_moe(
x,
w13,
w2,
scales,
ids,
routing,
*,
output_dtype,
out=None,
pair_w2_weight=None,
use_mtp=True,
):
"""Select native small-M kernels; unsupported shapes retain CUTLASS."""
if (
use_mtp
and x.ndim == 2
and 1 <= x.shape[0] <= 4
and x.shape[1] == 1024
and x.dtype == output_dtype == torch.bfloat16
and w13.shape == (512, 2688, 512)
and w2.shape == (512, 1024, 1344)
and ids.shape == (x.shape[0], 22)
):
return nvfp4_moe_mtp_mma(
x,
w13,
scales[1],
pair_w2_weight if pair_w2_weight is not None else w2,
scales[4],
scales[2],
scales[5],
scales[0],
scales[3],
ids,
routing,
out=out,
pair_layout=pair_w2_weight is not None,
)
if pair_w2_weight is not None:
raise RuntimeError("A paired NVFP4 weight reached an unsupported MoE path")
return None
@cache_once
def _transpose_module(rows: int, cols: int):
return load_jit(
"sm120_nvfp4_weight_transpose",
rows,
cols,
cuda_files=["gemm/sm120_nvfp4_weight_transpose.cuh"],
cuda_wrappers=[("run", f"sglang::Nvfp4WeightTranspose<{rows},{cols}>::run")],
extra_cuda_cflags=["-O3"],
)
class Nvfp4LayoutCycle:
"""Own the two views of a fixed, serial set of [512,1024,1344] weights."""
def __init__(self, weights):
if not weights:
raise ValueError("A layout cycle requires at least one expert weight")
first = weights[0]
if first.device.type != "cuda":
raise ValueError("The layout cycle requires CUDA weights")
for weight in weights:
if (
weight.shape != (512, 1024, 1344)
or weight.dtype != torch.uint8
or not weight.is_contiguous()
or weight.device != first.device
):
raise ValueError("Incompatible NVFP4 FC2 weight layout")
if len({weight.data_ptr() for weight in weights}) != len(weights):
raise ValueError("Expert weights in a layout cycle must not alias")
self.original = tuple(weights)
self.workspace_bytes = first.numel()
self.scratch = None
self.pair = ()
self.is_pair = False
self.switches = 0
# Compile before any graph capture can reach the state transition.
self._forward = _transpose_module(1024, 1344)
self._inverse = _transpose_module(1344, 1024)
def prepare_workspace(self):
if self.scratch is not None:
return
if torch.cuda.is_current_stream_capturing():
raise RuntimeError("NVFP4 workspace must be prepared before graph capture")
# Use runtime workspace headroom, like graph/kernel scratch allocations.
# Allocating this before Mamba/KV pool sizing incorrectly charges it as
# persistent checkpoint storage and can round the state pool below one
# request. The normal graph warmup prepares stable pointers before use.
self.scratch = torch.empty_like(self.original[0])
self.pair = tuple(
weight.view(512, 1344, 1024)
for weight in (self.scratch, *self.original[:-1])
)
def ensure(self, pair: bool):
if pair == self.is_pair:
return
if torch.cuda.is_current_stream_capturing():
raise RuntimeError("NVFP4 layout transitions must run before graph capture")
if pair:
self.prepare_workspace()
for original, paired in zip(self.original, self.pair):
self._forward.run(original, paired)
else:
for original, paired in reversed(tuple(zip(self.original, self.pair))):
self._inverse.run(paired, original)
self.is_pair = pair
self.switches += 1
+116 -3
View File
@@ -18,6 +18,85 @@ _SCORING_FUNC_MAP = {
}
@triton.jit
def _sm120_router_top22_kernel(
scores_ptr,
bias_ptr,
weights_ptr,
ids_ptr,
stride_sm,
stride_sn,
scale,
RENORMALIZE: tl.constexpr,
APPLY_SCALE: tl.constexpr,
USE_PDL: tl.constexpr,
):
row = tl.program_id(0)
n = tl.arange(0, 512)
if USE_PDL:
tl.extra.cuda.gdc_wait()
scores = tl.load(scores_ptr + row * stride_sm + n * stride_sn).to(tl.float32)
bias = tl.load(bias_ptr + n).to(tl.float32)
activated = tl.sigmoid(scores)
biased = activated + bias
biased = tl.where(biased == biased, biased, -1e30)
biased = tl.where(biased == 0.0, 0.0, biased)
bits = biased.to(tl.uint32, bitcast=True)
ordered = tl.where((bits & 0x80000000) != 0, ~bits, bits ^ 0x80000000)
keys = (ordered.to(tl.uint64) << 32) | (0xFFFFFFFF - n.to(tl.uint32)).to(tl.uint64)
winners = tl.topk(keys, 32)
ids = (0xFFFFFFFF - (winners & 0xFFFFFFFF).to(tl.uint32)).to(tl.int32)
k = tl.arange(0, 32)
values = tl.gather(activated, ids, axis=0)
values = tl.where(k < 22, values, 0.0)
total = tl.sum(values, 0)
if USE_PDL:
tl.extra.cuda.gdc_launch_dependents()
if RENORMALIZE:
values = values / tl.where(total > 0.0, total, 1.0)
if APPLY_SCALE:
values = values * scale
tl.store(weights_ptr + row * 22 + k, values, k < 22)
tl.store(ids_ptr + row * 22 + k, ids, k < 22)
def _topk_gate_small_m(
scores,
bias,
topk=22,
scoring_func="sigmoid",
num_fused_shared_experts=0,
renormalize=True,
routed_scaling_factor=1.0,
apply_routed_scaling_factor_on_output=False,
):
assert scores.is_cuda and bias.device == scores.device
assert scores.ndim == 2 and scores.shape[1] == 512
assert scores.dtype == torch.float32
assert 1 <= scores.shape[0] <= 16
assert bias.shape == (512,) and bias.dtype == torch.float32 and bias.is_contiguous()
assert topk == 22 and scoring_func == "sigmoid" and num_fused_shared_experts == 0
m = scores.shape[0]
weights = torch.empty((m, 22), device=scores.device, dtype=torch.float32)
ids = torch.empty((m, 22), device=scores.device, dtype=torch.int32)
pdl = is_arch_support_pdl()
_sm120_router_top22_kernel[(m,)](
scores,
bias,
weights,
ids,
scores.stride(0),
scores.stride(1),
float(routed_scaling_factor),
RENORMALIZE=bool(renormalize),
APPLY_SCALE=bool(apply_routed_scaling_factor_on_output),
USE_PDL=pdl,
num_warps=1,
**({"launch_pdl": True} if pdl else {}),
)
return weights, ids
@triton.jit
def _router_triton_kernel(
scores_ptr, # [M, N] raw logits, fp32/fp16/bf16 (upcast to fp32 on load)
@@ -328,12 +407,46 @@ def moe_fused_gate(
assert input_ids.shape == (scores.size(0),)
if packed_out is not None:
assert packed_out.dtype == torch.int32, "packed_out must be int32"
assert packed_out.shape == (scores.size(0), topk), (
"packed_out must be [M, topk]"
)
assert packed_out.shape == (
scores.size(0),
topk,
), "packed_out must be [M, topk]"
if routed_scaling_factor is None:
routed_scaling_factor = 1.0
from sglang.srt.environ import envs
if (
envs.SGLANG_SM120_MTP_ROUTER_TOP22.get()
and scores.is_cuda
and torch.cuda.get_device_capability(scores.device) == (12, 0)
and 2 <= scores.shape[0] <= 16
and scores.shape[1] == 512
and scores.dtype == torch.float32
and scores.stride(1) == 1
and bias is not None
and bias.dtype == torch.float32
and bias.is_contiguous()
and topk == 22
and scoring_func.lower() == "sigmoid"
and num_fused_shared_experts == 0
and num_expert_group <= 1
and moe_softcapping == 0.0
and input_ids is None
and num_token_non_padded is None
and renormalize_epsilon == 0.0
and packed_out is None
and not sqrtsoftplus_log1p
):
return _topk_gate_small_m(
scores,
bias,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
)
# K3 radix-select fast path: native-CUDA radix-select replaces the 16
# dependent argmax rounds (single CTA per token; ids bit-identical to this
# triton kernel incl. ties).
@@ -0,0 +1,41 @@
"""Pack a uniform-width verify tree mask into existing FlashInfer buffers."""
import triton
import triton.language as tl
@triton.jit
def _pack_verify_mask_kernel(
mask,
kv_indptr,
packed_indptr,
output,
Q: tl.constexpr,
BLOCK: tl.constexpr,
):
row = tl.program_id(0)
byte = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
bit = tl.arange(0, 8)
kv_start = tl.load(kv_indptr + row)
kv_end = tl.load(kv_indptr + row + 1)
count = (kv_end - kv_start) * Q
offsets = byte[:, None] * 8 + bit[None, :]
values = tl.load(mask + kv_start * Q + offsets, offsets < count, other=0)
packed = tl.sum(values.to(tl.uint32) << bit[None, :], axis=1)
start = tl.load(packed_indptr + row)
tl.store(output + start + byte, packed.to(tl.uint8), byte < tl.cdiv(count, 8))
def pack_verify_mask_into(
mask, kv_indptr, packed_indptr, output, *, batch_size, query_width, max_kv_len
):
_pack_verify_mask_kernel[
(batch_size, triton.cdiv(query_width * max_kv_len, 8 * 256))
](
mask.contiguous().view(-1),
kv_indptr,
packed_indptr,
output,
Q=query_width,
BLOCK=256,
)
+7
View File
@@ -1157,6 +1157,9 @@ class Envs:
SGLANG_TRTLLM_MOE_PDL_MAX_TOKENS = EnvInt(8192)
# Use FlashInfer's fused atomic CUTLASS/CuTe DSL MoE finalize.
SGLANG_FLASHINFER_MOE_FUSED_FINALIZE = EnvBool(False)
# Experimental Nemotron small-M NVFP4 path; validated only on SM120.
SGLANG_SM120_NVFP4_PHASE_LAYOUT = EnvBool(False)
SGLANG_SM120_MTP_ROUTER_TOP22 = EnvBool(False)
# Master switch for the experimental TRT-LLM LoRA fast path; when OFF (default) every
# fine-grained opt switch reads False, keeping non-experimental paths byte-identical.
SGLANG_EXPERIMENTAL_LORA_OPTI = EnvBool(False)
@@ -1453,6 +1456,10 @@ class Envs:
# into a small CUDA graph, collapsing its host dispatch cost to one launch.
# Experimental; auto-falls back to eager if the backend's prep is not capturable.
SGLANG_ENABLE_METADATA_GLUE_GRAPH = EnvBool(False)
# Reuse the resolved CPU sequence lengths for FlashInfer draft planning,
# avoiding a second blocking readback after generating the same indptr.
SGLANG_FLASHINFER_DRAFT_HOST_INDPTR = EnvBool(False)
SGLANG_FLASHINFER_VERIFY_HOST_PLAN = EnvBool(False)
SGLANG_OPT_FUSED_KDA_VERIFY = EnvBool(False)
# A/B: keep the DFLASH draft greedy head eager (not folded in-graph).
SGLANG_DFLASH_EAGER_DRAFT_SAMPLER = EnvBool(False)
@@ -210,9 +210,10 @@ def fast_prefill_plan(
kv_lens_host: torch.Tensor,
max_q_len: int,
max_kv_len: int,
uniform_verify_width: Optional[int] = None,
) -> None:
"""Sync-free ``BatchPrefillWithPagedKVCacheWrapper.plan`` for the EAGLE
draft-extend CUDA graph (FlashInfer fa2, cuda-graph mode only).
draft-extend or uniform-width verify CUDA graph (FlashInfer fa2 only).
Upstream plan() always does qo/paged_kv/last_page_len ``.to("cpu")`` to build
its host scheduling metadata, a blocking D2H that drains the GPU queue every
@@ -259,6 +260,30 @@ def fast_prefill_plan(
non_blocking=(paged_kv_indices.device == self.device) and non_blocking,
)
if custom_mask is not None:
# EAGLE's mask remains data-dependent: pack every actual tree-mask
# bit, including rejected branches. Only its row lengths are known
# on the host, so no device scalar read is needed for allocation.
assert uniform_verify_width is not None and page_size == 1
from sglang.kernels.ops.speculative.pack_verify_mask import (
pack_verify_mask_into,
)
packed_lengths = (kv_lens_host * uniform_verify_width + 7) // 8
packed_indptr = torch.zeros(batch_size + 1, dtype=torch.int32, device="cpu")
packed_indptr[1:] = torch.cumsum(packed_lengths, dim=0)
assert int(packed_indptr[-1]) <= self._custom_mask_buf.numel()
self._mask_indptr_buf.copy_(packed_indptr, non_blocking=non_blocking)
pack_verify_mask_into(
custom_mask,
paged_kv_indptr,
self._mask_indptr_buf,
self._custom_mask_buf,
batch_size=batch_size,
query_width=uniform_verify_width,
max_kv_len=max_kv_len,
)
self._cached_q_data_type = q_data_type
self._cached_kv_data_type = (
kv_data_type if kv_data_type is not None else q_data_type
@@ -855,9 +880,24 @@ class FlashInferAttnBackend(AttentionBackend):
# call_begin_forward therefore applies verbatim; installing the
# sync-free plan removes three blocking .to("cpu") reads per
# replay that otherwise stall the CPU behind the in-flight graph.
# EAGLE target-verify keeps the plain plan(): its spec input is
# not DFLASH_VERIFY, and this branch keys off the capture-time
# spec_info of these per-bs wrappers.
# EAGLE uses the separate opt-in branch below because it must
# also pack its actual tree mask on every replay.
for w in self.prefill_cuda_graph_metadata[bs]:
w.begin_forward = partial(fast_prefill_plan, w)
if (
in_capture
and envs.SGLANG_FLASHINFER_VERIFY_HOST_PLAN.get()
and forward_mode.is_target_verify()
and spec_info is not None
and spec_info.spec_input_type == SpecInputType.EAGLE_VERIFY
and spec_info.topk == 1
and seq_lens_cpu is not None
and self.prefill_backend == "fa2"
and self.dispatch_reason is None
and self.flashinfer_kv_cache_dtype
in (torch.bfloat16, torch.float16, torch.float8_e4m3fn, torch.float8_e5m2)
):
for w in self.prefill_cuda_graph_metadata[bs]:
w.begin_forward = partial(fast_prefill_plan, w)
@@ -2322,10 +2362,13 @@ class FlashInferIndicesUpdaterPrefill:
assert num_tokens_per_req is not None and num_tokens_per_req > 0, (
f"fast_prefill_plan replay requires num_tokens_per_req > 0 (got {num_tokens_per_req})"
)
assert use_custom_mask is None, (
"fast_prefill_plan does not support custom_mask; keep the plain plan()"
)
seq_lens_cpu_i32 = seq_lens_cpu.to(torch.int32)
eagle_verify = spec_info.spec_input_type == SpecInputType.EAGLE_VERIFY
assert use_custom_mask is None or eagle_verify
if eagle_verify:
# EAGLE carries the committed prefix length; its generated KV
# layout includes the complete candidate window after it.
seq_lens_cpu_i32 = seq_lens_cpu_i32 + num_tokens_per_req
qo_indptr_host = torch.arange(
0,
(bs + 1) * num_tokens_per_req,
@@ -2342,6 +2385,8 @@ class FlashInferIndicesUpdaterPrefill:
max_q_len=num_tokens_per_req,
max_kv_len=int(seq_lens_cpu_i32.max()),
)
if eagle_verify:
paged_plan_kwargs["uniform_verify_width"] = num_tokens_per_req
if window_left >= 0:
# selects the module with the per-element window mask compiled in
@@ -2369,6 +2414,30 @@ class FlashInferIndicesUpdaterPrefill:
)
def _build_draft_indptr_host(
seq_lens_cpu: torch.Tensor, topk: int, num_steps: int
) -> torch.Tensor:
"""Mirror the draft kernel's prefix sums from resolved host lengths.
At draft entry, positions repeat each committed sequence length ``topk``
times. Step i adds i+1 tokens per branch. This also covers the graph
runner's padded rows, whose CPU lengths have already been filled.
"""
assert seq_lens_cpu.device.type == "cpu" and seq_lens_cpu.ndim == 1
offsets = [0]
for length in seq_lens_cpu.tolist():
for _ in range(topk):
offsets.append(offsets[-1] + length)
return torch.tensor(
[
[offset + branch * (step + 1) for branch, offset in enumerate(offsets)]
for step in range(num_steps)
],
dtype=torch.int32,
device="cpu",
)
class FlashInferMultiStepDraftBackend:
"""
Wrap multiple flashinfer attention backends as one for multiple consecutive
@@ -2447,8 +2516,23 @@ class FlashInferMultiStepDraftBackend:
req_pool_indices=forward_batch.req_pool_indices,
bs=num_seqs,
)
host_lengths = forward_batch.seq_lens_cpu
use_host_indptr = (
envs.SGLANG_FLASHINFER_DRAFT_HOST_INDPTR.get()
and self.draft_window_size == 0
and host_lengths is not None
and host_lengths.device.type == "cpu"
and host_lengths.numel() == num_seqs
)
# The existing token-block kernel copies every index, but distributes
# long contexts across CTAs instead of serially walking them in 3 CTAs.
num_blocks = (
min(32, max(1, (max(host_lengths.tolist(), default=0) + 511) // 512))
if use_host_indptr
else 1
)
self.generate_draft_decode_kv_indices[
(self.speculative_num_steps, num_seqs, self.topk)
(self.speculative_num_steps * num_blocks, num_seqs, self.topk)
](
src.row_ids,
src.ids,
@@ -2468,24 +2552,33 @@ class FlashInferMultiStepDraftBackend:
ENTRY_PAGE_SIZE=src.entry_page_size,
v2p=src.v2p,
TRANSLATE=src.v2p is not None,
NUM_STEPS=self.speculative_num_steps if use_host_indptr else 0,
)
assert forward_batch.spec_info is not None
assert forward_batch.spec_info.is_draft_input()
# Copy the kv_indptr once to avoid multiple device-to-host copies in flashinfer's plan.
indptr_cpu_whole = self.kv_indptr[:, : bs + 1].cpu()
# resolve_seq_lens_cpu already supplies the committed lengths after
# acceptance/rollback. Do not drain the GPU queue to read them again.
indptr_cpu_whole = (
_build_draft_indptr_host(
host_lengths, self.topk, self.speculative_num_steps
)
if use_host_indptr
else self.kv_indptr[:, : bs + 1].cpu()
)
global global_override_indptr_cpu
for i in range(self.speculative_num_steps - 1):
forward_batch.spec_info.kv_indptr = self.kv_indptr[i, : bs + 1]
forward_batch.spec_info.kv_indices = kv_indices_buffer[i][
: draft_kv_indices_used_len(seq_lens_sum, self.topk, bs, i + 1)
]
global_override_indptr_cpu = indptr_cpu_whole[i]
call_fn(i, forward_batch)
global_override_indptr_cpu = None
previous_override = global_override_indptr_cpu
try:
for i in range(self.speculative_num_steps - 1):
forward_batch.spec_info.kv_indptr = self.kv_indptr[i, : bs + 1]
forward_batch.spec_info.kv_indices = kv_indices_buffer[i][
: draft_kv_indices_used_len(seq_lens_sum, self.topk, bs, i + 1)
]
global_override_indptr_cpu = indptr_cpu_whole[i]
call_fn(i, forward_batch)
finally:
global_override_indptr_cpu = previous_override
def init_forward_metadata(self, forward_batch: ForwardBatch):
kv_indices_width = draft_kv_indices_buffer_width(
@@ -26,7 +26,7 @@ from sglang.srt.layers.moe.moe_runner.base import (
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_flashinfer_available
from sglang.srt.utils.common import next_power_of_2
from sglang.srt.utils.common import is_sm120, next_power_of_2
if TYPE_CHECKING:
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
@@ -59,6 +59,7 @@ class FlashInferCutlassMoeQuantInfo(MoeQuantInfo):
swiglu_beta: Optional[torch.Tensor] = None
swiglu_limit: Optional[torch.Tensor] = None
apply_routed_scaling_factor: bool = True
pair_w2_weight: Optional[torch.Tensor] = None
def materialize_swiglu_params_for_cutlass(
@@ -250,6 +251,26 @@ def _prepare_input(
return x, x_sf, output_dtype, output_col
def _sm120_mtp_mma_supported() -> bool:
import os
# This implementation preserves FlashInfer's default FP4 quantizer and
# fused FC2 epilogue. Other quantization recipes retain the stock runner.
return (
is_sm120()
and envs.SGLANG_FLASHINFER_MOE_FUSED_FINALIZE.get()
and not envs.FLASHINFER_NVFP4_4OVER6.get()
and os.environ.get("FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH", "0") == "0"
and os.environ.get("TRTLLM_DISABLE_FP4_QUANT_FAST_MATH", "0") == "0"
)
def _flashinfer_autotune_active() -> bool:
from flashinfer.autotuner import AutoTuner
return AutoTuner.get().is_tuning_mode
def _run_flashinfer_cutlass(
*,
dispatch_output,
@@ -268,6 +289,35 @@ def _run_flashinfer_cutlass(
dispatch_output, quant_info, runner_config
)
if (
quant_info.quant_type == "fp4"
and not runner_config.is_gated
and runner_config.activation == "relu2"
and runner_config.moe_ep_size == runner_config.moe_tp_size == 1
and x_sf is None
and not enable_alltoall
and is_sm120()
):
from sglang.kernels.ops.gemm.sm120_nvfp4_moe import try_nvfp4_moe
result = try_nvfp4_moe(
x,
quant_info.w13_weight,
quant_info.w2_weight,
quant_info.quant_scales,
topk_ids,
topk_weights,
output_dtype=output_dtype,
out=output,
pair_w2_weight=quant_info.pair_w2_weight,
# Preserve FlashInfer target/draft tactic discovery during autotuning.
use_mtp=_sm120_mtp_mma_supported() and not _flashinfer_autotune_active(),
)
if result is not None:
return _maybe_apply_routed_scaling_factor(result, quant_info, runner_config)
if quant_info.pair_w2_weight is not None:
raise RuntimeError("A paired NVFP4 weight reached an unsupported MoE path")
if output is None:
with use_symmetric_memory(
get_parallel().tp_group, disabled=not is_allocation_symmetric()
@@ -3272,7 +3272,10 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
"apply_router_weight_on_input is not supported for Flashinfer"
)
swiglu_alpha, swiglu_beta, swiglu_limit = layer._cutlass_swiglu_params
slot = getattr(layer, "_nvfp4_layout_slot", None)
pair_weight = slot[0].pair[slot[1]] if slot and slot[0].is_pair else None
quant_info = FlashInferCutlassMoeQuantInfo(
pair_w2_weight=pair_weight,
quant_type="fp4",
w13_weight=layer.w13_weight,
w2_weight=layer.w2_weight,
@@ -317,8 +317,8 @@ def initialize_bf16_gemm_config() -> None:
backend = Bf16GemmBackend(backend_str)
if backend.is_gemv():
if torch.cuda.get_device_capability()[0] != 9:
raise ValueError("--bf16-gemm-backend gemv requires SM90 (Hopper)")
if torch.cuda.get_device_capability()[0] not in (9, 12):
raise ValueError("--bf16-gemm-backend gemv requires SM90 (Hopper) or SM12x")
global _hopper_bf16_gemv, _use_hopper_bf16_gemv
from sglang.kernels.ops.gemm.hopper_bf16_gemv import (
@@ -589,7 +589,7 @@ class UnquantizedLinearMethod(LinearMethodBase):
return tgemm.mm(x, layer.weight, bias, otype=x.dtype)
elif (
get_bf16_gemm_backend().is_cutedsl()
(get_bf16_gemm_backend().is_cutedsl() or get_bf16_gemm_backend().is_gemv())
and x.is_cuda
and x.dtype == torch.bfloat16
and layer.weight.dtype == torch.bfloat16
@@ -1953,6 +1953,9 @@ class ModelRunner:
reinit_attn_backend: bool = False,
split_forward_count: int = 1,
) -> ModelRunnerOutput:
prepare = getattr(self.model, "prepare_for_forward", None)
if prepare is not None:
prepare(forward_batch)
if has_forward_context():
ctx_mgr = contextlib.nullcontext()
else:
@@ -96,6 +96,25 @@ def has_standard_gqa_for_all_local_layers(
return attention_layer_count >= end_layer - start_layer
def has_hybrid_graph_handlers(
*,
attention_layers: list[Any],
moe_layers: list[Any],
start_layer: int,
end_layer: int,
) -> bool:
"""Accept pure MoE layers while rejecting gaps in the local decoder range."""
return (
0 <= start_layer < end_layer
and end_layer <= len(attention_layers)
and end_layer <= len(moe_layers)
and all(
attention_layers[i] is not None or moe_layers[i] is not None
for i in range(start_layer, end_layer)
)
)
def index_attention_layers_by_global_id(
attention_layers: list[Any],
mha_companion_layers: list[Any],
@@ -118,8 +137,13 @@ def index_attention_layers_by_global_id(
)
return attention_layers, mha_companion_layers
max_layer_id = max(int(layer.layer_id) for layer in populated)
indexed_attention = [None] * (max_layer_id + 1)
indexed_companions = [None] * (max_layer_id + 1)
# Keep trailing pure MoE layers represented as well. Their graph handler
# lives in moe_layers, and shrinking this list loses decoder alignment.
indexed_size = max(
max_layer_id + 1, len(layer_model.layers) if layer_model is not None else 0
)
indexed_attention = [None] * indexed_size
indexed_companions = [None] * indexed_size
has_reused_layers = False
for attention, companion in zip(attention_layers, mha_companion_layers):
if attention is None:
@@ -621,7 +645,17 @@ def capture_prefill_graph(
layer_model,
)
if not has_standard_gqa_for_all_local_layers(
supports_hybrid_graph = (
getattr(layer_model, "supports_hybrid_prefill_graph", False)
and prefill_backend == Backend.BREAKABLE
and has_hybrid_graph_handlers(
attention_layers=model_runner.attention_layers,
moe_layers=model_runner.moe_layers,
start_layer=model_runner.layer_info.start_layer,
end_layer=model_runner.layer_info.end_layer,
)
)
if not supports_hybrid_graph and not has_standard_gqa_for_all_local_layers(
attention_layer_count=sum(
layer is not None for layer in model_runner.attention_layers
),
@@ -24,6 +24,12 @@ class WeightExporter:
get_model: Callable[[], Any]
_weights_send_group: dict = field(default_factory=dict)
def _check_weight_export_supported(self):
for module in self.get_model().modules():
reason = getattr(module, "_weight_export_error", None)
if reason is not None:
raise RuntimeError(reason)
def init_weights_send_group_for_remote_instance(
self,
master_address,
@@ -33,6 +39,7 @@ class WeightExporter:
group_name,
backend="nccl",
):
self._check_weight_export_supported()
assert torch.distributed.is_initialized(), (
"Default torch process group must be initialized"
)
@@ -81,6 +88,7 @@ class WeightExporter:
ports,
group_name,
):
self._check_weight_export_supported()
assert torch.distributed.is_initialized(), (
"Default torch process group must be initialized"
)
@@ -124,6 +132,7 @@ class WeightExporter:
return success, message
def save_remote_model(self: WeightExporter, url: str):
self._check_weight_export_supported()
from sglang.srt.model_loader.loader import RemoteModelLoader
logger.info(f"Saving model to {url}")
@@ -135,6 +144,7 @@ class WeightExporter:
pattern: Optional[str] = None,
max_size: Optional[int] = None,
):
self._check_weight_export_supported()
from sglang.srt.model_loader.loader import ShardedStateLoader
logger.info(
@@ -150,6 +160,7 @@ class WeightExporter:
Only used for unit test with an unoptimized performance.
For optimized performance, please use torch.save and torch.load.
"""
self._check_weight_export_supported()
# TODO: (chenyang) Add support for Qwen models.
try:
return self.get_model().get_weights_by_name(
@@ -1709,6 +1709,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
"""
num_tokens = size
forward_batch, attn_backend = self.capture_prepare(num_tokens)
# Recapture can follow decode replay, which may change model-owned storage.
prepare = getattr(self.model_runner.model, "prepare_for_forward", None)
if prepare is not None:
prepare(forward_batch)
if self.enable_cp_bcg_capture:
assert self.prefill_cp_bcg_input is not None
self.prefill_cp_bcg_input.prepare(
@@ -447,6 +447,9 @@ class NemotronH_Omni_Reasoning_V3(NemotronH_Nano_VL_V2):
else None
)
def prepare_for_forward(self, forward_batch: ForwardBatch) -> None:
self.language_model.prepare_for_forward(forward_batch)
@property
def lm_head(self):
return self.language_model.lm_head
+168 -3
View File
@@ -18,7 +18,7 @@
"""Inference-only NemotronH model."""
from collections.abc import Iterable
from functools import partial
from functools import lru_cache, partial
import torch
from torch import nn
@@ -26,6 +26,7 @@ from torch import nn
from sglang.srt.compilation.compilation_config import register_split_op
from sglang.srt.configs import NemotronHConfig
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MLP, MOE
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
@@ -65,7 +66,11 @@ from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
from sglang.srt.model_executor.forward_batch_info import (
ForwardBatch,
ForwardMode,
PPProxyTensors,
)
from sglang.srt.model_executor.forward_context import get_attn_backend
from sglang.srt.model_executor.runner import get_is_capture_mode
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
@@ -84,12 +89,13 @@ from sglang.srt.model_loader.weight_utils import (
)
from sglang.srt.models.nemotron_h_utils import make_stage_boundary, stage_facts
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.runtime_context import get_exec, get_parallel, get_schedule
from sglang.srt.utils import (
LazyValue,
add_prefix,
get_current_device_stream_fast,
is_cuda,
is_sm120,
make_pp_layers,
)
from sglang.srt.utils.custom_op import register_custom_op
@@ -157,6 +163,58 @@ def _get_or_create_alt_stream(device_module):
return _alt_stream
@lru_cache(maxsize=None)
def _prefill_shared_expert_pool(device_index: int):
# Shared experts overlap the main graph, so their intermediates must use
# a separate pool. Calls on the shared stream are ordered across layers.
with torch.cuda.device(device_index):
return torch.cuda.graph_pool_handle()
class _PrefillSharedExpertCall:
def __init__(self, stream):
self.graph = None
self.stream = stream
self.ready = torch.cuda.Event(external=True)
self.done = torch.cuda.Event(external=True)
self.output = None
def launch(self, module, hidden_states):
main_stream = torch.cuda.current_stream()
with torch.cuda.stream(self.stream):
if self.graph is None:
# BCG warmup follows the same dependencies, without capture.
self.ready.wait()
self.output = module(hidden_states)
self.done.record()
self.output.record_stream(main_stream)
else:
self.graph.replay()
return self.output
def _capture_prefill_shared_expert(module, hidden_states, call):
call.graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(
call.graph,
pool=_prefill_shared_expert_pool(hidden_states.device.index),
stream=call.stream,
):
call.ready.wait()
call.output = module(hidden_states)
call.done.record()
# The ready record belongs to the preceding main-stream graph. Do not
# replay here: that graph has only been captured, not launched yet.
return call.output
@eager_on_graph(True, capture_stub=_capture_prefill_shared_expert)
def _launch_prefill_shared_expert(module, hidden_states, call):
# BCG retains this call, its graph, events and output until graph cleanup.
# A replay launches on the same explicit stream for every layer.
return call.launch(module, hidden_states)
def _latent_proj_fuses_shared_add(projection: nn.Module) -> bool:
return (
# LoRA swaps a wrapper module over this attribute after init,
@@ -288,7 +346,23 @@ class NemotronHMoE(nn.Module):
self,
hidden_states: torch.Tensor,
num_token_non_padded: torch.Tensor | None,
forward_batch: ForwardBatch | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if (
forward_batch is not None
and forward_batch.forward_mode == ForwardMode.EXTEND
and hidden_states.shape == (1024, 4096)
and hidden_states.dtype == torch.bfloat16
and get_parallel().tp_size == self.ep_size == 1
and self.n_routed_experts == 512
and self.moe_hidden_size == 1024
and self.shared_experts is not None
and is_in_breakable_cuda_graph()
and is_sm120()
):
return self._forward_core_prefill_overlap(
hidden_states, num_token_non_padded
)
if _is_cuda and (
not get_moe_a2a_backend().is_flashinfer() or get_is_capture_mode()
):
@@ -363,6 +437,38 @@ class NemotronHMoE(nn.Module):
return final_hidden_states, shared_output
def _forward_core_prefill_overlap(
self,
hidden_states: torch.Tensor,
num_token_non_padded: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# A multi-stream graph per hybrid segment creates one internal stream
# per executable. Keep each graph linear and launch shared experts on
# one explicit stream, preserving overlap with the routed experts.
call = _PrefillSharedExpertCall(_get_or_create_alt_stream(self.device_module))
call.ready.record()
shared_output = _launch_prefill_shared_expert(
self.shared_experts, hidden_states, call
)
router_logits = torch.mm(
hidden_states, self.gate.weight.t(), out_dtype=torch.float32
)
topk_output = self.topk(
hidden_states,
router_logits,
num_token_non_padded=num_token_non_padded,
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if not self.is_nextn
else None
),
)
if self.use_latent_moe:
hidden_states = self._apply_fc1_latent_proj(hidden_states)
final_hidden_states = self.experts(hidden_states, topk_output)
call.done.wait()
return final_hidden_states, shared_output
def get_moe_weights(self) -> list[torch.Tensor]:
return [
x.data
@@ -404,6 +510,7 @@ class NemotronHMoE(nn.Module):
if forward_batch is not None
else None
),
forward_batch=forward_batch,
)
if self.use_latent_moe:
@@ -760,6 +867,10 @@ class NemotronHModel(nn.Module):
self.norm_f = PPMissingLayer(return_tuple=True)
self.layers_to_capture: set[int] = set()
@property
def supports_hybrid_prefill_graph(self) -> bool:
return is_sm120()
def forward(
self,
input_ids: torch.Tensor,
@@ -856,6 +967,7 @@ class NemotronHForCausalLM(nn.Module):
):
super().__init__()
lora_config = None
self._use_nvfp4_phase_layout = envs.SGLANG_SM120_NVFP4_PHASE_LAYOUT.get()
self.config = config
self.quant_config = quant_config
self.model = self._init_model(
@@ -1027,6 +1139,58 @@ class NemotronHForCausalLM(nn.Module):
f"get_hidden_dim not implemented for {module_name}"
)
def prepare_for_forward(self, forward_batch: ForwardBatch) -> None:
if not getattr(self, "_use_nvfp4_phase_layout", False):
return
from sglang.kernels.ops.gemm.sm120_nvfp4_moe import Nvfp4LayoutCycle
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
_flashinfer_autotune_active,
_sm120_mtp_mma_supported,
)
cycle = getattr(self, "_nvfp4_layout", None)
if cycle is None:
if not (
get_parallel().tp_size
== get_parallel().ep_size
== get_parallel().pp_size
== 1
and get_schedule().max_running_requests == 1
and _sm120_mtp_mma_supported()
):
raise ValueError(
"NVFP4 phase layout requires serial SM120 CUTLASS MoE with TP/EP/PP=1"
)
experts = [
m.experts for m in self.model.modules() if isinstance(m, NemotronHMoE)
]
if not experts or not all(
getattr(m.quant_method, "enable_flashinfer_cutlass_moe", False)
and m.w13_weight.shape == (512, 2688, 512)
for m in experts
):
raise ValueError(
"NVFP4 phase layout requires compatible native NVFP4 experts"
)
cycle = self._nvfp4_layout = Nvfp4LayoutCycle(
[m.w2_weight for m in experts]
)
for index, module in enumerate(experts):
module._nvfp4_layout_slot = (cycle, index)
self._derived_weight_cache_error = (
"Online weight updates require disabling NVFP4 phase layout"
)
self._weight_export_error = (
"Weight export requires disabling NVFP4 phase layout"
)
mode = forward_batch.forward_mode
cycle.ensure(
(mode.is_target_verify() or mode.is_decode_or_idle())
and forward_batch.input_ids.numel() <= 4
and not _flashinfer_autotune_active()
)
@torch.no_grad()
def forward(
self,
@@ -1036,6 +1200,7 @@ class NemotronHForCausalLM(nn.Module):
input_embeds: torch.Tensor | None = None,
pp_proxy_tensors: PPProxyTensors | None = None,
):
self.prepare_for_forward(forward_batch)
hidden_states = self.model.forward(
input_ids, positions, forward_batch, pp_proxy_tensors, input_embeds
)