[Feat] nemotron: Port Nemotron 3.5 Super serving changes (#44518)
Co-authored-by: BBuf <1182563586@qq.com> Co-authored-by: elvischenv <219235043+elvischenv@users.noreply.github.com> Co-authored-by: Jimmy Shong <jimmy.shong@mac.attlocal.net> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
BBuf
elvischenv
Jimmy Shong
Claude Opus 5.5
parent
f726461287
commit
7c9a78efbe
@@ -37,12 +37,14 @@ __device__ __forceinline__ float dot8_f32(const float4 wv, const float4 xv) {
|
||||
return acc;
|
||||
}
|
||||
|
||||
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps>
|
||||
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps, bool kUsePDL>
|
||||
__global__ void __launch_bounds__(kNumWarps * 32)
|
||||
hopper_bf16_gemv_kernel(bf16_t* __restrict__ out, const bf16_t* __restrict__ x, const bf16_t* __restrict__ w) {
|
||||
__shared__ bf16_t sx[K];
|
||||
|
||||
PDLTriggerSecondary<kUsePDL>();
|
||||
const uint32_t tid = threadIdx.x;
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
for (uint32_t i = tid * kGemvVecSize; i < K; i += kNumWarps * 32 * kGemvVecSize) {
|
||||
*reinterpret_cast<float4*>(sx + i) = *reinterpret_cast<const float4*>(x + i);
|
||||
}
|
||||
@@ -118,9 +120,11 @@ __global__ void __launch_bounds__(kNumWarps * 32)
|
||||
}
|
||||
}
|
||||
|
||||
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps>
|
||||
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps, bool kUsePDL>
|
||||
struct HopperBf16GemvKernel {
|
||||
static_assert(K % (32 * kGemvVecSize * kUnroll) == 0, "K must cover full unrolled warp strides");
|
||||
static_assert(
|
||||
kUnroll == 1 ? (K % kGemvVecSize == 0) : (K % (32 * kGemvVecSize * kUnroll) == 0),
|
||||
"K must be 16B-aligned (kUnroll==1) or cover full unrolled warp strides");
|
||||
static_assert(K * sizeof(bf16_t) <= 48 * 1024, "activation row must fit static shared memory");
|
||||
|
||||
static void run(const tvm::ffi::TensorView x, const tvm::ffi::TensorView w, const tvm::ffi::TensorView out) {
|
||||
@@ -134,11 +138,12 @@ struct HopperBf16GemvKernel {
|
||||
|
||||
constexpr uint32_t kRowsPerBlock = kRows * kNumWarps;
|
||||
constexpr uint32_t kNumBlocks = (N + kRowsPerBlock - 1) / kRowsPerBlock;
|
||||
LaunchKernel(kNumBlocks, kNumWarps * 32, device.unwrap())(
|
||||
hopper_bf16_gemv_kernel<N, K, kRows, kUnroll, kNumWarps>,
|
||||
static_cast<bf16_t*>(out.data_ptr()),
|
||||
static_cast<const bf16_t*>(x.data_ptr()),
|
||||
static_cast<const bf16_t*>(w.data_ptr()));
|
||||
LaunchKernel(kNumBlocks, kNumWarps * 32, device.unwrap())
|
||||
.enable_pdl(kUsePDL)(
|
||||
hopper_bf16_gemv_kernel<N, K, kRows, kUnroll, kNumWarps, kUsePDL>,
|
||||
static_cast<bf16_t*>(out.data_ptr()),
|
||||
static_cast<const bf16_t*>(x.data_ptr()),
|
||||
static_cast<const bf16_t*>(w.data_ptr()));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ __device__ __forceinline__ float dot16_fp8_f32(const float4 wv, const float4 xv)
|
||||
return acc;
|
||||
}
|
||||
|
||||
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps>
|
||||
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps, bool kUsePDL>
|
||||
__global__ void __launch_bounds__(kNumWarps * 32) sm120_fp8_gemv_kernel(
|
||||
bf16_t* __restrict__ out,
|
||||
const uint8_t* __restrict__ x,
|
||||
@@ -47,7 +47,9 @@ __global__ void __launch_bounds__(kNumWarps * 32) sm120_fp8_gemv_kernel(
|
||||
const float* __restrict__ alpha) {
|
||||
__shared__ uint8_t sx[K];
|
||||
|
||||
PDLTriggerSecondary<kUsePDL>();
|
||||
const uint32_t tid = threadIdx.x;
|
||||
PDLWaitPrimary<kUsePDL>();
|
||||
for (uint32_t i = tid * kFp8VecSize; i < K; i += kNumWarps * 32 * kFp8VecSize) {
|
||||
*reinterpret_cast<float4*>(sx + i) = *reinterpret_cast<const float4*>(x + i);
|
||||
}
|
||||
@@ -123,9 +125,11 @@ __global__ void __launch_bounds__(kNumWarps * 32) sm120_fp8_gemv_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps>
|
||||
template <uint32_t N, uint32_t K, uint32_t kRows, uint32_t kUnroll, uint32_t kNumWarps, bool kUsePDL>
|
||||
struct Sm120Fp8GemvKernel {
|
||||
static_assert(K % (32 * kFp8VecSize * kUnroll) == 0, "K must cover full unrolled warp strides");
|
||||
static_assert(
|
||||
kUnroll == 1 ? (K % kFp8VecSize == 0) : (K % (32 * kFp8VecSize * kUnroll) == 0),
|
||||
"K must be 16B-aligned (kUnroll==1) or cover full unrolled warp strides");
|
||||
static_assert(K <= 48 * 1024, "activation row must fit static shared memory");
|
||||
|
||||
static void
|
||||
@@ -144,12 +148,13 @@ struct Sm120Fp8GemvKernel {
|
||||
|
||||
constexpr uint32_t kRowsPerBlock = kRows * kNumWarps;
|
||||
constexpr uint32_t kNumBlocks = (N + kRowsPerBlock - 1) / kRowsPerBlock;
|
||||
LaunchKernel(kNumBlocks, kNumWarps * 32, device.unwrap())(
|
||||
sm120_fp8_gemv_kernel<N, K, kRows, kUnroll, kNumWarps>,
|
||||
static_cast<bf16_t*>(out.data_ptr()),
|
||||
static_cast<const uint8_t*>(x.data_ptr()),
|
||||
static_cast<const uint8_t*>(w.data_ptr()),
|
||||
static_cast<const float*>(alpha.data_ptr()));
|
||||
LaunchKernel(kNumBlocks, kNumWarps * 32, device.unwrap())
|
||||
.enable_pdl(kUsePDL)(
|
||||
sm120_fp8_gemv_kernel<N, K, kRows, kUnroll, kNumWarps, kUsePDL>,
|
||||
static_cast<bf16_t*>(out.data_ptr()),
|
||||
static_cast<const uint8_t*>(x.data_ptr()),
|
||||
static_cast<const uint8_t*>(w.data_ptr()),
|
||||
static_cast<const float*>(alpha.data_ptr()));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,410 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2025 - 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
// Native SM120 NVFP4 MMA operand layout follows NVIDIA CUTLASS mma_sm120.hpp
|
||||
// and mma_traits_sm120.hpp, distributed under BSD-3-Clause. Exact installed-source hashes are recorded in the research
|
||||
// task history/moe-mtp-native-upstream.json.
|
||||
#include <sgl_kernel/tensor.h>
|
||||
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
|
||||
#include <cstdint>
|
||||
#include <cuda_fp8.h>
|
||||
|
||||
namespace sglang {
|
||||
namespace nvfp4_mtp_mma {
|
||||
using namespace device;
|
||||
|
||||
__device__ __forceinline__ float rcp(float x) {
|
||||
float y;
|
||||
asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x));
|
||||
return y;
|
||||
}
|
||||
__device__ __forceinline__ uint32_t fp4x2(float hi, float lo) {
|
||||
uint32_t d;
|
||||
asm volatile("{ .reg .b8 t; cvt.rn.satfinite.e2m1x2.f32 t, %1, %2; cvt.u32.u8 %0, t; }" : "=r"(d) : "f"(hi), "f"(lo));
|
||||
return d;
|
||||
}
|
||||
__device__ __forceinline__ void quantize16(const float* v, uint32_t& q0, uint32_t& q1, uint8_t& sf, float global) {
|
||||
float amax = 0.f;
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < 16; ++i)
|
||||
amax = fmaxf(amax, fabsf(v[i]));
|
||||
__nv_fp8_e4m3 f(global * (amax * rcp(6.f)));
|
||||
sf = f.__x;
|
||||
float scale = amax != 0.f ? rcp(static_cast<float>(f) * rcp(global)) : 0.f;
|
||||
q0 = 0;
|
||||
q1 = 0;
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < 4; ++i) {
|
||||
q0 |= fp4x2(v[2 * i + 1] * scale, v[2 * i] * scale) << (8 * i);
|
||||
q1 |= fp4x2(v[8 + 2 * i + 1] * scale, v[8 + 2 * i] * scale) << (8 * i);
|
||||
}
|
||||
}
|
||||
template <uint32_t G>
|
||||
__device__ __forceinline__ uint32_t sf_index(uint32_t row, uint32_t group) {
|
||||
return (row / 128) * (128 * G) + (row % 32) * 16 + ((row % 128) / 32) * 4 + (group / 4) * 512 + group % 4;
|
||||
}
|
||||
__device__ __forceinline__ void
|
||||
mma(float& d0,
|
||||
float& d1,
|
||||
float& d2,
|
||||
float& d3,
|
||||
uint32_t a0,
|
||||
uint32_t a1,
|
||||
uint32_t a2,
|
||||
uint32_t a3,
|
||||
uint32_t b0,
|
||||
uint32_t b1,
|
||||
uint32_t sfa,
|
||||
uint32_t sfb) {
|
||||
asm volatile(
|
||||
"mma.sync.aligned.kind::mxf4nvf4.block_scale.scale_vec::4X.m16n8k64.row.col.f32.e2m1.e2m1.f32.ue4m3 "
|
||||
"{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3}, {%10}, {0,0}, {%11}, {0,0};"
|
||||
: "+f"(d0), "+f"(d1), "+f"(d2), "+f"(d3)
|
||||
: "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1), "r"(sfa), "r"(sfb));
|
||||
}
|
||||
|
||||
// Stage 256 K elements in contiguous 128B row segments. Padding each shared
|
||||
// row by 16B avoids the eight-way ldmatrix bank conflict at a 128B stride.
|
||||
template <uint32_t K>
|
||||
__device__ __forceinline__ void
|
||||
prefetch(uint8_t* shared, const uint8_t* weights, uint32_t tile, uint32_t step, uint32_t lane) {
|
||||
#pragma unroll
|
||||
for (uint32_t j = 0; j < 4; ++j) {
|
||||
uint32_t row = lane / 8 + 4 * j, col = (lane % 8) * 16;
|
||||
uint32_t dst = static_cast<uint32_t>(__cvta_generic_to_shared(shared + row * 144 + col));
|
||||
uint32_t source_col = step * 128 + col;
|
||||
uint32_t bytes = source_col < K / 2 ? 16 : 0;
|
||||
const uint8_t* src = weights + (tile + row) * (K / 2) + (bytes ? source_col : 0);
|
||||
asm volatile("cp.async.cg.shared.global [%0], [%1], 16, %2;" ::"r"(dst), "l"(src), "r"(bytes) : "memory");
|
||||
}
|
||||
asm volatile("cp.async.commit_group;" ::: "memory");
|
||||
}
|
||||
__device__ __forceinline__ void
|
||||
load_a(uint32_t& a0, uint32_t& a1, uint32_t& a2, uint32_t& a3, const uint8_t* shared, uint32_t lane) {
|
||||
uint32_t addr = static_cast<uint32_t>(__cvta_generic_to_shared(shared + (lane % 16) * 144 + (lane / 16) * 16));
|
||||
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];"
|
||||
: "=r"(a0), "=r"(a1), "=r"(a2), "=r"(a3)
|
||||
: "r"(addr)
|
||||
: "memory");
|
||||
}
|
||||
|
||||
template <uint32_t N, uint32_t K, uint32_t TopK, bool PDL>
|
||||
__global__ __launch_bounds__(128) void fc1(
|
||||
const bf16_t* __restrict__ x,
|
||||
const uint8_t* __restrict__ w,
|
||||
const uint8_t* __restrict__ sf,
|
||||
const float* __restrict__ alpha,
|
||||
const float* __restrict__ global1,
|
||||
const float* __restrict__ global2,
|
||||
const int32_t* __restrict__ ids,
|
||||
uint32_t* __restrict__ z,
|
||||
uint8_t* __restrict__ zsf) {
|
||||
__shared__ uint32_t xq[K / 8];
|
||||
__shared__ uint8_t xs[K / 16];
|
||||
__shared__ float act[4][16];
|
||||
__shared__ __align__(16) uint8_t ws[4][2][2304];
|
||||
uint32_t tid = threadIdx.x, warp = tid / 32, lane = tid % 32;
|
||||
uint32_t token = blockIdx.z, slot = blockIdx.y;
|
||||
PDLTriggerSecondary<PDL>();
|
||||
PDLWaitPrimary<PDL>();
|
||||
if (tid < K / 16) {
|
||||
float v[16];
|
||||
#pragma unroll
|
||||
for (uint32_t i = 0; i < 16; ++i)
|
||||
v[i] = static_cast<float>(x[token * K + tid * 16 + i]);
|
||||
quantize16(v, xq[2 * tid], xq[2 * tid + 1], xs[tid], *global1);
|
||||
}
|
||||
__syncthreads();
|
||||
uint32_t expert = ids[token * TopK + slot];
|
||||
uint32_t tile = (blockIdx.x * 4 + warp) * 16;
|
||||
uint32_t row = tile + lane / 4, col = (lane % 4) * 4;
|
||||
const uint8_t* weights = w + static_cast<size_t>(expert) * N * K / 2;
|
||||
const uint8_t* scales = sf + static_cast<size_t>(expert) * N * (K / 16);
|
||||
float d0 = 0, d1 = 0, d2 = 0, d3 = 0;
|
||||
#pragma unroll
|
||||
for (uint32_t stage = 0; stage < 2; ++stage)
|
||||
prefetch<K>(ws[warp][stage], weights, tile, stage, lane);
|
||||
#pragma unroll 1
|
||||
for (uint32_t base = 0; base < K; base += 256) {
|
||||
asm volatile("cp.async.wait_group 1;" ::: "memory");
|
||||
__syncwarp();
|
||||
#pragma unroll
|
||||
for (uint32_t inner = 0; inner < 256; inner += 64) {
|
||||
if (base + inner >= K) break;
|
||||
uint32_t k = base + inner;
|
||||
uint32_t a0, a1, a2, a3;
|
||||
load_a(a0, a1, a2, a3, ws[warp][(base / 256) % 2] + inner / 2, lane);
|
||||
uint32_t sa =
|
||||
*reinterpret_cast<const uint32_t*>(scales + sf_index<K / 16>(tile + lane / 4 + (lane % 2) * 8, k / 16));
|
||||
uint32_t sb = *reinterpret_cast<const uint32_t*>(xs + k / 16);
|
||||
mma(d0, d1, d2, d3, a0, a1, a2, a3, xq[k / 8 + lane % 4], xq[k / 8 + lane % 4 + 4], sa, sb);
|
||||
}
|
||||
__syncwarp();
|
||||
prefetch<K>(ws[warp][(base / 256) % 2], weights, tile, base / 256 + 2, lane);
|
||||
}
|
||||
asm volatile("cp.async.wait_group 0;" ::: "memory");
|
||||
if (lane % 4 == 0) {
|
||||
float y0 = static_cast<float>(cast<bf16_t>(d0 * alpha[expert]));
|
||||
float y1 = static_cast<float>(cast<bf16_t>(d2 * alpha[expert]));
|
||||
y0 = fmaxf(y0, 0.f);
|
||||
y1 = fmaxf(y1, 0.f);
|
||||
act[warp][lane / 4] = static_cast<float>(cast<bf16_t>(y0 * y0));
|
||||
act[warp][lane / 4 + 8] = static_cast<float>(cast<bf16_t>(y1 * y1));
|
||||
}
|
||||
__syncwarp();
|
||||
if (lane == 0) {
|
||||
size_t qoff = (token * TopK + slot) * (N / 8) + tile / 8;
|
||||
size_t soff = (token * TopK + slot) * (N / 16) + tile / 16;
|
||||
quantize16(act[warp], z[qoff], z[qoff + 1], zsf[soff], *global2);
|
||||
}
|
||||
}
|
||||
|
||||
template <uint32_t N, uint32_t K, uint32_t TopK, bool PDL>
|
||||
__global__ __launch_bounds__(128) void fc2(
|
||||
const uint32_t* __restrict__ z,
|
||||
const uint8_t* __restrict__ zs,
|
||||
const uint8_t* __restrict__ w,
|
||||
const uint8_t* __restrict__ sf,
|
||||
const float* __restrict__ alpha,
|
||||
const int32_t* __restrict__ ids,
|
||||
const float* __restrict__ routing,
|
||||
bf16_t* __restrict__ partial) {
|
||||
__shared__ __align__(16) uint8_t ws[4][2][2304];
|
||||
uint32_t warp = threadIdx.x / 32, lane = threadIdx.x % 32;
|
||||
uint32_t token = blockIdx.z, slot = blockIdx.y, route = token * TopK + slot;
|
||||
PDLTriggerSecondary<PDL>();
|
||||
PDLWaitPrimary<PDL>();
|
||||
uint32_t expert = ids[route], tile = (blockIdx.x * 4 + warp) * 16;
|
||||
uint32_t row = tile + lane / 4, col = (lane % 4) * 4;
|
||||
const uint8_t* weights = w + static_cast<size_t>(expert) * N * K / 2;
|
||||
const uint8_t* scales = sf + static_cast<size_t>(expert) * N * (K / 16);
|
||||
z += route * (K / 8);
|
||||
zs += route * (K / 16);
|
||||
float d0 = 0, d1 = 0, d2 = 0, d3 = 0;
|
||||
#pragma unroll
|
||||
for (uint32_t stage = 0; stage < 2; ++stage)
|
||||
prefetch<K>(ws[warp][stage], weights, tile, stage, lane);
|
||||
#pragma unroll 1
|
||||
for (uint32_t base = 0; base < K; base += 256) {
|
||||
asm volatile("cp.async.wait_group 1;" ::: "memory");
|
||||
__syncwarp();
|
||||
#pragma unroll
|
||||
for (uint32_t inner = 0; inner < 256; inner += 64) {
|
||||
if (base + inner >= K) break;
|
||||
uint32_t k = base + inner;
|
||||
uint32_t a0, a1, a2, a3;
|
||||
load_a(a0, a1, a2, a3, ws[warp][(base / 256) % 2] + inner / 2, lane);
|
||||
uint32_t sa =
|
||||
*reinterpret_cast<const uint32_t*>(scales + sf_index<K / 16>(tile + lane / 4 + (lane % 2) * 8, k / 16));
|
||||
uint32_t sb = *reinterpret_cast<const uint32_t*>(zs + k / 16);
|
||||
mma(d0, d1, d2, d3, a0, a1, a2, a3, z[k / 8 + lane % 4], z[k / 8 + lane % 4 + 4], sa, sb);
|
||||
}
|
||||
__syncwarp();
|
||||
prefetch<K>(ws[warp][(base / 256) % 2], weights, tile, base / 256 + 2, lane);
|
||||
}
|
||||
asm volatile("cp.async.wait_group 0;" ::: "memory");
|
||||
if (lane % 4 == 0) {
|
||||
partial[route * N + row] = cast<bf16_t>((d0 * alpha[expert]) * routing[route]);
|
||||
partial[route * N + row + 8] = cast<bf16_t>((d2 * alpha[expert]) * routing[route]);
|
||||
}
|
||||
}
|
||||
template <uint32_t N, uint32_t TopK, bool PDL>
|
||||
__global__ void reduce(const bf16_t* __restrict__ partial, bf16_t* __restrict__ out, uint32_t rows) {
|
||||
PDLTriggerSecondary<PDL>();
|
||||
uint32_t i = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
PDLWaitPrimary<PDL>();
|
||||
if (i < rows * N) {
|
||||
uint32_t token = i / N, col = i % N;
|
||||
float value = 0.f;
|
||||
#pragma unroll
|
||||
for (uint32_t slot = 0; slot < TopK; ++slot)
|
||||
value += static_cast<float>(partial[(token * TopK + slot) * N + col]);
|
||||
out[i] = cast<bf16_t>(value);
|
||||
}
|
||||
}
|
||||
} // namespace nvfp4_mtp_mma
|
||||
|
||||
namespace nvfp4_pair_mma {
|
||||
using namespace nvfp4_mtp_mma;
|
||||
// Store each original pair of FP4 weight bits across contiguous output channels.
|
||||
// One 16-byte fetch then corresponds to exactly two activation values, enabling
|
||||
// zero-filled asynchronous loads without discarding any nonzero multiplication.
|
||||
template <unsigned N, unsigned K>
|
||||
__device__ __forceinline__ void
|
||||
pair_prefetch(uint8_t* shared, const uint8_t* weights, unsigned tile, unsigned step, unsigned lane, const uint32_t* z) {
|
||||
#pragma unroll
|
||||
for (unsigned j = 0; j < 2; ++j) {
|
||||
unsigned pair = lane + 32 * j, source_pair = step * 64 + pair;
|
||||
unsigned bytes = source_pair < K / 2 ? 16 : 0;
|
||||
if (bytes && (reinterpret_cast<const uint8_t*>(z)[source_pair] & 0x77u) == 0) bytes = 0;
|
||||
const uint8_t* src = weights + (bytes ? source_pair : 0) * N + tile;
|
||||
unsigned dst = static_cast<unsigned>(__cvta_generic_to_shared(shared + pair * 16));
|
||||
asm volatile("cp.async.cg.shared.global [%0], [%1], 16, %2;" ::"r"(dst), "l"(src), "r"(bytes) : "memory");
|
||||
}
|
||||
asm volatile("cp.async.commit_group;" ::: "memory");
|
||||
}
|
||||
__device__ __forceinline__ void
|
||||
pair_load_a(unsigned& a0, unsigned& a1, unsigned& a2, unsigned& a3, const uint8_t* s, unsigned lane) {
|
||||
unsigned addr = static_cast<unsigned>(__cvta_generic_to_shared(s + lane * 16));
|
||||
asm volatile("ldmatrix.sync.aligned.m16n16.x2.trans.shared.b8 {%0,%1,%2,%3}, [%4];"
|
||||
: "=r"(a0), "=r"(a1), "=r"(a2), "=r"(a3)
|
||||
: "r"(addr)
|
||||
: "memory");
|
||||
}
|
||||
template <uint32_t N, uint32_t K, uint32_t TopK, bool PDL>
|
||||
__global__ __launch_bounds__(128) void fc2(
|
||||
const uint32_t* __restrict__ z,
|
||||
const uint8_t* __restrict__ zs,
|
||||
const uint8_t* __restrict__ w,
|
||||
const uint8_t* __restrict__ sf,
|
||||
const float* __restrict__ alpha,
|
||||
const int32_t* __restrict__ ids,
|
||||
const float* __restrict__ routing,
|
||||
bf16_t* __restrict__ partial) {
|
||||
__shared__ __align__(16) uint8_t ws[4][2][1024];
|
||||
uint32_t warp = threadIdx.x / 32, lane = threadIdx.x % 32;
|
||||
uint32_t token = blockIdx.z, slot = blockIdx.y, route = token * TopK + slot;
|
||||
PDLTriggerSecondary<PDL>();
|
||||
PDLWaitPrimary<PDL>();
|
||||
uint32_t expert = ids[route], tile = (blockIdx.x * 4 + warp) * 16;
|
||||
uint32_t row = tile + lane / 4, col = (lane % 4) * 4;
|
||||
const uint8_t* weights = w + static_cast<size_t>(expert) * N * K / 2;
|
||||
const uint8_t* scales = sf + static_cast<size_t>(expert) * N * (K / 16);
|
||||
z += route * (K / 8);
|
||||
zs += route * (K / 16);
|
||||
float d0 = 0, d1 = 0, d2 = 0, d3 = 0;
|
||||
#pragma unroll
|
||||
for (uint32_t stage = 0; stage < 2; ++stage)
|
||||
pair_prefetch<N, K>(ws[warp][stage], weights, tile, stage, lane, z);
|
||||
#pragma unroll 1
|
||||
for (uint32_t base = 0; base < K; base += 128) {
|
||||
asm volatile("cp.async.wait_group 1;" ::: "memory");
|
||||
__syncwarp();
|
||||
#pragma unroll
|
||||
for (uint32_t inner = 0; inner < 128; inner += 64) {
|
||||
if (base + inner >= K) break;
|
||||
uint32_t k = base + inner;
|
||||
uint32_t a0, a1, a2, a3;
|
||||
pair_load_a(a0, a1, a2, a3, ws[warp][(base / 128) % 2] + (inner / 2) * 16, lane);
|
||||
uint32_t sa =
|
||||
*reinterpret_cast<const uint32_t*>(scales + sf_index<K / 16>(tile + lane / 4 + (lane % 2) * 8, k / 16));
|
||||
uint32_t sb = *reinterpret_cast<const uint32_t*>(zs + k / 16);
|
||||
mma(d0, d1, d2, d3, a0, a1, a2, a3, z[k / 8 + lane % 4], z[k / 8 + lane % 4 + 4], sa, sb);
|
||||
}
|
||||
__syncwarp();
|
||||
pair_prefetch<N, K>(ws[warp][(base / 128) % 2], weights, tile, base / 128 + 2, lane, z);
|
||||
}
|
||||
asm volatile("cp.async.wait_group 0;" ::: "memory");
|
||||
if (lane % 4 == 0) {
|
||||
partial[route * N + row] = cast<bf16_t>((d0 * alpha[expert]) * routing[route]);
|
||||
partial[route * N + row + 8] = cast<bf16_t>((d2 * alpha[expert]) * routing[route]);
|
||||
}
|
||||
}
|
||||
} // namespace nvfp4_pair_mma
|
||||
|
||||
template <uint32_t N1, uint32_t K1, uint32_t N2, uint32_t K2, uint32_t TopK, bool PDL, bool PairLayout>
|
||||
struct Sm120Nvfp4MoeMtpMma {
|
||||
static_assert(N1 == 2688 && K1 == 1024 && N2 == 1024 && K2 == 2688 && TopK == 22);
|
||||
/// Run 1 to 4 token rows with BF16 epilogues and deterministic reduction.
|
||||
static void
|
||||
run(tvm::ffi::TensorView x,
|
||||
tvm::ffi::TensorView w1,
|
||||
tvm::ffi::TensorView s1,
|
||||
tvm::ffi::TensorView w2,
|
||||
tvm::ffi::TensorView s2,
|
||||
tvm::ffi::TensorView a1,
|
||||
tvm::ffi::TensorView a2,
|
||||
tvm::ffi::TensorView g1,
|
||||
tvm::ffi::TensorView g2,
|
||||
tvm::ffi::TensorView ids,
|
||||
tvm::ffi::TensorView routing,
|
||||
tvm::ffi::TensorView z,
|
||||
tvm::ffi::TensorView zs,
|
||||
tvm::ffi::TensorView partial,
|
||||
tvm::ffi::TensorView out) {
|
||||
using namespace host;
|
||||
using namespace device;
|
||||
auto dev = SymbolicDevice{};
|
||||
dev.set_options<kDLCUDA>();
|
||||
auto rows = SymbolicSize{"token_rows"};
|
||||
TensorMatcher({rows, K1}).with_dtype<bf16_t>().with_device(dev).verify(x);
|
||||
CHECK_HOST(rows.unwrap() >= 1 && rows.unwrap() <= 4) << "Expected 1 <= M <= 4";
|
||||
const uint32_t M = static_cast<uint32_t>(rows.unwrap());
|
||||
TensorMatcher({512, N1, K1 / 2}).with_dtype<uint8_t>().with_device(dev).verify(w1);
|
||||
if constexpr (PairLayout)
|
||||
TensorMatcher({512, K2 / 2, N2}).with_dtype<uint8_t>().with_device(dev).verify(w2);
|
||||
else
|
||||
TensorMatcher({512, N2, K2 / 2}).with_dtype<uint8_t>().with_device(dev).verify(w2);
|
||||
TensorMatcher({512 * N1 * K1 / 16}).with_dtype<uint8_t>().with_device(dev).verify(s1);
|
||||
TensorMatcher({512 * N2 * K2 / 16}).with_dtype<uint8_t>().with_device(dev).verify(s2);
|
||||
TensorMatcher({512}).with_dtype<float>().with_device(dev).verify(a1).verify(a2);
|
||||
TensorMatcher({1}).with_dtype<float>().with_device(dev).verify(g1).verify(g2);
|
||||
TensorMatcher({M, TopK}).with_dtype<int32_t>().with_device(dev).verify(ids);
|
||||
TensorMatcher({M, TopK}).with_dtype<float>().with_device(dev).verify(routing);
|
||||
TensorMatcher({M * TopK * N1 / 8}).with_dtype<int32_t>().with_device(dev).verify(z);
|
||||
TensorMatcher({M * TopK * N1 / 16}).with_dtype<uint8_t>().with_device(dev).verify(zs);
|
||||
TensorMatcher({M * TopK * N2}).with_dtype<bf16_t>().with_device(dev).verify(partial);
|
||||
TensorMatcher({M, N2}).with_dtype<bf16_t>().with_device(dev).verify(out);
|
||||
LaunchKernel({N1 / 64, TopK, M}, 128, dev.unwrap())
|
||||
.enable_pdl(PDL)(
|
||||
nvfp4_mtp_mma::fc1<N1, K1, TopK, PDL>,
|
||||
static_cast<const bf16_t*>(x.data_ptr()),
|
||||
static_cast<const uint8_t*>(w1.data_ptr()),
|
||||
static_cast<const uint8_t*>(s1.data_ptr()),
|
||||
static_cast<const float*>(a1.data_ptr()),
|
||||
static_cast<const float*>(g1.data_ptr()),
|
||||
static_cast<const float*>(g2.data_ptr()),
|
||||
static_cast<const int32_t*>(ids.data_ptr()),
|
||||
static_cast<uint32_t*>(z.data_ptr()),
|
||||
static_cast<uint8_t*>(zs.data_ptr()));
|
||||
LaunchKernel({N2 / 64, TopK, M}, 128, dev.unwrap())
|
||||
.enable_pdl(PDL)(
|
||||
PairLayout ? nvfp4_pair_mma::fc2<N2, K2, TopK, PDL> : nvfp4_mtp_mma::fc2<N2, K2, TopK, PDL>,
|
||||
static_cast<const uint32_t*>(z.data_ptr()),
|
||||
static_cast<const uint8_t*>(zs.data_ptr()),
|
||||
static_cast<const uint8_t*>(w2.data_ptr()),
|
||||
static_cast<const uint8_t*>(s2.data_ptr()),
|
||||
static_cast<const float*>(a2.data_ptr()),
|
||||
static_cast<const int32_t*>(ids.data_ptr()),
|
||||
static_cast<const float*>(routing.data_ptr()),
|
||||
static_cast<bf16_t*>(partial.data_ptr()));
|
||||
LaunchKernel({(M * N2 + 255) / 256}, 256, dev.unwrap())
|
||||
.enable_pdl(PDL)(
|
||||
nvfp4_mtp_mma::reduce<N2, TopK, PDL>,
|
||||
static_cast<const bf16_t*>(partial.data_ptr()),
|
||||
static_cast<bf16_t*>(out.data_ptr()),
|
||||
M);
|
||||
}
|
||||
};
|
||||
} // namespace sglang
|
||||
@@ -0,0 +1,49 @@
|
||||
// Exact byte transpose for changing weight layout once per request phase.
|
||||
#include <sgl_kernel/tensor.h>
|
||||
|
||||
#include <sgl_kernel/type.cuh>
|
||||
#include <sgl_kernel/utils.cuh>
|
||||
namespace sglang {
|
||||
namespace nvfp4_weight_transpose {
|
||||
template <unsigned Rows, unsigned Cols, unsigned Tile>
|
||||
__global__ __launch_bounds__(256) void kernel(const uint8_t* __restrict__ x, uint8_t* __restrict__ y) {
|
||||
__shared__ uint8_t data[Tile][Tile + 4];
|
||||
unsigned tx = threadIdx.x % 32, ty = threadIdx.x / 32;
|
||||
unsigned rbase = blockIdx.y * Tile, cbase = blockIdx.x * Tile;
|
||||
size_t expert = static_cast<size_t>(blockIdx.z) * Rows * Cols;
|
||||
#pragma unroll
|
||||
for (unsigned j = 0; j < Tile / 8; ++j) {
|
||||
unsigned row = rbase + ty + j * 8, col = cbase + tx * 4;
|
||||
unsigned v = 0;
|
||||
if (row < Rows && col < Cols) v = *reinterpret_cast<const unsigned*>(x + expert + row * Cols + col);
|
||||
*reinterpret_cast<unsigned*>(&data[ty + j * 8][tx * 4]) = v;
|
||||
}
|
||||
__syncthreads();
|
||||
#pragma unroll
|
||||
for (unsigned j = 0; j < Tile / 8; ++j) {
|
||||
unsigned row = cbase + ty + j * 8, col = rbase + tx * 4;
|
||||
if (row < Cols && col < Rows) {
|
||||
unsigned v = static_cast<unsigned>(data[tx * 4 + 0][ty + j * 8]) |
|
||||
(static_cast<unsigned>(data[tx * 4 + 1][ty + j * 8]) << 8) |
|
||||
(static_cast<unsigned>(data[tx * 4 + 2][ty + j * 8]) << 16) |
|
||||
(static_cast<unsigned>(data[tx * 4 + 3][ty + j * 8]) << 24);
|
||||
*reinterpret_cast<unsigned*>(y + expert + row * Rows + col) = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace nvfp4_weight_transpose
|
||||
template <unsigned Rows, unsigned Cols>
|
||||
struct Nvfp4WeightTranspose {
|
||||
static void run(tvm::ffi::TensorView x, tvm::ffi::TensorView y) {
|
||||
using namespace host;
|
||||
auto dev = SymbolicDevice{};
|
||||
dev.set_options<kDLCUDA>();
|
||||
TensorMatcher({512, Rows, Cols}).with_dtype<uint8_t>().with_device(dev).verify(x);
|
||||
TensorMatcher({512, Cols, Rows}).with_dtype<uint8_t>().with_device(dev).verify(y);
|
||||
LaunchKernel({(Cols + 127) / 128, (Rows + 127) / 128, 512}, 256, dev.unwrap())(
|
||||
nvfp4_weight_transpose::kernel<Rows, Cols, 128>,
|
||||
static_cast<const uint8_t*>(x.data_ptr()),
|
||||
static_cast<uint8_t*>(y.data_ptr()));
|
||||
}
|
||||
};
|
||||
} // namespace sglang
|
||||
@@ -4,7 +4,12 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
|
||||
from sglang.kernels.jit.utils import (
|
||||
cache_once,
|
||||
is_arch_support_pdl,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
@@ -20,8 +25,18 @@ _MAX_K = 17408 # static smem limit (K * 2 bytes <= 48KB) with margin
|
||||
_MAX_N = 65536 # very large N (lm_head) is already at the bandwidth ceiling
|
||||
|
||||
|
||||
def _is_sm12x() -> bool:
|
||||
from sglang.kernels.jit.utils import get_jit_cuda_arch
|
||||
|
||||
return get_jit_cuda_arch().major >= 12
|
||||
|
||||
|
||||
def _config(n: int) -> tuple[int, int, int]:
|
||||
"""(rows_per_warp, k_unroll, num_warps) tuned on H200."""
|
||||
"""(rows_per_warp, k_unroll, num_warps) tuned on H200 (SM90) and
|
||||
DRAM-cold on RTX PRO 6000 (SM120), where kUnroll must stay 1 so the
|
||||
in-loop k < K guard also serves K % 512 != 0 shapes."""
|
||||
if _is_sm12x():
|
||||
return (1, 1, 8)
|
||||
if n >= 8192:
|
||||
return (2, 2, 8)
|
||||
return (1, 2, 8)
|
||||
@@ -30,7 +45,7 @@ def _config(n: int) -> tuple[int, int, int]:
|
||||
@cache_once
|
||||
def _jit_hopper_bf16_gemv_module(n: int, k: int) -> Module:
|
||||
rows, unroll, warps = _config(n)
|
||||
args = make_cpp_args(n, k, rows, unroll, warps)
|
||||
args = make_cpp_args(n, k, rows, unroll, warps, is_arch_support_pdl())
|
||||
return load_jit(
|
||||
"hopper_bf16_gemv",
|
||||
*args,
|
||||
@@ -41,6 +56,21 @@ def _jit_hopper_bf16_gemv_module(n: int, k: int) -> Module:
|
||||
|
||||
|
||||
def use_hopper_bf16_gemv(m: int, n: int, k: int) -> bool:
|
||||
if _is_sm12x():
|
||||
# SM120 (GDDR7): kUnroll==1 needs only 16B-aligned K (128 kept for
|
||||
# vector-alignment margin). cuBLAS M=1 bf16 trails the streaming GEMV
|
||||
# on small/odd N and very wide N; the 12288..32768 window (e.g. the
|
||||
# 18560-row bf16 in_proj) already runs at the DRAM ceiling under
|
||||
# cuBLAS, so leave it there.
|
||||
if not (
|
||||
m == 1
|
||||
and k % 128 == 0
|
||||
and 512 <= k <= _MAX_K
|
||||
and n % 8 == 0
|
||||
and 64 <= n <= 140000
|
||||
):
|
||||
return False
|
||||
return n < 12288 or n >= 32768
|
||||
if not (
|
||||
m == 1
|
||||
and k % 512 == 0
|
||||
|
||||
@@ -4,7 +4,12 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
|
||||
from sglang.kernels.jit.utils import (
|
||||
cache_once,
|
||||
is_arch_support_pdl,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tvm_ffi.module import Module
|
||||
@@ -22,16 +27,16 @@ _MAX_N = 32768 # very large N is served well by cuBLAS already
|
||||
|
||||
|
||||
def _config(n: int) -> tuple[int, int, int]:
|
||||
"""(rows_per_warp, k_unroll, num_warps)."""
|
||||
"""(rows_per_warp, k_unroll, num_warps); tuned DRAM-cold on RTX PRO 6000."""
|
||||
if n >= 8192:
|
||||
return (2, 1, 8)
|
||||
return (4, 1, 8)
|
||||
return (1, 1, 8)
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_sm120_fp8_gemv_module(n: int, k: int) -> Module:
|
||||
rows, unroll, warps = _config(n)
|
||||
args = make_cpp_args(n, k, rows, unroll, warps)
|
||||
args = make_cpp_args(n, k, rows, unroll, warps, is_arch_support_pdl())
|
||||
return load_jit(
|
||||
"sm120_fp8_gemv",
|
||||
*args,
|
||||
@@ -42,9 +47,12 @@ def _jit_sm120_fp8_gemv_module(n: int, k: int) -> Module:
|
||||
|
||||
|
||||
def use_sm120_fp8_gemv(m: int, n: int, k: int) -> bool:
|
||||
# kUnroll == 1 streams 16B per lane-step with an in-loop k < K guard, so
|
||||
# K only needs 16B alignment; keep 128 for conservative vector alignment
|
||||
# (covers the K=5376 shared-expert down_proj).
|
||||
return (
|
||||
m == 1
|
||||
and k % 512 == 0
|
||||
and k % 128 == 0
|
||||
and 512 <= k <= _MAX_K
|
||||
and n % 16 == 0
|
||||
and 256 <= n <= _MAX_N
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import (
|
||||
cache_once,
|
||||
load_jit,
|
||||
make_cpp_args,
|
||||
)
|
||||
|
||||
|
||||
@cache_once
|
||||
def _mtp_module(pair_layout: bool = False):
|
||||
args = make_cpp_args(2688, 1024, 1024, 2688, 22, False, pair_layout)
|
||||
return load_jit(
|
||||
"sm120_nvfp4_moe_mtp_mma",
|
||||
*args,
|
||||
cuda_files=["gemm/sm120_nvfp4_moe_mtp_mma.cuh"],
|
||||
cuda_wrappers=[("run", f"sglang::Sm120Nvfp4MoeMtpMma<{args}>::run")],
|
||||
extra_cuda_cflags=["-O3"],
|
||||
)
|
||||
|
||||
|
||||
def nvfp4_moe_mtp_mma(
|
||||
x, w1, s1, w2, s2, a1, a2, g1, g2, ids, routing, out=None, *, pair_layout=False
|
||||
):
|
||||
m = x.shape[0]
|
||||
module = _mtp_module(pair_layout)
|
||||
if out is None:
|
||||
out = torch.empty((m, 1024), dtype=torch.bfloat16, device=x.device)
|
||||
z = torch.empty(m * 22 * 2688 // 8, dtype=torch.int32, device=x.device)
|
||||
zs = torch.empty(m * 22 * 2688 // 16, dtype=torch.uint8, device=x.device)
|
||||
partial = torch.empty(m * 22 * 1024, dtype=torch.bfloat16, device=x.device)
|
||||
module.run(
|
||||
x,
|
||||
w1.view(torch.uint8),
|
||||
s1.view(torch.uint8).view(-1),
|
||||
w2.view(torch.uint8),
|
||||
s2.view(torch.uint8).view(-1),
|
||||
a1.view(-1),
|
||||
a2.view(-1),
|
||||
g1.view(-1),
|
||||
g2.view(-1),
|
||||
ids.to(torch.int32),
|
||||
routing.float(),
|
||||
z,
|
||||
zs,
|
||||
partial,
|
||||
out,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def try_nvfp4_moe(
|
||||
x,
|
||||
w13,
|
||||
w2,
|
||||
scales,
|
||||
ids,
|
||||
routing,
|
||||
*,
|
||||
output_dtype,
|
||||
out=None,
|
||||
pair_w2_weight=None,
|
||||
use_mtp=True,
|
||||
):
|
||||
"""Select native small-M kernels; unsupported shapes retain CUTLASS."""
|
||||
if (
|
||||
use_mtp
|
||||
and x.ndim == 2
|
||||
and 1 <= x.shape[0] <= 4
|
||||
and x.shape[1] == 1024
|
||||
and x.dtype == output_dtype == torch.bfloat16
|
||||
and w13.shape == (512, 2688, 512)
|
||||
and w2.shape == (512, 1024, 1344)
|
||||
and ids.shape == (x.shape[0], 22)
|
||||
):
|
||||
return nvfp4_moe_mtp_mma(
|
||||
x,
|
||||
w13,
|
||||
scales[1],
|
||||
pair_w2_weight if pair_w2_weight is not None else w2,
|
||||
scales[4],
|
||||
scales[2],
|
||||
scales[5],
|
||||
scales[0],
|
||||
scales[3],
|
||||
ids,
|
||||
routing,
|
||||
out=out,
|
||||
pair_layout=pair_w2_weight is not None,
|
||||
)
|
||||
if pair_w2_weight is not None:
|
||||
raise RuntimeError("A paired NVFP4 weight reached an unsupported MoE path")
|
||||
return None
|
||||
|
||||
|
||||
@cache_once
|
||||
def _transpose_module(rows: int, cols: int):
|
||||
return load_jit(
|
||||
"sm120_nvfp4_weight_transpose",
|
||||
rows,
|
||||
cols,
|
||||
cuda_files=["gemm/sm120_nvfp4_weight_transpose.cuh"],
|
||||
cuda_wrappers=[("run", f"sglang::Nvfp4WeightTranspose<{rows},{cols}>::run")],
|
||||
extra_cuda_cflags=["-O3"],
|
||||
)
|
||||
|
||||
|
||||
class Nvfp4LayoutCycle:
|
||||
"""Own the two views of a fixed, serial set of [512,1024,1344] weights."""
|
||||
|
||||
def __init__(self, weights):
|
||||
if not weights:
|
||||
raise ValueError("A layout cycle requires at least one expert weight")
|
||||
first = weights[0]
|
||||
if first.device.type != "cuda":
|
||||
raise ValueError("The layout cycle requires CUDA weights")
|
||||
for weight in weights:
|
||||
if (
|
||||
weight.shape != (512, 1024, 1344)
|
||||
or weight.dtype != torch.uint8
|
||||
or not weight.is_contiguous()
|
||||
or weight.device != first.device
|
||||
):
|
||||
raise ValueError("Incompatible NVFP4 FC2 weight layout")
|
||||
if len({weight.data_ptr() for weight in weights}) != len(weights):
|
||||
raise ValueError("Expert weights in a layout cycle must not alias")
|
||||
self.original = tuple(weights)
|
||||
self.workspace_bytes = first.numel()
|
||||
self.scratch = None
|
||||
self.pair = ()
|
||||
self.is_pair = False
|
||||
self.switches = 0
|
||||
# Compile before any graph capture can reach the state transition.
|
||||
self._forward = _transpose_module(1024, 1344)
|
||||
self._inverse = _transpose_module(1344, 1024)
|
||||
|
||||
def prepare_workspace(self):
|
||||
if self.scratch is not None:
|
||||
return
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
raise RuntimeError("NVFP4 workspace must be prepared before graph capture")
|
||||
# Use runtime workspace headroom, like graph/kernel scratch allocations.
|
||||
# Allocating this before Mamba/KV pool sizing incorrectly charges it as
|
||||
# persistent checkpoint storage and can round the state pool below one
|
||||
# request. The normal graph warmup prepares stable pointers before use.
|
||||
self.scratch = torch.empty_like(self.original[0])
|
||||
self.pair = tuple(
|
||||
weight.view(512, 1344, 1024)
|
||||
for weight in (self.scratch, *self.original[:-1])
|
||||
)
|
||||
|
||||
def ensure(self, pair: bool):
|
||||
if pair == self.is_pair:
|
||||
return
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
raise RuntimeError("NVFP4 layout transitions must run before graph capture")
|
||||
if pair:
|
||||
self.prepare_workspace()
|
||||
for original, paired in zip(self.original, self.pair):
|
||||
self._forward.run(original, paired)
|
||||
else:
|
||||
for original, paired in reversed(tuple(zip(self.original, self.pair))):
|
||||
self._inverse.run(paired, original)
|
||||
self.is_pair = pair
|
||||
self.switches += 1
|
||||
@@ -18,6 +18,85 @@ _SCORING_FUNC_MAP = {
|
||||
}
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _sm120_router_top22_kernel(
|
||||
scores_ptr,
|
||||
bias_ptr,
|
||||
weights_ptr,
|
||||
ids_ptr,
|
||||
stride_sm,
|
||||
stride_sn,
|
||||
scale,
|
||||
RENORMALIZE: tl.constexpr,
|
||||
APPLY_SCALE: tl.constexpr,
|
||||
USE_PDL: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
n = tl.arange(0, 512)
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_wait()
|
||||
scores = tl.load(scores_ptr + row * stride_sm + n * stride_sn).to(tl.float32)
|
||||
bias = tl.load(bias_ptr + n).to(tl.float32)
|
||||
activated = tl.sigmoid(scores)
|
||||
biased = activated + bias
|
||||
biased = tl.where(biased == biased, biased, -1e30)
|
||||
biased = tl.where(biased == 0.0, 0.0, biased)
|
||||
bits = biased.to(tl.uint32, bitcast=True)
|
||||
ordered = tl.where((bits & 0x80000000) != 0, ~bits, bits ^ 0x80000000)
|
||||
keys = (ordered.to(tl.uint64) << 32) | (0xFFFFFFFF - n.to(tl.uint32)).to(tl.uint64)
|
||||
winners = tl.topk(keys, 32)
|
||||
ids = (0xFFFFFFFF - (winners & 0xFFFFFFFF).to(tl.uint32)).to(tl.int32)
|
||||
k = tl.arange(0, 32)
|
||||
values = tl.gather(activated, ids, axis=0)
|
||||
values = tl.where(k < 22, values, 0.0)
|
||||
total = tl.sum(values, 0)
|
||||
if USE_PDL:
|
||||
tl.extra.cuda.gdc_launch_dependents()
|
||||
if RENORMALIZE:
|
||||
values = values / tl.where(total > 0.0, total, 1.0)
|
||||
if APPLY_SCALE:
|
||||
values = values * scale
|
||||
tl.store(weights_ptr + row * 22 + k, values, k < 22)
|
||||
tl.store(ids_ptr + row * 22 + k, ids, k < 22)
|
||||
|
||||
|
||||
def _topk_gate_small_m(
|
||||
scores,
|
||||
bias,
|
||||
topk=22,
|
||||
scoring_func="sigmoid",
|
||||
num_fused_shared_experts=0,
|
||||
renormalize=True,
|
||||
routed_scaling_factor=1.0,
|
||||
apply_routed_scaling_factor_on_output=False,
|
||||
):
|
||||
assert scores.is_cuda and bias.device == scores.device
|
||||
assert scores.ndim == 2 and scores.shape[1] == 512
|
||||
assert scores.dtype == torch.float32
|
||||
assert 1 <= scores.shape[0] <= 16
|
||||
assert bias.shape == (512,) and bias.dtype == torch.float32 and bias.is_contiguous()
|
||||
assert topk == 22 and scoring_func == "sigmoid" and num_fused_shared_experts == 0
|
||||
m = scores.shape[0]
|
||||
weights = torch.empty((m, 22), device=scores.device, dtype=torch.float32)
|
||||
ids = torch.empty((m, 22), device=scores.device, dtype=torch.int32)
|
||||
pdl = is_arch_support_pdl()
|
||||
_sm120_router_top22_kernel[(m,)](
|
||||
scores,
|
||||
bias,
|
||||
weights,
|
||||
ids,
|
||||
scores.stride(0),
|
||||
scores.stride(1),
|
||||
float(routed_scaling_factor),
|
||||
RENORMALIZE=bool(renormalize),
|
||||
APPLY_SCALE=bool(apply_routed_scaling_factor_on_output),
|
||||
USE_PDL=pdl,
|
||||
num_warps=1,
|
||||
**({"launch_pdl": True} if pdl else {}),
|
||||
)
|
||||
return weights, ids
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _router_triton_kernel(
|
||||
scores_ptr, # [M, N] raw logits, fp32/fp16/bf16 (upcast to fp32 on load)
|
||||
@@ -328,12 +407,46 @@ def moe_fused_gate(
|
||||
assert input_ids.shape == (scores.size(0),)
|
||||
if packed_out is not None:
|
||||
assert packed_out.dtype == torch.int32, "packed_out must be int32"
|
||||
assert packed_out.shape == (scores.size(0), topk), (
|
||||
"packed_out must be [M, topk]"
|
||||
)
|
||||
assert packed_out.shape == (
|
||||
scores.size(0),
|
||||
topk,
|
||||
), "packed_out must be [M, topk]"
|
||||
if routed_scaling_factor is None:
|
||||
routed_scaling_factor = 1.0
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
if (
|
||||
envs.SGLANG_SM120_MTP_ROUTER_TOP22.get()
|
||||
and scores.is_cuda
|
||||
and torch.cuda.get_device_capability(scores.device) == (12, 0)
|
||||
and 2 <= scores.shape[0] <= 16
|
||||
and scores.shape[1] == 512
|
||||
and scores.dtype == torch.float32
|
||||
and scores.stride(1) == 1
|
||||
and bias is not None
|
||||
and bias.dtype == torch.float32
|
||||
and bias.is_contiguous()
|
||||
and topk == 22
|
||||
and scoring_func.lower() == "sigmoid"
|
||||
and num_fused_shared_experts == 0
|
||||
and num_expert_group <= 1
|
||||
and moe_softcapping == 0.0
|
||||
and input_ids is None
|
||||
and num_token_non_padded is None
|
||||
and renormalize_epsilon == 0.0
|
||||
and packed_out is None
|
||||
and not sqrtsoftplus_log1p
|
||||
):
|
||||
return _topk_gate_small_m(
|
||||
scores,
|
||||
bias,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
|
||||
)
|
||||
|
||||
# K3 radix-select fast path: native-CUDA radix-select replaces the 16
|
||||
# dependent argmax rounds (single CTA per token; ids bit-identical to this
|
||||
# triton kernel incl. ties).
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Pack a uniform-width verify tree mask into existing FlashInfer buffers."""
|
||||
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pack_verify_mask_kernel(
|
||||
mask,
|
||||
kv_indptr,
|
||||
packed_indptr,
|
||||
output,
|
||||
Q: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
byte = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
|
||||
bit = tl.arange(0, 8)
|
||||
kv_start = tl.load(kv_indptr + row)
|
||||
kv_end = tl.load(kv_indptr + row + 1)
|
||||
count = (kv_end - kv_start) * Q
|
||||
offsets = byte[:, None] * 8 + bit[None, :]
|
||||
values = tl.load(mask + kv_start * Q + offsets, offsets < count, other=0)
|
||||
packed = tl.sum(values.to(tl.uint32) << bit[None, :], axis=1)
|
||||
start = tl.load(packed_indptr + row)
|
||||
tl.store(output + start + byte, packed.to(tl.uint8), byte < tl.cdiv(count, 8))
|
||||
|
||||
|
||||
def pack_verify_mask_into(
|
||||
mask, kv_indptr, packed_indptr, output, *, batch_size, query_width, max_kv_len
|
||||
):
|
||||
_pack_verify_mask_kernel[
|
||||
(batch_size, triton.cdiv(query_width * max_kv_len, 8 * 256))
|
||||
](
|
||||
mask.contiguous().view(-1),
|
||||
kv_indptr,
|
||||
packed_indptr,
|
||||
output,
|
||||
Q=query_width,
|
||||
BLOCK=256,
|
||||
)
|
||||
@@ -1157,6 +1157,9 @@ class Envs:
|
||||
SGLANG_TRTLLM_MOE_PDL_MAX_TOKENS = EnvInt(8192)
|
||||
# Use FlashInfer's fused atomic CUTLASS/CuTe DSL MoE finalize.
|
||||
SGLANG_FLASHINFER_MOE_FUSED_FINALIZE = EnvBool(False)
|
||||
# Experimental Nemotron small-M NVFP4 path; validated only on SM120.
|
||||
SGLANG_SM120_NVFP4_PHASE_LAYOUT = EnvBool(False)
|
||||
SGLANG_SM120_MTP_ROUTER_TOP22 = EnvBool(False)
|
||||
# Master switch for the experimental TRT-LLM LoRA fast path; when OFF (default) every
|
||||
# fine-grained opt switch reads False, keeping non-experimental paths byte-identical.
|
||||
SGLANG_EXPERIMENTAL_LORA_OPTI = EnvBool(False)
|
||||
@@ -1453,6 +1456,10 @@ class Envs:
|
||||
# into a small CUDA graph, collapsing its host dispatch cost to one launch.
|
||||
# Experimental; auto-falls back to eager if the backend's prep is not capturable.
|
||||
SGLANG_ENABLE_METADATA_GLUE_GRAPH = EnvBool(False)
|
||||
# Reuse the resolved CPU sequence lengths for FlashInfer draft planning,
|
||||
# avoiding a second blocking readback after generating the same indptr.
|
||||
SGLANG_FLASHINFER_DRAFT_HOST_INDPTR = EnvBool(False)
|
||||
SGLANG_FLASHINFER_VERIFY_HOST_PLAN = EnvBool(False)
|
||||
SGLANG_OPT_FUSED_KDA_VERIFY = EnvBool(False)
|
||||
# A/B: keep the DFLASH draft greedy head eager (not folded in-graph).
|
||||
SGLANG_DFLASH_EAGER_DRAFT_SAMPLER = EnvBool(False)
|
||||
|
||||
@@ -210,9 +210,10 @@ def fast_prefill_plan(
|
||||
kv_lens_host: torch.Tensor,
|
||||
max_q_len: int,
|
||||
max_kv_len: int,
|
||||
uniform_verify_width: Optional[int] = None,
|
||||
) -> None:
|
||||
"""Sync-free ``BatchPrefillWithPagedKVCacheWrapper.plan`` for the EAGLE
|
||||
draft-extend CUDA graph (FlashInfer fa2, cuda-graph mode only).
|
||||
draft-extend or uniform-width verify CUDA graph (FlashInfer fa2 only).
|
||||
|
||||
Upstream plan() always does qo/paged_kv/last_page_len ``.to("cpu")`` to build
|
||||
its host scheduling metadata, a blocking D2H that drains the GPU queue every
|
||||
@@ -259,6 +260,30 @@ def fast_prefill_plan(
|
||||
non_blocking=(paged_kv_indices.device == self.device) and non_blocking,
|
||||
)
|
||||
|
||||
if custom_mask is not None:
|
||||
# EAGLE's mask remains data-dependent: pack every actual tree-mask
|
||||
# bit, including rejected branches. Only its row lengths are known
|
||||
# on the host, so no device scalar read is needed for allocation.
|
||||
assert uniform_verify_width is not None and page_size == 1
|
||||
from sglang.kernels.ops.speculative.pack_verify_mask import (
|
||||
pack_verify_mask_into,
|
||||
)
|
||||
|
||||
packed_lengths = (kv_lens_host * uniform_verify_width + 7) // 8
|
||||
packed_indptr = torch.zeros(batch_size + 1, dtype=torch.int32, device="cpu")
|
||||
packed_indptr[1:] = torch.cumsum(packed_lengths, dim=0)
|
||||
assert int(packed_indptr[-1]) <= self._custom_mask_buf.numel()
|
||||
self._mask_indptr_buf.copy_(packed_indptr, non_blocking=non_blocking)
|
||||
pack_verify_mask_into(
|
||||
custom_mask,
|
||||
paged_kv_indptr,
|
||||
self._mask_indptr_buf,
|
||||
self._custom_mask_buf,
|
||||
batch_size=batch_size,
|
||||
query_width=uniform_verify_width,
|
||||
max_kv_len=max_kv_len,
|
||||
)
|
||||
|
||||
self._cached_q_data_type = q_data_type
|
||||
self._cached_kv_data_type = (
|
||||
kv_data_type if kv_data_type is not None else q_data_type
|
||||
@@ -855,9 +880,24 @@ class FlashInferAttnBackend(AttentionBackend):
|
||||
# call_begin_forward therefore applies verbatim; installing the
|
||||
# sync-free plan removes three blocking .to("cpu") reads per
|
||||
# replay that otherwise stall the CPU behind the in-flight graph.
|
||||
# EAGLE target-verify keeps the plain plan(): its spec input is
|
||||
# not DFLASH_VERIFY, and this branch keys off the capture-time
|
||||
# spec_info of these per-bs wrappers.
|
||||
# EAGLE uses the separate opt-in branch below because it must
|
||||
# also pack its actual tree mask on every replay.
|
||||
for w in self.prefill_cuda_graph_metadata[bs]:
|
||||
w.begin_forward = partial(fast_prefill_plan, w)
|
||||
|
||||
if (
|
||||
in_capture
|
||||
and envs.SGLANG_FLASHINFER_VERIFY_HOST_PLAN.get()
|
||||
and forward_mode.is_target_verify()
|
||||
and spec_info is not None
|
||||
and spec_info.spec_input_type == SpecInputType.EAGLE_VERIFY
|
||||
and spec_info.topk == 1
|
||||
and seq_lens_cpu is not None
|
||||
and self.prefill_backend == "fa2"
|
||||
and self.dispatch_reason is None
|
||||
and self.flashinfer_kv_cache_dtype
|
||||
in (torch.bfloat16, torch.float16, torch.float8_e4m3fn, torch.float8_e5m2)
|
||||
):
|
||||
for w in self.prefill_cuda_graph_metadata[bs]:
|
||||
w.begin_forward = partial(fast_prefill_plan, w)
|
||||
|
||||
@@ -2322,10 +2362,13 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
assert num_tokens_per_req is not None and num_tokens_per_req > 0, (
|
||||
f"fast_prefill_plan replay requires num_tokens_per_req > 0 (got {num_tokens_per_req})"
|
||||
)
|
||||
assert use_custom_mask is None, (
|
||||
"fast_prefill_plan does not support custom_mask; keep the plain plan()"
|
||||
)
|
||||
seq_lens_cpu_i32 = seq_lens_cpu.to(torch.int32)
|
||||
eagle_verify = spec_info.spec_input_type == SpecInputType.EAGLE_VERIFY
|
||||
assert use_custom_mask is None or eagle_verify
|
||||
if eagle_verify:
|
||||
# EAGLE carries the committed prefix length; its generated KV
|
||||
# layout includes the complete candidate window after it.
|
||||
seq_lens_cpu_i32 = seq_lens_cpu_i32 + num_tokens_per_req
|
||||
qo_indptr_host = torch.arange(
|
||||
0,
|
||||
(bs + 1) * num_tokens_per_req,
|
||||
@@ -2342,6 +2385,8 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
max_q_len=num_tokens_per_req,
|
||||
max_kv_len=int(seq_lens_cpu_i32.max()),
|
||||
)
|
||||
if eagle_verify:
|
||||
paged_plan_kwargs["uniform_verify_width"] = num_tokens_per_req
|
||||
|
||||
if window_left >= 0:
|
||||
# selects the module with the per-element window mask compiled in
|
||||
@@ -2369,6 +2414,30 @@ class FlashInferIndicesUpdaterPrefill:
|
||||
)
|
||||
|
||||
|
||||
def _build_draft_indptr_host(
|
||||
seq_lens_cpu: torch.Tensor, topk: int, num_steps: int
|
||||
) -> torch.Tensor:
|
||||
"""Mirror the draft kernel's prefix sums from resolved host lengths.
|
||||
|
||||
At draft entry, positions repeat each committed sequence length ``topk``
|
||||
times. Step i adds i+1 tokens per branch. This also covers the graph
|
||||
runner's padded rows, whose CPU lengths have already been filled.
|
||||
"""
|
||||
assert seq_lens_cpu.device.type == "cpu" and seq_lens_cpu.ndim == 1
|
||||
offsets = [0]
|
||||
for length in seq_lens_cpu.tolist():
|
||||
for _ in range(topk):
|
||||
offsets.append(offsets[-1] + length)
|
||||
return torch.tensor(
|
||||
[
|
||||
[offset + branch * (step + 1) for branch, offset in enumerate(offsets)]
|
||||
for step in range(num_steps)
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
)
|
||||
|
||||
|
||||
class FlashInferMultiStepDraftBackend:
|
||||
"""
|
||||
Wrap multiple flashinfer attention backends as one for multiple consecutive
|
||||
@@ -2447,8 +2516,23 @@ class FlashInferMultiStepDraftBackend:
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
bs=num_seqs,
|
||||
)
|
||||
host_lengths = forward_batch.seq_lens_cpu
|
||||
use_host_indptr = (
|
||||
envs.SGLANG_FLASHINFER_DRAFT_HOST_INDPTR.get()
|
||||
and self.draft_window_size == 0
|
||||
and host_lengths is not None
|
||||
and host_lengths.device.type == "cpu"
|
||||
and host_lengths.numel() == num_seqs
|
||||
)
|
||||
# The existing token-block kernel copies every index, but distributes
|
||||
# long contexts across CTAs instead of serially walking them in 3 CTAs.
|
||||
num_blocks = (
|
||||
min(32, max(1, (max(host_lengths.tolist(), default=0) + 511) // 512))
|
||||
if use_host_indptr
|
||||
else 1
|
||||
)
|
||||
self.generate_draft_decode_kv_indices[
|
||||
(self.speculative_num_steps, num_seqs, self.topk)
|
||||
(self.speculative_num_steps * num_blocks, num_seqs, self.topk)
|
||||
](
|
||||
src.row_ids,
|
||||
src.ids,
|
||||
@@ -2468,24 +2552,33 @@ class FlashInferMultiStepDraftBackend:
|
||||
ENTRY_PAGE_SIZE=src.entry_page_size,
|
||||
v2p=src.v2p,
|
||||
TRANSLATE=src.v2p is not None,
|
||||
NUM_STEPS=self.speculative_num_steps if use_host_indptr else 0,
|
||||
)
|
||||
|
||||
assert forward_batch.spec_info is not None
|
||||
assert forward_batch.spec_info.is_draft_input()
|
||||
|
||||
# Copy the kv_indptr once to avoid multiple device-to-host copies in flashinfer's plan.
|
||||
indptr_cpu_whole = self.kv_indptr[:, : bs + 1].cpu()
|
||||
# resolve_seq_lens_cpu already supplies the committed lengths after
|
||||
# acceptance/rollback. Do not drain the GPU queue to read them again.
|
||||
indptr_cpu_whole = (
|
||||
_build_draft_indptr_host(
|
||||
host_lengths, self.topk, self.speculative_num_steps
|
||||
)
|
||||
if use_host_indptr
|
||||
else self.kv_indptr[:, : bs + 1].cpu()
|
||||
)
|
||||
global global_override_indptr_cpu
|
||||
|
||||
for i in range(self.speculative_num_steps - 1):
|
||||
forward_batch.spec_info.kv_indptr = self.kv_indptr[i, : bs + 1]
|
||||
forward_batch.spec_info.kv_indices = kv_indices_buffer[i][
|
||||
: draft_kv_indices_used_len(seq_lens_sum, self.topk, bs, i + 1)
|
||||
]
|
||||
global_override_indptr_cpu = indptr_cpu_whole[i]
|
||||
call_fn(i, forward_batch)
|
||||
|
||||
global_override_indptr_cpu = None
|
||||
previous_override = global_override_indptr_cpu
|
||||
try:
|
||||
for i in range(self.speculative_num_steps - 1):
|
||||
forward_batch.spec_info.kv_indptr = self.kv_indptr[i, : bs + 1]
|
||||
forward_batch.spec_info.kv_indices = kv_indices_buffer[i][
|
||||
: draft_kv_indices_used_len(seq_lens_sum, self.topk, bs, i + 1)
|
||||
]
|
||||
global_override_indptr_cpu = indptr_cpu_whole[i]
|
||||
call_fn(i, forward_batch)
|
||||
finally:
|
||||
global_override_indptr_cpu = previous_override
|
||||
|
||||
def init_forward_metadata(self, forward_batch: ForwardBatch):
|
||||
kv_indices_width = draft_kv_indices_buffer_width(
|
||||
|
||||
@@ -26,7 +26,7 @@ from sglang.srt.layers.moe.moe_runner.base import (
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_flashinfer_available
|
||||
from sglang.srt.utils.common import next_power_of_2
|
||||
from sglang.srt.utils.common import is_sm120, next_power_of_2
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
|
||||
@@ -59,6 +59,7 @@ class FlashInferCutlassMoeQuantInfo(MoeQuantInfo):
|
||||
swiglu_beta: Optional[torch.Tensor] = None
|
||||
swiglu_limit: Optional[torch.Tensor] = None
|
||||
apply_routed_scaling_factor: bool = True
|
||||
pair_w2_weight: Optional[torch.Tensor] = None
|
||||
|
||||
|
||||
def materialize_swiglu_params_for_cutlass(
|
||||
@@ -250,6 +251,26 @@ def _prepare_input(
|
||||
return x, x_sf, output_dtype, output_col
|
||||
|
||||
|
||||
def _sm120_mtp_mma_supported() -> bool:
|
||||
import os
|
||||
|
||||
# This implementation preserves FlashInfer's default FP4 quantizer and
|
||||
# fused FC2 epilogue. Other quantization recipes retain the stock runner.
|
||||
return (
|
||||
is_sm120()
|
||||
and envs.SGLANG_FLASHINFER_MOE_FUSED_FINALIZE.get()
|
||||
and not envs.FLASHINFER_NVFP4_4OVER6.get()
|
||||
and os.environ.get("FLASHINFER_DISABLE_FP4_QUANT_FAST_MATH", "0") == "0"
|
||||
and os.environ.get("TRTLLM_DISABLE_FP4_QUANT_FAST_MATH", "0") == "0"
|
||||
)
|
||||
|
||||
|
||||
def _flashinfer_autotune_active() -> bool:
|
||||
from flashinfer.autotuner import AutoTuner
|
||||
|
||||
return AutoTuner.get().is_tuning_mode
|
||||
|
||||
|
||||
def _run_flashinfer_cutlass(
|
||||
*,
|
||||
dispatch_output,
|
||||
@@ -268,6 +289,35 @@ def _run_flashinfer_cutlass(
|
||||
dispatch_output, quant_info, runner_config
|
||||
)
|
||||
|
||||
if (
|
||||
quant_info.quant_type == "fp4"
|
||||
and not runner_config.is_gated
|
||||
and runner_config.activation == "relu2"
|
||||
and runner_config.moe_ep_size == runner_config.moe_tp_size == 1
|
||||
and x_sf is None
|
||||
and not enable_alltoall
|
||||
and is_sm120()
|
||||
):
|
||||
from sglang.kernels.ops.gemm.sm120_nvfp4_moe import try_nvfp4_moe
|
||||
|
||||
result = try_nvfp4_moe(
|
||||
x,
|
||||
quant_info.w13_weight,
|
||||
quant_info.w2_weight,
|
||||
quant_info.quant_scales,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
output_dtype=output_dtype,
|
||||
out=output,
|
||||
pair_w2_weight=quant_info.pair_w2_weight,
|
||||
# Preserve FlashInfer target/draft tactic discovery during autotuning.
|
||||
use_mtp=_sm120_mtp_mma_supported() and not _flashinfer_autotune_active(),
|
||||
)
|
||||
if result is not None:
|
||||
return _maybe_apply_routed_scaling_factor(result, quant_info, runner_config)
|
||||
if quant_info.pair_w2_weight is not None:
|
||||
raise RuntimeError("A paired NVFP4 weight reached an unsupported MoE path")
|
||||
|
||||
if output is None:
|
||||
with use_symmetric_memory(
|
||||
get_parallel().tp_group, disabled=not is_allocation_symmetric()
|
||||
|
||||
@@ -3272,7 +3272,10 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
"apply_router_weight_on_input is not supported for Flashinfer"
|
||||
)
|
||||
swiglu_alpha, swiglu_beta, swiglu_limit = layer._cutlass_swiglu_params
|
||||
slot = getattr(layer, "_nvfp4_layout_slot", None)
|
||||
pair_weight = slot[0].pair[slot[1]] if slot and slot[0].is_pair else None
|
||||
quant_info = FlashInferCutlassMoeQuantInfo(
|
||||
pair_w2_weight=pair_weight,
|
||||
quant_type="fp4",
|
||||
w13_weight=layer.w13_weight,
|
||||
w2_weight=layer.w2_weight,
|
||||
|
||||
@@ -317,8 +317,8 @@ def initialize_bf16_gemm_config() -> None:
|
||||
backend = Bf16GemmBackend(backend_str)
|
||||
|
||||
if backend.is_gemv():
|
||||
if torch.cuda.get_device_capability()[0] != 9:
|
||||
raise ValueError("--bf16-gemm-backend gemv requires SM90 (Hopper)")
|
||||
if torch.cuda.get_device_capability()[0] not in (9, 12):
|
||||
raise ValueError("--bf16-gemm-backend gemv requires SM90 (Hopper) or SM12x")
|
||||
|
||||
global _hopper_bf16_gemv, _use_hopper_bf16_gemv
|
||||
from sglang.kernels.ops.gemm.hopper_bf16_gemv import (
|
||||
@@ -589,7 +589,7 @@ class UnquantizedLinearMethod(LinearMethodBase):
|
||||
return tgemm.mm(x, layer.weight, bias, otype=x.dtype)
|
||||
|
||||
elif (
|
||||
get_bf16_gemm_backend().is_cutedsl()
|
||||
(get_bf16_gemm_backend().is_cutedsl() or get_bf16_gemm_backend().is_gemv())
|
||||
and x.is_cuda
|
||||
and x.dtype == torch.bfloat16
|
||||
and layer.weight.dtype == torch.bfloat16
|
||||
|
||||
@@ -1953,6 +1953,9 @@ class ModelRunner:
|
||||
reinit_attn_backend: bool = False,
|
||||
split_forward_count: int = 1,
|
||||
) -> ModelRunnerOutput:
|
||||
prepare = getattr(self.model, "prepare_for_forward", None)
|
||||
if prepare is not None:
|
||||
prepare(forward_batch)
|
||||
if has_forward_context():
|
||||
ctx_mgr = contextlib.nullcontext()
|
||||
else:
|
||||
|
||||
@@ -96,6 +96,25 @@ def has_standard_gqa_for_all_local_layers(
|
||||
return attention_layer_count >= end_layer - start_layer
|
||||
|
||||
|
||||
def has_hybrid_graph_handlers(
|
||||
*,
|
||||
attention_layers: list[Any],
|
||||
moe_layers: list[Any],
|
||||
start_layer: int,
|
||||
end_layer: int,
|
||||
) -> bool:
|
||||
"""Accept pure MoE layers while rejecting gaps in the local decoder range."""
|
||||
return (
|
||||
0 <= start_layer < end_layer
|
||||
and end_layer <= len(attention_layers)
|
||||
and end_layer <= len(moe_layers)
|
||||
and all(
|
||||
attention_layers[i] is not None or moe_layers[i] is not None
|
||||
for i in range(start_layer, end_layer)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def index_attention_layers_by_global_id(
|
||||
attention_layers: list[Any],
|
||||
mha_companion_layers: list[Any],
|
||||
@@ -118,8 +137,13 @@ def index_attention_layers_by_global_id(
|
||||
)
|
||||
return attention_layers, mha_companion_layers
|
||||
max_layer_id = max(int(layer.layer_id) for layer in populated)
|
||||
indexed_attention = [None] * (max_layer_id + 1)
|
||||
indexed_companions = [None] * (max_layer_id + 1)
|
||||
# Keep trailing pure MoE layers represented as well. Their graph handler
|
||||
# lives in moe_layers, and shrinking this list loses decoder alignment.
|
||||
indexed_size = max(
|
||||
max_layer_id + 1, len(layer_model.layers) if layer_model is not None else 0
|
||||
)
|
||||
indexed_attention = [None] * indexed_size
|
||||
indexed_companions = [None] * indexed_size
|
||||
has_reused_layers = False
|
||||
for attention, companion in zip(attention_layers, mha_companion_layers):
|
||||
if attention is None:
|
||||
@@ -621,7 +645,17 @@ def capture_prefill_graph(
|
||||
layer_model,
|
||||
)
|
||||
|
||||
if not has_standard_gqa_for_all_local_layers(
|
||||
supports_hybrid_graph = (
|
||||
getattr(layer_model, "supports_hybrid_prefill_graph", False)
|
||||
and prefill_backend == Backend.BREAKABLE
|
||||
and has_hybrid_graph_handlers(
|
||||
attention_layers=model_runner.attention_layers,
|
||||
moe_layers=model_runner.moe_layers,
|
||||
start_layer=model_runner.layer_info.start_layer,
|
||||
end_layer=model_runner.layer_info.end_layer,
|
||||
)
|
||||
)
|
||||
if not supports_hybrid_graph and not has_standard_gqa_for_all_local_layers(
|
||||
attention_layer_count=sum(
|
||||
layer is not None for layer in model_runner.attention_layers
|
||||
),
|
||||
|
||||
@@ -24,6 +24,12 @@ class WeightExporter:
|
||||
get_model: Callable[[], Any]
|
||||
_weights_send_group: dict = field(default_factory=dict)
|
||||
|
||||
def _check_weight_export_supported(self):
|
||||
for module in self.get_model().modules():
|
||||
reason = getattr(module, "_weight_export_error", None)
|
||||
if reason is not None:
|
||||
raise RuntimeError(reason)
|
||||
|
||||
def init_weights_send_group_for_remote_instance(
|
||||
self,
|
||||
master_address,
|
||||
@@ -33,6 +39,7 @@ class WeightExporter:
|
||||
group_name,
|
||||
backend="nccl",
|
||||
):
|
||||
self._check_weight_export_supported()
|
||||
assert torch.distributed.is_initialized(), (
|
||||
"Default torch process group must be initialized"
|
||||
)
|
||||
@@ -81,6 +88,7 @@ class WeightExporter:
|
||||
ports,
|
||||
group_name,
|
||||
):
|
||||
self._check_weight_export_supported()
|
||||
assert torch.distributed.is_initialized(), (
|
||||
"Default torch process group must be initialized"
|
||||
)
|
||||
@@ -124,6 +132,7 @@ class WeightExporter:
|
||||
return success, message
|
||||
|
||||
def save_remote_model(self: WeightExporter, url: str):
|
||||
self._check_weight_export_supported()
|
||||
from sglang.srt.model_loader.loader import RemoteModelLoader
|
||||
|
||||
logger.info(f"Saving model to {url}")
|
||||
@@ -135,6 +144,7 @@ class WeightExporter:
|
||||
pattern: Optional[str] = None,
|
||||
max_size: Optional[int] = None,
|
||||
):
|
||||
self._check_weight_export_supported()
|
||||
from sglang.srt.model_loader.loader import ShardedStateLoader
|
||||
|
||||
logger.info(
|
||||
@@ -150,6 +160,7 @@ class WeightExporter:
|
||||
Only used for unit test with an unoptimized performance.
|
||||
For optimized performance, please use torch.save and torch.load.
|
||||
"""
|
||||
self._check_weight_export_supported()
|
||||
# TODO: (chenyang) Add support for Qwen models.
|
||||
try:
|
||||
return self.get_model().get_weights_by_name(
|
||||
|
||||
@@ -1709,6 +1709,10 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
"""
|
||||
num_tokens = size
|
||||
forward_batch, attn_backend = self.capture_prepare(num_tokens)
|
||||
# Recapture can follow decode replay, which may change model-owned storage.
|
||||
prepare = getattr(self.model_runner.model, "prepare_for_forward", None)
|
||||
if prepare is not None:
|
||||
prepare(forward_batch)
|
||||
if self.enable_cp_bcg_capture:
|
||||
assert self.prefill_cp_bcg_input is not None
|
||||
self.prefill_cp_bcg_input.prepare(
|
||||
|
||||
@@ -447,6 +447,9 @@ class NemotronH_Omni_Reasoning_V3(NemotronH_Nano_VL_V2):
|
||||
else None
|
||||
)
|
||||
|
||||
def prepare_for_forward(self, forward_batch: ForwardBatch) -> None:
|
||||
self.language_model.prepare_for_forward(forward_batch)
|
||||
|
||||
@property
|
||||
def lm_head(self):
|
||||
return self.language_model.lm_head
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
"""Inference-only NemotronH model."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from functools import partial
|
||||
from functools import lru_cache, partial
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -26,6 +26,7 @@ from torch import nn
|
||||
from sglang.srt.compilation.compilation_config import register_split_op
|
||||
from sglang.srt.configs import NemotronHConfig
|
||||
from sglang.srt.configs.nemotron_h import ATTENTION, MAMBA, MLP, MOE
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
@@ -65,7 +66,11 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
ForwardBatch,
|
||||
ForwardMode,
|
||||
PPProxyTensors,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
@@ -84,12 +89,13 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
)
|
||||
from sglang.srt.models.nemotron_h_utils import make_stage_boundary, stage_facts
|
||||
from sglang.srt.models.utils import WeightsMapper
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_schedule
|
||||
from sglang.srt.utils import (
|
||||
LazyValue,
|
||||
add_prefix,
|
||||
get_current_device_stream_fast,
|
||||
is_cuda,
|
||||
is_sm120,
|
||||
make_pp_layers,
|
||||
)
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
@@ -157,6 +163,58 @@ def _get_or_create_alt_stream(device_module):
|
||||
return _alt_stream
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _prefill_shared_expert_pool(device_index: int):
|
||||
# Shared experts overlap the main graph, so their intermediates must use
|
||||
# a separate pool. Calls on the shared stream are ordered across layers.
|
||||
with torch.cuda.device(device_index):
|
||||
return torch.cuda.graph_pool_handle()
|
||||
|
||||
|
||||
class _PrefillSharedExpertCall:
|
||||
def __init__(self, stream):
|
||||
self.graph = None
|
||||
self.stream = stream
|
||||
self.ready = torch.cuda.Event(external=True)
|
||||
self.done = torch.cuda.Event(external=True)
|
||||
self.output = None
|
||||
|
||||
def launch(self, module, hidden_states):
|
||||
main_stream = torch.cuda.current_stream()
|
||||
with torch.cuda.stream(self.stream):
|
||||
if self.graph is None:
|
||||
# BCG warmup follows the same dependencies, without capture.
|
||||
self.ready.wait()
|
||||
self.output = module(hidden_states)
|
||||
self.done.record()
|
||||
self.output.record_stream(main_stream)
|
||||
else:
|
||||
self.graph.replay()
|
||||
return self.output
|
||||
|
||||
|
||||
def _capture_prefill_shared_expert(module, hidden_states, call):
|
||||
call.graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(
|
||||
call.graph,
|
||||
pool=_prefill_shared_expert_pool(hidden_states.device.index),
|
||||
stream=call.stream,
|
||||
):
|
||||
call.ready.wait()
|
||||
call.output = module(hidden_states)
|
||||
call.done.record()
|
||||
# The ready record belongs to the preceding main-stream graph. Do not
|
||||
# replay here: that graph has only been captured, not launched yet.
|
||||
return call.output
|
||||
|
||||
|
||||
@eager_on_graph(True, capture_stub=_capture_prefill_shared_expert)
|
||||
def _launch_prefill_shared_expert(module, hidden_states, call):
|
||||
# BCG retains this call, its graph, events and output until graph cleanup.
|
||||
# A replay launches on the same explicit stream for every layer.
|
||||
return call.launch(module, hidden_states)
|
||||
|
||||
|
||||
def _latent_proj_fuses_shared_add(projection: nn.Module) -> bool:
|
||||
return (
|
||||
# LoRA swaps a wrapper module over this attribute after init,
|
||||
@@ -288,7 +346,23 @@ class NemotronHMoE(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
num_token_non_padded: torch.Tensor | None,
|
||||
forward_batch: ForwardBatch | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if (
|
||||
forward_batch is not None
|
||||
and forward_batch.forward_mode == ForwardMode.EXTEND
|
||||
and hidden_states.shape == (1024, 4096)
|
||||
and hidden_states.dtype == torch.bfloat16
|
||||
and get_parallel().tp_size == self.ep_size == 1
|
||||
and self.n_routed_experts == 512
|
||||
and self.moe_hidden_size == 1024
|
||||
and self.shared_experts is not None
|
||||
and is_in_breakable_cuda_graph()
|
||||
and is_sm120()
|
||||
):
|
||||
return self._forward_core_prefill_overlap(
|
||||
hidden_states, num_token_non_padded
|
||||
)
|
||||
if _is_cuda and (
|
||||
not get_moe_a2a_backend().is_flashinfer() or get_is_capture_mode()
|
||||
):
|
||||
@@ -363,6 +437,38 @@ class NemotronHMoE(nn.Module):
|
||||
|
||||
return final_hidden_states, shared_output
|
||||
|
||||
def _forward_core_prefill_overlap(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
num_token_non_padded: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# A multi-stream graph per hybrid segment creates one internal stream
|
||||
# per executable. Keep each graph linear and launch shared experts on
|
||||
# one explicit stream, preserving overlap with the routed experts.
|
||||
call = _PrefillSharedExpertCall(_get_or_create_alt_stream(self.device_module))
|
||||
call.ready.record()
|
||||
shared_output = _launch_prefill_shared_expert(
|
||||
self.shared_experts, hidden_states, call
|
||||
)
|
||||
router_logits = torch.mm(
|
||||
hidden_states, self.gate.weight.t(), out_dtype=torch.float32
|
||||
)
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_token_non_padded=num_token_non_padded,
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if not self.is_nextn
|
||||
else None
|
||||
),
|
||||
)
|
||||
if self.use_latent_moe:
|
||||
hidden_states = self._apply_fc1_latent_proj(hidden_states)
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
call.done.wait()
|
||||
return final_hidden_states, shared_output
|
||||
|
||||
def get_moe_weights(self) -> list[torch.Tensor]:
|
||||
return [
|
||||
x.data
|
||||
@@ -404,6 +510,7 @@ class NemotronHMoE(nn.Module):
|
||||
if forward_batch is not None
|
||||
else None
|
||||
),
|
||||
forward_batch=forward_batch,
|
||||
)
|
||||
|
||||
if self.use_latent_moe:
|
||||
@@ -760,6 +867,10 @@ class NemotronHModel(nn.Module):
|
||||
self.norm_f = PPMissingLayer(return_tuple=True)
|
||||
self.layers_to_capture: set[int] = set()
|
||||
|
||||
@property
|
||||
def supports_hybrid_prefill_graph(self) -> bool:
|
||||
return is_sm120()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -856,6 +967,7 @@ class NemotronHForCausalLM(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
lora_config = None
|
||||
self._use_nvfp4_phase_layout = envs.SGLANG_SM120_NVFP4_PHASE_LAYOUT.get()
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.model = self._init_model(
|
||||
@@ -1027,6 +1139,58 @@ class NemotronHForCausalLM(nn.Module):
|
||||
f"get_hidden_dim not implemented for {module_name}"
|
||||
)
|
||||
|
||||
def prepare_for_forward(self, forward_batch: ForwardBatch) -> None:
|
||||
if not getattr(self, "_use_nvfp4_phase_layout", False):
|
||||
return
|
||||
from sglang.kernels.ops.gemm.sm120_nvfp4_moe import Nvfp4LayoutCycle
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
|
||||
_flashinfer_autotune_active,
|
||||
_sm120_mtp_mma_supported,
|
||||
)
|
||||
|
||||
cycle = getattr(self, "_nvfp4_layout", None)
|
||||
if cycle is None:
|
||||
if not (
|
||||
get_parallel().tp_size
|
||||
== get_parallel().ep_size
|
||||
== get_parallel().pp_size
|
||||
== 1
|
||||
and get_schedule().max_running_requests == 1
|
||||
and _sm120_mtp_mma_supported()
|
||||
):
|
||||
raise ValueError(
|
||||
"NVFP4 phase layout requires serial SM120 CUTLASS MoE with TP/EP/PP=1"
|
||||
)
|
||||
experts = [
|
||||
m.experts for m in self.model.modules() if isinstance(m, NemotronHMoE)
|
||||
]
|
||||
if not experts or not all(
|
||||
getattr(m.quant_method, "enable_flashinfer_cutlass_moe", False)
|
||||
and m.w13_weight.shape == (512, 2688, 512)
|
||||
for m in experts
|
||||
):
|
||||
raise ValueError(
|
||||
"NVFP4 phase layout requires compatible native NVFP4 experts"
|
||||
)
|
||||
cycle = self._nvfp4_layout = Nvfp4LayoutCycle(
|
||||
[m.w2_weight for m in experts]
|
||||
)
|
||||
for index, module in enumerate(experts):
|
||||
module._nvfp4_layout_slot = (cycle, index)
|
||||
self._derived_weight_cache_error = (
|
||||
"Online weight updates require disabling NVFP4 phase layout"
|
||||
)
|
||||
self._weight_export_error = (
|
||||
"Weight export requires disabling NVFP4 phase layout"
|
||||
)
|
||||
|
||||
mode = forward_batch.forward_mode
|
||||
cycle.ensure(
|
||||
(mode.is_target_verify() or mode.is_decode_or_idle())
|
||||
and forward_batch.input_ids.numel() <= 4
|
||||
and not _flashinfer_autotune_active()
|
||||
)
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
@@ -1036,6 +1200,7 @@ class NemotronHForCausalLM(nn.Module):
|
||||
input_embeds: torch.Tensor | None = None,
|
||||
pp_proxy_tensors: PPProxyTensors | None = None,
|
||||
):
|
||||
self.prepare_for_forward(forward_batch)
|
||||
hidden_states = self.model.forward(
|
||||
input_ids, positions, forward_batch, pp_proxy_tensors, input_embeds
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user