[Fix] gguf: Preserve MoE routing, weighting and allocation budgets (#45916)

This commit is contained in:
Cheng Wan
2026-10-23 12:29:51 -07:00
committed by GitHub
parent 1340044e97
commit e6451c1a95
36 changed files with 1592 additions and 189 deletions
+27 -2
View File
@@ -30,10 +30,12 @@ import json
import os
import re
import runpy
import shutil
import subprocess
from pathlib import Path
from setuptools import setup
from setuptools.command.sdist import sdist
try:
from setuptools_rust import Binding, RustExtension, build_rust
@@ -214,6 +216,29 @@ def _declared_rust_extensions():
return _pyproject_rust_extensions(_discovered_rust_extensions())
class Sdist(sdist):
"""Materialize shared GGUF headers in the standalone source distribution."""
def make_release_tree(self, base_dir, files):
super().make_release_tree(base_dir, files)
relative = Path("sglang/kernels/jit/include/sgl_kernel/quantization/gguf")
canonical = (
_PYTHON_DIR / "sglang/kernels/aot/include/sgl_kernel/quantization/gguf"
)
included = {Path(name) for name in files}
for header in sorted((_PYTHON_DIR / relative).glob("*")):
name = header.relative_to(_PYTHON_DIR)
if name not in included or not header.is_symlink():
continue
target = header.resolve(strict=True)
if target.parent != canonical.resolve():
raise ValueError(f"Shared GGUF header escapes its owner: {header}")
destination = Path(base_dir) / name
# make_release_tree may hard-link the source symlink itself.
destination.unlink()
shutil.copyfile(target, destination)
if build_rust is not None:
class BuildRust(build_rust):
@@ -241,8 +266,8 @@ if build_rust is not None:
super().run()
setup(
cmdclass={"build_rust": BuildRust},
cmdclass={"build_rust": BuildRust, "sdist": Sdist},
rust_extensions=_declared_rust_extensions(),
)
else:
setup()
setup(cmdclass={"sdist": Sdist})
@@ -155,6 +155,9 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) {
m.def("moe_sum(Tensor input, Tensor! output) -> ()");
m.impl("moe_sum", torch::kCUDA, &moe_sum);
m.def("apply_gguf_moe_weights(Tensor! outputs, Tensor weights, Tensor ids) -> ()");
m.impl("apply_gguf_moe_weights", torch::kCUDA, &apply_gguf_moe_weights);
// moe_fused_gate / kimi_k2_moe_fused_gate (AOT) retired: the CUDA gate/topk path
// now routes through the unified Triton router
// (python/sglang/kernels/ops/moe/moe_fused_gate.py).
@@ -114,6 +114,9 @@ TORCH_LIBRARY_EXPAND(sgl_kernel, m) {
m.def("moe_sum(Tensor input, Tensor! output) -> ()");
m.impl("moe_sum", torch::kMUSA, &moe_sum);
m.def("apply_gguf_moe_weights(Tensor! outputs, Tensor weights, Tensor ids) -> ()");
m.impl("apply_gguf_moe_weights", torch::kMUSA, &apply_gguf_moe_weights);
// moe_fused_gate / kimi_k2_moe_fused_gate (AOT gate kernels) retired: gate/topk
// is consolidated onto the unified Triton router (sglang issue #26771). sglang's
// MUSA path uses `mate.moe_fused_gate`, so dropping the sgl_kernel MUSA op here
@@ -0,0 +1,42 @@
#pragma once
#include <cstdint>
template <typename output_t, typename weight_t, typename id_t>
__global__ void apply_gguf_moe_weights_kernel(
output_t* __restrict__ outputs,
const weight_t* __restrict__ weights,
const id_t* __restrict__ ids,
const int64_t rows,
const int64_t hidden_size,
const int64_t topk,
const int64_t weight_stride0,
const int64_t weight_stride1,
const int64_t id_stride0,
const int64_t id_stride1) {
for (int64_t row = blockIdx.x; row < rows; row += gridDim.x) {
const int64_t token = row / topk;
const int64_t slot = row % topk;
const int64_t first_column = static_cast<int64_t>(blockIdx.y) * blockDim.x + threadIdx.x;
const int64_t column_step = static_cast<int64_t>(gridDim.y) * blockDim.x;
if (ids[token * id_stride0 + slot * id_stride1] < 0) {
for (int64_t column = first_column; column < hidden_size; column += column_step) {
outputs[row * hidden_size + column] = static_cast<output_t>(0.0f);
}
continue;
}
const float weight = static_cast<float>(weights[token * weight_stride0 + slot * weight_stride1]);
for (int64_t column = first_column; column < hidden_size; column += column_step) {
const int64_t index = row * hidden_size + column;
const float value = static_cast<float>(outputs[index]);
#ifdef USE_MUSA
const float product = value * weight;
#else
// SM90 enables fast math; preserve subnormal operands and products here.
float product;
asm("mul.rn.f32 %0, %1, %2;" : "=f"(product) : "f"(value), "f"(weight));
#endif
outputs[index] = static_cast<output_t>(product);
}
}
}
@@ -1,4 +1,5 @@
#include <ATen/ATen.h>
#include <ATen/MemoryOverlap.h>
#include <ATen/cuda/CUDAContext.h>
#include <c10/cuda/CUDAGuard.h>
#include <torch/all.h>
@@ -6,6 +7,14 @@
#include <ATen/cuda/Atomic.cuh>
#include <cub/cub.cuh>
#ifdef USE_MUSA
#include <musa_runtime.h>
#include "torch_musa/csrc/core/MUSAGuard.h"
#include "torch_musa/csrc/core/MUSAStream.h"
#endif
#include "gguf_moe_weights.cuh"
#include "utils.h"
template <typename scalar_t, int TOPK>
@@ -64,3 +73,78 @@ void moe_sum(
break;
}
}
void apply_gguf_moe_weights(torch::Tensor& outputs, const torch::Tensor& weights, const torch::Tensor& ids) {
#ifdef USE_MUSA
TORCH_CHECK(outputs.device().type() == torch::kMUSA, "outputs must be a MUSA tensor");
#else
TORCH_CHECK(outputs.is_cuda(), "outputs must be a CUDA tensor");
#endif
TORCH_CHECK(weights.device() == outputs.device() && ids.device() == outputs.device(), "tensors must share a device");
TORCH_CHECK(
outputs.layout() == at::kStrided && weights.layout() == at::kStrided && ids.layout() == at::kStrided,
"tensors must have strided layout");
TORCH_CHECK(outputs.dim() == 2 && outputs.is_contiguous(), "outputs must be contiguous [tokens * topk, hidden_size]");
TORCH_CHECK(weights.dim() == 2 && ids.dim() == 2, "weights and ids must have shape [tokens, topk]");
TORCH_CHECK(weights.sizes() == ids.sizes(), "weights and ids shapes must match");
TORCH_CHECK(outputs.size(0) == weights.numel(), "outputs rows must equal tokens * topk");
const auto supported_float = [](at::ScalarType dtype) {
return dtype == at::kHalf || dtype == at::kBFloat16 || dtype == at::kFloat;
};
TORCH_CHECK(supported_float(outputs.scalar_type()), "outputs must be float16, bfloat16, or float32");
TORCH_CHECK(supported_float(weights.scalar_type()), "weights must be float16, bfloat16, or float32");
TORCH_CHECK(ids.scalar_type() == at::kInt || ids.scalar_type() == at::kLong, "ids must be int32 or int64");
TORCH_CHECK(
weights.stride(0) >= 0 && weights.stride(1) >= 0 && ids.stride(0) >= 0 && ids.stride(1) >= 0,
"weights and ids strides must be nonnegative");
at::assert_no_overlap(outputs, weights);
at::assert_no_overlap(outputs, ids);
const int64_t numel = outputs.numel();
if (numel == 0) {
return;
}
#ifdef USE_MUSA
const c10::musa::OptionalMUSAGuard device_guard(device_of(outputs));
const musaStream_t stream = c10::musa::getCurrentMUSAStream();
#else
const at::cuda::OptionalCUDAGuard device_guard(device_of(outputs));
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
#endif
constexpr int threads = 256;
// Reuse each route's ID and score across four coalesced column groups.
constexpr int columns_per_block = threads * 4;
const dim3 blocks(
static_cast<unsigned int>(std::min<int64_t>(outputs.size(0), 65535)),
static_cast<unsigned int>(std::min<int64_t>((outputs.size(1) - 1) / columns_per_block + 1, 65535)));
DISPATCH_FLOAT_TYPES(outputs.scalar_type(), "apply_gguf_moe_weights_output", [&] {
using output_t = scalar_t;
DISPATCH_FLOAT_TYPES(weights.scalar_type(), "apply_gguf_moe_weights_weight", [&] {
using weight_t = scalar_t;
const auto launch = [&](auto* id_data) {
apply_gguf_moe_weights_kernel<<<blocks, threads, 0, stream>>>(
outputs.data_ptr<output_t>(),
weights.data_ptr<weight_t>(),
id_data,
outputs.size(0),
outputs.size(1),
weights.size(1),
weights.stride(0),
weights.stride(1),
ids.stride(0),
ids.stride(1));
};
if (ids.scalar_type() == at::kInt) {
launch(ids.data_ptr<int32_t>());
} else {
launch(ids.data_ptr<int64_t>());
}
});
});
#ifdef USE_MUSA
const auto error = musaGetLastError();
TORCH_CHECK(error == musaSuccess, "apply_gguf_moe_weights launch failed: ", musaGetErrorString(error));
#else
const auto error = cudaGetLastError();
TORCH_CHECK(error == cudaSuccess, "apply_gguf_moe_weights launch failed: ", cudaGetErrorString(error));
#endif
}
@@ -1,3 +1,24 @@
// Utility function
template <typename dst_t>
static __device__ __forceinline__ dst_t convert_from_half(half val) {
return val;
}
template <>
__device__ __forceinline__ c10::BFloat16 convert_from_half<c10::BFloat16>(half val) {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 || defined(USE_MUSA)
return __float2bfloat16(__half2float(val));
#else
return __half2float(val);
#endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 || defined(USE_MUSA)
}
template <>
__device__ __forceinline__ float convert_from_half<float>(half val) {
return __half2float(val);
}
// copied from
// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/dequantize.cuh
// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/convert.cu
@@ -9,68 +9,16 @@
// clang-format off
#include "utils.h"
#include "ggml-common.h"
#include "vecdotq.cuh"
#include <sgl_kernel/quantization/gguf/ggml-common.h>
#include <sgl_kernel/quantization/gguf/vecdotq.cuh>
#include "dequantize.cuh"
#include "mmvq.cuh"
#include "mmq.cuh"
#include "moe.cuh"
#include "moe_vec.cuh"
#include <sgl_kernel/quantization/gguf/moe_vec.cuh>
#include <sgl_kernel/quantization/gguf/quantize_q8_1.cuh>
// clang-format off
// Q8 gemv
template <typename scalar_t>
static __global__ void
quantize_q8_1(const scalar_t* __restrict__ x, void* __restrict__ vy, const int kx, const int kx_padded) {
const auto ix = blockDim.x * blockIdx.x + threadIdx.x;
if (ix >= kx_padded) {
return;
}
const auto iy = blockDim.y * blockIdx.y + threadIdx.y;
const int i_padded = iy * kx_padded + ix;
block_q8_1* y = (block_q8_1*)vy;
const int ib = i_padded / QK8_1; // block index
const int iqs = i_padded % QK8_1; // quant index
const float xi = ix < kx ? static_cast<float>(x[iy * kx + ix]) : 0.0f;
float amax = fabsf(xi);
float sum = xi;
#pragma unroll
for (int mask = 16; mask > 0; mask >>= 1) {
amax = fmaxf(amax, SGLANG_SHFL_XOR_SYNC_WIDTH(uint32_t(-1), amax, mask, 32));
sum += SGLANG_SHFL_XOR_SYNC_WIDTH(uint32_t(-1), sum, mask, 32);
}
const float d = amax / 127;
const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);
y[ib].qs[iqs] = q;
if (iqs > 0) {
return;
}
y[ib].ds.x = __float2half(d);
y[ib].ds.y = __float2half(sum);
}
template <typename scalar_t>
static void quantize_row_q8_1_cuda(const scalar_t* x, void* vy, const int kx, const int ky, cudaStream_t stream) {
const int64_t kx_padded = (kx + 512 - 1) / 512 * 512;
const int block_num_x = (kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;
constexpr int MAX_BLOCK_SIZE = 65535;
for (int off = 0; off < ky; off += MAX_BLOCK_SIZE) {
const int num_blocks_y = std::min(ky, off + MAX_BLOCK_SIZE) - off;
const dim3 num_blocks(block_num_x, num_blocks_y, 1);
const dim3 block_size(CUDA_DEQUANTIZE_BLOCK_SIZE, 1, 1);
quantize_q8_1<<<num_blocks, block_size, 0, stream>>>(
&x[off * kx], (int32_t*)vy + off * (kx_padded / 32 * 9), kx, kx_padded);
}
}
torch::Tensor ggml_dequantize(
torch::Tensor W, // quant weight
int64_t type,
@@ -114,18 +114,21 @@ static __device__ __forceinline__ void moe_q(
if (threadIdx.x < n_per_r / QK8_1) {
const auto kby = threadIdx.x % (WARP_SIZE_GGUF / QI8_1);
const int col_y_eff = token_offs[threadIdx.y] / top_k;
const int block_x = ib0 * (qk / QK8_1) + ir * (WARP_SIZE_GGUF / QI8_1) + kby;
if (col_y_eff < ncols_y && block_x < blocks_per_col_y) {
const half2* dsi_src = &y[col_y_eff * blocks_per_col_y + block_x].ds;
half2* dsi_dst = &tile_y_ds[threadIdx.y * (WARP_SIZE_GGUF / QI8_1) + kby];
#pragma unroll
for (int i = 0; i < mmq_x; i += nwarps) {
const int col_y_eff = token_offs[i / nwarps] / top_k;
if (col_y_eff < ncols_y && block_x < blocks_per_col_y) {
const half2* dsi_src = &y[col_y_eff * blocks_per_col_y + block_x].ds;
half2* dsi_dst = &tile_y_ds[(threadIdx.y + i) * (WARP_SIZE_GGUF / QI8_1) + kby];
if (need_sum) {
*dsi_dst = *dsi_src;
} else {
float* dfi_dst = (float*)dsi_dst;
*dfi_dst = __low2float(*dsi_src);
if (need_sum) {
*dsi_dst = *dsi_src;
} else {
float* dfi_dst = (float*)dsi_dst;
*dfi_dst = __low2float(*dsi_src);
}
}
}
}
@@ -0,0 +1,17 @@
#pragma once
#include <cuda_fp16.h>
#include <cuda_runtime.h>
#include <algorithm>
#include <cstdint>
#include <limits>
// AOT supplies these definitions for CUDA, ROCm and MUSA before this header.
#ifndef WARP_SIZE
#define WARP_SIZE 32
#endif
#ifndef SGLANG_SHFL_XOR_SYNC
#define SGLANG_SHFL_XOR_SYNC(mask, var, lane_mask) __shfl_xor_sync((mask), (var), (lane_mask))
#define SGLANG_SHFL_XOR_SYNC_WIDTH(mask, var, lane_mask, width) __shfl_xor_sync((mask), (var), (lane_mask), (width))
#endif
@@ -1,3 +1,7 @@
#pragma once
#include "device_compat.cuh"
// adapted from
// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/ggml-common.h
// copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-common.h
@@ -955,27 +959,6 @@ typedef float (*vec_dot_q_mul_mat_cuda_t)(
const int& j,
const int& k);
// Utility function
template <typename dst_t>
static __device__ __forceinline__ dst_t convert_from_half(half val) {
return val;
}
template <>
__device__ __forceinline__ c10::BFloat16 convert_from_half<c10::BFloat16>(half val) {
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 || defined(USE_MUSA)
return __float2bfloat16(__half2float(val));
#else
return __half2float(val);
#endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 || defined(USE_MUSA)
}
template <>
__device__ __forceinline__ float convert_from_half<float>(half val) {
return __half2float(val);
}
#if defined(USE_ROCM)
#ifndef __has_builtin
@@ -1,3 +1,7 @@
#pragma once
#include "vecdotq.cuh"
// copied from
// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/moe_vec.cuh
// copied and adapted from
@@ -0,0 +1,56 @@
#pragma once
#include "ggml-common.h"
// Q8 gemv
template <typename scalar_t>
static __global__ void
quantize_q8_1(const scalar_t* __restrict__ x, void* __restrict__ vy, const int kx, const int kx_padded) {
const auto ix = blockDim.x * blockIdx.x + threadIdx.x;
if (ix >= kx_padded) {
return;
}
const auto iy = blockDim.y * blockIdx.y + threadIdx.y;
const int i_padded = iy * kx_padded + ix;
block_q8_1* y = (block_q8_1*)vy;
const int ib = i_padded / QK8_1; // block index
const int iqs = i_padded % QK8_1; // quant index
const float xi = ix < kx ? static_cast<float>(x[iy * kx + ix]) : 0.0f;
float amax = fabsf(xi);
float sum = xi;
#pragma unroll
for (int mask = 16; mask > 0; mask >>= 1) {
amax = fmaxf(amax, SGLANG_SHFL_XOR_SYNC_WIDTH(uint32_t(-1), amax, mask, 32));
sum += SGLANG_SHFL_XOR_SYNC_WIDTH(uint32_t(-1), sum, mask, 32);
}
const float d = amax / 127;
const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);
y[ib].qs[iqs] = q;
if (iqs > 0) {
return;
}
y[ib].ds.x = __float2half(d);
y[ib].ds.y = __float2half(sum);
}
template <typename scalar_t>
static void quantize_row_q8_1_cuda(const scalar_t* x, void* vy, const int kx, const int ky, cudaStream_t stream) {
const int64_t kx_padded = (kx + 512 - 1) / 512 * 512;
const int block_num_x = (kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;
constexpr int MAX_BLOCK_SIZE = 65535;
for (int off = 0; off < ky; off += MAX_BLOCK_SIZE) {
const int num_blocks_y = std::min(ky, off + MAX_BLOCK_SIZE) - off;
const dim3 num_blocks(block_num_x, num_blocks_y, 1);
const dim3 block_size(CUDA_DEQUANTIZE_BLOCK_SIZE, 1, 1);
quantize_q8_1<<<num_blocks, block_size, 0, stream>>>(
&x[off * kx], (int32_t*)vy + off * (kx_padded / 32 * 9), kx, kx_padded);
}
}
@@ -1,3 +1,7 @@
#pragma once
#include "ggml-common.h"
// copied from
// https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/vecdotq.cuh
// copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/vecdotq.cuh
@@ -258,6 +258,10 @@ void moe_sum_reduce(at::Tensor& input, at::Tensor& output, double routed_scaling
void moe_sum(torch::Tensor& input, torch::Tensor& output);
/// \brief Weight contiguous expert rows in place, writing zero for negative expert IDs.
/// The paired GGUF projections accept negative IDs and bound padding metadata reads.
void apply_gguf_moe_weights(torch::Tensor& outputs, const torch::Tensor& weights, const torch::Tensor& ids);
void fp8_blockwise_scaled_grouped_mm(
torch::Tensor& output,
torch::Tensor& a_ptrs,
@@ -77,6 +77,7 @@ else:
)
from sgl_kernel.memory import weak_ref_tensor
from sgl_kernel.moe import (
apply_gguf_moe_weights,
apply_shuffle_mul_sum,
fp8_blockwise_scaled_grouped_mm,
fused_qk_norm_rope,
@@ -145,6 +146,7 @@ else:
)
_DEBUG_EXPORT_NAMES = [
"apply_gguf_moe_weights",
"apply_shuffle_mul_sum",
"apply_token_bitmask_inplace_cuda",
"build_tree_kernel_efficient",
@@ -94,6 +94,22 @@ def moe_sum_reduce(
)
def apply_gguf_moe_weights(
outputs: torch.Tensor,
weights: torch.Tensor,
ids: torch.Tensor,
) -> None:
"""Weight contiguous [tokens * topk, hidden_size] expert outputs in place.
Weights and IDs are strided [tokens, topk] tensors on the same device.
Outputs and weights independently support float16, bfloat16, and float32;
IDs support int32 or int64. Negative IDs write zero without reading the
corresponding output or weight. Valid products round to the output dtype.
Outputs must not overlap weights or IDs.
"""
torch.ops.sgl_kernel.apply_gguf_moe_weights.default(outputs, weights, ids)
def moe_sum(
input_tensor: torch.Tensor,
output_tensor: torch.Tensor,
@@ -0,0 +1,130 @@
#pragma once
#include <sgl_kernel/quantization/gguf/ggml-common.h>
#include <sgl_kernel/tensor.h>
#include <sgl_kernel/utils.cuh>
#include <sgl_kernel/quantization/gguf/moe_vec.cuh>
#include <sgl_kernel/quantization/gguf/quantize_q8_1.cuh>
#include <sgl_kernel/quantization/gguf/vecdotq.cuh>
#include <cstdint>
#include <limits>
namespace sglang {
namespace gguf {
template <int kQuantType>
struct MoeVecFormat;
#define GGUF_MOE_VEC_FORMAT(type, qk, block, launcher) \
template <> \
struct MoeVecFormat<type> { \
static constexpr int kBlockSize = qk; \
static constexpr int kBlockBytes = sizeof(block); \
static constexpr int kAlignment = alignof(block); \
template <typename scalar_t> \
static void \
run(const void* w, \
const void* q, \
scalar_t* y, \
const int* ids, \
int topk, \
int m, \
int k, \
int n, \
int stride, \
cudaStream_t stream) { \
launcher<scalar_t>(w, q, y, ids, topk, m, k, n, stride, stream); \
} \
}
GGUF_MOE_VEC_FORMAT(2, QK4_0, block_q4_0, moe_vec_q4_0_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(3, QK4_0, block_q4_1, moe_vec_q4_1_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(6, QK5_0, block_q5_0, moe_vec_q5_0_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(7, QK5_1, block_q5_1, moe_vec_q5_1_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(8, QK8_0, block_q8_0, moe_vec_q8_0_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(10, QK_K, block_q2_K, moe_vec_q2_K_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(11, QK_K, block_q3_K, moe_vec_q3_K_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(12, QK_K, block_q4_K, moe_vec_q4_K_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(13, QK_K, block_q5_K, moe_vec_q5_K_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(14, QK_K, block_q6_K, moe_vec_q6_K_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(16, QK_K, block_iq2_xxs, moe_vec_iq2_xxs_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(17, QK_K, block_iq2_xs, moe_vec_iq2_xs_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(18, QK_K, block_iq3_xxs, moe_vec_iq3_xxs_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(19, QK_K, block_iq1_s, moe_vec_iq1_s_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(20, QK4_NL, block_iq4_nl, moe_vec_iq4_nl_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(21, QK_K, block_iq3_s, moe_vec_iq3_s_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(22, QK_K, block_iq2_s, moe_vec_iq2_s_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(23, QK_K, block_iq4_xs, moe_vec_iq4_xs_q8_1_cuda);
GGUF_MOE_VEC_FORMAT(29, QK_K, block_iq1_m, moe_vec_iq1_m_q8_1_cuda);
#undef GGUF_MOE_VEC_FORMAT
} // namespace gguf
/// \brief Quantize input rows to Q8_1 and project compact GGUF expert routes.
/// \param expert_ids Contiguous int32 routes; negative IDs write zero without reading operands.
/// \param output Zero-initialized private output, matching the AOT vector projection lifetime.
/// \param quantized_input Private Q8_1 scratch, with the same padded-row format as the AOT ABI.
template <typename scalar_t, int kQuantType>
void gguf_moe_a8_vec(
tvm::ffi::TensorView input,
tvm::ffi::TensorView weight,
tvm::ffi::TensorView expert_ids,
tvm::ffi::TensorView output,
tvm::ffi::TensorView quantized_input,
int64_t top_k,
int64_t row,
int64_t tokens) {
using namespace host;
using Format = gguf::MoeVecFormat<kQuantType>;
constexpr int64_t kMaxIndex = std::numeric_limits<int32_t>::max();
CHECK_HOST(tokens >= 0 && tokens <= kMaxIndex && top_k > 0 && top_k <= kMaxIndex);
CHECK_HOST(tokens <= 65535 / top_k) << "GGUF vector routes exceed the CUDA z-grid limit";
CHECK_HOST(row > 0 && row <= kMaxIndex);
auto device = SymbolicDevice{};
auto columns = SymbolicSize{"columns"};
auto experts = SymbolicSize{"experts"};
TensorMatcher({tokens, columns}).with_dtype<scalar_t>().template with_device<kDLCUDA>(device).verify(input);
const int64_t k = columns.unwrap();
CHECK_HOST(k > 0 && k <= kMaxIndex - 511 && k % Format::kBlockSize == 0);
const int64_t packed_columns = k / Format::kBlockSize * Format::kBlockBytes;
TensorMatcher({experts, row, packed_columns})
.with_dtype<uint8_t>()
.with_device(device)
.ensure_alignment(std::max(2, Format::kAlignment))
.verify(weight);
CHECK_HOST(experts.unwrap() > 0 && experts.unwrap() <= kMaxIndex / row / (k / Format::kBlockSize));
TensorMatcher({-1, -1}).with_dtype<int32_t>().with_device(device).verify(expert_ids);
CHECK_HOST(expert_ids.numel() == tokens * top_k);
TensorMatcher({tokens * top_k, row}).with_dtype<scalar_t>().with_device(device).verify(output);
const int64_t padded = (k + 511) / 512 * 512;
TensorMatcher({tokens, padded / 32 * 9}).with_dtype<int32_t>().with_device(device).verify(quantized_input);
CHECK_HOST(tokens <= kMaxIndex / padded && tokens * top_k <= kMaxIndex / row);
if (tokens == 0) return;
const auto stream = LaunchKernel::resolve_device(device.unwrap());
quantize_row_q8_1_cuda<scalar_t>(
static_cast<const scalar_t*>(input.data_ptr()),
quantized_input.data_ptr(),
static_cast<int>(k),
static_cast<int>(tokens),
stream);
Format::template run<scalar_t>(
weight.data_ptr(),
quantized_input.data_ptr(),
static_cast<scalar_t*>(output.data_ptr()),
static_cast<const int32_t*>(expert_ids.data_ptr()),
static_cast<int>(top_k),
static_cast<int>(tokens),
static_cast<int>(k),
static_cast<int>(row),
static_cast<int>(padded / 32 * 9),
stream);
CHECK_CUDA(cudaGetLastError());
}
} // namespace sglang
@@ -53,7 +53,7 @@ __global__ void count_and_sort_expert_tokens_kernel(
}
}
template <typename scalar_t>
template <typename scalar_t, bool kPadExpertIds>
__global__ void moe_align_block_size_kernel(
const scalar_t* __restrict__ topk_ids,
int32_t* __restrict__ sorted_token_ids,
@@ -65,7 +65,8 @@ __global__ void moe_align_block_size_kernel(
int32_t* __restrict__ cumsum,
bool pad_sorted_token_ids,
const int32_t scan_size,
int32_t max_num_tokens_padded) {
int32_t max_num_tokens_padded,
int32_t max_num_m_blocks) {
// Use a separate thread block to populate sorted_token_ids
if (blockIdx.x == 1) {
if (pad_sorted_token_ids) {
@@ -210,11 +211,9 @@ __global__ void moe_align_block_size_kernel(
__syncthreads();
int offset = warp_sums[warp_id];
if (tid < scan_size) scan_buf[tid] = pre + offset;
if (tid < num_experts) prefix[tid] = pre + offset;
// Publish the prefix before other threads binary-search it below.
__syncthreads();
// Write prefix[0..num_experts - 1] and cumsum
if (tid < num_experts) prefix[tid] = scan_buf[tid];
#endif
if (tid <= num_experts) {
@@ -235,9 +234,14 @@ __global__ void moe_align_block_size_kernel(
}
expert_ids[i] = left - 2;
}
if constexpr (kPadExpertIds) {
for (int32_t i = num_blocks + tid; i < max_num_m_blocks; i += stride) {
expert_ids[i] = -1;
}
}
}
template <typename scalar_t, int32_t fill_threads>
template <typename scalar_t, int32_t fill_threads, bool kPadExpertIds>
__global__ void moe_align_block_size_small_batch_expert_kernel(
const scalar_t* __restrict__ topk_ids,
int32_t* __restrict__ sorted_token_ids,
@@ -247,7 +251,8 @@ __global__ void moe_align_block_size_small_batch_expert_kernel(
int32_t block_size,
size_t numel,
bool pad_sorted_token_ids,
int32_t max_num_tokens_padded) {
int32_t max_num_tokens_padded,
int32_t max_num_m_blocks) {
// Adapted from
// https://github.com/vllm-project/vllm/pull/29642/files#diff-5647b1413f4ae9aacba904eca8f8a8aee9079321eadff4c10101a2c6962dcc53R226
// Use an additional group of threads to fill sorted_token_ids.
@@ -310,6 +315,11 @@ __global__ void moe_align_block_size_small_batch_expert_kernel(
expert_ids[i / block_size] = tid - 1;
}
}
if constexpr (kPadExpertIds) {
for (int32_t i = cumsum[num_experts] / block_size + tid; i < max_num_m_blocks; i += stride) {
expert_ids[i] = -1;
}
}
for (size_t i = tid; i < numel; i += stride) {
int32_t expert_id = topk_ids[i] + 1;
@@ -328,7 +338,7 @@ __global__ void moe_align_block_size_small_batch_expert_kernel(
// With 1024 threads and EXPERTS_PER_THREAD=4, covers at most 4096 expert
// indices. Since num_experts includes the +1 offset bucket, this supports
// up to 4095 real experts.
template <typename scalar_t, int EXPERTS_PER_THREAD>
template <typename scalar_t, int EXPERTS_PER_THREAD, bool kPadExpertIds>
__global__ void moe_align_block_size_kernel_v2(
const scalar_t* __restrict__ topk_ids,
int32_t* __restrict__ sorted_token_ids,
@@ -340,7 +350,8 @@ __global__ void moe_align_block_size_kernel_v2(
size_t numel,
int32_t* __restrict__ cumsum,
bool pad_sorted_token_ids,
int32_t max_num_tokens_padded) {
int32_t max_num_tokens_padded,
int32_t max_num_m_blocks) {
// Use a separate thread block to populate sorted_token_ids
if (blockIdx.x == 1) {
if (pad_sorted_token_ids) {
@@ -442,11 +453,16 @@ __global__ void moe_align_block_size_kernel_v2(
}
}
}
if constexpr (kPadExpertIds) {
for (int32_t i = cumsum[num_experts] / block_size + tid; i < max_num_m_blocks; i += blockDim.x) {
expert_ids[i] = -1;
}
}
}
} // namespace moe
template <typename scalar_t>
template <typename scalar_t, bool kPadExpertIds>
struct MoeAlignBlockSizeKernel {
static void
run(tvm::ffi::TensorView topk_ids,
@@ -487,7 +503,7 @@ struct MoeAlignBlockSizeKernel {
constexpr int32_t fill_threads = 256;
const int32_t shared_mem_size = ((num_thread + 1) * num_experts + (num_experts + 1)) * sizeof(int32_t);
auto kernel = moe::moe_align_block_size_small_batch_expert_kernel<scalar_t, fill_threads>;
auto kernel = moe::moe_align_block_size_small_batch_expert_kernel<scalar_t, fill_threads, kPadExpertIds>;
LaunchKernel(dim3(1), dim3(fill_threads + num_thread), stream, shared_mem_size)(
kernel,
topk_ids_ptr,
@@ -498,12 +514,13 @@ struct MoeAlignBlockSizeKernel {
(int32_t)block_size,
numel,
pad_sorted_token_ids,
(int32_t)max_num_tokens_padded);
(int32_t)max_num_tokens_padded,
static_cast<int32_t>(expert_ids.numel()));
} else if (num_experts <= 1024) {
const size_t scan_size = host::round_up_pow2(static_cast<uint32_t>(num_experts));
const size_t shared_mem_size = (num_experts + (num_experts + 1) + scan_size + WARP_SIZE) * sizeof(int32_t);
auto align_kernel = moe::moe_align_block_size_kernel<scalar_t>;
auto align_kernel = moe::moe_align_block_size_kernel<scalar_t, kPadExpertIds>;
LaunchKernel(dim3(2), dim3(threads), stream, shared_mem_size)(
align_kernel,
topk_ids_ptr,
@@ -516,7 +533,8 @@ struct MoeAlignBlockSizeKernel {
cumsum_buffer_ptr,
pad_sorted_token_ids,
(int32_t)scan_size,
(int32_t)max_num_tokens_padded);
(int32_t)max_num_tokens_padded,
static_cast<int32_t>(expert_ids.numel()));
const int block_threads = std::min(256, threads);
const int num_blocks = (numel + block_threads - 1) / block_threads;
@@ -533,7 +551,7 @@ struct MoeAlignBlockSizeKernel {
auto launch_v2 = [&](auto ept_tag) {
constexpr int EPT = decltype(ept_tag)::value;
auto v2_kernel = moe::moe_align_block_size_kernel_v2<scalar_t, EPT>;
auto v2_kernel = moe::moe_align_block_size_kernel_v2<scalar_t, EPT, kPadExpertIds>;
LaunchKernel(dim3(2), dim3(threads), stream, shared_mem_size)(
v2_kernel,
topk_ids_ptr,
@@ -546,7 +564,8 @@ struct MoeAlignBlockSizeKernel {
numel,
cumsum_buffer_ptr,
pad_sorted_token_ids,
(int32_t)max_num_tokens_padded);
(int32_t)max_num_tokens_padded,
static_cast<int32_t>(expert_ids.numel()));
};
if (padded_num_experts <= 2048) {
@@ -0,0 +1 @@
../../../../../aot/include/sgl_kernel/quantization/gguf/device_compat.cuh
@@ -0,0 +1 @@
../../../../../aot/include/sgl_kernel/quantization/gguf/ggml-common.h
@@ -0,0 +1 @@
../../../../../aot/include/sgl_kernel/quantization/gguf/moe_vec.cuh
@@ -0,0 +1 @@
../../../../../aot/include/sgl_kernel/quantization/gguf/quantize_q8_1.cuh
@@ -0,0 +1 @@
../../../../../aot/include/sgl_kernel/quantization/gguf/vecdotq.cuh
+32
View File
@@ -81,6 +81,38 @@ register_kernel(
)
register_kernel(
KernelSpec(
op="moe.gguf_moe_a8_vec",
backend=KernelBackend.JIT,
target="sglang.kernels.ops.moe.gguf_moe_vec:gguf_moe_a8_vec",
capabilities=CUDA_ONLY,
)
)
register_kernel(
KernelSpec(
op="moe.apply_gguf_moe_weights",
backend=KernelBackend.TRITON,
target="sglang.kernels.ops.moe.gguf_moe_weights:apply_gguf_moe_weights",
capabilities=CUDA_ONLY,
)
)
register_kernel(
KernelSpec(
op="moe.apply_gguf_moe_weights",
backend=KernelBackend.AOT,
target="sglang.kernels.ops.moe.gguf_moe_weights_aot:apply_gguf_moe_weights",
capabilities=frozenset(
{CapabilityRequirement.CUDA, CapabilityRequirement.MUSA}
),
)
)
def moe_align_block_size(
topk_ids: torch.Tensor,
num_experts: int,
@@ -0,0 +1,85 @@
"""GGUF vector expert projections with consumer-owned inactive routes."""
from __future__ import annotations
from typing import TYPE_CHECKING
import torch
from sglang.kernels.jit.utils import (
cache_once,
get_jit_cuda_arch,
load_jit,
make_cpp_args,
)
from sglang.kernels.kernel_api_logging import debug_kernel_api
if TYPE_CHECKING:
from tvm_ffi.module import Module
@cache_once
def _jit_gguf_moe_vec_module(dtype: torch.dtype, quant_type: int) -> Module:
if dtype not in (torch.float16, torch.bfloat16, torch.float32):
raise ValueError(f"Unsupported GGUF input dtype: {dtype}")
if quant_type not in (
2,
3,
6,
7,
8,
10,
11,
12,
13,
14,
16,
17,
18,
19,
20,
21,
22,
23,
29,
):
raise ValueError(f"Unsupported GGUF vector quantization type: {quant_type}")
args = make_cpp_args(dtype, quant_type)
arch = get_jit_cuda_arch()
# Match the AOT common_ops SM90 fast-math build; its other build is precise.
flags = ["--use_fast_math"] if (arch.major, arch.minor) == (9, 0) else []
return load_jit(
"gguf_moe_vec",
*args,
cuda_files=["moe/gguf_moe_vec.cuh"],
cuda_wrappers=[("project", f"gguf_moe_a8_vec<{args}>")],
extra_cuda_cflags=flags,
)
@debug_kernel_api
def gguf_moe_a8_vec(
input: torch.Tensor,
weight: torch.Tensor,
topk_ids: torch.Tensor,
top_k: int,
type: int,
row: int,
tokens: int,
) -> torch.Tensor:
"""Project packed GGUF experts, writing zero for every negative route.
Inputs and uint8 packed weights are contiguous. IDs are contiguous int32
local expert indices or negative sentinels; their total size is tokens * top_k.
The down projection retains the original route shape with top_k=1.
Quantization, vector reduction and output rounding match the AOT operation.
"""
output = torch.zeros((tokens * top_k, row), dtype=input.dtype, device=input.device)
padded = (input.shape[1] + 511) // 512 * 512
quantized_input = torch.empty(
(tokens, padded // 32 * 9), dtype=torch.int32, device=input.device
)
_jit_gguf_moe_vec_module(input.dtype, int(type)).project(
input, weight, topk_ids, output, quantized_input, top_k, row, tokens
)
return output
@@ -0,0 +1,73 @@
"""Apply GGUF route weights without reading inactive expert outputs."""
import torch
import triton
import triton.language as tl
@triton.jit(
do_not_specialize=["weight_stride0", "weight_stride1", "id_stride0", "id_stride1"]
)
def _apply_gguf_moe_weights_kernel(
outputs,
weights,
expert_ids,
width: tl.constexpr,
topk: tl.constexpr,
weight_stride0,
weight_stride1,
id_stride0,
id_stride1,
BLOCK: tl.constexpr,
):
row = tl.program_id(0).to(tl.int64)
token = row // topk
slot = row % topk
active = tl.load(expert_ids + token * id_stride0 + slot * id_stride1) >= 0
columns = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
values = tl.load(
outputs + row * width + columns,
mask=active & (columns < width),
other=0,
).to(tl.float32)
factor = tl.load(
weights + token * weight_stride0 + slot * weight_stride1,
mask=active,
other=0,
).to(tl.float32)
tl.store(outputs + row * width + columns, values * factor, mask=columns < width)
def apply_gguf_moe_weights(
outputs: torch.Tensor, weights: torch.Tensor, expert_ids: torch.Tensor
) -> None:
"""Weight contiguous GGUF rows in place, zeroing routes with negative IDs.
``outputs`` is ``[tokens * topk, hidden]`` and routing tensors are
``[tokens, topk]``. Products round to the output dtype before the caller's
native ``moe_sum``; inactive rows read neither outputs nor weights.
"""
assert outputs.ndim == weights.ndim == expert_ids.ndim == 2
assert outputs.is_contiguous()
assert weights.shape == expert_ids.shape
assert outputs.shape[0] == expert_ids.numel()
assert outputs.device == weights.device == expert_ids.device
assert outputs.dtype in (torch.float16, torch.bfloat16, torch.float32)
assert weights.dtype in (torch.float16, torch.bfloat16, torch.float32)
assert expert_ids.dtype in (torch.int32, torch.int64)
if outputs.numel() == 0:
return
width = outputs.shape[1]
block = min(1024, triton.next_power_of_2(width))
_apply_gguf_moe_weights_kernel[(outputs.shape[0], triton.cdiv(width, block))](
outputs,
weights,
expert_ids,
width,
weights.shape[1],
weights.stride(0),
weights.stride(1),
expert_ids.stride(0),
expert_ids.stride(1),
BLOCK=block,
)
@@ -0,0 +1,36 @@
"""Native GGUF route weighting for CUDA and MUSA builds."""
from typing import Callable, Optional
import torch
def apply_gguf_moe_weights(
outputs: torch.Tensor, weights: torch.Tensor, expert_ids: torch.Tensor
) -> None:
"""Weight rows in place, rounding each product before native MoE reduction."""
import sgl_kernel # noqa: F401
torch.ops.sgl_kernel.apply_gguf_moe_weights.default(outputs, weights, expert_ids)
def resolve_gguf_moe_weights(device_type: str) -> Optional[Callable]:
"""Resolve once at backend import, retaining compatibility with older wheels.
This native operator ships with negative-ID-safe GGUF vector projections,
early MMQ slack bounds, and matching Q8_1 value/scale row indexing. Its
device registration, rather than the Python wrapper's presence, establishes
that the rebuilt provider can consume raw routes without Python clearing.
"""
import sgl_kernel # noqa: F401
try:
torch.ops.sgl_kernel.apply_gguf_moe_weights.default
except AttributeError:
return None
key = torch._C._dispatch_key_for_device(device_type)
if not torch._C._dispatch_has_kernel_for_dispatch_key(
"sgl_kernel::apply_gguf_moe_weights", key
):
return None
return apply_gguf_moe_weights
+6 -3
View File
@@ -11,8 +11,8 @@ if TYPE_CHECKING:
@cache_once
def _jit_moe_align_module(dtype: torch.dtype) -> Module:
args = make_cpp_args(dtype)
def _jit_moe_align_module(dtype: torch.dtype, pad_expert_ids: bool) -> Module:
args = make_cpp_args(dtype, pad_expert_ids)
return load_jit(
"moe_align_block_size",
*args,
@@ -32,8 +32,11 @@ def moe_align_block_size(
num_tokens_post_pad: torch.Tensor,
cumsum_buffer: torch.Tensor,
pad_sorted_token_ids: bool = False,
*,
pad_expert_ids: bool = False,
) -> None:
module = _jit_moe_align_module(topk_ids.dtype)
"""Align routes; optionally define unused expert-map capacity as -1."""
module = _jit_moe_align_module(topk_ids.dtype, pad_expert_ids)
module.moe_align_block_size(
topk_ids,
num_experts,
+14 -2
View File
@@ -59,9 +59,10 @@ class DeviceType(str, Enum):
CUDA = "cuda"
HIP = "hip"
MUSA = "musa"
NPU = "npu" # Ascend NPU (torch_npu / sgl_kernel_npu)
CPU = "cpu"
# TODO(RFC #29630): XPU / MUSA / ... as backends land.
# TODO(RFC #29630): XPU / ... as backends land.
class PlatformInfo(msgspec.Struct, frozen=True):
@@ -101,6 +102,14 @@ class PlatformInfo(msgspec.Struct, frozen=True):
npu = getattr(torch, "npu", None)
if npu is not None and npu.is_available():
return cls(device_type="npu")
musa = getattr(torch, "musa", None)
if musa is not None and musa.is_available():
return cls(device_type="musa")
if (
getattr(torch.version, "musa", None) is not None
and torch.cuda.is_available()
):
return cls(device_type="musa")
if torch.cuda.is_available():
major, minor = torch.cuda.get_device_capability()
return cls(
@@ -126,7 +135,8 @@ class CapabilityRequirement(msgspec.Struct, frozen=True):
(``min_cuda_arch`` / ``max_cuda_arch`` apply only when ``device == CUDA``).
The device-only cases are so common that they are exposed as class constants
(``CapabilityRequirement.CUDA`` / ``.HIP`` / ``.NPU``); use :meth:`cuda` for an
(``CapabilityRequirement.CUDA`` / ``.HIP`` / ``.MUSA`` / ``.NPU``);
use :meth:`cuda` for an
arch-bounded CUDA requirement (e.g. ``CapabilityRequirement.cuda(
min_sm=(10, 0))`` for SM100+).
"""
@@ -139,6 +149,7 @@ class CapabilityRequirement(msgspec.Struct, frozen=True):
# instances of the class itself). ClassVar keeps them out of msgspec fields.
CUDA: ClassVar[CapabilityRequirement]
HIP: ClassVar[CapabilityRequirement]
MUSA: ClassVar[CapabilityRequirement]
NPU: ClassVar[CapabilityRequirement]
@classmethod
@@ -164,6 +175,7 @@ class CapabilityRequirement(msgspec.Struct, frozen=True):
CapabilityRequirement.CUDA = CapabilityRequirement(device=DeviceType.CUDA)
CapabilityRequirement.HIP = CapabilityRequirement(device=DeviceType.HIP)
CapabilityRequirement.MUSA = CapabilityRequirement(device=DeviceType.MUSA)
CapabilityRequirement.NPU = CapabilityRequirement(device=DeviceType.NPU)
CUDA_ONLY: FrozenSet[CapabilityRequirement] = frozenset({CapabilityRequirement.CUDA})
@@ -34,6 +34,8 @@ def moe_align_block_size(
block_size: int,
num_experts: int,
ignore_invalid_expert: bool = False,
*,
pad_expert_ids: bool = False,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Aligns the token distribution across experts to be compatible with block
@@ -72,8 +74,13 @@ def moe_align_block_size(
- The padding ensures that the total number of tokens is now divisible
by block_size for proper block matrix operations.
"""
if pad_expert_ids and (not _is_cuda or ignore_invalid_expert):
raise ValueError(
"Padding expert IDs requires CUDA and the invalid-expert bucket"
)
# ===== TO BE REFACTORED ====
if _SGLANG_EXPERIMENTAL_LORA_OPTI:
if _SGLANG_EXPERIMENTAL_LORA_OPTI and not pad_expert_ids:
from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs
if lora_envs.SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN.get() and num_experts <= 8191:
@@ -116,6 +123,7 @@ def moe_align_block_size(
and topk_ids.numel() <= SMALL_NUMEL_LIMIT
and num_experts + 1 > _CUDA_SMALL_BATCH_MAX_BUCKETS
and not ignore_invalid_expert
and not pad_expert_ids
):
moe_align_small_numel(
topk_ids,
@@ -128,11 +136,11 @@ def moe_align_block_size(
return sorted_ids, expert_ids, num_tokens_post_pad
# ===== TO BE REFACTORED ====
use_jit_align = False
use_jit_align = pad_expert_ids
if _SGLANG_EXPERIMENTAL_LORA_OPTI:
from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs
use_jit_align = lora_envs.SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN.get()
use_jit_align |= lora_envs.SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN.get()
if use_jit_align:
from sglang.kernels.ops.moe.moe_align import (
moe_align_block_size as jit_moe_align_block_size,
@@ -147,6 +155,7 @@ def moe_align_block_size(
num_tokens_post_pad,
cumsum_buffer,
True,
pad_expert_ids=pad_expert_ids,
)
# ===== END TO BE REFACTORED ====
else:
+60 -35
View File
@@ -42,18 +42,20 @@ _is_hip = is_hip()
_is_xpu = is_xpu()
_is_musa = is_musa()
_is_npu = is_npu()
_musa_apply_gguf_moe_weights = None
if _is_cuda:
from sgl_kernel import moe_sum
from sgl_kernel.quantization import (
ggml_dequantize,
ggml_moe_a8,
ggml_moe_a8_vec,
ggml_mul_mat_a8,
ggml_mul_mat_vec_a8,
)
from sglang.kernels.ops.activation.activation import gelu_and_mul, silu_and_mul
from sglang.kernels.ops.moe.gguf_moe_vec import gguf_moe_a8_vec as ggml_moe_a8_vec
from sglang.kernels.ops.moe.gguf_moe_weights import apply_gguf_moe_weights
elif _is_musa:
from sgl_kernel import gelu_and_mul, moe_sum, silu_and_mul
from sgl_kernel.quantization import (
@@ -63,6 +65,10 @@ elif _is_musa:
ggml_mul_mat_a8,
ggml_mul_mat_vec_a8,
)
from sglang.kernels.ops.moe.gguf_moe_weights_aot import resolve_gguf_moe_weights
_musa_apply_gguf_moe_weights = resolve_gguf_moe_weights("musa")
elif _is_npu:
from gguf import dequantize as gguf_dequantize
else:
@@ -243,15 +249,31 @@ def fused_moe_gguf(
qweight_type2: int,
activation: str,
) -> torch.Tensor:
def act(x: torch.Tensor):
def act(x: torch.Tensor, *, routed: bool = False):
if activation == "silu":
return silu_and_mul(x)
activation_fn = silu_and_mul
elif activation == "gelu":
return gelu_and_mul(x)
raise ValueError(f"Unsupported activation: {activation}")
activation_fn = gelu_and_mul
else:
raise ValueError(f"Unsupported activation: {activation}")
if routed and _is_musa and not native_musa:
x.masked_fill_(inactive.reshape(-1, 1), 0.0)
return activation_fn(x)
inactive = topk_ids < 0
x = x.masked_fill(inactive.all(dim=-1, keepdim=True), 0.0)
def sum_expert_outputs(out: torch.Tensor):
if _is_cuda:
apply_gguf_moe_weights(out, topk_weights, topk_ids)
elif native_musa:
_musa_apply_gguf_moe_weights(out, topk_weights, topk_ids)
else:
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
valid_weights = topk_weights.masked_fill(inactive, 0.0)
out.view(*topk_ids.shape, -1).mul_(valid_weights.unsqueeze(-1))
moe_sum(out.view(*topk_ids.shape, -1), out_hidden_states)
native_musa = _is_musa and _musa_apply_gguf_moe_weights is not None
if _is_musa and not native_musa:
inactive = topk_ids < 0
out_hidden_states = torch.empty_like(x)
# unless we decent expert reuse we are better off running moe_vec kernel
if (
@@ -265,13 +287,14 @@ def fused_moe_gguf(
BLOCK_SIZE = ggml_moe_get_block_size(qweight_type)
sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size(
topk_ids, BLOCK_SIZE, E
topk_ids, BLOCK_SIZE, E, pad_expert_ids=_is_cuda
)
# Released MMQ kernels inspect one slack block before their bound check.
block_starts = (
torch.arange(expert_ids.numel(), device=expert_ids.device) * BLOCK_SIZE
)
expert_ids.masked_fill_(block_starts >= num_tokens_post_padded, -1)
if _is_musa and not native_musa:
# Released MMQ kernels inspect a slack block before their bound check.
block_starts = (
torch.arange(expert_ids.numel(), device=expert_ids.device) * BLOCK_SIZE
)
expert_ids.masked_fill_(block_starts >= num_tokens_post_padded, -1)
out = ggml_moe_a8(
x,
w1,
@@ -283,8 +306,7 @@ def fused_moe_gguf(
top_k,
num_tokens,
)
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
out = act(out)
out = act(out, routed=True)
out = ggml_moe_a8(
out,
w2,
@@ -296,34 +318,31 @@ def fused_moe_gguf(
1,
num_tokens * top_k,
)
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
valid_weights = topk_weights.masked_fill(inactive, 0.0)
out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_(
valid_weights.view(num_tokens, top_k, 1)
)
# TODO(FlamingoPg): maybe we can use moe_sum_reduce here?
moe_sum(out, out_hidden_states)
sum_expert_outputs(out)
elif qweight_type2 in MMVQ_QUANT_TYPES and qweight_type in MMVQ_QUANT_TYPES:
num_tokens, _ = x.shape
E, N, _ = w1.shape
top_k = topk_ids.shape[1]
# Older vector kernels form weight addresses before checking -1.
# They have no per-expert capacity buffer; discard their dummy slots.
safe_ids = topk_ids.clamp_min(0)
if _is_cuda or native_musa:
# The vector ABI reads compact int32 IDs. Normal routes keep their
# storage; exceptional int64 routes canonicalize negative IDs.
safe_ids = (
topk_ids.contiguous()
if topk_ids.dtype == torch.int32
else topk_ids.clamp_min(-1).to(torch.int32).contiguous()
)
else:
# Older vector kernels form weight addresses before checking -1.
safe_ids = topk_ids.clamp_min(0)
out = ggml_moe_a8_vec(x, w1, safe_ids, top_k, qweight_type, N, num_tokens)
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
out = act(out)
out = act(out, routed=True)
out = ggml_moe_a8_vec(
out, w2, safe_ids, 1, qweight_type2, w2.shape[1], num_tokens * top_k
)
out.masked_fill_(inactive.reshape(-1, 1), 0.0)
valid_weights = topk_weights.masked_fill(inactive, 0.0)
out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_(
valid_weights.view(num_tokens, top_k, 1)
)
moe_sum(out, out_hidden_states)
sum_expert_outputs(out)
else:
logger.warning_once(
"There is no support for fast MoE kernel "
@@ -332,7 +351,7 @@ def fused_moe_gguf(
)
for tok, (w, idx) in enumerate(zip(topk_weights, topk_ids)):
inp = x[tok].reshape((1,) + x.shape[1:])
current_hidden_state = torch.zeros_like(inp)
current_hidden_state = None
for ww, ii in zip(w, idx):
if ii < 0:
continue
@@ -345,8 +364,14 @@ def fused_moe_gguf(
current_state = fused_mul_mat_gguf(
out, expert_down, qweight_type2
).mul_(ww)
current_hidden_state.add_(current_state)
out_hidden_states[tok] = current_hidden_state
if current_hidden_state is None:
current_hidden_state = current_state
else:
current_hidden_state.add_(current_state)
if current_hidden_state is None:
out_hidden_states[tok].zero_()
else:
out_hidden_states[tok] = current_hidden_state
return out_hidden_states
@@ -0,0 +1,65 @@
"""Compare all-valid GGUF weighting with the original in-place product."""
import sgl_kernel # noqa: F401
import torch
from sglang.kernels.jit.benchmark import marker
from sglang.kernels.ops.moe.gguf_moe_vec import gguf_moe_a8_vec
from sglang.kernels.ops.moe.gguf_moe_weights_aot import resolve_gguf_moe_weights
from sglang.srt.utils import is_musa
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=10, stage="nightly", runner_config="1-gpu-large")
def torch_weight(outputs, weights, ids):
outputs.view(*ids.shape, -1).mul_(weights.unsqueeze(-1))
@marker.parametrize("tokens", [1, 64, 1024, 8192], [1024])
@marker.parametrize("hidden", [4096, 7168], [4096])
@marker.parametrize(
"dtype", [torch.float16, torch.bfloat16, torch.float32], [torch.float16]
)
@marker.benchmark("provider", ["aot", "torch"])
def benchmark(tokens, hidden, dtype, provider):
device = "musa" if is_musa() else "cuda"
native = resolve_gguf_moe_weights(device)
if provider == "aot" and native is None:
marker.skip("Native GGUF weighting requires a rebuilt kernel provider")
outputs = torch.ones(tokens * 8, hidden, device=device, dtype=dtype)
# Unit weights keep repeated in-place calls numerically stable.
weights = torch.ones(tokens, 8, device=device)
ids = torch.zeros(tokens, 8, dtype=torch.int32, device=device)
return marker.do_bench(
native if provider == "aot" else torch_weight,
input_args=(outputs, weights, ids),
memory_output=(outputs,),
)
@marker.parametrize("tokens", [1, 32, 64, 128], [64])
@marker.parametrize(
"dtype", [torch.float16, torch.bfloat16, torch.float32], [torch.float16]
)
@marker.benchmark("provider", ["jit", "aot"])
def benchmark_vector(tokens, dtype, provider):
"""Compare the same all-valid Q4_0 vector operation, including private scratch."""
from sgl_kernel.quantization import ggml_moe_a8_vec
if is_musa():
marker.skip("Local GGUF vector projection is CUDA-only")
inputs = torch.full((tokens, 4096), 0.25, device="cuda", dtype=dtype)
packed = torch.full(
(4, 128, 4096 // 32 * 18), 0x31, device="cuda", dtype=torch.uint8
)
ids = torch.zeros(tokens, 2, dtype=torch.int32, device="cuda")
return marker.do_bench(
gguf_moe_a8_vec if provider == "jit" else ggml_moe_a8_vec,
input_args=(inputs, packed, ids, 2, 2, 128, tokens),
)
if __name__ == "__main__":
benchmark.run()
benchmark_vector.run()
@@ -0,0 +1,235 @@
"""GGUF route weighting preserves per-slot rounding and skips inactive data."""
import unittest
import sgl_kernel # noqa: F401
import torch
from gguf import GGML_QUANT_SIZES, GGMLQuantizationType
from sglang.kernels.ops.moe.gguf_moe_vec import gguf_moe_a8_vec
from sglang.kernels.ops.moe.gguf_moe_weights_aot import resolve_gguf_moe_weights
from sglang.srt.utils import is_musa
from sglang.test.ci.ci_register import register_cuda_ci, register_musa_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=90, stage="nightly", runner_config="1-gpu-large")
# backend-specific: MUSA uses the native weighting operator in its serving path.
register_musa_ci(est_time=6, suite="nightly-musa-1-gpu", nightly=True)
class TestGgufMoeWeights(CustomTestCase):
@unittest.skipIf(is_musa(), "Local GGUF vector projection is CUDA-only")
@torch.inference_mode()
def test_vector_formats_match_native_and_replay_inactive_routes(self):
"""Every vector format retains real provider math and skips negative operands."""
from sgl_kernel.quantization import ggml_moe_a8_vec
generator = torch.Generator(device="cuda").manual_seed(314)
base_ids = torch.tensor(
[[0, 3], [1, 2], [3, 0]], dtype=torch.int32, device="cuda"
)
# These valid packed bytes give finite, nonzero scales, integer codes and
# codebook indices, including IQ1_M's distributed FP16 scale bits.
for quant_type in (
2,
3,
6,
7,
8,
10,
11,
12,
13,
14,
16,
17,
18,
19,
20,
21,
22,
23,
29,
):
block_size, block_bytes = GGML_QUANT_SIZES[GGMLQuantizationType(quant_type)]
packed = torch.empty(
4, 17, 256 // block_size * block_bytes, dtype=torch.uint8, device="cuda"
)
for expert in range(4):
packed[expert].fill_(0x31 + expert)
dtypes = (
(torch.float16, torch.bfloat16, torch.float32)
if quant_type in (2, 8, 10, 29)
else (torch.float16,)
)
for dtype in dtypes:
with self.subTest(quant_type=quant_type, dtype=dtype):
inputs = (
torch.randn(
3, 256, generator=generator, device="cuda", dtype=dtype
)
/ 4
)
original = inputs.clone()
ids = base_ids.clone()
expected = ggml_moe_a8_vec(
inputs, packed, ids, 2, quant_type, 17, 3
)
self.assertTrue(bool(torch.isfinite(expected).all()))
self.assertGreater(float(expected.abs().max()), 0.0)
actual = gguf_moe_a8_vec(inputs, packed, ids, 2, quant_type, 17, 3)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
# The down projection has one input per original route, while
# its IDs retain [original_tokens, original_topk] shape.
down = inputs.repeat_interleave(2, dim=0)
torch.testing.assert_close(
gguf_moe_a8_vec(down, packed, ids, 1, quant_type, 17, 6),
ggml_moe_a8_vec(down, packed, ids, 1, quant_type, 17, 6),
rtol=0,
atol=0,
)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured = gguf_moe_a8_vec(
inputs, packed, ids, 2, quant_type, 17, 3
)
original_pointer = ids.data_ptr()
for routes in (
[[-1, 3], [1, -3], [-7, -1]],
[[-1, -3], [-7, -1], [-1, -1]],
base_ids,
):
ids.copy_(
torch.as_tensor(routes, dtype=ids.dtype, device=ids.device)
)
inputs.copy_(original)
inputs[(ids < 0).all(dim=1)] = float("nan")
captured.fill_(float("nan"))
graph.replay()
live = (ids >= 0).flatten()
self.assertTrue(bool(torch.isfinite(captured).all()))
self.assertEqual(int(torch.count_nonzero(captured[~live])), 0)
torch.testing.assert_close(
captured[live], expected[live], rtol=0, atol=0
)
self.assertEqual(ids.data_ptr(), original_pointer)
empty = gguf_moe_a8_vec(
inputs[:0], packed, ids[:0], 2, quant_type, 17, 0
)
self.assertEqual(empty.shape, (0, 17))
def test_rounding_and_poisoned_routes(self):
"""Weights round per slot before summation; dropped NaNs contribute zero."""
device = "musa" if is_musa() else "cuda"
providers = {}
native = resolve_gguf_moe_weights(device)
if device == "musa":
self.assertIsNotNone(native, "MUSA kernel tests require a rebuilt provider")
else:
from sglang.kernels.ops.moe.gguf_moe_weights import apply_gguf_moe_weights
providers["triton"] = apply_gguf_moe_weights
if native is not None:
providers["aot"] = native
generator = torch.Generator(device=device).manual_seed(836)
for provider, apply_weights in providers.items():
for dtype in (torch.float16, torch.bfloat16, torch.float32):
for weight_dtype in (torch.float16, torch.bfloat16, torch.float32):
for strided in (False, True):
with self.subTest(
provider=provider,
dtype=dtype,
weight_dtype=weight_dtype,
strided=strided,
):
outputs = torch.randn(
12, 257, generator=generator, device=device, dtype=dtype
)
weights = torch.randn(
4,
3,
generator=generator,
device=device,
dtype=weight_dtype,
)
ids = torch.tensor(
[[0, 1, 2], [0, -3, 2], [-1, -1, -1], [1, 0, -1]],
device=device,
dtype=torch.int64 if strided else torch.int32,
)
if strided:
weights = weights.t().contiguous().t()
ids = ids.t().contiguous().t()
inactive = ids < 0
outputs[inactive.flatten()] = float("nan")
weights[inactive] = float("nan")
expected = outputs.clone()
expected[inactive.flatten()] = 0
expected.view(4, 3, 257).mul_(
weights.masked_fill(inactive, 0).unsqueeze(-1)
)
apply_weights(outputs, weights, ids)
torch.testing.assert_close(
outputs, expected, rtol=0, atol=0
)
self.assertTrue(bool(torch.isfinite(outputs).all()))
ids.fill_(-1)
weights.fill_(float("nan"))
outputs.fill_(float("nan"))
apply_weights(outputs, weights, ids)
self.assertEqual(int(torch.count_nonzero(outputs)), 0)
apply_weights(outputs[:0], weights[:0], ids[:0])
@unittest.skipIf(is_musa(), "CUDA fast-math regression")
def test_subnormal_operands_and_products(self):
"""A tiny score can produce a normal result even in the SM90 fast-math build."""
native = resolve_gguf_moe_weights("cuda")
if native is None:
self.skipTest("Native GGUF weighting requires a rebuilt kernel provider")
for dtype in (torch.float32, torch.bfloat16):
for weight_dtype in (torch.float32, torch.bfloat16):
for strided in (False, True):
with self.subTest(
dtype=dtype, weight_dtype=weight_dtype, strided=strided
):
outputs = (
torch.tensor(
[1e38, 1e38, float("nan"), 1e-40, 2.0**-126, -0.0],
dtype=dtype,
device="cuda",
)
.view(6, 1)
.expand(6, 257)
.clone()
)
weights = torch.tensor(
[[1e-40, -1e-40, float("nan")], [1e38, 0.5, 1.0]],
dtype=weight_dtype,
device="cuda",
)
ids = torch.tensor(
[[0, 1, -1], [2, 3, 4]],
dtype=torch.int64 if strided else torch.int32,
device="cuda",
)
if strided:
weights = weights.t().contiguous().t()
ids = ids.t().contiguous().t()
# FP64 exactly represents products of FP32 operands;
# round to FP32 before narrowing to the output dtype.
expected = outputs.cpu().double().view(2, 3, 257)
expected = expected * weights.cpu().double().unsqueeze(-1)
expected[ids.cpu() < 0] = 0
expected = expected.float().to(dtype).view_as(outputs).cuda()
self.assertGreater(float(expected[0, 0]), 1e-3)
self.assertGreater(float(expected[3, 0]), 1e-3)
self.assertNotEqual(float(expected[4, 0]), 0.0)
native(outputs, weights, ids)
torch.testing.assert_close(outputs, expected, rtol=0, atol=0)
self.assertTrue(
torch.equal(outputs.signbit(), expected.signbit())
)
if __name__ == "__main__":
unittest.main()
@@ -286,5 +286,82 @@ def test_moe_align_block_size_pad_fill_stays_in_bounds(
)
@pytest.mark.parametrize("dtype", [torch.int32, torch.int64])
@pytest.mark.parametrize("num_tokens,num_experts", [(65, 4), (512, 128), (65, 1025)])
def test_moe_align_block_size_completes_expert_capacity(dtype, num_tokens, num_experts):
"""Unused expert-map slots must be inactive after every graph replay."""
block_size, topk = 4, 2
original = (
torch.arange(num_tokens * topk, device="cuda", dtype=dtype).reshape(
num_tokens, topk
)
% num_experts
)
routes = original.clone()
capacity = min(
routes.numel() * block_size,
routes.numel() + (num_experts + 1) * (block_size - 1),
)
capacity = ceil_div(capacity, 4) * 4
sorted_ids = torch.empty(capacity, device="cuda", dtype=torch.int32)
blocks = ceil_div(capacity, block_size)
# Guard words make an out-of-bounds capacity fill observable.
expert_storage = torch.full(
(blocks + 4,), num_experts, device="cuda", dtype=torch.int32
)
expert_ids = expert_storage[:blocks]
count = torch.empty(1, device="cuda", dtype=torch.int32)
cumsum = torch.empty(num_experts + 2, device="cuda", dtype=torch.int32)
def align():
moe_align_block_size(
routes,
num_experts + 1,
block_size,
sorted_ids,
expert_ids,
count,
cumsum,
True,
pad_expert_ids=True,
)
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
align()
torch.cuda.current_stream().wait_stream(stream)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
align()
for live in (num_tokens, num_tokens // 2, 0, num_tokens):
routes.copy_(original)
routes[live:].fill_(-1)
sorted_ids.fill_(-99)
expert_storage.fill_(num_experts)
graph.replay()
used = count.item()
assert used % block_size == 0
assert torch.all(expert_ids[used // block_size :] == -1)
assert torch.all(expert_storage[blocks:] == num_experts)
assert torch.all(sorted_ids[used:] == routes.numel())
# Every real route appears exactly once in its expert's contiguous region.
flat_routes = routes.flatten()
expected_start = 0
for expert in torch.unique(flat_routes).tolist():
expected = torch.where(flat_routes == expert)[0]
padded = ceil_div(expected.numel(), block_size) * block_size
end = expected_start + padded
assert torch.all(
expert_ids[expected_start // block_size : end // block_size] == expert
)
actual = sorted_ids[expected_start:end]
actual = actual[actual < routes.numel()].long().sort().values
assert torch.equal(actual, expected)
expected_start = end
assert expected_start == used
if __name__ == "__main__":
sys.exit(pytest.main([__file__]))
@@ -5,6 +5,7 @@ import re
import subprocess
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
@@ -21,6 +22,7 @@ _CPU = PlatformInfo(device_type="cpu")
_SM90 = PlatformInfo(device_type="cuda", cuda_arch_major=9, cuda_arch_minor=0)
_SM100 = PlatformInfo(device_type="cuda", cuda_arch_major=10, cuda_arch_minor=0)
_HIP = PlatformInfo(device_type="hip")
_MUSA = PlatformInfo(device_type="musa")
def test_single_backend_resolves_without_backend():
@@ -40,6 +42,15 @@ def test_fp8_scaled_mm_requires_explicit_registry_backend(monkeypatch):
)
def test_gguf_vector_registry_is_cuda_local_jit():
spec = K.select_kernel("moe.gguf_moe_a8_vec")
assert spec.backend is KernelBackend.JIT
assert spec.target == "sglang.kernels.ops.moe.gguf_moe_vec:gguf_moe_a8_vec"
assert K.capabilities_satisfied(spec.capabilities, _SM90)
assert not K.capabilities_satisfied(spec.capabilities, _MUSA)
assert not K.capabilities_satisfied(spec.capabilities, _CPU)
def test_unknown_op_or_backend_raises():
with pytest.raises(KeyError):
K.select_kernel("does_not.exist")
@@ -55,6 +66,7 @@ def test_unknown_op_or_backend_raises():
(PlatformInfo(device_type="cuda", cuda_arch_major=8, cuda_arch_minor=0), False),
(_CPU, False),
(_HIP, False),
(_MUSA, False),
],
)
def test_qwen_qkv_registry_accepts_hopper(platform, eligible):
@@ -90,6 +102,12 @@ def test_activation_default_backend(monkeypatch, device, expect):
(Cap.CUDA, _SM90, True),
(Cap.CUDA, _HIP, False),
(Cap.HIP, _HIP, True),
(Cap.MUSA, _MUSA, True),
(Cap.MUSA, _CPU, False),
(Cap.MUSA, _SM90, False),
(Cap.MUSA, _HIP, False),
(Cap.CUDA, _MUSA, False),
(Cap.HIP, _MUSA, False),
(Cap.cuda(min_sm=(10, 0)), _SM90, False),
(Cap.cuda(min_sm=(10, 0)), _SM100, True),
(Cap.cuda(max_sm=(9, 0)), _SM100, False),
@@ -99,6 +117,32 @@ def test_capability_is_satisfied_by(req, plat, ok):
assert req.is_satisfied_by(plat) is ok
@pytest.mark.parametrize(
"musa_version, cuda_available, musa_available, expected_device",
[
("4.0", True, None, "musa"),
(None, True, None, "cuda"),
("4.0", False, None, "cpu"),
("4.0", False, True, "musa"),
],
)
def test_platform_detection_distinguishes_musa_cuda_alias(
monkeypatch, musa_version, cuda_available, musa_available, expected_device
):
"""Native MUSA remains eligible before the CUDA compatibility alias is ready."""
torch = SimpleNamespace(
version=SimpleNamespace(hip=None, musa=musa_version),
cuda=SimpleNamespace(
is_available=lambda: cuda_available,
get_device_capability=lambda: (9, 0),
),
)
if musa_available is not None:
torch.musa = SimpleNamespace(is_available=lambda: musa_available)
monkeypatch.setitem(sys.modules, "torch", torch)
assert PlatformInfo.detect().device_type == expected_device
def test_capabilities_or_semantics():
both = {Cap.CUDA, Cap.HIP}
assert K.capabilities_satisfied(both, _SM90)
@@ -99,7 +99,10 @@ class TestGgufInactiveRoutes(CustomTestCase):
# GGUF GEMMs quantize each 32 input values to Q8_1, including an FP16
# scale and input sum. CUDA roundf rounds halfway values away from zero.
blocks = inputs.float().unflatten(-1, (-1, 32))
scale = blocks.abs().amax(dim=-1, keepdim=True) / 127
amax = blocks.abs().amax(dim=-1, keepdim=True)
# A scalar divisor lets PyTorch multiply by its rounded reciprocal,
# changing Q8_1 codes at ties compared with the native FP32 division.
scale = amax / torch.full_like(amax, 127.0)
denominator = torch.where(scale == 0, 1.0, scale)
codes = blocks.sign() * torch.floor(blocks.abs() / denominator + 0.5)
quantized = codes * scale.half().float()
@@ -143,7 +146,7 @@ class TestGgufInactiveRoutes(CustomTestCase):
"""Changing the live count must zero dropped GGUF routes on every replay."""
for quant_type in (GGMLQuantizationType.Q4_0, GGMLQuantizationType.Q8_0):
packed, dense = self._weights(quant_type)
for tokens, partial in ((64, 48), (96, 71)):
for tokens, partial in ((64, 48), (65, 48), (96, 71)):
with self.subTest(quant_type=quant_type, tokens=tokens):
self._check_graph(packed, dense, quant_type, tokens, partial)
@@ -186,20 +189,12 @@ class TestGgufInactiveRoutes(CustomTestCase):
native_outputs = []
native_mmq = gguf_runtime.ggml_moe_a8
native_align = gguf_runtime.moe_align_block_size
def poison_alignment(*args, **kwargs):
sorted_ids, expert_ids, count = native_align(*args, **kwargs)
block_starts = (
torch.arange(expert_ids.numel(), device=expert_ids.device) * 4
)
expert_ids.masked_fill_(block_starts >= count, 4)
return sorted_ids, expert_ids, count
def record_mmq_outputs(*args, **kwargs):
if not torch.cuda.is_current_stream_capturing():
# Fail before native dispatch if a slack block can index expert E.
self.assertTrue(bool((args[3] < args[1].shape[0]).all()))
used_blocks = args[4].item() // 4
self.assertTrue(bool((args[3][used_blocks:] == -1).all()))
result = native_mmq(*args, **kwargs)
if torch.cuda.is_current_stream_capturing():
native_outputs.append(result)
@@ -212,10 +207,7 @@ class TestGgufInactiveRoutes(CustomTestCase):
if tokens > 64
else nullcontext()
)
with (
observe_mmq,
patch.object(gguf_runtime, "moe_align_block_size", poison_alignment),
):
with observe_mmq:
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
@@ -226,8 +218,7 @@ class TestGgufInactiveRoutes(CustomTestCase):
with torch.cuda.graph(graph):
output, ids = run()
self.assertEqual(len(native_outputs), 2 if tokens > 64 else 0)
# Dropped slots must overwrite poisoned native intermediates rather
# than inherit any previous live route or allocator contents.
# Dropped routes must not consume poisoned native intermediates.
for live in (tokens, partial, 0, partial // 2, tokens):
hidden.copy_(base_hidden)
hidden[live:].fill_(float("nan"))
@@ -238,10 +229,6 @@ class TestGgufInactiveRoutes(CustomTestCase):
for intermediate in native_outputs:
intermediate.fill_(float("nan"))
graph.replay()
for intermediate in native_outputs:
self.assertTrue(torch.isfinite(intermediate).all().item())
inactive = intermediate.view(tokens, 2, -1)[live:]
self.assertEqual(torch.count_nonzero(inactive).item(), 0)
self.assertTrue(torch.isfinite(output).all().item())
self.assertTrue((ids[live:] == -1).all().item())
self.assertEqual(torch.count_nonzero(output[live:]).item(), 0)
@@ -254,6 +241,81 @@ class TestGgufInactiveRoutes(CustomTestCase):
output[:live].float(), expected, atol=2e-3, rtol=0.06
)
@torch.inference_mode()
def test_mmq_prefix_preserves_activation_memory_budget(self):
"""Preparing MMQ routes must not duplicate the caller's activation matrix."""
tokens, hidden, intermediate, experts = 8192, 4096, 128, 4
inputs = torch.ones(tokens, hidden, device="cuda", dtype=torch.float16)
ids = torch.zeros(tokens, 2, device="cuda", dtype=torch.int32)
weights = torch.full((tokens, 2), 0.5, device="cuda")
packed_up = torch.empty(
experts,
2 * intermediate,
hidden // 32 * 18,
device="cuda",
dtype=torch.uint8,
)
packed_down = torch.empty(
experts,
hidden,
intermediate // 32 * 18,
device="cuda",
dtype=torch.uint8,
)
class ProjectionReached(Exception):
pass
def stop_before_projection(*args, **kwargs):
raise ProjectionReached
def run_prefix():
with self.assertRaises(ProjectionReached):
fused_moe_gguf(
inputs,
packed_up,
packed_down,
weights,
ids,
GGMLQuantizationType.Q4_0,
GGMLQuantizationType.Q4_0,
"silu",
)
from torch.utils._python_dispatch import TorchDispatchMode
allocations = []
class ObserveAllocations(TorchDispatchMode):
def __torch_dispatch__(self, func, types, args=(), kwargs=None):
result = func(*args, **(kwargs or {}))
input_storage = {
arg.untyped_storage().data_ptr()
for arg in args
if isinstance(arg, torch.Tensor)
}
if (
isinstance(result, torch.Tensor)
and result.numel()
and result.untyped_storage().data_ptr() not in input_storage
):
allocations.append(result.untyped_storage().nbytes())
return result
with patch.object(gguf_runtime, "ggml_moe_a8", stop_before_projection):
run_prefix()
torch.cuda.synchronize()
allocated = torch.cuda.memory_allocated()
torch.cuda.reset_peak_memory_stats()
with ObserveAllocations():
run_prefix()
torch.cuda.synchronize()
peak = torch.cuda.max_memory_allocated() - allocated
# One output and four alignment buffers; no intermediate masks or ranges.
self.assertLessEqual(len(allocations), 5)
budget = inputs.numel() * inputs.element_size() + 128 * 1024
self.assertLessEqual(peak, budget)
@torch.inference_mode()
def test_nonlocal_routes_preserve_local_contributions(self):
"""A nonlocal slot must not erase the same token's valid expert result."""
@@ -308,34 +370,310 @@ class TestGgufInactiveRoutes(CustomTestCase):
output[token].float(), expected, atol=2e-3, rtol=0.06
)
@torch.inference_mode()
def test_cuda_vector_uses_original_ids_without_route_preparation(self):
"""Real CUDA projections consume negative routes without a clamp launch."""
from sgl_kernel.quantization import ggml_moe_a8_vec as legacy_vector
from torch.utils._python_dispatch import TorchDispatchMode
ids = torch.tensor(
[[-1, 1], [0, -3], [-1, -1]], device="cuda", dtype=torch.int32
)
scores = torch.tensor(
[[float("nan"), 0.75], [0.5, float("nan")], [float("nan"), float("nan")]],
device="cuda",
)
hidden = torch.full((3, 256), 0.25, device="cuda", dtype=torch.float16)
hidden[-1].fill_(float("nan"))
raw_ids = ids.clone()
route_ops = []
actual_vector = gguf_runtime.ggml_moe_a8_vec
class ObserveRoutePreparation(TorchDispatchMode):
def __torch_dispatch__(self, func, types, args=(), kwargs=None):
if func in (
torch.ops.aten.clamp_min.default,
torch.ops.aten.clamp_min.out,
torch.ops.aten._to_copy.default,
):
route_ops.append(str(func))
return func(*args, **(kwargs or {}))
def observe_vector(inputs, weights, routes, *args):
self.assertEqual(routes.data_ptr(), ids.data_ptr())
return actual_vector(inputs, weights, routes, *args)
for quant_type in (GGMLQuantizationType.Q4_0, GGMLQuantizationType.Q8_0):
packed, _ = self._weights(quant_type)
with patch.object(gguf_runtime, "ggml_moe_a8_vec", legacy_vector):
expected = fused_moe_gguf(
hidden.nan_to_num(),
*packed,
scores.nan_to_num(),
ids.clamp_min(0),
quant_type,
quant_type,
"silu",
)
route_ops.clear()
with (
ObserveRoutePreparation(),
patch.object(gguf_runtime, "ggml_moe_a8_vec", observe_vector),
):
actual = fused_moe_gguf(
hidden, *packed, scores, ids, quant_type, quant_type, "silu"
)
self.assertEqual(route_ops, [])
self.assertTrue(bool(torch.isfinite(actual).all()))
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
torch.testing.assert_close(ids, raw_ids, rtol=0, atol=0)
@torch.inference_mode()
def test_musa_branch_preserves_input_and_route_budgets(self):
"""MUSA consumers exclude NaNs without input copies or route temporaries.
CUDA execution covers Python dispatch; native MUSA runs in its own CI lane.
"""
from sgl_kernel import gelu_and_mul as legacy_gelu_and_mul
from sgl_kernel import silu_and_mul as legacy_silu_and_mul
from torch.utils._python_dispatch import TorchDispatchMode
from sglang.kernels.ops.moe.gguf_moe_weights_aot import resolve_gguf_moe_weights
from sglang.srt.utils import is_musa
finalizer = resolve_gguf_moe_weights("musa" if is_musa() else "cuda")
finalizers = (None,) if finalizer is None else (None, finalizer)
for finalizer in finalizers:
with self.subTest(native_finalizer=finalizer is not None):
for quant_type in (
GGMLQuantizationType.Q4_0,
GGMLQuantizationType.Q8_0,
):
packed, _ = self._weights(quant_type)
for tokens in (3, 96):
for activation in ("silu", "gelu"):
with self.subTest(
quant_type=quant_type,
tokens=tokens,
activation=activation,
):
repeats = (tokens + 2) // 3
ids = torch.tensor(
[[0, 1], [1, -1], [-1, -1]],
device="cuda",
dtype=torch.int32,
).repeat(repeats, 1)[:tokens]
weights = torch.tensor(
[
[0.25, 0.75],
[0.5, float("nan")],
[float("nan"), float("nan")],
],
device="cuda",
).repeat(repeats, 1)[:tokens]
inactive = ids < 0
inputs = (
torch.linspace(
0.125,
2.0,
tokens,
device="cuda",
dtype=torch.float16,
)
.unsqueeze(1)
.expand(tokens, 256)
.contiguous()
)
inputs[inactive.all(dim=-1)] = float("nan")
provider_name = (
"ggml_moe_a8" if tokens > 64 else "ggml_moe_a8_vec"
)
native = (
gguf_runtime.ggml_moe_a8
if tokens > 64
else gguf_runtime.ggml_moe_a8_vec
)
calls = 0
route_costs = []
class ObserveRouteCosts(TorchDispatchMode):
def __torch_dispatch__(
self, func, types, args=(), kwargs=None
):
if func in (
torch.ops.aten.lt.Scalar,
torch.ops.aten.clamp_min.default,
torch.ops.aten.clamp_min.out,
torch.ops.aten.masked_fill.Scalar,
torch.ops.aten.masked_fill_.Scalar,
torch.ops.aten.arange.default,
):
route_costs.append(str(func))
return func(*args, **(kwargs or {}))
def observe_projection(
projection_input, *args, **kwargs
):
nonlocal calls
if calls == 0:
self.assertEqual(
projection_input.data_ptr(),
inputs.data_ptr(),
)
if finalizer is not None and tokens <= 64:
self.assertIs(args[1], ids)
calls += 1
return native(projection_input, *args, **kwargs)
with (
patch.object(gguf_runtime, "_is_cuda", False),
patch.object(gguf_runtime, "_is_musa", True),
patch.object(
gguf_runtime,
"_musa_apply_gguf_moe_weights",
None,
),
patch.object(
gguf_runtime,
"silu_and_mul",
legacy_silu_and_mul,
),
patch.object(
gguf_runtime,
"gelu_and_mul",
legacy_gelu_and_mul,
),
):
expected = fused_moe_gguf(
inputs.nan_to_num(),
*packed,
weights.masked_fill(inactive, 0),
ids.clamp_min(0),
quant_type,
quant_type,
activation,
)
with (
ObserveRouteCosts(),
patch.object(
gguf_runtime,
provider_name,
observe_projection,
),
patch.object(
gguf_runtime,
"_musa_apply_gguf_moe_weights",
finalizer,
),
):
actual = fused_moe_gguf(
inputs,
*packed,
weights,
ids,
quant_type,
quant_type,
activation,
)
self.assertEqual(calls, 2)
if finalizer is not None:
self.assertEqual(route_costs, [])
self.assertGreater(expected.abs().max().item(), 1e-3)
self.assertTrue(torch.isfinite(actual).all().item())
self.assertEqual(
torch.count_nonzero(
actual[inactive.all(dim=-1)]
).item(),
0,
)
torch.testing.assert_close(
actual, expected, rtol=0, atol=0
)
@torch.inference_mode()
def test_unquantized_fallback_drops_negative_ids(self):
"""The slow path must not alias a dropped route to the last expert."""
"""Dropped routes stay zero without adding a valid-route accumulator."""
from torch.utils._python_dispatch import TorchDispatchMode
generator = torch.Generator(device="cuda").manual_seed(17)
w13 = torch.randn(2, 32, 16, generator=generator, device="cuda").half() / 16
w2 = torch.randn(2, 16, 16, generator=generator, device="cuda").half() / 16
hidden = torch.ones(2, 16, device="cuda", dtype=torch.float16)
hidden[1].fill_(float("nan"))
ids = torch.tensor([[0, -1], [-1, -1]], device="cuda", dtype=torch.int32)
hidden = torch.ones(4, 16, device="cuda", dtype=torch.float16)
hidden[-1].fill_(float("nan"))
ids = torch.tensor(
[[0, -3, -1], [0, 1, 0], [-1, 1, 0], [-1, -3, -1]],
device="cuda",
dtype=torch.int32,
)
weights = torch.tensor(
[[1.0, float("nan")], [float("nan"), float("nan")]], device="cuda"
[
[1.0, float("nan"), float("nan")],
[0.2, 0.3, 0.5],
[float("nan"), 0.75, 0.25],
[float("nan"), float("nan"), float("nan")],
],
device="cuda",
)
output = fused_moe_gguf(
hidden,
w13,
w2,
weights,
ids,
GGMLQuantizationType.F16,
GGMLQuantizationType.F16,
"silu",
original_hidden, original_ids, original_weights = (
hidden.clone(),
ids.clone(),
weights.clone(),
)
projected = hidden[0] @ w13[0].t()
gate, up = projected.chunk(2)
expected = (torch.nn.functional.silu(gate) * up) @ w2[0].t()
observed = {
op: []
for op in (
torch.ops.aten.zeros_like.default,
torch.ops.aten.add_.Tensor,
torch.ops.aten.mul_.Tensor,
torch.ops.aten.zero_.default,
)
}
class ObserveTensorOps(TorchDispatchMode):
def __torch_dispatch__(self, func, types, args=(), kwargs=None):
if func in observed:
observed[func].append(args[0].data_ptr())
return func(*args, **(kwargs or {}))
with ObserveTensorOps():
output = fused_moe_gguf(
hidden,
w13,
w2,
weights,
ids,
GGMLQuantizationType.F16,
GGMLQuantizationType.F16,
"silu",
)
projected = torch.einsum("th,enh->ten", hidden.nan_to_num(), w13)
gate, up = projected.float().chunk(2, dim=-1)
activated = (torch.nn.functional.silu(gate) * up).to(hidden.dtype)
experts = torch.einsum("tek,eok->teo", activated, w2)
selected = experts.gather(
1, ids.clamp_min(0).long().unsqueeze(-1).expand(-1, -1, hidden.shape[1])
)
expected = (
selected.float() * weights.masked_fill(ids < 0, 0).unsqueeze(-1)
).sum(dim=1)
self.assertGreater(expected.abs().max().item(), 1e-3)
self.assertTrue(torch.isfinite(output).all().item())
self.assertEqual(torch.count_nonzero(output[1]).item(), 0)
torch.testing.assert_close(output[0], expected, atol=1e-3, rtol=1e-3)
self.assertEqual(torch.count_nonzero(output[-1]).item(), 0)
torch.testing.assert_close(output.float(), expected, atol=1e-3, rtol=1e-3)
torch.testing.assert_close(hidden, original_hidden, equal_nan=True)
torch.testing.assert_close(ids, original_ids)
torch.testing.assert_close(weights, original_weights, equal_nan=True)
self.assertNotEqual(output.data_ptr(), hidden.data_ptr())
self.assertEqual(observed[torch.ops.aten.zeros_like.default], [])
self.assertEqual(
observed[torch.ops.aten.zero_.default], [output[-1].data_ptr()]
)
weighted_outputs = observed[torch.ops.aten.mul_.Tensor]
self.assertEqual(len(weighted_outputs), 6)
self.assertEqual(
observed[torch.ops.aten.add_.Tensor],
[weighted_outputs[1], weighted_outputs[1], weighted_outputs[4]],
)
if __name__ == "__main__":