From e6451c1a956d6140df13841943bdec68671e506c Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:29:51 -0700 Subject: [PATCH] [Fix] gguf: Preserve MoE routing, weighting and allocation budgets (#45916) --- python/setup.py | 29 +- .../kernels/aot/csrc/common_extension.cc | 3 + .../kernels/aot/csrc/common_extension_musa.cc | 3 + .../kernels/aot/csrc/moe/gguf_moe_weights.cuh | 42 ++ python/sglang/kernels/aot/csrc/moe/moe_sum.cu | 84 ++++ .../aot/csrc/quantization/gguf/dequantize.cuh | 21 + .../aot/csrc/quantization/gguf/gguf_kernel.cu | 60 +-- .../aot/csrc/quantization/gguf/moe.cuh | 21 +- .../quantization/gguf/device_compat.cuh | 17 + .../quantization/gguf/ggml-common.h | 25 +- .../sgl_kernel}/quantization/gguf/moe_vec.cuh | 4 + .../quantization/gguf/quantize_q8_1.cuh | 56 +++ .../sgl_kernel}/quantization/gguf/vecdotq.cuh | 4 + .../kernels/aot/include/sgl_kernel_ops.h | 4 + .../kernels/aot/python/sgl_kernel/__init__.py | 2 + .../kernels/aot/python/sgl_kernel/moe.py | 16 + .../kernels/jit/csrc/moe/gguf_moe_vec.cuh | 130 ++++++ .../kernels/jit/csrc/moe/moe_align_kernel.cu | 53 ++- .../quantization/gguf/device_compat.cuh | 1 + .../quantization/gguf/ggml-common.h | 1 + .../sgl_kernel/quantization/gguf/moe_vec.cuh | 1 + .../quantization/gguf/quantize_q8_1.cuh | 1 + .../sgl_kernel/quantization/gguf/vecdotq.cuh | 1 + python/sglang/kernels/ops/moe/__init__.py | 32 ++ python/sglang/kernels/ops/moe/gguf_moe_vec.py | 85 ++++ .../kernels/ops/moe/gguf_moe_weights.py | 73 +++ .../kernels/ops/moe/gguf_moe_weights_aot.py | 36 ++ python/sglang/kernels/ops/moe/moe_align.py | 9 +- python/sglang/kernels/spec.py | 16 +- .../triton_utils/moe_align_block_size.py | 15 +- python/sglang/srt/layers/quantization/gguf.py | 95 ++-- .../benchmark/moe/bench_gguf_moe_weights.py | 65 +++ .../kernels/ops/moe/test_gguf_moe_weights.py | 235 ++++++++++ .../ops/moe/test_moe_align_block_size.py | 77 ++++ .../unit/kernels/test_kernels_namespace.py | 44 ++ .../quantization/test_gguf_inactive_routes.py | 420 ++++++++++++++++-- 36 files changed, 1592 insertions(+), 189 deletions(-) create mode 100644 python/sglang/kernels/aot/csrc/moe/gguf_moe_weights.cuh create mode 100644 python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/device_compat.cuh rename python/sglang/kernels/aot/{csrc => include/sgl_kernel}/quantization/gguf/ggml-common.h (99%) rename python/sglang/kernels/aot/{csrc => include/sgl_kernel}/quantization/gguf/moe_vec.cuh (99%) create mode 100644 python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/quantize_q8_1.cuh rename python/sglang/kernels/aot/{csrc => include/sgl_kernel}/quantization/gguf/vecdotq.cuh (99%) create mode 100644 python/sglang/kernels/jit/csrc/moe/gguf_moe_vec.cuh create mode 120000 python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/device_compat.cuh create mode 120000 python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/ggml-common.h create mode 120000 python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/moe_vec.cuh create mode 120000 python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/quantize_q8_1.cuh create mode 120000 python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/vecdotq.cuh create mode 100644 python/sglang/kernels/ops/moe/gguf_moe_vec.py create mode 100644 python/sglang/kernels/ops/moe/gguf_moe_weights.py create mode 100644 python/sglang/kernels/ops/moe/gguf_moe_weights_aot.py create mode 100644 test/registered/kernels/benchmark/moe/bench_gguf_moe_weights.py create mode 100644 test/registered/kernels/ops/moe/test_gguf_moe_weights.py diff --git a/python/setup.py b/python/setup.py index d014ac2c84..12b322a858 100644 --- a/python/setup.py +++ b/python/setup.py @@ -30,10 +30,12 @@ import json import os import re import runpy +import shutil import subprocess from pathlib import Path from setuptools import setup +from setuptools.command.sdist import sdist try: from setuptools_rust import Binding, RustExtension, build_rust @@ -214,6 +216,29 @@ def _declared_rust_extensions(): return _pyproject_rust_extensions(_discovered_rust_extensions()) +class Sdist(sdist): + """Materialize shared GGUF headers in the standalone source distribution.""" + + def make_release_tree(self, base_dir, files): + super().make_release_tree(base_dir, files) + relative = Path("sglang/kernels/jit/include/sgl_kernel/quantization/gguf") + canonical = ( + _PYTHON_DIR / "sglang/kernels/aot/include/sgl_kernel/quantization/gguf" + ) + included = {Path(name) for name in files} + for header in sorted((_PYTHON_DIR / relative).glob("*")): + name = header.relative_to(_PYTHON_DIR) + if name not in included or not header.is_symlink(): + continue + target = header.resolve(strict=True) + if target.parent != canonical.resolve(): + raise ValueError(f"Shared GGUF header escapes its owner: {header}") + destination = Path(base_dir) / name + # make_release_tree may hard-link the source symlink itself. + destination.unlink() + shutil.copyfile(target, destination) + + if build_rust is not None: class BuildRust(build_rust): @@ -241,8 +266,8 @@ if build_rust is not None: super().run() setup( - cmdclass={"build_rust": BuildRust}, + cmdclass={"build_rust": BuildRust, "sdist": Sdist}, rust_extensions=_declared_rust_extensions(), ) else: - setup() + setup(cmdclass={"sdist": Sdist}) diff --git a/python/sglang/kernels/aot/csrc/common_extension.cc b/python/sglang/kernels/aot/csrc/common_extension.cc index 49816a619f..300cf4f83f 100644 --- a/python/sglang/kernels/aot/csrc/common_extension.cc +++ b/python/sglang/kernels/aot/csrc/common_extension.cc @@ -155,6 +155,9 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { m.def("moe_sum(Tensor input, Tensor! output) -> ()"); m.impl("moe_sum", torch::kCUDA, &moe_sum); + m.def("apply_gguf_moe_weights(Tensor! outputs, Tensor weights, Tensor ids) -> ()"); + m.impl("apply_gguf_moe_weights", torch::kCUDA, &apply_gguf_moe_weights); + // moe_fused_gate / kimi_k2_moe_fused_gate (AOT) retired: the CUDA gate/topk path // now routes through the unified Triton router // (python/sglang/kernels/ops/moe/moe_fused_gate.py). diff --git a/python/sglang/kernels/aot/csrc/common_extension_musa.cc b/python/sglang/kernels/aot/csrc/common_extension_musa.cc index 098cd46290..5708ebb0f7 100644 --- a/python/sglang/kernels/aot/csrc/common_extension_musa.cc +++ b/python/sglang/kernels/aot/csrc/common_extension_musa.cc @@ -114,6 +114,9 @@ TORCH_LIBRARY_EXPAND(sgl_kernel, m) { m.def("moe_sum(Tensor input, Tensor! output) -> ()"); m.impl("moe_sum", torch::kMUSA, &moe_sum); + m.def("apply_gguf_moe_weights(Tensor! outputs, Tensor weights, Tensor ids) -> ()"); + m.impl("apply_gguf_moe_weights", torch::kMUSA, &apply_gguf_moe_weights); + // moe_fused_gate / kimi_k2_moe_fused_gate (AOT gate kernels) retired: gate/topk // is consolidated onto the unified Triton router (sglang issue #26771). sglang's // MUSA path uses `mate.moe_fused_gate`, so dropping the sgl_kernel MUSA op here diff --git a/python/sglang/kernels/aot/csrc/moe/gguf_moe_weights.cuh b/python/sglang/kernels/aot/csrc/moe/gguf_moe_weights.cuh new file mode 100644 index 0000000000..35d04b7741 --- /dev/null +++ b/python/sglang/kernels/aot/csrc/moe/gguf_moe_weights.cuh @@ -0,0 +1,42 @@ +#pragma once + +#include + +template +__global__ void apply_gguf_moe_weights_kernel( + output_t* __restrict__ outputs, + const weight_t* __restrict__ weights, + const id_t* __restrict__ ids, + const int64_t rows, + const int64_t hidden_size, + const int64_t topk, + const int64_t weight_stride0, + const int64_t weight_stride1, + const int64_t id_stride0, + const int64_t id_stride1) { + for (int64_t row = blockIdx.x; row < rows; row += gridDim.x) { + const int64_t token = row / topk; + const int64_t slot = row % topk; + const int64_t first_column = static_cast(blockIdx.y) * blockDim.x + threadIdx.x; + const int64_t column_step = static_cast(gridDim.y) * blockDim.x; + if (ids[token * id_stride0 + slot * id_stride1] < 0) { + for (int64_t column = first_column; column < hidden_size; column += column_step) { + outputs[row * hidden_size + column] = static_cast(0.0f); + } + continue; + } + const float weight = static_cast(weights[token * weight_stride0 + slot * weight_stride1]); + for (int64_t column = first_column; column < hidden_size; column += column_step) { + const int64_t index = row * hidden_size + column; + const float value = static_cast(outputs[index]); +#ifdef USE_MUSA + const float product = value * weight; +#else + // SM90 enables fast math; preserve subnormal operands and products here. + float product; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(product) : "f"(value), "f"(weight)); +#endif + outputs[index] = static_cast(product); + } + } +} diff --git a/python/sglang/kernels/aot/csrc/moe/moe_sum.cu b/python/sglang/kernels/aot/csrc/moe/moe_sum.cu index af8cacf126..904a24dc7b 100644 --- a/python/sglang/kernels/aot/csrc/moe/moe_sum.cu +++ b/python/sglang/kernels/aot/csrc/moe/moe_sum.cu @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6,6 +7,14 @@ #include #include +#ifdef USE_MUSA +#include + +#include "torch_musa/csrc/core/MUSAGuard.h" +#include "torch_musa/csrc/core/MUSAStream.h" +#endif + +#include "gguf_moe_weights.cuh" #include "utils.h" template @@ -64,3 +73,78 @@ void moe_sum( break; } } + +void apply_gguf_moe_weights(torch::Tensor& outputs, const torch::Tensor& weights, const torch::Tensor& ids) { +#ifdef USE_MUSA + TORCH_CHECK(outputs.device().type() == torch::kMUSA, "outputs must be a MUSA tensor"); +#else + TORCH_CHECK(outputs.is_cuda(), "outputs must be a CUDA tensor"); +#endif + TORCH_CHECK(weights.device() == outputs.device() && ids.device() == outputs.device(), "tensors must share a device"); + TORCH_CHECK( + outputs.layout() == at::kStrided && weights.layout() == at::kStrided && ids.layout() == at::kStrided, + "tensors must have strided layout"); + TORCH_CHECK(outputs.dim() == 2 && outputs.is_contiguous(), "outputs must be contiguous [tokens * topk, hidden_size]"); + TORCH_CHECK(weights.dim() == 2 && ids.dim() == 2, "weights and ids must have shape [tokens, topk]"); + TORCH_CHECK(weights.sizes() == ids.sizes(), "weights and ids shapes must match"); + TORCH_CHECK(outputs.size(0) == weights.numel(), "outputs rows must equal tokens * topk"); + const auto supported_float = [](at::ScalarType dtype) { + return dtype == at::kHalf || dtype == at::kBFloat16 || dtype == at::kFloat; + }; + TORCH_CHECK(supported_float(outputs.scalar_type()), "outputs must be float16, bfloat16, or float32"); + TORCH_CHECK(supported_float(weights.scalar_type()), "weights must be float16, bfloat16, or float32"); + TORCH_CHECK(ids.scalar_type() == at::kInt || ids.scalar_type() == at::kLong, "ids must be int32 or int64"); + TORCH_CHECK( + weights.stride(0) >= 0 && weights.stride(1) >= 0 && ids.stride(0) >= 0 && ids.stride(1) >= 0, + "weights and ids strides must be nonnegative"); + at::assert_no_overlap(outputs, weights); + at::assert_no_overlap(outputs, ids); + const int64_t numel = outputs.numel(); + if (numel == 0) { + return; + } +#ifdef USE_MUSA + const c10::musa::OptionalMUSAGuard device_guard(device_of(outputs)); + const musaStream_t stream = c10::musa::getCurrentMUSAStream(); +#else + const at::cuda::OptionalCUDAGuard device_guard(device_of(outputs)); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); +#endif + constexpr int threads = 256; + // Reuse each route's ID and score across four coalesced column groups. + constexpr int columns_per_block = threads * 4; + const dim3 blocks( + static_cast(std::min(outputs.size(0), 65535)), + static_cast(std::min((outputs.size(1) - 1) / columns_per_block + 1, 65535))); + DISPATCH_FLOAT_TYPES(outputs.scalar_type(), "apply_gguf_moe_weights_output", [&] { + using output_t = scalar_t; + DISPATCH_FLOAT_TYPES(weights.scalar_type(), "apply_gguf_moe_weights_weight", [&] { + using weight_t = scalar_t; + const auto launch = [&](auto* id_data) { + apply_gguf_moe_weights_kernel<<>>( + outputs.data_ptr(), + weights.data_ptr(), + id_data, + outputs.size(0), + outputs.size(1), + weights.size(1), + weights.stride(0), + weights.stride(1), + ids.stride(0), + ids.stride(1)); + }; + if (ids.scalar_type() == at::kInt) { + launch(ids.data_ptr()); + } else { + launch(ids.data_ptr()); + } + }); + }); +#ifdef USE_MUSA + const auto error = musaGetLastError(); + TORCH_CHECK(error == musaSuccess, "apply_gguf_moe_weights launch failed: ", musaGetErrorString(error)); +#else + const auto error = cudaGetLastError(); + TORCH_CHECK(error == cudaSuccess, "apply_gguf_moe_weights launch failed: ", cudaGetErrorString(error)); +#endif +} diff --git a/python/sglang/kernels/aot/csrc/quantization/gguf/dequantize.cuh b/python/sglang/kernels/aot/csrc/quantization/gguf/dequantize.cuh index 57cb1a27cf..6915e2fd71 100644 --- a/python/sglang/kernels/aot/csrc/quantization/gguf/dequantize.cuh +++ b/python/sglang/kernels/aot/csrc/quantization/gguf/dequantize.cuh @@ -1,3 +1,24 @@ +// Utility function + +template +static __device__ __forceinline__ dst_t convert_from_half(half val) { + return val; +} + +template <> +__device__ __forceinline__ c10::BFloat16 convert_from_half(half val) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 || defined(USE_MUSA) + return __float2bfloat16(__half2float(val)); +#else + return __half2float(val); +#endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 || defined(USE_MUSA) +} + +template <> +__device__ __forceinline__ float convert_from_half(half val) { + return __half2float(val); +} + // copied from // https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/dequantize.cuh // copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/convert.cu diff --git a/python/sglang/kernels/aot/csrc/quantization/gguf/gguf_kernel.cu b/python/sglang/kernels/aot/csrc/quantization/gguf/gguf_kernel.cu index 1a4955f517..fd9835b7da 100644 --- a/python/sglang/kernels/aot/csrc/quantization/gguf/gguf_kernel.cu +++ b/python/sglang/kernels/aot/csrc/quantization/gguf/gguf_kernel.cu @@ -9,68 +9,16 @@ // clang-format off #include "utils.h" -#include "ggml-common.h" -#include "vecdotq.cuh" +#include +#include #include "dequantize.cuh" #include "mmvq.cuh" #include "mmq.cuh" #include "moe.cuh" -#include "moe_vec.cuh" +#include +#include // clang-format off -// Q8 gemv -template -static __global__ void -quantize_q8_1(const scalar_t* __restrict__ x, void* __restrict__ vy, const int kx, const int kx_padded) { - const auto ix = blockDim.x * blockIdx.x + threadIdx.x; - if (ix >= kx_padded) { - return; - } - const auto iy = blockDim.y * blockIdx.y + threadIdx.y; - const int i_padded = iy * kx_padded + ix; - - block_q8_1* y = (block_q8_1*)vy; - - const int ib = i_padded / QK8_1; // block index - const int iqs = i_padded % QK8_1; // quant index - - const float xi = ix < kx ? static_cast(x[iy * kx + ix]) : 0.0f; - float amax = fabsf(xi); - float sum = xi; - -#pragma unroll - for (int mask = 16; mask > 0; mask >>= 1) { - amax = fmaxf(amax, SGLANG_SHFL_XOR_SYNC_WIDTH(uint32_t(-1), amax, mask, 32)); - sum += SGLANG_SHFL_XOR_SYNC_WIDTH(uint32_t(-1), sum, mask, 32); - } - - const float d = amax / 127; - const int8_t q = amax == 0.0f ? 0 : roundf(xi / d); - - y[ib].qs[iqs] = q; - - if (iqs > 0) { - return; - } - - y[ib].ds.x = __float2half(d); - y[ib].ds.y = __float2half(sum); -} - -template -static void quantize_row_q8_1_cuda(const scalar_t* x, void* vy, const int kx, const int ky, cudaStream_t stream) { - const int64_t kx_padded = (kx + 512 - 1) / 512 * 512; - const int block_num_x = (kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE; - constexpr int MAX_BLOCK_SIZE = 65535; - for (int off = 0; off < ky; off += MAX_BLOCK_SIZE) { - const int num_blocks_y = std::min(ky, off + MAX_BLOCK_SIZE) - off; - const dim3 num_blocks(block_num_x, num_blocks_y, 1); - const dim3 block_size(CUDA_DEQUANTIZE_BLOCK_SIZE, 1, 1); - quantize_q8_1<<>>( - &x[off * kx], (int32_t*)vy + off * (kx_padded / 32 * 9), kx, kx_padded); - } -} - torch::Tensor ggml_dequantize( torch::Tensor W, // quant weight int64_t type, diff --git a/python/sglang/kernels/aot/csrc/quantization/gguf/moe.cuh b/python/sglang/kernels/aot/csrc/quantization/gguf/moe.cuh index 6b48201c5a..91d273b862 100644 --- a/python/sglang/kernels/aot/csrc/quantization/gguf/moe.cuh +++ b/python/sglang/kernels/aot/csrc/quantization/gguf/moe.cuh @@ -114,18 +114,21 @@ static __device__ __forceinline__ void moe_q( if (threadIdx.x < n_per_r / QK8_1) { const auto kby = threadIdx.x % (WARP_SIZE_GGUF / QI8_1); - const int col_y_eff = token_offs[threadIdx.y] / top_k; const int block_x = ib0 * (qk / QK8_1) + ir * (WARP_SIZE_GGUF / QI8_1) + kby; - if (col_y_eff < ncols_y && block_x < blocks_per_col_y) { - const half2* dsi_src = &y[col_y_eff * blocks_per_col_y + block_x].ds; - half2* dsi_dst = &tile_y_ds[threadIdx.y * (WARP_SIZE_GGUF / QI8_1) + kby]; +#pragma unroll + for (int i = 0; i < mmq_x; i += nwarps) { + const int col_y_eff = token_offs[i / nwarps] / top_k; + if (col_y_eff < ncols_y && block_x < blocks_per_col_y) { + const half2* dsi_src = &y[col_y_eff * blocks_per_col_y + block_x].ds; + half2* dsi_dst = &tile_y_ds[(threadIdx.y + i) * (WARP_SIZE_GGUF / QI8_1) + kby]; - if (need_sum) { - *dsi_dst = *dsi_src; - } else { - float* dfi_dst = (float*)dsi_dst; - *dfi_dst = __low2float(*dsi_src); + if (need_sum) { + *dsi_dst = *dsi_src; + } else { + float* dfi_dst = (float*)dsi_dst; + *dfi_dst = __low2float(*dsi_src); + } } } } diff --git a/python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/device_compat.cuh b/python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/device_compat.cuh new file mode 100644 index 0000000000..e69157b8bb --- /dev/null +++ b/python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/device_compat.cuh @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +#include +#include +#include + +// AOT supplies these definitions for CUDA, ROCm and MUSA before this header. +#ifndef WARP_SIZE +#define WARP_SIZE 32 +#endif +#ifndef SGLANG_SHFL_XOR_SYNC +#define SGLANG_SHFL_XOR_SYNC(mask, var, lane_mask) __shfl_xor_sync((mask), (var), (lane_mask)) +#define SGLANG_SHFL_XOR_SYNC_WIDTH(mask, var, lane_mask, width) __shfl_xor_sync((mask), (var), (lane_mask), (width)) +#endif diff --git a/python/sglang/kernels/aot/csrc/quantization/gguf/ggml-common.h b/python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/ggml-common.h similarity index 99% rename from python/sglang/kernels/aot/csrc/quantization/gguf/ggml-common.h rename to python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/ggml-common.h index 88c21a4ab3..7b825df5dd 100644 --- a/python/sglang/kernels/aot/csrc/quantization/gguf/ggml-common.h +++ b/python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/ggml-common.h @@ -1,3 +1,7 @@ +#pragma once + +#include "device_compat.cuh" + // adapted from // https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/ggml-common.h // copied from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-common.h @@ -955,27 +959,6 @@ typedef float (*vec_dot_q_mul_mat_cuda_t)( const int& j, const int& k); -// Utility function - -template -static __device__ __forceinline__ dst_t convert_from_half(half val) { - return val; -} - -template <> -__device__ __forceinline__ c10::BFloat16 convert_from_half(half val) { -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 || defined(USE_MUSA) - return __float2bfloat16(__half2float(val)); -#else - return __half2float(val); -#endif // defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 || defined(USE_MUSA) -} - -template <> -__device__ __forceinline__ float convert_from_half(half val) { - return __half2float(val); -} - #if defined(USE_ROCM) #ifndef __has_builtin diff --git a/python/sglang/kernels/aot/csrc/quantization/gguf/moe_vec.cuh b/python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/moe_vec.cuh similarity index 99% rename from python/sglang/kernels/aot/csrc/quantization/gguf/moe_vec.cuh rename to python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/moe_vec.cuh index 174736144b..386f5109eb 100644 --- a/python/sglang/kernels/aot/csrc/quantization/gguf/moe_vec.cuh +++ b/python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/moe_vec.cuh @@ -1,3 +1,7 @@ +#pragma once + +#include "vecdotq.cuh" + // copied from // https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/moe_vec.cuh // copied and adapted from diff --git a/python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/quantize_q8_1.cuh b/python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/quantize_q8_1.cuh new file mode 100644 index 0000000000..ecae2f62ff --- /dev/null +++ b/python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/quantize_q8_1.cuh @@ -0,0 +1,56 @@ +#pragma once + +#include "ggml-common.h" + +// Q8 gemv +template +static __global__ void +quantize_q8_1(const scalar_t* __restrict__ x, void* __restrict__ vy, const int kx, const int kx_padded) { + const auto ix = blockDim.x * blockIdx.x + threadIdx.x; + if (ix >= kx_padded) { + return; + } + const auto iy = blockDim.y * blockIdx.y + threadIdx.y; + const int i_padded = iy * kx_padded + ix; + + block_q8_1* y = (block_q8_1*)vy; + + const int ib = i_padded / QK8_1; // block index + const int iqs = i_padded % QK8_1; // quant index + + const float xi = ix < kx ? static_cast(x[iy * kx + ix]) : 0.0f; + float amax = fabsf(xi); + float sum = xi; + +#pragma unroll + for (int mask = 16; mask > 0; mask >>= 1) { + amax = fmaxf(amax, SGLANG_SHFL_XOR_SYNC_WIDTH(uint32_t(-1), amax, mask, 32)); + sum += SGLANG_SHFL_XOR_SYNC_WIDTH(uint32_t(-1), sum, mask, 32); + } + + const float d = amax / 127; + const int8_t q = amax == 0.0f ? 0 : roundf(xi / d); + + y[ib].qs[iqs] = q; + + if (iqs > 0) { + return; + } + + y[ib].ds.x = __float2half(d); + y[ib].ds.y = __float2half(sum); +} + +template +static void quantize_row_q8_1_cuda(const scalar_t* x, void* vy, const int kx, const int ky, cudaStream_t stream) { + const int64_t kx_padded = (kx + 512 - 1) / 512 * 512; + const int block_num_x = (kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE; + constexpr int MAX_BLOCK_SIZE = 65535; + for (int off = 0; off < ky; off += MAX_BLOCK_SIZE) { + const int num_blocks_y = std::min(ky, off + MAX_BLOCK_SIZE) - off; + const dim3 num_blocks(block_num_x, num_blocks_y, 1); + const dim3 block_size(CUDA_DEQUANTIZE_BLOCK_SIZE, 1, 1); + quantize_q8_1<<>>( + &x[off * kx], (int32_t*)vy + off * (kx_padded / 32 * 9), kx, kx_padded); + } +} diff --git a/python/sglang/kernels/aot/csrc/quantization/gguf/vecdotq.cuh b/python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/vecdotq.cuh similarity index 99% rename from python/sglang/kernels/aot/csrc/quantization/gguf/vecdotq.cuh rename to python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/vecdotq.cuh index 08b4cd269c..19e990d4da 100644 --- a/python/sglang/kernels/aot/csrc/quantization/gguf/vecdotq.cuh +++ b/python/sglang/kernels/aot/include/sgl_kernel/quantization/gguf/vecdotq.cuh @@ -1,3 +1,7 @@ +#pragma once + +#include "ggml-common.h" + // copied from // https://github.com/vllm-project/vllm/blob/4492e3a55428e161ca8db381edc28263e5da4c8d/csrc/quantization/gguf/vecdotq.cuh // copied and adapted from https://github.com/ggerganov/llama.cpp/blob/b2899/ggml-cuda/vecdotq.cuh diff --git a/python/sglang/kernels/aot/include/sgl_kernel_ops.h b/python/sglang/kernels/aot/include/sgl_kernel_ops.h index 0a2ce7a3b0..f9d2f29e89 100644 --- a/python/sglang/kernels/aot/include/sgl_kernel_ops.h +++ b/python/sglang/kernels/aot/include/sgl_kernel_ops.h @@ -258,6 +258,10 @@ void moe_sum_reduce(at::Tensor& input, at::Tensor& output, double routed_scaling void moe_sum(torch::Tensor& input, torch::Tensor& output); +/// \brief Weight contiguous expert rows in place, writing zero for negative expert IDs. +/// The paired GGUF projections accept negative IDs and bound padding metadata reads. +void apply_gguf_moe_weights(torch::Tensor& outputs, const torch::Tensor& weights, const torch::Tensor& ids); + void fp8_blockwise_scaled_grouped_mm( torch::Tensor& output, torch::Tensor& a_ptrs, diff --git a/python/sglang/kernels/aot/python/sgl_kernel/__init__.py b/python/sglang/kernels/aot/python/sgl_kernel/__init__.py index 17c7f1bad2..543968535f 100644 --- a/python/sglang/kernels/aot/python/sgl_kernel/__init__.py +++ b/python/sglang/kernels/aot/python/sgl_kernel/__init__.py @@ -77,6 +77,7 @@ else: ) from sgl_kernel.memory import weak_ref_tensor from sgl_kernel.moe import ( + apply_gguf_moe_weights, apply_shuffle_mul_sum, fp8_blockwise_scaled_grouped_mm, fused_qk_norm_rope, @@ -145,6 +146,7 @@ else: ) _DEBUG_EXPORT_NAMES = [ + "apply_gguf_moe_weights", "apply_shuffle_mul_sum", "apply_token_bitmask_inplace_cuda", "build_tree_kernel_efficient", diff --git a/python/sglang/kernels/aot/python/sgl_kernel/moe.py b/python/sglang/kernels/aot/python/sgl_kernel/moe.py index c01a0ac374..1b9dbabf32 100755 --- a/python/sglang/kernels/aot/python/sgl_kernel/moe.py +++ b/python/sglang/kernels/aot/python/sgl_kernel/moe.py @@ -94,6 +94,22 @@ def moe_sum_reduce( ) +def apply_gguf_moe_weights( + outputs: torch.Tensor, + weights: torch.Tensor, + ids: torch.Tensor, +) -> None: + """Weight contiguous [tokens * topk, hidden_size] expert outputs in place. + + Weights and IDs are strided [tokens, topk] tensors on the same device. + Outputs and weights independently support float16, bfloat16, and float32; + IDs support int32 or int64. Negative IDs write zero without reading the + corresponding output or weight. Valid products round to the output dtype. + Outputs must not overlap weights or IDs. + """ + torch.ops.sgl_kernel.apply_gguf_moe_weights.default(outputs, weights, ids) + + def moe_sum( input_tensor: torch.Tensor, output_tensor: torch.Tensor, diff --git a/python/sglang/kernels/jit/csrc/moe/gguf_moe_vec.cuh b/python/sglang/kernels/jit/csrc/moe/gguf_moe_vec.cuh new file mode 100644 index 0000000000..ac08fa12d8 --- /dev/null +++ b/python/sglang/kernels/jit/csrc/moe/gguf_moe_vec.cuh @@ -0,0 +1,130 @@ +#pragma once + +#include +#include + +#include + +#include +#include +#include + +#include +#include + +namespace sglang { +namespace gguf { + +template +struct MoeVecFormat; + +#define GGUF_MOE_VEC_FORMAT(type, qk, block, launcher) \ + template <> \ + struct MoeVecFormat { \ + static constexpr int kBlockSize = qk; \ + static constexpr int kBlockBytes = sizeof(block); \ + static constexpr int kAlignment = alignof(block); \ + template \ + static void \ + run(const void* w, \ + const void* q, \ + scalar_t* y, \ + const int* ids, \ + int topk, \ + int m, \ + int k, \ + int n, \ + int stride, \ + cudaStream_t stream) { \ + launcher(w, q, y, ids, topk, m, k, n, stride, stream); \ + } \ + } + +GGUF_MOE_VEC_FORMAT(2, QK4_0, block_q4_0, moe_vec_q4_0_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(3, QK4_0, block_q4_1, moe_vec_q4_1_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(6, QK5_0, block_q5_0, moe_vec_q5_0_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(7, QK5_1, block_q5_1, moe_vec_q5_1_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(8, QK8_0, block_q8_0, moe_vec_q8_0_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(10, QK_K, block_q2_K, moe_vec_q2_K_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(11, QK_K, block_q3_K, moe_vec_q3_K_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(12, QK_K, block_q4_K, moe_vec_q4_K_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(13, QK_K, block_q5_K, moe_vec_q5_K_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(14, QK_K, block_q6_K, moe_vec_q6_K_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(16, QK_K, block_iq2_xxs, moe_vec_iq2_xxs_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(17, QK_K, block_iq2_xs, moe_vec_iq2_xs_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(18, QK_K, block_iq3_xxs, moe_vec_iq3_xxs_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(19, QK_K, block_iq1_s, moe_vec_iq1_s_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(20, QK4_NL, block_iq4_nl, moe_vec_iq4_nl_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(21, QK_K, block_iq3_s, moe_vec_iq3_s_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(22, QK_K, block_iq2_s, moe_vec_iq2_s_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(23, QK_K, block_iq4_xs, moe_vec_iq4_xs_q8_1_cuda); +GGUF_MOE_VEC_FORMAT(29, QK_K, block_iq1_m, moe_vec_iq1_m_q8_1_cuda); + +#undef GGUF_MOE_VEC_FORMAT + +} // namespace gguf + +/// \brief Quantize input rows to Q8_1 and project compact GGUF expert routes. +/// \param expert_ids Contiguous int32 routes; negative IDs write zero without reading operands. +/// \param output Zero-initialized private output, matching the AOT vector projection lifetime. +/// \param quantized_input Private Q8_1 scratch, with the same padded-row format as the AOT ABI. +template +void gguf_moe_a8_vec( + tvm::ffi::TensorView input, + tvm::ffi::TensorView weight, + tvm::ffi::TensorView expert_ids, + tvm::ffi::TensorView output, + tvm::ffi::TensorView quantized_input, + int64_t top_k, + int64_t row, + int64_t tokens) { + using namespace host; + using Format = gguf::MoeVecFormat; + constexpr int64_t kMaxIndex = std::numeric_limits::max(); + CHECK_HOST(tokens >= 0 && tokens <= kMaxIndex && top_k > 0 && top_k <= kMaxIndex); + CHECK_HOST(tokens <= 65535 / top_k) << "GGUF vector routes exceed the CUDA z-grid limit"; + CHECK_HOST(row > 0 && row <= kMaxIndex); + + auto device = SymbolicDevice{}; + auto columns = SymbolicSize{"columns"}; + auto experts = SymbolicSize{"experts"}; + TensorMatcher({tokens, columns}).with_dtype().template with_device(device).verify(input); + const int64_t k = columns.unwrap(); + CHECK_HOST(k > 0 && k <= kMaxIndex - 511 && k % Format::kBlockSize == 0); + const int64_t packed_columns = k / Format::kBlockSize * Format::kBlockBytes; + TensorMatcher({experts, row, packed_columns}) + .with_dtype() + .with_device(device) + .ensure_alignment(std::max(2, Format::kAlignment)) + .verify(weight); + CHECK_HOST(experts.unwrap() > 0 && experts.unwrap() <= kMaxIndex / row / (k / Format::kBlockSize)); + TensorMatcher({-1, -1}).with_dtype().with_device(device).verify(expert_ids); + CHECK_HOST(expert_ids.numel() == tokens * top_k); + TensorMatcher({tokens * top_k, row}).with_dtype().with_device(device).verify(output); + const int64_t padded = (k + 511) / 512 * 512; + TensorMatcher({tokens, padded / 32 * 9}).with_dtype().with_device(device).verify(quantized_input); + CHECK_HOST(tokens <= kMaxIndex / padded && tokens * top_k <= kMaxIndex / row); + if (tokens == 0) return; + + const auto stream = LaunchKernel::resolve_device(device.unwrap()); + quantize_row_q8_1_cuda( + static_cast(input.data_ptr()), + quantized_input.data_ptr(), + static_cast(k), + static_cast(tokens), + stream); + Format::template run( + weight.data_ptr(), + quantized_input.data_ptr(), + static_cast(output.data_ptr()), + static_cast(expert_ids.data_ptr()), + static_cast(top_k), + static_cast(tokens), + static_cast(k), + static_cast(row), + static_cast(padded / 32 * 9), + stream); + CHECK_CUDA(cudaGetLastError()); +} + +} // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/moe/moe_align_kernel.cu b/python/sglang/kernels/jit/csrc/moe/moe_align_kernel.cu index 85be81df64..ec72b4a4fe 100644 --- a/python/sglang/kernels/jit/csrc/moe/moe_align_kernel.cu +++ b/python/sglang/kernels/jit/csrc/moe/moe_align_kernel.cu @@ -53,7 +53,7 @@ __global__ void count_and_sort_expert_tokens_kernel( } } -template +template __global__ void moe_align_block_size_kernel( const scalar_t* __restrict__ topk_ids, int32_t* __restrict__ sorted_token_ids, @@ -65,7 +65,8 @@ __global__ void moe_align_block_size_kernel( int32_t* __restrict__ cumsum, bool pad_sorted_token_ids, const int32_t scan_size, - int32_t max_num_tokens_padded) { + int32_t max_num_tokens_padded, + int32_t max_num_m_blocks) { // Use a separate thread block to populate sorted_token_ids if (blockIdx.x == 1) { if (pad_sorted_token_ids) { @@ -210,11 +211,9 @@ __global__ void moe_align_block_size_kernel( __syncthreads(); int offset = warp_sums[warp_id]; - if (tid < scan_size) scan_buf[tid] = pre + offset; + if (tid < num_experts) prefix[tid] = pre + offset; + // Publish the prefix before other threads binary-search it below. __syncthreads(); - - // Write prefix[0..num_experts - 1] and cumsum - if (tid < num_experts) prefix[tid] = scan_buf[tid]; #endif if (tid <= num_experts) { @@ -235,9 +234,14 @@ __global__ void moe_align_block_size_kernel( } expert_ids[i] = left - 2; } + if constexpr (kPadExpertIds) { + for (int32_t i = num_blocks + tid; i < max_num_m_blocks; i += stride) { + expert_ids[i] = -1; + } + } } -template +template __global__ void moe_align_block_size_small_batch_expert_kernel( const scalar_t* __restrict__ topk_ids, int32_t* __restrict__ sorted_token_ids, @@ -247,7 +251,8 @@ __global__ void moe_align_block_size_small_batch_expert_kernel( int32_t block_size, size_t numel, bool pad_sorted_token_ids, - int32_t max_num_tokens_padded) { + int32_t max_num_tokens_padded, + int32_t max_num_m_blocks) { // Adapted from // https://github.com/vllm-project/vllm/pull/29642/files#diff-5647b1413f4ae9aacba904eca8f8a8aee9079321eadff4c10101a2c6962dcc53R226 // Use an additional group of threads to fill sorted_token_ids. @@ -310,6 +315,11 @@ __global__ void moe_align_block_size_small_batch_expert_kernel( expert_ids[i / block_size] = tid - 1; } } + if constexpr (kPadExpertIds) { + for (int32_t i = cumsum[num_experts] / block_size + tid; i < max_num_m_blocks; i += stride) { + expert_ids[i] = -1; + } + } for (size_t i = tid; i < numel; i += stride) { int32_t expert_id = topk_ids[i] + 1; @@ -328,7 +338,7 @@ __global__ void moe_align_block_size_small_batch_expert_kernel( // With 1024 threads and EXPERTS_PER_THREAD=4, covers at most 4096 expert // indices. Since num_experts includes the +1 offset bucket, this supports // up to 4095 real experts. -template +template __global__ void moe_align_block_size_kernel_v2( const scalar_t* __restrict__ topk_ids, int32_t* __restrict__ sorted_token_ids, @@ -340,7 +350,8 @@ __global__ void moe_align_block_size_kernel_v2( size_t numel, int32_t* __restrict__ cumsum, bool pad_sorted_token_ids, - int32_t max_num_tokens_padded) { + int32_t max_num_tokens_padded, + int32_t max_num_m_blocks) { // Use a separate thread block to populate sorted_token_ids if (blockIdx.x == 1) { if (pad_sorted_token_ids) { @@ -442,11 +453,16 @@ __global__ void moe_align_block_size_kernel_v2( } } } + if constexpr (kPadExpertIds) { + for (int32_t i = cumsum[num_experts] / block_size + tid; i < max_num_m_blocks; i += blockDim.x) { + expert_ids[i] = -1; + } + } } } // namespace moe -template +template struct MoeAlignBlockSizeKernel { static void run(tvm::ffi::TensorView topk_ids, @@ -487,7 +503,7 @@ struct MoeAlignBlockSizeKernel { constexpr int32_t fill_threads = 256; const int32_t shared_mem_size = ((num_thread + 1) * num_experts + (num_experts + 1)) * sizeof(int32_t); - auto kernel = moe::moe_align_block_size_small_batch_expert_kernel; + auto kernel = moe::moe_align_block_size_small_batch_expert_kernel; LaunchKernel(dim3(1), dim3(fill_threads + num_thread), stream, shared_mem_size)( kernel, topk_ids_ptr, @@ -498,12 +514,13 @@ struct MoeAlignBlockSizeKernel { (int32_t)block_size, numel, pad_sorted_token_ids, - (int32_t)max_num_tokens_padded); + (int32_t)max_num_tokens_padded, + static_cast(expert_ids.numel())); } else if (num_experts <= 1024) { const size_t scan_size = host::round_up_pow2(static_cast(num_experts)); const size_t shared_mem_size = (num_experts + (num_experts + 1) + scan_size + WARP_SIZE) * sizeof(int32_t); - auto align_kernel = moe::moe_align_block_size_kernel; + auto align_kernel = moe::moe_align_block_size_kernel; LaunchKernel(dim3(2), dim3(threads), stream, shared_mem_size)( align_kernel, topk_ids_ptr, @@ -516,7 +533,8 @@ struct MoeAlignBlockSizeKernel { cumsum_buffer_ptr, pad_sorted_token_ids, (int32_t)scan_size, - (int32_t)max_num_tokens_padded); + (int32_t)max_num_tokens_padded, + static_cast(expert_ids.numel())); const int block_threads = std::min(256, threads); const int num_blocks = (numel + block_threads - 1) / block_threads; @@ -533,7 +551,7 @@ struct MoeAlignBlockSizeKernel { auto launch_v2 = [&](auto ept_tag) { constexpr int EPT = decltype(ept_tag)::value; - auto v2_kernel = moe::moe_align_block_size_kernel_v2; + auto v2_kernel = moe::moe_align_block_size_kernel_v2; LaunchKernel(dim3(2), dim3(threads), stream, shared_mem_size)( v2_kernel, topk_ids_ptr, @@ -546,7 +564,8 @@ struct MoeAlignBlockSizeKernel { numel, cumsum_buffer_ptr, pad_sorted_token_ids, - (int32_t)max_num_tokens_padded); + (int32_t)max_num_tokens_padded, + static_cast(expert_ids.numel())); }; if (padded_num_experts <= 2048) { diff --git a/python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/device_compat.cuh b/python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/device_compat.cuh new file mode 120000 index 0000000000..b6984ccdc0 --- /dev/null +++ b/python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/device_compat.cuh @@ -0,0 +1 @@ +../../../../../aot/include/sgl_kernel/quantization/gguf/device_compat.cuh \ No newline at end of file diff --git a/python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/ggml-common.h b/python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/ggml-common.h new file mode 120000 index 0000000000..f67784a18d --- /dev/null +++ b/python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/ggml-common.h @@ -0,0 +1 @@ +../../../../../aot/include/sgl_kernel/quantization/gguf/ggml-common.h \ No newline at end of file diff --git a/python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/moe_vec.cuh b/python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/moe_vec.cuh new file mode 120000 index 0000000000..7bdbdde591 --- /dev/null +++ b/python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/moe_vec.cuh @@ -0,0 +1 @@ +../../../../../aot/include/sgl_kernel/quantization/gguf/moe_vec.cuh \ No newline at end of file diff --git a/python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/quantize_q8_1.cuh b/python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/quantize_q8_1.cuh new file mode 120000 index 0000000000..57b6bb69eb --- /dev/null +++ b/python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/quantize_q8_1.cuh @@ -0,0 +1 @@ +../../../../../aot/include/sgl_kernel/quantization/gguf/quantize_q8_1.cuh \ No newline at end of file diff --git a/python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/vecdotq.cuh b/python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/vecdotq.cuh new file mode 120000 index 0000000000..4a02775ee9 --- /dev/null +++ b/python/sglang/kernels/jit/include/sgl_kernel/quantization/gguf/vecdotq.cuh @@ -0,0 +1 @@ +../../../../../aot/include/sgl_kernel/quantization/gguf/vecdotq.cuh \ No newline at end of file diff --git a/python/sglang/kernels/ops/moe/__init__.py b/python/sglang/kernels/ops/moe/__init__.py index 173b817c61..b3a4f6672c 100644 --- a/python/sglang/kernels/ops/moe/__init__.py +++ b/python/sglang/kernels/ops/moe/__init__.py @@ -81,6 +81,38 @@ register_kernel( ) +register_kernel( + KernelSpec( + op="moe.gguf_moe_a8_vec", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.moe.gguf_moe_vec:gguf_moe_a8_vec", + capabilities=CUDA_ONLY, + ) +) + + +register_kernel( + KernelSpec( + op="moe.apply_gguf_moe_weights", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.moe.gguf_moe_weights:apply_gguf_moe_weights", + capabilities=CUDA_ONLY, + ) +) + + +register_kernel( + KernelSpec( + op="moe.apply_gguf_moe_weights", + backend=KernelBackend.AOT, + target="sglang.kernels.ops.moe.gguf_moe_weights_aot:apply_gguf_moe_weights", + capabilities=frozenset( + {CapabilityRequirement.CUDA, CapabilityRequirement.MUSA} + ), + ) +) + + def moe_align_block_size( topk_ids: torch.Tensor, num_experts: int, diff --git a/python/sglang/kernels/ops/moe/gguf_moe_vec.py b/python/sglang/kernels/ops/moe/gguf_moe_vec.py new file mode 100644 index 0000000000..65e31a0746 --- /dev/null +++ b/python/sglang/kernels/ops/moe/gguf_moe_vec.py @@ -0,0 +1,85 @@ +"""GGUF vector expert projections with consumer-owned inactive routes.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from sglang.kernels.jit.utils import ( + cache_once, + get_jit_cuda_arch, + load_jit, + make_cpp_args, +) +from sglang.kernels.kernel_api_logging import debug_kernel_api + +if TYPE_CHECKING: + from tvm_ffi.module import Module + + +@cache_once +def _jit_gguf_moe_vec_module(dtype: torch.dtype, quant_type: int) -> Module: + if dtype not in (torch.float16, torch.bfloat16, torch.float32): + raise ValueError(f"Unsupported GGUF input dtype: {dtype}") + if quant_type not in ( + 2, + 3, + 6, + 7, + 8, + 10, + 11, + 12, + 13, + 14, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 29, + ): + raise ValueError(f"Unsupported GGUF vector quantization type: {quant_type}") + args = make_cpp_args(dtype, quant_type) + arch = get_jit_cuda_arch() + # Match the AOT common_ops SM90 fast-math build; its other build is precise. + flags = ["--use_fast_math"] if (arch.major, arch.minor) == (9, 0) else [] + return load_jit( + "gguf_moe_vec", + *args, + cuda_files=["moe/gguf_moe_vec.cuh"], + cuda_wrappers=[("project", f"gguf_moe_a8_vec<{args}>")], + extra_cuda_cflags=flags, + ) + + +@debug_kernel_api +def gguf_moe_a8_vec( + input: torch.Tensor, + weight: torch.Tensor, + topk_ids: torch.Tensor, + top_k: int, + type: int, + row: int, + tokens: int, +) -> torch.Tensor: + """Project packed GGUF experts, writing zero for every negative route. + + Inputs and uint8 packed weights are contiguous. IDs are contiguous int32 + local expert indices or negative sentinels; their total size is tokens * top_k. + The down projection retains the original route shape with top_k=1. + Quantization, vector reduction and output rounding match the AOT operation. + """ + output = torch.zeros((tokens * top_k, row), dtype=input.dtype, device=input.device) + padded = (input.shape[1] + 511) // 512 * 512 + quantized_input = torch.empty( + (tokens, padded // 32 * 9), dtype=torch.int32, device=input.device + ) + _jit_gguf_moe_vec_module(input.dtype, int(type)).project( + input, weight, topk_ids, output, quantized_input, top_k, row, tokens + ) + return output diff --git a/python/sglang/kernels/ops/moe/gguf_moe_weights.py b/python/sglang/kernels/ops/moe/gguf_moe_weights.py new file mode 100644 index 0000000000..08015600bd --- /dev/null +++ b/python/sglang/kernels/ops/moe/gguf_moe_weights.py @@ -0,0 +1,73 @@ +"""Apply GGUF route weights without reading inactive expert outputs.""" + +import torch +import triton +import triton.language as tl + + +@triton.jit( + do_not_specialize=["weight_stride0", "weight_stride1", "id_stride0", "id_stride1"] +) +def _apply_gguf_moe_weights_kernel( + outputs, + weights, + expert_ids, + width: tl.constexpr, + topk: tl.constexpr, + weight_stride0, + weight_stride1, + id_stride0, + id_stride1, + BLOCK: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + token = row // topk + slot = row % topk + active = tl.load(expert_ids + token * id_stride0 + slot * id_stride1) >= 0 + columns = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK) + values = tl.load( + outputs + row * width + columns, + mask=active & (columns < width), + other=0, + ).to(tl.float32) + factor = tl.load( + weights + token * weight_stride0 + slot * weight_stride1, + mask=active, + other=0, + ).to(tl.float32) + tl.store(outputs + row * width + columns, values * factor, mask=columns < width) + + +def apply_gguf_moe_weights( + outputs: torch.Tensor, weights: torch.Tensor, expert_ids: torch.Tensor +) -> None: + """Weight contiguous GGUF rows in place, zeroing routes with negative IDs. + + ``outputs`` is ``[tokens * topk, hidden]`` and routing tensors are + ``[tokens, topk]``. Products round to the output dtype before the caller's + native ``moe_sum``; inactive rows read neither outputs nor weights. + """ + assert outputs.ndim == weights.ndim == expert_ids.ndim == 2 + assert outputs.is_contiguous() + assert weights.shape == expert_ids.shape + assert outputs.shape[0] == expert_ids.numel() + assert outputs.device == weights.device == expert_ids.device + assert outputs.dtype in (torch.float16, torch.bfloat16, torch.float32) + assert weights.dtype in (torch.float16, torch.bfloat16, torch.float32) + assert expert_ids.dtype in (torch.int32, torch.int64) + if outputs.numel() == 0: + return + width = outputs.shape[1] + block = min(1024, triton.next_power_of_2(width)) + _apply_gguf_moe_weights_kernel[(outputs.shape[0], triton.cdiv(width, block))]( + outputs, + weights, + expert_ids, + width, + weights.shape[1], + weights.stride(0), + weights.stride(1), + expert_ids.stride(0), + expert_ids.stride(1), + BLOCK=block, + ) diff --git a/python/sglang/kernels/ops/moe/gguf_moe_weights_aot.py b/python/sglang/kernels/ops/moe/gguf_moe_weights_aot.py new file mode 100644 index 0000000000..653903913b --- /dev/null +++ b/python/sglang/kernels/ops/moe/gguf_moe_weights_aot.py @@ -0,0 +1,36 @@ +"""Native GGUF route weighting for CUDA and MUSA builds.""" + +from typing import Callable, Optional + +import torch + + +def apply_gguf_moe_weights( + outputs: torch.Tensor, weights: torch.Tensor, expert_ids: torch.Tensor +) -> None: + """Weight rows in place, rounding each product before native MoE reduction.""" + import sgl_kernel # noqa: F401 + + torch.ops.sgl_kernel.apply_gguf_moe_weights.default(outputs, weights, expert_ids) + + +def resolve_gguf_moe_weights(device_type: str) -> Optional[Callable]: + """Resolve once at backend import, retaining compatibility with older wheels. + + This native operator ships with negative-ID-safe GGUF vector projections, + early MMQ slack bounds, and matching Q8_1 value/scale row indexing. Its + device registration, rather than the Python wrapper's presence, establishes + that the rebuilt provider can consume raw routes without Python clearing. + """ + import sgl_kernel # noqa: F401 + + try: + torch.ops.sgl_kernel.apply_gguf_moe_weights.default + except AttributeError: + return None + key = torch._C._dispatch_key_for_device(device_type) + if not torch._C._dispatch_has_kernel_for_dispatch_key( + "sgl_kernel::apply_gguf_moe_weights", key + ): + return None + return apply_gguf_moe_weights diff --git a/python/sglang/kernels/ops/moe/moe_align.py b/python/sglang/kernels/ops/moe/moe_align.py index 0b071056aa..9dd8460151 100644 --- a/python/sglang/kernels/ops/moe/moe_align.py +++ b/python/sglang/kernels/ops/moe/moe_align.py @@ -11,8 +11,8 @@ if TYPE_CHECKING: @cache_once -def _jit_moe_align_module(dtype: torch.dtype) -> Module: - args = make_cpp_args(dtype) +def _jit_moe_align_module(dtype: torch.dtype, pad_expert_ids: bool) -> Module: + args = make_cpp_args(dtype, pad_expert_ids) return load_jit( "moe_align_block_size", *args, @@ -32,8 +32,11 @@ def moe_align_block_size( num_tokens_post_pad: torch.Tensor, cumsum_buffer: torch.Tensor, pad_sorted_token_ids: bool = False, + *, + pad_expert_ids: bool = False, ) -> None: - module = _jit_moe_align_module(topk_ids.dtype) + """Align routes; optionally define unused expert-map capacity as -1.""" + module = _jit_moe_align_module(topk_ids.dtype, pad_expert_ids) module.moe_align_block_size( topk_ids, num_experts, diff --git a/python/sglang/kernels/spec.py b/python/sglang/kernels/spec.py index 90e6c6c180..6aea153b4a 100644 --- a/python/sglang/kernels/spec.py +++ b/python/sglang/kernels/spec.py @@ -59,9 +59,10 @@ class DeviceType(str, Enum): CUDA = "cuda" HIP = "hip" + MUSA = "musa" NPU = "npu" # Ascend NPU (torch_npu / sgl_kernel_npu) CPU = "cpu" - # TODO(RFC #29630): XPU / MUSA / ... as backends land. + # TODO(RFC #29630): XPU / ... as backends land. class PlatformInfo(msgspec.Struct, frozen=True): @@ -101,6 +102,14 @@ class PlatformInfo(msgspec.Struct, frozen=True): npu = getattr(torch, "npu", None) if npu is not None and npu.is_available(): return cls(device_type="npu") + musa = getattr(torch, "musa", None) + if musa is not None and musa.is_available(): + return cls(device_type="musa") + if ( + getattr(torch.version, "musa", None) is not None + and torch.cuda.is_available() + ): + return cls(device_type="musa") if torch.cuda.is_available(): major, minor = torch.cuda.get_device_capability() return cls( @@ -126,7 +135,8 @@ class CapabilityRequirement(msgspec.Struct, frozen=True): (``min_cuda_arch`` / ``max_cuda_arch`` apply only when ``device == CUDA``). The device-only cases are so common that they are exposed as class constants - (``CapabilityRequirement.CUDA`` / ``.HIP`` / ``.NPU``); use :meth:`cuda` for an + (``CapabilityRequirement.CUDA`` / ``.HIP`` / ``.MUSA`` / ``.NPU``); + use :meth:`cuda` for an arch-bounded CUDA requirement (e.g. ``CapabilityRequirement.cuda( min_sm=(10, 0))`` for SM100+). """ @@ -139,6 +149,7 @@ class CapabilityRequirement(msgspec.Struct, frozen=True): # instances of the class itself). ClassVar keeps them out of msgspec fields. CUDA: ClassVar[CapabilityRequirement] HIP: ClassVar[CapabilityRequirement] + MUSA: ClassVar[CapabilityRequirement] NPU: ClassVar[CapabilityRequirement] @classmethod @@ -164,6 +175,7 @@ class CapabilityRequirement(msgspec.Struct, frozen=True): CapabilityRequirement.CUDA = CapabilityRequirement(device=DeviceType.CUDA) CapabilityRequirement.HIP = CapabilityRequirement(device=DeviceType.HIP) +CapabilityRequirement.MUSA = CapabilityRequirement(device=DeviceType.MUSA) CapabilityRequirement.NPU = CapabilityRequirement(device=DeviceType.NPU) CUDA_ONLY: FrozenSet[CapabilityRequirement] = frozenset({CapabilityRequirement.CUDA}) diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py b/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py index 6ad8973393..0b63a3a909 100644 --- a/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/moe_align_block_size.py @@ -34,6 +34,8 @@ def moe_align_block_size( block_size: int, num_experts: int, ignore_invalid_expert: bool = False, + *, + pad_expert_ids: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """ Aligns the token distribution across experts to be compatible with block @@ -72,8 +74,13 @@ def moe_align_block_size( - The padding ensures that the total number of tokens is now divisible by block_size for proper block matrix operations. """ + if pad_expert_ids and (not _is_cuda or ignore_invalid_expert): + raise ValueError( + "Padding expert IDs requires CUDA and the invalid-expert bucket" + ) + # ===== TO BE REFACTORED ==== - if _SGLANG_EXPERIMENTAL_LORA_OPTI: + if _SGLANG_EXPERIMENTAL_LORA_OPTI and not pad_expert_ids: from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs if lora_envs.SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN.get() and num_experts <= 8191: @@ -116,6 +123,7 @@ def moe_align_block_size( and topk_ids.numel() <= SMALL_NUMEL_LIMIT and num_experts + 1 > _CUDA_SMALL_BATCH_MAX_BUCKETS and not ignore_invalid_expert + and not pad_expert_ids ): moe_align_small_numel( topk_ids, @@ -128,11 +136,11 @@ def moe_align_block_size( return sorted_ids, expert_ids, num_tokens_post_pad # ===== TO BE REFACTORED ==== - use_jit_align = False + use_jit_align = pad_expert_ids if _SGLANG_EXPERIMENTAL_LORA_OPTI: from sglang.srt.lora.trtllm_lora_temp.environ import lora_envs - use_jit_align = lora_envs.SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN.get() + use_jit_align |= lora_envs.SGLANG_OPT_USE_JIT_KERNEL_MOE_ALIGN.get() if use_jit_align: from sglang.kernels.ops.moe.moe_align import ( moe_align_block_size as jit_moe_align_block_size, @@ -147,6 +155,7 @@ def moe_align_block_size( num_tokens_post_pad, cumsum_buffer, True, + pad_expert_ids=pad_expert_ids, ) # ===== END TO BE REFACTORED ==== else: diff --git a/python/sglang/srt/layers/quantization/gguf.py b/python/sglang/srt/layers/quantization/gguf.py index efbae869bb..01790f01b9 100644 --- a/python/sglang/srt/layers/quantization/gguf.py +++ b/python/sglang/srt/layers/quantization/gguf.py @@ -42,18 +42,20 @@ _is_hip = is_hip() _is_xpu = is_xpu() _is_musa = is_musa() _is_npu = is_npu() +_musa_apply_gguf_moe_weights = None if _is_cuda: from sgl_kernel import moe_sum from sgl_kernel.quantization import ( ggml_dequantize, ggml_moe_a8, - ggml_moe_a8_vec, ggml_mul_mat_a8, ggml_mul_mat_vec_a8, ) from sglang.kernels.ops.activation.activation import gelu_and_mul, silu_and_mul + from sglang.kernels.ops.moe.gguf_moe_vec import gguf_moe_a8_vec as ggml_moe_a8_vec + from sglang.kernels.ops.moe.gguf_moe_weights import apply_gguf_moe_weights elif _is_musa: from sgl_kernel import gelu_and_mul, moe_sum, silu_and_mul from sgl_kernel.quantization import ( @@ -63,6 +65,10 @@ elif _is_musa: ggml_mul_mat_a8, ggml_mul_mat_vec_a8, ) + + from sglang.kernels.ops.moe.gguf_moe_weights_aot import resolve_gguf_moe_weights + + _musa_apply_gguf_moe_weights = resolve_gguf_moe_weights("musa") elif _is_npu: from gguf import dequantize as gguf_dequantize else: @@ -243,15 +249,31 @@ def fused_moe_gguf( qweight_type2: int, activation: str, ) -> torch.Tensor: - def act(x: torch.Tensor): + def act(x: torch.Tensor, *, routed: bool = False): if activation == "silu": - return silu_and_mul(x) + activation_fn = silu_and_mul elif activation == "gelu": - return gelu_and_mul(x) - raise ValueError(f"Unsupported activation: {activation}") + activation_fn = gelu_and_mul + else: + raise ValueError(f"Unsupported activation: {activation}") + if routed and _is_musa and not native_musa: + x.masked_fill_(inactive.reshape(-1, 1), 0.0) + return activation_fn(x) - inactive = topk_ids < 0 - x = x.masked_fill(inactive.all(dim=-1, keepdim=True), 0.0) + def sum_expert_outputs(out: torch.Tensor): + if _is_cuda: + apply_gguf_moe_weights(out, topk_weights, topk_ids) + elif native_musa: + _musa_apply_gguf_moe_weights(out, topk_weights, topk_ids) + else: + out.masked_fill_(inactive.reshape(-1, 1), 0.0) + valid_weights = topk_weights.masked_fill(inactive, 0.0) + out.view(*topk_ids.shape, -1).mul_(valid_weights.unsqueeze(-1)) + moe_sum(out.view(*topk_ids.shape, -1), out_hidden_states) + + native_musa = _is_musa and _musa_apply_gguf_moe_weights is not None + if _is_musa and not native_musa: + inactive = topk_ids < 0 out_hidden_states = torch.empty_like(x) # unless we decent expert reuse we are better off running moe_vec kernel if ( @@ -265,13 +287,14 @@ def fused_moe_gguf( BLOCK_SIZE = ggml_moe_get_block_size(qweight_type) sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( - topk_ids, BLOCK_SIZE, E + topk_ids, BLOCK_SIZE, E, pad_expert_ids=_is_cuda ) - # Released MMQ kernels inspect one slack block before their bound check. - block_starts = ( - torch.arange(expert_ids.numel(), device=expert_ids.device) * BLOCK_SIZE - ) - expert_ids.masked_fill_(block_starts >= num_tokens_post_padded, -1) + if _is_musa and not native_musa: + # Released MMQ kernels inspect a slack block before their bound check. + block_starts = ( + torch.arange(expert_ids.numel(), device=expert_ids.device) * BLOCK_SIZE + ) + expert_ids.masked_fill_(block_starts >= num_tokens_post_padded, -1) out = ggml_moe_a8( x, w1, @@ -283,8 +306,7 @@ def fused_moe_gguf( top_k, num_tokens, ) - out.masked_fill_(inactive.reshape(-1, 1), 0.0) - out = act(out) + out = act(out, routed=True) out = ggml_moe_a8( out, w2, @@ -296,34 +318,31 @@ def fused_moe_gguf( 1, num_tokens * top_k, ) - out.masked_fill_(inactive.reshape(-1, 1), 0.0) - valid_weights = topk_weights.masked_fill(inactive, 0.0) - out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_( - valid_weights.view(num_tokens, top_k, 1) - ) # TODO(FlamingoPg): maybe we can use moe_sum_reduce here? - moe_sum(out, out_hidden_states) + sum_expert_outputs(out) elif qweight_type2 in MMVQ_QUANT_TYPES and qweight_type in MMVQ_QUANT_TYPES: num_tokens, _ = x.shape E, N, _ = w1.shape top_k = topk_ids.shape[1] - # Older vector kernels form weight addresses before checking -1. - # They have no per-expert capacity buffer; discard their dummy slots. - safe_ids = topk_ids.clamp_min(0) + if _is_cuda or native_musa: + # The vector ABI reads compact int32 IDs. Normal routes keep their + # storage; exceptional int64 routes canonicalize negative IDs. + safe_ids = ( + topk_ids.contiguous() + if topk_ids.dtype == torch.int32 + else topk_ids.clamp_min(-1).to(torch.int32).contiguous() + ) + else: + # Older vector kernels form weight addresses before checking -1. + safe_ids = topk_ids.clamp_min(0) out = ggml_moe_a8_vec(x, w1, safe_ids, top_k, qweight_type, N, num_tokens) - out.masked_fill_(inactive.reshape(-1, 1), 0.0) - out = act(out) + out = act(out, routed=True) out = ggml_moe_a8_vec( out, w2, safe_ids, 1, qweight_type2, w2.shape[1], num_tokens * top_k ) - out.masked_fill_(inactive.reshape(-1, 1), 0.0) - valid_weights = topk_weights.masked_fill(inactive, 0.0) - out = out.reshape(num_tokens, top_k, w2.shape[1]).mul_( - valid_weights.view(num_tokens, top_k, 1) - ) - moe_sum(out, out_hidden_states) + sum_expert_outputs(out) else: logger.warning_once( "There is no support for fast MoE kernel " @@ -332,7 +351,7 @@ def fused_moe_gguf( ) for tok, (w, idx) in enumerate(zip(topk_weights, topk_ids)): inp = x[tok].reshape((1,) + x.shape[1:]) - current_hidden_state = torch.zeros_like(inp) + current_hidden_state = None for ww, ii in zip(w, idx): if ii < 0: continue @@ -345,8 +364,14 @@ def fused_moe_gguf( current_state = fused_mul_mat_gguf( out, expert_down, qweight_type2 ).mul_(ww) - current_hidden_state.add_(current_state) - out_hidden_states[tok] = current_hidden_state + if current_hidden_state is None: + current_hidden_state = current_state + else: + current_hidden_state.add_(current_state) + if current_hidden_state is None: + out_hidden_states[tok].zero_() + else: + out_hidden_states[tok] = current_hidden_state return out_hidden_states diff --git a/test/registered/kernels/benchmark/moe/bench_gguf_moe_weights.py b/test/registered/kernels/benchmark/moe/bench_gguf_moe_weights.py new file mode 100644 index 0000000000..83f1eabc3c --- /dev/null +++ b/test/registered/kernels/benchmark/moe/bench_gguf_moe_weights.py @@ -0,0 +1,65 @@ +"""Compare all-valid GGUF weighting with the original in-place product.""" + +import sgl_kernel # noqa: F401 +import torch + +from sglang.kernels.jit.benchmark import marker +from sglang.kernels.ops.moe.gguf_moe_vec import gguf_moe_a8_vec +from sglang.kernels.ops.moe.gguf_moe_weights_aot import resolve_gguf_moe_weights +from sglang.srt.utils import is_musa +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=10, stage="nightly", runner_config="1-gpu-large") + + +def torch_weight(outputs, weights, ids): + outputs.view(*ids.shape, -1).mul_(weights.unsqueeze(-1)) + + +@marker.parametrize("tokens", [1, 64, 1024, 8192], [1024]) +@marker.parametrize("hidden", [4096, 7168], [4096]) +@marker.parametrize( + "dtype", [torch.float16, torch.bfloat16, torch.float32], [torch.float16] +) +@marker.benchmark("provider", ["aot", "torch"]) +def benchmark(tokens, hidden, dtype, provider): + device = "musa" if is_musa() else "cuda" + native = resolve_gguf_moe_weights(device) + if provider == "aot" and native is None: + marker.skip("Native GGUF weighting requires a rebuilt kernel provider") + outputs = torch.ones(tokens * 8, hidden, device=device, dtype=dtype) + # Unit weights keep repeated in-place calls numerically stable. + weights = torch.ones(tokens, 8, device=device) + ids = torch.zeros(tokens, 8, dtype=torch.int32, device=device) + return marker.do_bench( + native if provider == "aot" else torch_weight, + input_args=(outputs, weights, ids), + memory_output=(outputs,), + ) + + +@marker.parametrize("tokens", [1, 32, 64, 128], [64]) +@marker.parametrize( + "dtype", [torch.float16, torch.bfloat16, torch.float32], [torch.float16] +) +@marker.benchmark("provider", ["jit", "aot"]) +def benchmark_vector(tokens, dtype, provider): + """Compare the same all-valid Q4_0 vector operation, including private scratch.""" + from sgl_kernel.quantization import ggml_moe_a8_vec + + if is_musa(): + marker.skip("Local GGUF vector projection is CUDA-only") + inputs = torch.full((tokens, 4096), 0.25, device="cuda", dtype=dtype) + packed = torch.full( + (4, 128, 4096 // 32 * 18), 0x31, device="cuda", dtype=torch.uint8 + ) + ids = torch.zeros(tokens, 2, dtype=torch.int32, device="cuda") + return marker.do_bench( + gguf_moe_a8_vec if provider == "jit" else ggml_moe_a8_vec, + input_args=(inputs, packed, ids, 2, 2, 128, tokens), + ) + + +if __name__ == "__main__": + benchmark.run() + benchmark_vector.run() diff --git a/test/registered/kernels/ops/moe/test_gguf_moe_weights.py b/test/registered/kernels/ops/moe/test_gguf_moe_weights.py new file mode 100644 index 0000000000..35ebb50d8c --- /dev/null +++ b/test/registered/kernels/ops/moe/test_gguf_moe_weights.py @@ -0,0 +1,235 @@ +"""GGUF route weighting preserves per-slot rounding and skips inactive data.""" + +import unittest + +import sgl_kernel # noqa: F401 +import torch +from gguf import GGML_QUANT_SIZES, GGMLQuantizationType + +from sglang.kernels.ops.moe.gguf_moe_vec import gguf_moe_a8_vec +from sglang.kernels.ops.moe.gguf_moe_weights_aot import resolve_gguf_moe_weights +from sglang.srt.utils import is_musa +from sglang.test.ci.ci_register import register_cuda_ci, register_musa_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=90, stage="nightly", runner_config="1-gpu-large") +# backend-specific: MUSA uses the native weighting operator in its serving path. +register_musa_ci(est_time=6, suite="nightly-musa-1-gpu", nightly=True) + + +class TestGgufMoeWeights(CustomTestCase): + @unittest.skipIf(is_musa(), "Local GGUF vector projection is CUDA-only") + @torch.inference_mode() + def test_vector_formats_match_native_and_replay_inactive_routes(self): + """Every vector format retains real provider math and skips negative operands.""" + from sgl_kernel.quantization import ggml_moe_a8_vec + + generator = torch.Generator(device="cuda").manual_seed(314) + base_ids = torch.tensor( + [[0, 3], [1, 2], [3, 0]], dtype=torch.int32, device="cuda" + ) + # These valid packed bytes give finite, nonzero scales, integer codes and + # codebook indices, including IQ1_M's distributed FP16 scale bits. + for quant_type in ( + 2, + 3, + 6, + 7, + 8, + 10, + 11, + 12, + 13, + 14, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 29, + ): + block_size, block_bytes = GGML_QUANT_SIZES[GGMLQuantizationType(quant_type)] + packed = torch.empty( + 4, 17, 256 // block_size * block_bytes, dtype=torch.uint8, device="cuda" + ) + for expert in range(4): + packed[expert].fill_(0x31 + expert) + dtypes = ( + (torch.float16, torch.bfloat16, torch.float32) + if quant_type in (2, 8, 10, 29) + else (torch.float16,) + ) + for dtype in dtypes: + with self.subTest(quant_type=quant_type, dtype=dtype): + inputs = ( + torch.randn( + 3, 256, generator=generator, device="cuda", dtype=dtype + ) + / 4 + ) + original = inputs.clone() + ids = base_ids.clone() + expected = ggml_moe_a8_vec( + inputs, packed, ids, 2, quant_type, 17, 3 + ) + self.assertTrue(bool(torch.isfinite(expected).all())) + self.assertGreater(float(expected.abs().max()), 0.0) + actual = gguf_moe_a8_vec(inputs, packed, ids, 2, quant_type, 17, 3) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + # The down projection has one input per original route, while + # its IDs retain [original_tokens, original_topk] shape. + down = inputs.repeat_interleave(2, dim=0) + torch.testing.assert_close( + gguf_moe_a8_vec(down, packed, ids, 1, quant_type, 17, 6), + ggml_moe_a8_vec(down, packed, ids, 1, quant_type, 17, 6), + rtol=0, + atol=0, + ) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = gguf_moe_a8_vec( + inputs, packed, ids, 2, quant_type, 17, 3 + ) + original_pointer = ids.data_ptr() + for routes in ( + [[-1, 3], [1, -3], [-7, -1]], + [[-1, -3], [-7, -1], [-1, -1]], + base_ids, + ): + ids.copy_( + torch.as_tensor(routes, dtype=ids.dtype, device=ids.device) + ) + inputs.copy_(original) + inputs[(ids < 0).all(dim=1)] = float("nan") + captured.fill_(float("nan")) + graph.replay() + live = (ids >= 0).flatten() + self.assertTrue(bool(torch.isfinite(captured).all())) + self.assertEqual(int(torch.count_nonzero(captured[~live])), 0) + torch.testing.assert_close( + captured[live], expected[live], rtol=0, atol=0 + ) + self.assertEqual(ids.data_ptr(), original_pointer) + empty = gguf_moe_a8_vec( + inputs[:0], packed, ids[:0], 2, quant_type, 17, 0 + ) + self.assertEqual(empty.shape, (0, 17)) + + def test_rounding_and_poisoned_routes(self): + """Weights round per slot before summation; dropped NaNs contribute zero.""" + device = "musa" if is_musa() else "cuda" + providers = {} + native = resolve_gguf_moe_weights(device) + if device == "musa": + self.assertIsNotNone(native, "MUSA kernel tests require a rebuilt provider") + else: + from sglang.kernels.ops.moe.gguf_moe_weights import apply_gguf_moe_weights + + providers["triton"] = apply_gguf_moe_weights + if native is not None: + providers["aot"] = native + generator = torch.Generator(device=device).manual_seed(836) + for provider, apply_weights in providers.items(): + for dtype in (torch.float16, torch.bfloat16, torch.float32): + for weight_dtype in (torch.float16, torch.bfloat16, torch.float32): + for strided in (False, True): + with self.subTest( + provider=provider, + dtype=dtype, + weight_dtype=weight_dtype, + strided=strided, + ): + outputs = torch.randn( + 12, 257, generator=generator, device=device, dtype=dtype + ) + weights = torch.randn( + 4, + 3, + generator=generator, + device=device, + dtype=weight_dtype, + ) + ids = torch.tensor( + [[0, 1, 2], [0, -3, 2], [-1, -1, -1], [1, 0, -1]], + device=device, + dtype=torch.int64 if strided else torch.int32, + ) + if strided: + weights = weights.t().contiguous().t() + ids = ids.t().contiguous().t() + inactive = ids < 0 + outputs[inactive.flatten()] = float("nan") + weights[inactive] = float("nan") + expected = outputs.clone() + expected[inactive.flatten()] = 0 + expected.view(4, 3, 257).mul_( + weights.masked_fill(inactive, 0).unsqueeze(-1) + ) + apply_weights(outputs, weights, ids) + torch.testing.assert_close( + outputs, expected, rtol=0, atol=0 + ) + self.assertTrue(bool(torch.isfinite(outputs).all())) + ids.fill_(-1) + weights.fill_(float("nan")) + outputs.fill_(float("nan")) + apply_weights(outputs, weights, ids) + self.assertEqual(int(torch.count_nonzero(outputs)), 0) + apply_weights(outputs[:0], weights[:0], ids[:0]) + + @unittest.skipIf(is_musa(), "CUDA fast-math regression") + def test_subnormal_operands_and_products(self): + """A tiny score can produce a normal result even in the SM90 fast-math build.""" + native = resolve_gguf_moe_weights("cuda") + if native is None: + self.skipTest("Native GGUF weighting requires a rebuilt kernel provider") + for dtype in (torch.float32, torch.bfloat16): + for weight_dtype in (torch.float32, torch.bfloat16): + for strided in (False, True): + with self.subTest( + dtype=dtype, weight_dtype=weight_dtype, strided=strided + ): + outputs = ( + torch.tensor( + [1e38, 1e38, float("nan"), 1e-40, 2.0**-126, -0.0], + dtype=dtype, + device="cuda", + ) + .view(6, 1) + .expand(6, 257) + .clone() + ) + weights = torch.tensor( + [[1e-40, -1e-40, float("nan")], [1e38, 0.5, 1.0]], + dtype=weight_dtype, + device="cuda", + ) + ids = torch.tensor( + [[0, 1, -1], [2, 3, 4]], + dtype=torch.int64 if strided else torch.int32, + device="cuda", + ) + if strided: + weights = weights.t().contiguous().t() + ids = ids.t().contiguous().t() + # FP64 exactly represents products of FP32 operands; + # round to FP32 before narrowing to the output dtype. + expected = outputs.cpu().double().view(2, 3, 257) + expected = expected * weights.cpu().double().unsqueeze(-1) + expected[ids.cpu() < 0] = 0 + expected = expected.float().to(dtype).view_as(outputs).cuda() + self.assertGreater(float(expected[0, 0]), 1e-3) + self.assertGreater(float(expected[3, 0]), 1e-3) + self.assertNotEqual(float(expected[4, 0]), 0.0) + native(outputs, weights, ids) + torch.testing.assert_close(outputs, expected, rtol=0, atol=0) + self.assertTrue( + torch.equal(outputs.signbit(), expected.signbit()) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/moe/test_moe_align_block_size.py b/test/registered/kernels/ops/moe/test_moe_align_block_size.py index 558916c709..c6f83e9ea0 100644 --- a/test/registered/kernels/ops/moe/test_moe_align_block_size.py +++ b/test/registered/kernels/ops/moe/test_moe_align_block_size.py @@ -286,5 +286,82 @@ def test_moe_align_block_size_pad_fill_stays_in_bounds( ) +@pytest.mark.parametrize("dtype", [torch.int32, torch.int64]) +@pytest.mark.parametrize("num_tokens,num_experts", [(65, 4), (512, 128), (65, 1025)]) +def test_moe_align_block_size_completes_expert_capacity(dtype, num_tokens, num_experts): + """Unused expert-map slots must be inactive after every graph replay.""" + block_size, topk = 4, 2 + original = ( + torch.arange(num_tokens * topk, device="cuda", dtype=dtype).reshape( + num_tokens, topk + ) + % num_experts + ) + routes = original.clone() + capacity = min( + routes.numel() * block_size, + routes.numel() + (num_experts + 1) * (block_size - 1), + ) + capacity = ceil_div(capacity, 4) * 4 + sorted_ids = torch.empty(capacity, device="cuda", dtype=torch.int32) + blocks = ceil_div(capacity, block_size) + # Guard words make an out-of-bounds capacity fill observable. + expert_storage = torch.full( + (blocks + 4,), num_experts, device="cuda", dtype=torch.int32 + ) + expert_ids = expert_storage[:blocks] + count = torch.empty(1, device="cuda", dtype=torch.int32) + cumsum = torch.empty(num_experts + 2, device="cuda", dtype=torch.int32) + + def align(): + moe_align_block_size( + routes, + num_experts + 1, + block_size, + sorted_ids, + expert_ids, + count, + cumsum, + True, + pad_expert_ids=True, + ) + + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + align() + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + align() + + for live in (num_tokens, num_tokens // 2, 0, num_tokens): + routes.copy_(original) + routes[live:].fill_(-1) + sorted_ids.fill_(-99) + expert_storage.fill_(num_experts) + graph.replay() + used = count.item() + assert used % block_size == 0 + assert torch.all(expert_ids[used // block_size :] == -1) + assert torch.all(expert_storage[blocks:] == num_experts) + assert torch.all(sorted_ids[used:] == routes.numel()) + # Every real route appears exactly once in its expert's contiguous region. + flat_routes = routes.flatten() + expected_start = 0 + for expert in torch.unique(flat_routes).tolist(): + expected = torch.where(flat_routes == expert)[0] + padded = ceil_div(expected.numel(), block_size) * block_size + end = expected_start + padded + assert torch.all( + expert_ids[expected_start // block_size : end // block_size] == expert + ) + actual = sorted_ids[expected_start:end] + actual = actual[actual < routes.numel()].long().sort().values + assert torch.equal(actual, expected) + expected_start = end + assert expected_start == used + + if __name__ == "__main__": sys.exit(pytest.main([__file__])) diff --git a/test/registered/unit/kernels/test_kernels_namespace.py b/test/registered/unit/kernels/test_kernels_namespace.py index a287e8a3cf..9c5f0d5d33 100644 --- a/test/registered/unit/kernels/test_kernels_namespace.py +++ b/test/registered/unit/kernels/test_kernels_namespace.py @@ -5,6 +5,7 @@ import re import subprocess import sys from pathlib import Path +from types import SimpleNamespace import pytest @@ -21,6 +22,7 @@ _CPU = PlatformInfo(device_type="cpu") _SM90 = PlatformInfo(device_type="cuda", cuda_arch_major=9, cuda_arch_minor=0) _SM100 = PlatformInfo(device_type="cuda", cuda_arch_major=10, cuda_arch_minor=0) _HIP = PlatformInfo(device_type="hip") +_MUSA = PlatformInfo(device_type="musa") def test_single_backend_resolves_without_backend(): @@ -40,6 +42,15 @@ def test_fp8_scaled_mm_requires_explicit_registry_backend(monkeypatch): ) +def test_gguf_vector_registry_is_cuda_local_jit(): + spec = K.select_kernel("moe.gguf_moe_a8_vec") + assert spec.backend is KernelBackend.JIT + assert spec.target == "sglang.kernels.ops.moe.gguf_moe_vec:gguf_moe_a8_vec" + assert K.capabilities_satisfied(spec.capabilities, _SM90) + assert not K.capabilities_satisfied(spec.capabilities, _MUSA) + assert not K.capabilities_satisfied(spec.capabilities, _CPU) + + def test_unknown_op_or_backend_raises(): with pytest.raises(KeyError): K.select_kernel("does_not.exist") @@ -55,6 +66,7 @@ def test_unknown_op_or_backend_raises(): (PlatformInfo(device_type="cuda", cuda_arch_major=8, cuda_arch_minor=0), False), (_CPU, False), (_HIP, False), + (_MUSA, False), ], ) def test_qwen_qkv_registry_accepts_hopper(platform, eligible): @@ -90,6 +102,12 @@ def test_activation_default_backend(monkeypatch, device, expect): (Cap.CUDA, _SM90, True), (Cap.CUDA, _HIP, False), (Cap.HIP, _HIP, True), + (Cap.MUSA, _MUSA, True), + (Cap.MUSA, _CPU, False), + (Cap.MUSA, _SM90, False), + (Cap.MUSA, _HIP, False), + (Cap.CUDA, _MUSA, False), + (Cap.HIP, _MUSA, False), (Cap.cuda(min_sm=(10, 0)), _SM90, False), (Cap.cuda(min_sm=(10, 0)), _SM100, True), (Cap.cuda(max_sm=(9, 0)), _SM100, False), @@ -99,6 +117,32 @@ def test_capability_is_satisfied_by(req, plat, ok): assert req.is_satisfied_by(plat) is ok +@pytest.mark.parametrize( + "musa_version, cuda_available, musa_available, expected_device", + [ + ("4.0", True, None, "musa"), + (None, True, None, "cuda"), + ("4.0", False, None, "cpu"), + ("4.0", False, True, "musa"), + ], +) +def test_platform_detection_distinguishes_musa_cuda_alias( + monkeypatch, musa_version, cuda_available, musa_available, expected_device +): + """Native MUSA remains eligible before the CUDA compatibility alias is ready.""" + torch = SimpleNamespace( + version=SimpleNamespace(hip=None, musa=musa_version), + cuda=SimpleNamespace( + is_available=lambda: cuda_available, + get_device_capability=lambda: (9, 0), + ), + ) + if musa_available is not None: + torch.musa = SimpleNamespace(is_available=lambda: musa_available) + monkeypatch.setitem(sys.modules, "torch", torch) + assert PlatformInfo.detect().device_type == expected_device + + def test_capabilities_or_semantics(): both = {Cap.CUDA, Cap.HIP} assert K.capabilities_satisfied(both, _SM90) diff --git a/test/registered/unit/layers/quantization/test_gguf_inactive_routes.py b/test/registered/unit/layers/quantization/test_gguf_inactive_routes.py index a2dccd470f..769515abf4 100644 --- a/test/registered/unit/layers/quantization/test_gguf_inactive_routes.py +++ b/test/registered/unit/layers/quantization/test_gguf_inactive_routes.py @@ -99,7 +99,10 @@ class TestGgufInactiveRoutes(CustomTestCase): # GGUF GEMMs quantize each 32 input values to Q8_1, including an FP16 # scale and input sum. CUDA roundf rounds halfway values away from zero. blocks = inputs.float().unflatten(-1, (-1, 32)) - scale = blocks.abs().amax(dim=-1, keepdim=True) / 127 + amax = blocks.abs().amax(dim=-1, keepdim=True) + # A scalar divisor lets PyTorch multiply by its rounded reciprocal, + # changing Q8_1 codes at ties compared with the native FP32 division. + scale = amax / torch.full_like(amax, 127.0) denominator = torch.where(scale == 0, 1.0, scale) codes = blocks.sign() * torch.floor(blocks.abs() / denominator + 0.5) quantized = codes * scale.half().float() @@ -143,7 +146,7 @@ class TestGgufInactiveRoutes(CustomTestCase): """Changing the live count must zero dropped GGUF routes on every replay.""" for quant_type in (GGMLQuantizationType.Q4_0, GGMLQuantizationType.Q8_0): packed, dense = self._weights(quant_type) - for tokens, partial in ((64, 48), (96, 71)): + for tokens, partial in ((64, 48), (65, 48), (96, 71)): with self.subTest(quant_type=quant_type, tokens=tokens): self._check_graph(packed, dense, quant_type, tokens, partial) @@ -186,20 +189,12 @@ class TestGgufInactiveRoutes(CustomTestCase): native_outputs = [] native_mmq = gguf_runtime.ggml_moe_a8 - native_align = gguf_runtime.moe_align_block_size - - def poison_alignment(*args, **kwargs): - sorted_ids, expert_ids, count = native_align(*args, **kwargs) - block_starts = ( - torch.arange(expert_ids.numel(), device=expert_ids.device) * 4 - ) - expert_ids.masked_fill_(block_starts >= count, 4) - return sorted_ids, expert_ids, count - def record_mmq_outputs(*args, **kwargs): if not torch.cuda.is_current_stream_capturing(): # Fail before native dispatch if a slack block can index expert E. self.assertTrue(bool((args[3] < args[1].shape[0]).all())) + used_blocks = args[4].item() // 4 + self.assertTrue(bool((args[3][used_blocks:] == -1).all())) result = native_mmq(*args, **kwargs) if torch.cuda.is_current_stream_capturing(): native_outputs.append(result) @@ -212,10 +207,7 @@ class TestGgufInactiveRoutes(CustomTestCase): if tokens > 64 else nullcontext() ) - with ( - observe_mmq, - patch.object(gguf_runtime, "moe_align_block_size", poison_alignment), - ): + with observe_mmq: stream = torch.cuda.Stream() stream.wait_stream(torch.cuda.current_stream()) with torch.cuda.stream(stream): @@ -226,8 +218,7 @@ class TestGgufInactiveRoutes(CustomTestCase): with torch.cuda.graph(graph): output, ids = run() self.assertEqual(len(native_outputs), 2 if tokens > 64 else 0) - # Dropped slots must overwrite poisoned native intermediates rather - # than inherit any previous live route or allocator contents. + # Dropped routes must not consume poisoned native intermediates. for live in (tokens, partial, 0, partial // 2, tokens): hidden.copy_(base_hidden) hidden[live:].fill_(float("nan")) @@ -238,10 +229,6 @@ class TestGgufInactiveRoutes(CustomTestCase): for intermediate in native_outputs: intermediate.fill_(float("nan")) graph.replay() - for intermediate in native_outputs: - self.assertTrue(torch.isfinite(intermediate).all().item()) - inactive = intermediate.view(tokens, 2, -1)[live:] - self.assertEqual(torch.count_nonzero(inactive).item(), 0) self.assertTrue(torch.isfinite(output).all().item()) self.assertTrue((ids[live:] == -1).all().item()) self.assertEqual(torch.count_nonzero(output[live:]).item(), 0) @@ -254,6 +241,81 @@ class TestGgufInactiveRoutes(CustomTestCase): output[:live].float(), expected, atol=2e-3, rtol=0.06 ) + @torch.inference_mode() + def test_mmq_prefix_preserves_activation_memory_budget(self): + """Preparing MMQ routes must not duplicate the caller's activation matrix.""" + tokens, hidden, intermediate, experts = 8192, 4096, 128, 4 + inputs = torch.ones(tokens, hidden, device="cuda", dtype=torch.float16) + ids = torch.zeros(tokens, 2, device="cuda", dtype=torch.int32) + weights = torch.full((tokens, 2), 0.5, device="cuda") + packed_up = torch.empty( + experts, + 2 * intermediate, + hidden // 32 * 18, + device="cuda", + dtype=torch.uint8, + ) + packed_down = torch.empty( + experts, + hidden, + intermediate // 32 * 18, + device="cuda", + dtype=torch.uint8, + ) + + class ProjectionReached(Exception): + pass + + def stop_before_projection(*args, **kwargs): + raise ProjectionReached + + def run_prefix(): + with self.assertRaises(ProjectionReached): + fused_moe_gguf( + inputs, + packed_up, + packed_down, + weights, + ids, + GGMLQuantizationType.Q4_0, + GGMLQuantizationType.Q4_0, + "silu", + ) + + from torch.utils._python_dispatch import TorchDispatchMode + + allocations = [] + + class ObserveAllocations(TorchDispatchMode): + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + result = func(*args, **(kwargs or {})) + input_storage = { + arg.untyped_storage().data_ptr() + for arg in args + if isinstance(arg, torch.Tensor) + } + if ( + isinstance(result, torch.Tensor) + and result.numel() + and result.untyped_storage().data_ptr() not in input_storage + ): + allocations.append(result.untyped_storage().nbytes()) + return result + + with patch.object(gguf_runtime, "ggml_moe_a8", stop_before_projection): + run_prefix() + torch.cuda.synchronize() + allocated = torch.cuda.memory_allocated() + torch.cuda.reset_peak_memory_stats() + with ObserveAllocations(): + run_prefix() + torch.cuda.synchronize() + peak = torch.cuda.max_memory_allocated() - allocated + # One output and four alignment buffers; no intermediate masks or ranges. + self.assertLessEqual(len(allocations), 5) + budget = inputs.numel() * inputs.element_size() + 128 * 1024 + self.assertLessEqual(peak, budget) + @torch.inference_mode() def test_nonlocal_routes_preserve_local_contributions(self): """A nonlocal slot must not erase the same token's valid expert result.""" @@ -308,34 +370,310 @@ class TestGgufInactiveRoutes(CustomTestCase): output[token].float(), expected, atol=2e-3, rtol=0.06 ) + @torch.inference_mode() + def test_cuda_vector_uses_original_ids_without_route_preparation(self): + """Real CUDA projections consume negative routes without a clamp launch.""" + from sgl_kernel.quantization import ggml_moe_a8_vec as legacy_vector + from torch.utils._python_dispatch import TorchDispatchMode + + ids = torch.tensor( + [[-1, 1], [0, -3], [-1, -1]], device="cuda", dtype=torch.int32 + ) + scores = torch.tensor( + [[float("nan"), 0.75], [0.5, float("nan")], [float("nan"), float("nan")]], + device="cuda", + ) + hidden = torch.full((3, 256), 0.25, device="cuda", dtype=torch.float16) + hidden[-1].fill_(float("nan")) + raw_ids = ids.clone() + route_ops = [] + actual_vector = gguf_runtime.ggml_moe_a8_vec + + class ObserveRoutePreparation(TorchDispatchMode): + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + if func in ( + torch.ops.aten.clamp_min.default, + torch.ops.aten.clamp_min.out, + torch.ops.aten._to_copy.default, + ): + route_ops.append(str(func)) + return func(*args, **(kwargs or {})) + + def observe_vector(inputs, weights, routes, *args): + self.assertEqual(routes.data_ptr(), ids.data_ptr()) + return actual_vector(inputs, weights, routes, *args) + + for quant_type in (GGMLQuantizationType.Q4_0, GGMLQuantizationType.Q8_0): + packed, _ = self._weights(quant_type) + with patch.object(gguf_runtime, "ggml_moe_a8_vec", legacy_vector): + expected = fused_moe_gguf( + hidden.nan_to_num(), + *packed, + scores.nan_to_num(), + ids.clamp_min(0), + quant_type, + quant_type, + "silu", + ) + route_ops.clear() + with ( + ObserveRoutePreparation(), + patch.object(gguf_runtime, "ggml_moe_a8_vec", observe_vector), + ): + actual = fused_moe_gguf( + hidden, *packed, scores, ids, quant_type, quant_type, "silu" + ) + self.assertEqual(route_ops, []) + self.assertTrue(bool(torch.isfinite(actual).all())) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + torch.testing.assert_close(ids, raw_ids, rtol=0, atol=0) + + @torch.inference_mode() + def test_musa_branch_preserves_input_and_route_budgets(self): + """MUSA consumers exclude NaNs without input copies or route temporaries. + + CUDA execution covers Python dispatch; native MUSA runs in its own CI lane. + """ + from sgl_kernel import gelu_and_mul as legacy_gelu_and_mul + from sgl_kernel import silu_and_mul as legacy_silu_and_mul + from torch.utils._python_dispatch import TorchDispatchMode + + from sglang.kernels.ops.moe.gguf_moe_weights_aot import resolve_gguf_moe_weights + from sglang.srt.utils import is_musa + + finalizer = resolve_gguf_moe_weights("musa" if is_musa() else "cuda") + finalizers = (None,) if finalizer is None else (None, finalizer) + for finalizer in finalizers: + with self.subTest(native_finalizer=finalizer is not None): + for quant_type in ( + GGMLQuantizationType.Q4_0, + GGMLQuantizationType.Q8_0, + ): + packed, _ = self._weights(quant_type) + for tokens in (3, 96): + for activation in ("silu", "gelu"): + with self.subTest( + quant_type=quant_type, + tokens=tokens, + activation=activation, + ): + repeats = (tokens + 2) // 3 + ids = torch.tensor( + [[0, 1], [1, -1], [-1, -1]], + device="cuda", + dtype=torch.int32, + ).repeat(repeats, 1)[:tokens] + weights = torch.tensor( + [ + [0.25, 0.75], + [0.5, float("nan")], + [float("nan"), float("nan")], + ], + device="cuda", + ).repeat(repeats, 1)[:tokens] + inactive = ids < 0 + inputs = ( + torch.linspace( + 0.125, + 2.0, + tokens, + device="cuda", + dtype=torch.float16, + ) + .unsqueeze(1) + .expand(tokens, 256) + .contiguous() + ) + inputs[inactive.all(dim=-1)] = float("nan") + provider_name = ( + "ggml_moe_a8" if tokens > 64 else "ggml_moe_a8_vec" + ) + native = ( + gguf_runtime.ggml_moe_a8 + if tokens > 64 + else gguf_runtime.ggml_moe_a8_vec + ) + calls = 0 + route_costs = [] + + class ObserveRouteCosts(TorchDispatchMode): + def __torch_dispatch__( + self, func, types, args=(), kwargs=None + ): + if func in ( + torch.ops.aten.lt.Scalar, + torch.ops.aten.clamp_min.default, + torch.ops.aten.clamp_min.out, + torch.ops.aten.masked_fill.Scalar, + torch.ops.aten.masked_fill_.Scalar, + torch.ops.aten.arange.default, + ): + route_costs.append(str(func)) + return func(*args, **(kwargs or {})) + + def observe_projection( + projection_input, *args, **kwargs + ): + nonlocal calls + if calls == 0: + self.assertEqual( + projection_input.data_ptr(), + inputs.data_ptr(), + ) + if finalizer is not None and tokens <= 64: + self.assertIs(args[1], ids) + calls += 1 + return native(projection_input, *args, **kwargs) + + with ( + patch.object(gguf_runtime, "_is_cuda", False), + patch.object(gguf_runtime, "_is_musa", True), + patch.object( + gguf_runtime, + "_musa_apply_gguf_moe_weights", + None, + ), + patch.object( + gguf_runtime, + "silu_and_mul", + legacy_silu_and_mul, + ), + patch.object( + gguf_runtime, + "gelu_and_mul", + legacy_gelu_and_mul, + ), + ): + expected = fused_moe_gguf( + inputs.nan_to_num(), + *packed, + weights.masked_fill(inactive, 0), + ids.clamp_min(0), + quant_type, + quant_type, + activation, + ) + with ( + ObserveRouteCosts(), + patch.object( + gguf_runtime, + provider_name, + observe_projection, + ), + patch.object( + gguf_runtime, + "_musa_apply_gguf_moe_weights", + finalizer, + ), + ): + actual = fused_moe_gguf( + inputs, + *packed, + weights, + ids, + quant_type, + quant_type, + activation, + ) + self.assertEqual(calls, 2) + if finalizer is not None: + self.assertEqual(route_costs, []) + self.assertGreater(expected.abs().max().item(), 1e-3) + self.assertTrue(torch.isfinite(actual).all().item()) + self.assertEqual( + torch.count_nonzero( + actual[inactive.all(dim=-1)] + ).item(), + 0, + ) + torch.testing.assert_close( + actual, expected, rtol=0, atol=0 + ) + @torch.inference_mode() def test_unquantized_fallback_drops_negative_ids(self): - """The slow path must not alias a dropped route to the last expert.""" + """Dropped routes stay zero without adding a valid-route accumulator.""" + from torch.utils._python_dispatch import TorchDispatchMode + generator = torch.Generator(device="cuda").manual_seed(17) w13 = torch.randn(2, 32, 16, generator=generator, device="cuda").half() / 16 w2 = torch.randn(2, 16, 16, generator=generator, device="cuda").half() / 16 - hidden = torch.ones(2, 16, device="cuda", dtype=torch.float16) - hidden[1].fill_(float("nan")) - ids = torch.tensor([[0, -1], [-1, -1]], device="cuda", dtype=torch.int32) + hidden = torch.ones(4, 16, device="cuda", dtype=torch.float16) + hidden[-1].fill_(float("nan")) + ids = torch.tensor( + [[0, -3, -1], [0, 1, 0], [-1, 1, 0], [-1, -3, -1]], + device="cuda", + dtype=torch.int32, + ) weights = torch.tensor( - [[1.0, float("nan")], [float("nan"), float("nan")]], device="cuda" + [ + [1.0, float("nan"), float("nan")], + [0.2, 0.3, 0.5], + [float("nan"), 0.75, 0.25], + [float("nan"), float("nan"), float("nan")], + ], + device="cuda", ) - output = fused_moe_gguf( - hidden, - w13, - w2, - weights, - ids, - GGMLQuantizationType.F16, - GGMLQuantizationType.F16, - "silu", + original_hidden, original_ids, original_weights = ( + hidden.clone(), + ids.clone(), + weights.clone(), ) - projected = hidden[0] @ w13[0].t() - gate, up = projected.chunk(2) - expected = (torch.nn.functional.silu(gate) * up) @ w2[0].t() + observed = { + op: [] + for op in ( + torch.ops.aten.zeros_like.default, + torch.ops.aten.add_.Tensor, + torch.ops.aten.mul_.Tensor, + torch.ops.aten.zero_.default, + ) + } + + class ObserveTensorOps(TorchDispatchMode): + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + if func in observed: + observed[func].append(args[0].data_ptr()) + return func(*args, **(kwargs or {})) + + with ObserveTensorOps(): + output = fused_moe_gguf( + hidden, + w13, + w2, + weights, + ids, + GGMLQuantizationType.F16, + GGMLQuantizationType.F16, + "silu", + ) + projected = torch.einsum("th,enh->ten", hidden.nan_to_num(), w13) + gate, up = projected.float().chunk(2, dim=-1) + activated = (torch.nn.functional.silu(gate) * up).to(hidden.dtype) + experts = torch.einsum("tek,eok->teo", activated, w2) + selected = experts.gather( + 1, ids.clamp_min(0).long().unsqueeze(-1).expand(-1, -1, hidden.shape[1]) + ) + expected = ( + selected.float() * weights.masked_fill(ids < 0, 0).unsqueeze(-1) + ).sum(dim=1) + self.assertGreater(expected.abs().max().item(), 1e-3) self.assertTrue(torch.isfinite(output).all().item()) - self.assertEqual(torch.count_nonzero(output[1]).item(), 0) - torch.testing.assert_close(output[0], expected, atol=1e-3, rtol=1e-3) + self.assertEqual(torch.count_nonzero(output[-1]).item(), 0) + torch.testing.assert_close(output.float(), expected, atol=1e-3, rtol=1e-3) + torch.testing.assert_close(hidden, original_hidden, equal_nan=True) + torch.testing.assert_close(ids, original_ids) + torch.testing.assert_close(weights, original_weights, equal_nan=True) + self.assertNotEqual(output.data_ptr(), hidden.data_ptr()) + self.assertEqual(observed[torch.ops.aten.zeros_like.default], []) + self.assertEqual( + observed[torch.ops.aten.zero_.default], [output[-1].data_ptr()] + ) + weighted_outputs = observed[torch.ops.aten.mul_.Tensor] + self.assertEqual(len(weighted_outputs), 6) + self.assertEqual( + observed[torch.ops.aten.add_.Tensor], + [weighted_outputs[1], weighted_outputs[1], weighted_outputs[4]], + ) if __name__ == "__main__":