Release sglang kernel 0.4.0 (#20440)

Co-authored-by: Baizhou Zhang <sobereddiezhang@gmail.com>
This commit is contained in:
Xiaoyu Zhang
2026-03-16 20:34:58 +08:00
committed by GitHub
co-authored by Baizhou Zhang
parent 3d58cd16d9
commit 15097c5c3b
36 changed files with 188 additions and 146 deletions
@@ -8,12 +8,14 @@ from sglang.jit_kernel.awq_marlin_repack import (
)
from sglang.srt.layers.quantization.utils import pack_cols, quantize_weights
try:
from sgl_kernel import awq_marlin_moe_repack as aot_awq_marlin_moe_repack
AOT_AVAILABLE = True
except ImportError:
AOT_AVAILABLE = False
def _has_aot_awq_marlin_moe_repack() -> bool:
return hasattr(torch.ops.sgl_kernel, "awq_marlin_moe_repack") and hasattr(
torch.ops.sgl_kernel.awq_marlin_moe_repack, "default"
)
AOT_AVAILABLE = _has_aot_awq_marlin_moe_repack()
def awq_pack(
@@ -68,7 +70,9 @@ def test_awq_marlin_moe_repack_jit_vs_aot(
perm = torch.empty((num_experts, 0), dtype=torch.int32, device="cuda")
out_jit = jit_awq_marlin_moe_repack(b_q_weight, perm, size_k, size_n, num_bits)
out_aot = aot_awq_marlin_moe_repack(b_q_weight, perm, size_k, size_n, num_bits)
out_aot = torch.ops.sgl_kernel.awq_marlin_moe_repack.default(
b_q_weight, perm, size_k, size_n, num_bits
)
torch.cuda.synchronize()
@@ -9,12 +9,14 @@ from sglang.jit_kernel.awq_marlin_repack import (
from sglang.srt.layers.quantization.utils import pack_cols, quantize_weights
from sglang.test.test_marlin_utils import get_weight_perm, marlin_weights
try:
from sgl_kernel import awq_marlin_repack as aot_awq_marlin_repack
AOT_AVAILABLE = True
except ImportError:
AOT_AVAILABLE = False
def _has_aot_awq_marlin_repack() -> bool:
return hasattr(torch.ops.sgl_kernel, "awq_marlin_repack") and hasattr(
torch.ops.sgl_kernel.awq_marlin_repack, "default"
)
AOT_AVAILABLE = _has_aot_awq_marlin_repack()
def awq_pack(
@@ -58,7 +60,9 @@ def test_awq_marlin_repack_jit_vs_aot(num_bits, k_tiles, n_tiles, group_size):
q_w_awq = awq_pack(q_w, num_bits, size_k, size_n)
out_jit = jit_awq_marlin_repack(q_w_awq, size_k, size_n, num_bits)
out_aot = aot_awq_marlin_repack(q_w_awq, size_k, size_n, num_bits)
out_aot = torch.ops.sgl_kernel.awq_marlin_repack.default(
q_w_awq, size_k, size_n, num_bits
)
torch.cuda.synchronize()
@@ -2,7 +2,6 @@ import itertools
import pytest
import torch
from sgl_kernel import moe_wna16_marlin_gemm as aot_moe_wna16_marlin_gemm
from sgl_kernel.scalar_type import scalar_types
from sglang.jit_kernel.moe_wna16_marlin import moe_wna16_marlin_gemm
@@ -10,6 +9,15 @@ from sglang.srt.layers.moe.fused_moe_triton import moe_align_block_size
from sglang.test.test_marlin_utils import awq_marlin_quantize, marlin_quantize
def _has_aot_moe_wna16_marlin_gemm() -> bool:
return hasattr(torch.ops.sgl_kernel, "moe_wna16_marlin_gemm") and hasattr(
torch.ops.sgl_kernel.moe_wna16_marlin_gemm, "default"
)
AOT_AVAILABLE = _has_aot_moe_wna16_marlin_gemm()
def stack_and_dev(tensors: list[torch.Tensor]):
dev = tensors[0].device
return torch.stack(tensors, dim=0).to(dev)
@@ -143,7 +151,7 @@ def _run_single_gemm_aot(
is_k_full,
use_atomic_add,
):
return aot_moe_wna16_marlin_gemm(
return torch.ops.sgl_kernel.moe_wna16_marlin_gemm.default(
a,
c,
qweight,
@@ -224,6 +232,9 @@ TEST_CASES = generate_test_cases()
def test_moe_wna16_marlin_gemm(
m, n, k, e, topk, dtype, group_size, act_order, quant_type
):
if not AOT_AVAILABLE:
pytest.skip("sgl_kernel moe_wna16_marlin_gemm AOT op not available")
torch.manual_seed(0)
has_zp = quant_type in [scalar_types.uint4, scalar_types.uint8]