Release sglang kernel 0.4.0 (#20440)
Co-authored-by: Baizhou Zhang <sobereddiezhang@gmail.com>
This commit is contained in:
co-authored by
Baizhou Zhang
parent
3d58cd16d9
commit
15097c5c3b
@@ -8,12 +8,14 @@ from sglang.jit_kernel.awq_marlin_repack import (
|
||||
)
|
||||
from sglang.srt.layers.quantization.utils import pack_cols, quantize_weights
|
||||
|
||||
try:
|
||||
from sgl_kernel import awq_marlin_moe_repack as aot_awq_marlin_moe_repack
|
||||
|
||||
AOT_AVAILABLE = True
|
||||
except ImportError:
|
||||
AOT_AVAILABLE = False
|
||||
def _has_aot_awq_marlin_moe_repack() -> bool:
|
||||
return hasattr(torch.ops.sgl_kernel, "awq_marlin_moe_repack") and hasattr(
|
||||
torch.ops.sgl_kernel.awq_marlin_moe_repack, "default"
|
||||
)
|
||||
|
||||
|
||||
AOT_AVAILABLE = _has_aot_awq_marlin_moe_repack()
|
||||
|
||||
|
||||
def awq_pack(
|
||||
@@ -68,7 +70,9 @@ def test_awq_marlin_moe_repack_jit_vs_aot(
|
||||
perm = torch.empty((num_experts, 0), dtype=torch.int32, device="cuda")
|
||||
|
||||
out_jit = jit_awq_marlin_moe_repack(b_q_weight, perm, size_k, size_n, num_bits)
|
||||
out_aot = aot_awq_marlin_moe_repack(b_q_weight, perm, size_k, size_n, num_bits)
|
||||
out_aot = torch.ops.sgl_kernel.awq_marlin_moe_repack.default(
|
||||
b_q_weight, perm, size_k, size_n, num_bits
|
||||
)
|
||||
|
||||
torch.cuda.synchronize()
|
||||
|
||||
|
||||
@@ -9,12 +9,14 @@ from sglang.jit_kernel.awq_marlin_repack import (
|
||||
from sglang.srt.layers.quantization.utils import pack_cols, quantize_weights
|
||||
from sglang.test.test_marlin_utils import get_weight_perm, marlin_weights
|
||||
|
||||
try:
|
||||
from sgl_kernel import awq_marlin_repack as aot_awq_marlin_repack
|
||||
|
||||
AOT_AVAILABLE = True
|
||||
except ImportError:
|
||||
AOT_AVAILABLE = False
|
||||
def _has_aot_awq_marlin_repack() -> bool:
|
||||
return hasattr(torch.ops.sgl_kernel, "awq_marlin_repack") and hasattr(
|
||||
torch.ops.sgl_kernel.awq_marlin_repack, "default"
|
||||
)
|
||||
|
||||
|
||||
AOT_AVAILABLE = _has_aot_awq_marlin_repack()
|
||||
|
||||
|
||||
def awq_pack(
|
||||
@@ -58,7 +60,9 @@ def test_awq_marlin_repack_jit_vs_aot(num_bits, k_tiles, n_tiles, group_size):
|
||||
q_w_awq = awq_pack(q_w, num_bits, size_k, size_n)
|
||||
|
||||
out_jit = jit_awq_marlin_repack(q_w_awq, size_k, size_n, num_bits)
|
||||
out_aot = aot_awq_marlin_repack(q_w_awq, size_k, size_n, num_bits)
|
||||
out_aot = torch.ops.sgl_kernel.awq_marlin_repack.default(
|
||||
q_w_awq, size_k, size_n, num_bits
|
||||
)
|
||||
|
||||
torch.cuda.synchronize()
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ import itertools
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from sgl_kernel import moe_wna16_marlin_gemm as aot_moe_wna16_marlin_gemm
|
||||
from sgl_kernel.scalar_type import scalar_types
|
||||
|
||||
from sglang.jit_kernel.moe_wna16_marlin import moe_wna16_marlin_gemm
|
||||
@@ -10,6 +9,15 @@ from sglang.srt.layers.moe.fused_moe_triton import moe_align_block_size
|
||||
from sglang.test.test_marlin_utils import awq_marlin_quantize, marlin_quantize
|
||||
|
||||
|
||||
def _has_aot_moe_wna16_marlin_gemm() -> bool:
|
||||
return hasattr(torch.ops.sgl_kernel, "moe_wna16_marlin_gemm") and hasattr(
|
||||
torch.ops.sgl_kernel.moe_wna16_marlin_gemm, "default"
|
||||
)
|
||||
|
||||
|
||||
AOT_AVAILABLE = _has_aot_moe_wna16_marlin_gemm()
|
||||
|
||||
|
||||
def stack_and_dev(tensors: list[torch.Tensor]):
|
||||
dev = tensors[0].device
|
||||
return torch.stack(tensors, dim=0).to(dev)
|
||||
@@ -143,7 +151,7 @@ def _run_single_gemm_aot(
|
||||
is_k_full,
|
||||
use_atomic_add,
|
||||
):
|
||||
return aot_moe_wna16_marlin_gemm(
|
||||
return torch.ops.sgl_kernel.moe_wna16_marlin_gemm.default(
|
||||
a,
|
||||
c,
|
||||
qweight,
|
||||
@@ -224,6 +232,9 @@ TEST_CASES = generate_test_cases()
|
||||
def test_moe_wna16_marlin_gemm(
|
||||
m, n, k, e, topk, dtype, group_size, act_order, quant_type
|
||||
):
|
||||
if not AOT_AVAILABLE:
|
||||
pytest.skip("sgl_kernel moe_wna16_marlin_gemm AOT op not available")
|
||||
|
||||
torch.manual_seed(0)
|
||||
|
||||
has_zp = quant_type in [scalar_types.uint4, scalar_types.uint8]
|
||||
|
||||
Reference in New Issue
Block a user