Co-authored-by: kernel-design-agents <kernel-design-agents@users.noreply.github.com>
453 lines
18 KiB
Python
453 lines
18 KiB
Python
"""GPU-free import / registry / selector tests for ``sglang.kernels`` (RFC #29630)."""
|
|
|
|
import ast
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
import sglang.kernels as K
|
|
import sglang.kernels.fused_op as fo
|
|
import sglang.kernels.selector as sel
|
|
from sglang.kernels import KernelBackend, PlatformInfo
|
|
from sglang.kernels.spec import CapabilityRequirement as Cap
|
|
from sglang.test.ci.ci_register import register_cpu_ci
|
|
|
|
register_cpu_ci(est_time=24, suite="base-a-test-cpu")
|
|
|
|
_CPU = PlatformInfo(device_type="cpu")
|
|
_SM90 = PlatformInfo(device_type="cuda", cuda_arch_major=9, cuda_arch_minor=0)
|
|
_SM100 = PlatformInfo(device_type="cuda", cuda_arch_major=10, cuda_arch_minor=0)
|
|
_HIP = PlatformInfo(device_type="hip")
|
|
|
|
|
|
def test_single_backend_resolves_without_backend():
|
|
assert (
|
|
K.select_kernel("kvcache.reshape_and_cache_flash").backend
|
|
is KernelBackend.TRITON
|
|
)
|
|
|
|
|
|
def test_fp8_scaled_mm_requires_explicit_registry_backend(monkeypatch):
|
|
monkeypatch.setattr(sel, "_platform", lambda: _SM90)
|
|
with pytest.raises(ValueError, match="multiple backends"):
|
|
K.select_kernel("gemm.fp8_scaled_mm")
|
|
assert (
|
|
K.select_kernel("gemm.fp8_scaled_mm", backend=KernelBackend.AOT).backend
|
|
is KernelBackend.AOT
|
|
)
|
|
|
|
|
|
def test_unknown_op_or_backend_raises():
|
|
with pytest.raises(KeyError):
|
|
K.select_kernel("does_not.exist")
|
|
with pytest.raises(KeyError):
|
|
K.select_kernel("gemm.fp8_scaled_mm", backend=KernelBackend.TRITON)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"platform, eligible",
|
|
[
|
|
(_SM90, True),
|
|
(_SM100, True),
|
|
(PlatformInfo(device_type="cuda", cuda_arch_major=8, cuda_arch_minor=0), False),
|
|
(_CPU, False),
|
|
(_HIP, False),
|
|
],
|
|
)
|
|
def test_qwen_qkv_registry_accepts_hopper(platform, eligible):
|
|
spec = K.select_kernel("diffusion.qwen_qkv_epilogue")
|
|
assert spec.backend is KernelBackend.JIT
|
|
assert K.capabilities_satisfied(spec.capabilities, platform) is eligible
|
|
|
|
|
|
def test_multi_backend_requires_explicit_backend(monkeypatch):
|
|
# Device is a hard eligibility filter, not a ranking: >1 usable backend on
|
|
# the current device means selection must name one.
|
|
monkeypatch.setattr(sel, "_platform", lambda: _SM90)
|
|
with pytest.raises(ValueError):
|
|
K.select_kernel("activation.silu_and_mul")
|
|
spec = K.select_kernel("activation.silu_and_mul", backend=KernelBackend.JIT)
|
|
assert spec.backend is KernelBackend.JIT
|
|
assert spec.target == "sglang.kernels.ops.activation:_SILU_AND_MUL.forward_jit"
|
|
|
|
|
|
@pytest.mark.parametrize("device, expect", [("cuda", "jit"), ("hip", "aot")])
|
|
def test_activation_default_backend(monkeypatch, device, expect):
|
|
# silu_and_mul default matches production: jit on CUDA, aot (sgl_kernel) on HIP.
|
|
from sglang.kernels.ops.activation import _SILU_AND_MUL
|
|
|
|
monkeypatch.setattr(fo, "_platform", lambda: PlatformInfo(device_type=device))
|
|
assert _SILU_AND_MUL.auto_selected_backend().value == expect
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"req, plat, ok",
|
|
[
|
|
(Cap.CUDA, _CPU, False),
|
|
(Cap.CUDA, _SM90, True),
|
|
(Cap.CUDA, _HIP, False),
|
|
(Cap.HIP, _HIP, True),
|
|
(Cap.cuda(min_sm=(10, 0)), _SM90, False),
|
|
(Cap.cuda(min_sm=(10, 0)), _SM100, True),
|
|
(Cap.cuda(max_sm=(9, 0)), _SM100, False),
|
|
],
|
|
)
|
|
def test_capability_is_satisfied_by(req, plat, ok):
|
|
assert req.is_satisfied_by(plat) is ok
|
|
|
|
|
|
def test_capabilities_or_semantics():
|
|
both = {Cap.CUDA, Cap.HIP}
|
|
assert K.capabilities_satisfied(both, _SM90)
|
|
assert K.capabilities_satisfied(both, _HIP)
|
|
assert not K.capabilities_satisfied(both, _CPU)
|
|
assert K.capabilities_satisfied((), _CPU) # empty = unrestricted
|
|
assert K.capabilities_satisfied(Cap.CUDA, _SM90) # single tolerated
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"relative_path",
|
|
(
|
|
"srt/utils/common.py",
|
|
"multimodal_gen/runtime/utils/common.py",
|
|
),
|
|
)
|
|
def test_amx_backend_probe_is_lazy(relative_path):
|
|
loader = (
|
|
"package = importlib.util.find_spec('sglang'); "
|
|
"path = pathlib.Path(next(iter(package.submodule_search_locations))) / "
|
|
f"{relative_path!r}; "
|
|
"spec = importlib.util.spec_from_file_location('_common_under_test', path); "
|
|
"module = importlib.util.module_from_spec(spec); "
|
|
"sys.modules[spec.name] = module; "
|
|
"spec.loader.exec_module(module)"
|
|
)
|
|
code = "; ".join(
|
|
(
|
|
"import builtins, importlib.util, pathlib, sys",
|
|
"from unittest import mock",
|
|
"real_import = builtins.__import__",
|
|
"import_mock = mock.Mock(wraps=real_import)",
|
|
"builtins.__import__ = import_mock",
|
|
loader,
|
|
"builtins.__import__ = real_import",
|
|
"attempted = any(call.args and call.args[0] == 'sgl_kernel' "
|
|
"for call in import_mock.call_args_list)",
|
|
"print('DIRTY' if attempted else 'CLEAN')",
|
|
)
|
|
)
|
|
result = subprocess.run(
|
|
[sys.executable, "-c", code], capture_output=True, text=True
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
assert "CLEAN" in result.stdout
|
|
|
|
|
|
def test_import_stays_metadata_only():
|
|
# Importing the namespace must not pull in the AOT backend (sgl_kernel) or
|
|
# the JIT compilation infra (sglang.kernels.jit), which import torch / nvcc.
|
|
code = (
|
|
"import sys, sglang.kernels.ops; "
|
|
"print('DIRTY' if any(m in sys.modules for m in "
|
|
"('sgl_kernel', 'cutlass', 'flydsl', 'aiter', "
|
|
"'sglang.kernels.ops.gemm.kimi_k3', "
|
|
"'sglang.kernels.ops.activation.softcap', "
|
|
"'sglang.kernels.ops.attention.dllm_kv_pack', "
|
|
"'sglang.kernels.ops.attention.flash_attention_v4', "
|
|
"'sglang.kernels.ops.attention.gemma_qkv_norm_rope', "
|
|
"'sglang.kernels.ops.layernorm.rmsnorm_fanout', "
|
|
"'sglang.kernels.ops.speculative.row_argmax')) or any("
|
|
"m.startswith('sglang.kernels.jit') for m in sys.modules) else 'CLEAN')"
|
|
)
|
|
r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
|
|
assert r.returncode == 0, r.stderr
|
|
assert "CLEAN" in r.stdout
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"op, backend, device",
|
|
[
|
|
("activation.situ_and_mul", KernelBackend.JIT, "cuda"),
|
|
("moe.situ_and_mul_masked_post_quant", KernelBackend.JIT, "cuda"),
|
|
("attention.attn_res_fused_tma", KernelBackend.JIT, "cuda"),
|
|
("attention.attn_res_hip", KernelBackend.TRITON, "hip"),
|
|
("elementwise.kimi_k3_mla_output_gate", KernelBackend.JIT, "cuda"),
|
|
("attention.fused_kda_decode_mtp_dspark", KernelBackend.CUTE_DSL, "cuda"),
|
|
("attention.flydsl_kimi_k3_kda_decode_with_f_b", KernelBackend.FLYDSL, "hip"),
|
|
("communication.all_reduce_push_res", KernelBackend.JIT, "cuda"),
|
|
("communication.gemm_ag_up_proj", KernelBackend.JIT, "cuda"),
|
|
("communication.reduce_scatter_res", KernelBackend.JIT, "cuda"),
|
|
],
|
|
)
|
|
def test_kimi_k3_kernels_are_inventoried_by_operator(op, backend, device):
|
|
spec = K.select_kernel(op, backend=backend)
|
|
group = op.split(".")[0]
|
|
assert spec.target.startswith(f"sglang.kernels.ops.{group}.")
|
|
assert K.capabilities_satisfied(spec.capabilities, PlatformInfo(device_type=device))
|
|
assert not K.capabilities_satisfied(spec.capabilities, _CPU)
|
|
|
|
|
|
def _readme() -> str:
|
|
return (Path(K.__file__).resolve().parent / "README.md").read_text()
|
|
|
|
|
|
def test_readme_operator_groups_match_registry():
|
|
"""The README group list is the documented contract for ``ops._GROUPS``."""
|
|
paragraph = _readme().split("Operator groups", 1)[1].split("\n\n", 1)[0]
|
|
documented = re.findall(r"`([a-z_0-9]+)`", paragraph)
|
|
assert len(documented) == len(set(documented)), documented
|
|
assert set(documented) == set(K.ops.__all__) == set(K.ops._GROUPS)
|
|
|
|
|
|
def test_readme_layout_lists_every_top_level_directory():
|
|
"""A new directory directly under ``kernels/`` must be documented first."""
|
|
layout = _readme().split("## Layout", 1)[1].split("```", 2)[1]
|
|
documented = set(re.findall(r"^ ([a-z_0-9]+)/", layout, re.MULTILINE))
|
|
root = Path(K.__file__).resolve().parent
|
|
actual = {p.name for p in root.iterdir() if p.is_dir() and p.name != "__pycache__"}
|
|
assert documented == actual
|
|
|
|
|
|
def test_operator_and_test_groups_agree():
|
|
"""A new root-level model bundle must not bypass logical op grouping."""
|
|
root = Path(K.__file__).resolve().parents[3]
|
|
ops = root / "python/sglang/kernels/ops"
|
|
groups = set(K.ops.__all__)
|
|
assert {p.name for p in ops.glob("*.py")} == {"__init__.py"}
|
|
assert {p.name for p in ops.iterdir() if (p / "__init__.py").is_file()} == groups
|
|
for kind in ("ops", "benchmark"):
|
|
tests = root / "test/registered/kernels" / kind
|
|
actual = {p.relative_to(tests).parts[0] for p in tests.rglob("*.py")}
|
|
assert actual <= groups, actual - groups
|
|
|
|
|
|
def test_agent_kernels_register_their_backend():
|
|
"""Agent-generated kernels are listed by backend, so directory and backend must agree."""
|
|
agent_backends = {
|
|
"sglang.kernels.kda_kernels.": KernelBackend.KDA,
|
|
"sglang.kernels.cake_kernels.": KernelBackend.CAKE,
|
|
}
|
|
for spec in K.registry.all_specs():
|
|
expected = next(
|
|
(b for p, b in agent_backends.items() if spec.target.startswith(p)), None
|
|
)
|
|
if expected is not None or spec.backend in agent_backends.values():
|
|
assert spec.backend is expected, spec
|
|
|
|
|
|
def test_inventory_targets_survive_module_moves():
|
|
"""Moving a module without its lazy target otherwise breaks only on first call."""
|
|
root = Path(K.__file__).resolve().parents[3] / "python"
|
|
for spec in K.registry.all_specs():
|
|
module, _ = spec.target.split(":", 1)
|
|
if not module.startswith("sglang.kernels.ops."):
|
|
continue
|
|
path = root.joinpath(*module.split("."))
|
|
assert path.with_suffix(".py").is_file() or (path / "__init__.py").is_file(), (
|
|
spec
|
|
)
|
|
|
|
|
|
def test_reclassified_public_entry_points_are_inventoried():
|
|
"""Public compute functions in these formerly unregistered modules need metadata."""
|
|
root = Path(K.__file__).resolve().parent / "ops"
|
|
targets = {spec.target for spec in K.registry.all_specs()}
|
|
modules = (
|
|
"attention.dllm_kv_pack",
|
|
"attention.gemma_qkv_norm_rope",
|
|
"attention.minicpm_sala.get_block_table",
|
|
"attention.fast_topk",
|
|
"attention.dsa.kpool_topk_transform",
|
|
"attention.dsa.indexer_k",
|
|
"attention.dsv4.wo_a",
|
|
"attention.helion.kda_replayssm",
|
|
"attention.helion.kda_verify",
|
|
"attention.minimax_sparse.common.index",
|
|
"attention.minimax_sparse.decode.indexer_cp",
|
|
"attention.minimax_sparse.decode.topk_sparse",
|
|
"attention.minimax_sparse.prefill.gluon_prefill",
|
|
"attention.minimax_sparse.prefill.topk_sparse",
|
|
"embeddings.qwen4_ngram",
|
|
"elementwise.qwen4_gate",
|
|
"elementwise.row_scale",
|
|
"elementwise.hc_combine_decode",
|
|
"mamba.qwen4_short_conv",
|
|
"mamba.lfm_short_conv",
|
|
"mamba.kda_prefill_track",
|
|
"gemm.dsv4_wo_a",
|
|
"gemm.inkling_rel_proj",
|
|
"gemm.hopper_bf16_gemv",
|
|
"gemm.sm120_fp8_gemv",
|
|
"gemm.fp8_blockwise_gemm",
|
|
"gemm.gptq_marlin",
|
|
"layernorm.rmsnorm_hf",
|
|
"layernorm.rmsnorm_fanout",
|
|
"layernorm.grouped_gemma_rmsnorm",
|
|
"moe.dsv4",
|
|
"activation.silu_and_mul_clamp",
|
|
"moe.gemma4_routing",
|
|
"moe.shared_expert_gate",
|
|
"memory.adler32",
|
|
"memory.row_compact",
|
|
"kv_canary.plan.api",
|
|
"kv_canary.write",
|
|
"kv_canary.verify",
|
|
"kv_canary.scatter_req_token_ids",
|
|
"mm.process.image",
|
|
"speculative.row_argmax",
|
|
)
|
|
for module in modules:
|
|
tree = ast.parse(
|
|
root.joinpath(*module.split(".")).with_suffix(".py").read_text()
|
|
)
|
|
for node in tree.body:
|
|
if not isinstance(node, ast.FunctionDef):
|
|
continue
|
|
if (
|
|
node.name.startswith(("_", "can_", "is_", "use_"))
|
|
or node.name == "make_name"
|
|
):
|
|
continue
|
|
assert f"sglang.kernels.ops.{module}:{node.name}" in targets
|
|
|
|
|
|
def _module_defines(module: str, attr: str) -> bool:
|
|
"""AST check that ``module`` defines ``attr`` (``name`` or ``name.method``).
|
|
|
|
For ``instance.method`` the instance's class must define ``method``.
|
|
"""
|
|
root = Path(K.__file__).resolve().parents[2]
|
|
tree = ast.parse(root.joinpath(*module.split(".")).with_suffix(".py").read_text())
|
|
top = {}
|
|
for node in tree.body:
|
|
if isinstance(node, (ast.FunctionDef, ast.ClassDef)):
|
|
top[node.name] = node
|
|
elif isinstance(node, ast.Assign):
|
|
for target in node.targets:
|
|
if isinstance(target, ast.Name):
|
|
top[target.id] = node.value
|
|
name, _, method = attr.partition(".")
|
|
node = top.get(name)
|
|
if node is None or not method:
|
|
return node is not None
|
|
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
|
|
node = top.get(node.func.id)
|
|
return isinstance(node, ast.ClassDef) and any(
|
|
isinstance(item, ast.FunctionDef) and item.name == method for item in node.body
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"op, backends",
|
|
[
|
|
("attention.muse_spark_qk_norm_rope_store", {"torch", "triton", "helion"}),
|
|
("communication.muse_spark_all_gather", {"jit"}),
|
|
("communication.muse_spark_all_gather_norm_residual", {"jit"}),
|
|
(
|
|
"communication.muse_spark_moe_finalize_all_reduce",
|
|
{"torch", "cute_dsl", "helion"},
|
|
),
|
|
("communication.all_gather_matmul", {"cute_dsl"}),
|
|
("communication.matmul_reduce_scatter", {"cute_dsl"}),
|
|
("elementwise.row_sum", {"jit"}),
|
|
("gemm.muse_spark_router_gemm", {"jit"}),
|
|
("gemm.muse_spark_router_gemv", {"cute_dsl"}),
|
|
("layernorm.muse_spark_norm_gate", {"torch", "jit", "helion", "triton"}),
|
|
("layernorm.muse_spark_norm_residual", {"torch", "jit", "triton"}),
|
|
("layernorm.muse_spark_rmsnorm_nvfp4_quant", {"torch", "jit", "helion"}),
|
|
("moe.muse_spark_routing", {"torch", "jit", "triton"}),
|
|
("moe.muse_spark_post_expert_finalize", {"triton"}),
|
|
],
|
|
)
|
|
def test_muse_spark_kernels_are_inventoried_by_operator(op, backends):
|
|
specs = [s for s in K.registry.all_specs() if s.op == op]
|
|
assert {s.backend.value for s in specs} == backends
|
|
group = op.split(".")[0]
|
|
for spec in specs:
|
|
module, attr = spec.target.split(":", 1)
|
|
assert module.startswith(f"sglang.kernels.ops.{group}."), spec
|
|
assert _module_defines(module, attr), spec
|
|
if spec.backend is not KernelBackend.TORCH:
|
|
assert not K.capabilities_satisfied(spec.capabilities, _CPU), spec
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"op, sm, expected",
|
|
[
|
|
("gemm.hopper_bf16_gemv", (9, 0), True),
|
|
("gemm.hopper_bf16_gemv", (10, 0), False),
|
|
("elementwise.hc_combine_apply_norm", (8, 9), False),
|
|
("elementwise.hc_combine_apply_norm", (9, 0), True),
|
|
("communication.moe_finalize_shared_gate_all_reduce", (8, 9), False),
|
|
("communication.moe_finalize_shared_gate_all_reduce", (10, 3), True),
|
|
("gemm.fp8_blockwise_scaled_mm", (12, 0), True),
|
|
("gemm.fp8_blockwise_scaled_mm", (12, 1), True),
|
|
("gemm.fp8_blockwise_scaled_mm", (10, 0), False),
|
|
("attention.fused_rope_wo_a_bf16", (9, 0), False),
|
|
("attention.fused_rope_wo_a_bf16", (10, 0), True),
|
|
("attention.fused_rope_wo_a_bf16", (10, 3), True),
|
|
("attention.fused_rope_wo_a_bf16", (12, 0), False),
|
|
("attention.deep_select_topk", (8, 0), False),
|
|
("attention.deep_select_topk", (9, 0), True),
|
|
("attention.deep_select_topk", (10, 0), True),
|
|
("attention.deep_select_topk", (10, 1), False),
|
|
("attention.deep_select_topk", (10, 3), True),
|
|
("attention.deep_select_topk", (12, 0), False),
|
|
("attention.chunk_kda_fwd", (9, 0), False),
|
|
("attention.chunk_kda_fwd", (10, 0), True),
|
|
("attention.chunk_kda_fwd", (10, 1), False),
|
|
("attention.chunk_kda_fwd", (10, 3), True),
|
|
("attention.chunk_kda_fwd", (12, 0), False),
|
|
("attention.absorbed_bmm_concat_cast_q_fp8", (9, 0), True),
|
|
("attention.absorbed_bmm_concat_cast_q_fp8", (10, 0), False),
|
|
("gemm.convrot_int8_fused_linear", (8, 9), False),
|
|
("gemm.convrot_int8_fused_linear", (9, 0), True),
|
|
("gemm.convrot_int8_fused_linear", (10, 0), True),
|
|
("gemm.convrot_int8_fused_linear", (10, 3), False),
|
|
("gemm.convrot_int8_fused_linear", (12, 0), True),
|
|
("gemm.convrot_int8_fused_linear", (12, 1), True),
|
|
("gemm.convrot_int8_fused_linear", (12, 2), False),
|
|
],
|
|
)
|
|
def test_registered_architecture_boundaries(op, sm, expected):
|
|
spec = K.registry.get_backend(op, KernelBackend.JIT)
|
|
platform = PlatformInfo(
|
|
device_type="cuda", cuda_arch_major=sm[0], cuda_arch_minor=sm[1]
|
|
)
|
|
assert K.capabilities_satisfied(spec.capabilities, platform) is expected
|
|
assert not K.capabilities_satisfied(spec.capabilities, _CPU)
|
|
|
|
|
|
def test_convrot_int8_specs_match_wrapper_capabilities():
|
|
"""The gemm group cannot import the wrapper, so this pins its three registry
|
|
entries to the wrapper's SUPPORTED_CAPABILITIES table."""
|
|
from sglang.kernels.ops.gemm import convrot_int8
|
|
|
|
expected = frozenset(
|
|
Cap.cuda(min_sm=sm, max_sm=sm) for sm in convrot_int8.SUPPORTED_CAPABILITIES
|
|
)
|
|
for op in (
|
|
"gemm.convrot_rotate_quantize_activation",
|
|
"gemm.convrot_int8_fused_linear",
|
|
"gemm.convrot_int8_linear_prequant",
|
|
):
|
|
spec = K.registry.get_backend(op, KernelBackend.JIT)
|
|
assert spec.capabilities == expected, op
|
|
|
|
|
|
def test_deep_select_spec_matches_wrapper_architectures():
|
|
"""The group cannot import the wrapper, so this is the only link between the two SM lists."""
|
|
from sglang.kernels.ops.attention import deep_select
|
|
|
|
spec = K.registry.get_backend("attention.deep_select_topk", KernelBackend.JIT)
|
|
assert spec.capabilities == frozenset(
|
|
Cap.cuda(min_sm=sm, max_sm=sm) for sm in deep_select.SUPPORTED_CUDA_ARCHS
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(pytest.main([__file__]))
|