From 38ec649048df9b7d4d2c3355a55d7eb724f572f3 Mon Sep 17 00:00:00 2001 From: Xiaoyu Zhang <1182563586@qq.com> Date: Sat, 26 Sep 2026 12:30:44 +0800 Subject: [PATCH] [Refactor] Restore logical kernel groups and test organization (#41243) --- .claude/skills/add-jit-kernel/SKILL.md | 34 +- .claude/skills/add-sgl-kernel/SKILL.md | 19 + .claude/skills/kernel-organization/SKILL.md | 82 + .claude/skills/write-sglang-test/SKILL.md | 4 + .../triton_flashinfer_cudnn.py | 2 +- .../deepseek/benchmark_deepgemm_fp8_gemm.py | 2 +- .../benchmark_deepgemm_fp8_gemm_blackwell.py | 6 +- benchmark/kernels/quantization/README.md | 2 +- .../quantization/tuning_block_wise_kernel.py | 6 +- python/sglang/kernels/README.md | 29 +- .../kernels/jit/csrc/kimi_k3/comm/gemm_ar.cuh | 1577 ----------------- python/sglang/kernels/ops/__init__.py | 1 - .../sglang/kernels/ops/attention/__init__.py | 62 +- .../ops/attention/dsa/dequant_k_cache.py | 4 - .../dsa/indexer_k.py} | 0 .../dsa}/kpool_topk_transform.py | 0 .../ops/attention/dsa/quant_k_cache.py | 125 -- .../ops/attention/dsa/transform_index.py | 11 - .../kernels/ops/attention/dsv4/__init__.py | 16 - .../ops/attention/dsv4/dequant_k_cache.py | 30 - .../sglang/kernels/ops/attention/dsv4/wo_a.py | 133 -- .../{elementwise => attention}/fast_topk.py | 0 .../kernels/ops/attention/log_scaling_tau.py | 2 +- .../{ => attention}/minicpm_sala/__init__.py | 8 +- .../minicpm_sala/get_block_table.py | 0 .../kernels/ops/communication/__init__.py | 1 - .../kernels/ops/communication/gemm_ar.py | 216 --- .../kernels/ops/elementwise/__init__.py | 30 + .../kernels/ops/elementwise/qwen4_gate.py | 155 ++ .../row_scale.py} | 12 - .../sglang/kernels/ops/embeddings/__init__.py | 13 +- .../kernels/ops/embeddings/qwen4_ngram.py | 114 ++ python/sglang/kernels/ops/gemm/__init__.py | 137 ++ .../dsv4/gemm.py => gemm/bf16_fp32.py} | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...0,dtype=fp8_w8a8,block_shape=[32, 32].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...0,dtype=fp8_w8a8,block_shape=[32, 32].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...VIDIA_L40S,dtype=fp8_w8a8_channelwise.json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...0,dtype=fp8_w8a8,block_shape=[32, 32].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...VIDIA_L40S,dtype=fp8_w8a8_channelwise.json | 0 ...0,dtype=fp8_w8a8,block_shape=[32, 32].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...VIDIA_L40S,dtype=fp8_w8a8_channelwise.json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...0,dtype=fp8_w8a8,block_shape=[32, 32].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...0,dtype=fp8_w8a8,block_shape=[32, 32].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...VIDIA_L40S,dtype=fp8_w8a8_channelwise.json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...type=int8_w8a8,block_shape=[128, 128].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 ...0,dtype=fp8_w8a8,block_shape=[32, 32].json | 0 ...dtype=fp8_w8a8,block_shape=[128, 128].json | 0 .../{quantization => gemm}/configs/README.md | 0 python/sglang/kernels/ops/gemm/dsv4_wo_a.py | 137 ++ python/sglang/kernels/ops/gemm/fp8_kernel.py | 1134 ++++++++++++ .../ops/{quantization => gemm}/gptq_marlin.py | 0 .../ops/gemm/hc_mix.py} | 0 .../{attention => gemm}/inkling_rel_proj.py | 0 python/sglang/kernels/ops/gemm/int8_kernel.py | 229 +++ .../sglang/kernels/ops/layernorm/__init__.py | 27 +- .../kernels/ops/layernorm/gemma4_fused_ops.py | 183 -- .../dsv4 => layernorm}/rms_normalize_hip.py | 0 python/sglang/kernels/ops/mamba/__init__.py | 35 + .../qwen4_short_conv.py} | 261 +-- python/sglang/kernels/ops/memory/__init__.py | 37 +- .../sglang/kernels/ops/memory/row_compact.py | 20 + python/sglang/kernels/ops/mm/__init__.py | 24 +- python/sglang/kernels/ops/moe/__init__.py | 66 + .../{attention/dsv4/moe.py => moe/dsv4.py} | 6 +- .../sglang/kernels/ops/moe/gemma4_routing.py | 186 ++ .../kernels/ops/quantization/__init__.py | 14 +- .../ops/quantization/dsv32/__init__.py | 11 - .../dsv4 => quantization}/fp8_cvt.py | 6 +- .../kernels/ops/quantization/fp8_kernel.py | 1117 +----------- .../kernels/ops/quantization/int8_kernel.py | 223 +-- python/sglang/srt/environ.py | 10 +- .../srt/layers/attention/dsa/dsa_indexer.py | 4 +- .../layers/attention/dsa/kpool_fp8_index.py | 2 +- .../srt/layers/attention/dsv4/compress_hip.py | 2 +- .../srt/layers/attention/dsv4/compressor.py | 7 +- .../srt/layers/attention/dsv4/dsv41_sparse.py | 2 +- .../srt/layers/attention/minicpm/backend.py | 2 +- .../sglang/srt/layers/attention/qsa/kernel.py | 2 +- .../srt/layers/attention/qsa/qsa_indexer.py | 2 +- python/sglang/srt/layers/attn_residual.py | 4 +- .../srt/layers/communication/__init__.py | 1 + .../{ => communication}/k3_ar_fusion.py | 0 .../{ => communication}/k3_sp_collective.py | 2 +- python/sglang/srt/layers/hyperconnection.py | 2 +- python/sglang/srt/layers/k3_gemm_ar.py | 93 - python/sglang/srt/layers/moe/cutlass_moe.py | 2 +- python/sglang/srt/layers/moe/hash_topk.py | 2 +- python/sglang/srt/layers/moe/mega_moe.py | 2 +- .../srt/layers/moe/moe_runner/deep_gemm.py | 5 +- .../srt/layers/moe/moe_runner/humming.py | 4 +- .../moe/moe_runner/triton_utils/fused_moe.py | 2 +- python/sglang/srt/layers/moe/topk.py | 2 +- .../srt/layers/quantization/expert_pack.py | 2 +- .../srt/layers/quantization/fp8_utils.py | 12 +- .../srt/layers/quantization/int8_utils.py | 6 +- .../srt/layers/quantization/marlin_utils.py | 6 +- .../layers/quantization/marlin_utils_fp4.py | 2 +- .../layers/quantization/marlin_utils_fp8.py | 2 +- .../model_runner_components/weight_updater.py | 2 +- python/sglang/srt/models/deepseek_v2.py | 4 +- python/sglang/srt/models/deepseek_v4.py | 6 +- python/sglang/srt/models/gemma4_causal.py | 4 +- .../sglang/srt/models/inkling_common/attn.py | 4 +- python/sglang/srt/models/kimi_k3.py | 12 +- python/sglang/srt/models/longcat_flash.py | 8 +- python/sglang/srt/models/qwen4_exp.py | 10 +- .../sglang/test/kernels/fp8.py | 64 +- scripts/ci/cuda/warmup_deep_gemm.py | 2 +- .../attention/check_dsa_dequant_k_cache.py | 2 + .../attention/check_dsa_quant_k_cache.py | 133 ++ .../attention/check_dsa_transform_index.py | 16 + .../attention/check_dsv4_dequant_k_cache.py | 37 + .../attention}/bench_gdn_replayssm_fold.py | 2 +- test/manual/quant/test_awq_archived.py | 2 +- test/manual/quant/test_block_fp8.py | 3 +- .../test_autoround_quantization.py | 0 .../{quant => e2e/quantization}/test_awq.py | 0 .../quantization}/test_fp8kv_triton.py | 0 .../{quant => e2e/quantization}/test_gguf.py | 0 .../quantization}/test_gptqmodel_dynamic.py | 0 .../quantization}/test_int4fp8_moe.py | 0 .../test_llama8b_nvfp4_kv_cache_sm120.py | 0 .../quantization}/test_modelopt_fp8.py | 0 .../quantization}/test_nvfp4_gemm_sm120.py | 0 .../quantization}/test_quark_mxfp4.py | 0 .../quantization}/test_w8a8_quantization.py | 1 - .../minicpm_sala/bench_get_block_table.py | 2 +- .../gemm/bench_bf16xfp32_router_gemm.py | 2 +- .../{moe => sampling}/bench_renorm.py | 0 .../activation/test_fused_op_gpu_parity.py | 55 + .../minicpm_sala/test_get_block_table.py | 2 +- .../minicpm_sala/test_minicpm_compress_k.py | 0 .../ops/{ => attention}/qsa/test_qsa.py | 0 .../{ => attention}/qsa/test_qsa_indexer.py | 0 .../qsa/test_qsa_strided_zero_fill.py | 0 .../attention/test_dsv32_indexer_fusion.py | 8 +- .../test_fast_topk.py | 2 +- .../test_kpool_topk_transform_fused.py | 4 +- .../ops/attention/test_mla_scatter_concat.py | 100 ++ .../ops/attention/test_replayssm_ring_fold.py | 142 ++ .../kernels/ops/attention/test_vision_rope.py | 65 + .../ops/communication/test_collectives.py | 38 - .../test_nvfp4_scaled_mm.py} | 0 .../ops/dllm/test_gemma4_sampling_cuda.py | 127 -- .../kernels/ops/elementwise/test_add3.py | 26 + .../test_row_scale.py} | 26 +- .../kernels/ops/gemm/test_dsv4_wo_a.py | 85 + .../kernels/ops/gemm/test_fp8_kernel.py | 46 + .../test_gptq_marlin.py | 2 +- .../test_hc_mix.py} | 2 +- .../ops}/gemm/test_hopper_bf16_gemv.py | 0 .../test_inkling_rel_proj.py | 2 +- .../ops}/gemm/test_linear_bf16_fp32_hpc.py | 2 +- .../test_nvfp4_marlin.py | 0 .../ops/gemm/test_tiny_gemm_variants.py | 35 + .../ops/gemm}/test_triton_scaled_mm.py | 2 +- .../kimi_k3/test_kimi_k3_prerequisite_ops.py | 496 ------ .../ops/layernorm/test_fused_op_gpu_parity.py | 18 +- .../test_fused_rms_fp8_group_quant.py | 0 .../ops/layernorm/test_gemma4_qkv_norm.py | 57 + .../ops/layernorm/test_mxfp8_epilogue.py | 57 +- .../test_verify_commit_triton.py | 0 .../kernels/ops/memory/test_row_compact.py | 38 + test/registered/kernels/ops/mm/test_image.py | 44 + .../ops/moe}/test_block_int8.py | 0 .../ops/moe/test_gemma4_expert_activation.py | 62 + .../test_gemma4_fused_routing.py | 2 +- .../ops/moe/test_humming_fp8_dispatch.py | 7 +- .../ops/moe}/test_int8_kernel.py | 0 .../ops/moe}/test_marlin_moe.py | 0 .../ops/moe/test_mega_moe_pre_dispatch.py | 2 +- .../ops/moe/test_route_quant_prerequisites.py | 164 ++ .../ops/quantization}/test_awq_dequant.py | 0 .../ops/quantization}/test_dsv4_fp8_cast.py | 2 +- .../ops/quantization/test_fp8_kernel.py | 34 + .../test_per_token_group_quant.py | 2 +- .../ops/{moe => sampling}/test_renorm.py | 0 .../kernels/ops/sampling/test_top_p_renorm.py | 38 + .../configs}/test_quant_config_parsing.py | 0 .../disaggregation/test_kv_checksum.py | 0 .../algorithm/test_gemma4_sampling_cuda.py | 43 + .../kernels}/test_fused_op.py | 0 .../kernels}/test_kernels_namespace.py | 96 +- .../test_fp8_blockwise_linear_backends.py | 5 +- .../layers/quantization}/test_fp8_utils.py | 0 .../quantization}/test_is_layer_skipped.py | 0 .../quantization}/test_nvfp4_embedding.py | 0 .../pool_host}/test_kvcacheio_asymmetric.py | 0 .../test_unified_swa_tail_allocation.py | 0 .../runner_backend}/test_cuda_graph_dedup.py | 0 .../test_compensated_mhc_update_guard.py | 2 +- .../test_derived_weight_cache.py | 2 +- .../test_modelopt_fp8_scaled_mm.py} | 0 357 files changed, 4018 insertions(+), 4973 deletions(-) create mode 100644 .claude/skills/kernel-organization/SKILL.md delete mode 100644 python/sglang/kernels/jit/csrc/kimi_k3/comm/gemm_ar.cuh rename python/sglang/kernels/ops/{quantization/dsv32/elementwise.py => attention/dsa/indexer_k.py} (100%) rename python/sglang/kernels/ops/{moe => attention/dsa}/kpool_topk_transform.py (100%) rename python/sglang/kernels/ops/{elementwise => attention}/fast_topk.py (100%) rename python/sglang/kernels/ops/{ => attention}/minicpm_sala/__init__.py (66%) rename python/sglang/kernels/ops/{ => attention}/minicpm_sala/get_block_table.py (100%) delete mode 100644 python/sglang/kernels/ops/communication/gemm_ar.py create mode 100644 python/sglang/kernels/ops/elementwise/qwen4_gate.py rename python/sglang/kernels/ops/{attention/inkling_row_scale.py => elementwise/row_scale.py} (71%) create mode 100644 python/sglang/kernels/ops/embeddings/qwen4_ngram.py rename python/sglang/kernels/ops/{attention/dsv4/gemm.py => gemm/bf16_fp32.py} (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1024,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1152,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1280,K=5120,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1536,K=1536,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1536,K=1536,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1536,K=1536,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1536,K=1536,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1536,K=1536,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1536,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1536,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1536,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1536,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1536,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1536,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1536,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1536,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1536,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=1792,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=2048,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=2048,K=4096,device_name=NVIDIA_L40,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=2048,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=2048,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=2048,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=2048,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=2048,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=2048,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=2304,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=2304,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=2304,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=2304,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=2304,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=2304,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=24576,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=24576,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=24576,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=24576,K=4096,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=24576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=24576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=24576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=24576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=24576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=24576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=256,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=256,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=256,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=256,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=256,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=25600,K=6144,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=3072,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=3072,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=3072,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=3072,K=1536,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=3072,K=1536,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=3072,K=1536,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=3072,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=3072,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=3072,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=3072,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=32768,K=1024,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=32768,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=32768,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=32768,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=32768,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=32768,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=32768,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=32768,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=32768,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=32768,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=36864,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=36864,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4096,K=12288,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4096,K=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4096,K=2048,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4096,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4096,K=4096,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4096,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4096,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4096,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4096,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4096,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4096,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4096,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4096,K=8192,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4608,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4608,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4608,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4608,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4608,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4608,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=4608,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=512,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=512,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=512,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=512,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=512,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=512,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=512,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=5120,K=1024,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=5120,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=5120,K=2048,device_name=NVIDIA_L40,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=5120,K=3200,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=5120,K=576,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=576,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=576,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=576,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=6144,K=4096,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=6400,K=5120,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=1024,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=1024,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=1024,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=1152,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=1152,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=1152,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=1152,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=1152,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=1152,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=16384,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=16384,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=16384,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=16384,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=16384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=16384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=16384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=16384,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=18432,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=18432,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=18432,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=18432,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=18432,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=18432,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=2048,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=2048,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=2048,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=2048,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=2304,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=2304,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=2304,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=2304,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=2304,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=2304,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=2304,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=7168,K=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=8192,K=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/N=8192,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/configs/README.md (100%) create mode 100644 python/sglang/kernels/ops/gemm/dsv4_wo_a.py create mode 100644 python/sglang/kernels/ops/gemm/fp8_kernel.py rename python/sglang/kernels/ops/{quantization => gemm}/gptq_marlin.py (100%) rename python/sglang/{srt/layers/hc_mix_triton.py => kernels/ops/gemm/hc_mix.py} (100%) rename python/sglang/kernels/ops/{attention => gemm}/inkling_rel_proj.py (100%) create mode 100644 python/sglang/kernels/ops/gemm/int8_kernel.py rename python/sglang/kernels/ops/{attention/dsv4 => layernorm}/rms_normalize_hip.py (100%) rename python/sglang/kernels/ops/{qwen4_ple.py => mamba/qwen4_short_conv.py} (54%) create mode 100644 python/sglang/kernels/ops/memory/row_compact.py rename python/sglang/kernels/ops/{attention/dsv4/moe.py => moe/dsv4.py} (99%) create mode 100644 python/sglang/kernels/ops/moe/gemma4_routing.py delete mode 100644 python/sglang/kernels/ops/quantization/dsv32/__init__.py rename python/sglang/kernels/ops/{attention/dsv4 => quantization}/fp8_cvt.py (95%) create mode 100644 python/sglang/srt/layers/communication/__init__.py rename python/sglang/srt/layers/{ => communication}/k3_ar_fusion.py (100%) rename python/sglang/srt/layers/{ => communication}/k3_sp_collective.py (99%) delete mode 100644 python/sglang/srt/layers/k3_gemm_ar.py rename test/registered/quant/test_fp8_kernel.py => python/sglang/test/kernels/fp8.py (53%) create mode 100644 test/manual/kernels/attention/check_dsa_dequant_k_cache.py create mode 100644 test/manual/kernels/attention/check_dsa_quant_k_cache.py create mode 100644 test/manual/kernels/attention/check_dsa_transform_index.py create mode 100644 test/manual/kernels/attention/check_dsv4_dequant_k_cache.py rename {python/sglang/kernels/ops/attention/fla => test/manual/kernels/benchmark/attention}/bench_gdn_replayssm_fold.py (98%) rename test/registered/{quant => e2e/quantization}/test_autoround_quantization.py (100%) rename test/registered/{quant => e2e/quantization}/test_awq.py (100%) rename test/registered/{quant => e2e/quantization}/test_fp8kv_triton.py (100%) rename test/registered/{quant => e2e/quantization}/test_gguf.py (100%) rename test/registered/{quant => e2e/quantization}/test_gptqmodel_dynamic.py (100%) rename test/registered/{quant => e2e/quantization}/test_int4fp8_moe.py (100%) rename test/registered/{quant => e2e/quantization}/test_llama8b_nvfp4_kv_cache_sm120.py (100%) rename test/registered/{quant => e2e/quantization}/test_modelopt_fp8.py (100%) rename test/registered/{quant => e2e/quantization}/test_nvfp4_gemm_sm120.py (100%) rename test/registered/{quant => e2e/quantization}/test_quark_mxfp4.py (100%) rename test/registered/{quant => e2e/quantization}/test_w8a8_quantization.py (99%) rename test/registered/kernels/benchmark/{ => attention}/minicpm_sala/bench_get_block_table.py (95%) rename test/registered/kernels/benchmark/{moe => sampling}/bench_renorm.py (100%) create mode 100644 test/registered/kernels/ops/activation/test_fused_op_gpu_parity.py rename test/registered/kernels/ops/{ => attention}/minicpm_sala/test_get_block_table.py (98%) rename test/registered/kernels/ops/{ => attention}/minicpm_sala/test_minicpm_compress_k.py (100%) rename test/registered/kernels/ops/{ => attention}/qsa/test_qsa.py (100%) rename test/registered/kernels/ops/{ => attention}/qsa/test_qsa_indexer.py (100%) rename test/registered/kernels/ops/{ => attention}/qsa/test_qsa_strided_zero_fill.py (100%) rename test/registered/kernels/ops/{elementwise => attention}/test_fast_topk.py (98%) rename test/registered/kernels/ops/{moe => attention}/test_kpool_topk_transform_fused.py (97%) create mode 100644 test/registered/kernels/ops/attention/test_mla_scatter_concat.py create mode 100644 test/registered/kernels/ops/attention/test_replayssm_ring_fold.py create mode 100644 test/registered/kernels/ops/attention/test_vision_rope.py rename test/registered/kernels/ops/{quantization/test_diffusion_nvfp4_scaled_mm.py => diffusion/test_nvfp4_scaled_mm.py} (100%) delete mode 100644 test/registered/kernels/ops/dllm/test_gemma4_sampling_cuda.py create mode 100644 test/registered/kernels/ops/elementwise/test_add3.py rename test/registered/kernels/ops/{attention/test_inkling_row_scale.py => elementwise/test_row_scale.py} (65%) create mode 100644 test/registered/kernels/ops/gemm/test_dsv4_wo_a.py create mode 100644 test/registered/kernels/ops/gemm/test_fp8_kernel.py rename test/registered/kernels/ops/{quantization => gemm}/test_gptq_marlin.py (97%) rename test/registered/kernels/ops/{hyperconnection/test_hc_mix_triton.py => gemm/test_hc_mix.py} (98%) rename test/registered/{ => kernels/ops}/gemm/test_hopper_bf16_gemv.py (100%) rename test/registered/kernels/ops/{attention => gemm}/test_inkling_rel_proj.py (97%) rename test/registered/{ => kernels/ops}/gemm/test_linear_bf16_fp32_hpc.py (98%) rename test/registered/kernels/ops/{quantization => gemm}/test_nvfp4_marlin.py (100%) create mode 100644 test/registered/kernels/ops/gemm/test_tiny_gemm_variants.py rename test/registered/{quant => kernels/ops/gemm}/test_triton_scaled_mm.py (98%) delete mode 100644 test/registered/kernels/ops/kimi_k3/test_kimi_k3_prerequisite_ops.py rename test/registered/{quant => kernels/ops/layernorm}/test_fused_rms_fp8_group_quant.py (100%) create mode 100644 test/registered/kernels/ops/layernorm/test_gemma4_qkv_norm.py rename test/registered/kernels/ops/{speculative => mamba}/test_verify_commit_triton.py (100%) create mode 100644 test/registered/kernels/ops/memory/test_row_compact.py create mode 100644 test/registered/kernels/ops/mm/test_image.py rename test/registered/{quant => kernels/ops/moe}/test_block_int8.py (100%) create mode 100644 test/registered/kernels/ops/moe/test_gemma4_expert_activation.py rename test/registered/kernels/ops/{layernorm => moe}/test_gemma4_fused_routing.py (97%) rename test/registered/{quant => kernels/ops/moe}/test_int8_kernel.py (100%) rename test/registered/{quant => kernels/ops/moe}/test_marlin_moe.py (100%) create mode 100644 test/registered/kernels/ops/moe/test_route_quant_prerequisites.py rename test/registered/{quant => kernels/ops/quantization}/test_awq_dequant.py (100%) rename test/registered/{e2e/dsv4 => kernels/ops/quantization}/test_dsv4_fp8_cast.py (98%) create mode 100644 test/registered/kernels/ops/quantization/test_fp8_kernel.py rename test/registered/kernels/ops/{moe => sampling}/test_renorm.py (100%) create mode 100644 test/registered/kernels/ops/sampling/test_top_p_renorm.py rename test/registered/{quant => unit/configs}/test_quant_config_parsing.py (100%) rename test/registered/{kernels/ops => unit}/disaggregation/test_kv_checksum.py (100%) create mode 100644 test/registered/unit/dllm/algorithm/test_gemma4_sampling_cuda.py rename test/registered/{kernels/ops/layernorm => unit/kernels}/test_fused_op.py (100%) rename test/registered/{kernels/ops/layernorm => unit/kernels}/test_kernels_namespace.py (68%) rename test/registered/{quant => unit/layers/quantization}/test_fp8_utils.py (100%) rename test/registered/{quant => unit/layers/quantization}/test_is_layer_skipped.py (100%) rename test/registered/{quant => unit/layers/quantization}/test_nvfp4_embedding.py (100%) rename test/registered/{kernels/ops/kvcache => unit/mem_cache/pool_host}/test_kvcacheio_asymmetric.py (100%) rename test/registered/{kernels/ops/kvcache => unit/mem_cache}/test_unified_swa_tail_allocation.py (100%) rename test/registered/{kernels/ops/cuda_graph => unit/model_executor/runner_backend}/test_cuda_graph_dedup.py (100%) rename test/registered/{kernels/ops/quantization/test_diffusion_modelopt_fp8_scaled_mm.py => unit/multimodal_gen/test_modelopt_fp8_scaled_mm.py} (100%) diff --git a/.claude/skills/add-jit-kernel/SKILL.md b/.claude/skills/add-jit-kernel/SKILL.md index 11fa1d02e6..b5289b80d0 100644 --- a/.claude/skills/add-jit-kernel/SKILL.md +++ b/.claude/skills/add-jit-kernel/SKILL.md @@ -1,10 +1,14 @@ --- name: add-jit-kernel -description: Step-by-step tutorial for adding a new lightweight JIT CUDA kernel to sglang's jit_kernel module +description: Step-by-step tutorial for adding a new lightweight JIT CUDA kernel to sglang.kernels JIT infrastructure and public operator groups --- # Tutorial: Adding a New JIT Kernel to SGLang +Apply [kernel-organization](../kernel-organization/SKILL.md) for the public +operator namespace, logical grouping, lazy registry metadata, and test placement. +The implementation tutorial below does not replace that API contract. + This tutorial walks through adding a simple element-wise scale operation as a JIT kernel. We'll implement `scale(x, factor) = x * factor` to demonstrate the complete workflow. ## Goal @@ -497,6 +501,32 @@ def scale(src: torch.Tensor, factor: float, out: torch.Tensor | None = None) -> --- +## Register the public entry point (required) + +Add metadata in `python/sglang/kernels/ops/elementwise/__init__.py` so the +operator appears in the registry without importing its implementation: + +```python +from sglang.kernels.registry import register_kernel +from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec + +register_kernel( + KernelSpec( + op="elementwise.scale", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.elementwise.scale:scale", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +``` + +Record the devices the implementation actually supports; the CUDA-only scale +example above does not imply that every JIT kernel is CUDA-only. Runtime and +correctness tests import `scale` from `sglang.kernels.ops.elementwise.scale`. +Do not import the implementation eagerly in the group initializer. When an +existing `BaseFusedOp` owns the operation, add the implementation there instead +of registering a second conflicting op/backend pair. + ## Step 3 (optional): Tune JIT build flags If your kernel uses some math functions like `expf` or `sinf`, consider enabling `--use_fast_math` for better performance (with a potential precision tradeoff): @@ -724,7 +754,7 @@ cd test && python3 run_suite.py --hw cuda --suite base-b-kernel-benchmark-test-1 - `python/sglang/kernels/jit/include/sgl_kernel/cta.cuh` — `cta::reduce_max` - `python/sglang/kernels/jit/include/sgl_kernel/atomic.cuh` — `atomic::max` - `python/sglang/kernels/jit/include/sgl_kernel/runtime.cuh` — occupancy / SM count helpers -- `python/sglang/kernels/jit/csrc/add_constant.cuh` — minimal runnable reference +- `python/sglang/kernels/jit/csrc/elementwise/add_constant.cuh` — minimal runnable reference - `python/sglang/kernels/jit/csrc/elementwise/rmsnorm.cuh` — real example using `TensorMatcher` + `LaunchKernel` + `tile::Memory` - `python/sglang/kernels/jit/csrc/elementwise/qknorm.cuh` — real example using `runtime::get_blocks_per_sm` + persistent kernel pattern - `python/sglang/kernels/jit/benchmark/marker.py` — `benchmark`, `parametrize`, `do_bench`, `BenchResult` diff --git a/.claude/skills/add-sgl-kernel/SKILL.md b/.claude/skills/add-sgl-kernel/SKILL.md index 1916a4abbb..6aaad777d5 100644 --- a/.claude/skills/add-sgl-kernel/SKILL.md +++ b/.claude/skills/add-sgl-kernel/SKILL.md @@ -5,6 +5,10 @@ description: Step-by-step tutorial for adding a heavyweight AOT CUDA/C++ kernel # Tutorial: Adding a New Kernel to `sgl-kernel` (AOT / Heavyweight) +Apply [kernel-organization](../kernel-organization/SKILL.md) for the public +operator namespace, logical grouping, lazy registry metadata, and test placement. +The implementation tutorial below does not replace that API contract. + This tutorial walks through adding a simple element-wise scale operation as an AOT kernel. We'll implement `scale(x, factor) = x * factor` to demonstrate the complete workflow. ## Goal @@ -194,6 +198,21 @@ Then re-export it from `python/sglang/kernels/aot/python/sgl_kernel/__init__.py` --- +## SGLang integration (required for runtime use) + +After exposing the AOT wheel symbol, add a lazy wrapper and a `KernelSpec` under +`python/sglang/kernels/ops//`. Set `backend=KernelBackend.AOT`, record +actual device/architecture support, and keep `sgl_kernel` imports inside the +implementation path. SGLang runtime and integration tests import that wrapper. +For this example, use `sglang.kernels.ops.elementwise.scale`. + +The wheel-level tests below validate its standalone API/build. They do not +replace CI-registered SGLang correctness tests in +`test/registered/kernels/ops/elementwise/test_scale.py` and benchmarks in +`test/registered/kernels/benchmark/elementwise/bench_scale.py`. Follow +`write-sglang-test` for their registration and CI budget; do not register tests +under the shipped `python/sglang/` package. + ## Step 6: Write tests (required) Create `python/sglang/kernels/aot/tests/test_scale.py`: diff --git a/.claude/skills/kernel-organization/SKILL.md b/.claude/skills/kernel-organization/SKILL.md new file mode 100644 index 0000000000..40081866b4 --- /dev/null +++ b/.claude/skills/kernel-organization/SKILL.md @@ -0,0 +1,82 @@ +--- +name: kernel-organization +description: Apply the SGLang kernels RFC when adding, moving, splitting, or reviewing kernel APIs, registry metadata, kernel tests, benchmarks, and model-specific implementations. Use with add-jit-kernel, add-sgl-kernel, and write-sglang-test for placement and migration checks. +--- + +# Kernel organization + +Read `python/sglang/kernels/README.md` and `test/README.md` before choosing a +location. [RFC #29630](https://github.com/sgl-project/sglang/issues/29630) +establishes the namespace; subsequent migrations #32148 and #40922 clarify +logical grouping and remove the old `_jit_` filename prefix. + +## Choose the owner + +- Put callable SGLang kernel APIs under `sglang.kernels.ops.`. Runtime + and integration tests import from that namespace, including its submodules. + The `sgl_kernel` wheel retains its own public API and packaging tests. +- Group by computation, not model or GPU: GEMM and GEMV in `gemm`, expert + routing in `moe`, attention index selection in `attention`, sampling in + `sampling`, normalization in `layernorm`. A quantized GEMM is still a GEMM; + quantization alone belongs in `quantization`. +- Model-specific files/subpackages and tuning data are allowed **inside** a + logical group. Do not add a model bundle or an implementation file directly + under `ops/`. Propose a new logical group only for a distinct responsibility, + and update `ops._GROUPS`, documentation, and tests together. +- Classify a fused operator by its complete contract. Do not split a fused + kernel into separate launches just to separate norm, RoPE, or quantization. + Split unrelated public entry points that happen to share a source file. +- Keep shared CUDA build/runtime infrastructure in `kernels/jit`; operator + wrappers call it from their logical group. Keep process groups, communicator + state, model dispatch, and buffer ownership in `srt`. K3-specific adapters in + `srt/layers/communication/` need not pretend to be generic interfaces. + +## Register the public entry point + +Add lazy `KernelSpec` metadata in the owning group's `__init__.py`, or use the +existing `BaseFusedOp` registration when the operation has interchangeable +backends. Group imports must not import GPU implementations or compile kernels. + +Use `.` for the op id and `module:callable` for the target. Preserve +existing backend dispatch and describe actual device/architecture restrictions +with `CapabilityRequirement`; JIT/AOT are not device types. Input shape/dtype +checks remain part of the entry point's contract. Registering a torch custom op +does not register it in the SGLang kernel inventory. Predicates, private JIT +factories, reference helpers, and runtime classes are not separate kernel APIs. + +## Place and preserve tests + +- Kernel numerical tests: `test/registered/kernels/ops//`. +- CI microbenchmarks: `test/registered/kernels/benchmark//`. +- Inventory/selector tests: `test/registered/unit/kernels/`. +- Runtime unit tests: `test/registered/unit//`, following the tested + runtime module. A CUDA allocation does not make a runtime test a kernel test. +- Non-CI smoke scripts and benchmarks: `test/manual/kernels/`; do not leave + standalone test/benchmark entry points in production operator modules. +- Shared test helpers: `sglang.test.kernels`. Preserve vendored upstream trees + and AOT wheel packaging boundaries instead of reorganizing them incidentally. + +For a move/split, preserve assertions, parametrization, fixtures, platform skips, +execution entry points, and CI stages/runners. Apportion existing time estimates +across split files; do not duplicate the original budget for every output file +or silently drop a registration. Follow `write-sglang-test` for CI registration. + +## Verify a migration + +1. Search all runtime, test, benchmark, documentation, patch-string, and lazy + registry references before deleting the old path. Check relative imports and + package re-exports as well as direct imports. Do not leave forwarding shims. +2. Keep tuning files with the GEMM that loads them and preserve relative lookup + behavior. Check wheel/package inclusion as well as source-tree execution. +3. Preserve module singleton state: every runtime caller must import the same + new communication adapter, not a second copy of its buffer registry. +4. Separate relocation commits from semantic changes. Use + `mechanical-refactor-verify` to reproduce moves, and compare test inventories + and CI registration coverage before/after. Only delete an experimental path + after checking its call sites, flags, source, and dedicated tests together. +5. Run the namespace/dispatch CPU tests, the registered-test validation hook, + pre-commit, and relevant GPU tests when available. State exactly which GPU + checks ran; import/AST checks do not prove numerical or performance parity. + +Do not infer violations solely from a model name inside a group, a missing +`_jit_` prefix, use of a runtime utility, or absence of `BaseFusedOp` inheritance. diff --git a/.claude/skills/write-sglang-test/SKILL.md b/.claude/skills/write-sglang-test/SKILL.md index c932523dd0..d0ddcdbe91 100644 --- a/.claude/skills/write-sglang-test/SKILL.md +++ b/.claude/skills/write-sglang-test/SKILL.md @@ -5,6 +5,10 @@ description: Guide for writing SGLang CI/UT tests. Covers CustomTestCase, CI reg # Writing SGLang CI / UT Tests +Apply [kernel-organization](../kernel-organization/SKILL.md) for the public +operator namespace, logical grouping, lazy registry metadata, and test placement. +The implementation tutorial below does not replace that API contract. + This skill covers **how to write and register tests**. For CI pipeline internals (stage ordering, fail-fast, gating, partitioning, debugging CI failures), see the [CI workflow guide](../ci-workflow-guide/SKILL.md). Whether a case is worth adding at all is decided by [`unit-test-admission`](../../rules/unit-test-admission.md) — read it before writing the case, not after. ## Core Rules diff --git a/benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py b/benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py index 3a9f9be50e..96f839c4fd 100644 --- a/benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py +++ b/benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py @@ -7,7 +7,7 @@ import torch.utils.benchmark as benchmark from flashinfer import BatchDecodeWithPagedKVCacheWrapper from sglang.kernels.ops.attention.decode_attention import decode_attention_fwd -from sglang.kernels.ops.attention.flashinfer_backend import should_use_tensor_core +from sglang.srt.layers.attention.flashinfer_backend import should_use_tensor_core def benchmark_forward( diff --git a/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py b/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py index e994ea5df0..cf9b866e0f 100644 --- a/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py +++ b/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py @@ -11,7 +11,7 @@ from sglang.benchmark.deepseek_utils import ( per_block_cast_to_fp8, per_token_cast_to_fp8, ) -from sglang.kernels.ops.quantization.fp8_kernel import ( +from sglang.kernels.ops.gemm.fp8_kernel import ( w8a8_block_fp8_matmul_deepgemm as w8a8_block_fp8_matmul, ) diff --git a/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm_blackwell.py b/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm_blackwell.py index 8fae14b191..b49b2c58f9 100644 --- a/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm_blackwell.py +++ b/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm_blackwell.py @@ -9,10 +9,8 @@ from sglang.benchmark.deepseek_utils import ( get_weight_shapes, per_block_cast_to_fp8, ) -from sglang.kernels.ops.quantization.fp8_kernel import ( - sglang_per_token_group_quant_fp8, - w8a8_block_fp8_matmul_deepgemm, -) +from sglang.kernels.ops.gemm.fp8_kernel import w8a8_block_fp8_matmul_deepgemm +from sglang.kernels.ops.quantization.fp8_kernel import sglang_per_token_group_quant_fp8 from sglang.srt.layers.quantization.fp8_utils import requant_weight_ue8m0 BLOCK_SIZE = 128 diff --git a/benchmark/kernels/quantization/README.md b/benchmark/kernels/quantization/README.md index acf6f0b0d1..e76d4a237a 100644 --- a/benchmark/kernels/quantization/README.md +++ b/benchmark/kernels/quantization/README.md @@ -78,7 +78,7 @@ python benchmark/kernels/quantization/tuning_block_wise_kernel.py --N 5120 --K 1 ## Output -Generates JSON config files saved to `python/sglang/srt/layers/quantization/configs/`: +Generates JSON config files saved to `python/sglang/kernels/ops/gemm/configs/`: ``` N={N},K={K},device_name={DEVICE},dtype=fp8_w8a8,block_shape=[128,128].json ``` diff --git a/benchmark/kernels/quantization/tuning_block_wise_kernel.py b/benchmark/kernels/quantization/tuning_block_wise_kernel.py index f2c4cca0ab..0c5ddc0275 100644 --- a/benchmark/kernels/quantization/tuning_block_wise_kernel.py +++ b/benchmark/kernels/quantization/tuning_block_wise_kernel.py @@ -28,11 +28,11 @@ from tqdm import tqdm mp.set_start_method("spawn", force=True) from sglang.benchmark.deepseek_utils import get_weight_shapes -from sglang.kernels.ops.quantization.fp8_kernel import ( +from sglang.kernels.ops.gemm.fp8_kernel import ( _w8a8_block_fp8_matmul, _w8a8_block_fp8_matmul_unrolledx4, ) -from sglang.kernels.ops.quantization.int8_kernel import _w8a8_block_int8_matmul +from sglang.kernels.ops.gemm.int8_kernel import _w8a8_block_int8_matmul from sglang.srt.utils import ( get_device, get_device_core_count, @@ -491,7 +491,7 @@ if __name__ == "__main__": parser.add_argument( "--save-path", type=str, - default="python/sglang/kernels/ops/quantization/configs", + default="python/sglang/kernels/ops/gemm/configs", ) args = parser.parse_args() diff --git a/python/sglang/kernels/README.md b/python/sglang/kernels/README.md index 5a9b160f90..d4c538390c 100644 --- a/python/sglang/kernels/README.md +++ b/python/sglang/kernels/README.md @@ -26,7 +26,7 @@ sglang/kernels/ Operator groups (all populated): `activation`, `attention`, `communication`, `diffusion`, `elementwise`, `embeddings`, `gemm`, `grammar`, `kv_canary`, -`kvcache`, `layernorm`, `lplb`, `mamba`, `memory`, `moe`, `quantization`, +`kvcache`, `layernorm`, `lplb`, `mamba`, `memory`, `mm`, `moe`, `quantization`, `sampling`, `speculative`. Place model-specific implementations and tuning data inside the corresponding @@ -36,9 +36,32 @@ under `ops/`. As of the RFC #29630 finale (#32072) the legacy `sglang.jit_kernel` package has been **removed**: its shared build/runtime infra moved to `sglang.kernels.jit` and each JIT-backed operator into its group as -`sglang.kernels.ops.._jit_`. Tests and benchmarks live under +`sglang.kernels.ops..` (the old `_jit_` filename prefix was removed +in #32148). Tests and benchmarks live under `test/registered/kernels/` (`ops//` for tests, `benchmark//` for -benchmarks); shared test helpers are in `sglang.test.kernels`. +benchmarks); shared test helpers are in `sglang.test.kernels`. Standalone scripts +that are not CI-registered belong in `test/manual/kernels/`, outside the shipped +operator package. Keep vendored upstream trees under their existing maintenance +policy. + +Classify the public computation, not the model that first used it: expert +routing belongs in `moe`, matrix multiplication (including quantized GEMM and +its tuning data) in `gemm`, and attention index selection in `attention`. +Keep a fused operation intact; an attention kernel that fuses RoPE with a +projection may stay in `attention`. A model name inside a logical group is fine. + +Runtime integration that owns process groups, symmetric buffers, or model +dispatch stays in `srt`; the K3 communication adapters live in +`srt/layers/communication/`, while their callable kernels live in +`kernels/ops/communication/`. Tests of this runtime state belong with the +runtime subsystem, even when they need a GPU. Kernel inventory/dispatch tests +belong in `test/registered/unit/kernels/`. + +When moving an operator, update its callers, lazy `KernelSpec` targets and op +ids, tests, benchmark imports, and tuning-data paths together. Preserve CI +stage/runner registrations and test cases when splitting files. See the +[kernel organization skill](../../../.claude/skills/kernel-organization/SKILL.md) +for the review checklist. ## How it works diff --git a/python/sglang/kernels/jit/csrc/kimi_k3/comm/gemm_ar.cuh b/python/sglang/kernels/jit/csrc/kimi_k3/comm/gemm_ar.cuh deleted file mode 100644 index ea3498c8f0..0000000000 --- a/python/sglang/kernels/jit/csrc/kimi_k3/comm/gemm_ar.cuh +++ /dev/null @@ -1,1577 +0,0 @@ -// K3 fused o_proj GEMM + all-reduce for decode (bf16, TP row-parallel). -// -// CONTRACT (per rank r of R): out[M, 7168] = sum_r x_r[M, K] @ W_r[7168, K]^T -// bf16 in/out, fp32 accumulate, partials round to bf16 pre-sum (same -// semantics as the unfused GEMM + bf16 ring AR). -// M in [1, 512], rounded up to a tuned cell {8,16,32,64,128,256,512}; out -// must have `cell` rows — rows [M, cell) are clobbered with zeros. -// Comm plane: pure NVLink P2P (unicast pushes + per-rank flag reductions); -// one-shot AR below the two-shot threshold, two-shot RS+AG above. -// Requires SM100+ with full P2P; tuned on GB300 (sm_103a). - -#include -#include - -#include - -#include -#include - -#include -#include -#include -#include -#include -#include - -// Local PTX / TMA primitives -// Only what this kernel issues, kept in-file on purpose: these are raw-ISA -// shapes (sm_100+ tcgen05, the cta_group::1 multicast TMA load) that no shared -// sglang header wraps, and splitting them out bought a dozen headers with -// exactly one consumer. Anything cute already provides goes through cute -// (`set_block_rank` below, the tensor-map driver wrapper in `w_maps`). - -namespace sglang { - -namespace device::ptx { - -// ---- generic → shared address conversion (PTX ISA §10.4) -------------------- - -// ---- cvt: pack 2 fp32 into one bf16x2 (PTX ISA §9.7.9.21) ------------------ -// -// PACKED-PAIR ORDERING: `cvt.bf16x2.f32 d, a, b` puts cvt(a) in d's UPPER -// half and cvt(b) in the LOWER half. Read back from little-endian memory as a -// bf16 array, LOWER lands at column i and UPPER at i+1 — so for cells (c0, c1) -// destined for adjacent slots [i, i+1] pass `cvt_pack_f32x2_to(c1, c0)`. -struct bf16 { - using packed2_t = uint32_t; -}; - -template -static __device__ __forceinline__ typename Dst::packed2_t cvt_pack_f32x2_to(float a, float b); - -template <> -__device__ __forceinline__ uint32_t cvt_pack_f32x2_to(float a, float b) { - uint32_t d; - asm volatile("cvt.rn.bf16x2.f32 %0, %1, %2;" : "=r"(d) : "f"(a), "f"(b)); - return d; -} - -// ---- ldmatrix (PTX ISA §9.7.14.5.15) --------------------------------------- - -// Warp-collective load of 4 (8x8) BF16 matrices from smem into mma.sync -// fragments. `row_addr` is this lane's 16-byte-aligned row base: lanes 0-7 -// supply matrix 0, 8-15 matrix 1, 16-23 matrix 2, 24-31 matrix 3. -static SGL_DEVICE void ldmatrix_x4_b16(uint32_t row_addr, uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3) { - asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared::cta.b16 {%0, %1, %2, %3}, [%4];" - : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) - : "r"(row_addr)); -} - -static SGL_DEVICE void ldmatrix_x2_b16(uint32_t row_addr, uint32_t& r0, uint32_t& r1) { - asm volatile("ldmatrix.sync.aligned.m8n8.x2.shared::cta.b16 {%0, %1}, [%2];" : "=r"(r0), "=r"(r1) : "r"(row_addr)); -} - -// ---- warp-level mma.sync (PTX ISA §9.7.14) --------------------------------- - -// D += A*B, bf16 x bf16 -> f32. The warp-register form: co-resides freely (no -// TMEM, no 1-CTA/SM cap), which is why the small-M members use it instead of -// tcgen05. -static __device__ __forceinline__ void -mma_m16n8k16_bf16f32(float4& d, uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, uint32_t b1) { - asm volatile( - "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " - "{%0,%1,%2,%3},{%4,%5,%6,%7},{%8,%9},{%0,%1,%2,%3};" - : "+f"(d.x), "+f"(d.y), "+f"(d.z), "+f"(d.w) - : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1)); -} - -// ---- tcgen05 matrix descriptors (PTX ISA §9.7.16.4) ------------------------ -// -// 0 = K-major (innermost dim is K, the "TN" convention), 1 = MN-major. -enum class Major : uint8_t { - K = 0, - MN = 1, -}; - -enum class F16Type : uint8_t { F16 = 0, BF16 = 1 }; // kind::f16 atype/btype -enum class DType : uint8_t { F16 = 0, F32 = 1, S32 = 2 }; - -// Smem matrix descriptor. The public PTX spec has errors at bits 46-60; the -// layout below is the one our gate tests verify: -// 0-13: start_address >> 4 16-29: leading_byte_offset >> 4 -// 32-45: stride_byte_offset >> 4 46-47: version (=1, Blackwell — larger -// shapes such as 128B-swizzle BLOCK_K>16 produce garbage at 0) -// 49-51: base_offset 52: lbo_mode (=0, relative byte offset) -// 61-63: layout_type (0=None, 2=128B, 4=64B, 6=32B) -__host__ __device__ static __forceinline__ constexpr uint64_t -mma_smem_desc(uint32_t matrix_addr, uint32_t lbo, uint32_t sbo, uint32_t base_offset, int swizzle_bytes) { - auto enc = [](uint32_t x) -> uint64_t { return (uint64_t)((x & 0x3FFFFu) >> 4); }; - uint8_t code = (swizzle_bytes == 128) ? 2u : (swizzle_bytes == 64) ? 4u : (swizzle_bytes == 32) ? 6u : 0u; - uint64_t d = 0; - d |= enc(matrix_addr); // bits 0-13 - d |= enc(lbo) << 16; // bits 16-29 - d |= enc(sbo) << 32; // bits 32-45 - d |= uint64_t(1u) << 46; // bits 46-47 = version = 1 - d |= uint64_t(base_offset & 0x7u) << 49; // bits 49-51 - d |= uint64_t(code & 0x7u) << 61; // bits 61-63 - return d; -} - -// K-major operand (A as (M, K), B as (N, K) in a TN GEMM). `T` is a size proxy -// (uint16_t for BF16/FP16); the dtype semantics live in the instruction -// descriptor. K-major mandates SWIZZLE_BYTES == BLOCK_K * sizeof(T). -template -__host__ __device__ static __forceinline__ constexpr uint64_t -mma_smem_desc_k_major(uint32_t addr, uint32_t base_offset = 0) { - constexpr int K_BYTES = BLOCK_K * int(sizeof(T)); - static_assert(SWIZZLE_BYTES == K_BYTES, "K-major requires swizzle bytes == BLOCK_K * sizeof(T)"); - return mma_smem_desc(addr, /*lbo=*/0u, /*sbo=*/8u * uint32_t(K_BYTES), base_offset, SWIZZLE_BYTES); -} - -// Instruction descriptor, kind::f16 (PTX ISA Table 44). -__host__ __device__ static __forceinline__ constexpr uint32_t mma_inst_desc_f16( - uint32_t M, - uint32_t N, - F16Type a_type = F16Type::BF16, - F16Type b_type = F16Type::BF16, - DType d_type = DType::F32, - Major a_major = Major::K, - Major b_major = Major::K, - bool negate_a = false, - bool negate_b = false) { - uint32_t d = 0; - d |= (static_cast(d_type) & 0x3u) << 4; - d |= (static_cast(a_type) & 0x7u) << 7; - d |= (static_cast(b_type) & 0x7u) << 10; - if (negate_a) d |= 1u << 13; - if (negate_b) d |= 1u << 14; - d |= (static_cast(a_major) & 0x1u) << 15; - d |= (static_cast(b_major) & 0x1u) << 16; - d |= ((N >> 3) & 0x3Fu) << 17; - d |= ((M >> 4) & 0x1Fu) << 24; - return d; -} - -// Cross-CTA arrive with `.release.cta` ordering. A plain cluster arrive carries -// NO release, so prior memory ops (notably a retired `tcgen05.ld` TMEM drain on -// the arriving warp) are not guaranteed visible before a peer warp's `.acquire` -// wait returns — and the peer then overwriting that TMEM is a real race (spec -// §8.8: a release pattern requires `mbarrier.arrive.release`). -// -// `cute::set_block_rank` is the `mapa.shared::cluster` that retargets a local -// smem offset at CTA `cta_rank`; the `.shared::cluster` qualifier on the arrive -// is what makes it cross-CTA (the plain `.shared` form hits the local mbar -// whatever the address bits say). -static SGL_DEVICE void mbar_arrive_cluster_release(uint64_t* bar, uint32_t cta_rank) { - const uint32_t mapped = cute::set_block_rank(to_shared(bar), cta_rank); - asm volatile("mbarrier.arrive.release.cta.shared::cluster.b64 _, [%0], 1;" ::"r"(mapped)); -} - -// ---- cluster / warp sync (PTX ISA §9.7.13, §9.7.4) ------------------------- - -// Cluster barrier with explicit release/acquire — the publish/observe boundary -// between `mbarrier.init` and any `.shared::cluster` use of those mbars. -static SGL_DEVICE void cluster_sync_rel_acq() { - asm volatile("barrier.cluster.arrive.release.aligned;"); - asm volatile("barrier.cluster.wait.acquire.aligned;"); -} - -// True on exactly one lane of the issuing warp — guards single-issuer sites -// (mbar init, TMA issue, MMA issue, TMEM alloc) without gating on lane_id. -static SGL_DEVICE bool elect_one() { - uint32_t pred; - asm volatile( - "{\n\t.reg .pred p;\n\t" - "elect.sync _|p, 0xffffffff;\n\t" - "selp.b32 %0, 1, 0, p;\n\t}\n" - : "=r"(pred)); - return pred != 0; -} - -static SGL_DEVICE uint32_t cluster_cta_rank() { - uint32_t rank; - asm("mov.u32 %0, %%cluster_ctarank;" : "=r"(rank)); - return rank; -} - -// ---- tcgen05 (PTX ISA §9.7.16) --------------------------------------------- -// -// Lifecycle (mandatory order, §9.7.16.7.1): alloc (one warp, n_cols a power of -// 2 in [32, 512], TMEM address written to smem) -> __syncthreads + read taddr -// -> ld/mma -> dealloc -> relinquish before kernel exit. -// -// Each warp can only touch its own 32-lane TMEM band (§9.7.16.8.1): warp 0 -> -// lanes 0-31, warp 1 -> 32-63, and so on. -// -// After an MMA, `tcgen05_commit_arrive` + `mbar_wait_parity` + -// `tcgen05_fence_after_thread_sync` before reading the result with -// `tcgen05_ld_*`; the fence is mandatory or the register reads may see stale -// values even though the mbarrier signaled "MMA done". -static SGL_DEVICE void tcgen05_alloc(uint32_t smem_addr_for_taddr, uint32_t n_cols) { - asm volatile( - "tcgen05.alloc.cta_group::1.sync.aligned.shared::cta.b32 [%0], %1;" ::"r"(smem_addr_for_taddr), "r"(n_cols)); -} - -static SGL_DEVICE void tcgen05_dealloc(uint32_t taddr, uint32_t n_cols) { - asm volatile("tcgen05.dealloc.cta_group::1.sync.aligned.b32 %0, %1;" ::"r"(taddr), "r"(n_cols)); -} - -static SGL_DEVICE void tcgen05_relinquish() { - asm volatile("tcgen05.relinquish_alloc_permit.cta_group::1.sync.aligned;"); -} - -// .32x32b.x8: 8 b32 per lane = 8 TMEM columns. Per-lane 8 FP32 -> 4 bf16x2 -// packs = one int4, the natural fit for a BF16 epilogue draining a column band -// with 16-byte smem stores. -static SGL_DEVICE void tcgen05_ld_32x32b_x8( - uint32_t taddr, - uint32_t& r0, - uint32_t& r1, - uint32_t& r2, - uint32_t& r3, - uint32_t& r4, - uint32_t& r5, - uint32_t& r6, - uint32_t& r7) { - asm volatile( - "tcgen05.ld.sync.aligned.32x32b.x8.b32 " - " {%0, %1, %2, %3, %4, %5, %6, %7}, [%8];" - : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3), "=r"(r4), "=r"(r5), "=r"(r6), "=r"(r7) - : "r"(taddr)); -} - -static SGL_DEVICE void tcgen05_ld_32x32b_x8(uint32_t taddr, uint32_t* dst) { - tcgen05_ld_32x32b_x8(taddr, dst[0], dst[1], dst[2], dst[3], dst[4], dst[5], dst[6], dst[7]); -} - -// Blocks until this thread's prior TMEM->reg drains retired. The "memory" -// clobber is load-bearing: ptxas lowers the wait to per-load scoreboard waits -// on the dependent register consumers, so an `mbarrier.arrive` that does not -// read the drained registers gets HOISTED above the last drain (observed in -// SASS) and the next tile's MMA reuses TMEM still being read. -static SGL_DEVICE void tcgen05_wait_ld() { - asm volatile("tcgen05.wait::ld.sync.aligned;" ::: "memory"); -} - -// Commit prior tcgen05.mma to an mbarrier. Spec accepts `.shared::cluster` or -// no state space, NOT `.shared::cta`. -static SGL_DEVICE void tcgen05_commit_arrive(uint64_t* bar) { - asm volatile("tcgen05.commit.cta_group::1.mbarrier::arrive::one.b64 [%0];" ::"r"(to_shared(bar))); -} - -static SGL_DEVICE void tcgen05_fence_after_thread_sync() { - asm volatile("tcgen05.fence::after_thread_sync;"); -} - -// kind::f16 MMA, cta_group::1. Valid dense shapes: M in {64, 128}, N in -// {8, 16, ..., 256} step 8, K = 16. Off-table shapes are NOT rejected by -// ptxas — they hit cudaErrorIllegalInstruction at runtime. -static SGL_DEVICE void -tcgen05_mma_f16(uint32_t d, uint64_t desc_a, uint64_t desc_b, uint32_t inst_desc_high, uint32_t scale_c) { - asm volatile( - "{\n\t.reg .pred p;\n\t" - "setp.ne.b32 p, %4, 0;\n\t" - "tcgen05.mma.cta_group::1.kind::f16 [%0], %1, %2, %3, p;\n\t}\n" ::"r"(d), - "l"(desc_a), - "l"(desc_b), - "r"(inst_desc_high), - "r"(scale_c)); -} - -// ---- TMA (PTX ISA §9.7.9.25) ----------------------------------------------- -// -// COORDINATE CONVENTION: the tensor map's globalDim is (inner, outer) — dim 0 -// is the stride-1 axis. The load calls take (x = inner offset, y = outer -// offset). Mismatch and you load the transposed tile, often with right-looking -// magnitudes but scrambled per-cell pairing. -// -// Completion is mbarrier-based for loads: arm with -// `mbar_arrive_expect_tx(bar, BYTES)` before issuing, then `mbar_wait_parity`. - -// Warm the cache line holding the tensor-map descriptor so the first load of -// the persistent loop does not pay the descriptor fetch. `tmap` is a generic -// address into the __grid_constant__ CUtensorMap param; generic addressing -// resolves it to .param (§9.7.9.15). -static SGL_DEVICE void prefetch_tensormap(const void* tmap) { - asm volatile("prefetch.tensormap [%0];" ::"l"(tmap) : "memory"); -} - -// global -> shared::cta 2D tile load. -static SGL_DEVICE void -cp_async_bulk_tensor_2d_load(uint32_t dst_smem, const CUtensorMap* tmap, int32_t x, int32_t y, uint64_t* bar) { - asm volatile( - "cp.async.bulk.tensor.2d.shared::cta.global.tile.mbarrier::complete_tx::bytes" - " [%0], [%1, {%2, %3}], [%4];" ::"r"(dst_smem), - "l"(tmap), - "r"(x), - "r"(y), - "r"(to_shared(bar)) - : "memory"); -} - -// MULTICAST load, cta_group::1: one leader CTA issues, and every CTA whose bit -// is set in `multicast_mask` receives the same bytes at the same CTA-relative -// smem offset AND its OWN tx-count decrement on its own local mbar at -// `bar`'s offset (spec §9.7.9.25, .cta_group::1 bullet). -// -// Contrast with the cta_group::2 form, which CONSOLIDATES all completion onto -// one CTA's mbar (bit 24 of the mbar address cleared). Here each peer keeps its -// own `expect_tx` accounting and only the leader issues, so the follower's smem -// and mbar are both served by the leader's single DRAM read — do NOT clear bit -// 24, that is the cta_group::2 trick and would mis-route the signal. -static SGL_DEVICE void cp_async_bulk_tensor_2d_load_multicast_cg1( - uint32_t dst_smem, - const CUtensorMap* tmap, - int32_t x, - int32_t y, - uint64_t* bar, - uint16_t multicast_mask = 0b11, - uint64_t cache_hint = 0x0ULL) { - const uint32_t mbar_addr = to_shared(bar); - asm volatile( - "cp.async.bulk.tensor.2d.cta_group::1.shared::cluster.global" - ".mbarrier::complete_tx::bytes.multicast::cluster.L2::cache_hint" - " [%0], [%1, {%4, %5}], [%2], %3, %6;" ::"r"(dst_smem), - "l"(tmap), - "r"(mbar_addr), - "h"(multicast_mask), - "r"(x), - "r"(y), - "l"(cache_hint) - : "memory"); -} - -} // namespace device::ptx - -namespace swz { - -// CPU-side 128B-swizzle math. The TMA hardware permutes the (row x col) atom -// layout when it stores a tile into smem, so reading a known cell back applies -// the same permutation: row stride = 128 B = 8 atoms = 64 BF16 cols, and -// smem_atom = logical_atom XOR (r & 7), an 8-row period (PTX ISA §5.5.7, -// Figures 23-37). Verified on B300 (sm_103a) by a load + read-back roundtrip. -// -// Returns the smem column index in BF16 units within the row. -__host__ __device__ inline uint32_t smem_col_128b_bf16(uint32_t r, uint32_t c) { - return c ^ ((r & 7u) << 3); // (r & 7) atoms shifted; atom = 8 BF16 cols -} - -} // namespace swz - -namespace tmap { - -// Thin wrapper over the tensor-map encoder. Coordinate convention: dim 0 is -// the innermost (stride-1) axis, so globalDim = {cols, rows} and globalStrides -// (length rank-1) carries the row stride in BYTES. -// -// K-major MMA feeds require swizzle == BLOCK_K bytes; the smem descriptor -// assumes equality and a smaller inner box is encoder-legal but silently -// corrupts the MMA load. -// -// The encode is the one driver-API call in this file (there is no runtime-API -// tensor-map encoder), so it goes through cutlass's dlopen-based driver wrapper -// like the other JIT kernels that encode tensor maps — that keeps the module -// off `-lcuda`. -inline CUtensorMap encode_tiled_2d( - void* global_ptr, - CUtensorMapDataType dtype, - uint64_t global_rows, - uint64_t global_cols, - uint64_t row_stride_bytes, - uint32_t box_rows, - uint32_t box_cols, - CUtensorMapSwizzle swizzle = CU_TENSOR_MAP_SWIZZLE_NONE, - CUtensorMapL2promotion promo = CU_TENSOR_MAP_L2_PROMOTION_NONE) { - cuuint64_t global_dim[2] = {global_cols, global_rows}; - cuuint64_t global_strides[1] = {row_stride_bytes}; - cuuint32_t box_dim[2] = {box_cols, box_rows}; - cuuint32_t element_strides[2] = {1, 1}; - - CUtensorMap m{}; - const CUresult res = CUTLASS_CUDA_DRIVER_WRAPPER_CALL(cuTensorMapEncodeTiled)( - &m, - dtype, - /*rank=*/2, - global_ptr, - global_dim, - global_strides, - box_dim, - element_strides, - CU_TENSOR_MAP_INTERLEAVE_NONE, - swizzle, - promo, - CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); - if (res != CUDA_SUCCESS) { - std::fprintf( - stderr, - "gemm_ar: cuTensorMapEncodeTiled failed (%d) at %s:%d — rows=%llu cols=%llu " - "row_stride=%llu box=%ux%u swizzle=%d\n", - int(res), - __FILE__, - __LINE__, - (unsigned long long)global_rows, - (unsigned long long)global_cols, - (unsigned long long)row_stride_bytes, - box_rows, - box_cols, - int(swizzle)); - std::abort(); - } - return m; -} - -} // namespace tmap - -namespace dense_gemm_mainloop { - -// GROUP_N L2-stripe raster: linear tile index -> (bid_m, bid_n), walking the -// output grid in N-stripes of width GROUP_N so the same chunk of B stays -// resident in L2 across all M-rows of the stripe (DeepGEMM's -// get_swizzled_block_idx). `cluster_grid_m` is the M-axis tile count in CLUSTER -// units (= grid_m / CTA_GROUP); `grid_n` is NOT halved by CTA_GROUP. The tail -// stripe is shorter than GROUP_N when grid_n % GROUP_N != 0. -template -__device__ __forceinline__ int2 group_n_swizzle(int linear, int crank, int cluster_grid_m, int grid_n) { - if constexpr (CTA_GROUP == 1) { - const int num_blocks_per_group = cluster_grid_m * GROUP_N; - const int group_idx = linear / num_blocks_per_group; - const int first_n = group_idx * GROUP_N; - const int in_group = linear - group_idx * num_blocks_per_group; - const int num_n_in_group = grid_n - first_n < GROUP_N ? grid_n - first_n : GROUP_N; - const int bid_m = in_group / num_n_in_group; - const int bid_n = first_n + (in_group % num_n_in_group); - (void)crank; - return {bid_m, bid_n}; - } else { - const int c = linear / CTA_GROUP; - const int r = linear & (CTA_GROUP - 1); - const int num_blocks_per_group = cluster_grid_m * GROUP_N; - const int group_idx = c / num_blocks_per_group; - const int first_n = group_idx * GROUP_N; - const int in_group = c - group_idx * num_blocks_per_group; - const int num_n_in_group = grid_n - first_n < GROUP_N ? grid_n - first_n : GROUP_N; - const int cluster_bid_m = in_group / num_n_in_group; - const int cluster_bid_n = first_n + (in_group % num_n_in_group); - const int bid_m = cluster_bid_m * CTA_GROUP + r; - const int bid_n = cluster_bid_n; - (void)crank; // `r` already encodes the intra-cluster lane. - return {bid_m, bid_n}; - } -} - -} // namespace dense_gemm_mainloop - -namespace oproj_ar { - -// ---------------------------------------------------------------- constants -#ifndef OPROJ_N // output dim (columns of W / of out). The -#define OPROJ_N 7168 // default is the Kimi-K3 o_proj shape; other -#endif // shapes compile via -DOPROJ_N (see asserts). -constexpr int kN = OPROJ_N; -static_assert( - kN % 256 == 0 && kN >= 256, - "N must be a multiple of 256 (member tile table: 128-row " - "strips + BN up to 256); relaxing this needs a BN-table edit"); -constexpr int kBK = 64; // K per stage (128 B rows → swizzle-128B) -constexpr int kBNRows = 48; // B-box rows per stage (6 n8-tiles) -constexpr int kTilesMax = 6; // n8-tiles per CTA (5-tile CTAs pad, never push) -constexpr int kCWarps = 6; // consumer warp w owns n8-tile w -constexpr int kThreads = (kCWarps + 1) * 32; // +1 dedicated TMA producer warp -constexpr int kMMax = 512; // bs64 — sizes the shared slot layout -constexpr int kRing = 64; // epoch flag/gather ring (monotonic values) - -enum class Comm { kNone, kMc, kPeer, kMcPull, kTwoShot, kTwoShotPeer }; - -// Shared-region layout (BYTES, M-independent: sized at kMMax so every arm and -// every bs cell reuses one region). Parity-double-buffered payloads; the flag -// ring lives on its own 2 MB page. Slot reuse across epochs e / e+2 is safe -// with 2 parities because each rank's launch e+1 spin-waits epoch e+1 AFTER -// its own e-reduce (per-rank stream order). -// Slots are TILE-MAJOR ([n8-tile][m][8 cols]), NOT [m][n]: a warp's push for -// one tile is then a CONTIGUOUS 128 B fabric write instead of 32 scattered -// 4-16 B m-strided writes — lane-scatter runs ~0.26x on this fabric -// and the scattered form's ack-drain dominated the bs8 -// boundary (stamped 22 us at idle). The reduce un-transposes locally. -constexpr size_t kSlotBytes1 = size_t(kMMax) * kN * 2; // one [M,N] bf16 -constexpr __host__ __device__ size_t slot_off(int parity, int src, int R) { - return (size_t(parity) * R + src) * kSlotBytes1; -} -constexpr __host__ __device__ size_t pull_off(int parity, int R) { // [2][M,N] above slots - return (2 * size_t(R) + parity) * kSlotBytes1; -} -constexpr __host__ __device__ size_t flags_off(int R) { - const size_t end = (2 * size_t(R) + 2) * kSlotBytes1; - return (end + (size_t(2) << 20)) & ~((size_t(2) << 20) - 1); -} -// second flag family: "epoch e's slots fully REDUCED", at PER-CTA granularity. -// PDL's wait pairs with the prior grid's TRIGGER (not completion), and with -// 2-CTA/SM residency a fast rank's e+2 pushes can overwrite a slot replica a -// straggling rank's e-reduce still reads — so epoch e's push phase guards on -// done[e-2]. Per-CTA flags (the overwriter of tile t IS every rank's CTA-t) -// keep the publish fully parallel: no second grid-wide gather chain. -constexpr int kMaxCTA = 256; -constexpr int kFams = 7; // flag/gather/done ring FAMILY per dispatch - // CELL {8,16,32,64,128,256,512}. Monotonic ring - // targets assume every epoch of a ring came - // from the same gridDim (and only two-shot - // bumps the boundary-2 ring words), so one fam - // per cell removes the host-side ring reset a - // cell change needed under the original 3-fam - // grid-class split — the reset was a collective - // and blocked CUDA-graph capture. -constexpr __host__ __device__ size_t done_off(int R) { - return flags_off(R) + size_t(kFams) * 512; -} -constexpr __host__ __device__ size_t region_bytes(int R) { - return done_off(R) + size_t(kFams) * kRing * kMaxCTA * 4 + 512; -} - -// PDL (programmatic dependent launch): the NEXT launch on the stream may -// start its feed while THIS grid sits in the boundary spin + reduce; its -// epilogue then griddepcontrol.wait's until this grid fully completes, so -// slots/flags/out stay race-free. Serving overlaps the next layer's kernels -// the same way; an unfused cublas composite cannot cooperate across the -// vendor-kernel boundary. -SGL_DEVICE uint32_t bf2_u32(float2 f) { - const __nv_bfloat162 p = __float22bfloat162_rn(f); - return *reinterpret_cast(&p); -} - -// ------------------------------------------------------------------ params - -template -struct Params { - uint8_t* mc_base; // MC VA (null when no MC object — kPeer runs) - uint8_t* uc_base[R]; // per-rank unicast VAs of the shared region - uint32_t* gather; // device-local u32[kRing] - __nv_bfloat16* out; // [M,N] local output - const __nv_bfloat16* partial_in; // GEMM_ON=false input [M,N] - // Device-resident per-fam CTA ticket counters. Every CTA takes one ticket - // at entry and divides by the (family-stable) gridDim to recover the - // launch epoch. All CTAs have taken their tickets before this grid - // triggers a PDL successor, so successive launches receive disjoint, - // contiguous ticket ranges without a separate bump kernel. Device state - // instead of a launch arg keeps CUDA-graph replays advancing the epoch. - uint32_t* epoch_base; - int my_rank; - int fam; // ring family (dispatch cell) — see kFams -}; - -// -------------------------------------------------------------- CTA strips -// kN/8 = 896 n8-tiles over gridDim CTAs: first `rem` CTAs own base+1 tiles. -struct Strip { - int t0, nt; - static SGL_DEVICE Strip make(int cta, int ncta) { - const int kT = kN / 8; - const int base = kT / ncta, rem = kT % ncta; - Strip s; - if (cta < rem) { - s.nt = base + 1; - s.t0 = cta * (base + 1); - } else { - s.nt = base; - s.t0 = rem * (base + 1) + (cta - rem) * base; - } - return s; - } -}; - -// ------------------------------------------------------------------ kernel -// mbar contract (recipes/mbar_handshake_design): full[s] count=1, arrive = -// producer's arrive_expect_tx(B+A bytes) + TMA complete_tx; empty[s] count = -// kCWarps (one elected lane per consumer warp after its last stage read). -// Producer = warp kCWarps lane 0; it never consumes, so the ring never -// self-blocks. Ring reuse distance = S stages, issue j waits empty parity -// ((j-S)/S)&1 — both derived from S. -// C = TMA cluster size for the A feed: ONE leader multicast per stage fills -// all C CTAs' A slots (cp_async_bulk_tensor_2d_load_multicast_cg1: each CTA's -// local full-bar gets its own tx-decrement, so expect_tx is unchanged). Per-CTA -// A bytes drop C-fold — the M-scaling A-tax (LEDGER O1) is A riding every -// CTA's capped TMA pipe. Contract deltas at C>1: the LEADER's empty[s] count -// = kCWarps*C (followers' consumers cluster-arrive it — release variant, so -// their slot reads are performed-before the leader's next multicast write); -// followers skip their own A issue but keep full expect_tx. -template -__global__ void __launch_bounds__(kThreads) oproj_ar_kernel( - const __grid_constant__ CUtensorMap w_map, - const __grid_constant__ CUtensorMap x_map, - const __grid_constant__ Params prm) { - constexpr int Mp = (M + 15) & ~15; // mma m16 padding (x buffer padded) - constexpr int MT = Mp / 16; - constexpr int KSTEPS = K / (kBK * CH); // OUTER stages: CH k-chunks batched - constexpr int kBBytes = kBNRows * kBK * 2, kABytes = Mp * kBK * 2; - constexpr int kStB = CH * kBBytes, kStA = CH * kABytes; - static_assert(K % (kBK * CH) == 0); - - const int tid = threadIdx.x, warp = tid >> 5, lane = tid & 31; - const Strip strip = Strip::make(blockIdx.x, gridDim.x); - __shared__ uint32_t cta_epoch; - if (tid == 0) cta_epoch = atomicAdd(prm.epoch_base + prm.fam, 1u) / uint32_t(gridDim.x); - __syncthreads(); - const uint32_t epoch = cta_epoch; - const int parity = int(epoch & 1); - const int ring = int(epoch % kRing); - // pinned wait-set: flag VA + monotonic targets resolved before any spin - const size_t foff = flags_off(R) + size_t(prm.fam) * 512; - const size_t doff2 = done_off(R) + size_t(prm.fam) * kRing * kMaxCTA * 4; - uint32_t* const flag_local = reinterpret_cast(prm.uc_base[prm.my_rank] + foff) + ring; - uint32_t* const done_local = reinterpret_cast(prm.uc_base[prm.my_rank] + doff2) + size_t(blockIdx.x); - uint32_t* const gather_fam = prm.gather + size_t(prm.fam) * 2 * kRing; - const uint32_t wrap = epoch / kRing + 1; - const uint32_t flag_target = wrap * R; - const uint32_t gath_target = wrap * gridDim.x; - - float4 acc[MT]; -#pragma unroll - for (int i = 0; i < MT; ++i) - acc[i] = make_float4(0.f, 0.f, 0.f, 0.f); - - extern __shared__ __align__(1024) uint8_t smem[]; - uint8_t* b_st = smem; // [S][CH][kBBytes] - uint8_t* a_st = b_st + size_t(S) * kStB; // [S][CH][kABytes] - uint64_t* fullb = reinterpret_cast(a_st + size_t(S) * kStA); - uint64_t* emptyb = fullb + S; - - const uint32_t crank = C > 1 ? device::ptx::cluster_cta_rank() : 0; - { - if (tid == 0) { - device::ptx::prefetch_tensormap(&w_map); - device::ptx::prefetch_tensormap(&x_map); -#pragma unroll - for (int s = 0; s < S; ++s) { - device::ptx::mbar_init(fullb + s, 1); - device::ptx::mbar_init(emptyb + s, crank == 0 ? kCWarps * C : kCWarps); - } - } - __syncthreads(); - if constexpr (C > 1) device::ptx::cluster_sync_rel_acq(); - - if (warp == kCWarps) { - // ---- producer: the whole K stream, one thread ----------------- - // k-phase rotation: spreads B's DRAM pages across CTAs; fp32 - // accumulation order changes per CTA — a sum, gate-covered. - // CLUSTER-uniform: all members consume the same A flight. - const int phase = ((int(blockIdx.x) / C) * KSTEPS) / (int(gridDim.x) / C); - if (lane == 0) { - for (int j = 0; j < KSTEPS; ++j) { - const int slot = j % S; - const int jj = (j + phase) % KSTEPS; - if (j >= S) device::ptx::mbar_wait_parity(emptyb + slot, ((j - S) / S) & 1); - device::ptx::mbar_arrive_expect_tx(fullb + slot, kStB + kStA); -#pragma unroll - for (int c = 0; c < CH; ++c) { - device::ptx::cp_async_bulk_tensor_2d_load( - device::ptx::to_shared(b_st + size_t(slot) * kStB + c * kBBytes), - &w_map, - (jj * CH + c) * kBK, - strip.t0 * 8, - fullb + slot); - if constexpr (C > 1) { - if (crank == 0) - device::ptx::cp_async_bulk_tensor_2d_load_multicast_cg1( - device::ptx::to_shared(a_st + size_t(slot) * kStA + c * kABytes), - &x_map, - (jj * CH + c) * kBK, - 0, - fullb + slot, - uint16_t((1u << C) - 1)); - } else { - device::ptx::cp_async_bulk_tensor_2d_load( - device::ptx::to_shared(a_st + size_t(slot) * kStA + c * kABytes), - &x_map, - (jj * CH + c) * kBK, - 0, - fullb + slot); - } - } - } - } - } else { - // ---- consumers: warp w owns n8-tile (strip.t0 + w) ------------ - // A fragments load straight from gmem (x is L2-hot and tiny; the - // LSU pipe is idle here) — the TMA path stays a pure-B stream. - const int b_row = (lane & 7) + warp * 8; // row within the B box - const int b_ka = (lane >> 3) & 1; // k-atom half (x2) - const int a_row = lane & 15, a_ka = lane >> 4; - for (int s = 0; s < KSTEPS; ++s) { - const int slot = s % S; - device::ptx::mbar_wait_parity(fullb + slot, (s / S) & 1); -#pragma unroll - for (int c = 0; c < CH; ++c) { - const uint32_t b_base = device::ptx::to_shared(b_st + size_t(slot) * kStB + c * kBBytes); -#pragma unroll - for (int k16 = 0; k16 < kBK / 16; ++k16) { - uint32_t b0, b1; - device::ptx::ldmatrix_x2_b16( - b_base + uint32_t(b_row) * (kBK * 2) + swz::smem_col_128b_bf16(b_row, (k16 * 2 + b_ka) * 8) * 2, - b0, - b1); -#pragma unroll - for (int mt = 0; mt < MT; ++mt) { - uint32_t a0, a1, a2, a3; - device::ptx::ldmatrix_x4_b16( - device::ptx::to_shared( - a_st + size_t(slot) * kStA + c * kABytes + uint32_t(mt * 16 + a_row) * (kBK * 2) + - swz::smem_col_128b_bf16(a_row, (k16 * 2 + a_ka) * 8) * 2), - a0, - a1, - a2, - a3); - device::ptx::mma_m16n8k16_bf16f32(acc[mt], a0, a1, a2, a3, b0, b1); - } - } - } - if (lane == 0) { - device::ptx::mbar_arrive(emptyb + slot); - if constexpr (C > 1) - if (crank != 0) device::ptx::mbar_arrive_cluster_release(emptyb + slot, 0); - } - } - } - } - - // ---- epilogue: push ---------------------------------------------------- - __syncthreads(); // whole CTA past its smem/feed reads - device::PDLTriggerSecondary(); // next launch streams weights under our - // push+boundary+reduce (needs 2-CTA/SM - // co-residency: 100% smem carveout + this - // kernel's smem ≤ ~113 KB) - device::PDLWaitPrimary(); // prior grid reached ITS trigger (k-loop end) — NOT done - { - // guard: epoch e-2 (same parity) fully reduced everywhere before we - // overwrite its slots. Steady-state this is already set (~one hot - // acquire); it binds only when a boundary straggles. - if (tid == 0 && epoch >= 2) { - const uint32_t e2 = epoch - 2; - const uint32_t tgt = (e2 / kRing + 1) * R; - while (device::ptx::load_acquire_sys(done_local + size_t(e2 % kRing) * kMaxCTA) < tgt) { - } - } - __syncthreads(); - } - const int tig = lane & 3, grp = lane >> 2; - const int n0 = (strip.t0 + warp) * 8; - const bool own = warp < strip.nt; // phantom 6th tile / producer never push - - // tile-major slot offset: tile t (global n8 index), row m, byte offset - auto slot_tm = [&](uint8_t* base, size_t sb, int t, int m) { return base + sb + (size_t(t) * M + m) * 16; }; - - { - const size_t sb = slot_off(parity, prm.my_rank, R); - if (own) { - const int t = strip.t0 + warp; -#pragma unroll - for (int mt = 0; mt < MT; ++mt) { -#pragma unroll - for (int half = 0; half < 2; ++half) { - const int m = mt * 16 + grp + half * 8; - if (m < M) { - const uint32_t v = - half ? bf2_u32(make_float2(acc[mt].z, acc[mt].w)) : bf2_u32(make_float2(acc[mt].x, acc[mt].y)); - // lanes (grp,tig) of one warp land contiguous: 128 B - // per (mt,half) group - const size_t boff = size_t(m) * 16 + tig * 4; - { -#pragma unroll - for (int r = 0; r < R; ++r) - *reinterpret_cast(slot_tm(prm.uc_base[r], sb, t, 0) + boff) = v; - } - } - } - } - } - } - - // ---- boundary (gather + flag + per-CTA local-replica spin) ------------- - __syncthreads(); - if (tid == 0) { - const uint32_t old = device::ptx::atomic_add_acq_rel_gpu(gather_fam + ring, 1); - if (old + 1 == gath_target) { - // ONE completing-CTA fence: publishes every CTA's pushes via the - // acq_rel gather chain. Measured 0.9 us better than per-CTA - // fences at bs1 (11.04 vs 11.97) — the parallel-drain theory lost. - device::ptx::fence_release_sys(); - { -#pragma unroll - for (int r = 0; r < R; ++r) - device::ptx::red_add_relaxed_sys(reinterpret_cast(prm.uc_base[r] + foff) + ring, 1); - } - } - while (device::ptx::load_acquire_sys(flag_local) < flag_target) { - } - } - __syncthreads(); - - // ---- reduce: each CTA finishes its own tiles from the LOCAL replica ---- - const int units = strip.nt * M; - for (int u = tid; u < units; u += kThreads) { - const int t = u / M, m = u % M, c0 = (strip.t0 + t) * 8; - const size_t soff = (size_t(strip.t0 + t) * M + m) * 16; // tile-major - uint4 res; - { - float2 s[4] = {{0.f, 0.f}, {0.f, 0.f}, {0.f, 0.f}, {0.f, 0.f}}; -#pragma unroll - for (int r = 0; r < R; ++r) { - const uint4 v = *reinterpret_cast(prm.uc_base[prm.my_rank] + slot_off(parity, r, R) + soff); - const uint32_t w4[4] = {v.x, v.y, v.z, v.w}; -#pragma unroll - for (int j = 0; j < 4; ++j) { - const float2 f = __bfloat1622float2(*reinterpret_cast(&w4[j])); - s[j].x += f.x; - s[j].y += f.y; - } - } - res = make_uint4(bf2_u32(s[0]), bf2_u32(s[1]), bf2_u32(s[2]), bf2_u32(s[3])); - } - *reinterpret_cast(reinterpret_cast(prm.out) + (size_t(m) * kN + c0) * 2) = res; - } - - // ---- done-publish: per-CTA, fence-free. The beacon carries no payload: - // the remote guard only needs its VALUE. This CTA's reduce loads are - // data-flow-complete (their values fed the out stores) before the - // syncthreads, so the relaxed beacon cannot pass an unfinished read; a - // fence here stamped at ~4 us draining against the co-resident feed. - __syncthreads(); - if (tid == 0) { - const size_t off = size_t(ring) * kMaxCTA + blockIdx.x; - { -#pragma unroll - for (int r = 0; r < R; ++r) - device::ptx::red_add_relaxed_sys(reinterpret_cast(prm.uc_base[r] + doff2) + off, 1); - } - } -} - -constexpr int kD3BM = 128, kD3BK = 64; -constexpr __host__ __device__ int d3_ns(int bn) { // ring depth by stage size - // deepest ring the smem budget fits, capped at the mbar array size. The - // old 12/11/4 step form silently ran M>=128 at NS=4 (BN64 stage 24.5 KB - // fits 9) — a 4-deep ring can't hide DRAM latency and was the dominant - // M=128 pole (21.4 us vs DG 17.2 at-shape). - const int stage = kD3BM * kD3BK * 2 + bn * kD3BK * 2; - const int ns = 220 * 1024 / stage; - return ns > 12 ? 12 : ns; -} -constexpr int kD3GroupN = 16, kD3KPer = kD3BK / 16; -constexpr int kD3Threads = 8 * 32; // warps 0,1,4-7 live -constexpr int kD3ABytes = kD3BM * kD3BK * 2; -// per-M BLOCK_N: smaller BN = more tiles = more feed pipes (grid-starvation -// at BN=256 measured M-flat ~30.7 us: 28 CTAs); bigger BN = less A-dup. -// bytes/(min(tiles,152)) + tensor-floor optimum: 64 for M<=128, 128 above. -// re-swept 2026-07-23 at floored d3_ns: M=128 BN64/NS9 16.41 vs BN128/NS6 -// 20.50 (ring depth, not BN, was the old 21.4 pole); 256 BN128/NS6 20.51; -// 512 BN256/NS4 30.9 (BN128's 224-on-152 nu-imbalance regressed to 38.9). -constexpr __host__ __device__ int d3_bn(int Mp) { - return Mp <= 128 ? 64 : (Mp == 256 ? 128 : 256); -} - -// drain-order slot offset: unit = (tile t, n-block nb, epi-warp w, lane l), -// 16 B each. Total = num_tiles * (BN/8) * 4 * 32 * 16 = Mpad*N*2 bytes. -SGL_DEVICE size_t d3_slot_off(int t, int nb, int w, int l, int nblk_per_tile) { - return ((size_t(t) * nblk_per_tile + nb) * 4u + w) * 32u * 16u + size_t(l) * 16u; -} - -// SWAP (M<=64): swapAB on the dense ring — A = W (M-slot carries the 7168 -// n-rows in 128-row strips, NO small-M padding tax), B = x^T (N-slot = Mp, -// as small as 8). Tiles = 56 n-strips; the drain's lane-rows become n and -// its cols become m — the same drain-order slots work, only the out mapping -// transposes (local scatter, cheap at these payloads). -template -__global__ void __launch_bounds__(kD3Threads) oproj_dense_ar_kernel( - const __grid_constant__ CUtensorMap x_tmap, // A = x [kMMax, K] - const __grid_constant__ CUtensorMap w_tmap, // B = W [kN, K] - const __grid_constant__ Params prm) { - constexpr bool SWAP = (M <= 64); - // two-shot planes share RS/reduce/out-copy; they differ only in the AG + - // flag transport (kTwoShot = NVLS multimem, kTwoShotPeer = pure P2P). - constexpr bool k2S = (COMM == Comm::kTwoShot || COMM == Comm::kTwoShotPeer); - // SWAP = the DG decode recipe (O9h): token-slot tiles of 16/32 m-cols — - // tiles = 56 strips × (Mp/tok) fills the grid (112 pipes at M≥32; the - // 56-tile form measured flat ~19.6-20.5, R11) — with a 12-deep ring. - // UMMA M=128 requires N ≥ 16 step 16 (bs1 pads the token slot to 16). - constexpr int Mp = SWAP ? (M < 16 ? 16 : ((M + 15) & ~15)) : (M + kD3BM - 1) / kD3BM * kD3BM; - constexpr int kD3BN = SWAP ? (Mp <= 32 ? 16 : 32) : d3_bn(Mp); - constexpr int kD3BBytes = kD3BN * kD3BK * 2; - constexpr int kGridM = SWAP ? kN / kD3BM : Mp / kD3BM; - constexpr int kGridN = SWAP ? Mp / kD3BN : kN / kD3BN; - constexpr int kTiles = kGridM * kGridN; - constexpr int kIters = K / kD3BK; - constexpr int kNBlk = kD3BN / 8; - - constexpr int kD3NS = d3_ns(kD3BN); - // tcgen05.alloc column count must be a power of two >= 32 - constexpr int kTmemCols = 2 * kD3BN <= 32 ? 32 - : 2 * kD3BN <= 64 ? 64 - : 2 * kD3BN <= 128 ? 128 - : 2 * kD3BN <= 256 ? 256 - : 512; - extern __shared__ __align__(1024) uint8_t smem_buf[]; - const uint32_t smem_base = device::ptx::to_shared(smem_buf); - constexpr uint32_t kSmemAOff = 0, kSmemBOff = kD3NS * kD3ABytes; - - __shared__ __align__(8) uint64_t tma_mbars[12]; - __shared__ __align__(8) uint64_t mma_mbars[12]; - __shared__ __align__(8) uint64_t mainloop_mbars[2]; - __shared__ __align__(8) uint64_t epi_mbars[2]; - __shared__ __align__(4) uint32_t s_taddr[1]; - __shared__ uint32_t cta_epoch; - - const int tid = threadIdx.x, warp_id = tid >> 5, lane_id = tid & 31; - const int bid = int(blockIdx.x), num_bids = int(gridDim.x); - if (tid == 0) cta_epoch = atomicAdd(prm.epoch_base + prm.fam, 1u) / uint32_t(gridDim.x); - __syncthreads(); - const uint32_t epoch = cta_epoch; - const int parity = int(epoch & 1); - const int ring = int(epoch % kRing); - const size_t foff = flags_off(R) + size_t(prm.fam) * 512; - const size_t doff2 = done_off(R) + size_t(prm.fam) * kRing * kMaxCTA * 4; - uint32_t* const flag_local = reinterpret_cast(prm.uc_base[prm.my_rank] + foff) + ring; - uint32_t* const done_local = reinterpret_cast(prm.uc_base[prm.my_rank] + doff2) + size_t(blockIdx.x); - uint32_t* const gather_fam = prm.gather + size_t(prm.fam) * 2 * kRing; - const uint32_t wrap = epoch / kRing + 1; - const uint32_t flag_target = wrap * R; - const uint32_t gath_target = wrap * gridDim.x; - const size_t sb = slot_off(parity, prm.my_rank, R); - - if (warp_id == 0 && device::ptx::elect_one()) { - for (int i = 0; i < kD3NS; ++i) { - device::ptx::mbar_init(&tma_mbars[i], 1); - device::ptx::mbar_init(&mma_mbars[i], 1); - } - for (int i = 0; i < 2; ++i) { - device::ptx::mbar_init(&mainloop_mbars[i], 1); - device::ptx::mbar_init(&epi_mbars[i], 4 * 32); - } - } else if (warp_id == 1) { - device::ptx::tcgen05_alloc(device::ptx::to_shared(s_taddr), kTmemCols); - } - __syncthreads(); - const uint32_t taddr = s_taddr[0]; - - constexpr uint32_t i_desc = device::ptx::mma_inst_desc_f16( - kD3BM, - kD3BN, - device::ptx::F16Type::BF16, - device::ptx::F16Type::BF16, - device::ptx::DType::F32, - device::ptx::Major::K, - device::ptx::Major::K); - auto tile_mn = [&](int linear) -> int2 { - return dense_gemm_mainloop::group_n_swizzle<1, kD3GroupN>(linear, 0, kGridM, kGridN); - }; - - // Prefetch the first ring of input-independent weight stages before the - // PDL dependency. SWAP changes which operand slot contains W, but never - // changes which tensor map is safe to touch here. - if (warp_id == 0 && device::ptx::elect_one()) { - constexpr int kPrefetch = kIters < kD3NS ? kIters : kD3NS; - const int2 mn = tile_mn(bid); -#pragma unroll - for (int k = 0; k < kPrefetch; ++k) { - device::ptx::mbar_arrive_expect_tx(&tma_mbars[k], kD3ABytes + kD3BBytes); - device::ptx::cp_async_bulk_tensor_2d_load( - smem_base + (SWAP ? kSmemAOff + k * kD3ABytes : kSmemBOff + k * kD3BBytes), - &w_tmap, - k * kD3BK, - (SWAP ? mn.x * kD3BM : mn.y * kD3BN), - &tma_mbars[k]); - } - } - - // x and all slot traffic remain behind the dependency. - device::PDLWaitPrimary(); - - // done-guard before any slot write (W3; PDL wait pairs with the trigger) - { - if (tid == 0 && epoch >= 2) { - const uint32_t e2 = epoch - 2; - const uint32_t tgt = (e2 / kRing + 1) * R; - while (device::ptx::load_acquire_sys(done_local + size_t(e2 % kRing) * kMaxCTA) < tgt) { - } - } - __syncthreads(); - } - - if (warp_id == 0 && device::ptx::elect_one()) { - // TMA issuer (simple persistent) — verbatim dense_1cta fp8out shape - int stage = 0, mma_phase = 1; - for (int t = bid; t < kTiles; t += num_bids) { - const int2 mn = tile_mn(t); - for (int k = 0; k < kIters; ++k) { - device::ptx::mbar_wait_parity(&mma_mbars[stage], mma_phase); - constexpr bool kDropA = false; - const bool prefetched = (t == bid && k < kD3NS); - if (!prefetched) device::ptx::mbar_arrive_expect_tx(&tma_mbars[stage], (kDropA ? 0 : kD3ABytes) + kD3BBytes); - if constexpr (!kDropA) { - // In SWAP, A is the prefetched weight; otherwise A is x. - if (!prefetched || !SWAP) - device::ptx::cp_async_bulk_tensor_2d_load( - smem_base + kSmemAOff + stage * kD3ABytes, - SWAP ? &w_tmap : &x_tmap, - k * kD3BK, - mn.x * kD3BM, - &tma_mbars[stage]); - } - // In SWAP, B is x; otherwise B is the prefetched weight. - if (!prefetched || SWAP) - device::ptx::cp_async_bulk_tensor_2d_load( - smem_base + kSmemBOff + stage * kD3BBytes, - SWAP ? &x_tmap : &w_tmap, - k * kD3BK, - mn.y * kD3BN, - &tma_mbars[stage]); - if (++stage == kD3NS) { - stage = 0; - mma_phase ^= 1; - } - } - } - } else if (warp_id == 1 && device::ptx::elect_one()) { - // MMA issuer with 2-stage TMEM ping-pong - int stage = 0, tma_phase = 0, ml_stage = 0, epi_phase = 1; - for (int t = bid; t < kTiles; t += num_bids) { - device::ptx::mbar_wait_parity(&epi_mbars[ml_stage], epi_phase); - const uint32_t tmem_d = taddr + uint32_t(ml_stage) * kD3BN; - for (int k = 0; k < kIters; ++k) { - device::ptx::mbar_wait_parity(&tma_mbars[stage], tma_phase); - device::ptx::tcgen05_fence_after_thread_sync(); - const uint32_t a_smem = smem_base + kSmemAOff + stage * kD3ABytes; - const uint32_t b_smem = smem_base + kSmemBOff + stage * kD3BBytes; -#pragma unroll - for (int k2 = 0; k2 < kD3KPer; ++k2) { - const uint64_t da = device::ptx::mma_smem_desc_k_major(a_smem + uint32_t(k2) * 32); - const uint64_t db = device::ptx::mma_smem_desc_k_major(b_smem + uint32_t(k2) * 32); - device::ptx::tcgen05_mma_f16(tmem_d, da, db, i_desc, (k == 0 && k2 == 0) ? 0u : 1u); - } - device::ptx::tcgen05_commit_arrive(&mma_mbars[stage]); - if (++stage == kD3NS) { - stage = 0; - tma_phase ^= 1; - } - } - device::ptx::tcgen05_commit_arrive(&mainloop_mbars[ml_stage]); - ml_stage ^= 1; - if (ml_stage == 0) epi_phase ^= 1; - } - } else if (warp_id >= 4) { - // epilogue: BF16 drain (dense_1cta idiom) → coalesced comm push - const int epi_warp = warp_id & 3; - const uint32_t taddr_lane = uint32_t(epi_warp * 32) << 16; - int ml_stage = 0, ml_phase = 0; - for (int t = bid; t < kTiles; t += num_bids) { - const int2 mn = tile_mn(t); - device::ptx::mbar_wait_parity(&mainloop_mbars[ml_stage], ml_phase); - device::ptx::tcgen05_fence_after_thread_sync(); - const uint32_t tmem_d_base = taddr + uint32_t(ml_stage) * kD3BN; - const int row = mn.x * kD3BM + epi_warp * 32 + lane_id; // SWAP: n -#pragma unroll 4 - for (int nb = 0; nb < kNBlk; ++nb) { - const uint32_t taddr_n = tmem_d_base + uint32_t(nb) * 8 + taddr_lane; - uint32_t r0, r1, r2, r3, r4, r5, r6, r7; - device::ptx::tcgen05_ld_32x32b_x8(taddr_n, r0, r1, r2, r3, r4, r5, r6, r7); - device::ptx::tcgen05_wait_ld(); - uint4 v; - v.x = device::ptx::cvt_pack_f32x2_to(__int_as_float(r1), __int_as_float(r0)); - v.y = device::ptx::cvt_pack_f32x2_to(__int_as_float(r3), __int_as_float(r2)); - v.z = device::ptx::cvt_pack_f32x2_to(__int_as_float(r5), __int_as_float(r4)); - v.w = device::ptx::cvt_pack_f32x2_to(__int_as_float(r7), __int_as_float(r6)); - if (SWAP || row < M) { // SWAP masks pad m-cols below - { - const size_t off = sb + d3_slot_off(t, nb, epi_warp, lane_id, kNBlk); - if constexpr (k2S) - // RS: unicast to tile-owner only — 1x egress and, - // spread over the tile loop, absorbed under the - // mainloop (O9b: one-shot is R x-payload INGRESS- - // bound; the composite pays its RS serially) - *reinterpret_cast(prm.uc_base[t % R] + off) = v; - else { -#pragma unroll - for (int r = 0; r < R; ++r) - *reinterpret_cast(prm.uc_base[r] + off) = v; - } - } - } - } - (void)device::ptx::mbar_arrive(&epi_mbars[ml_stage]); - ml_stage ^= 1; - if (ml_stage == 0) ml_phase ^= 1; - } - } - __syncthreads(); - device::PDLTriggerSecondary(); - if (warp_id == 1) { - device::ptx::tcgen05_dealloc(taddr, kTmemCols); - device::ptx::tcgen05_relinquish(); - } - - // ---- boundary (fam rings) — verbatim member-1 contract ---------------- - if (tid == 0) { - const uint32_t old = device::ptx::atomic_add_acq_rel_gpu(gather_fam + ring, 1); - if (old + 1 == gath_target) { - device::ptx::fence_release_sys(); - { -#pragma unroll - for (int r = 0; r < R; ++r) - device::ptx::red_add_relaxed_sys(reinterpret_cast(prm.uc_base[r] + foff) + ring, 1); - } - } - while (device::ptx::load_acquire_sys(flag_local) < flag_target) { - } - } - __syncthreads(); - - if constexpr (k2S) { - // ---- owner-reduce + AG: reduce MY tiles, store the result to all - // replicas' pull region (kTwoShot: one mm.st, fabric replicates; - // kTwoShotPeer: R unicast stores — (R-1)× the egress, same ingress); - // then boundary 2 gates the out-copy --------------------------------- - for (int u = tid + bid * kD3Threads; u < kTiles * kNBlk * 4 * 32; u += num_bids * kD3Threads) { - const int l = u & 31, w = (u >> 5) & 3, nb = (u >> 7) % kNBlk; - const int t = u / (kNBlk * 128); - if (t % R != prm.my_rank) continue; // not my slab - const int2 mn = tile_mn(t); - if (!SWAP && mn.x * kD3BM + w * 32 + l >= M) continue; - const size_t soff = d3_slot_off(t, nb, w, l, kNBlk); - float2 acc2[4] = {{0.f, 0.f}, {0.f, 0.f}, {0.f, 0.f}, {0.f, 0.f}}; -#pragma unroll - for (int r = 0; r < R; ++r) { - const uint4 vv = *reinterpret_cast(prm.uc_base[prm.my_rank] + slot_off(parity, r, R) + soff); - const uint32_t w4[4] = {vv.x, vv.y, vv.z, vv.w}; -#pragma unroll - for (int j = 0; j < 4; ++j) { - const float2 f = __bfloat1622float2(*reinterpret_cast(&w4[j])); - acc2[j].x += f.x; - acc2[j].y += f.y; - } - } - const uint4 res = make_uint4(bf2_u32(acc2[0]), bf2_u32(acc2[1]), bf2_u32(acc2[2]), bf2_u32(acc2[3])); - { -#pragma unroll - for (int r = 0; r < R; ++r) - *reinterpret_cast(prm.uc_base[r] + pull_off(parity, R) + soff) = res; - } - } - // boundary 2 (second flag/gather ring words at +256 B / +kRing) - __syncthreads(); - if (tid == 0) { - uint32_t* const g2 = gather_fam + kRing; // AG gather ring - const uint32_t old = device::ptx::atomic_add_acq_rel_gpu(g2 + ring, 1); - if (old + 1 == gath_target) { - device::ptx::fence_release_sys(); - { -#pragma unroll - for (int r = 0; r < R; ++r) - device::ptx::red_add_relaxed_sys(reinterpret_cast(prm.uc_base[r] + foff + 256) + ring, 1); - } - } - while (device::ptx::load_acquire_sys(reinterpret_cast(prm.uc_base[prm.my_rank] + foff + 256) + ring) < - flag_target) { - } - } - __syncthreads(); - // out-copy: every CTA writes its grid-partition of out from the - // LOCAL reduced replica - for (int u = tid + bid * kD3Threads; u < kTiles * kNBlk * 4 * 32; u += num_bids * kD3Threads) { - const int l = u & 31, w = (u >> 5) & 3, nb = (u >> 7) % kNBlk; - const int t = u / (kNBlk * 128); - const int2 mn = tile_mn(t); - const int row = mn.x * kD3BM + w * 32 + l; // SWAP: row = n, full - if (!SWAP && row >= M) continue; // range; m masked below - const uint4 res = *reinterpret_cast( - prm.uc_base[prm.my_rank] + pull_off(parity, R) + d3_slot_off(t, nb, w, l, kNBlk)); - if constexpr (SWAP) { - const uint32_t u4[4] = {res.x, res.y, res.z, res.w}; -#pragma unroll - for (int j = 0; j < 8; ++j) { - const int mm = mn.y * kD3BN + nb * 8 + j; - if (mm < M) - *reinterpret_cast(reinterpret_cast(prm.out) + (size_t(mm) * kN + row) * 2) = - uint16_t((u4[j >> 1] >> ((j & 1) * 16)) & 0xFFFFu); - } - } else { - *reinterpret_cast( - reinterpret_cast(prm.out) + (size_t(row) * kN + size_t(mn.y) * kD3BN + nb * 8) * 2) = res; - } - } - __syncthreads(); - if (tid == 0) { - const size_t off = size_t(ring) * kMaxCTA + blockIdx.x; - { -#pragma unroll - for (int r = 0; r < R; ++r) - device::ptx::red_add_relaxed_sys(reinterpret_cast(prm.uc_base[r] + doff2) + off, 1); - } - } - return; - } - - // ---- reduce: descatter drain-order slots → out[m][n] ------------------- - // unit = (t, nb, w, l): row = mn.x*BM + w*32 + l; cols = mn.y*BN + nb*8. - const int units = kTiles * kNBlk * 4 * 32; - for (int u = tid + bid * kD3Threads; u < units; u += num_bids * kD3Threads) { - const int l = u & 31, w = (u >> 5) & 3, nb = (u >> 7) % kNBlk; - const int t = u / (kNBlk * 128); - const int2 mn = tile_mn(t); - const int row = mn.x * kD3BM + w * 32 + l; - if (!SWAP && row >= M) continue; - const size_t soff = d3_slot_off(t, nb, w, l, kNBlk); - uint4 res; - { - float2 acc2[4] = {{0.f, 0.f}, {0.f, 0.f}, {0.f, 0.f}, {0.f, 0.f}}; -#pragma unroll - for (int r = 0; r < R; ++r) { - const uint4 vv = *reinterpret_cast(prm.uc_base[prm.my_rank] + slot_off(parity, r, R) + soff); - const uint32_t w4[4] = {vv.x, vv.y, vv.z, vv.w}; -#pragma unroll - for (int j = 0; j < 4; ++j) { - const float2 f = __bfloat1622float2(*reinterpret_cast(&w4[j])); - acc2[j].x += f.x; - acc2[j].y += f.y; - } - } - res = make_uint4(bf2_u32(acc2[0]), bf2_u32(acc2[1]), bf2_u32(acc2[2]), bf2_u32(acc2[3])); - } - if constexpr (SWAP) { - const uint32_t u4[4] = {res.x, res.y, res.z, res.w}; -#pragma unroll - for (int j = 0; j < 8; ++j) { - const int mm = mn.y * kD3BN + nb * 8 + j; - if (mm < M) - *reinterpret_cast(reinterpret_cast(prm.out) + (size_t(mm) * kN + row) * 2) = - uint16_t((u4[j >> 1] >> ((j & 1) * 16)) & 0xFFFFu); - } - } else { - *reinterpret_cast( - reinterpret_cast(prm.out) + (size_t(row) * kN + size_t(mn.y) * kD3BN + nb * 8) * 2) = res; - } - } - - // ---- done publish (fence-free beacon, W4) ------------------------------ - __syncthreads(); - if (tid == 0) { - const size_t off = size_t(ring) * kMaxCTA + blockIdx.x; - { -#pragma unroll - for (int r = 0; r < R; ++r) - device::ptx::red_add_relaxed_sys(reinterpret_cast(prm.uc_base[r] + doff2) + off, 1); - } - } -} - -template -struct Launcher3 { - static constexpr bool kSwap = (M <= 64); - static constexpr int Mp = kSwap ? (M < 16 ? 16 : ((M + 15) & ~15)) : (M + kD3BM - 1) / kD3BM * kD3BM; - static constexpr int kBN = kSwap ? (Mp <= 32 ? 16 : 32) : d3_bn(Mp); - static constexpr int kTiles = kSwap ? (kN / kD3BM) * (Mp / kBN) : (Mp / kD3BM) * (kN / kBN); - static constexpr int kGrid = kTiles < 152 ? kTiles : 152; - static constexpr size_t kSmem = size_t(d3_ns(kBN)) * (kD3ABytes + kBN * kD3BK * 2); - static void set_smem_attr() { - CHECK_CUDA(cudaFuncSetAttribute( - oproj_dense_ar_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, int(kSmem))); - CHECK_CUDA(cudaFuncSetAttribute( - oproj_dense_ar_kernel, cudaFuncAttributePreferredSharedMemoryCarveout, 100)); - } - static void - launch(const CUtensorMap& x_tmap, const CUtensorMap& w_tmap, const Params& prm, cudaStream_t stream, bool pdl) { - cudaLaunchConfig_t cfg{}; - cudaLaunchAttribute attr[1]; - int na = 0; - if (pdl) { - attr[na].id = cudaLaunchAttributeProgrammaticStreamSerialization; - attr[na].val.programmaticStreamSerializationAllowed = 1; - ++na; - } - cfg.gridDim = dim3(unsigned(kGrid)); - cfg.blockDim = dim3(kD3Threads); - cfg.dynamicSmemBytes = kSmem; - cfg.stream = stream; - cfg.attrs = attr; - cfg.numAttrs = unsigned(na); - CHECK_CUDA(cudaLaunchKernelEx(&cfg, oproj_dense_ar_kernel, x_tmap, w_tmap, prm)); - } -}; - -template < - int M, - int K, - int R, - Comm COMM, - bool GEMM_ON, - int S = ((M + 15) & ~15) <= 16 ? 6 : (((M + 15) & ~15) == 32 ? 4 : 3), - int CH = 2, - int C = 1> // cluster-multicast A axis: C=2 REFUTED as-built at - // S=3/4 rings (LEDGER R9 — pair-lockstep pacing beats - // the halved A bytes); flip here to test siblings -struct Launcher { - static constexpr int Mp = (M + 15) & ~15; - static constexpr size_t kSmem = GEMM_ON - ? size_t(S) * CH * (kBNRows * kBK * 2 + Mp * kBK * 2) + 2 * S * sizeof(uint64_t) - : 4096; // AR-only path never touches the feed ring - // once per device, before first launch - static void set_smem_attr() { - CHECK_CUDA(cudaFuncSetAttribute( - oproj_ar_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, int(kSmem))); - // 100% carveout → the SM smem config fits TWO CTAs (this grid's tail - // + the next PDL grid's feed); the default config blocks dual - // residency and with it the whole tail-hiding scheme. - CHECK_CUDA(cudaFuncSetAttribute( - oproj_ar_kernel, cudaFuncAttributePreferredSharedMemoryCarveout, 100)); - } - // pdl=false = the NON-COOPERATIVE-neighbor regime: no programmatic - // serialization attribute, so successive kernels fully serialize — the - // AR tail is exposed, as it is in a serving stack whose adjacent kernels - // don't PDL-cooperate or can't co-reside. In-kernel griddepcontrol ops - // are no-ops without the attribute; the done-guard is trivially met. - static void launch( - const CUtensorMap& w_map, - const CUtensorMap& x_map, - const Params& prm, - int ncta, - cudaStream_t stream, - bool pdl) { - cudaLaunchConfig_t cfg{}; - cudaLaunchAttribute attr[2]; - int na = 0; - if (pdl) { - attr[na].id = cudaLaunchAttributeProgrammaticStreamSerialization; - attr[na].val.programmaticStreamSerializationAllowed = 1; - ++na; - } - if constexpr (C > 1) { - attr[na].id = cudaLaunchAttributeClusterDimension; - attr[na].val.clusterDim.x = C; - attr[na].val.clusterDim.y = 1; - attr[na].val.clusterDim.z = 1; - ++na; - } - cfg.gridDim = dim3(unsigned(ncta)); - cfg.blockDim = dim3(kThreads); - cfg.dynamicSmemBytes = kSmem; - cfg.stream = stream; - cfg.attrs = attr; - cfg.numAttrs = unsigned(na); - CHECK_CUDA(cudaLaunchKernelEx(&cfg, oproj_ar_kernel, w_map, x_map, prm)); - } -}; - -} // namespace oproj_ar - -} // namespace sglang - -// ================= sglang tvm-ffi adapter ================= - -#include // For TensorMatcher, SymbolicSize, SymbolicDevice -#include // For CHECK_HOST - -#include // For bf16_t, TVMFFIEnvGetStream - -#include - -#include -#include -#include - -namespace sglang { - -namespace oproj_ar_ffi { - -using namespace oproj_ar; -using tvm::ffi::TensorView; - -constexpr int kCellList[7] = {8, 16, 32, 64, 128, 256, 512}; - -inline int cell_of(int m) { - for (int c : kCellList) - if (m <= c) return c; - return -1; -} - -template -struct GemmArKernel { - static_assert(R >= 2 && R <= 8, "R outside the validated 2..8 range"); - static_assert(K % 128 == 0 && K >= 128, "K must be a multiple of 128"); - - static constexpr int kTwoShotMinM = R >= 8 ? 128 : 256; - - // one ring family per dispatch cell (see kFams) - static int64_t fam_of(int64_t m) { - const int cell = cell_of(int(m)); - CHECK_HOST(cell > 0) << "gemm_ar: M=" << m << " outside [1, " << kMMax << "]"; - for (int i = 0; i < 7; ++i) - if (kCellList[i] == cell) return i; - return -1; - } - - static int64_t cell_of_ffi(int64_t m) { - return cell_of(int(m)); - } - static int64_t region_nbytes() { - return int64_t(region_bytes(R)); - } - static int64_t gather_words() { - return int64_t(kFams) * 2 * kRing; - } - static int64_t num_fams() { - return kFams; - } - static int64_t max_tokens() { - return kMMax; - } - - // Per-weight-pointer W tensor maps (encode once; weights are static). - struct WMaps { - CUtensorMap w48, w64, w128, w256; - }; - - static const WMaps& w_maps(void* w) { - static std::unordered_map cache; - static std::mutex mu; - std::lock_guard lk(mu); - auto it = cache.find(w); - if (it == cache.end()) { - auto enc = [&](uint32_t box_rows) { - return tmap::encode_tiled_2d( - w, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, kN, K, size_t(K) * 2, box_rows, kBK, CU_TENSOR_MAP_SWIZZLE_128B); - }; - it = cache.emplace(w, WMaps{enc(kBNRows), enc(64), enc(128), enc(256)}).first; - } - return it->second; - } - - // x tensor map over the caller's [M, K] tensor: global rows = M, TMA - // zero-fills the [M, cell) padding rows out-of-bounds. - static CUtensorMap x_map(void* x, int m, uint32_t box_rows) { - return tmap::encode_tiled_2d( - x, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, uint64_t(m), K, size_t(K) * 2, box_rows, kBK, CU_TENSOR_MAP_SWIZZLE_128B); - } - - static void set_smem_attrs_once() { - static bool done = [] { - Launcher<8, K, R, Comm::kPeer, true>::set_smem_attr(); - Launcher<16, K, R, Comm::kPeer, true>::set_smem_attr(); - Launcher3<32, K, R, Comm::kPeer>::set_smem_attr(); - Launcher3<64, K, R, Comm::kPeer>::set_smem_attr(); - if constexpr (kTwoShotMinM > 128) Launcher3<128, K, R, Comm::kPeer>::set_smem_attr(); - if constexpr (kTwoShotMinM <= 128) Launcher3<128, K, R, Comm::kTwoShotPeer>::set_smem_attr(); - Launcher3<256, K, R, Comm::kTwoShotPeer>::set_smem_attr(); - Launcher3<512, K, R, Comm::kTwoShotPeer>::set_smem_attr(); - return true; - }(); - (void)done; - } - - template - static void enqueue_cell(const WMaps& wm, void* x, int m, const Params& prm, cudaStream_t stream, bool pdl) { - if constexpr (CELL <= 16) { - const CUtensorMap xm = x_map(x, m, 16); - Launcher::launch(wm.w48, xm, prm, kM1Grid_(), stream, pdl); - } else if constexpr (CELL < kTwoShotMinM) { - const CUtensorMap xm = x_map(x, m, CELL <= 32 ? 16 : (CELL <= 64 ? 32 : 128)); - const CUtensorMap& wmap = CELL <= 64 ? wm.w128 : wm.w64; - Launcher3::launch(xm, wmap, prm, stream, pdl); - } else { - const CUtensorMap xm = x_map(x, m, 128); - const CUtensorMap& wmap = CELL == 128 ? wm.w64 : (CELL == 256 ? wm.w128 : wm.w256); - Launcher3::launch(xm, wmap, prm, stream, pdl); - } - } - - static constexpr int kM1Grid_() { - return 152; - } - - // per-rank UC VAs of the comm region, stashed host-side ONCE at init: - // per-call CPU-tensor derefs from inside the op are not reliable in every - // execution context (observed dangling under the sglang scheduler). - static std::array& bases_store() { - static std::array a{}; - return a; - } - - static void set_bases(TensorView uc_bases) { - using namespace host; - TensorMatcher({R}).with_dtype().verify(uc_bases); - const int64_t* b = static_cast(uc_bases.data_ptr()); - for (int r = 0; r < R; ++r) - bases_store()[r] = reinterpret_cast(b[r]); - } - - static void - run(TensorView out, - TensorView x, - TensorView w, - TensorView gather, // [kFams * 2 * kRing] int32 CUDA, device-local - TensorView epochs, // [kFams] int32 CUDA: device-resident CTA ticket counters - int64_t my_rank) { - using namespace host; - - auto M = SymbolicSize{"num_tokens"}; - auto CellRows = SymbolicSize{"cell_rows"}; - auto device = SymbolicDevice{}; - device.set_options(); - TensorMatcher({M, K}).with_dtype().with_device(device).verify(x); - TensorMatcher({kN, K}).with_dtype().with_device(device).verify(w); - TensorMatcher({CellRows, kN}).with_dtype().with_device(device).verify(out); - - const int m = int(M.unwrap()); - const int cell = cell_of(m); - CHECK_HOST(cell > 0) << "gemm_ar: M=" << m << " outside [1, " << kMMax << "]"; - CHECK_HOST(int64_t(CellRows.unwrap()) == cell) - << "out must have cell(M)=" << cell << " rows, got " << CellRows.unwrap(); - CHECK_HOST(my_rank >= 0 && my_rank < R); - CHECK_HOST(bases_store()[0] != nullptr) << "gemm_ar: set_bases not called"; - TensorMatcher({int64_t(kFams) * 2 * kRing}).with_dtype().verify(gather); - TensorMatcher({kFams}).with_dtype().verify(epochs); - - set_smem_attrs_once(); - - const DLDevice dev = device.unwrap(); - const auto stream = static_cast(::TVMFFIEnvGetStream(dev.device_type, dev.device_id)); - - Params prm{}; - prm.mc_base = nullptr; // pure-P2P plane - for (int r = 0; r < R; ++r) - prm.uc_base[r] = bases_store()[r]; - prm.gather = static_cast(gather.data_ptr()); - prm.out = static_cast<__nv_bfloat16*>(out.data_ptr()); - prm.partial_in = nullptr; - prm.epoch_base = static_cast(epochs.data_ptr()); - prm.my_rank = int(my_rank); - prm.fam = int(fam_of(m)); - - const bool pdl = kUsePDL; - const WMaps& wm = w_maps(w.data_ptr()); - switch (cell) { - case 8: - enqueue_cell<8>(wm, x.data_ptr(), m, prm, stream, pdl); - break; - case 16: - enqueue_cell<16>(wm, x.data_ptr(), m, prm, stream, pdl); - break; - case 32: - enqueue_cell<32>(wm, x.data_ptr(), m, prm, stream, pdl); - break; - case 64: - enqueue_cell<64>(wm, x.data_ptr(), m, prm, stream, pdl); - break; - case 128: - enqueue_cell<128>(wm, x.data_ptr(), m, prm, stream, pdl); - break; - case 256: - enqueue_cell<256>(wm, x.data_ptr(), m, prm, stream, pdl); - break; - default: - enqueue_cell<512>(wm, x.data_ptr(), m, prm, stream, pdl); - break; - } - CHECK_CUDA(cudaGetLastError()) << "gemm_ar launch (cell=" << cell << ")"; - } -}; - -} // namespace oproj_ar_ffi - -using oproj_ar_ffi::GemmArKernel; - -} // namespace sglang diff --git a/python/sglang/kernels/ops/__init__.py b/python/sglang/kernels/ops/__init__.py index 42cc5ccf9b..ee1c1b2a11 100644 --- a/python/sglang/kernels/ops/__init__.py +++ b/python/sglang/kernels/ops/__init__.py @@ -27,7 +27,6 @@ _GROUPS = ( "layernorm", "mamba", "memory", - "minicpm_sala", "mm", "moe", "quantization", diff --git a/python/sglang/kernels/ops/attention/__init__.py b/python/sglang/kernels/ops/attention/__init__.py index 037a927156..f64f3fe2de 100644 --- a/python/sglang/kernels/ops/attention/__init__.py +++ b/python/sglang/kernels/ops/attention/__init__.py @@ -184,7 +184,6 @@ for _mod, _fn in [ ("dsa.cp_split", "dsa_cp_interleave_q_seqs_kernel"), ("dsv4.fp4_indexer", "quantize_fp4_indexer_tensor"), ("dsv4.fp4_indexer", "store_fp4_index_k_cache"), - ("dsv4.rms_normalize_hip", "rms_normalize_triton"), ]: register_kernel( KernelSpec( @@ -287,3 +286,64 @@ for _mod, _fn, _backend, _device in [ ) ) del _mod, _fn, _backend, _device + + +# Public entry points inventoried by logical operator group (RFC #29630). +register_kernel( + KernelSpec( + op="attention.get_block_table", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.attention.minicpm_sala.get_block_table:get_block_table", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="attention.fast_kpool_topk_transform_fused", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.attention.dsa.kpool_topk_transform:fast_kpool_topk_transform_fused", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="attention.fast_topk", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.attention.fast_topk:fast_topk", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="attention.fused_k_indexer_norm_rope", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.attention.dsa.indexer_k:fused_k_indexer_norm_rope", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="attention.fused_k_indexer_norm_rope_store", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.attention.dsa.indexer_k:fused_k_indexer_norm_rope_store", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="attention.fused_rope_wo_a_bf16", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.attention.dsv4.wo_a:fused_rope_wo_a_bf16", + capabilities=frozenset( + {CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))} + ), + ) +) +register_kernel( + KernelSpec( + op="attention.fp4_index_logits_decode", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.attention.dsv4.fp4_indexer:fp4_index_logits_decode", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) diff --git a/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py b/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py index 56649c1e7a..befdff24c2 100644 --- a/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py +++ b/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py @@ -557,10 +557,6 @@ def _gather_dequant_requant_fp8_paged_kernel( tl.store(dst_ptr, data, mask=mask) -if __name__ == "__main__": - raise Exception("UT is in quant_k_cache.py") - - @triton.jit def _concat_cast_kv_fp8_pad_kernel( out_ptr, diff --git a/python/sglang/kernels/ops/quantization/dsv32/elementwise.py b/python/sglang/kernels/ops/attention/dsa/indexer_k.py similarity index 100% rename from python/sglang/kernels/ops/quantization/dsv32/elementwise.py rename to python/sglang/kernels/ops/attention/dsa/indexer_k.py diff --git a/python/sglang/kernels/ops/moe/kpool_topk_transform.py b/python/sglang/kernels/ops/attention/dsa/kpool_topk_transform.py similarity index 100% rename from python/sglang/kernels/ops/moe/kpool_topk_transform.py rename to python/sglang/kernels/ops/attention/dsa/kpool_topk_transform.py diff --git a/python/sglang/kernels/ops/attention/dsa/quant_k_cache.py b/python/sglang/kernels/ops/attention/dsa/quant_k_cache.py index d2e524093e..536aae4658 100644 --- a/python/sglang/kernels/ops/attention/dsa/quant_k_cache.py +++ b/python/sglang/kernels/ops/attention/dsa/quant_k_cache.py @@ -397,128 +397,3 @@ def _quantize_k_cache_fast_kernel( data = tl.load(src_ptr, mask=mask) tl.store(dst_ptr, data, mask=mask) - - -if __name__ == "__main__": - import dequant_k_cache - - for num_blocks, block_size in [ - (1, 1), - (10, 64), - ]: - dim_nope_and_rope = 512 + 64 - - input_k_cache = torch.randn( - (num_blocks, block_size, 1, dim_nope_and_rope), - dtype=torch.bfloat16, - device="cuda", - ) - - ref_quant = _quantize_k_cache_ref(input_k_cache) - actual_quant = _quantize_k_cache_fast_wrapped(input_k_cache) - - ref_ref_dequant = dequant_k_cache._dequantize_k_cache_slow(ref_quant) - ref_actual_dequant = dequant_k_cache._dequantize_k_cache_fast_wrapped(ref_quant) - actual_actual_dequant = dequant_k_cache._dequantize_k_cache_fast_wrapped( - actual_quant - ) - - print(f"{ref_ref_dequant=}") - print(f"{actual_actual_dequant=}") - print(f"{actual_actual_dequant - ref_ref_dequant=}") - print(f"{torch.mean(ref_ref_dequant - actual_actual_dequant)=}") - - # TODO too different? - torch.testing.assert_close( - ref_ref_dequant, ref_actual_dequant, atol=0.2, rtol=0.2 - ) - torch.testing.assert_close( - ref_ref_dequant, actual_actual_dequant, atol=0.2, rtol=0.2 - ) - - # test dequant_k_cache_paged - page_table_1 = torch.arange( - num_blocks * block_size, dtype=torch.int32, device="cuda" - ) - actual_dequant_paged = dequant_k_cache.dequantize_k_cache_paged( - actual_quant, page_table_1 - ).reshape(actual_actual_dequant.shape) - print(f"{torch.mean(actual_actual_dequant - actual_dequant_paged)=}") - torch.testing.assert_close( - ref_ref_dequant, actual_dequant_paged, atol=0.2, rtol=0.2 - ) - - print("Passed") - - # Test quantize_k_cache_separate: verify output matches concat path - print("\nTesting quantize_k_cache_separate...") - for num_tokens in [64, 100]: - dim_nope = 512 - dim_rope = 64 - - k_nope = torch.randn( - num_tokens, 1, dim_nope, dtype=torch.bfloat16, device="cuda" - ) - k_rope = torch.randn( - num_tokens, 1, dim_rope, dtype=torch.bfloat16, device="cuda" - ) - - # Old path: concat then quantize - k_concat = torch.cat([k_nope, k_rope], dim=-1).squeeze(1) # (num_tokens, 576) - old_output = quantize_k_cache(k_concat.unsqueeze(1).unsqueeze(1)) # 4D input - old_output = old_output.squeeze(1).squeeze(1) # Back to (num_tokens, 656) - - # New path: quantize separately - nope_part, rope_part = quantize_k_cache_separate(k_nope, k_rope) - new_bytes = torch.cat([nope_part.squeeze(1), rope_part.squeeze(1)], dim=-1) - - # Compare byte-level equality - old_bytes = old_output.view(torch.uint8) - - if old_bytes.shape != new_bytes.shape: - raise RuntimeError( - f"Shape mismatch: {old_bytes.shape} vs {new_bytes.shape}" - ) - - diff_bytes = (old_bytes != new_bytes).sum().item() - if diff_bytes > 0: - max_diff = (old_bytes.float() - new_bytes.float()).abs().max().item() - raise RuntimeError( - f"quantize_k_cache_separate output doesn't match concat path: " - f"{diff_bytes} differing bytes, max_diff={max_diff}" - ) - - print(f" num_tokens={num_tokens}: PASSED (outputs match byte-wise)") - - print("quantize_k_cache_separate tests passed!") - - print("\nDo benchmark...") - - for num_blocks, block_size in [ - (1, 64), - (64, 64), - (128, 64), - (256, 64), - (512, 64), - (1024, 64), - (2048, 64), - ]: - dim_nope_and_rope = 512 + 64 - - input_k_cache = torch.randn( - (num_blocks, block_size, 1, dim_nope_and_rope), - dtype=torch.bfloat16, - device="cuda", - ) - - actual_quant = _quantize_k_cache_fast_wrapped(input_k_cache) - - page_table_1 = torch.arange( - num_blocks * block_size, dtype=torch.int32, device="cuda" - ) - - def run_ans(): - return dequant_k_cache.dequantize_k_cache_paged(actual_quant, page_table_1) - - ans_time: float = triton.testing.do_bench(run_ans, warmup=10, rep=20) / 1000 # type: ignore - print(f"seq_kv: {num_blocks * block_size}, time: {ans_time * 1e6: 4.0f} us") diff --git a/python/sglang/kernels/ops/attention/dsa/transform_index.py b/python/sglang/kernels/ops/attention/dsa/transform_index.py index 506add4704..8c2d0c2f5a 100644 --- a/python/sglang/kernels/ops/attention/dsa/transform_index.py +++ b/python/sglang/kernels/ops/attention/dsa/transform_index.py @@ -375,14 +375,3 @@ def transform_index_page_table_prefill_ref( ) offset += l return result - - -if __name__ == "__main__": - bs, topk, max_seqlen = 10, 2048, 3000 - page_table = torch.randint(0, 100, (bs, max_seqlen), device="cuda") - topk_indices = torch.full((bs, topk), -1, device="cuda") - topk_indices[:, :1600] = torch.arange(1600).unsqueeze(0).repeat(bs, 1) - ref_result = transform_index_page_table_decode_ref(page_table, topk_indices) - result = transform_index_page_table_decode_fast(page_table, topk_indices) - assert torch.all(result == ref_result) - print("Passed") diff --git a/python/sglang/kernels/ops/attention/dsv4/__init__.py b/python/sglang/kernels/ops/attention/dsv4/__init__.py index a99daa3135..a92f72bb1a 100644 --- a/python/sglang/kernels/ops/attention/dsv4/__init__.py +++ b/python/sglang/kernels/ops/attention/dsv4/__init__.py @@ -23,15 +23,6 @@ from .elementwise import ( fused_rope_inplace, ) from .fp8_wo_a import sglang_per_token_group_quant_fp8_dsv4_wo_a -from .gemm import linear_bf16_fp32 -from .moe import ( - hash_topk, - mask_topk_ids, - mega_moe_pre_dispatch, - silu_and_mul_clamp, - silu_and_mul_contig_post_quant, - silu_and_mul_masked_post_quant, -) from .topk import ( plan_topk_v2, topk_transform_paged, @@ -56,17 +47,10 @@ __all__ = [ "fused_k_norm_rope_flashmla", "sglang_per_token_group_quant_fp8_dsv4_wo_a", "make_name", - "linear_bf16_fp32", "get_paged_mqa_logits_metadata", "triton_create_paged_compress_data", "topk_transform_paged", "topk_transform_paged_v2", "topk_transform_ragged_v2", "plan_topk_v2", - "hash_topk", - "mega_moe_pre_dispatch", - "mask_topk_ids", - "silu_and_mul_clamp", - "silu_and_mul_masked_post_quant", - "silu_and_mul_contig_post_quant", ] diff --git a/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py b/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py index 9d24953b13..2cbe2e0069 100644 --- a/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py +++ b/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py @@ -572,33 +572,3 @@ def gather_dequant_requant_fp8_paged_ref( ) out[: active.shape[0]] = active return out - - -if __name__ == "__main__": - assert torch.cuda.is_available(), "this self-test needs a CUDA device" - torch.manual_seed(0) - device = "cuda" - - page_size = 64 - num_pages = 8 - num_tokens = 333 - raw_bytes = page_size * (NOPE_ROPE_BYTES + PADDED_SCALE_PER_TOKEN) - bytes_per_page = ( - (raw_bytes + NOPE_ROPE_BYTES - 1) // NOPE_ROPE_BYTES - ) * NOPE_ROPE_BYTES - - quant_k_cache = torch.randint( - 0, 256, (num_pages, bytes_per_page), dtype=torch.uint8, device=device - ) - page_table = torch.randint( - 0, num_pages * page_size, (num_tokens,), dtype=torch.int32, device=device - ) - - out_kernel = dequantize_k_cache_paged(quant_k_cache, page_table, page_size) - out_ref = dequantize_k_cache_paged_ref(quant_k_cache, page_table, page_size) - - torch.testing.assert_close(out_kernel, out_ref, atol=0, rtol=0, equal_nan=True) - print( - f"OK: kernel matches torch ref for {num_tokens} tokens " - f"(page_size={page_size}, bytes_per_page={bytes_per_page})" - ) diff --git a/python/sglang/kernels/ops/attention/dsv4/wo_a.py b/python/sglang/kernels/ops/attention/dsv4/wo_a.py index f9218e4213..08df08123f 100644 --- a/python/sglang/kernels/ops/attention/dsv4/wo_a.py +++ b/python/sglang/kernels/ops/attention/dsv4/wo_a.py @@ -3,11 +3,8 @@ from __future__ import annotations from typing import TYPE_CHECKING, Sequence import torch -import triton -import triton.language as tl from sglang.kernels.jit.utils import cache_once, cuda_stubs_dir, load_jit -from sglang.kernels.ops.layernorm.mxfp8_epilogue import ue8m0_scale if TYPE_CHECKING: from tvm_ffi.module import Module @@ -52,133 +49,3 @@ def fused_rope_wo_a_bf16( y = x.new_empty((num_tokens, N_OUT)) module.run(x, weight, freqs_cis, positions, None, None, y) return (y,) - - -@triton.jit -def _wo_a_bf16_gemv_kernel(X, W, Y, R: tl.constexpr, D: tl.constexpr, BN: tl.constexpr): - group = tl.program_id(1) - rows = tl.program_id(0) * BN + tl.arange(0, BN) - columns = tl.arange(0, D) - x = tl.load(X + group * D + columns).to(tl.float32) - w = tl.load( - W + (group * R + rows[:, None]) * D + columns[None, :], - rows[:, None] < R, - 0, - ).to(tl.float32) - result = tl.sum(w * x[None, :], axis=1) - tl.store(Y + group * R + rows, result, rows < R) - - -def wo_a_bf16_gemv(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: - """Compute ``einsum('tgd,grd->tgr', x, weight)`` for one token.""" - assert x.shape[0] == 1 and x.ndim == weight.ndim == 3 - assert x.dtype == weight.dtype == torch.bfloat16 - assert x.is_cuda and x.device == weight.device - assert x.is_contiguous() and weight.is_contiguous() - groups, rows, dim = weight.shape - assert x.shape[1:] == (groups, dim) and dim == triton.next_power_of_2(dim) - result = torch.empty((1, groups, rows), dtype=x.dtype, device=x.device) - # One output row per CTA keeps register use low and exposes enough - # independent weight loads for single-token decode. - _wo_a_bf16_gemv_kernel[(rows, groups)]( - x, - weight, - result, - rows, - dim, - 1, - num_warps=4, - enable_fp_fusion=False, - ) - return result - - -@triton.jit -def _wo_a_partial(X, W, P, M: tl.constexpr, SX: tl.constexpr): - tile, group, split = tl.program_id(0), tl.program_id(1), tl.program_id(2) - m = tl.arange(0, 16) - n = tile * 64 + tl.arange(0, 64) - k = split * 512 + tl.arange(0, 128) - acc = tl.zeros((16, 64), tl.float32) - for i in range(4): - offsets = k + i * 128 - x = tl.load( - X + m[:, None] * SX + group * 4096 + offsets[None, :], m[:, None] < M, 0 - ) - w = tl.load(W + (group * 1024 + n[None, :]) * 4096 + offsets[:, None]) - acc += tl.dot(x, w) - tl.store( - P + ((split * M + m[:, None]) * 2 + group) * 1024 + n[None, :], - acc, - m[:, None] < M, - ) - - -@triton.jit -def _wo_a_reduce(P, Y, E: tl.constexpr): - i = tl.program_id(0) * 256 + tl.arange(0, 256) - split = tl.arange(0, 8) - values = tl.load(P + split[:, None] * E + i[None, :], i[None, :] < E, 0) - tl.store(Y + i, tl.sum(values, 0), i < E) - - -def wo_a_bf16_small_batch(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: - """Compute ``einsum('tgd,grd->tgr', x, weight)`` for the TP4 WO-A shape. - Partial sums stay in FP32 until the final BF16, token-major store.""" - m = x.shape[0] - assert 2 <= m <= 8 and x.shape[1:] == (2, 4096) - assert weight.shape == (2, 1024, 4096) and weight.is_contiguous() - assert x.dtype == weight.dtype == torch.bfloat16 - assert x.is_cuda and x.device == weight.device - assert x.stride(2) == 1 and x.stride(1) == 4096 and x.stride(0) >= 8192 - result = torch.empty((m, 2, 1024), dtype=x.dtype, device=x.device) - partial = torch.empty((8, m, 2, 1024), dtype=torch.float32, device=x.device) - _wo_a_partial[(16, 2, 8)]( - x, weight, partial, m, x.stride(0), num_warps=4, num_stages=3 - ) - _wo_a_reduce[(triton.cdiv(m * 2048, 256),)](partial, result, m * 2048, num_warps=4) - return result - - -@triton.jit -def _wo_a_reduce_quant(P, Q, S, M: tl.constexpr): - row, tile = tl.program_id(0), tl.program_id(1) - i = tile * 256 + tl.arange(0, 256) - split = tl.arange(0, 8) - v = tl.load(P + split[:, None] * (M * 2048) + row * 2048 + i[None, :]) - y = tl.sum(v, 0).to(tl.bfloat16).to(tl.float32).reshape((8, 32)) - amax = tl.max(tl.abs(y), 1) - sf, inv = ue8m0_scale(amax) - quant = tl.minimum(tl.maximum(y * inv[:, None], -448.0), 448.0).to(tl.float8e4nv) - tl.store(Q + row * 2048 + i, quant.reshape((256,))) - col = tile * 8 + tl.arange(0, 8) - off = (col // 4) * 512 + row * 16 + col % 4 - tl.store(S + off, sf.to(tl.uint8)) - # Zero only padding rows; valid scale bytes have disjoint writers above. - for z in tl.static_range(triton.cdiv(8192, M * 8 * 256)): - s = (row * 8 + tile) * 256 + tl.arange(0, 256) + z * (M * 8 * 256) - sr = (s % 512) // 16 + ((s % 16) // 4) * 32 - tl.store(S + s, 0, (s < 8192) & (sr >= M)) - - -def _quantize_partial(p): - m = p.shape[1] - q = torch.empty((m, 2048), device=p.device, dtype=torch.float8_e4m3fn) - s = torch.empty(8192, device=p.device, dtype=torch.uint8) - _wo_a_reduce_quant[(m, 8)](p, q, s, m, num_warps=4) - return q, s - - -def wo_a_bf16_small_batch_mxfp8(x: torch.Tensor, weight: torch.Tensor): - """WO-A with BF16 rounding followed by FlashInfer-compatible MXFP8 quantization.""" - m = x.shape[0] - assert 2 <= m <= 8 and x.shape[1:] == (2, 4096) - assert x.dtype == weight.dtype == torch.bfloat16 - assert weight.shape == (2, 1024, 4096) and weight.is_contiguous() - assert x.is_cuda and x.device == weight.device - assert x.stride(2) == 1 and x.stride(1) == 4096 and x.stride(0) >= 8192 - partial = torch.empty((8, m, 2, 1024), dtype=torch.float32, device=x.device) - _wo_a_partial[(16, 2, 8)]( - x, weight, partial, m, x.stride(0), num_warps=4, num_stages=3 - ) - return _quantize_partial(partial) diff --git a/python/sglang/kernels/ops/elementwise/fast_topk.py b/python/sglang/kernels/ops/attention/fast_topk.py similarity index 100% rename from python/sglang/kernels/ops/elementwise/fast_topk.py rename to python/sglang/kernels/ops/attention/fast_topk.py diff --git a/python/sglang/kernels/ops/attention/log_scaling_tau.py b/python/sglang/kernels/ops/attention/log_scaling_tau.py index 7502421dad..3884d732b8 100644 --- a/python/sglang/kernels/ops/attention/log_scaling_tau.py +++ b/python/sglang/kernels/ops/attention/log_scaling_tau.py @@ -50,7 +50,7 @@ def apply_log_scaling_tau(x: torch.Tensor, tau: torch.Tensor) -> torch.Tensor: # Vectorized JIT kernel (16B loads, one row divide per vector) -- # bit-identical output (same fp32-mul + bf16-round), ~2-3x the # scalar triton kernel below at every size. - from sglang.kernels.ops.attention.inkling_row_scale import row_scale_bf16 + from sglang.kernels.ops.elementwise.row_scale import row_scale_bf16 x2d = torch.as_strided(x, (rows, inner), (x.stride(0), 1)) return row_scale_bf16(x2d, tau.reshape(rows).float()).view(x.shape) diff --git a/python/sglang/kernels/ops/minicpm_sala/__init__.py b/python/sglang/kernels/ops/attention/minicpm_sala/__init__.py similarity index 66% rename from python/sglang/kernels/ops/minicpm_sala/__init__.py rename to python/sglang/kernels/ops/attention/minicpm_sala/__init__.py index 7a7978bd4f..8b9e7904f9 100644 --- a/python/sglang/kernels/ops/minicpm_sala/__init__.py +++ b/python/sglang/kernels/ops/attention/minicpm_sala/__init__.py @@ -5,12 +5,16 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any if TYPE_CHECKING: - from sglang.kernels.ops.minicpm_sala.get_block_table import get_block_table + from sglang.kernels.ops.attention.minicpm_sala.get_block_table import ( + get_block_table, + ) def __getattr__(name: str) -> Any: if name == "get_block_table": - from sglang.kernels.ops.minicpm_sala.get_block_table import get_block_table + from sglang.kernels.ops.attention.minicpm_sala.get_block_table import ( + get_block_table, + ) globals()[name] = get_block_table return get_block_table diff --git a/python/sglang/kernels/ops/minicpm_sala/get_block_table.py b/python/sglang/kernels/ops/attention/minicpm_sala/get_block_table.py similarity index 100% rename from python/sglang/kernels/ops/minicpm_sala/get_block_table.py rename to python/sglang/kernels/ops/attention/minicpm_sala/get_block_table.py diff --git a/python/sglang/kernels/ops/communication/__init__.py b/python/sglang/kernels/ops/communication/__init__.py index 2f8ee65a28..6994b74b6e 100644 --- a/python/sglang/kernels/ops/communication/__init__.py +++ b/python/sglang/kernels/ops/communication/__init__.py @@ -19,7 +19,6 @@ for _mod, _fn in [ ("all_reduce_residual", "all_reduce_pull_res"), ("all_reduce_residual", "all_reduce_pull_norm"), ("gemm_ag", "gemm_ag_up_proj"), - ("gemm_ar", "o_proj_gemm_ar"), ("sp_collective", "reduce_scatter_res"), ("sp_collective", "reduce_scatter_pull"), ("sp_collective", "all_gather"), diff --git a/python/sglang/kernels/ops/communication/gemm_ar.py b/python/sglang/kernels/ops/communication/gemm_ar.py deleted file mode 100644 index 9d398e5046..0000000000 --- a/python/sglang/kernels/ops/communication/gemm_ar.py +++ /dev/null @@ -1,216 +0,0 @@ -"""K3 fused o_proj GEMM + all-reduce for decode (bf16, TP row-parallel). - -One entry point over ``csrc/kimi_k3/comm/gemm_ar.cuh``: a single kernel per -rank computes the local ``x_r [M, K] @ W_r [7168, K]^T`` partial AND the -cross-rank sum — the epilogue pushes finished tiles straight into a -peer-mapped P2P comm region, one flag boundary, then a tile-local reduce -writes the fully reduced ``out [M, 7168]`` on every rank. Replaces the -o_proj GEMM + NCCL all-reduce pair with one launch (see GEMM_AR_README.md). - -Contracts: - -* bf16 only; ``out = sum_r bf16(x_r @ W_r^T)`` (partials round to bf16 - pre-sum — same numerics as the unfused bf16 GEMM + ring AR). -* M in [1, 512]; internally rounded up to a tuned cell {8, 16, 32, 64, - 128, 256, 512}. ``out`` is allocated with ``cell`` rows and sliced. -* SM100+ with full NVLink P2P (fabric/MNNVL across nodes); perf-tuned on - GB300 (sm_103a). -* CUDA-graph compatible: the per-cell launch epoch lives in device memory - (read at kernel entry, bumped by a trailing kernel), so replays advance - it naturally. Each dispatch cell owns its own flag-ring family — no - host-side ring reset, ever. -* All TP ranks must call :func:`o_proj_gemm_ar` with the same M in - lockstep (same stream order of cells on every rank). -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, NamedTuple, Optional - -import torch - -from sglang.kernels.jit.utils import ( - cache_once, - is_arch_support_pdl, - load_jit, - make_cpp_args, -) -from sglang.srt.utils.custom_op import register_custom_op - -if TYPE_CHECKING: - from tvm_ffi.module import Module - -N = 7168 # K3 hidden size (OPROJ_N compile-time default in gemm_ar.cuh) -MAX_TOKENS = 512 # kMMax - - -@cache_once -def _jit_module(k: int, world_size: int) -> Module: - args = make_cpp_args( - k, - world_size, - is_arch_support_pdl(), - ) - cls = f"GemmArKernel<{args}>" - return load_jit( - "kimi_k3_gemm_ar", - *args, - cuda_files=["kimi_k3/comm/gemm_ar.cuh"], - cuda_wrappers=[ - ("run", f"{cls}::run"), - ("set_bases", f"{cls}::set_bases"), - ("region_nbytes", f"{cls}::region_nbytes"), - ("gather_words", f"{cls}::gather_words"), - ("num_fams", f"{cls}::num_fams"), - ], - extra_cuda_cflags=["-O3"], - extra_dependencies=["cutlass"], - ) - - -class _State(NamedTuple): - world_size: int - rank: int - region: tuple # (slab tensor, peer buffer views) — keeps mappings alive - uc_bases: torch.Tensor # [R] int64 CPU: per-rank UC VAs of the region - gather: torch.Tensor # [kFams * 2 * kRing] int32 CUDA, device-local - epochs: torch.Tensor # [kFams] int32 CUDA: device-resident CTA ticket counters - - -_STATE: Optional[_State] = None - - -def init( - *, - world_size: int, - rank: int, - group: torch.distributed.ProcessGroup, - k: int, -) -> None: - """Allocate + rendezvous the P2P comm region (collective; call once from - every TP rank, BEFORE any CUDA-graph capture). ``group`` is the TP CPU - (gloo) group used for the symm-mem rendezvous.""" - global _STATE - if _STATE is not None: - return - # the empty_strided_p2p + get_buffer path is the one that exchanges - # fabric handles and maps every peer (incl. cross-node MNNVL) into this - # process — the mem-pool rendezvous(tensor, group_name) API leaves - # remote-node (and sometimes even local) peers unmapped. - from torch._C._distributed_c10d import _SymmetricMemory - - mod = _jit_module(k, world_size) - nbytes = int(mod.region_nbytes()) - device = torch.device("cuda", torch.cuda.current_device()) - if torch.__version__ < "2.11.0": - import torch.distributed._symmetric_memory as torch_symm_mem - - torch_symm_mem.enable_symm_mem_for_group(group.group_name) - region = _SymmetricMemory.empty_strided_p2p( - (nbytes,), [1], torch.uint8, device, group.group_name - ) - symm = _SymmetricMemory.rendezvous(region) - region.zero_() - torch.cuda.synchronize() - torch.distributed.barrier(group=group) - # keep the peer buffer tensors alive alongside the region - peer_bufs = [symm.get_buffer(r, [nbytes], torch.uint8) for r in range(world_size)] - ptrs = [t.data_ptr() for t in peer_bufs] - import logging - - logging.getLogger(__name__).info( - "gemm_ar comm region: rank=%d nbytes=%d uc_bases=%s", - rank, - nbytes, - [hex(p) for p in ptrs], - ) - assert all(p != 0 for p in ptrs), f"gemm_ar: null peer pointers {ptrs}" - # explicit cpu: model build may run under a cuda default-device context, - # and a silently-cuda tensor here means the host-side deref in set_bases - # reads a device pointer (segfault) - uc_bases = torch.tensor(ptrs, dtype=torch.int64, device="cpu") - gather = torch.zeros(int(mod.gather_words()), dtype=torch.int32, device=device) - epochs = torch.zeros(int(mod.num_fams()), dtype=torch.int32, device=device) - torch.cuda.synchronize() - _STATE = _State( - world_size=world_size, - rank=rank, - region=(region, peer_bufs), - uc_bases=uc_bases, - gather=gather, - epochs=epochs, - ) - - -def initialized() -> bool: - return _STATE is not None - - -@cache_once -def _module_with_bases(k: int, world_size: int) -> Module: - """The per-K JIT module with the comm-region base addresses stashed - host-side (per-call CPU-tensor derefs from inside the custom op segfault - under the sglang scheduler, so the module holds them in a static).""" - state = _STATE - assert state is not None - mod = _jit_module(k, world_size) - mod.set_bases(state.uc_bases) - return mod - - -def fits(x: torch.Tensor) -> bool: - """Whether this o_proj input can take the fused GEMM+AR path.""" - return ( - _STATE is not None - and x.dim() == 2 - and x.dtype == torch.bfloat16 - and 0 < x.shape[0] <= MAX_TOKENS - and x.stride(1) == 1 - and x.stride(0) == x.shape[1] - ) - - -@register_custom_op(mutates_args=["out", "epochs"]) -def _gemm_ar_op( - k: int, - world_size: int, - out: torch.Tensor, - x: torch.Tensor, - weight: torch.Tensor, - gather: torch.Tensor, - epochs: torch.Tensor, - my_rank: int, -) -> None: - _module_with_bases(k, world_size).run(out, x, weight, gather, epochs, my_rank) - - -def _cell_of(m: int) -> int: - for c in (8, 16, 32, 64, 128, 256, 512): - if m <= c: - return c - raise ValueError(f"gemm_ar: M={m} outside [1, {MAX_TOKENS}]") - - -def o_proj_gemm_ar(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: - """Fully reduced ``sum_r x_r @ weight_r^T`` on every rank, one kernel. - - ``x`` is the TP-local [M, K] o_proj input, ``weight`` the TP-local - [7168, K] o_proj weight shard. Caller checked :func:`fits`; all ranks - call in lockstep with the same M. - """ - state = _STATE - assert state is not None - m = x.shape[0] - cell = _cell_of(m) - out = torch.empty((cell, N), dtype=torch.bfloat16, device=x.device) - _gemm_ar_op( - weight.shape[1], - state.world_size, - out, - x, - weight, - state.gather, - state.epochs, - state.rank, - ) - return out[:m] diff --git a/python/sglang/kernels/ops/elementwise/__init__.py b/python/sglang/kernels/ops/elementwise/__init__.py index da0d838fcb..0fa42400cb 100644 --- a/python/sglang/kernels/ops/elementwise/__init__.py +++ b/python/sglang/kernels/ops/elementwise/__init__.py @@ -8,4 +8,34 @@ Individual functions register (or are imported) under the functional op id they logically belong to. """ +from sglang.kernels.registry import register_kernel +from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec + __all__ = [] + + +# Public entry points inventoried by logical operator group (RFC #29630). +register_kernel( + KernelSpec( + op="elementwise.fused_qwen4_gate_value", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.elementwise.qwen4_gate:fused_qwen4_gate_value", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="elementwise.fused_qwen4_gate_reduce", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.elementwise.qwen4_gate:fused_qwen4_gate_reduce", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="elementwise.row_scale_bf16", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.elementwise.row_scale:row_scale_bf16", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) diff --git a/python/sglang/kernels/ops/elementwise/qwen4_gate.py b/python/sglang/kernels/ops/elementwise/qwen4_gate.py new file mode 100644 index 0000000000..1959fb23eb --- /dev/null +++ b/python/sglang/kernels/ops/elementwise/qwen4_gate.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +_QWEN4_HC_COUNT = 4 +_QWEN4_HIDDEN_SIZE = 2560 + + +@triton.jit +def _round_bf16_to_fp32(value): + """RNE-round fp32 to BF16 precision while retaining an fp32 register.""" + + bits = value.to(tl.int32, bitcast=True) + rounding_bias = 0x7FFF + ((bits >> 16) & 1) + rounded_bits = (bits + rounding_bias) & -65536 + return rounded_bits.to(tl.float32, bitcast=True) + + +@triton.jit +def _qwen4_gate_value_kernel( + gate_ptr, + value_ptr, + output_ptr, + num_tokens, + HC_COUNT: tl.constexpr, + HIDDEN_SIZE: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + token_group = tl.program_id(0) + token = token_group // HC_COUNT + hidden = tl.arange(0, BLOCK_SIZE) + mask = (token < num_tokens) & (hidden < HIDDEN_SIZE) + + # `gate` arrives already rounded to bf16 by the eager multiply/reduce/divide; + # every remaining eager bf16 rounding boundary is reproduced below. + gate = tl.load(gate_ptr + token_group).to(tl.float32) + magnitude = tl.maximum(tl.abs(gate), 1.0e-6) + root = _round_bf16_to_fp32(tl.sqrt(magnitude)) + sign = tl.where(gate > 0.0, 1.0, tl.where(gate < 0.0, -1.0, 0.0)) + transformed = _round_bf16_to_fp32(root * sign) + activated = _round_bf16_to_fp32(tl.sigmoid(transformed)) + + value = tl.load(value_ptr + token * HIDDEN_SIZE + hidden, mask=mask, other=0.0).to( + tl.float32 + ) + output_offset = token_group * HIDDEN_SIZE + hidden + tl.store(output_ptr + output_offset, activated * value, mask=mask) + + +def can_fuse_qwen4_gate_value(gate: torch.Tensor, value: torch.Tensor) -> bool: + """Return whether inputs match Qwen4's fixed BF16 gate/value contract.""" + + return ( + gate.is_cuda + and gate.dtype == torch.bfloat16 + and gate.dim() == 3 + and gate.shape[1:] == (_QWEN4_HC_COUNT, 1) + and gate.is_contiguous() + and value.is_cuda + and value.dtype == gate.dtype + and value.shape == (gate.shape[0], _QWEN4_HIDDEN_SIZE) + and value.is_contiguous() + ) + + +def fused_qwen4_gate_value(gate: torch.Tensor, value: torch.Tensor) -> torch.Tensor: + """Apply Qwen4's post-reduction gate and value broadcast in one kernel.""" + + if not can_fuse_qwen4_gate_value(gate, value): + raise ValueError("unsupported input for fused Qwen4 PLE gate/value") + output = torch.empty( + (gate.shape[0], _QWEN4_HC_COUNT, _QWEN4_HIDDEN_SIZE), + dtype=value.dtype, + device=value.device, + ) + if gate.shape[0]: + _qwen4_gate_value_kernel[(gate.shape[0] * _QWEN4_HC_COUNT,)]( + gate, + value, + output, + gate.shape[0], + HC_COUNT=_QWEN4_HC_COUNT, + HIDDEN_SIZE=_QWEN4_HIDDEN_SIZE, + BLOCK_SIZE=4096, + num_warps=8, + ) + return output + + +@triton.jit +def _qwen4_gate_reduce_kernel( + key_ptr, + query_ptr, + value_ptr, + output_ptr, + HIDDEN_SIZE: tl.constexpr, + HC_COUNT: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + group = tl.program_id(0) + hidden = tl.arange(0, BLOCK_SIZE) + mask = hidden < HIDDEN_SIZE + key = tl.load(key_ptr + group * HIDDEN_SIZE + hidden, mask, other=0).to(tl.float32) + query = tl.load(query_ptr + group * HIDDEN_SIZE + hidden, mask, other=0).to( + tl.float32 + ) + product = _round_bf16_to_fp32(key * query) + gate = _round_bf16_to_fp32(tl.sum(product, 0)) + gate = _round_bf16_to_fp32(gate * (HIDDEN_SIZE**-0.5)) + magnitude = _round_bf16_to_fp32(tl.maximum(tl.abs(gate), 1.0e-6)) + root = _round_bf16_to_fp32(tl.sqrt(magnitude)) + sign = tl.where(gate > 0, 1.0, tl.where(gate < 0, -1.0, 0.0)) + activated = _round_bf16_to_fp32(tl.sigmoid(_round_bf16_to_fp32(root * sign))) + value = tl.load( + value_ptr + (group // HC_COUNT) * HIDDEN_SIZE + hidden, mask, other=0 + ).to(tl.float32) + tl.store(output_ptr + group * HIDDEN_SIZE + hidden, activated * value, mask) + + +def can_fuse_qwen4_gate_reduce(key, query, value): + return ( + key.is_cuda + and key.dtype == torch.bfloat16 + and key.ndim == 3 + and key.shape[1:] == (_QWEN4_HC_COUNT, _QWEN4_HIDDEN_SIZE) + and key.is_contiguous() + and query.shape == key.shape + and query.dtype == key.dtype + and query.device == key.device + and query.is_contiguous() + and value.shape == (key.shape[0], _QWEN4_HIDDEN_SIZE) + and value.dtype == key.dtype + and value.device == key.device + and value.is_contiguous() + ) + + +def fused_qwen4_gate_reduce(key, query, value): + if not can_fuse_qwen4_gate_reduce(key, query, value): + raise ValueError("unsupported input for Qwen4 PLE gate reduction") + output = torch.empty_like(key) + if key.shape[0]: + _qwen4_gate_reduce_kernel[(key.shape[0] * _QWEN4_HC_COUNT,)]( + key, + query, + value, + output, + HIDDEN_SIZE=_QWEN4_HIDDEN_SIZE, + HC_COUNT=_QWEN4_HC_COUNT, + BLOCK_SIZE=triton.next_power_of_2(_QWEN4_HIDDEN_SIZE), + enable_fp_fusion=False, + ) + return output diff --git a/python/sglang/kernels/ops/attention/inkling_row_scale.py b/python/sglang/kernels/ops/elementwise/row_scale.py similarity index 71% rename from python/sglang/kernels/ops/attention/inkling_row_scale.py rename to python/sglang/kernels/ops/elementwise/row_scale.py index 48ca313882..ec8420c0e5 100644 --- a/python/sglang/kernels/ops/attention/inkling_row_scale.py +++ b/python/sglang/kernels/ops/elementwise/row_scale.py @@ -45,15 +45,3 @@ def row_scale_bf16( module = _jit_row_scale_module(is_arch_support_pdl()) module.run(x, tau, out) return out - - -def row_compact_bf16(x: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor: - """Contiguous copy of row-strided ``x`` ([rows, inner] bf16, inner - contiguous, inner % 8 == 0, 16B-aligned rows) -- the tau-less flavor of - ``row_scale_bf16``. Beats the TensorIterator strided copy that einsum's - reshape would otherwise run on such inputs.""" - if out is None: - out = torch.empty(x.shape, dtype=x.dtype, device=x.device) - module = _jit_row_scale_module(is_arch_support_pdl()) - module.run_compact(x, out) - return out diff --git a/python/sglang/kernels/ops/embeddings/__init__.py b/python/sglang/kernels/ops/embeddings/__init__.py index d318222ccb..07d916b344 100644 --- a/python/sglang/kernels/ops/embeddings/__init__.py +++ b/python/sglang/kernels/ops/embeddings/__init__.py @@ -1,7 +1,7 @@ """Embedding kernels.""" from sglang.kernels.registry import register_kernel -from sglang.kernels.spec import KernelBackend, KernelSpec +from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec register_kernel( KernelSpec( @@ -55,3 +55,14 @@ register_kernel( ) __all__ = [] + + +# Public entry points inventoried by logical operator group (RFC #29630). +register_kernel( + KernelSpec( + op="embeddings.fused_qwen4_ngram_hash", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.embeddings.qwen4_ngram:fused_qwen4_ngram_hash", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) diff --git a/python/sglang/kernels/ops/embeddings/qwen4_ngram.py b/python/sglang/kernels/ops/embeddings/qwen4_ngram.py new file mode 100644 index 0000000000..58392dd125 --- /dev/null +++ b/python/sglang/kernels/ops/embeddings/qwen4_ngram.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +_QWEN4_NGRAM_SIZE = 3 +_QWEN4_HEADS_PER_NGRAM = 8 +_QWEN4_NGRAM_HEADS = 16 + + +@triton.jit +def _qwen4_ngram_hash_kernel( + contexts_ptr, + multipliers_ptr, + vocab_sizes_ptr, + offsets_ptr, + output_ptr, + num_outputs, + eos_token_id, + NGRAM_SIZE: tl.constexpr, + HEADS_PER_NGRAM: tl.constexpr, + NGRAM_HEADS: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + output_idx = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = output_idx < num_outputs + token_idx = output_idx // NGRAM_HEADS + head_idx = output_idx % NGRAM_HEADS + context_base = token_idx * NGRAM_SIZE + + token_0 = tl.load(contexts_ptr + context_base, mask=mask, other=0) + token_1 = tl.load(contexts_ptr + context_base + 1, mask=mask, other=0) + token_2 = tl.load(contexts_ptr + context_base + 2, mask=mask, other=0) + multiplier_0 = tl.load(multipliers_ptr) + multiplier_1 = tl.load(multipliers_ptr + 1) + multiplier_2 = tl.load(multipliers_ptr + 2) + + # Only the final position of each three-token context is materialized. + previous_1 = tl.where(token_1 == eos_token_id, eos_token_id, token_1) + previous_2 = tl.where( + (token_0 == eos_token_id) | (token_1 == eos_token_id), + eos_token_id, + token_0, + ) + mixed = (token_2 * multiplier_0) ^ (previous_1 * multiplier_1) + mixed_3 = mixed ^ (previous_2 * multiplier_2) + mixed = tl.where(head_idx < HEADS_PER_NGRAM, mixed, mixed_3) + + vocab_size = tl.load(vocab_sizes_ptr + head_idx, mask=mask, other=1) + offset = tl.load(offsets_ptr + head_idx, mask=mask, other=0) + tl.store(output_ptr + output_idx, mixed % vocab_size + offset, mask=mask) + + +def can_fuse_qwen4_ngram_hash( + contexts: torch.Tensor, + multipliers: torch.Tensor, + vocab_sizes: torch.Tensor, + offsets: torch.Tensor, +) -> bool: + """Return whether inputs match the fixed Qwen4 PLE hash contract.""" + + return ( + contexts.is_cuda + and contexts.dtype == torch.long + and contexts.dim() == 2 + and contexts.shape[1] == _QWEN4_NGRAM_SIZE + and contexts.is_contiguous() + and multipliers.is_cuda + and multipliers.dtype == torch.long + and multipliers.numel() == _QWEN4_NGRAM_SIZE + and vocab_sizes.is_cuda + and vocab_sizes.dtype == torch.long + and vocab_sizes.numel() == _QWEN4_NGRAM_HEADS + and offsets.is_cuda + and offsets.dtype == torch.long + and offsets.numel() == _QWEN4_NGRAM_HEADS + ) + + +def fused_qwen4_ngram_hash( + contexts: torch.Tensor, + multipliers: torch.Tensor, + vocab_sizes: torch.Tensor, + offsets: torch.Tensor, + eos_token_id: int, +) -> torch.Tensor: + """Return the 16 Qwen4 PLE N-gram IDs in one kernel launch.""" + + if not can_fuse_qwen4_ngram_hash(contexts, multipliers, vocab_sizes, offsets): + raise ValueError("unsupported input for fused Qwen4 PLE N-gram hash") + output = torch.empty( + (contexts.shape[0], _QWEN4_NGRAM_HEADS), + dtype=torch.long, + device=contexts.device, + ) + num_outputs = output.numel() + if num_outputs: + block_size = 256 + _qwen4_ngram_hash_kernel[(triton.cdiv(num_outputs, block_size),)]( + contexts, + multipliers, + vocab_sizes, + offsets, + output, + num_outputs, + eos_token_id, + NGRAM_SIZE=_QWEN4_NGRAM_SIZE, + HEADS_PER_NGRAM=_QWEN4_HEADS_PER_NGRAM, + NGRAM_HEADS=_QWEN4_NGRAM_HEADS, + BLOCK_SIZE=block_size, + num_warps=4, + ) + return output diff --git a/python/sglang/kernels/ops/gemm/__init__.py b/python/sglang/kernels/ops/gemm/__init__.py index 9f9cb3b1b5..49a962d3bb 100644 --- a/python/sglang/kernels/ops/gemm/__init__.py +++ b/python/sglang/kernels/ops/gemm/__init__.py @@ -422,3 +422,140 @@ for _mod, _fn in _TRITON_KERNELS: ) ) del _mod, _fn + + +# Public entry points inventoried by logical operator group (RFC #29630). +register_kernel( + KernelSpec( + op="gemm.linear_bf16_fp32", + backend=KernelBackend.TORCH, + target="sglang.kernels.ops.gemm.bf16_fp32:_linear_bf16_fp32_cublas", + description=( + "Torch/cuBLAS implementation. The direct linear_bf16_fp32 entry also " + "retains its environment-controlled HPC-Ops, AITER and DeepGEMM dispatch." + ), + ) +) +register_kernel( + KernelSpec( + op="gemm.wo_a_bf16_gemv", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.gemm.dsv4_wo_a:wo_a_bf16_gemv", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="gemm.wo_a_bf16_small_batch", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.gemm.dsv4_wo_a:wo_a_bf16_small_batch", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="gemm.wo_a_bf16_small_batch_mxfp8", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.gemm.dsv4_wo_a:wo_a_bf16_small_batch_mxfp8", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="gemm.hopper_bf16_gemv", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.gemm.hopper_bf16_gemv:hopper_bf16_gemv", + capabilities=frozenset( + {CapabilityRequirement.cuda(min_sm=(9, 0), max_sm=(9, 0))} + ), + ) +) +register_kernel( + KernelSpec( + op="gemm.fp8_blockwise_scaled_mm", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.gemm.fp8_blockwise_gemm:fp8_blockwise_scaled_mm", + capabilities=frozenset( + {CapabilityRequirement.cuda(min_sm=(12, 0), max_sm=(12, 9))} + ), + ) +) +register_kernel( + KernelSpec( + op="gemm.sm120_fp8_gemv", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.gemm.sm120_fp8_gemv:sm120_fp8_gemv", + capabilities=frozenset( + {CapabilityRequirement.cuda(min_sm=(12, 0), max_sm=(12, 9))} + ), + ) +) +register_kernel( + KernelSpec( + op="gemm.gptq_marlin_gemm", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.gemm.gptq_marlin:gptq_marlin_gemm", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="gemm.rel_proj_small_t", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.gemm.inkling_rel_proj:rel_proj_small_t", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="gemm.fused_hc_mix", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.gemm.hc_mix:fused_hc_mix", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="gemm.w8a8_block_fp8_matmul", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.gemm.fp8_kernel:w8a8_block_fp8_matmul", + ) +) +register_kernel( + KernelSpec( + op="gemm.w8a8_block_fp8_matmul_triton", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.gemm.fp8_kernel:w8a8_block_fp8_matmul_triton", + capabilities=frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP}), + ) +) +register_kernel( + KernelSpec( + op="gemm.w8a8_block_fp8_matmul_deepgemm", + backend=KernelBackend.DEEPGEMM, + target="sglang.kernels.ops.gemm.fp8_kernel:w8a8_block_fp8_matmul_deepgemm", + capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}), + ) +) +register_kernel( + KernelSpec( + op="gemm.w8a8_mxfp8_matmul_deepgemm", + backend=KernelBackend.DEEPGEMM, + target="sglang.kernels.ops.gemm.fp8_kernel:w8a8_mxfp8_matmul_deepgemm", + capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(10, 0))}), + ) +) +register_kernel( + KernelSpec( + op="gemm.triton_scaled_mm", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.gemm.fp8_kernel:triton_scaled_mm", + ) +) +register_kernel( + KernelSpec( + op="gemm.w8a8_block_int8_matmul", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.gemm.int8_kernel:w8a8_block_int8_matmul", + ) +) diff --git a/python/sglang/kernels/ops/attention/dsv4/gemm.py b/python/sglang/kernels/ops/gemm/bf16_fp32.py similarity index 100% rename from python/sglang/kernels/ops/attention/dsv4/gemm.py rename to python/sglang/kernels/ops/gemm/bf16_fp32.py diff --git a/python/sglang/kernels/ops/quantization/configs/N=1024,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1024,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1024,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1024,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1152,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=1152,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1152,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json rename to python/sglang/kernels/ops/gemm/configs/N=1152,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1280,K=5120,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1280,K=5120,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1280,K=5120,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1280,K=5120,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1536,K=1536,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1536,K=1536,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1536,K=1536,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1536,K=1536,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1536,K=1536,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1536,K=1536,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1536,K=1536,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1536,K=1536,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1536,K=1536,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1536,K=1536,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1536,K=1536,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1536,K=1536,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1536,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=1536,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=1792,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=1792,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=1792,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json rename to python/sglang/kernels/ops/gemm/configs/N=1792,K=5120,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=2048,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=2048,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=2048,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=2048,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=2048,K=4096,device_name=NVIDIA_L40,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=2048,K=4096,device_name=NVIDIA_L40,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=2048,K=4096,device_name=NVIDIA_L40,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=2048,K=4096,device_name=NVIDIA_L40,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=2048,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=2048,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=2048,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=2048,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=2048,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=2048,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=2048,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=2048,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=2048,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=2048,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=2048,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=2048,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=2048,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=2048,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=2048,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=2048,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=2048,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=2048,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=2048,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=2048,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=2048,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=2048,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=2048,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=2048,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=2304,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=2304,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=2304,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=2304,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=2304,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=2304,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=2304,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=2304,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=2304,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=2304,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=2304,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=2304,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=2304,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=2304,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=24576,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=24576,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=24576,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=24576,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=24576,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=24576,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=24576,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=24576,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=24576,K=4096,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json b/python/sglang/kernels/ops/gemm/configs/N=24576,K=4096,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=24576,K=4096,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json rename to python/sglang/kernels/ops/gemm/configs/N=24576,K=4096,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json diff --git a/python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=24576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=24576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=256,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=256,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=256,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=256,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=256,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=256,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=256,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=256,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=256,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=256,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=256,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=256,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=256,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=256,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=256,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=256,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=256,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=256,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=256,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=256,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=25600,K=6144,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=25600,K=6144,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=25600,K=6144,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json rename to python/sglang/kernels/ops/gemm/configs/N=25600,K=6144,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=3072,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=3072,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=3072,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=3072,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=3072,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=3072,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=3072,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=3072,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=3072,K=1536,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=3072,K=1536,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=3072,K=1536,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=3072,K=1536,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=3072,K=1536,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=3072,K=1536,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=3072,K=1536,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=3072,K=1536,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=3072,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=3072,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=3072,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=3072,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=3072,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=3072,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=3072,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=3072,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=3072,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=3072,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=32768,K=1024,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=32768,K=1024,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=32768,K=1024,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=32768,K=1024,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=32768,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=32768,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=36864,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=36864,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=36864,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=36864,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4096,K=12288,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json b/python/sglang/kernels/ops/gemm/configs/N=4096,K=12288,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4096,K=12288,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json rename to python/sglang/kernels/ops/gemm/configs/N=4096,K=12288,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4096,K=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=4096,K=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4096,K=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json rename to python/sglang/kernels/ops/gemm/configs/N=4096,K=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4096,K=2048,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4096,K=2048,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4096,K=2048,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4096,K=2048,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4096,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4096,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4096,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4096,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4096,K=4096,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json b/python/sglang/kernels/ops/gemm/configs/N=4096,K=4096,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4096,K=4096,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json rename to python/sglang/kernels/ops/gemm/configs/N=4096,K=4096,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4096,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4096,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4096,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4096,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4096,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4096,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4096,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4096,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4096,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4096,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4096,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4096,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4096,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4096,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4096,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4096,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4096,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4096,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4096,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4096,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4096,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4096,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4096,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4096,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4096,K=8192,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4096,K=8192,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4096,K=8192,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4096,K=8192,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4608,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4608,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4608,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4608,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4608,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4608,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4608,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4608,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4608,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4608,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4608,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4608,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4608,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4608,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4608,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4608,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=4608,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=4608,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=512,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=512,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=512,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=512,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=512,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=512,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=512,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=512,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=512,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=512,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=512,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=512,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=512,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=512,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=512,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=512,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=512,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=512,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=512,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=512,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=512,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=512,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=512,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=512,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=5120,K=1024,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=5120,K=1024,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=5120,K=1024,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=5120,K=1024,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=5120,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=5120,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=5120,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json rename to python/sglang/kernels/ops/gemm/configs/N=5120,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=5120,K=2048,device_name=NVIDIA_L40,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=5120,K=2048,device_name=NVIDIA_L40,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=5120,K=2048,device_name=NVIDIA_L40,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=5120,K=2048,device_name=NVIDIA_L40,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=5120,K=3200,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=5120,K=3200,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=5120,K=3200,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=5120,K=3200,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=5120,K=576,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=5120,K=576,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=5120,K=576,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json rename to python/sglang/kernels/ops/gemm/configs/N=5120,K=576,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=6144,K=4096,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json b/python/sglang/kernels/ops/gemm/configs/N=6144,K=4096,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=6144,K=4096,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json rename to python/sglang/kernels/ops/gemm/configs/N=6144,K=4096,device_name=NVIDIA_L40S,dtype=fp8_w8a8_channelwise.json diff --git a/python/sglang/kernels/ops/quantization/configs/N=6400,K=5120,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=6400,K=5120,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=6400,K=5120,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=6400,K=5120,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=1024,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=1024,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=1024,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=1024,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=1024,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=1024,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=1024,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=1024,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=1152,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=1152,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=1152,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=1152,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=1152,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=1152,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=1152,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=1152,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=1152,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=1152,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=1152,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=1152,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=1152,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=1152,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=16384,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=16384,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=18432,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=18432,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=2048,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=2048,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=2048,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=2048,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=2048,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=2048,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=2048,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=2048,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=2048,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=2048,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=2048,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=2048,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=2304,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=2304,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=2304,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=2304,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=2304,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=2304,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=2304,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=2304,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=2304,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=2304,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=2304,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=2304,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=2304,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=2304,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=2304,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=2304,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=2304,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=2304,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=7168,K=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=7168,K=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=7168,K=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=7168,K=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=8192,K=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json b/python/sglang/kernels/ops/gemm/configs/N=8192,K=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=8192,K=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json rename to python/sglang/kernels/ops/gemm/configs/N=8192,K=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[32, 32].json diff --git a/python/sglang/kernels/ops/quantization/configs/N=8192,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/kernels/ops/gemm/configs/N=8192,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/N=8192,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json rename to python/sglang/kernels/ops/gemm/configs/N=8192,K=4096,device_name=AMD_Instinct_MI455X,dtype=fp8_w8a8,block_shape=[128, 128].json diff --git a/python/sglang/kernels/ops/quantization/configs/README.md b/python/sglang/kernels/ops/gemm/configs/README.md similarity index 100% rename from python/sglang/kernels/ops/quantization/configs/README.md rename to python/sglang/kernels/ops/gemm/configs/README.md diff --git a/python/sglang/kernels/ops/gemm/dsv4_wo_a.py b/python/sglang/kernels/ops/gemm/dsv4_wo_a.py new file mode 100644 index 0000000000..8ed5c910b1 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/dsv4_wo_a.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from sglang.kernels.ops.layernorm.mxfp8_epilogue import ue8m0_scale + + +@triton.jit +def _wo_a_bf16_gemv_kernel(X, W, Y, R: tl.constexpr, D: tl.constexpr, BN: tl.constexpr): + group = tl.program_id(1) + rows = tl.program_id(0) * BN + tl.arange(0, BN) + columns = tl.arange(0, D) + x = tl.load(X + group * D + columns).to(tl.float32) + w = tl.load( + W + (group * R + rows[:, None]) * D + columns[None, :], + rows[:, None] < R, + 0, + ).to(tl.float32) + result = tl.sum(w * x[None, :], axis=1) + tl.store(Y + group * R + rows, result, rows < R) + + +def wo_a_bf16_gemv(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: + """Compute ``einsum('tgd,grd->tgr', x, weight)`` for one token.""" + assert x.shape[0] == 1 and x.ndim == weight.ndim == 3 + assert x.dtype == weight.dtype == torch.bfloat16 + assert x.is_cuda and x.device == weight.device + assert x.is_contiguous() and weight.is_contiguous() + groups, rows, dim = weight.shape + assert x.shape[1:] == (groups, dim) and dim == triton.next_power_of_2(dim) + result = torch.empty((1, groups, rows), dtype=x.dtype, device=x.device) + # One output row per CTA keeps register use low and exposes enough + # independent weight loads for single-token decode. + _wo_a_bf16_gemv_kernel[(rows, groups)]( + x, + weight, + result, + rows, + dim, + 1, + num_warps=4, + enable_fp_fusion=False, + ) + return result + + +@triton.jit +def _wo_a_partial(X, W, P, M: tl.constexpr, SX: tl.constexpr): + tile, group, split = tl.program_id(0), tl.program_id(1), tl.program_id(2) + m = tl.arange(0, 16) + n = tile * 64 + tl.arange(0, 64) + k = split * 512 + tl.arange(0, 128) + acc = tl.zeros((16, 64), tl.float32) + for i in range(4): + offsets = k + i * 128 + x = tl.load( + X + m[:, None] * SX + group * 4096 + offsets[None, :], m[:, None] < M, 0 + ) + w = tl.load(W + (group * 1024 + n[None, :]) * 4096 + offsets[:, None]) + acc += tl.dot(x, w) + tl.store( + P + ((split * M + m[:, None]) * 2 + group) * 1024 + n[None, :], + acc, + m[:, None] < M, + ) + + +@triton.jit +def _wo_a_reduce(P, Y, E: tl.constexpr): + i = tl.program_id(0) * 256 + tl.arange(0, 256) + split = tl.arange(0, 8) + values = tl.load(P + split[:, None] * E + i[None, :], i[None, :] < E, 0) + tl.store(Y + i, tl.sum(values, 0), i < E) + + +def wo_a_bf16_small_batch(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: + """Compute ``einsum('tgd,grd->tgr', x, weight)`` for the TP4 WO-A shape. + Partial sums stay in FP32 until the final BF16, token-major store.""" + m = x.shape[0] + assert 2 <= m <= 8 and x.shape[1:] == (2, 4096) + assert weight.shape == (2, 1024, 4096) and weight.is_contiguous() + assert x.dtype == weight.dtype == torch.bfloat16 + assert x.is_cuda and x.device == weight.device + assert x.stride(2) == 1 and x.stride(1) == 4096 and x.stride(0) >= 8192 + result = torch.empty((m, 2, 1024), dtype=x.dtype, device=x.device) + partial = torch.empty((8, m, 2, 1024), dtype=torch.float32, device=x.device) + _wo_a_partial[(16, 2, 8)]( + x, weight, partial, m, x.stride(0), num_warps=4, num_stages=3 + ) + _wo_a_reduce[(triton.cdiv(m * 2048, 256),)](partial, result, m * 2048, num_warps=4) + return result + + +@triton.jit +def _wo_a_reduce_quant(P, Q, S, M: tl.constexpr): + row, tile = tl.program_id(0), tl.program_id(1) + i = tile * 256 + tl.arange(0, 256) + split = tl.arange(0, 8) + v = tl.load(P + split[:, None] * (M * 2048) + row * 2048 + i[None, :]) + y = tl.sum(v, 0).to(tl.bfloat16).to(tl.float32).reshape((8, 32)) + amax = tl.max(tl.abs(y), 1) + sf, inv = ue8m0_scale(amax) + quant = tl.minimum(tl.maximum(y * inv[:, None], -448.0), 448.0).to(tl.float8e4nv) + tl.store(Q + row * 2048 + i, quant.reshape((256,))) + col = tile * 8 + tl.arange(0, 8) + off = (col // 4) * 512 + row * 16 + col % 4 + tl.store(S + off, sf.to(tl.uint8)) + # Zero only padding rows; valid scale bytes have disjoint writers above. + for z in tl.static_range(triton.cdiv(8192, M * 8 * 256)): + s = (row * 8 + tile) * 256 + tl.arange(0, 256) + z * (M * 8 * 256) + sr = (s % 512) // 16 + ((s % 16) // 4) * 32 + tl.store(S + s, 0, (s < 8192) & (sr >= M)) + + +def _quantize_partial(p): + m = p.shape[1] + q = torch.empty((m, 2048), device=p.device, dtype=torch.float8_e4m3fn) + s = torch.empty(8192, device=p.device, dtype=torch.uint8) + _wo_a_reduce_quant[(m, 8)](p, q, s, m, num_warps=4) + return q, s + + +def wo_a_bf16_small_batch_mxfp8(x: torch.Tensor, weight: torch.Tensor): + """WO-A with BF16 rounding followed by FlashInfer-compatible MXFP8 quantization.""" + m = x.shape[0] + assert 2 <= m <= 8 and x.shape[1:] == (2, 4096) + assert x.dtype == weight.dtype == torch.bfloat16 + assert weight.shape == (2, 1024, 4096) and weight.is_contiguous() + assert x.is_cuda and x.device == weight.device + assert x.stride(2) == 1 and x.stride(1) == 4096 and x.stride(0) >= 8192 + partial = torch.empty((8, m, 2, 1024), dtype=torch.float32, device=x.device) + _wo_a_partial[(16, 2, 8)]( + x, weight, partial, m, x.stride(0), num_warps=4, num_stages=3 + ) + return _quantize_partial(partial) diff --git a/python/sglang/kernels/ops/gemm/fp8_kernel.py b/python/sglang/kernels/ops/gemm/fp8_kernel.py new file mode 100644 index 0000000000..46234dc6eb --- /dev/null +++ b/python/sglang/kernels/ops/gemm/fp8_kernel.py @@ -0,0 +1,1134 @@ +import functools +import json +import logging +import os +from typing import Any, Dict, List, Optional, Tuple + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers import deep_gemm_wrapper +from sglang.srt.runtime_context import get_platform +from sglang.srt.utils import ( + get_device_core_count, + get_device_name, + is_cuda, + is_gfx1250_supported, + is_hip, + log_info_on_rank0, +) +from sglang.srt.utils.custom_op import register_custom_op + +_is_cuda = is_cuda() +_is_hip = is_hip() +_is_gfx1250 = is_gfx1250_supported() +logger = logging.getLogger(__name__) + + +@register_custom_op(mutates_args=["C"]) +def deep_gemm_fp8_fp8_bf16_nt( + A: torch.Tensor, + As: torch.Tensor, + B: torch.Tensor, + Bs: torch.Tensor, + C: torch.Tensor, +) -> None: + deep_gemm_wrapper.gemm_nt_f8f8bf16((A, As), (B, Bs), C) + + +@register_custom_op(mutates_args=["C"]) +def deep_gemm_mxfp8_fp8_bf16_nt( + A: torch.Tensor, + As: torch.Tensor, + B: torch.Tensor, + Bs: torch.Tensor, + C: torch.Tensor, +) -> None: + deep_gemm_wrapper.gemm_nt_mxfp8_f8f8bf16((A, As), (B, Bs), C) + + +@triton.jit +def _w8a8_block_fp8_matmul( + # Pointers to inputs and output + A, + B, + C, + As, + Bs, + # Shape for matmul + M, + N, + K, + # Block size for block-wise quantization + group_n, + group_k, + # Stride for inputs and output + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + stride_As_m, + stride_As_k, + stride_Bs_k, + stride_Bs_n, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, + needs_masking: tl.constexpr, +): + """Triton-accelerated function used to perform linear operations (dot + product) on input tensors `A` and `B` with block-wise quantization, and store the result in output + tensor `C`. + """ + + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) + b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) + + As_ptrs = As + offs_am * stride_As_m + offs_bsn = offs_bn // group_n + Bs_ptrs = Bs + offs_bsn * stride_Bs_n + n_tiles_k_per_group_k = group_k // BLOCK_SIZE_K + + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + if needs_masking: + a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k * BLOCK_SIZE_K, other=0.0) + b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0) + else: + a = tl.load(a_ptrs) + b = tl.load(b_ptrs) + + a_s = tl.load(As_ptrs) + b_s = tl.load(Bs_ptrs) + + scale_step_k = tl.where((k + 1) % n_tiles_k_per_group_k == 0, 1, 0) + accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + As_ptrs += scale_step_k * stride_As_k + Bs_ptrs += scale_step_k * stride_Bs_k + + if C.dtype.element_ty == tl.bfloat16: + c = accumulator.to(tl.bfloat16) + elif C.dtype.element_ty == tl.float16: + c = accumulator.to(tl.float16) + else: + c = accumulator.to(tl.float32) + + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + tl.store(c_ptrs, c, mask=c_mask) + + +@triton.jit +def _w8a8_block_fp8_matmul_hopper( + # Pointers to inputs and output + A, + B, + C, + As, + Bs, + # Shape for matmul + M, + N, + K, + # Block size for block-wise quantization + group_n, + group_k, + # Stride for inputs and output + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + stride_As_m, + stride_As_k, + stride_Bs_k, + stride_Bs_n, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, + needs_masking: tl.constexpr, + SWAP_AB: tl.constexpr = False, + SPLIT_K: tl.constexpr = 1, +): + pid = tl.program_id(axis=0) + split = tl.program_id(axis=1) + tiles_per_split = tl.cdiv(tl.cdiv(K, BLOCK_SIZE_K), SPLIT_K) + first_tile = split * tiles_per_split + C += split * M * N + num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) + b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) + + As_ptrs = As + offs_am * stride_As_m + offs_bsn = offs_bn // group_n + Bs_ptrs = Bs + offs_bsn * stride_Bs_n + n_tiles_k_per_group_k = group_k // BLOCK_SIZE_K + + a_ptrs += first_tile * BLOCK_SIZE_K * stride_ak + b_ptrs += first_tile * BLOCK_SIZE_K * stride_bk + As_ptrs += (first_tile // n_tiles_k_per_group_k) * stride_As_k + Bs_ptrs += (first_tile // n_tiles_k_per_group_k) * stride_Bs_k + + # Small-M Hopper configs transpose the MMA so the weight tile occupies M. + if SWAP_AB: + accumulator = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_M), dtype=tl.float32) + else: + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + for k in range( + first_tile, tl.minimum(first_tile + tiles_per_split, tl.cdiv(K, BLOCK_SIZE_K)) + ): + if needs_masking: + a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k * BLOCK_SIZE_K, other=0.0) + b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0) + else: + a = tl.load(a_ptrs) + b = tl.load(b_ptrs) + + a_s = tl.load(As_ptrs) + b_s = tl.load(Bs_ptrs) + + scale_step_k = tl.where((k + 1) % n_tiles_k_per_group_k == 0, 1, 0) + if SWAP_AB: + accumulator += ( + tl.dot(tl.trans(b), tl.trans(a)) * b_s[:, None] * a_s[None, :] + ) + else: + accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + As_ptrs += scale_step_k * stride_As_k + Bs_ptrs += scale_step_k * stride_Bs_k + + if SWAP_AB: + accumulator = tl.trans(accumulator) + + if C.dtype.element_ty == tl.bfloat16: + c = accumulator.to(tl.bfloat16) + elif C.dtype.element_ty == tl.float16: + c = accumulator.to(tl.float16) + else: + c = accumulator.to(tl.float32) + + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + tl.store(c_ptrs, c, mask=c_mask) + + +@triton.jit +def _reduce_block_fp8_split_k( + Parts, Out, ELEMENTS: tl.constexpr, SPLITS: tl.constexpr, BLOCK: tl.constexpr +): + offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + splits = tl.arange(0, SPLITS) + values = tl.load( + Parts + splits[:, None] * ELEMENTS + offsets[None, :], + offsets[None, :] < ELEMENTS, + 0.0, + ) + tl.store(Out + offsets, tl.sum(values, axis=0), offsets < ELEMENTS) + + +@triton.jit +def _w8a8_block_fp8_matmul_gfx1250( + # Pointers to inputs and output + A, + B, + C, + As, + Bs, + # Shape for matmul + M, + N, + K, + # Block size for block-wise quantization + group_n, + group_k, + # Stride for inputs and output + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + stride_As_m, + stride_As_k, + stride_Bs_k, + stride_Bs_n, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, + needs_masking: tl.constexpr, +): + """ + gfx1250 (RDNA4) block-fp8 matmul. + The shared ``_w8a8_block_fp8_matmul`` is unusable on gfx1250. + 1. fp8 ``tl.dot`` faults at runtime + 2. software pipelining (``num_stages`` > 1) miscompiles and yields NaN. + 3. the ``offs % M`` / ``offs % N`` modulo-wrap index trick is + intermittently miscompiled into out-of-bounds addresses. + """ + + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + # No modulo-wrap on gfx1250; use explicit masks instead. + offs_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_bn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + m_mask = offs_am < M + n_mask = offs_bn < N + + offs_am_c = tl.where(m_mask, offs_am, 0) + offs_bn_c = tl.where(n_mask, offs_bn, 0) + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = A + (offs_am_c[:, None] * stride_am + offs_k[None, :] * stride_ak) + b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn_c[None, :] * stride_bn) + + As_ptrs = As + offs_am_c * stride_As_m + offs_bsn = offs_bn_c // group_n + Bs_ptrs = Bs + offs_bsn * stride_Bs_n + n_tiles_k_per_group_k = group_k // BLOCK_SIZE_K + + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + k_mask = offs_k < K - k * BLOCK_SIZE_K + a = tl.load(a_ptrs, mask=m_mask[:, None] & k_mask[None, :], other=0.0) + b = tl.load(b_ptrs, mask=k_mask[:, None] & n_mask[None, :], other=0.0) + + a_s = tl.load(As_ptrs, mask=m_mask, other=0.0) + b_s = tl.load(Bs_ptrs, mask=n_mask, other=0.0) + + scale_step_k = tl.where((k + 1) % n_tiles_k_per_group_k == 0, 1, 0) + + # Upcast fp8 to bf16 in-register + accumulator += ( + tl.dot(a.to(tl.bfloat16), b.to(tl.bfloat16)) * a_s[:, None] * b_s[None, :] + ) + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + As_ptrs += scale_step_k * stride_As_k + Bs_ptrs += scale_step_k * stride_Bs_k + + if C.dtype.element_ty == tl.bfloat16: + c = accumulator.to(tl.bfloat16) + elif C.dtype.element_ty == tl.float16: + c = accumulator.to(tl.float16) + else: + c = accumulator.to(tl.float32) + + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + tl.store(c_ptrs, c, mask=c_mask) + + +@triton.jit +def _w8a8_block_fp8_matmul_unrolledx4( + # Pointers to inputs and output + A, + B, + C, + As, + Bs, + # Shape for matmul + M, + N, + K, + # Block size for block-wise quantization + group_n, + group_k, + # Stride for inputs and output + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + stride_As_m, + stride_As_k, + stride_Bs_k, + stride_Bs_n, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, + needs_masking: tl.constexpr, +): + """Triton-accelerated function used to perform linear operations (dot + product) on input tensors `A` and `B` with block-wise quantization, and store the result in output + tensor `C`. + """ + + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) + b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) + + As_ptrs = As + offs_am * stride_As_m + offs_bsn = offs_bn // group_n + Bs_ptrs = Bs + offs_bsn * stride_Bs_n + scale_step_k = BLOCK_SIZE_K // group_k + + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + # manually unroll to 4 iterations + UNROLL_FACTOR = 4 + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K * UNROLL_FACTOR)): + # 1st iteration + if needs_masking: + a = tl.load( + a_ptrs, + mask=offs_k[None, :] < K - (k * UNROLL_FACTOR) * BLOCK_SIZE_K, + other=0.0, + ) + b = tl.load( + b_ptrs, + mask=offs_k[:, None] < K - (k * UNROLL_FACTOR) * BLOCK_SIZE_K, + other=0.0, + ) + else: + a = tl.load(a_ptrs) + b = tl.load(b_ptrs) + + a_s = tl.load(As_ptrs) + b_s = tl.load(Bs_ptrs) + + accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + As_ptrs += scale_step_k * stride_As_k + Bs_ptrs += scale_step_k * stride_Bs_k + + # 2nd iteration + if needs_masking: + a = tl.load( + a_ptrs, + mask=offs_k[None, :] < K - (k * UNROLL_FACTOR + 1) * BLOCK_SIZE_K, + other=0.0, + ) + b = tl.load( + b_ptrs, + mask=offs_k[:, None] < K - (k * UNROLL_FACTOR + 1) * BLOCK_SIZE_K, + other=0.0, + ) + else: + a = tl.load(a_ptrs) + b = tl.load(b_ptrs) + + a_s = tl.load(As_ptrs) + b_s = tl.load(Bs_ptrs) + + accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + As_ptrs += scale_step_k * stride_As_k + Bs_ptrs += scale_step_k * stride_Bs_k + + # 3rd iteration + if needs_masking: + a = tl.load( + a_ptrs, + mask=offs_k[None, :] < K - (k * UNROLL_FACTOR + 2) * BLOCK_SIZE_K, + other=0.0, + ) + b = tl.load( + b_ptrs, + mask=offs_k[:, None] < K - (k * UNROLL_FACTOR + 2) * BLOCK_SIZE_K, + other=0.0, + ) + else: + a = tl.load(a_ptrs) + b = tl.load(b_ptrs) + + a_s = tl.load(As_ptrs) + b_s = tl.load(Bs_ptrs) + + accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + As_ptrs += scale_step_k * stride_As_k + Bs_ptrs += scale_step_k * stride_Bs_k + + # 4th iteration + if needs_masking: + a = tl.load( + a_ptrs, + mask=offs_k[None, :] < K - (k * UNROLL_FACTOR + 3) * BLOCK_SIZE_K, + other=0.0, + ) + b = tl.load( + b_ptrs, + mask=offs_k[:, None] < K - (k * UNROLL_FACTOR + 3) * BLOCK_SIZE_K, + other=0.0, + ) + else: + a = tl.load(a_ptrs) + b = tl.load(b_ptrs) + + a_s = tl.load(As_ptrs) + b_s = tl.load(Bs_ptrs) + + accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + As_ptrs += scale_step_k * stride_As_k + Bs_ptrs += scale_step_k * stride_Bs_k + + if C.dtype.element_ty == tl.bfloat16: + c = accumulator.to(tl.bfloat16) + elif C.dtype.element_ty == tl.float16: + c = accumulator.to(tl.float16) + else: + c = accumulator.to(tl.float32) + + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + tl.store(c_ptrs, c, mask=c_mask) + + +@functools.lru_cache +def get_w8a8_block_fp8_configs( + N: int, K: int, block_n: int, block_k: int +) -> Optional[Dict[int, Any]]: + """ + Return optimized configurations for the w8a8 block fp8 kernel. + + The return value will be a dictionary that maps an irregular grid of + batch sizes to configurations of the w8a8 block fp8 kernel. To evaluate the + kernel on a given batch size bs, the closest batch size in the grid should + be picked and the associated configuration chosen to invoke the kernel. + """ + + # Skip config lookup during torch.compile to avoid non-Tensor ops (e.g., device name). + # Returning None forces the caller to use the default config path during compile. + if torch._dynamo.is_compiling(): + return None + + # First look up if an optimized configuration is available in the configs + # directory + device_name = get_device_name().replace(" ", "_") + json_file_name = f"N={N},K={K},device_name={device_name},dtype=fp8_w8a8,block_shape=[{block_n}, {block_k}].json" + + config_file_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name + ) + if os.path.exists(config_file_path): + with open(config_file_path) as f: + log_info_on_rank0( + logger, + f"Using configuration from {config_file_path} for W8A8 Block FP8 kernel.", + ) + raw = {int(key): val for key, val in json.load(f).items()} + + sanitized = {} + clamped_ms = [] + for m_key, cfg in raw.items(): + if cfg["BLOCK_SIZE_K"] < block_k and ( + not _is_cuda or block_k % cfg["BLOCK_SIZE_K"] != 0 + ): + clamped_ms.append((m_key, cfg["BLOCK_SIZE_K"])) + cfg = {**cfg, "BLOCK_SIZE_K": block_k} + sanitized[m_key] = cfg + if clamped_ms: + logger.warning( + "Clamped BLOCK_SIZE_K up to %d in tuned config %s for entries %s " + "(scale stepping requires BLOCK_SIZE_K >= block_k).", + block_k, + json_file_name, + clamped_ms, + ) + + return sanitized + + # If no optimized configuration is available, we will use the default + # configuration + logger.warning( + ( + "Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! " + "Config file not found at %s" + ), + config_file_path, + ) + return None + + +@functools.lru_cache +def get_w8a8_channelwise_fp8_configs(N: int, K: int) -> Optional[Dict[int, Any]]: + """ + Return tuned Triton configurations for the per-token/per-channel w8a8 fp8 + GEMM (`triton_scaled_mm`), for one weight shape on the current device. + + The return value maps an irregular grid of token counts M to + `triton_scaled_mm` tile configs; the closest M in the grid should be picked + for a given batch. A null entry means CUTLASS was faster at that M, so the + grid keeps the point (to stop a neighbouring M snapping onto it) but the + caller must fall back. A missing file / shape likewise means "keep the + default". + """ + # Intentional: the tuned lookup (host-side device name + file I/O) isn't + # traceable, so under torch.compile return None and fall back to CUTLASS -- + # same as mainline's block-FP8 get_w8a8_block_fp8_configs. + if torch._dynamo.is_compiling(): + return None + + device_name = get_device_name().replace(" ", "_") + json_file_name = ( + f"N={N},K={K},device_name={device_name},dtype=fp8_w8a8_channelwise.json" + ) + config_file_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name + ) + if not os.path.exists(config_file_path): + return None + + with open(config_file_path) as f: + log_info_on_rank0( + logger, + f"Using configuration from {config_file_path} for W8A8 channelwise FP8 GEMM.", + ) + return {int(key): val for key, val in json.load(f).items()} + + +def get_w8a8_channelwise_fp8_config(N: int, K: int, M: int) -> Optional[Dict[str, int]]: + """Tuned `triton_scaled_mm` config for this GEMM, or None to keep the default. + + N / K are the weight's output / contraction dims and M the token count. None + means no tuned config applies -- the caller keeps its existing kernel choice + -- either because this device / shape was never tuned, or the nearest tuned + M is a point where CUTLASS won. The caller gates on the feature flag. + """ + configs = get_w8a8_channelwise_fp8_configs(N, K) + if not configs: + return None + return configs[min(configs.keys(), key=lambda x: abs(x - M))] + + +def select_w8a8_block_fp8_matmul_kernel(M, N, META): + return _w8a8_block_fp8_matmul + + +if _is_hip: + + def use_w8a8_block_fp8_matmul_unrolledx4(M, N, META): + # Use manually unrolledx4 kernel on AMD GPU when the grid size is small. + # Empirical testing shows the sweet spot lies when it's less than the # of + # compute units available on the device. + num_workgroups = triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv( + N, META["BLOCK_SIZE_N"] + ) + num_workgroups <= get_device_core_count() + + def select_w8a8_block_fp8_matmul_kernel(M, N, META): + if use_w8a8_block_fp8_matmul_unrolledx4(M, N, META): + return _w8a8_block_fp8_matmul_unrolledx4 + else: + return _w8a8_block_fp8_matmul + + +def prepare_block_fp8_matmul_inputs( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + block_size: List[int], + output_dtype: torch.dtype = torch.float16, +) -> Tuple[int, int, int]: + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + + assert A.shape[-1] == B.shape[-1] + assert A.shape[:-1] == As.shape[:-1] + assert A.is_contiguous() + + if As.dtype == torch.float: + assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1] + elif As.dtype == torch.int: + assert triton.cdiv(triton.cdiv(A.shape[-1], block_k), 4) == As.shape[-1], ( + f"{A.shape=} {As.shape=} {block_size=}" + ) + else: + raise NotImplementedError + + M = A.numel() // A.shape[-1] + + assert B.ndim == 2 + assert B.is_contiguous() + assert Bs.ndim == 2 + N, K = B.shape + + if Bs.dtype == torch.float: + assert triton.cdiv(N, block_n) == Bs.shape[0] + assert triton.cdiv(K, block_k) == Bs.shape[1] + elif Bs.dtype == torch.int: + assert N == Bs.shape[0], f"{B.shape=} {Bs.shape=} {block_size=}" + assert triton.cdiv(triton.cdiv(K, block_k), 4) == Bs.shape[1], ( + f"{B.shape=} {Bs.shape=} {block_size=}" + ) + else: + raise NotImplementedError + + C_shape = A.shape[:-1] + (N,) + C = A.new_empty(C_shape, dtype=output_dtype) + + return M, N, K, C + + +def w8a8_block_fp8_matmul_deepgemm( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + block_size: List[int], + output_dtype: torch.dtype, +) -> torch.Tensor: + M, N, K, C = prepare_block_fp8_matmul_inputs(A, B, As, Bs, block_size, output_dtype) + + # Deepgemm only supports output tensor type as bfloat16 + assert C.dtype == torch.bfloat16 and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM + + deep_gemm_fp8_fp8_bf16_nt(A, As, B, Bs, C) + + return C + + +def w8a8_mxfp8_matmul_deepgemm( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + output_dtype: torch.dtype, +) -> torch.Tensor: + assert A.is_contiguous() and B.is_contiguous() + assert output_dtype == torch.bfloat16 and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM + + M = A.numel() // A.shape[-1] + N, K = B.shape + C = A.new_empty(A.shape[:-1] + (N,), dtype=output_dtype) + + deep_gemm_mxfp8_fp8_bf16_nt(A, As, B, Bs, C) + + return C + + +def w8a8_block_fp8_matmul_triton( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + block_size: List[int], + output_dtype: torch.dtype = torch.float16, +) -> torch.Tensor: + """This function performs matrix multiplication with block-wise quantization. + + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + + Args: + A: The input tensor, e.g., activation. + B: The input tensor, e.g., weight. + As: The per-token-group quantization scale for `A`. + Bs: The per-block quantization scale for `B`. + block_size: The block size for per-block quantization. It should be 2-dim, e.g., [128, 128]. + output_dytpe: The dtype of the returned tensor. + + Returns: + torch.Tensor: The result of matmul. + """ + + M, N, K, C = prepare_block_fp8_matmul_inputs(A, B, As, Bs, block_size, output_dtype) + + block_n, block_k = block_size + + configs = get_w8a8_block_fp8_configs(N, K, block_size[0], block_size[1]) + if configs: + # If an optimal configuration map has been found, look up the + # optimal config + config = configs[min(configs.keys(), key=lambda x: abs(x - M))] + else: + # Default config + # Block-wise quant: BLOCK_SIZE_K must be divisible by block_size[1] + config = { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": block_size[0], + "BLOCK_SIZE_K": block_size[1], + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + } + + # Split-K accumulates K in SPLIT_K separate fp32 partials, so its results + # do not match the single-accumulator kernels bit-for-bit. + hopper_tuned = get_platform().is_sm90 and ( + config.get("SWAP_AB", False) or config.get("SPLIT_K", 1) > 1 + ) + if hopper_tuned: + kernel = _w8a8_block_fp8_matmul_hopper + elif _is_gfx1250: + config = {**config, "num_stages": 1} + kernel = _w8a8_block_fp8_matmul_gfx1250 + else: + kernel = select_w8a8_block_fp8_matmul_kernel(M, N, config) + + split_k = config.get("SPLIT_K", 1) if hopper_tuned else 1 + if split_k > 1: + assert split_k & (split_k - 1) == 0 + partials = torch.empty((split_k, M, N), device=A.device, dtype=torch.float32) + else: + partials = C + + needs_masking = bool(K % config["BLOCK_SIZE_K"] != 0) + + def grid(META): + blocks = triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv( + N, META["BLOCK_SIZE_N"] + ) + return (blocks, split_k) if hopper_tuned else (blocks,) + + kernel[grid]( + A, + B, + partials, + As, + Bs, + M, + N, + K, + block_n, + block_k, + A.stride(-2), + A.stride(-1), + B.stride(1), + B.stride(0), + C.stride(-2), + C.stride(-1), + As.stride(-2), + As.stride(-1), + Bs.stride(1), + Bs.stride(0), + **config, + needs_masking=needs_masking, + ) + + if split_k > 1: + _reduce_block_fp8_split_k[(triton.cdiv(M * N, 256),)]( + partials, C, M * N, split_k, 256 + ) + + return C + + +def w8a8_block_fp8_matmul( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + block_size: List[int], + output_dtype: torch.dtype = torch.float16, +) -> torch.Tensor: + if output_dtype == torch.bfloat16 and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM: + return w8a8_block_fp8_matmul_deepgemm( + A, B, As, Bs, block_size, output_dtype=output_dtype + ) + + return w8a8_block_fp8_matmul_triton( + A, B, As, Bs, block_size, output_dtype=output_dtype + ) + + +def is_weak_contiguous(x: torch.Tensor): + strides = x.stride() + sizes = x.shape + is_not_transpose = strides[0] == 1 and (strides[1] >= max(1, sizes[0])) + is_transpose = strides[1] == 1 and (strides[0] >= max(1, sizes[1])) + return is_transpose or is_not_transpose + + +def _as_column_scale(scale: torch.Tensor, expected_len: int) -> torch.Tensor: + if scale.dim() <= 1: + return scale.reshape(-1, 1) + if scale.dim() == 2: + if scale.shape[1] == 1: + return scale + if scale.shape[0] == 1 and scale.shape[1] == expected_len: + return scale.t() + return scale + + +@triton.jit +def scaled_mm_kernel( + a_ptr, + b_ptr, + scale_a_ptr, + scale_b_ptr, + c_ptr, + bias_ptr, + M, + N, + K, + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + ACCUMULATOR_DTYPE: tl.constexpr, + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + BLOCK_SIZE_SCALE_A: tl.constexpr, + BLOCK_SIZE_SCALE_B: tl.constexpr, +): + pid = tl.program_id(axis=0) + + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + + pid_m = pid // num_pid_n + pid_n = pid % num_pid_n + + accumulator_dtype = ACCUMULATOR_DTYPE + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=accumulator_dtype) + + # NOTE: Some tensor inputs are so large, they will cause int32 overflow + # so it is necessary to use tl.int64 for all the offsets, else SEGV will + # eventually occur. + + # Offsets and masks. + offsets_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64) + masks_am = offsets_am < M + + offsets_bn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64) + masks_bn = offsets_bn < N + + offsets_k = tl.arange(0, BLOCK_SIZE_K).to(tl.int64) + offsets_a = stride_am * offsets_am[:, None] + stride_ak * offsets_k[None, :] + offsets_b = stride_bk * offsets_k[:, None] + stride_bn * offsets_bn[None, :] + + # NOTE: BLOCK_SIZE_SCALE_A could be 1 or BLOCK_SIZE_M, so need to create + # appropriate offsets and masks for each case. Same goes for + # BLOCK_SIZE_SCALE_B. + offsets_scale_am = ( + tl.arange(0, BLOCK_SIZE_SCALE_A) + + (BLOCK_SIZE_SCALE_A > 1) * pid_m * BLOCK_SIZE_M + ) + masks_scale_am = offsets_scale_am < M + + offsets_scale_bn = ( + tl.arange(0, BLOCK_SIZE_SCALE_B) + + (BLOCK_SIZE_SCALE_B > 1) * pid_n * BLOCK_SIZE_N + ) + masks_scale_bn = offsets_scale_bn < N + + a_ptrs = a_ptr + offsets_a + b_ptrs = b_ptr + offsets_b + + scale_a_ptrs = scale_a_ptr + offsets_scale_am + scale_b_ptrs = scale_b_ptr + offsets_scale_bn + + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + masks_k = offsets_k < K + masks_a = masks_am[:, None] & masks_k[None, :] + a = tl.load(a_ptrs, mask=masks_a) + + masks_b = masks_k[:, None] & masks_bn[None, :] + b = tl.load(b_ptrs, mask=masks_b) + + # Accumulate results. + accumulator = tl.dot(a, b, accumulator, out_dtype=accumulator_dtype) + + offsets_k += BLOCK_SIZE_K + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + # Apply scale at end. + masks_scale_a = masks_scale_am[:, None] & (tl.arange(0, 1) < 1)[:, None] + scale_a = tl.load(scale_a_ptrs[:, None], masks_scale_a) + # Need to broadcast to the appropriate size, if scale_a is already + # (BLOCK_SIZE_M, 1) then it will broadcast to its own shape. Same goes + # for scale_b below. + scale_a = scale_a.broadcast_to((BLOCK_SIZE_M, 1)) + accumulator = scale_a * accumulator.to(tl.float32) + + masks_scale_b = masks_scale_bn[:, None] & (tl.arange(0, 1) < 1)[None, :] + scale_b = tl.load(scale_b_ptrs[:, None], masks_scale_b) + scale_b = scale_b.broadcast_to((BLOCK_SIZE_N, 1)) + accumulator = scale_b.T * accumulator.to(tl.float32) + + # Convert to output format. + c = accumulator.to(c_ptr.type.element_ty) + + # Add bias, it's already in output format, so add it after conversion. + if bias_ptr: + offsets_bias = offsets_bn + bias_ptrs = bias_ptr + offsets_bias + bias_mask = offsets_bias < N + bias = tl.load(bias_ptrs, bias_mask) + c += bias + + # Save output + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64) + offs_cm = offs_cm.to(tl.int64) + offs_cn = offs_cn.to(tl.int64) + c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + + tl.store(c_ptrs, c, mask=c_mask) + + +def triton_scaled_mm( + input: torch.Tensor, + weight: torch.Tensor, + scale_a: torch.Tensor, + scale_b: torch.Tensor, + out_dtype: type[torch.dtype], + bias: Optional[torch.Tensor] = None, + block_size_m: int = 32, + block_size_n: int = 32, + block_size_k: int = 32, + use_heuristic=True, + num_warps: Optional[int] = None, + num_stages: Optional[int] = None, +) -> torch.Tensor: + M, K = input.shape + N = weight.shape[1] + + assert N > 0 and K > 0 and M > 0 + assert weight.shape[0] == K + assert input.dtype == weight.dtype + + scale_a = _as_column_scale(scale_a, M) + scale_b = _as_column_scale(scale_b, N) + + assert scale_a.dim() == 2 and scale_b.dim() == 2 + assert scale_a.dtype == scale_b.dtype and scale_a.is_floating_point() + assert scale_a.shape[1] == 1 and (scale_a.shape[0] == 1 or scale_a.shape[0] == M) + assert scale_b.shape[1] == 1 and (scale_b.shape[0] == 1 or scale_b.shape[0] == N) + assert out_dtype.is_floating_point + assert bias is None or bias.is_floating_point() + assert is_weak_contiguous(input) + assert is_weak_contiguous(weight) + + grid = lambda META: ( + triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), + ) + + result = torch.empty((M, N), dtype=out_dtype, device=input.device) + + has_scalar = lambda x: x.shape[0] == 1 and x.shape[1] == 1 + + if use_heuristic: + is_small_N = N < 8192 + next_power_of_2_M = max(32, triton.next_power_of_2(M)) + if next_power_of_2_M <= 32: + tile_shape = (64, 64, 256) if is_small_N else (64, 128, 256) + elif next_power_of_2_M <= 64: + tile_shape = (64, 64, 256) + elif next_power_of_2_M <= 128: + tile_shape = (64, 128, 128) + else: + tile_shape = (128, 128, 128) + block_size_m, block_size_n, block_size_k = tile_shape + # else: use the block_size_{m,n,k} the caller passed (e.g. a tuned config). + + block_size_sa = 1 if has_scalar(scale_a) else block_size_m + block_size_sb = 1 if has_scalar(scale_b) else block_size_n + + accumulator_dtype = tl.float32 if input.is_floating_point() else tl.int32 + + # num_warps / num_stages are triton.jit launch kwargs, and triton rejects an + # explicit None -- pass them only when the caller set them, so the default + # path keeps triton's own defaults. + launch_kwargs = {} + if num_warps is not None: + launch_kwargs["num_warps"] = num_warps + if num_stages is not None: + launch_kwargs["num_stages"] = num_stages + + # A = input, B = weight, C = result + # A = M x K, B = K x N, C = M x N + scaled_mm_kernel[grid]( + input, + weight, + scale_a, + scale_b, + result, + bias, + M, + N, + K, + input.stride(0), + input.stride(1), + weight.stride(0), + weight.stride(1), + result.stride(0), + result.stride(1), + accumulator_dtype, + BLOCK_SIZE_M=block_size_m, + BLOCK_SIZE_N=block_size_n, + BLOCK_SIZE_K=block_size_k, + BLOCK_SIZE_SCALE_A=block_size_sa, + BLOCK_SIZE_SCALE_B=block_size_sb, + **launch_kwargs, + ) + + return result.to(out_dtype) diff --git a/python/sglang/kernels/ops/quantization/gptq_marlin.py b/python/sglang/kernels/ops/gemm/gptq_marlin.py similarity index 100% rename from python/sglang/kernels/ops/quantization/gptq_marlin.py rename to python/sglang/kernels/ops/gemm/gptq_marlin.py diff --git a/python/sglang/srt/layers/hc_mix_triton.py b/python/sglang/kernels/ops/gemm/hc_mix.py similarity index 100% rename from python/sglang/srt/layers/hc_mix_triton.py rename to python/sglang/kernels/ops/gemm/hc_mix.py diff --git a/python/sglang/kernels/ops/attention/inkling_rel_proj.py b/python/sglang/kernels/ops/gemm/inkling_rel_proj.py similarity index 100% rename from python/sglang/kernels/ops/attention/inkling_rel_proj.py rename to python/sglang/kernels/ops/gemm/inkling_rel_proj.py diff --git a/python/sglang/kernels/ops/gemm/int8_kernel.py b/python/sglang/kernels/ops/gemm/int8_kernel.py new file mode 100644 index 0000000000..3eee2f93c6 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/int8_kernel.py @@ -0,0 +1,229 @@ +import functools +import json +import logging +import os +from typing import Any, Dict, List, Optional + +import torch +import triton +import triton.language as tl + +from sglang.srt.utils import get_device_name + +logger = logging.getLogger(__name__) + + +@triton.jit +def _w8a8_block_int8_matmul( + # Pointers to inputs and output + A, + B, + C, + As, + Bs, + # Shape for matmul + M, + N, + K, + # Block size for block-wise quantization + group_n, + group_k, + # Stride for inputs and output + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + stride_As_m, + stride_As_k, + stride_Bs_k, + stride_Bs_n, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, +): + """Triton-accelerated function used to perform linear operations (dot + product) on input tensors `A` and `B` with block-wise quantization, and store the result in output + tensor `C`. + """ + + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) + b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) + + As_ptrs = As + offs_am * stride_As_m + offs_bsn = offs_bn // group_n + Bs_ptrs = Bs + offs_bsn * stride_Bs_n + + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k * BLOCK_SIZE_K, other=0.0) + b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0) + + k_start = k * BLOCK_SIZE_K + offs_ks = k_start // group_k + a_s = tl.load(As_ptrs + offs_ks * stride_As_k) + b_s = tl.load(Bs_ptrs + offs_ks * stride_Bs_k) + + accumulator += tl.dot(a, b).to(tl.float32) * a_s[:, None] * b_s[None, :] + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + if C.dtype.element_ty == tl.bfloat16: + c = accumulator.to(tl.bfloat16) + elif C.dtype.element_ty == tl.float16: + c = accumulator.to(tl.float16) + else: + c = accumulator.to(tl.float32) + + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + tl.store(c_ptrs, c, mask=c_mask) + + +@functools.lru_cache +def get_w8a8_block_int8_configs( + N: int, K: int, block_n: int, block_k: int +) -> Optional[Dict[int, Any]]: + """ + Return optimized configurations for the w8a8 block fp8 kernel. + + The return value will be a dictionary that maps an irregular grid of + batch sizes to configurations of the w8a8 block fp8 kernel. To evaluate the + kernel on a given batch size bs, the closest batch size in the grid should + be picked and the associated configuration chosen to invoke the kernel. + """ + + # First look up if an optimized configuration is available in the configs + # directory + device_name = get_device_name().replace(" ", "_") + json_file_name = f"N={N},K={K},device_name={device_name},dtype=int8_w8a8,block_shape=[{block_n}, {block_k}].json" + + config_file_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name + ) + if os.path.exists(config_file_path): + with open(config_file_path) as f: + logger.info( + "Using configuration from %s for W8A8 Block INT8 kernel.", + config_file_path, + ) + # If a configuration has been found, return it + return {int(key): val for key, val in json.load(f).items()} + + # If no optimized configuration is available, we will use the default + # configuration + logger.warning( + ( + "Using default W8A8 Block INT8 kernel config. Performance might be sub-optimal! " + "Config file not found at %s" + ), + config_file_path, + ) + return None + + +def w8a8_block_int8_matmul( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + block_size: List[int], + output_dtype: torch.dtype = torch.float16, +) -> torch.Tensor: + """This function performs matrix multiplication with block-wise quantization. + + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + + Args: + A: The input tensor, e.g., activation. + B: The input tensor, e.g., weight. + As: The per-token-group quantization scale for `A`. + Bs: The per-block quantization scale for `B`. + block_size: The block size for per-block quantization. It should be 2-dim, e.g., [128, 128]. + output_dytpe: The dtype of the returned tensor. + + Returns: + torch.Tensor: The result of matmul. + """ + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + + assert A.shape[-1] == B.shape[-1] + assert A.shape[:-1] == As.shape[:-1] and A.is_contiguous() + assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1] + M = A.numel() // A.shape[-1] + + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 + N, K = B.shape + assert triton.cdiv(N, block_n) == Bs.shape[0] + assert triton.cdiv(K, block_k) == Bs.shape[1] + + C_shape = A.shape[:-1] + (N,) + C = A.new_empty(C_shape, dtype=output_dtype) + + configs = get_w8a8_block_int8_configs(N, K, block_size[0], block_size[1]) + if configs: + # If an optimal configuration map has been found, look up the + # optimal config + config = configs[min(configs.keys(), key=lambda x: abs(x - M))] + else: + # Default config + # Block-wise quant: BLOCK_SIZE_K must be divisible by block_size[1] + config = { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": block_size[0], + "BLOCK_SIZE_K": block_size[1], + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + } + + def grid(META): + return ( + triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), + ) + + _w8a8_block_int8_matmul[grid]( + A, + B, + C, + As, + Bs, + M, + N, + K, + block_n, + block_k, + A.stride(-2), + A.stride(-1), + B.stride(1), + B.stride(0), + C.stride(-2), + C.stride(-1), + As.stride(-2), + As.stride(-1), + Bs.stride(1), + Bs.stride(0), + **config, + ) + + return C diff --git a/python/sglang/kernels/ops/layernorm/__init__.py b/python/sglang/kernels/ops/layernorm/__init__.py index 46959aef5a..79048da919 100644 --- a/python/sglang/kernels/ops/layernorm/__init__.py +++ b/python/sglang/kernels/ops/layernorm/__init__.py @@ -511,7 +511,6 @@ from sglang.kernels.spec import KernelSpec # Triton / TileLang kernels migrated from srt/layers top-level strays # (RFC #29630, Phase 2.5); registered for inventory. _PHASE25_KERNELS = [ - ("gemma4_fused_ops", "gemma4_fused_routing", "triton"), ("gemma4_fused_ops", "gemma_qkv_rmsnorm", "triton"), ("mhc_head", "fused_hc_head", "triton"), ("hy4_ihc", "fused_hy4_ihc_pre", "triton"), @@ -555,3 +554,29 @@ for _fn in ("fused_dual_residual_rmsnorm", "fused_rmsnorm"): ) ) del _fn + + +# Public entry points inventoried by logical operator group (RFC #29630). +register_kernel( + KernelSpec( + op="layernorm.rmsnorm_hf", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.layernorm.rmsnorm_hf:rmsnorm_hf", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="layernorm.grouped_gemma_rmsnorm", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.layernorm.grouped_gemma_rmsnorm:grouped_gemma_rmsnorm", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="layernorm.rms_normalize_triton", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.layernorm.rms_normalize_hip:rms_normalize_triton", + ) +) diff --git a/python/sglang/kernels/ops/layernorm/gemma4_fused_ops.py b/python/sglang/kernels/ops/layernorm/gemma4_fused_ops.py index fc1902fdb8..a9e4eed6b7 100644 --- a/python/sglang/kernels/ops/layernorm/gemma4_fused_ops.py +++ b/python/sglang/kernels/ops/layernorm/gemma4_fused_ops.py @@ -317,75 +317,6 @@ def gemma_qkv_rmsnorm( ) -@triton.jit -def _gemma_routing_post_topk_kernel( - Logits_ptr, - Ids_ptr, - Scale_ptr, - Out_weights_ptr, - Out_ids_ptr, - stride_l, - stride_ow, - stride_oi, - K: tl.constexpr, - BLOCK_K: tl.constexpr, -): - """Fused: softmax(topk_logits) * per_expert_scale[topk_ids] → float32 weights, int32 ids. - - One program per token. K is the number of top-k experts (e.g. 8). - """ - row = tl.program_id(0) - cols = tl.arange(0, BLOCK_K) - mask = cols < K - - logits = tl.load( - Logits_ptr + row * stride_l + cols, mask=mask, other=float("-inf") - ).to(tl.float32) - ids_i64 = tl.load(Ids_ptr + row * stride_l + cols, mask=mask, other=0) - - # Stable softmax - max_val = tl.max(logits, axis=0) - exp_val = tl.exp(logits - max_val) - sum_exp = tl.sum(exp_val, axis=0) - weights = exp_val / sum_exp - - # Gather per_expert_scale and multiply - scale = tl.load(Scale_ptr + ids_i64, mask=mask, other=1.0).to(tl.float32) - weights = weights * scale - - tl.store(Out_weights_ptr + row * stride_ow + cols, weights, mask=mask) - tl.store(Out_ids_ptr + row * stride_oi + cols, ids_i64.to(tl.int32), mask=mask) - - -def gemma_routing_post_topk( - topk_logits: torch.Tensor, - topk_ids: torch.Tensor, - per_expert_scale: torch.Tensor, -) -> tuple[torch.Tensor, torch.Tensor]: - """Fused softmax + scale-gather + casts for Gemma4 routing. - - Replaces: softmax(topk_logits) * per_expert_scale[topk_ids] → (f32, i32). - """ - B, K = topk_logits.shape - BLOCK_K = triton.next_power_of_2(K) - out_weights = torch.empty((B, K), dtype=torch.float32, device=topk_logits.device) - out_ids = torch.empty((B, K), dtype=torch.int32, device=topk_logits.device) - - _gemma_routing_post_topk_kernel[(B,)]( - topk_logits, - topk_ids, - per_expert_scale, - out_weights, - out_ids, - topk_logits.stride(0), - out_weights.stride(0), - out_ids.stride(0), - K=K, - BLOCK_K=BLOCK_K, - ) - return out_weights, out_ids - - def gemma_dual_rmsnorm_residual_scalar( x1: torch.Tensor, weight1: torch.Tensor, @@ -424,117 +355,3 @@ def gemma_dual_rmsnorm_residual_scalar( BLOCK_SIZE=BLOCK_SIZE, ) return out - - -@triton.jit -def _gemma4_routing_kernel( - gating_ptr, # [T, E] router logits, any float dtype - per_expert_scale_ptr, # [E] per-expert scale (any float dtype) - topk_weights_ptr, # [T, K] fp32 out - topk_ids_ptr, # [T, K] int32 out - stride_g_t, # stride of gating in the token dim - E: tl.constexpr, - K: tl.constexpr, - BLOCK_E: tl.constexpr, -): - pid = tl.program_id(0) - offs_e = tl.arange(0, BLOCK_E) - valid = offs_e < E - - logits = tl.load( - gating_ptr + pid * stride_g_t + offs_e, - mask=valid, - other=-float("inf"), - ).to(tl.float32) - - # Pack (sort_key, expert_id) into one int64 so a single signed-ascending - # tl.sort yields logits in descending float order. The key bijection is - # anti-monotone on the float value, and the <<32 shift moves its high bit - # into the int64 sign bit. Ties break by expert id ascending. Invalid - # lanes use a max key so they sort last. - MIN32 = -2147483648 - logit_bits = logits.to(tl.int32, bitcast=True) - sign = logit_bits >> 31 - key = tl.where(sign == 0, logit_bits ^ -1, logit_bits ^ MIN32) - key = tl.where(valid, key, 0x7FFFFFFF) - sk64 = key.to(tl.int64) & 0x00000000FFFFFFFF - packed = (sk64 << 32) | offs_e.to(tl.int64) - - sorted_p = tl.sort(packed, descending=False) - all_keys = ((sorted_p >> 32) & 0x00000000FFFFFFFF).to(tl.int32) - all_ids = (sorted_p & 0x00000000FFFFFFFF).to(tl.int32) - - # Invert the key bijection to recover the original logit value. - sign_k = all_keys >> 31 - all_bits = tl.where(sign_k < 0, all_keys ^ -1, all_keys ^ MIN32) - all_logits = all_bits.to(tl.float32, bitcast=True) - - # softmax over the top-K logits; max sits at index 0 (sorted descending). - top_mask = offs_e < K - max_l = tl.max(tl.where(top_mask, all_logits, -float("inf")), axis=0) - raw_exp = tl.where(top_mask, tl.exp(all_logits - max_l), 0.0) - - denom = tl.sum(raw_exp, axis=0) - denom = tl.where(denom > 0.0, denom, 1.0) - weights = raw_exp / denom - - scales = tl.load( - per_expert_scale_ptr + all_ids.to(tl.int64), - mask=top_mask, - other=1.0, - ).to(tl.float32) - weights = weights * scales - - base_off = pid * K + offs_e - tl.store(topk_weights_ptr + base_off, weights, mask=top_mask) - tl.store(topk_ids_ptr + base_off, all_ids, mask=top_mask) - - -def gemma4_fused_routing( - gating_output: torch.Tensor, - per_expert_scale: torch.Tensor, - topk: int, -) -> tuple[torch.Tensor, torch.Tensor]: - """One-launch Gemma4 router. - - Args: - gating_output: [T, E] router logits in any floating dtype; will be - cast to fp32 inside the kernel. - per_expert_scale: [E] per-expert scale, any floating dtype. - topk: number of experts to keep per token. - - Returns: - topk_weights: [T, topk] fp32 (matches SGLang TopK contract). - topk_ids: [T, topk] int32 (matches SGLang TopK contract). - """ - assert gating_output.dim() == 2, "expected [T, E] router logits" - assert per_expert_scale.dim() == 1 - assert per_expert_scale.shape[0] == gating_output.shape[1] - T, E = gating_output.shape - assert topk <= E, f"topk ({topk}) must be <= E ({E})" - assert E <= 1024, f"gemma4_fused_routing only supports E<=1024, got E={E}" - - gating_output = gating_output.contiguous() - per_expert_scale = per_expert_scale.contiguous() - - BLOCK_E = triton.next_power_of_2(E) - topk_weights = torch.empty( - (T, topk), dtype=torch.float32, device=gating_output.device - ) - topk_ids = torch.empty((T, topk), dtype=torch.int32, device=gating_output.device) - - if T == 0: - return topk_weights, topk_ids - - _gemma4_routing_kernel[(T,)]( - gating_output, - per_expert_scale, - topk_weights, - topk_ids, - gating_output.stride(0), - E, - topk, - BLOCK_E, - num_warps=1, - ) - return topk_weights, topk_ids diff --git a/python/sglang/kernels/ops/attention/dsv4/rms_normalize_hip.py b/python/sglang/kernels/ops/layernorm/rms_normalize_hip.py similarity index 100% rename from python/sglang/kernels/ops/attention/dsv4/rms_normalize_hip.py rename to python/sglang/kernels/ops/layernorm/rms_normalize_hip.py diff --git a/python/sglang/kernels/ops/mamba/__init__.py b/python/sglang/kernels/ops/mamba/__init__.py index e4664256b1..d3800b71eb 100644 --- a/python/sglang/kernels/ops/mamba/__init__.py +++ b/python/sglang/kernels/ops/mamba/__init__.py @@ -114,3 +114,38 @@ for _mod, _fn in [ ) ) del _mod, _fn + + +# Public entry points inventoried by logical operator group (RFC #29630). +register_kernel( + KernelSpec( + op="mamba.fused_qwen4_short_conv_state", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.mamba.qwen4_short_conv:fused_qwen4_short_conv_state", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="mamba.fused_qwen4_verify_conv", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.mamba.qwen4_short_conv:fused_qwen4_verify_conv", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="mamba.fused_lfm_short_conv_prefill", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.mamba.lfm_short_conv:fused_lfm_short_conv_prefill", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="mamba.fused_lfm_short_conv_decode", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.mamba.lfm_short_conv:fused_lfm_short_conv_decode", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) diff --git a/python/sglang/kernels/ops/qwen4_ple.py b/python/sglang/kernels/ops/mamba/qwen4_short_conv.py similarity index 54% rename from python/sglang/kernels/ops/qwen4_ple.py rename to python/sglang/kernels/ops/mamba/qwen4_short_conv.py index 6749ae98ec..d586d4f1e3 100644 --- a/python/sglang/kernels/ops/qwen4_ple.py +++ b/python/sglang/kernels/ops/mamba/qwen4_short_conv.py @@ -1,205 +1,14 @@ -"""Fused kernels for Qwen4 PLE decode and target verification.""" - from __future__ import annotations import torch import triton import triton.language as tl -_QWEN4_NGRAM_SIZE = 3 -_QWEN4_HEADS_PER_NGRAM = 8 -_QWEN4_NGRAM_HEADS = 16 -_QWEN4_HC_COUNT = 4 -_QWEN4_HIDDEN_SIZE = 2560 +from sglang.kernels.ops.elementwise.qwen4_gate import _round_bf16_to_fp32 + _QWEN4_MAX_SHORT_CONV_STATE_LEN = 16 -@triton.jit -def _round_bf16_to_fp32(value): - """RNE-round fp32 to BF16 precision while retaining an fp32 register.""" - - bits = value.to(tl.int32, bitcast=True) - rounding_bias = 0x7FFF + ((bits >> 16) & 1) - rounded_bits = (bits + rounding_bias) & -65536 - return rounded_bits.to(tl.float32, bitcast=True) - - -@triton.jit -def _qwen4_ngram_hash_kernel( - contexts_ptr, - multipliers_ptr, - vocab_sizes_ptr, - offsets_ptr, - output_ptr, - num_outputs, - eos_token_id, - NGRAM_SIZE: tl.constexpr, - HEADS_PER_NGRAM: tl.constexpr, - NGRAM_HEADS: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - output_idx = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - mask = output_idx < num_outputs - token_idx = output_idx // NGRAM_HEADS - head_idx = output_idx % NGRAM_HEADS - context_base = token_idx * NGRAM_SIZE - - token_0 = tl.load(contexts_ptr + context_base, mask=mask, other=0) - token_1 = tl.load(contexts_ptr + context_base + 1, mask=mask, other=0) - token_2 = tl.load(contexts_ptr + context_base + 2, mask=mask, other=0) - multiplier_0 = tl.load(multipliers_ptr) - multiplier_1 = tl.load(multipliers_ptr + 1) - multiplier_2 = tl.load(multipliers_ptr + 2) - - # Only the final position of each three-token context is materialized. - previous_1 = tl.where(token_1 == eos_token_id, eos_token_id, token_1) - previous_2 = tl.where( - (token_0 == eos_token_id) | (token_1 == eos_token_id), - eos_token_id, - token_0, - ) - mixed = (token_2 * multiplier_0) ^ (previous_1 * multiplier_1) - mixed_3 = mixed ^ (previous_2 * multiplier_2) - mixed = tl.where(head_idx < HEADS_PER_NGRAM, mixed, mixed_3) - - vocab_size = tl.load(vocab_sizes_ptr + head_idx, mask=mask, other=1) - offset = tl.load(offsets_ptr + head_idx, mask=mask, other=0) - tl.store(output_ptr + output_idx, mixed % vocab_size + offset, mask=mask) - - -def can_fuse_qwen4_ngram_hash( - contexts: torch.Tensor, - multipliers: torch.Tensor, - vocab_sizes: torch.Tensor, - offsets: torch.Tensor, -) -> bool: - """Return whether inputs match the fixed Qwen4 PLE hash contract.""" - - return ( - contexts.is_cuda - and contexts.dtype == torch.long - and contexts.dim() == 2 - and contexts.shape[1] == _QWEN4_NGRAM_SIZE - and contexts.is_contiguous() - and multipliers.is_cuda - and multipliers.dtype == torch.long - and multipliers.numel() == _QWEN4_NGRAM_SIZE - and vocab_sizes.is_cuda - and vocab_sizes.dtype == torch.long - and vocab_sizes.numel() == _QWEN4_NGRAM_HEADS - and offsets.is_cuda - and offsets.dtype == torch.long - and offsets.numel() == _QWEN4_NGRAM_HEADS - ) - - -def fused_qwen4_ngram_hash( - contexts: torch.Tensor, - multipliers: torch.Tensor, - vocab_sizes: torch.Tensor, - offsets: torch.Tensor, - eos_token_id: int, -) -> torch.Tensor: - """Return the 16 Qwen4 PLE N-gram IDs in one kernel launch.""" - - if not can_fuse_qwen4_ngram_hash(contexts, multipliers, vocab_sizes, offsets): - raise ValueError("unsupported input for fused Qwen4 PLE N-gram hash") - output = torch.empty( - (contexts.shape[0], _QWEN4_NGRAM_HEADS), - dtype=torch.long, - device=contexts.device, - ) - num_outputs = output.numel() - if num_outputs: - block_size = 256 - _qwen4_ngram_hash_kernel[(triton.cdiv(num_outputs, block_size),)]( - contexts, - multipliers, - vocab_sizes, - offsets, - output, - num_outputs, - eos_token_id, - NGRAM_SIZE=_QWEN4_NGRAM_SIZE, - HEADS_PER_NGRAM=_QWEN4_HEADS_PER_NGRAM, - NGRAM_HEADS=_QWEN4_NGRAM_HEADS, - BLOCK_SIZE=block_size, - num_warps=4, - ) - return output - - -@triton.jit -def _qwen4_gate_value_kernel( - gate_ptr, - value_ptr, - output_ptr, - num_tokens, - HC_COUNT: tl.constexpr, - HIDDEN_SIZE: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - token_group = tl.program_id(0) - token = token_group // HC_COUNT - hidden = tl.arange(0, BLOCK_SIZE) - mask = (token < num_tokens) & (hidden < HIDDEN_SIZE) - - # `gate` arrives already rounded to bf16 by the eager multiply/reduce/divide; - # every remaining eager bf16 rounding boundary is reproduced below. - gate = tl.load(gate_ptr + token_group).to(tl.float32) - magnitude = tl.maximum(tl.abs(gate), 1.0e-6) - root = _round_bf16_to_fp32(tl.sqrt(magnitude)) - sign = tl.where(gate > 0.0, 1.0, tl.where(gate < 0.0, -1.0, 0.0)) - transformed = _round_bf16_to_fp32(root * sign) - activated = _round_bf16_to_fp32(tl.sigmoid(transformed)) - - value = tl.load(value_ptr + token * HIDDEN_SIZE + hidden, mask=mask, other=0.0).to( - tl.float32 - ) - output_offset = token_group * HIDDEN_SIZE + hidden - tl.store(output_ptr + output_offset, activated * value, mask=mask) - - -def can_fuse_qwen4_gate_value(gate: torch.Tensor, value: torch.Tensor) -> bool: - """Return whether inputs match Qwen4's fixed BF16 gate/value contract.""" - - return ( - gate.is_cuda - and gate.dtype == torch.bfloat16 - and gate.dim() == 3 - and gate.shape[1:] == (_QWEN4_HC_COUNT, 1) - and gate.is_contiguous() - and value.is_cuda - and value.dtype == gate.dtype - and value.shape == (gate.shape[0], _QWEN4_HIDDEN_SIZE) - and value.is_contiguous() - ) - - -def fused_qwen4_gate_value(gate: torch.Tensor, value: torch.Tensor) -> torch.Tensor: - """Apply Qwen4's post-reduction gate and value broadcast in one kernel.""" - - if not can_fuse_qwen4_gate_value(gate, value): - raise ValueError("unsupported input for fused Qwen4 PLE gate/value") - output = torch.empty( - (gate.shape[0], _QWEN4_HC_COUNT, _QWEN4_HIDDEN_SIZE), - dtype=value.dtype, - device=value.device, - ) - if gate.shape[0]: - _qwen4_gate_value_kernel[(gate.shape[0] * _QWEN4_HC_COUNT,)]( - gate, - value, - output, - gate.shape[0], - HC_COUNT=_QWEN4_HC_COUNT, - HIDDEN_SIZE=_QWEN4_HIDDEN_SIZE, - BLOCK_SIZE=4096, - num_warps=8, - ) - return output - - @triton.jit def _qwen4_short_conv_state_kernel( state_ptr, @@ -309,72 +118,6 @@ def fused_qwen4_short_conv_state( return conv_input -@triton.jit -def _qwen4_gate_reduce_kernel( - key_ptr, - query_ptr, - value_ptr, - output_ptr, - HIDDEN_SIZE: tl.constexpr, - HC_COUNT: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - group = tl.program_id(0) - hidden = tl.arange(0, BLOCK_SIZE) - mask = hidden < HIDDEN_SIZE - key = tl.load(key_ptr + group * HIDDEN_SIZE + hidden, mask, other=0).to(tl.float32) - query = tl.load(query_ptr + group * HIDDEN_SIZE + hidden, mask, other=0).to( - tl.float32 - ) - product = _round_bf16_to_fp32(key * query) - gate = _round_bf16_to_fp32(tl.sum(product, 0)) - gate = _round_bf16_to_fp32(gate * (HIDDEN_SIZE**-0.5)) - magnitude = _round_bf16_to_fp32(tl.maximum(tl.abs(gate), 1.0e-6)) - root = _round_bf16_to_fp32(tl.sqrt(magnitude)) - sign = tl.where(gate > 0, 1.0, tl.where(gate < 0, -1.0, 0.0)) - activated = _round_bf16_to_fp32(tl.sigmoid(_round_bf16_to_fp32(root * sign))) - value = tl.load( - value_ptr + (group // HC_COUNT) * HIDDEN_SIZE + hidden, mask, other=0 - ).to(tl.float32) - tl.store(output_ptr + group * HIDDEN_SIZE + hidden, activated * value, mask) - - -def can_fuse_qwen4_gate_reduce(key, query, value): - return ( - key.is_cuda - and key.dtype == torch.bfloat16 - and key.ndim == 3 - and key.shape[1:] == (_QWEN4_HC_COUNT, _QWEN4_HIDDEN_SIZE) - and key.is_contiguous() - and query.shape == key.shape - and query.dtype == key.dtype - and query.device == key.device - and query.is_contiguous() - and value.shape == (key.shape[0], _QWEN4_HIDDEN_SIZE) - and value.dtype == key.dtype - and value.device == key.device - and value.is_contiguous() - ) - - -def fused_qwen4_gate_reduce(key, query, value): - if not can_fuse_qwen4_gate_reduce(key, query, value): - raise ValueError("unsupported input for Qwen4 PLE gate reduction") - output = torch.empty_like(key) - if key.shape[0]: - _qwen4_gate_reduce_kernel[(key.shape[0] * _QWEN4_HC_COUNT,)]( - key, - query, - value, - output, - HIDDEN_SIZE=_QWEN4_HIDDEN_SIZE, - HC_COUNT=_QWEN4_HC_COUNT, - BLOCK_SIZE=triton.next_power_of_2(_QWEN4_HIDDEN_SIZE), - enable_fp_fusion=False, - ) - return output - - @triton.jit def _qwen4_verify_conv_prepare_kernel( x_ptr, diff --git a/python/sglang/kernels/ops/memory/__init__.py b/python/sglang/kernels/ops/memory/__init__.py index 7a4aaa0d7e..7d3f2fe8f5 100644 --- a/python/sglang/kernels/ops/memory/__init__.py +++ b/python/sglang/kernels/ops/memory/__init__.py @@ -6,7 +6,7 @@ The Triton kernels migrated here live in this package """ from sglang.kernels.registry import register_kernel -from sglang.kernels.spec import KernelBackend, KernelSpec +from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec # (module, public_fn) migrated from mem_cache/triton_ops. _TRITON_KERNELS = [ @@ -48,3 +48,38 @@ register_kernel( target="sglang.kernels.ops.memory.memcpy_triton:memcpy_triton", ) ) + + +# Public entry points inventoried by logical operator group (RFC #29630). +register_kernel( + KernelSpec( + op="memory.adler32_checksum", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.memory.adler32:adler32_checksum", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="memory.adler32_regions_checksum", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.memory.adler32:adler32_regions_checksum", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="memory.adler32_strided_checksum", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.memory.adler32:adler32_strided_checksum", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="memory.row_compact_bf16", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.memory.row_compact:row_compact_bf16", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) diff --git a/python/sglang/kernels/ops/memory/row_compact.py b/python/sglang/kernels/ops/memory/row_compact.py new file mode 100644 index 0000000000..f19477d43f --- /dev/null +++ b/python/sglang/kernels/ops/memory/row_compact.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import torch + +from sglang.kernels.jit.utils import ( + is_arch_support_pdl, +) +from sglang.kernels.ops.elementwise.row_scale import _jit_row_scale_module + + +def row_compact_bf16(x: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor: + """Contiguous copy of row-strided ``x`` ([rows, inner] bf16, inner + contiguous, inner % 8 == 0, 16B-aligned rows) -- the tau-less flavor of + ``row_scale_bf16``. Beats the TensorIterator strided copy that einsum's + reshape would otherwise run on such inputs.""" + if out is None: + out = torch.empty(x.shape, dtype=x.dtype, device=x.device) + module = _jit_row_scale_module(is_arch_support_pdl()) + module.run_compact(x, out) + return out diff --git a/python/sglang/kernels/ops/mm/__init__.py b/python/sglang/kernels/ops/mm/__init__.py index 794777ad94..c7569b8a3b 100644 --- a/python/sglang/kernels/ops/mm/__init__.py +++ b/python/sglang/kernels/ops/mm/__init__.py @@ -1,3 +1,25 @@ -"""Multimodal kernels.""" +"""Multimodal input-processing kernels; implementations are imported lazily.""" + +from sglang.kernels.registry import register_kernel +from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec __all__ = ["process"] + + +# Public entry points inventoried by logical operator group (RFC #29630). +register_kernel( + KernelSpec( + op="mm.normalize_and_patchify", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.mm.process.image:normalize_and_patchify", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) + +register_kernel( + KernelSpec( + op="mm.normalize_and_patchify", + backend=KernelBackend.TORCH, + target="sglang.kernels.ops.mm.process.image:_normalize_and_patchify_torch", + ) +) diff --git a/python/sglang/kernels/ops/moe/__init__.py b/python/sglang/kernels/ops/moe/__init__.py index aa6000a748..02657fded5 100644 --- a/python/sglang/kernels/ops/moe/__init__.py +++ b/python/sglang/kernels/ops/moe/__init__.py @@ -245,3 +245,69 @@ register_kernel( capabilities=frozenset({CapabilityRequirement.CUDA}), ) ) + + +# Public entry points inventoried by logical operator group (RFC #29630). +register_kernel( + KernelSpec( + op="moe.gemma_routing_post_topk", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.moe.gemma4_routing:gemma_routing_post_topk", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) +register_kernel( + KernelSpec( + op="moe.gemma4_fused_routing", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.moe.gemma4_routing:gemma4_fused_routing", + ) +) +register_kernel( + KernelSpec( + op="moe.mask_topk_ids", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.moe.dsv4:mask_topk_ids", + capabilities=frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP}), + ) +) +register_kernel( + KernelSpec( + op="moe.hash_topk", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.moe.dsv4:hash_topk", + capabilities=frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP}), + ) +) +register_kernel( + KernelSpec( + op="moe.mega_moe_pre_dispatch", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.moe.dsv4:mega_moe_pre_dispatch", + capabilities=frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP}), + ) +) +register_kernel( + KernelSpec( + op="moe.silu_and_mul_clamp", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.moe.dsv4:silu_and_mul_clamp", + capabilities=frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP}), + ) +) +register_kernel( + KernelSpec( + op="moe.silu_and_mul_masked_post_quant", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.moe.dsv4:silu_and_mul_masked_post_quant", + capabilities=frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP}), + ) +) +register_kernel( + KernelSpec( + op="moe.silu_and_mul_contig_post_quant", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.moe.dsv4:silu_and_mul_contig_post_quant", + capabilities=frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP}), + ) +) diff --git a/python/sglang/kernels/ops/attention/dsv4/moe.py b/python/sglang/kernels/ops/moe/dsv4.py similarity index 99% rename from python/sglang/kernels/ops/attention/dsv4/moe.py rename to python/sglang/kernels/ops/moe/dsv4.py index 05736cf4ca..8abfa94912 100644 --- a/python/sglang/kernels/ops/attention/dsv4/moe.py +++ b/python/sglang/kernels/ops/moe/dsv4.py @@ -11,11 +11,13 @@ from sglang.kernels.jit.utils import ( ) from sglang.srt.utils import is_xpu -from .utils import make_name - _is_xpu = is_xpu() +def make_name(name: str) -> str: + return f"dpsk_v4_{name}" + + @cache_once def _jit_mask_topk_module(): return load_jit( diff --git a/python/sglang/kernels/ops/moe/gemma4_routing.py b/python/sglang/kernels/ops/moe/gemma4_routing.py new file mode 100644 index 0000000000..7ab3b8a7d6 --- /dev/null +++ b/python/sglang/kernels/ops/moe/gemma4_routing.py @@ -0,0 +1,186 @@ +import torch +import triton +import triton.language as tl + + +@triton.jit +def _gemma_routing_post_topk_kernel( + Logits_ptr, + Ids_ptr, + Scale_ptr, + Out_weights_ptr, + Out_ids_ptr, + stride_l, + stride_ow, + stride_oi, + K: tl.constexpr, + BLOCK_K: tl.constexpr, +): + """Fused: softmax(topk_logits) * per_expert_scale[topk_ids] → float32 weights, int32 ids. + + One program per token. K is the number of top-k experts (e.g. 8). + """ + row = tl.program_id(0) + cols = tl.arange(0, BLOCK_K) + mask = cols < K + + logits = tl.load( + Logits_ptr + row * stride_l + cols, mask=mask, other=float("-inf") + ).to(tl.float32) + ids_i64 = tl.load(Ids_ptr + row * stride_l + cols, mask=mask, other=0) + + # Stable softmax + max_val = tl.max(logits, axis=0) + exp_val = tl.exp(logits - max_val) + sum_exp = tl.sum(exp_val, axis=0) + weights = exp_val / sum_exp + + # Gather per_expert_scale and multiply + scale = tl.load(Scale_ptr + ids_i64, mask=mask, other=1.0).to(tl.float32) + weights = weights * scale + + tl.store(Out_weights_ptr + row * stride_ow + cols, weights, mask=mask) + tl.store(Out_ids_ptr + row * stride_oi + cols, ids_i64.to(tl.int32), mask=mask) + + +def gemma_routing_post_topk( + topk_logits: torch.Tensor, + topk_ids: torch.Tensor, + per_expert_scale: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Fused softmax + scale-gather + casts for Gemma4 routing. + + Replaces: softmax(topk_logits) * per_expert_scale[topk_ids] → (f32, i32). + """ + B, K = topk_logits.shape + BLOCK_K = triton.next_power_of_2(K) + out_weights = torch.empty((B, K), dtype=torch.float32, device=topk_logits.device) + out_ids = torch.empty((B, K), dtype=torch.int32, device=topk_logits.device) + + _gemma_routing_post_topk_kernel[(B,)]( + topk_logits, + topk_ids, + per_expert_scale, + out_weights, + out_ids, + topk_logits.stride(0), + out_weights.stride(0), + out_ids.stride(0), + K=K, + BLOCK_K=BLOCK_K, + ) + return out_weights, out_ids + + +@triton.jit +def _gemma4_routing_kernel( + gating_ptr, # [T, E] router logits, any float dtype + per_expert_scale_ptr, # [E] per-expert scale (any float dtype) + topk_weights_ptr, # [T, K] fp32 out + topk_ids_ptr, # [T, K] int32 out + stride_g_t, # stride of gating in the token dim + E: tl.constexpr, + K: tl.constexpr, + BLOCK_E: tl.constexpr, +): + pid = tl.program_id(0) + offs_e = tl.arange(0, BLOCK_E) + valid = offs_e < E + + logits = tl.load( + gating_ptr + pid * stride_g_t + offs_e, + mask=valid, + other=-float("inf"), + ).to(tl.float32) + + # Pack (sort_key, expert_id) into one int64 so a single signed-ascending + # tl.sort yields logits in descending float order. The key bijection is + # anti-monotone on the float value, and the <<32 shift moves its high bit + # into the int64 sign bit. Ties break by expert id ascending. Invalid + # lanes use a max key so they sort last. + MIN32 = -2147483648 + logit_bits = logits.to(tl.int32, bitcast=True) + sign = logit_bits >> 31 + key = tl.where(sign == 0, logit_bits ^ -1, logit_bits ^ MIN32) + key = tl.where(valid, key, 0x7FFFFFFF) + sk64 = key.to(tl.int64) & 0x00000000FFFFFFFF + packed = (sk64 << 32) | offs_e.to(tl.int64) + + sorted_p = tl.sort(packed, descending=False) + all_keys = ((sorted_p >> 32) & 0x00000000FFFFFFFF).to(tl.int32) + all_ids = (sorted_p & 0x00000000FFFFFFFF).to(tl.int32) + + # Invert the key bijection to recover the original logit value. + sign_k = all_keys >> 31 + all_bits = tl.where(sign_k < 0, all_keys ^ -1, all_keys ^ MIN32) + all_logits = all_bits.to(tl.float32, bitcast=True) + + # softmax over the top-K logits; max sits at index 0 (sorted descending). + top_mask = offs_e < K + max_l = tl.max(tl.where(top_mask, all_logits, -float("inf")), axis=0) + raw_exp = tl.where(top_mask, tl.exp(all_logits - max_l), 0.0) + + denom = tl.sum(raw_exp, axis=0) + denom = tl.where(denom > 0.0, denom, 1.0) + weights = raw_exp / denom + + scales = tl.load( + per_expert_scale_ptr + all_ids.to(tl.int64), + mask=top_mask, + other=1.0, + ).to(tl.float32) + weights = weights * scales + + base_off = pid * K + offs_e + tl.store(topk_weights_ptr + base_off, weights, mask=top_mask) + tl.store(topk_ids_ptr + base_off, all_ids, mask=top_mask) + + +def gemma4_fused_routing( + gating_output: torch.Tensor, + per_expert_scale: torch.Tensor, + topk: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """One-launch Gemma4 router. + + Args: + gating_output: [T, E] router logits in any floating dtype; will be + cast to fp32 inside the kernel. + per_expert_scale: [E] per-expert scale, any floating dtype. + topk: number of experts to keep per token. + + Returns: + topk_weights: [T, topk] fp32 (matches SGLang TopK contract). + topk_ids: [T, topk] int32 (matches SGLang TopK contract). + """ + assert gating_output.dim() == 2, "expected [T, E] router logits" + assert per_expert_scale.dim() == 1 + assert per_expert_scale.shape[0] == gating_output.shape[1] + T, E = gating_output.shape + assert topk <= E, f"topk ({topk}) must be <= E ({E})" + assert E <= 1024, f"gemma4_fused_routing only supports E<=1024, got E={E}" + + gating_output = gating_output.contiguous() + per_expert_scale = per_expert_scale.contiguous() + + BLOCK_E = triton.next_power_of_2(E) + topk_weights = torch.empty( + (T, topk), dtype=torch.float32, device=gating_output.device + ) + topk_ids = torch.empty((T, topk), dtype=torch.int32, device=gating_output.device) + + if T == 0: + return topk_weights, topk_ids + + _gemma4_routing_kernel[(T,)]( + gating_output, + per_expert_scale, + topk_weights, + topk_ids, + gating_output.stride(0), + E, + topk, + BLOCK_E, + num_warps=1, + ) + return topk_weights, topk_ids diff --git a/python/sglang/kernels/ops/quantization/__init__.py b/python/sglang/kernels/ops/quantization/__init__.py index 2032116e64..97baf5a4fc 100644 --- a/python/sglang/kernels/ops/quantization/__init__.py +++ b/python/sglang/kernels/ops/quantization/__init__.py @@ -172,15 +172,12 @@ _TRITON_KERNELS = [ ("fp8_kernel", "sglang_per_token_group_quant_8bit"), ("fp8_kernel", "sglang_per_token_quant_fp8"), ("fp8_kernel", "static_quant_fp8"), - ("fp8_kernel", "w8a8_block_fp8_matmul"), ("fp8_kernel", "per_tensor_quant_mla_fp8"), ("fp8_kernel", "per_token_group_quant_mla_deep_gemm_masked_fp8"), ("fp8_kernel", "per_token_group_quant_fp8_hopper_moe_mn_major"), ("fp8_kernel", "per_group_transpose"), - ("fp8_kernel", "triton_scaled_mm"), ("int8_kernel", "per_token_quant_int8"), ("int8_kernel", "per_token_group_quant_int8"), - ("int8_kernel", "w8a8_block_int8_matmul"), ("awq_triton", "awq_dequantize_triton"), ("awq_triton", "awq_gemm_triton"), ("mxfp8_amd_gfx95", "mxfp8_e4m3_quantize"), @@ -207,3 +204,14 @@ register_kernel( description="Fused NVFP4 GEMM + SwiGLU + NVFP4 quant (CuTe DSL, SM100).", ) ) + + +# Public entry points inventoried by logical operator group (RFC #29630). +register_kernel( + KernelSpec( + op="quantization.cvt_fp8_e4m3", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.quantization.fp8_cvt:cvt_fp8_e4m3", + capabilities=frozenset({CapabilityRequirement.CUDA}), + ) +) diff --git a/python/sglang/kernels/ops/quantization/dsv32/__init__.py b/python/sglang/kernels/ops/quantization/dsv32/__init__.py deleted file mode 100644 index a1b357c5f5..0000000000 --- a/python/sglang/kernels/ops/quantization/dsv32/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""DSA only.""" - -from .elementwise import ( - fused_k_indexer_norm_rope, - fused_k_indexer_norm_rope_store, -) - -__all__ = [ - "fused_k_indexer_norm_rope", - "fused_k_indexer_norm_rope_store", -] diff --git a/python/sglang/kernels/ops/attention/dsv4/fp8_cvt.py b/python/sglang/kernels/ops/quantization/fp8_cvt.py similarity index 95% rename from python/sglang/kernels/ops/attention/dsv4/fp8_cvt.py rename to python/sglang/kernels/ops/quantization/fp8_cvt.py index 9ff05c89d3..178aa2947d 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp8_cvt.py +++ b/python/sglang/kernels/ops/quantization/fp8_cvt.py @@ -6,12 +6,14 @@ import torch from sglang.kernels.jit.utils import cache_once, load_jit -from .utils import make_name - if TYPE_CHECKING: from tvm_ffi.module import Module +def make_name(name: str) -> str: + return f"dpsk_v4_{name}" + + @cache_once def _jit_fp8_cvt_module() -> Module: return load_jit( diff --git a/python/sglang/kernels/ops/quantization/fp8_kernel.py b/python/sglang/kernels/ops/quantization/fp8_kernel.py index 3f783ed8b5..8ec785e18b 100644 --- a/python/sglang/kernels/ops/quantization/fp8_kernel.py +++ b/python/sglang/kernels/ops/quantization/fp8_kernel.py @@ -12,12 +12,9 @@ # limitations under the License. # ============================================================================== -import functools -import json import logging -import os from functools import lru_cache -from typing import Any, Dict, List, Optional, Tuple +from typing import Optional, Tuple import torch import triton @@ -25,22 +22,16 @@ import triton.language as tl from sglang.kernels.jit.utils import is_arch_support_pdl from sglang.kernels.ops.quantization.fp8_utils import fp8_dtype_to_triton -from sglang.srt.layers import deep_gemm_wrapper -from sglang.srt.runtime_context import get_platform from sglang.srt.utils import ( ceil_align, get_bool_env_var, - get_device_core_count, - get_device_name, is_cpu, is_cuda, is_gfx1250_supported, is_hip, is_musa, is_xpu, - log_info_on_rank0, ) -from sglang.srt.utils.custom_op import register_custom_op from sglang.srt.utils.patch_torch import register_fake_if_exists _is_hip = is_hip() @@ -121,28 +112,6 @@ else: fp8_min = -fp8_max -@register_custom_op(mutates_args=["C"]) -def deep_gemm_fp8_fp8_bf16_nt( - A: torch.Tensor, - As: torch.Tensor, - B: torch.Tensor, - Bs: torch.Tensor, - C: torch.Tensor, -) -> None: - deep_gemm_wrapper.gemm_nt_f8f8bf16((A, As), (B, Bs), C) - - -@register_custom_op(mutates_args=["C"]) -def deep_gemm_mxfp8_fp8_bf16_nt( - A: torch.Tensor, - As: torch.Tensor, - B: torch.Tensor, - Bs: torch.Tensor, - C: torch.Tensor, -) -> None: - deep_gemm_wrapper.gemm_nt_mxfp8_f8f8bf16((A, As), (B, Bs), C) - - @triton.jit def _per_token_group_quant_8bit( # Pointers to inputs and output @@ -928,853 +897,7 @@ def static_quant_fp8( return x_q, x_s -@triton.jit -def _w8a8_block_fp8_matmul( - # Pointers to inputs and output - A, - B, - C, - As, - Bs, - # Shape for matmul - M, - N, - K, - # Block size for block-wise quantization - group_n, - group_k, - # Stride for inputs and output - stride_am, - stride_ak, - stride_bk, - stride_bn, - stride_cm, - stride_cn, - stride_As_m, - stride_As_k, - stride_Bs_k, - stride_Bs_n, - # Meta-parameters - BLOCK_SIZE_M: tl.constexpr, - BLOCK_SIZE_N: tl.constexpr, - BLOCK_SIZE_K: tl.constexpr, - GROUP_SIZE_M: tl.constexpr, - needs_masking: tl.constexpr, -): - """Triton-accelerated function used to perform linear operations (dot - product) on input tensors `A` and `B` with block-wise quantization, and store the result in output - tensor `C`. - """ - - pid = tl.program_id(axis=0) - num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) - num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) - num_pid_in_group = GROUP_SIZE_M * num_pid_n - group_id = pid // num_pid_in_group - first_pid_m = group_id * GROUP_SIZE_M - group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) - pid_m = first_pid_m + (pid % group_size_m) - pid_n = (pid % num_pid_in_group) // group_size_m - - offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M - offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N - offs_k = tl.arange(0, BLOCK_SIZE_K) - a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) - b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) - - As_ptrs = As + offs_am * stride_As_m - offs_bsn = offs_bn // group_n - Bs_ptrs = Bs + offs_bsn * stride_Bs_n - n_tiles_k_per_group_k = group_k // BLOCK_SIZE_K - - accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) - for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): - if needs_masking: - a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k * BLOCK_SIZE_K, other=0.0) - b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0) - else: - a = tl.load(a_ptrs) - b = tl.load(b_ptrs) - - a_s = tl.load(As_ptrs) - b_s = tl.load(Bs_ptrs) - - scale_step_k = tl.where((k + 1) % n_tiles_k_per_group_k == 0, 1, 0) - accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] - a_ptrs += BLOCK_SIZE_K * stride_ak - b_ptrs += BLOCK_SIZE_K * stride_bk - As_ptrs += scale_step_k * stride_As_k - Bs_ptrs += scale_step_k * stride_Bs_k - - if C.dtype.element_ty == tl.bfloat16: - c = accumulator.to(tl.bfloat16) - elif C.dtype.element_ty == tl.float16: - c = accumulator.to(tl.float16) - else: - c = accumulator.to(tl.float32) - - offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) - offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) - c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] - c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) - tl.store(c_ptrs, c, mask=c_mask) - - -@triton.jit -def _w8a8_block_fp8_matmul_hopper( - # Pointers to inputs and output - A, - B, - C, - As, - Bs, - # Shape for matmul - M, - N, - K, - # Block size for block-wise quantization - group_n, - group_k, - # Stride for inputs and output - stride_am, - stride_ak, - stride_bk, - stride_bn, - stride_cm, - stride_cn, - stride_As_m, - stride_As_k, - stride_Bs_k, - stride_Bs_n, - # Meta-parameters - BLOCK_SIZE_M: tl.constexpr, - BLOCK_SIZE_N: tl.constexpr, - BLOCK_SIZE_K: tl.constexpr, - GROUP_SIZE_M: tl.constexpr, - needs_masking: tl.constexpr, - SWAP_AB: tl.constexpr = False, - SPLIT_K: tl.constexpr = 1, -): - - pid = tl.program_id(axis=0) - split = tl.program_id(axis=1) - tiles_per_split = tl.cdiv(tl.cdiv(K, BLOCK_SIZE_K), SPLIT_K) - first_tile = split * tiles_per_split - C += split * M * N - num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) - num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) - num_pid_in_group = GROUP_SIZE_M * num_pid_n - group_id = pid // num_pid_in_group - first_pid_m = group_id * GROUP_SIZE_M - group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) - pid_m = first_pid_m + (pid % group_size_m) - pid_n = (pid % num_pid_in_group) // group_size_m - - offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M - offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N - offs_k = tl.arange(0, BLOCK_SIZE_K) - a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) - b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) - - As_ptrs = As + offs_am * stride_As_m - offs_bsn = offs_bn // group_n - Bs_ptrs = Bs + offs_bsn * stride_Bs_n - n_tiles_k_per_group_k = group_k // BLOCK_SIZE_K - - a_ptrs += first_tile * BLOCK_SIZE_K * stride_ak - b_ptrs += first_tile * BLOCK_SIZE_K * stride_bk - As_ptrs += (first_tile // n_tiles_k_per_group_k) * stride_As_k - Bs_ptrs += (first_tile // n_tiles_k_per_group_k) * stride_Bs_k - - # Small-M Hopper configs transpose the MMA so the weight tile occupies M. - if SWAP_AB: - accumulator = tl.zeros((BLOCK_SIZE_N, BLOCK_SIZE_M), dtype=tl.float32) - else: - accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) - for k in range( - first_tile, tl.minimum(first_tile + tiles_per_split, tl.cdiv(K, BLOCK_SIZE_K)) - ): - if needs_masking: - a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k * BLOCK_SIZE_K, other=0.0) - b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0) - else: - a = tl.load(a_ptrs) - b = tl.load(b_ptrs) - - a_s = tl.load(As_ptrs) - b_s = tl.load(Bs_ptrs) - - scale_step_k = tl.where((k + 1) % n_tiles_k_per_group_k == 0, 1, 0) - if SWAP_AB: - accumulator += ( - tl.dot(tl.trans(b), tl.trans(a)) * b_s[:, None] * a_s[None, :] - ) - else: - accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] - a_ptrs += BLOCK_SIZE_K * stride_ak - b_ptrs += BLOCK_SIZE_K * stride_bk - As_ptrs += scale_step_k * stride_As_k - Bs_ptrs += scale_step_k * stride_Bs_k - - if SWAP_AB: - accumulator = tl.trans(accumulator) - - if C.dtype.element_ty == tl.bfloat16: - c = accumulator.to(tl.bfloat16) - elif C.dtype.element_ty == tl.float16: - c = accumulator.to(tl.float16) - else: - c = accumulator.to(tl.float32) - - offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) - offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) - c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] - c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) - tl.store(c_ptrs, c, mask=c_mask) - - -@triton.jit -def _reduce_block_fp8_split_k( - Parts, Out, ELEMENTS: tl.constexpr, SPLITS: tl.constexpr, BLOCK: tl.constexpr -): - offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) - splits = tl.arange(0, SPLITS) - values = tl.load( - Parts + splits[:, None] * ELEMENTS + offsets[None, :], - offsets[None, :] < ELEMENTS, - 0.0, - ) - tl.store(Out + offsets, tl.sum(values, axis=0), offsets < ELEMENTS) - - -@triton.jit -def _w8a8_block_fp8_matmul_gfx1250( - # Pointers to inputs and output - A, - B, - C, - As, - Bs, - # Shape for matmul - M, - N, - K, - # Block size for block-wise quantization - group_n, - group_k, - # Stride for inputs and output - stride_am, - stride_ak, - stride_bk, - stride_bn, - stride_cm, - stride_cn, - stride_As_m, - stride_As_k, - stride_Bs_k, - stride_Bs_n, - # Meta-parameters - BLOCK_SIZE_M: tl.constexpr, - BLOCK_SIZE_N: tl.constexpr, - BLOCK_SIZE_K: tl.constexpr, - GROUP_SIZE_M: tl.constexpr, - needs_masking: tl.constexpr, -): - """ - gfx1250 (RDNA4) block-fp8 matmul. - The shared ``_w8a8_block_fp8_matmul`` is unusable on gfx1250. - 1. fp8 ``tl.dot`` faults at runtime - 2. software pipelining (``num_stages`` > 1) miscompiles and yields NaN. - 3. the ``offs % M`` / ``offs % N`` modulo-wrap index trick is - intermittently miscompiled into out-of-bounds addresses. - """ - - pid = tl.program_id(axis=0) - num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) - num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) - num_pid_in_group = GROUP_SIZE_M * num_pid_n - group_id = pid // num_pid_in_group - first_pid_m = group_id * GROUP_SIZE_M - group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) - pid_m = first_pid_m + (pid % group_size_m) - pid_n = (pid % num_pid_in_group) // group_size_m - - # No modulo-wrap on gfx1250; use explicit masks instead. - offs_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) - offs_bn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) - m_mask = offs_am < M - n_mask = offs_bn < N - - offs_am_c = tl.where(m_mask, offs_am, 0) - offs_bn_c = tl.where(n_mask, offs_bn, 0) - offs_k = tl.arange(0, BLOCK_SIZE_K) - a_ptrs = A + (offs_am_c[:, None] * stride_am + offs_k[None, :] * stride_ak) - b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn_c[None, :] * stride_bn) - - As_ptrs = As + offs_am_c * stride_As_m - offs_bsn = offs_bn_c // group_n - Bs_ptrs = Bs + offs_bsn * stride_Bs_n - n_tiles_k_per_group_k = group_k // BLOCK_SIZE_K - - accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) - for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): - k_mask = offs_k < K - k * BLOCK_SIZE_K - a = tl.load(a_ptrs, mask=m_mask[:, None] & k_mask[None, :], other=0.0) - b = tl.load(b_ptrs, mask=k_mask[:, None] & n_mask[None, :], other=0.0) - - a_s = tl.load(As_ptrs, mask=m_mask, other=0.0) - b_s = tl.load(Bs_ptrs, mask=n_mask, other=0.0) - - scale_step_k = tl.where((k + 1) % n_tiles_k_per_group_k == 0, 1, 0) - - # Upcast fp8 to bf16 in-register - accumulator += ( - tl.dot(a.to(tl.bfloat16), b.to(tl.bfloat16)) * a_s[:, None] * b_s[None, :] - ) - a_ptrs += BLOCK_SIZE_K * stride_ak - b_ptrs += BLOCK_SIZE_K * stride_bk - As_ptrs += scale_step_k * stride_As_k - Bs_ptrs += scale_step_k * stride_Bs_k - - if C.dtype.element_ty == tl.bfloat16: - c = accumulator.to(tl.bfloat16) - elif C.dtype.element_ty == tl.float16: - c = accumulator.to(tl.float16) - else: - c = accumulator.to(tl.float32) - - offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) - offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) - c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] - c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) - tl.store(c_ptrs, c, mask=c_mask) - - -@triton.jit -def _w8a8_block_fp8_matmul_unrolledx4( - # Pointers to inputs and output - A, - B, - C, - As, - Bs, - # Shape for matmul - M, - N, - K, - # Block size for block-wise quantization - group_n, - group_k, - # Stride for inputs and output - stride_am, - stride_ak, - stride_bk, - stride_bn, - stride_cm, - stride_cn, - stride_As_m, - stride_As_k, - stride_Bs_k, - stride_Bs_n, - # Meta-parameters - BLOCK_SIZE_M: tl.constexpr, - BLOCK_SIZE_N: tl.constexpr, - BLOCK_SIZE_K: tl.constexpr, - GROUP_SIZE_M: tl.constexpr, - needs_masking: tl.constexpr, -): - """Triton-accelerated function used to perform linear operations (dot - product) on input tensors `A` and `B` with block-wise quantization, and store the result in output - tensor `C`. - """ - - pid = tl.program_id(axis=0) - num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) - num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) - num_pid_in_group = GROUP_SIZE_M * num_pid_n - group_id = pid // num_pid_in_group - first_pid_m = group_id * GROUP_SIZE_M - group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) - pid_m = first_pid_m + (pid % group_size_m) - pid_n = (pid % num_pid_in_group) // group_size_m - - offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M - offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N - offs_k = tl.arange(0, BLOCK_SIZE_K) - a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) - b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) - - As_ptrs = As + offs_am * stride_As_m - offs_bsn = offs_bn // group_n - Bs_ptrs = Bs + offs_bsn * stride_Bs_n - scale_step_k = BLOCK_SIZE_K // group_k - - accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) - # manually unroll to 4 iterations - UNROLL_FACTOR = 4 - for k in range(0, tl.cdiv(K, BLOCK_SIZE_K * UNROLL_FACTOR)): - # 1st iteration - if needs_masking: - a = tl.load( - a_ptrs, - mask=offs_k[None, :] < K - (k * UNROLL_FACTOR) * BLOCK_SIZE_K, - other=0.0, - ) - b = tl.load( - b_ptrs, - mask=offs_k[:, None] < K - (k * UNROLL_FACTOR) * BLOCK_SIZE_K, - other=0.0, - ) - else: - a = tl.load(a_ptrs) - b = tl.load(b_ptrs) - - a_s = tl.load(As_ptrs) - b_s = tl.load(Bs_ptrs) - - accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] - a_ptrs += BLOCK_SIZE_K * stride_ak - b_ptrs += BLOCK_SIZE_K * stride_bk - As_ptrs += scale_step_k * stride_As_k - Bs_ptrs += scale_step_k * stride_Bs_k - - # 2nd iteration - if needs_masking: - a = tl.load( - a_ptrs, - mask=offs_k[None, :] < K - (k * UNROLL_FACTOR + 1) * BLOCK_SIZE_K, - other=0.0, - ) - b = tl.load( - b_ptrs, - mask=offs_k[:, None] < K - (k * UNROLL_FACTOR + 1) * BLOCK_SIZE_K, - other=0.0, - ) - else: - a = tl.load(a_ptrs) - b = tl.load(b_ptrs) - - a_s = tl.load(As_ptrs) - b_s = tl.load(Bs_ptrs) - - accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] - a_ptrs += BLOCK_SIZE_K * stride_ak - b_ptrs += BLOCK_SIZE_K * stride_bk - As_ptrs += scale_step_k * stride_As_k - Bs_ptrs += scale_step_k * stride_Bs_k - - # 3rd iteration - if needs_masking: - a = tl.load( - a_ptrs, - mask=offs_k[None, :] < K - (k * UNROLL_FACTOR + 2) * BLOCK_SIZE_K, - other=0.0, - ) - b = tl.load( - b_ptrs, - mask=offs_k[:, None] < K - (k * UNROLL_FACTOR + 2) * BLOCK_SIZE_K, - other=0.0, - ) - else: - a = tl.load(a_ptrs) - b = tl.load(b_ptrs) - - a_s = tl.load(As_ptrs) - b_s = tl.load(Bs_ptrs) - - accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] - a_ptrs += BLOCK_SIZE_K * stride_ak - b_ptrs += BLOCK_SIZE_K * stride_bk - As_ptrs += scale_step_k * stride_As_k - Bs_ptrs += scale_step_k * stride_Bs_k - - # 4th iteration - if needs_masking: - a = tl.load( - a_ptrs, - mask=offs_k[None, :] < K - (k * UNROLL_FACTOR + 3) * BLOCK_SIZE_K, - other=0.0, - ) - b = tl.load( - b_ptrs, - mask=offs_k[:, None] < K - (k * UNROLL_FACTOR + 3) * BLOCK_SIZE_K, - other=0.0, - ) - else: - a = tl.load(a_ptrs) - b = tl.load(b_ptrs) - - a_s = tl.load(As_ptrs) - b_s = tl.load(Bs_ptrs) - - accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] - a_ptrs += BLOCK_SIZE_K * stride_ak - b_ptrs += BLOCK_SIZE_K * stride_bk - As_ptrs += scale_step_k * stride_As_k - Bs_ptrs += scale_step_k * stride_Bs_k - - if C.dtype.element_ty == tl.bfloat16: - c = accumulator.to(tl.bfloat16) - elif C.dtype.element_ty == tl.float16: - c = accumulator.to(tl.float16) - else: - c = accumulator.to(tl.float32) - - offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) - offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) - c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] - c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) - tl.store(c_ptrs, c, mask=c_mask) - - -@functools.lru_cache -def get_w8a8_block_fp8_configs( - N: int, K: int, block_n: int, block_k: int -) -> Optional[Dict[int, Any]]: - """ - Return optimized configurations for the w8a8 block fp8 kernel. - - The return value will be a dictionary that maps an irregular grid of - batch sizes to configurations of the w8a8 block fp8 kernel. To evaluate the - kernel on a given batch size bs, the closest batch size in the grid should - be picked and the associated configuration chosen to invoke the kernel. - """ - - # Skip config lookup during torch.compile to avoid non-Tensor ops (e.g., device name). - # Returning None forces the caller to use the default config path during compile. - if torch._dynamo.is_compiling(): - return None - - # First look up if an optimized configuration is available in the configs - # directory - device_name = get_device_name().replace(" ", "_") - json_file_name = f"N={N},K={K},device_name={device_name},dtype=fp8_w8a8,block_shape=[{block_n}, {block_k}].json" - - config_file_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name - ) - if os.path.exists(config_file_path): - with open(config_file_path) as f: - log_info_on_rank0( - logger, - f"Using configuration from {config_file_path} for W8A8 Block FP8 kernel.", - ) - raw = {int(key): val for key, val in json.load(f).items()} - - sanitized = {} - clamped_ms = [] - for m_key, cfg in raw.items(): - if cfg["BLOCK_SIZE_K"] < block_k and ( - not _is_cuda or block_k % cfg["BLOCK_SIZE_K"] != 0 - ): - clamped_ms.append((m_key, cfg["BLOCK_SIZE_K"])) - cfg = {**cfg, "BLOCK_SIZE_K": block_k} - sanitized[m_key] = cfg - if clamped_ms: - logger.warning( - "Clamped BLOCK_SIZE_K up to %d in tuned config %s for entries %s " - "(scale stepping requires BLOCK_SIZE_K >= block_k).", - block_k, - json_file_name, - clamped_ms, - ) - - return sanitized - - # If no optimized configuration is available, we will use the default - # configuration - logger.warning( - ( - "Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! " - "Config file not found at %s" - ), - config_file_path, - ) - return None - - -@functools.lru_cache -def get_w8a8_channelwise_fp8_configs(N: int, K: int) -> Optional[Dict[int, Any]]: - """ - Return tuned Triton configurations for the per-token/per-channel w8a8 fp8 - GEMM (`triton_scaled_mm`), for one weight shape on the current device. - - The return value maps an irregular grid of token counts M to - `triton_scaled_mm` tile configs; the closest M in the grid should be picked - for a given batch. A null entry means CUTLASS was faster at that M, so the - grid keeps the point (to stop a neighbouring M snapping onto it) but the - caller must fall back. A missing file / shape likewise means "keep the - default". - """ - # Intentional: the tuned lookup (host-side device name + file I/O) isn't - # traceable, so under torch.compile return None and fall back to CUTLASS -- - # same as mainline's block-FP8 get_w8a8_block_fp8_configs. - if torch._dynamo.is_compiling(): - return None - - device_name = get_device_name().replace(" ", "_") - json_file_name = ( - f"N={N},K={K},device_name={device_name},dtype=fp8_w8a8_channelwise.json" - ) - config_file_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name - ) - if not os.path.exists(config_file_path): - return None - - with open(config_file_path) as f: - log_info_on_rank0( - logger, - f"Using configuration from {config_file_path} for W8A8 channelwise FP8 GEMM.", - ) - return {int(key): val for key, val in json.load(f).items()} - - -def get_w8a8_channelwise_fp8_config(N: int, K: int, M: int) -> Optional[Dict[str, int]]: - """Tuned `triton_scaled_mm` config for this GEMM, or None to keep the default. - - N / K are the weight's output / contraction dims and M the token count. None - means no tuned config applies -- the caller keeps its existing kernel choice - -- either because this device / shape was never tuned, or the nearest tuned - M is a point where CUTLASS won. The caller gates on the feature flag. - """ - configs = get_w8a8_channelwise_fp8_configs(N, K) - if not configs: - return None - return configs[min(configs.keys(), key=lambda x: abs(x - M))] - - -def select_w8a8_block_fp8_matmul_kernel(M, N, META): - return _w8a8_block_fp8_matmul - - -if _is_hip: - - def use_w8a8_block_fp8_matmul_unrolledx4(M, N, META): - # Use manually unrolledx4 kernel on AMD GPU when the grid size is small. - # Empirical testing shows the sweet spot lies when it's less than the # of - # compute units available on the device. - num_workgroups = triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv( - N, META["BLOCK_SIZE_N"] - ) - num_workgroups <= get_device_core_count() - - def select_w8a8_block_fp8_matmul_kernel(M, N, META): - if use_w8a8_block_fp8_matmul_unrolledx4(M, N, META): - return _w8a8_block_fp8_matmul_unrolledx4 - else: - return _w8a8_block_fp8_matmul - - -def prepare_block_fp8_matmul_inputs( - A: torch.Tensor, - B: torch.Tensor, - As: torch.Tensor, - Bs: torch.Tensor, - block_size: List[int], - output_dtype: torch.dtype = torch.float16, -) -> Tuple[int, int, int]: - assert len(block_size) == 2 - block_n, block_k = block_size[0], block_size[1] - - assert A.shape[-1] == B.shape[-1] - assert A.shape[:-1] == As.shape[:-1] - assert A.is_contiguous() - - if As.dtype == torch.float: - assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1] - elif As.dtype == torch.int: - assert triton.cdiv(triton.cdiv(A.shape[-1], block_k), 4) == As.shape[-1], ( - f"{A.shape=} {As.shape=} {block_size=}" - ) - else: - raise NotImplementedError - - M = A.numel() // A.shape[-1] - - assert B.ndim == 2 - assert B.is_contiguous() - assert Bs.ndim == 2 - N, K = B.shape - - if Bs.dtype == torch.float: - assert triton.cdiv(N, block_n) == Bs.shape[0] - assert triton.cdiv(K, block_k) == Bs.shape[1] - elif Bs.dtype == torch.int: - assert N == Bs.shape[0], f"{B.shape=} {Bs.shape=} {block_size=}" - assert triton.cdiv(triton.cdiv(K, block_k), 4) == Bs.shape[1], ( - f"{B.shape=} {Bs.shape=} {block_size=}" - ) - else: - raise NotImplementedError - - C_shape = A.shape[:-1] + (N,) - C = A.new_empty(C_shape, dtype=output_dtype) - - return M, N, K, C - - -def w8a8_block_fp8_matmul_deepgemm( - A: torch.Tensor, - B: torch.Tensor, - As: torch.Tensor, - Bs: torch.Tensor, - block_size: List[int], - output_dtype: torch.dtype, -) -> torch.Tensor: - M, N, K, C = prepare_block_fp8_matmul_inputs(A, B, As, Bs, block_size, output_dtype) - - # Deepgemm only supports output tensor type as bfloat16 - assert C.dtype == torch.bfloat16 and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM - - deep_gemm_fp8_fp8_bf16_nt(A, As, B, Bs, C) - - return C - - -def w8a8_mxfp8_matmul_deepgemm( - A: torch.Tensor, - B: torch.Tensor, - As: torch.Tensor, - Bs: torch.Tensor, - output_dtype: torch.dtype, -) -> torch.Tensor: - assert A.is_contiguous() and B.is_contiguous() - assert output_dtype == torch.bfloat16 and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM - - M = A.numel() // A.shape[-1] - N, K = B.shape - C = A.new_empty(A.shape[:-1] + (N,), dtype=output_dtype) - - deep_gemm_mxfp8_fp8_bf16_nt(A, As, B, Bs, C) - - return C - - -def w8a8_block_fp8_matmul_triton( - A: torch.Tensor, - B: torch.Tensor, - As: torch.Tensor, - Bs: torch.Tensor, - block_size: List[int], - output_dtype: torch.dtype = torch.float16, -) -> torch.Tensor: - """This function performs matrix multiplication with block-wise quantization. - - It takes two input tensors `A` and `B` with scales `As` and `Bs`. - The output is returned in the specified `output_dtype`. - - Args: - A: The input tensor, e.g., activation. - B: The input tensor, e.g., weight. - As: The per-token-group quantization scale for `A`. - Bs: The per-block quantization scale for `B`. - block_size: The block size for per-block quantization. It should be 2-dim, e.g., [128, 128]. - output_dytpe: The dtype of the returned tensor. - - Returns: - torch.Tensor: The result of matmul. - """ - - M, N, K, C = prepare_block_fp8_matmul_inputs(A, B, As, Bs, block_size, output_dtype) - - block_n, block_k = block_size - - configs = get_w8a8_block_fp8_configs(N, K, block_size[0], block_size[1]) - if configs: - # If an optimal configuration map has been found, look up the - # optimal config - config = configs[min(configs.keys(), key=lambda x: abs(x - M))] - else: - # Default config - # Block-wise quant: BLOCK_SIZE_K must be divisible by block_size[1] - config = { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": block_size[0], - "BLOCK_SIZE_K": block_size[1], - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - } - - # Split-K accumulates K in SPLIT_K separate fp32 partials, so its results - # do not match the single-accumulator kernels bit-for-bit. - hopper_tuned = get_platform().is_sm90 and ( - config.get("SWAP_AB", False) or config.get("SPLIT_K", 1) > 1 - ) - if hopper_tuned: - kernel = _w8a8_block_fp8_matmul_hopper - elif _is_gfx1250: - config = {**config, "num_stages": 1} - kernel = _w8a8_block_fp8_matmul_gfx1250 - else: - kernel = select_w8a8_block_fp8_matmul_kernel(M, N, config) - - split_k = config.get("SPLIT_K", 1) if hopper_tuned else 1 - if split_k > 1: - assert split_k & (split_k - 1) == 0 - partials = torch.empty((split_k, M, N), device=A.device, dtype=torch.float32) - else: - partials = C - - needs_masking = bool(K % config["BLOCK_SIZE_K"] != 0) - - def grid(META): - blocks = triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv( - N, META["BLOCK_SIZE_N"] - ) - return (blocks, split_k) if hopper_tuned else (blocks,) - - kernel[grid]( - A, - B, - partials, - As, - Bs, - M, - N, - K, - block_n, - block_k, - A.stride(-2), - A.stride(-1), - B.stride(1), - B.stride(0), - C.stride(-2), - C.stride(-1), - As.stride(-2), - As.stride(-1), - Bs.stride(1), - Bs.stride(0), - **config, - needs_masking=needs_masking, - ) - - if split_k > 1: - _reduce_block_fp8_split_k[(triton.cdiv(M * N, 256),)]( - partials, C, M * N, split_k, 256 - ) - - return C - - # universal entry point, for testing purposes -def w8a8_block_fp8_matmul( - A: torch.Tensor, - B: torch.Tensor, - As: torch.Tensor, - Bs: torch.Tensor, - block_size: List[int], - output_dtype: torch.dtype = torch.float16, -) -> torch.Tensor: - if output_dtype == torch.bfloat16 and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM: - return w8a8_block_fp8_matmul_deepgemm( - A, B, As, Bs, block_size, output_dtype=output_dtype - ) - - return w8a8_block_fp8_matmul_triton( - A, B, As, Bs, block_size, output_dtype=output_dtype - ) @triton.jit @@ -2082,7 +1205,6 @@ else: num_token_padding: Optional[int] = None, use_per_token_if_dynamic: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: - assert input.ndim == 2, f"Expected 2D input tensor, got {input.ndim}D" shape = input.shape if num_token_padding: @@ -2267,243 +1389,6 @@ def per_group_transpose( return trans_a -def is_weak_contiguous(x: torch.Tensor): - strides = x.stride() - sizes = x.shape - is_not_transpose = strides[0] == 1 and (strides[1] >= max(1, sizes[0])) - is_transpose = strides[1] == 1 and (strides[0] >= max(1, sizes[1])) - return is_transpose or is_not_transpose - - -def _as_column_scale(scale: torch.Tensor, expected_len: int) -> torch.Tensor: - if scale.dim() <= 1: - return scale.reshape(-1, 1) - if scale.dim() == 2: - if scale.shape[1] == 1: - return scale - if scale.shape[0] == 1 and scale.shape[1] == expected_len: - return scale.t() - return scale - - -@triton.jit -def scaled_mm_kernel( - a_ptr, - b_ptr, - scale_a_ptr, - scale_b_ptr, - c_ptr, - bias_ptr, - M, - N, - K, - stride_am, - stride_ak, - stride_bk, - stride_bn, - stride_cm, - stride_cn, - ACCUMULATOR_DTYPE: tl.constexpr, - BLOCK_SIZE_M: tl.constexpr, - BLOCK_SIZE_N: tl.constexpr, - BLOCK_SIZE_K: tl.constexpr, - BLOCK_SIZE_SCALE_A: tl.constexpr, - BLOCK_SIZE_SCALE_B: tl.constexpr, -): - pid = tl.program_id(axis=0) - - num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) - - pid_m = pid // num_pid_n - pid_n = pid % num_pid_n - - accumulator_dtype = ACCUMULATOR_DTYPE - accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=accumulator_dtype) - - # NOTE: Some tensor inputs are so large, they will cause int32 overflow - # so it is necessary to use tl.int64 for all the offsets, else SEGV will - # eventually occur. - - # Offsets and masks. - offsets_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64) - masks_am = offsets_am < M - - offsets_bn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64) - masks_bn = offsets_bn < N - - offsets_k = tl.arange(0, BLOCK_SIZE_K).to(tl.int64) - offsets_a = stride_am * offsets_am[:, None] + stride_ak * offsets_k[None, :] - offsets_b = stride_bk * offsets_k[:, None] + stride_bn * offsets_bn[None, :] - - # NOTE: BLOCK_SIZE_SCALE_A could be 1 or BLOCK_SIZE_M, so need to create - # appropriate offsets and masks for each case. Same goes for - # BLOCK_SIZE_SCALE_B. - offsets_scale_am = ( - tl.arange(0, BLOCK_SIZE_SCALE_A) - + (BLOCK_SIZE_SCALE_A > 1) * pid_m * BLOCK_SIZE_M - ) - masks_scale_am = offsets_scale_am < M - - offsets_scale_bn = ( - tl.arange(0, BLOCK_SIZE_SCALE_B) - + (BLOCK_SIZE_SCALE_B > 1) * pid_n * BLOCK_SIZE_N - ) - masks_scale_bn = offsets_scale_bn < N - - a_ptrs = a_ptr + offsets_a - b_ptrs = b_ptr + offsets_b - - scale_a_ptrs = scale_a_ptr + offsets_scale_am - scale_b_ptrs = scale_b_ptr + offsets_scale_bn - - for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): - masks_k = offsets_k < K - masks_a = masks_am[:, None] & masks_k[None, :] - a = tl.load(a_ptrs, mask=masks_a) - - masks_b = masks_k[:, None] & masks_bn[None, :] - b = tl.load(b_ptrs, mask=masks_b) - - # Accumulate results. - accumulator = tl.dot(a, b, accumulator, out_dtype=accumulator_dtype) - - offsets_k += BLOCK_SIZE_K - a_ptrs += BLOCK_SIZE_K * stride_ak - b_ptrs += BLOCK_SIZE_K * stride_bk - - # Apply scale at end. - masks_scale_a = masks_scale_am[:, None] & (tl.arange(0, 1) < 1)[:, None] - scale_a = tl.load(scale_a_ptrs[:, None], masks_scale_a) - # Need to broadcast to the appropriate size, if scale_a is already - # (BLOCK_SIZE_M, 1) then it will broadcast to its own shape. Same goes - # for scale_b below. - scale_a = scale_a.broadcast_to((BLOCK_SIZE_M, 1)) - accumulator = scale_a * accumulator.to(tl.float32) - - masks_scale_b = masks_scale_bn[:, None] & (tl.arange(0, 1) < 1)[None, :] - scale_b = tl.load(scale_b_ptrs[:, None], masks_scale_b) - scale_b = scale_b.broadcast_to((BLOCK_SIZE_N, 1)) - accumulator = scale_b.T * accumulator.to(tl.float32) - - # Convert to output format. - c = accumulator.to(c_ptr.type.element_ty) - - # Add bias, it's already in output format, so add it after conversion. - if bias_ptr: - offsets_bias = offsets_bn - bias_ptrs = bias_ptr + offsets_bias - bias_mask = offsets_bias < N - bias = tl.load(bias_ptrs, bias_mask) - c += bias - - # Save output - offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64) - offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64) - offs_cm = offs_cm.to(tl.int64) - offs_cn = offs_cn.to(tl.int64) - c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] - c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) - - tl.store(c_ptrs, c, mask=c_mask) - - # input - [M, K] # weight - [K, N] # Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/compressed_tensors/triton_scaled_mm.py -def triton_scaled_mm( - input: torch.Tensor, - weight: torch.Tensor, - scale_a: torch.Tensor, - scale_b: torch.Tensor, - out_dtype: type[torch.dtype], - bias: Optional[torch.Tensor] = None, - block_size_m: int = 32, - block_size_n: int = 32, - block_size_k: int = 32, - use_heuristic=True, - num_warps: Optional[int] = None, - num_stages: Optional[int] = None, -) -> torch.Tensor: - M, K = input.shape - N = weight.shape[1] - - assert N > 0 and K > 0 and M > 0 - assert weight.shape[0] == K - assert input.dtype == weight.dtype - - scale_a = _as_column_scale(scale_a, M) - scale_b = _as_column_scale(scale_b, N) - - assert scale_a.dim() == 2 and scale_b.dim() == 2 - assert scale_a.dtype == scale_b.dtype and scale_a.is_floating_point() - assert scale_a.shape[1] == 1 and (scale_a.shape[0] == 1 or scale_a.shape[0] == M) - assert scale_b.shape[1] == 1 and (scale_b.shape[0] == 1 or scale_b.shape[0] == N) - assert out_dtype.is_floating_point - assert bias is None or bias.is_floating_point() - assert is_weak_contiguous(input) - assert is_weak_contiguous(weight) - - grid = lambda META: ( - triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), - ) - - result = torch.empty((M, N), dtype=out_dtype, device=input.device) - - has_scalar = lambda x: x.shape[0] == 1 and x.shape[1] == 1 - - if use_heuristic: - is_small_N = N < 8192 - next_power_of_2_M = max(32, triton.next_power_of_2(M)) - if next_power_of_2_M <= 32: - tile_shape = (64, 64, 256) if is_small_N else (64, 128, 256) - elif next_power_of_2_M <= 64: - tile_shape = (64, 64, 256) - elif next_power_of_2_M <= 128: - tile_shape = (64, 128, 128) - else: - tile_shape = (128, 128, 128) - block_size_m, block_size_n, block_size_k = tile_shape - # else: use the block_size_{m,n,k} the caller passed (e.g. a tuned config). - - block_size_sa = 1 if has_scalar(scale_a) else block_size_m - block_size_sb = 1 if has_scalar(scale_b) else block_size_n - - accumulator_dtype = tl.float32 if input.is_floating_point() else tl.int32 - - # num_warps / num_stages are triton.jit launch kwargs, and triton rejects an - # explicit None -- pass them only when the caller set them, so the default - # path keeps triton's own defaults. - launch_kwargs = {} - if num_warps is not None: - launch_kwargs["num_warps"] = num_warps - if num_stages is not None: - launch_kwargs["num_stages"] = num_stages - - # A = input, B = weight, C = result - # A = M x K, B = K x N, C = M x N - scaled_mm_kernel[grid]( - input, - weight, - scale_a, - scale_b, - result, - bias, - M, - N, - K, - input.stride(0), - input.stride(1), - weight.stride(0), - weight.stride(1), - result.stride(0), - result.stride(1), - accumulator_dtype, - BLOCK_SIZE_M=block_size_m, - BLOCK_SIZE_N=block_size_n, - BLOCK_SIZE_K=block_size_k, - BLOCK_SIZE_SCALE_A=block_size_sa, - BLOCK_SIZE_SCALE_B=block_size_sb, - **launch_kwargs, - ) - - return result.to(out_dtype) diff --git a/python/sglang/kernels/ops/quantization/int8_kernel.py b/python/sglang/kernels/ops/quantization/int8_kernel.py index 3fafeb39f2..c1daae04a4 100644 --- a/python/sglang/kernels/ops/quantization/int8_kernel.py +++ b/python/sglang/kernels/ops/quantization/int8_kernel.py @@ -1,15 +1,12 @@ -import functools -import json import logging -import os -from typing import Any, Dict, List, Optional, Tuple +from typing import Tuple import torch import triton import triton.language as tl from triton.language.extra import libdevice -from sglang.srt.utils import get_device_name, is_cuda, is_hip +from sglang.srt.utils import is_cuda, is_hip _is_cuda = is_cuda() _is_hip = is_hip() @@ -210,219 +207,3 @@ def sglang_per_token_group_quant_int8( ) return per_token_group_quant(x, group_size=group_size, out_dtype=dtype) - - -@triton.jit -def _w8a8_block_int8_matmul( - # Pointers to inputs and output - A, - B, - C, - As, - Bs, - # Shape for matmul - M, - N, - K, - # Block size for block-wise quantization - group_n, - group_k, - # Stride for inputs and output - stride_am, - stride_ak, - stride_bk, - stride_bn, - stride_cm, - stride_cn, - stride_As_m, - stride_As_k, - stride_Bs_k, - stride_Bs_n, - # Meta-parameters - BLOCK_SIZE_M: tl.constexpr, - BLOCK_SIZE_N: tl.constexpr, - BLOCK_SIZE_K: tl.constexpr, - GROUP_SIZE_M: tl.constexpr, -): - """Triton-accelerated function used to perform linear operations (dot - product) on input tensors `A` and `B` with block-wise quantization, and store the result in output - tensor `C`. - """ - - pid = tl.program_id(axis=0) - num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) - num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) - num_pid_in_group = GROUP_SIZE_M * num_pid_n - group_id = pid // num_pid_in_group - first_pid_m = group_id * GROUP_SIZE_M - group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) - pid_m = first_pid_m + (pid % group_size_m) - pid_n = (pid % num_pid_in_group) // group_size_m - - offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M - offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N - offs_k = tl.arange(0, BLOCK_SIZE_K) - a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) - b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) - - As_ptrs = As + offs_am * stride_As_m - offs_bsn = offs_bn // group_n - Bs_ptrs = Bs + offs_bsn * stride_Bs_n - - accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) - for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): - a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k * BLOCK_SIZE_K, other=0.0) - b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0) - - k_start = k * BLOCK_SIZE_K - offs_ks = k_start // group_k - a_s = tl.load(As_ptrs + offs_ks * stride_As_k) - b_s = tl.load(Bs_ptrs + offs_ks * stride_Bs_k) - - accumulator += tl.dot(a, b).to(tl.float32) * a_s[:, None] * b_s[None, :] - a_ptrs += BLOCK_SIZE_K * stride_ak - b_ptrs += BLOCK_SIZE_K * stride_bk - - if C.dtype.element_ty == tl.bfloat16: - c = accumulator.to(tl.bfloat16) - elif C.dtype.element_ty == tl.float16: - c = accumulator.to(tl.float16) - else: - c = accumulator.to(tl.float32) - - offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) - offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) - c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] - c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) - tl.store(c_ptrs, c, mask=c_mask) - - -@functools.lru_cache -def get_w8a8_block_int8_configs( - N: int, K: int, block_n: int, block_k: int -) -> Optional[Dict[int, Any]]: - """ - Return optimized configurations for the w8a8 block fp8 kernel. - - The return value will be a dictionary that maps an irregular grid of - batch sizes to configurations of the w8a8 block fp8 kernel. To evaluate the - kernel on a given batch size bs, the closest batch size in the grid should - be picked and the associated configuration chosen to invoke the kernel. - """ - - # First look up if an optimized configuration is available in the configs - # directory - device_name = get_device_name().replace(" ", "_") - json_file_name = f"N={N},K={K},device_name={device_name},dtype=int8_w8a8,block_shape=[{block_n}, {block_k}].json" - - config_file_path = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name - ) - if os.path.exists(config_file_path): - with open(config_file_path) as f: - logger.info( - "Using configuration from %s for W8A8 Block INT8 kernel.", - config_file_path, - ) - # If a configuration has been found, return it - return {int(key): val for key, val in json.load(f).items()} - - # If no optimized configuration is available, we will use the default - # configuration - logger.warning( - ( - "Using default W8A8 Block INT8 kernel config. Performance might be sub-optimal! " - "Config file not found at %s" - ), - config_file_path, - ) - return None - - -def w8a8_block_int8_matmul( - A: torch.Tensor, - B: torch.Tensor, - As: torch.Tensor, - Bs: torch.Tensor, - block_size: List[int], - output_dtype: torch.dtype = torch.float16, -) -> torch.Tensor: - """This function performs matrix multiplication with block-wise quantization. - - It takes two input tensors `A` and `B` with scales `As` and `Bs`. - The output is returned in the specified `output_dtype`. - - Args: - A: The input tensor, e.g., activation. - B: The input tensor, e.g., weight. - As: The per-token-group quantization scale for `A`. - Bs: The per-block quantization scale for `B`. - block_size: The block size for per-block quantization. It should be 2-dim, e.g., [128, 128]. - output_dytpe: The dtype of the returned tensor. - - Returns: - torch.Tensor: The result of matmul. - """ - assert len(block_size) == 2 - block_n, block_k = block_size[0], block_size[1] - - assert A.shape[-1] == B.shape[-1] - assert A.shape[:-1] == As.shape[:-1] and A.is_contiguous() - assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1] - M = A.numel() // A.shape[-1] - - assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 - N, K = B.shape - assert triton.cdiv(N, block_n) == Bs.shape[0] - assert triton.cdiv(K, block_k) == Bs.shape[1] - - C_shape = A.shape[:-1] + (N,) - C = A.new_empty(C_shape, dtype=output_dtype) - - configs = get_w8a8_block_int8_configs(N, K, block_size[0], block_size[1]) - if configs: - # If an optimal configuration map has been found, look up the - # optimal config - config = configs[min(configs.keys(), key=lambda x: abs(x - M))] - else: - # Default config - # Block-wise quant: BLOCK_SIZE_K must be divisible by block_size[1] - config = { - "BLOCK_SIZE_M": 64, - "BLOCK_SIZE_N": block_size[0], - "BLOCK_SIZE_K": block_size[1], - "GROUP_SIZE_M": 32, - "num_warps": 4, - "num_stages": 3, - } - - def grid(META): - return ( - triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), - ) - - _w8a8_block_int8_matmul[grid]( - A, - B, - C, - As, - Bs, - M, - N, - K, - block_n, - block_k, - A.stride(-2), - A.stride(-1), - B.stride(1), - B.stride(0), - C.stride(-2), - C.stride(-1), - As.stride(-2), - As.stride(-1), - Bs.stride(1), - Bs.stride(0), - **config, - ) - - return C diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index d6c7e0da20..11ef39e53c 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1024,7 +1024,7 @@ class Envs: # token count, run the Triton w8a8 FP8 GEMM with it; otherwise keep the # default CUTLASS path. Only takes effect on a GPU with a matching # dtype=fp8_w8a8_channelwise config JSON under - # kernels/ops/quantization/configs/ (currently L40S), so it is a no-op on + # kernels/ops/gemm/configs/ (currently L40S), so it is a no-op on # any other GPU / untuned shape even when enabled. SGLANG_ENABLE_FP8_GEMM_CONFIG_TUNE = EnvBool(True) @@ -1754,22 +1754,18 @@ class Envs: # output and the latent|shared MoE reduce; everything else falls back to # the regular all-reduce path. Auto-enabled on SM100/SM103 when # CustomAllReduceV2 with multicast is available; set 0/1 to override in - # either direction. See srt/layers/k3_ar_fusion.py. + # either direction. See srt/layers/communication/k3_ar_fusion.py. SGLANG_K3_AR_FUSION = EnvBool(False) # K3 SP-MoE fused residual + reduce-scatter and matching all-gather over # CustomAllReduceV2's MNNVL push workspace. Auto-probed for the validated # TP8 GB300 configuration; set 0/1 to override. See - # srt/layers/k3_sp_collective.py. + # srt/layers/communication/k3_sp_collective.py. SGLANG_K3_SP_COLLECTIVE = EnvBool(False) # Keep K3's post-MoE residual stream token-sharded between consecutive # SP-MoE layers. The next attention-residual aggregation and snapshot # bank write run on the local shard, then only the normalized attention # input is all-gathered. Requires SGLANG_K3_SP_COLLECTIVE. SGLANG_K3_SP_ATTN_RES = EnvBool(False) - # Fused o_proj GEMM + all-reduce (bf16, TP 2..8, SM100+): one - # kernel computes the TP-local o_proj partial and the cross-rank sum over - # a P2P comm region, replacing the GEMM + NCCL AR pair at M <= 512. - SGLANG_K3_GEMM_AR = EnvBool(False) # Merge the router gate and routed_expert_down_proj weights so the K3 MoE # front reads hidden_states once, and run the top-k plus the bf16 cast in one # epilogue kernel. See kernels/ops/moe/moe_front.py. Default on. diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py index c14c42aa1b..361ab1b5d1 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py @@ -155,11 +155,11 @@ if _is_cuda or _is_hip: ) if _is_cuda: - from sglang.kernels.ops.attention.dsv4 import fused_q_indexer_rope_first_quant - from sglang.kernels.ops.quantization.dsv32 import ( + from sglang.kernels.ops.attention.dsa.indexer_k import ( fused_k_indexer_norm_rope, fused_k_indexer_norm_rope_store, ) + from sglang.kernels.ops.attention.dsv4 import fused_q_indexer_rope_first_quant @register_custom_op(mutates_args=["topk_indices"]) @register_split_op() diff --git a/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py b/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py index 7c8cc44e6f..c3ef36e90b 100644 --- a/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py +++ b/python/sglang/srt/layers/attention/dsa/kpool_fp8_index.py @@ -621,7 +621,7 @@ def topk_from_pooled_history_logits( ) if group_topk in (128, 160, 192, 224, 256, 512): - from sglang.kernels.ops.moe.kpool_topk_transform import ( + from sglang.kernels.ops.attention.dsa.kpool_topk_transform import ( fast_kpool_topk_transform_fused, ) diff --git a/python/sglang/srt/layers/attention/dsv4/compress_hip.py b/python/sglang/srt/layers/attention/dsv4/compress_hip.py index 5c0addf332..6c43d91e2f 100644 --- a/python/sglang/srt/layers/attention/dsv4/compress_hip.py +++ b/python/sglang/srt/layers/attention/dsv4/compress_hip.py @@ -31,7 +31,7 @@ if TYPE_CHECKING: ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch -from sglang.kernels.ops.attention.dsv4.rms_normalize_hip import rms_normalize_triton +from sglang.kernels.ops.layernorm.rms_normalize_hip import rms_normalize_triton class DeepseekRefRMSNorm(nn.Module): diff --git a/python/sglang/srt/layers/attention/dsv4/compressor.py b/python/sglang/srt/layers/attention/dsv4/compressor.py index 26772b2dba..81cfd1af2c 100644 --- a/python/sglang/srt/layers/attention/dsv4/compressor.py +++ b/python/sglang/srt/layers/attention/dsv4/compressor.py @@ -6,16 +6,14 @@ import torch import torch.nn as nn from sglang.kernels.fused_op import BaseFusedOp -from sglang.kernels.ops.attention.dsv4 import ( - linear_bf16_fp32, - triton_create_paged_compress_data, -) +from sglang.kernels.ops.attention.dsv4 import triton_create_paged_compress_data from sglang.kernels.ops.attention.dsv4.compress_old import ( CompressorDecodePlan, CompressorPrefillPlan, compress_forward, compress_fused_norm_rope_inplace, ) +from sglang.kernels.ops.gemm.bf16_fp32 import linear_bf16_fp32 from sglang.srt.configs.deepseek_v4 import DeepSeekV4Config from sglang.srt.environ import envs from sglang.srt.layers.attention.dsa.utils import dsa_use_prefill_cp @@ -442,7 +440,6 @@ class Compressor(BaseFusedOp): return ret def compute_kv_score(self, x: torch.Tensor, forward_batch: ForwardBatch): - kv_score = self._compute_wkv_gate(x) # CUDA path: delegate to backend diff --git a/python/sglang/srt/layers/attention/dsv4/dsv41_sparse.py b/python/sglang/srt/layers/attention/dsv4/dsv41_sparse.py index 797294c4d5..2ffce44b4a 100644 --- a/python/sglang/srt/layers/attention/dsv4/dsv41_sparse.py +++ b/python/sglang/srt/layers/attention/dsv4/dsv41_sparse.py @@ -11,11 +11,11 @@ from typing import Optional, Tuple import torch from torch import nn -from sglang.kernels.ops.attention.dsv4 import linear_bf16_fp32 from sglang.kernels.ops.attention.dsv4.torch_quant import ( fake_quant_compressed_kv, fake_quant_fp4, ) +from sglang.kernels.ops.gemm.bf16_fp32 import linear_bf16_fp32 from sglang.kernels.ops.layernorm.rmsnorm_fp32 import rmsnorm_fp32 from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.quantization.base_config import QuantizationConfig diff --git a/python/sglang/srt/layers/attention/minicpm/backend.py b/python/sglang/srt/layers/attention/minicpm/backend.py index d32a82bde9..0dad0b644c 100644 --- a/python/sglang/srt/layers/attention/minicpm/backend.py +++ b/python/sglang/srt/layers/attention/minicpm/backend.py @@ -30,7 +30,7 @@ if TYPE_CHECKING: from sglang.srt.model_executor.model_runner import ModelRunner -from sglang.kernels.ops.minicpm_sala import get_block_table +from sglang.kernels.ops.attention.minicpm_sala import get_block_table from sglang.srt.layers.attention.minicpm.sparse_utils import ( CompressionLevelMetadata, MiniCPMSparseMetadata, diff --git a/python/sglang/srt/layers/attention/qsa/kernel.py b/python/sglang/srt/layers/attention/qsa/kernel.py index 56070c4964..8a24ba4395 100644 --- a/python/sglang/srt/layers/attention/qsa/kernel.py +++ b/python/sglang/srt/layers/attention/qsa/kernel.py @@ -34,7 +34,7 @@ def qsa_fast_topk( if topk == 512: # Prefer the JIT kernel: it ships with the sglang python package, # so top-k 512 works regardless of the installed sgl_kernel version. - from sglang.kernels.ops.elementwise.fast_topk import fast_topk + from sglang.kernels.ops.attention.fast_topk import fast_topk return fast_topk(logits, lengths, topk=512, row_starts=starts) diff --git a/python/sglang/srt/layers/attention/qsa/qsa_indexer.py b/python/sglang/srt/layers/attention/qsa/qsa_indexer.py index 618981b245..a308419db6 100644 --- a/python/sglang/srt/layers/attention/qsa/qsa_indexer.py +++ b/python/sglang/srt/layers/attention/qsa/qsa_indexer.py @@ -498,7 +498,7 @@ class QSAIndexer(MultiPlatformOp): if logits.is_cuda and self.block_topk == 512: # Decode rows start at zero, so compressed lengths double as row lengths; # skip the generic zero-fill + subtract. - from sglang.kernels.ops.elementwise.fast_topk import fast_topk + from sglang.kernels.ops.attention.fast_topk import fast_topk block_indices = fast_topk( logits, diff --git a/python/sglang/srt/layers/attn_residual.py b/python/sglang/srt/layers/attn_residual.py index cd49676085..50add4e7d6 100644 --- a/python/sglang/srt/layers/attn_residual.py +++ b/python/sglang/srt/layers/attn_residual.py @@ -558,7 +558,7 @@ class AttnResidual: bank = self.block_residual[rows] cw = get_cw(score_proj, score_norm, dtype=torch.bfloat16) assert score_norm.variance_epsilon == out_norm.variance_epsilon - from sglang.srt.layers import k3_sp_collective + from sglang.srt.layers.communication import k3_sp_collective normed = k3_sp_collective.attn_res_all_gather( prefix, @@ -595,7 +595,7 @@ class AttnResidual: bank = self.block_residual[rows] cw = get_cw(score_proj, score_norm, dtype=torch.bfloat16) assert score_norm.variance_epsilon == out_norm.variance_epsilon - from sglang.srt.layers import k3_sp_collective + from sglang.srt.layers.communication import k3_sp_collective return k3_sp_collective.reduce_scatter_attn_res( hidden_states, diff --git a/python/sglang/srt/layers/communication/__init__.py b/python/sglang/srt/layers/communication/__init__.py new file mode 100644 index 0000000000..f28267317f --- /dev/null +++ b/python/sglang/srt/layers/communication/__init__.py @@ -0,0 +1 @@ +"""Runtime integration and state for fused communication operators.""" diff --git a/python/sglang/srt/layers/k3_ar_fusion.py b/python/sglang/srt/layers/communication/k3_ar_fusion.py similarity index 100% rename from python/sglang/srt/layers/k3_ar_fusion.py rename to python/sglang/srt/layers/communication/k3_ar_fusion.py diff --git a/python/sglang/srt/layers/k3_sp_collective.py b/python/sglang/srt/layers/communication/k3_sp_collective.py similarity index 99% rename from python/sglang/srt/layers/k3_sp_collective.py rename to python/sglang/srt/layers/communication/k3_sp_collective.py index e542a56611..a21deaf2ef 100644 --- a/python/sglang/srt/layers/k3_sp_collective.py +++ b/python/sglang/srt/layers/communication/k3_sp_collective.py @@ -16,7 +16,7 @@ from typing import TYPE_CHECKING, Optional import torch from sglang.srt.environ import envs -from sglang.srt.layers import k3_ar_fusion +from sglang.srt.layers.communication import k3_ar_fusion from sglang.srt.runtime_context import get_exec, get_parallel if TYPE_CHECKING: diff --git a/python/sglang/srt/layers/hyperconnection.py b/python/sglang/srt/layers/hyperconnection.py index c9439db590..cac970b0e1 100644 --- a/python/sglang/srt/layers/hyperconnection.py +++ b/python/sglang/srt/layers/hyperconnection.py @@ -5,7 +5,7 @@ import torch import torch.nn as nn import torch.nn.functional as F -from sglang.srt.layers.hc_mix_triton import fused_hc_mix, fused_hc_mix_supported +from sglang.kernels.ops.gemm.hc_mix import fused_hc_mix, fused_hc_mix_supported class HyperConnectionConfig(msgspec.Struct, frozen=True): diff --git a/python/sglang/srt/layers/k3_gemm_ar.py b/python/sglang/srt/layers/k3_gemm_ar.py deleted file mode 100644 index 0fd074432b..0000000000 --- a/python/sglang/srt/layers/k3_gemm_ar.py +++ /dev/null @@ -1,93 +0,0 @@ -"""K3 fused o_proj GEMM+AR dispatch (``SGLANG_K3_GEMM_AR``). - -Glue between the model and ``kernels.ops.communication.gemm_ar``: lazily allocates -the P2P comm region on the TP group and swaps the o_proj RowParallelLinear -forward for the single fused GEMM+all-reduce kernel at decode shapes -(falling through to the regular GEMM + AR path whenever the input doesn't -fit — prefill, capture, non-2D input). Eager-mode only. -""" - -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING - -import torch - -from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_parallel - -if TYPE_CHECKING: - from sglang.srt.layers.linear import RowParallelLinear - -logger = logging.getLogger(__name__) - -_INITIALIZED = False -_ENABLED = False - - -def _init() -> bool: - global _INITIALIZED, _ENABLED - if _INITIALIZED: - return _ENABLED - _INITIALIZED = True - if not envs.SGLANG_K3_GEMM_AR.get(): - return False - - world_size = get_parallel().tp_size - if not (2 <= world_size <= 8): - logger.warning("SGLANG_K3_GEMM_AR requires 2 <= TP <= 8; disabled.") - return False - if torch.cuda.get_device_capability() < (10, 0): - logger.warning("SGLANG_K3_GEMM_AR requires SM100+; disabled.") - return False - _ENABLED = True - logger.info("K3 fused o_proj GEMM+AR enabled (world_size=%d)", world_size) - return True - - -def maybe_wrap_o_proj(o_proj: RowParallelLinear) -> None: - """SGLANG_K3_GEMM_AR: route decode-shaped o_proj calls through the fused - GEMM+all-reduce kernel; everything else falls through to the original - forward. The comm region + JIT compile happen HERE (model build, weight - shape known, well before CUDA-graph capture) — capture must only see the - ready-to-launch path.""" - if not _init(): - return - from sglang.kernels.ops.communication import gemm_ar as mod - from sglang.srt.runtime_context import get_parallel - - parallel = get_parallel() - world_size = parallel.tp_size - if not o_proj.reduce_results or o_proj.tp_size != world_size: - return - weight = o_proj.weight - if not ( - isinstance(weight, torch.Tensor) - and weight.shape[0] == mod.N - and weight.shape[1] % 128 == 0 - ): - return - - mod.init( - world_size=world_size, - rank=parallel.tp_rank, - group=get_parallel().tp_group.cpu_group, - k=weight.shape[1], - ) - # per-K compile + base-address stash, pre-capture - mod._module_with_bases(weight.shape[1], world_size) - - inner = o_proj.forward - - def _gemm_ar_forward(x, *args, **kwargs): - weight = o_proj.weight - if ( - isinstance(weight, torch.Tensor) - and weight.dtype == torch.bfloat16 - and mod.fits(x) - ): - return mod.o_proj_gemm_ar(x, weight), None - return inner(x, *args, **kwargs) - - o_proj.forward = _gemm_ar_forward diff --git a/python/sglang/srt/layers/moe/cutlass_moe.py b/python/sglang/srt/layers/moe/cutlass_moe.py index 6510beda9f..918c51c99b 100755 --- a/python/sglang/srt/layers/moe/cutlass_moe.py +++ b/python/sglang/srt/layers/moe/cutlass_moe.py @@ -275,7 +275,7 @@ def cutlass_fused_experts_fp8( if swiglu_limit is None: silu_and_mul(c1, intermediate) else: - from sglang.kernels.ops.attention.dsv4 import silu_and_mul_clamp + from sglang.kernels.ops.moe.dsv4 import silu_and_mul_clamp silu_and_mul_clamp(c1, intermediate, swiglu_limit) diff --git a/python/sglang/srt/layers/moe/hash_topk.py b/python/sglang/srt/layers/moe/hash_topk.py index a9a4389097..d32c600ff2 100644 --- a/python/sglang/srt/layers/moe/hash_topk.py +++ b/python/sglang/srt/layers/moe/hash_topk.py @@ -232,7 +232,7 @@ class HashTopK(nn.Module): if _is_xpu: topk_weights, topk_ids = self._forward_xpu(router_logits, input_ids) elif envs.SGLANG_OPT_USE_FUSED_HASH_TOPK.get(): - from sglang.kernels.ops.attention.dsv4 import hash_topk + from sglang.kernels.ops.moe.dsv4 import hash_topk topk_weights, topk_ids = hash_topk( router_logits=router_logits, diff --git a/python/sglang/srt/layers/moe/mega_moe.py b/python/sglang/srt/layers/moe/mega_moe.py index 2478611b14..cddcc1f473 100644 --- a/python/sglang/srt/layers/moe/mega_moe.py +++ b/python/sglang/srt/layers/moe/mega_moe.py @@ -21,7 +21,7 @@ from typing import TYPE_CHECKING, Optional import torch -from sglang.kernels.ops.attention.dsv4 import mega_moe_pre_dispatch +from sglang.kernels.ops.moe.dsv4 import mega_moe_pre_dispatch from sglang.srt.environ import envs from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp diff --git a/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py b/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py index 0a3a71fdb9..bbff20fac6 100644 --- a/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py +++ b/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py @@ -8,7 +8,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.attention.dsv4 import ( +from sglang.kernels.ops.moe.dsv4 import ( silu_and_mul_clamp, silu_and_mul_masked_post_quant, ) @@ -398,7 +398,7 @@ class DeepGemmRunnerCore(MoeRunnerCore): quant_info: DeepGemmMoeQuantInfo, running_state: dict, ) -> torch.Tensor: - from sglang.kernels.ops.attention.dsv4 import silu_and_mul_contig_post_quant + from sglang.kernels.ops.moe.dsv4 import silu_and_mul_contig_post_quant from sglang.kernels.ops.moe.ep_moe_kernels import tma_align_input_scale from sglang.kernels.ops.quantization.fp8_kernel import ( create_per_token_group_quant_fp8_output_scale, @@ -631,7 +631,6 @@ class DeepGemmRunnerCore(MoeRunnerCore): quant_info: DeepGemmMoeQuantInfo, running_state: dict, ) -> torch.Tensor: - hidden_states = runner_input.hidden_states all_tokens = running_state["all_tokens"] hidden_states_device = running_state["hidden_states_device"] diff --git a/python/sglang/srt/layers/moe/moe_runner/humming.py b/python/sglang/srt/layers/moe/moe_runner/humming.py index 7063565b29..93b6941fab 100644 --- a/python/sglang/srt/layers/moe/moe_runner/humming.py +++ b/python/sglang/srt/layers/moe/moe_runner/humming.py @@ -426,9 +426,7 @@ class HummingRunnerCore(MoeRunnerCore): and num_experts <= min(256, num_threads) ) if use_fused_masked_act_quant: - from sglang.kernels.ops.attention.dsv4.moe import ( - silu_and_mul_masked_post_quant, - ) + from sglang.kernels.ops.moe.dsv4 import silu_and_mul_masked_post_quant down_input = torch.empty( (num_experts, max_tokens, intermediate), diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py index cdc2a84dd7..d4baddfe68 100644 --- a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py @@ -728,7 +728,7 @@ def _fused_moe_kernel_sequence( if not filter_expert: if swiglu_limit_for_silu_and_mul_clamp is not None: - from sglang.kernels.ops.attention.dsv4 import silu_and_mul_clamp + from sglang.kernels.ops.moe.dsv4 import silu_and_mul_clamp silu_and_mul_clamp( intermediate_cache1.view(-1, N), diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index 67b69edd16..b6b106fc70 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -86,7 +86,7 @@ except ImportError: pass from sglang.kernels.fused_op import BaseFusedOp -from sglang.kernels.ops.attention.dsv4 import mask_topk_ids +from sglang.kernels.ops.moe.dsv4 import mask_topk_ids from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) diff --git a/python/sglang/srt/layers/quantization/expert_pack.py b/python/sglang/srt/layers/quantization/expert_pack.py index 50951e9144..8e22c43362 100644 --- a/python/sglang/srt/layers/quantization/expert_pack.py +++ b/python/sglang/srt/layers/quantization/expert_pack.py @@ -33,7 +33,7 @@ def _clamped_swiglu( ) -> torch.Tensor: if limit is not None: if gate.is_cuda: - from sglang.kernels.ops.attention.dsv4 import silu_and_mul_clamp + from sglang.kernels.ops.moe.dsv4 import silu_and_mul_clamp gate_up = torch.cat((gate, up), dim=-1) output = torch.empty_like(gate) diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 10daac0bdc..fd86e97283 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -7,11 +7,16 @@ from typing import Callable, List, Optional, Tuple, Union import torch +from sglang.kernels.ops.gemm.fp8_kernel import ( + get_w8a8_channelwise_fp8_config, + triton_scaled_mm, + w8a8_block_fp8_matmul_deepgemm, + w8a8_block_fp8_matmul_triton, +) from sglang.kernels.ops.quantization.fp8_kernel import ( fp8_dtype, fp8_max, fp8_min, - get_w8a8_channelwise_fp8_config, is_fp8_fnuz, per_token_group_quant_fp8, scaled_fp8_quant, @@ -19,9 +24,6 @@ from sglang.kernels.ops.quantization.fp8_kernel import ( sglang_per_token_group_quant_fp8_row_padded, sglang_per_token_quant_fp8, static_quant_fp8, - triton_scaled_mm, - w8a8_block_fp8_matmul_deepgemm, - w8a8_block_fp8_matmul_triton, ) from sglang.srt.environ import envs from sglang.srt.hardware_backend.npu.utils import is_npu_arch35 @@ -815,9 +817,9 @@ def _deepgemm_w8a8_mxfp8_linear_with_fallback( bias: Optional[torch.Tensor] = None, weight_scale_swizzled: Optional[torch.Tensor] = None, ) -> torch.Tensor: + from sglang.kernels.ops.gemm.fp8_kernel import w8a8_mxfp8_matmul_deepgemm from sglang.kernels.ops.quantization.fp8_kernel import ( sglang_per_token_group_quant_fp8, - w8a8_mxfp8_matmul_deepgemm, ) assert input_scale is None diff --git a/python/sglang/srt/layers/quantization/int8_utils.py b/python/sglang/srt/layers/quantization/int8_utils.py index 27506f623e..3d91a44432 100644 --- a/python/sglang/srt/layers/quantization/int8_utils.py +++ b/python/sglang/srt/layers/quantization/int8_utils.py @@ -2,10 +2,8 @@ from typing import List, Optional import torch -from sglang.kernels.ops.quantization.int8_kernel import ( - per_token_group_quant_int8, - w8a8_block_int8_matmul, -) +from sglang.kernels.ops.gemm.int8_kernel import w8a8_block_int8_matmul +from sglang.kernels.ops.quantization.int8_kernel import per_token_group_quant_int8 def apply_w8a8_block_int8_linear( diff --git a/python/sglang/srt/layers/quantization/marlin_utils.py b/python/sglang/srt/layers/quantization/marlin_utils.py index 5e5c7c8967..e972584d54 100644 --- a/python/sglang/srt/layers/quantization/marlin_utils.py +++ b/python/sglang/srt/layers/quantization/marlin_utils.py @@ -30,7 +30,7 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo _is_cuda = is_cuda() if _is_cuda: - from sglang.kernels.ops.quantization.gptq_marlin import gptq_marlin_gemm + from sglang.kernels.ops.gemm.gptq_marlin import gptq_marlin_gemm logger = logging.getLogger(__name__) @@ -107,7 +107,6 @@ def _check_marlin_supported( has_zp: bool, device_capability: Optional[int] = None, ) -> tuple[bool, Optional[str]]: - if device_capability is None: major, minor = get_device_capability() capability = major * 10 + minor if major is not None else None @@ -165,7 +164,6 @@ def verify_marlin_supports_shape( input_size: int, group_size: int, ) -> None: - # Validate output_size_per_partition if output_size_per_partition % GPTQ_MARLIN_MIN_THREAD_N != 0: raise ValueError( @@ -313,7 +311,6 @@ def get_scale_perms(): def marlin_permute_scales( s: torch.Tensor, size_k: int, size_n: int, group_size: int ) -> torch.Tensor: - scale_perm, scale_perm_single = get_scale_perms() if group_size < size_k and group_size != -1: s = s.reshape((-1, len(scale_perm)))[:, scale_perm] @@ -441,7 +438,6 @@ def maybe_warn_marlin_atomic_add_env(): def should_use_atomic_add_reduce( m: int, n: int, k: int, device: torch.device, dtype: torch.dtype ) -> bool: - # the performance of atomicAdd is better than global reduce # only when m*n is small and k is large if n >= 2048 or k < 2048 or device.type != "cuda": diff --git a/python/sglang/srt/layers/quantization/marlin_utils_fp4.py b/python/sglang/srt/layers/quantization/marlin_utils_fp4.py index 0032824524..1a4b5d5269 100644 --- a/python/sglang/srt/layers/quantization/marlin_utils_fp4.py +++ b/python/sglang/srt/layers/quantization/marlin_utils_fp4.py @@ -19,7 +19,7 @@ from sglang.srt.utils.custom_op import register_custom_op _is_cuda = is_cuda() if _is_cuda: - from sglang.kernels.ops.quantization.gptq_marlin import gptq_marlin_gemm + from sglang.kernels.ops.gemm.gptq_marlin import gptq_marlin_gemm from sglang.kernels.ops.quantization.gptq_marlin_repack import gptq_marlin_repack ScalarType, scalar_types = get_scalar_types() diff --git a/python/sglang/srt/layers/quantization/marlin_utils_fp8.py b/python/sglang/srt/layers/quantization/marlin_utils_fp8.py index ff711fc335..0eba5b3d0f 100644 --- a/python/sglang/srt/layers/quantization/marlin_utils_fp8.py +++ b/python/sglang/srt/layers/quantization/marlin_utils_fp8.py @@ -17,7 +17,7 @@ from sglang.srt.utils.custom_op import register_custom_op _is_cuda = is_cuda() if _is_cuda: - from sglang.kernels.ops.quantization.gptq_marlin import gptq_marlin_gemm + from sglang.kernels.ops.gemm.gptq_marlin import gptq_marlin_gemm from sglang.kernels.ops.quantization.gptq_marlin_repack import gptq_marlin_repack ScalarType, scalar_types = get_scalar_types() diff --git a/python/sglang/srt/model_executor/model_runner_components/weight_updater.py b/python/sglang/srt/model_executor/model_runner_components/weight_updater.py index 0e0d8db22e..5d852a6136 100644 --- a/python/sglang/srt/model_executor/model_runner_components/weight_updater.py +++ b/python/sglang/srt/model_executor/model_runner_components/weight_updater.py @@ -66,7 +66,7 @@ def _unsupported_derived_weight_cache_error( reason = getattr(module, "_derived_weight_cache_error", None) if reason is not None: return reason - from sglang.kernels.ops.attention.dsv4.gemm import hpc_bf16xfp32_gemm_enabled + from sglang.kernels.ops.gemm.bf16_fp32 import hpc_bf16xfp32_gemm_enabled if hpc_bf16xfp32_gemm_enabled(): return ( diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index c758365644..896fcb7979 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -30,7 +30,7 @@ import torch.nn.functional as F from torch import nn from transformers import PretrainedConfig -from sglang.kernels.ops.attention.dsv4 import ( +from sglang.kernels.ops.moe.dsv4 import ( silu_and_mul_clamp, silu_and_mul_contig_post_quant, ) @@ -549,7 +549,7 @@ class MoEGate(nn.Module): logits = F.linear(hidden_states, self.weight, None) else: # cuBLAS bf16 x bf16 -> fp32 GEMM (torch.mm's out_dtype kwarg is CUDA-only) - from sglang.kernels.ops.attention.dsv4 import linear_bf16_fp32 + from sglang.kernels.ops.gemm.bf16_fp32 import linear_bf16_fp32 logits = linear_bf16_fp32(hidden_states, self.weight) diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 422c59bf2a..cd7e928648 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -32,13 +32,13 @@ from sglang.kernels.ops.attention.dsv4 import ( sglang_per_token_group_quant_fp8_dsv4_wo_a, ) from sglang.kernels.ops.attention.dsv4.wo_a import MAX_M as _FUSED_WO_A_MAX_TOKENS -from sglang.kernels.ops.attention.dsv4.wo_a import ( - fused_rope_wo_a_bf16, +from sglang.kernels.ops.attention.dsv4.wo_a import fused_rope_wo_a_bf16 +from sglang.kernels.ops.attention.flash_mla_sm120 import SM120_DECODE_MAX_TOKENS +from sglang.kernels.ops.gemm.dsv4_wo_a import ( wo_a_bf16_gemv, wo_a_bf16_small_batch, wo_a_bf16_small_batch_mxfp8, ) -from sglang.kernels.ops.attention.flash_mla_sm120 import SM120_DECODE_MAX_TOKENS from sglang.kernels.ops.layernorm.mhc_post_split_h import mhc_post_split_h from sglang.kernels.ops.quantization.fp8_kernel import ( sglang_per_token_group_quant_fp8, diff --git a/python/sglang/srt/models/gemma4_causal.py b/python/sglang/srt/models/gemma4_causal.py index 99187c539f..eec706fd6b 100644 --- a/python/sglang/srt/models/gemma4_causal.py +++ b/python/sglang/srt/models/gemma4_causal.py @@ -25,10 +25,12 @@ from transformers import ( ) from sglang.kernels.ops.layernorm.gemma4_fused_ops import ( - gemma4_fused_routing, gemma_dual_rmsnorm_residual_scalar, gemma_qkv_rmsnorm, gemma_rmsnorm_residual_scalar, +) +from sglang.kernels.ops.moe.gemma4_routing import ( + gemma4_fused_routing, gemma_routing_post_topk, ) from sglang.srt.layers.layernorm import Gemma4RMSNorm, RMSNorm diff --git a/python/sglang/srt/models/inkling_common/attn.py b/python/sglang/srt/models/inkling_common/attn.py index 75e6aa701f..5659b3efc3 100644 --- a/python/sglang/srt/models/inkling_common/attn.py +++ b/python/sglang/srt/models/inkling_common/attn.py @@ -9,14 +9,14 @@ from torch import nn from sglang.kernels.ops.attention.flash_attn.cute.batch_invariance import ( is_batch_invariant, ) -from sglang.kernels.ops.attention.inkling_rel_proj import rel_proj_small_t -from sglang.kernels.ops.attention.inkling_row_scale import row_compact_bf16 from sglang.kernels.ops.attention.log_scaling_tau import ( apply_log_scaling_tau as _apply_log_scaling_tau, ) from sglang.kernels.ops.attention.score_mod import ( relative_bias_score_mod as triton_relative_bias_score_mod, ) +from sglang.kernels.ops.gemm.inkling_rel_proj import rel_proj_small_t +from sglang.kernels.ops.memory.row_compact import row_compact_bf16 from sglang.srt.environ import envs from sglang.srt.layers.linear import MergedColumnParallelLinear, RowParallelLinear from sglang.srt.layers.quantization.base_config import QuantizationConfig diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index a8410480e4..52d275c2b9 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -30,14 +30,10 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( ) from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder -from sglang.srt.layers import ( - k3_ar_fusion, - k3_gemm_ar, - k3_sp_collective, - zero_copy_context, -) +from sglang.srt.layers import zero_copy_context from sglang.srt.layers.activation import SiluAndMul, SituAndMul from sglang.srt.layers.attn_residual import AttnResidual, aggregate_stream, get_cw +from sglang.srt.layers.communication import k3_ar_fusion, k3_sp_collective from sglang.srt.layers.dcp.planner import prepare_decode_context_parallel_metadata from sglang.srt.layers.dp_attention import ( dp_gather_replicate, @@ -774,7 +770,7 @@ class KimiK3MoE(nn.Module): applies the norm).""" import deep_gemm - from sglang.kernels.ops.attention.dsv4 import mega_moe_pre_dispatch + from sglang.kernels.ops.moe.dsv4 import mega_moe_pre_dispatch from sglang.srt.environ import envs from sglang.srt.layers.moe.mega_moe import ( _configure_mega_moe_deep_gemm_num_sms, @@ -1786,7 +1782,6 @@ class KimiK3DeltaAttention(nn.Module): self.all_reduce_fusion = False self.o_proj.reduce_results = True self.o_proj.use_dp_attention_reduce = True - k3_gemm_ar.maybe_wrap_o_proj(self.o_proj) conv_weights = self.qkv_conv1d.weight.squeeze(1) bias = self.qkv_conv1d.bias @@ -2238,7 +2233,6 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA): self.all_reduce_fusion = False self.o_proj.reduce_results = True self.o_proj.use_dp_attention_reduce = True - k3_gemm_ar.maybe_wrap_o_proj(self.o_proj) if self.all_reduce_fusion: # Hand the GEMM a slice of the persistent symmetric buffer # (k3_ar_fusion.symm_buffer); the fused AR reduces it in place. diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index e37641c240..353a560ae4 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -37,8 +37,10 @@ from typing import Iterable, List, Optional, Tuple import torch from torch import nn -from sglang.kernels.ops.attention.dsv4 import linear_bf16_fp32 -from sglang.kernels.ops.attention.dsv4.gemm import mark_hpc_bf16xfp32_gemm_enabled +from sglang.kernels.ops.gemm.bf16_fp32 import ( + linear_bf16_fp32, + mark_hpc_bf16xfp32_gemm_enabled, +) from sglang.kernels.ops.moe.ep_moe_kernels import zero_experts_compute_triton from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz from sglang.srt.configs import LongcatFlashConfig @@ -749,7 +751,6 @@ class LongcatFlashForCausalLM(nn.Module): ) def post_load_weights(self, weight_names=None): - # Perform post-processing after loading weights if weight_names is None: layer_ids = range(self.config.num_hidden_layers) @@ -961,7 +962,6 @@ class LongcatFlashForCausalLM(nn.Module): requant_weight_ue8m0_inplace(w[0], w[1], weight_block_size) def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): - stacked_params_mapping = [ # (param_name, shard_name, shard_id) ("gate_up_proj", "gate_proj", 0), diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py index 61866302e9..68c46cb2d3 100644 --- a/python/sglang/srt/models/qwen4_exp.py +++ b/python/sglang/srt/models/qwen4_exp.py @@ -640,7 +640,7 @@ class Qwen4ExpNGramEmbedding(nn.Module): ) -> torch.Tensor: contexts = contexts.to(torch.long) if self.enable_ple_fusion and decode_sized: - from sglang.kernels.ops.qwen4_ple import ( + from sglang.kernels.ops.embeddings.qwen4_ngram import ( can_fuse_qwen4_ngram_hash, fused_qwen4_ngram_hash, ) @@ -1020,7 +1020,7 @@ class Qwen4ExpPLELayer(nn.Module): if batch.use_decode_fast_path: # With row_width=1 the padded/transpose path is x.unsqueeze(-1), # and each state boundary is a one-column shift; conv and SiLU stay native. - from sglang.kernels.ops.qwen4_ple import ( + from sglang.kernels.ops.mamba.qwen4_short_conv import ( can_fuse_qwen4_short_conv_state, fused_qwen4_short_conv_state, ) @@ -1236,7 +1236,7 @@ class Qwen4ExpPLELayer(nn.Module): query = hidden_states.reshape(token_count, hc_count, hidden_size) key_normed = self._apply_ple_norm(self.norm_key, key) query_normed = self._apply_ple_norm(self.norm_query, query) - from sglang.kernels.ops.qwen4_ple import ( + from sglang.kernels.ops.elementwise.qwen4_gate import ( can_fuse_qwen4_gate_reduce, fused_qwen4_gate_reduce, ) @@ -1253,7 +1253,7 @@ class Qwen4ExpPLELayer(nn.Module): gate = gate / math.sqrt(hidden_size) fused_gate_value = False if batch.use_decode_fast_path: - from sglang.kernels.ops.qwen4_ple import ( + from sglang.kernels.ops.elementwise.qwen4_gate import ( can_fuse_qwen4_gate_value, fused_qwen4_gate_value, ) @@ -1269,7 +1269,7 @@ class Qwen4ExpPLELayer(nn.Module): gated_value = gated_value.flatten(-2) gated_value_normed = gated_value_normed.flatten(-2) if self.ple_embedding.enable_ple_fusion and batch.mode.is_target_verify(): - from sglang.kernels.ops.qwen4_ple import ( + from sglang.kernels.ops.mamba.qwen4_short_conv import ( can_fuse_qwen4_verify_conv, fused_qwen4_verify_conv, ) diff --git a/test/registered/quant/test_fp8_kernel.py b/python/sglang/test/kernels/fp8.py similarity index 53% rename from test/registered/quant/test_fp8_kernel.py rename to python/sglang/test/kernels/fp8.py index 6ee0b8a148..16fa06d69d 100644 --- a/test/registered/quant/test_fp8_kernel.py +++ b/python/sglang/test/kernels/fp8.py @@ -1,21 +1,10 @@ -import unittest +"""Shared input/reference construction for FP8 quantization and GEMM tests.""" import torch -from sglang.kernels.ops.quantization.fp8_kernel import ( - per_token_group_quant_fp8, - w8a8_block_fp8_matmul, -) -from sglang.test.ci.ci_register import register_cuda_ci +from sglang.srt.utils import get_device from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=9, stage="base-b", runner_config="1-gpu-large") - -from sglang.srt.utils import get_device, is_cuda, is_xpu - -_is_cuda = is_cuda() -_is_xpu = is_xpu() - device = get_device() @@ -94,52 +83,3 @@ class TestFP8Base(CustomTestCase): quant_B = quant_B.reshape(K_aligned, N_aligned).to(out_dtype)[:K, :N] scale = scale.reshape(K_aligned // group_size, N_aligned // group_size) return B, quant_B, scale - - -class TestPerTokenGroupQuantFP8(TestFP8Base): - def test_per_token_group_quant_fp8(self): - if _is_cuda and torch.cuda.get_device_capability()[0] < 9: - return - - A, A_quant_gt, scale_gt = self._make_A( - M=self.M, K=self.K, group_size=self.group_size, out_dtype=self.quant_type - ) - A_quant, scale = per_token_group_quant_fp8( - x=A.to(torch.bfloat16), group_size=self.group_size - ) - torch.testing.assert_close(scale, scale_gt) - diff = (A_quant.to(torch.float16) - A_quant_gt.to(torch.float16)).abs() - diff_count = (diff > 1e-5).count_nonzero() - assert diff_count / diff.numel() < 1e-4 - - -class TestW8A8BlockFP8Matmul(TestFP8Base): - def test_w8a8_block_fp8_matmul(self): - if _is_cuda and torch.cuda.get_device_capability()[0] < 9: - return - elif _is_xpu: - # XPU doesn't provide traditional capability info like CUDA - pass - else: - return - - A, A_quant_gt, A_scale_gt = self._make_A( - M=self.M, K=self.K, group_size=self.group_size, out_dtype=self.quant_type - ) - B, B_quant_gt, B_scale_gt = self._make_B( - K=self.K, N=self.N, group_size=self.group_size, out_dtype=self.quant_type - ) - C_gt = A.to(self.output_type) @ B.to(self.output_type) - C = w8a8_block_fp8_matmul( - A=A_quant_gt, - B=B_quant_gt.T.contiguous(), - As=A_scale_gt, - Bs=B_scale_gt.T.contiguous(), - block_size=[128, 128], - output_dtype=self.output_type, - ) - torch.testing.assert_close(C, C_gt, atol=0.5, rtol=1e-4) - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/ci/cuda/warmup_deep_gemm.py b/scripts/ci/cuda/warmup_deep_gemm.py index 91b23679dd..edf59679e2 100644 --- a/scripts/ci/cuda/warmup_deep_gemm.py +++ b/scripts/ci/cuda/warmup_deep_gemm.py @@ -120,7 +120,7 @@ def compute_deepseek_v2v3_shapes(config, tp): Shape derivation based on: - MoE: python/sglang/srt/layers/moe/fused_moe_triton/layer.py - MLA: python/sglang/srt/models/deepseek_v2.py - - FP8: python/sglang/kernels/ops/quantization/fp8_kernel.py + - FP8 GEMM: python/sglang/kernels/ops/gemm/fp8_kernel.py """ shapes = [] diff --git a/test/manual/kernels/attention/check_dsa_dequant_k_cache.py b/test/manual/kernels/attention/check_dsa_dequant_k_cache.py new file mode 100644 index 0000000000..9c69e1562e --- /dev/null +++ b/test/manual/kernels/attention/check_dsa_dequant_k_cache.py @@ -0,0 +1,2 @@ +if __name__ == "__main__": + raise Exception("UT is in quant_k_cache.py") diff --git a/test/manual/kernels/attention/check_dsa_quant_k_cache.py b/test/manual/kernels/attention/check_dsa_quant_k_cache.py new file mode 100644 index 0000000000..83f4570844 --- /dev/null +++ b/test/manual/kernels/attention/check_dsa_quant_k_cache.py @@ -0,0 +1,133 @@ +import torch +import triton + +from sglang.kernels.ops.attention.dsa.quant_k_cache import ( + _quantize_k_cache_fast_wrapped, + _quantize_k_cache_ref, + quantize_k_cache, + quantize_k_cache_separate, +) + +if __name__ == "__main__": + import dequant_k_cache + + for num_blocks, block_size in [ + (1, 1), + (10, 64), + ]: + dim_nope_and_rope = 512 + 64 + + input_k_cache = torch.randn( + (num_blocks, block_size, 1, dim_nope_and_rope), + dtype=torch.bfloat16, + device="cuda", + ) + + ref_quant = _quantize_k_cache_ref(input_k_cache) + actual_quant = _quantize_k_cache_fast_wrapped(input_k_cache) + + ref_ref_dequant = dequant_k_cache._dequantize_k_cache_slow(ref_quant) + ref_actual_dequant = dequant_k_cache._dequantize_k_cache_fast_wrapped(ref_quant) + actual_actual_dequant = dequant_k_cache._dequantize_k_cache_fast_wrapped( + actual_quant + ) + + print(f"{ref_ref_dequant=}") + print(f"{actual_actual_dequant=}") + print(f"{actual_actual_dequant - ref_ref_dequant=}") + print(f"{torch.mean(ref_ref_dequant - actual_actual_dequant)=}") + + # TODO too different? + torch.testing.assert_close( + ref_ref_dequant, ref_actual_dequant, atol=0.2, rtol=0.2 + ) + torch.testing.assert_close( + ref_ref_dequant, actual_actual_dequant, atol=0.2, rtol=0.2 + ) + + # test dequant_k_cache_paged + page_table_1 = torch.arange( + num_blocks * block_size, dtype=torch.int32, device="cuda" + ) + actual_dequant_paged = dequant_k_cache.dequantize_k_cache_paged( + actual_quant, page_table_1 + ).reshape(actual_actual_dequant.shape) + print(f"{torch.mean(actual_actual_dequant - actual_dequant_paged)=}") + torch.testing.assert_close( + ref_ref_dequant, actual_dequant_paged, atol=0.2, rtol=0.2 + ) + + print("Passed") + + # Test quantize_k_cache_separate: verify output matches concat path + print("\nTesting quantize_k_cache_separate...") + for num_tokens in [64, 100]: + dim_nope = 512 + dim_rope = 64 + + k_nope = torch.randn( + num_tokens, 1, dim_nope, dtype=torch.bfloat16, device="cuda" + ) + k_rope = torch.randn( + num_tokens, 1, dim_rope, dtype=torch.bfloat16, device="cuda" + ) + + # Old path: concat then quantize + k_concat = torch.cat([k_nope, k_rope], dim=-1).squeeze(1) # (num_tokens, 576) + old_output = quantize_k_cache(k_concat.unsqueeze(1).unsqueeze(1)) # 4D input + old_output = old_output.squeeze(1).squeeze(1) # Back to (num_tokens, 656) + + # New path: quantize separately + nope_part, rope_part = quantize_k_cache_separate(k_nope, k_rope) + new_bytes = torch.cat([nope_part.squeeze(1), rope_part.squeeze(1)], dim=-1) + + # Compare byte-level equality + old_bytes = old_output.view(torch.uint8) + + if old_bytes.shape != new_bytes.shape: + raise RuntimeError( + f"Shape mismatch: {old_bytes.shape} vs {new_bytes.shape}" + ) + + diff_bytes = (old_bytes != new_bytes).sum().item() + if diff_bytes > 0: + max_diff = (old_bytes.float() - new_bytes.float()).abs().max().item() + raise RuntimeError( + f"quantize_k_cache_separate output doesn't match concat path: " + f"{diff_bytes} differing bytes, max_diff={max_diff}" + ) + + print(f" num_tokens={num_tokens}: PASSED (outputs match byte-wise)") + + print("quantize_k_cache_separate tests passed!") + + print("\nDo benchmark...") + + for num_blocks, block_size in [ + (1, 64), + (64, 64), + (128, 64), + (256, 64), + (512, 64), + (1024, 64), + (2048, 64), + ]: + dim_nope_and_rope = 512 + 64 + + input_k_cache = torch.randn( + (num_blocks, block_size, 1, dim_nope_and_rope), + dtype=torch.bfloat16, + device="cuda", + ) + + actual_quant = _quantize_k_cache_fast_wrapped(input_k_cache) + + page_table_1 = torch.arange( + num_blocks * block_size, dtype=torch.int32, device="cuda" + ) + + def run_ans(): + return dequant_k_cache.dequantize_k_cache_paged(actual_quant, page_table_1) + + ans_time: float = triton.testing.do_bench(run_ans, warmup=10, rep=20) / 1000 # type: ignore + print(f"seq_kv: {num_blocks * block_size}, time: {ans_time * 1e6: 4.0f} us") diff --git a/test/manual/kernels/attention/check_dsa_transform_index.py b/test/manual/kernels/attention/check_dsa_transform_index.py new file mode 100644 index 0000000000..fd15f0c8eb --- /dev/null +++ b/test/manual/kernels/attention/check_dsa_transform_index.py @@ -0,0 +1,16 @@ +import torch + +from sglang.kernels.ops.attention.dsa.transform_index import ( + transform_index_page_table_decode_fast, + transform_index_page_table_decode_ref, +) + +if __name__ == "__main__": + bs, topk, max_seqlen = 10, 2048, 3000 + page_table = torch.randint(0, 100, (bs, max_seqlen), device="cuda") + topk_indices = torch.full((bs, topk), -1, device="cuda") + topk_indices[:, :1600] = torch.arange(1600).unsqueeze(0).repeat(bs, 1) + ref_result = transform_index_page_table_decode_ref(page_table, topk_indices) + result = transform_index_page_table_decode_fast(page_table, topk_indices) + assert torch.all(result == ref_result) + print("Passed") diff --git a/test/manual/kernels/attention/check_dsv4_dequant_k_cache.py b/test/manual/kernels/attention/check_dsv4_dequant_k_cache.py new file mode 100644 index 0000000000..23c77f2093 --- /dev/null +++ b/test/manual/kernels/attention/check_dsv4_dequant_k_cache.py @@ -0,0 +1,37 @@ +import torch + +from sglang.kernels.ops.attention.dsv4.dequant_k_cache import ( + NOPE_ROPE_BYTES, + PADDED_SCALE_PER_TOKEN, + dequantize_k_cache_paged, + dequantize_k_cache_paged_ref, +) + +if __name__ == "__main__": + assert torch.cuda.is_available(), "this self-test needs a CUDA device" + torch.manual_seed(0) + device = "cuda" + + page_size = 64 + num_pages = 8 + num_tokens = 333 + raw_bytes = page_size * (NOPE_ROPE_BYTES + PADDED_SCALE_PER_TOKEN) + bytes_per_page = ( + (raw_bytes + NOPE_ROPE_BYTES - 1) // NOPE_ROPE_BYTES + ) * NOPE_ROPE_BYTES + + quant_k_cache = torch.randint( + 0, 256, (num_pages, bytes_per_page), dtype=torch.uint8, device=device + ) + page_table = torch.randint( + 0, num_pages * page_size, (num_tokens,), dtype=torch.int32, device=device + ) + + out_kernel = dequantize_k_cache_paged(quant_k_cache, page_table, page_size) + out_ref = dequantize_k_cache_paged_ref(quant_k_cache, page_table, page_size) + + torch.testing.assert_close(out_kernel, out_ref, atol=0, rtol=0, equal_nan=True) + print( + f"OK: kernel matches torch ref for {num_tokens} tokens " + f"(page_size={page_size}, bytes_per_page={bytes_per_page})" + ) diff --git a/python/sglang/kernels/ops/attention/fla/bench_gdn_replayssm_fold.py b/test/manual/kernels/benchmark/attention/bench_gdn_replayssm_fold.py similarity index 98% rename from python/sglang/kernels/ops/attention/fla/bench_gdn_replayssm_fold.py rename to test/manual/kernels/benchmark/attention/bench_gdn_replayssm_fold.py index 2bc372643e..785d7e92c1 100644 --- a/python/sglang/kernels/ops/attention/fla/bench_gdn_replayssm_fold.py +++ b/test/manual/kernels/benchmark/attention/bench_gdn_replayssm_fold.py @@ -3,7 +3,7 @@ Qwen3.5-397B at TP4). The kernel is bound by the mandatory checkpoint read+write (``bs * layers * HV * K * V * 4 B * 2``), so the reported GB/s approximates achieved HBM bandwidth. -Run: ``python -m sglang.kernels.ops.attention.fla.bench_gdn_replayssm_fold`` +Run: ``python test/manual/kernels/benchmark/attention/bench_gdn_replayssm_fold.py`` """ from __future__ import annotations diff --git a/test/manual/quant/test_awq_archived.py b/test/manual/quant/test_awq_archived.py index 2e558006ef..84cce39026 100644 --- a/test/manual/quant/test_awq_archived.py +++ b/test/manual/quant/test_awq_archived.py @@ -1,4 +1,4 @@ -"""Archived test classes split out of test/registered/quant/test_awq.py; not in CI.""" +"""Archived test classes split out of test/registered/e2e/quantization/test_awq.py; not in CI.""" import unittest from types import SimpleNamespace diff --git a/test/manual/quant/test_block_fp8.py b/test/manual/quant/test_block_fp8.py index 8356a3a931..d96bb782e7 100644 --- a/test/manual/quant/test_block_fp8.py +++ b/test/manual/quant/test_block_fp8.py @@ -4,12 +4,12 @@ from functools import lru_cache import torch +from sglang.kernels.ops.gemm.fp8_kernel import w8a8_block_fp8_matmul from sglang.kernels.ops.quantization.fp8_kernel import ( per_tensor_quant_mla_fp8, per_token_group_quant_fp8, per_token_group_quant_mla_deep_gemm_masked_fp8, static_quant_fp8, - w8a8_block_fp8_matmul, ) from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import fused_moe @@ -747,7 +747,6 @@ class TestW8A8BlockFP8BatchedDeepGemm(CustomTestCase): ) def test_w8a8_block_fp8_batched_deep_gemm(self): - for params in itertools.product( self.M, self.N, diff --git a/test/registered/quant/test_autoround_quantization.py b/test/registered/e2e/quantization/test_autoround_quantization.py similarity index 100% rename from test/registered/quant/test_autoround_quantization.py rename to test/registered/e2e/quantization/test_autoround_quantization.py diff --git a/test/registered/quant/test_awq.py b/test/registered/e2e/quantization/test_awq.py similarity index 100% rename from test/registered/quant/test_awq.py rename to test/registered/e2e/quantization/test_awq.py diff --git a/test/registered/quant/test_fp8kv_triton.py b/test/registered/e2e/quantization/test_fp8kv_triton.py similarity index 100% rename from test/registered/quant/test_fp8kv_triton.py rename to test/registered/e2e/quantization/test_fp8kv_triton.py diff --git a/test/registered/quant/test_gguf.py b/test/registered/e2e/quantization/test_gguf.py similarity index 100% rename from test/registered/quant/test_gguf.py rename to test/registered/e2e/quantization/test_gguf.py diff --git a/test/registered/quant/test_gptqmodel_dynamic.py b/test/registered/e2e/quantization/test_gptqmodel_dynamic.py similarity index 100% rename from test/registered/quant/test_gptqmodel_dynamic.py rename to test/registered/e2e/quantization/test_gptqmodel_dynamic.py diff --git a/test/registered/quant/test_int4fp8_moe.py b/test/registered/e2e/quantization/test_int4fp8_moe.py similarity index 100% rename from test/registered/quant/test_int4fp8_moe.py rename to test/registered/e2e/quantization/test_int4fp8_moe.py diff --git a/test/registered/quant/test_llama8b_nvfp4_kv_cache_sm120.py b/test/registered/e2e/quantization/test_llama8b_nvfp4_kv_cache_sm120.py similarity index 100% rename from test/registered/quant/test_llama8b_nvfp4_kv_cache_sm120.py rename to test/registered/e2e/quantization/test_llama8b_nvfp4_kv_cache_sm120.py diff --git a/test/registered/quant/test_modelopt_fp8.py b/test/registered/e2e/quantization/test_modelopt_fp8.py similarity index 100% rename from test/registered/quant/test_modelopt_fp8.py rename to test/registered/e2e/quantization/test_modelopt_fp8.py diff --git a/test/registered/quant/test_nvfp4_gemm_sm120.py b/test/registered/e2e/quantization/test_nvfp4_gemm_sm120.py similarity index 100% rename from test/registered/quant/test_nvfp4_gemm_sm120.py rename to test/registered/e2e/quantization/test_nvfp4_gemm_sm120.py diff --git a/test/registered/quant/test_quark_mxfp4.py b/test/registered/e2e/quantization/test_quark_mxfp4.py similarity index 100% rename from test/registered/quant/test_quark_mxfp4.py rename to test/registered/e2e/quantization/test_quark_mxfp4.py diff --git a/test/registered/quant/test_w8a8_quantization.py b/test/registered/e2e/quantization/test_w8a8_quantization.py similarity index 99% rename from test/registered/quant/test_w8a8_quantization.py rename to test/registered/e2e/quantization/test_w8a8_quantization.py index 4989b79aae..f15aac6cf5 100644 --- a/test/registered/quant/test_w8a8_quantization.py +++ b/test/registered/e2e/quantization/test_w8a8_quantization.py @@ -78,7 +78,6 @@ class BaseW8A8Test(CustomTestCase): return response.json() def test_throughput(self): - max_tokens = 256 tic = time.perf_counter() res = self.run_decode(max_tokens) diff --git a/test/registered/kernels/benchmark/minicpm_sala/bench_get_block_table.py b/test/registered/kernels/benchmark/attention/minicpm_sala/bench_get_block_table.py similarity index 95% rename from test/registered/kernels/benchmark/minicpm_sala/bench_get_block_table.py rename to test/registered/kernels/benchmark/attention/minicpm_sala/bench_get_block_table.py index 406dba6504..a83f1c7f0c 100644 --- a/test/registered/kernels/benchmark/minicpm_sala/bench_get_block_table.py +++ b/test/registered/kernels/benchmark/attention/minicpm_sala/bench_get_block_table.py @@ -1,7 +1,7 @@ import torch from sglang.kernels.jit.benchmark import marker -from sglang.kernels.ops.minicpm_sala import get_block_table +from sglang.kernels.ops.attention.minicpm_sala import get_block_table from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci( diff --git a/test/registered/kernels/benchmark/gemm/bench_bf16xfp32_router_gemm.py b/test/registered/kernels/benchmark/gemm/bench_bf16xfp32_router_gemm.py index f4b55b2f39..4f114099ab 100644 --- a/test/registered/kernels/benchmark/gemm/bench_bf16xfp32_router_gemm.py +++ b/test/registered/kernels/benchmark/gemm/bench_bf16xfp32_router_gemm.py @@ -18,7 +18,7 @@ import torch from sglang.kernels.jit.benchmark import marker from sglang.kernels.jit.benchmark.utils import create_random from sglang.kernels.jit.utils import get_jit_cuda_arch, is_hip_runtime -from sglang.kernels.ops.attention.dsv4.gemm import ( +from sglang.kernels.ops.gemm.bf16_fp32 import ( _linear_bf16_fp32_hpc, mark_hpc_bf16xfp32_gemm_enabled, ) diff --git a/test/registered/kernels/benchmark/moe/bench_renorm.py b/test/registered/kernels/benchmark/sampling/bench_renorm.py similarity index 100% rename from test/registered/kernels/benchmark/moe/bench_renorm.py rename to test/registered/kernels/benchmark/sampling/bench_renorm.py diff --git a/test/registered/kernels/ops/activation/test_fused_op_gpu_parity.py b/test/registered/kernels/ops/activation/test_fused_op_gpu_parity.py new file mode 100644 index 0000000000..3a4dcbb804 --- /dev/null +++ b/test/registered/kernels/ops/activation/test_fused_op_gpu_parity.py @@ -0,0 +1,55 @@ +"""Every-backend-vs-native parity for BaseFusedOp ops on real GPU (RFC #29630). + +For each reworked fused op, run every backend eligible on this platform and +assert it matches the pure-torch ``forward_native`` reference within dtype +tolerance. New backends are picked up automatically. +""" + +import pytest +import torch + +from sglang.kernels.spec import KernelBackend +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=4, stage="extra-a", runner_config="1-gpu-small") + +_SKIP = {KernelBackend.TORCH, KernelBackend.TORCH_COMPILE} + +_TOL = { + torch.float16: dict(atol=1e-2, rtol=1e-2), + torch.bfloat16: dict(atol=2e-2, rtol=2e-2), +} + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") + + +def _eligible(op): + return [ + b for b in op.available_backends() if b not in _SKIP and op.backend_eligible(b) + ] + + +def _close(got, ref, dtype, msg): + torch.testing.assert_close(got, ref, **_TOL[dtype], msg=msg) + + +@pytest.mark.parametrize( + "op_attr", ["_SILU_AND_MUL", "_GELU_AND_MUL", "_GELU_TANH_AND_MUL"] +) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("shape", [(1, 8192), (128, 8192)]) +def test_gated_activation(op_attr, dtype, shape): + import sglang.kernels.ops.activation as act + + torch.manual_seed(0) + op = getattr(act, op_attr) + x = torch.randn(shape, dtype=dtype, device="cuda") + ref = op.forward_native(x) + for b in _eligible(op): + _close(op.forward(x, backend=b), ref, dtype, f"{op.op} {b.value}") + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__])) diff --git a/test/registered/kernels/ops/minicpm_sala/test_get_block_table.py b/test/registered/kernels/ops/attention/minicpm_sala/test_get_block_table.py similarity index 98% rename from test/registered/kernels/ops/minicpm_sala/test_get_block_table.py rename to test/registered/kernels/ops/attention/minicpm_sala/test_get_block_table.py index ea7f8994e4..d3306e9d01 100644 --- a/test/registered/kernels/ops/minicpm_sala/test_get_block_table.py +++ b/test/registered/kernels/ops/attention/minicpm_sala/test_get_block_table.py @@ -1,7 +1,7 @@ import pytest import torch -from sglang.kernels.ops.minicpm_sala.get_block_table import get_block_table +from sglang.kernels.ops.attention.minicpm_sala.get_block_table import get_block_table from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") diff --git a/test/registered/kernels/ops/minicpm_sala/test_minicpm_compress_k.py b/test/registered/kernels/ops/attention/minicpm_sala/test_minicpm_compress_k.py similarity index 100% rename from test/registered/kernels/ops/minicpm_sala/test_minicpm_compress_k.py rename to test/registered/kernels/ops/attention/minicpm_sala/test_minicpm_compress_k.py diff --git a/test/registered/kernels/ops/qsa/test_qsa.py b/test/registered/kernels/ops/attention/qsa/test_qsa.py similarity index 100% rename from test/registered/kernels/ops/qsa/test_qsa.py rename to test/registered/kernels/ops/attention/qsa/test_qsa.py diff --git a/test/registered/kernels/ops/qsa/test_qsa_indexer.py b/test/registered/kernels/ops/attention/qsa/test_qsa_indexer.py similarity index 100% rename from test/registered/kernels/ops/qsa/test_qsa_indexer.py rename to test/registered/kernels/ops/attention/qsa/test_qsa_indexer.py diff --git a/test/registered/kernels/ops/qsa/test_qsa_strided_zero_fill.py b/test/registered/kernels/ops/attention/qsa/test_qsa_strided_zero_fill.py similarity index 100% rename from test/registered/kernels/ops/qsa/test_qsa_strided_zero_fill.py rename to test/registered/kernels/ops/attention/qsa/test_qsa_strided_zero_fill.py diff --git a/test/registered/kernels/ops/attention/test_dsv32_indexer_fusion.py b/test/registered/kernels/ops/attention/test_dsv32_indexer_fusion.py index 268a1c4dff..c42cc8c350 100644 --- a/test/registered/kernels/ops/attention/test_dsv32_indexer_fusion.py +++ b/test/registered/kernels/ops/attention/test_dsv32_indexer_fusion.py @@ -17,15 +17,15 @@ from __future__ import annotations import pytest import torch +from sglang.kernels.ops.attention.dsa.indexer_k import ( + fused_k_indexer_norm_rope, + fused_k_indexer_norm_rope_store, +) from sglang.kernels.ops.attention.dsv4 import fused_q_indexer_rope_first_quant from sglang.kernels.ops.attention.fused_store_index_cache import ( can_use_dsa_fused_store, fused_store_index_k_cache, ) -from sglang.kernels.ops.quantization.dsv32 import ( - fused_k_indexer_norm_rope, - fused_k_indexer_norm_rope_store, -) from sglang.srt.utils import is_hip from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci diff --git a/test/registered/kernels/ops/elementwise/test_fast_topk.py b/test/registered/kernels/ops/attention/test_fast_topk.py similarity index 98% rename from test/registered/kernels/ops/elementwise/test_fast_topk.py rename to test/registered/kernels/ops/attention/test_fast_topk.py index c203acb49a..a08eb15c61 100644 --- a/test/registered/kernels/ops/elementwise/test_fast_topk.py +++ b/test/registered/kernels/ops/attention/test_fast_topk.py @@ -1,7 +1,7 @@ import pytest import torch -from sglang.kernels.ops.elementwise.fast_topk import fast_topk +from sglang.kernels.ops.attention.fast_topk import fast_topk from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") diff --git a/test/registered/kernels/ops/moe/test_kpool_topk_transform_fused.py b/test/registered/kernels/ops/attention/test_kpool_topk_transform_fused.py similarity index 97% rename from test/registered/kernels/ops/moe/test_kpool_topk_transform_fused.py rename to test/registered/kernels/ops/attention/test_kpool_topk_transform_fused.py index 48f80cec8b..edafdfc33d 100644 --- a/test/registered/kernels/ops/moe/test_kpool_topk_transform_fused.py +++ b/test/registered/kernels/ops/attention/test_kpool_topk_transform_fused.py @@ -18,7 +18,9 @@ import unittest import torch -from sglang.kernels.ops.moe.kpool_topk_transform import fast_kpool_topk_transform_fused +from sglang.kernels.ops.attention.dsa.kpool_topk_transform import ( + fast_kpool_topk_transform_fused, +) from sglang.test.ci.ci_register import register_amd_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/registered/kernels/ops/attention/test_mla_scatter_concat.py b/test/registered/kernels/ops/attention/test_mla_scatter_concat.py new file mode 100644 index 0000000000..fb2b380448 --- /dev/null +++ b/test/registered/kernels/ops/attention/test_mla_scatter_concat.py @@ -0,0 +1,100 @@ +"""MLA scatter and query concatenation parity.""" + +import unittest + +import torch + +from sglang.kernels.ops.attention.concat_mla import concat_mla_absorb_q +from sglang.kernels.ops.attention.set_mla_kv_concat_q import ( + can_use_set_mla_kv_concat_q, + can_use_set_mla_kv_concat_q_fp8, + set_mla_kv_concat_q, + set_mla_kv_concat_q_fp8, +) +from sglang.kernels.ops.attention.utils import concat_mla_absorb_q_general +from sglang.kernels.ops.kvcache.set_mla_kv_buffer import set_mla_kv_buffer +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +NOPE_DIM = 512 + +ROPE_DIM = 64 + +MLA_DIM = NOPE_DIM + ROPE_DIM + +MLA_PAGES = 256 + + +def _make_mla_inputs(batch_size, num_heads, seed): + generator = torch.Generator(device="cuda").manual_seed(seed) + + def randn(*shape): + return ( + torch.randn(*shape, generator=generator, device="cuda", dtype=torch.float32) + .mul(0.1) + .to(torch.bfloat16) + ) + + pool = randn(MLA_PAGES, MLA_DIM) + latent = randn(batch_size, MLA_DIM) + query = randn(batch_size, num_heads, MLA_DIM) + loc = ( + torch.randperm(MLA_PAGES - 1, generator=generator, device="cuda")[:batch_size] + + 1 + ).to(torch.int64) + return ( + pool, + loc, + latent[:, :NOPE_DIM], + latent[:, NOPE_DIM:], + query[..., :NOPE_DIM], + query[..., NOPE_DIM:], + ) + + +class TestMLAScatterConcat(CustomTestCase): + def test_mla_scatter_concat_bf16_and_fp8(self): + batch_size, num_heads = 64, 8 + pool, loc, k_nope, k_rope, q_nope, q_rope = _make_mla_inputs( + batch_size, num_heads, seed=0 + ) + + if not can_use_set_mla_kv_concat_q(NOPE_DIM * 2, ROPE_DIM * 2): + self.skipTest("fused MLA scatter+concat requires SM90+") + pool_ref = pool.clone() + query = set_mla_kv_concat_q(pool, loc, k_nope, k_rope, q_nope, q_rope) + set_mla_kv_buffer(pool_ref, loc, k_nope, k_rope) + query_ref = concat_mla_absorb_q(q_nope, q_rope) + self.assertTrue(torch.equal(pool, pool_ref)) + self.assertTrue(torch.equal(query, query_ref)) + + if not can_use_set_mla_kv_concat_q_fp8(): + self.skipTest("fused FP8 MLA scatter+concat requires SM90+") + fp8_pool = torch.zeros( + MLA_PAGES, MLA_DIM, device="cuda", dtype=torch.float8_e4m3fn + ) + fp8_ref = fp8_pool.clone() + fp8_query = set_mla_kv_concat_q_fp8( + fp8_pool, loc, k_nope, k_rope, q_nope, q_rope + ) + row = torch.cat([k_nope, k_rope], dim=-1).to(torch.float8_e4m3fn) + fp8_ref[loc] = row + fp8_query_ref = concat_mla_absorb_q_general(q_nope, q_rope).to( + torch.float8_e4m3fn + ) + self.assertTrue( + torch.equal(fp8_pool.view(torch.uint8), fp8_ref.view(torch.uint8)) + ) + self.assertTrue( + torch.equal( + fp8_query.view(torch.uint8), + fp8_query_ref.view(torch.uint8), + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/attention/test_replayssm_ring_fold.py b/test/registered/kernels/ops/attention/test_replayssm_ring_fold.py new file mode 100644 index 0000000000..716b3eb1c7 --- /dev/null +++ b/test/registered/kernels/ops/attention/test_replayssm_ring_fold.py @@ -0,0 +1,142 @@ +"""ReplaySSM ring folding parity.""" + +import unittest + +import torch + +from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( + fused_sigmoid_gating_delta_rule_update, +) +from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( + commit_kda_replayssm_spec, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +class TestReplaySSMRingFold(CustomTestCase): + def test_replayssm_ring_fold(self): + batch_size, num_steps = 8, 4 + num_value_heads, num_key_heads = 8, 2 + key_dim = value_dim = 128 + ring_size = 16 + torch.manual_seed(6) + + q = torch.randn( + batch_size, + num_steps, + num_key_heads, + key_dim, + device="cuda", + ) + k = torch.randn_like(q) + v = torch.randn( + batch_size, + num_steps, + num_value_heads, + value_dim, + device="cuda", + ) + a = torch.randn( + batch_size, + num_steps, + num_value_heads, + key_dim, + device="cuda", + ) + b = torch.randn(batch_size, num_steps, num_value_heads, device="cuda") + a_log = torch.randn(num_value_heads, device="cuda") + dt_bias = torch.randn(num_value_heads, key_dim, device="cuda") + slots = torch.arange(1, batch_size + 1, device="cuda", dtype=torch.int32) + slots[-1] = -1 + num_slots = batch_size + 1 + state = torch.randn( + num_slots, + num_value_heads, + value_dim, + key_dim, + device="cuda", + ) + intermediate = torch.zeros( + num_slots, + num_steps, + num_value_heads, + value_dim, + key_dim, + device="cuda", + ) + raw_v = torch.zeros( + num_slots, + num_value_heads, + ring_size, + value_dim, + device="cuda", + ) + raw_k = torch.zeros( + num_slots, + num_key_heads, + ring_size, + key_dim, + device="cuda", + ) + gate = torch.zeros_like(raw_v) + beta = torch.zeros( + num_slots, + num_value_heads, + ring_size, + device="cuda", + ) + + fused_sigmoid_gating_delta_rule_update( + A_log=a_log, + a=a, + dt_bias=dt_bias, + softplus_beta=1.0, + softplus_threshold=20.0, + q=q, + k=k, + v=v, + b=b, + initial_state_source=state, + initial_state_indices=slots, + scale=key_dim**-0.5, + use_qk_l2norm_in_kernel=True, + is_kda=True, + lower_bound=-5.0, + disable_state_update=True, + intermediate_states_buffer=intermediate, + intermediate_state_indices=slots, + cache_steps=num_steps, + cache_ring=True, + replayssm_rawv=raw_v, + replayssm_rawk=raw_k, + replayssm_g=gate, + replayssm_beta=beta, + ) + checkpoint = state.clone() + commit_kda_replayssm_spec( + checkpoint, + raw_v, + raw_k, + gate, + beta, + slots, + torch.full((batch_size,), num_steps, device="cuda", dtype=torch.int32), + max_cache_len=ring_size, + num_k_heads=num_key_heads, + use_qk_l2norm_in_kernel=True, + null_block_id=-1, + ) + for slot in slots[:-1].tolist(): + expected = intermediate[slot, num_steps - 1] + actual = checkpoint[slot] + relative_error = ( + actual - expected + ).abs().max() / expected.abs().max().clamp_min(1e-6) + self.assertLess(relative_error.item(), 1e-3) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/attention/test_vision_rope.py b/test/registered/kernels/ops/attention/test_vision_rope.py new file mode 100644 index 0000000000..e504fae887 --- /dev/null +++ b/test/registered/kernels/ops/attention/test_vision_rope.py @@ -0,0 +1,65 @@ +"""Vision complex RoPE parity and input constraints.""" + +import unittest + +import torch + +from sglang.kernels.ops.attention.vision_rope import ( + apply_fused_qk_complex_rope, + apply_fused_qk_complex_rope_inplace, + prepare_fused_qk_complex_rope_inplace, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +class TestVisionRoPE(CustomTestCase): + def test_vision_rope(self): + torch.manual_seed(4) + qkv = torch.randn(480, 3, 12, 128, device="cuda", dtype=torch.bfloat16) + q, k, _ = qkv.unbind(1) + angles = torch.randn(480, 64, device="cuda") + freqs = torch.polar(torch.ones_like(angles), angles) + freqs_expanded = freqs.unsqueeze(-2) + + def reference(x): + value = torch.view_as_complex(x.float().view(*x.shape[:-1], -1, 2)) + return torch.view_as_real(value * freqs_expanded).flatten(-2).type_as(x) + + actual_q, actual_k = apply_fused_qk_complex_rope(q, k, freqs) + atol = 2 * torch.finfo(torch.bfloat16).eps + torch.testing.assert_close(actual_q, reference(q), rtol=0, atol=atol) + torch.testing.assert_close(actual_k, reference(k), rtol=0, atol=atol) + + def test_vision_rope_inplace(self): + # VisionAttention hands the applier contiguous q/k, which is what the + # in-place kernel requires; mirror that rather than qkv.unbind views. + for dtype in (torch.bfloat16, torch.float16): + torch.manual_seed(4) + q = torch.randn(480, 12, 128, device="cuda", dtype=dtype) + k = torch.randn(480, 12, 128, device="cuda", dtype=dtype) + angles = torch.randn(480, 64, device="cuda") + freqs = torch.polar(torch.ones_like(angles), angles) + freqs_expanded = freqs.unsqueeze(-2) + + def reference(x): + value = torch.view_as_complex(x.float().view(*x.shape[:-1], -1, 2)) + return torch.view_as_real(value * freqs_expanded).flatten(-2).type_as(x) + + expected_q, expected_k = reference(q), reference(k) + prepared = prepare_fused_qk_complex_rope_inplace(freqs) + actual_q, actual_k = apply_fused_qk_complex_rope_inplace(q, k, prepared) + + atol = 2 * torch.finfo(dtype).eps + torch.testing.assert_close(actual_q, expected_q, rtol=0, atol=atol) + torch.testing.assert_close(actual_k, expected_k, rtol=0, atol=atol) + + def test_vision_rope_inplace_rejects_non_complex_frequencies(self): + with self.assertRaises(ValueError): + prepare_fused_qk_complex_rope_inplace(torch.randn(8, 64, device="cuda")) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/communication/test_collectives.py b/test/registered/kernels/ops/communication/test_collectives.py index 4e58b28ea6..d326bbe7d6 100644 --- a/test/registered/kernels/ops/communication/test_collectives.py +++ b/test/registered/kernels/ops/communication/test_collectives.py @@ -13,7 +13,6 @@ from sglang.kernels.ops.attention import attn_res from sglang.kernels.ops.communication import all_reduce_residual as all_reduce from sglang.kernels.ops.communication import ( gemm_ag, - gemm_ar, sp_collective, ) from sglang.kernels.ops.communication.mp import register_comm_cleanup @@ -28,7 +27,6 @@ register_cuda_ci(est_time=34, stage="base-c", runner_config="4-gpu-b200") register_cuda_ci(est_time=480, stage="nightly", runner_config="8-gpu-b200") _HIDDEN_SIZE = 7168 -_GEMM_AR_K_TOTAL = 12288 _GEMM_AG_WORLD_SIZE = 8 _MB = 1024 * 1024 _SP_TUNING = sp_collective.Tuning(num_blocks=1, block_size=256) @@ -80,18 +78,6 @@ def _init_comm(): return comm -@cache_once -def _init_gemm_ar(): - cpu_group, _ = _init_world() - world_size = dist.get_world_size() - gemm_ar.init( - world_size=world_size, - rank=dist.get_rank(), - group=cpu_group, - k=_GEMM_AR_K_TOTAL // world_size, - ) - - def _symmetric_tensor(shape): from torch._C._distributed_c10d import _SymmetricMemory @@ -248,29 +234,6 @@ def test_gemm_all_gather(): torch.testing.assert_close(output, expected, rtol=3e-2, atol=3e-2) -@torch.inference_mode() -def test_gemm_all_reduce(): - _require_sm100() - _init_gemm_ar() - rank, world_size = dist.get_rank(), dist.get_world_size() - local_k = _GEMM_AR_K_TOTAL // world_size - generator = torch.Generator().manual_seed(40 + rank) - x = torch.randn(1, local_k, generator=generator).to( - device=_device(), dtype=torch.bfloat16 - ) - weight = torch.randn(gemm_ar.N, local_k, generator=generator).to( - device=_device(), dtype=torch.bfloat16 - ) - expected = (x.float() @ weight.float().t()).to(torch.bfloat16).float() - _, nccl_group = _init_world() - dist.all_reduce(expected, group=nccl_group) - - output = gemm_ar.o_proj_gemm_ar(x, weight) - torch.cuda.synchronize() - bad = ((output.float() - expected).abs() > 0.05 + 0.02 * expected.abs()).sum() - assert bad.item() <= output.numel() / 1000 - - @torch.inference_mode() def test_attention_residual_direct_all_gather(): _require_sm100() @@ -343,7 +306,6 @@ def _precompile(num_gpus): for world_size in num_gpus: all_reduce._jit_module(world_size) sp_collective._jit_module(world_size) - gemm_ar._jit_module(_GEMM_AR_K_TOTAL // world_size, world_size) if _GEMM_AG_WORLD_SIZE in num_gpus: gemm_ag._jit_module() attn_res._jit_fused_tma_module(4, 1, 200) diff --git a/test/registered/kernels/ops/quantization/test_diffusion_nvfp4_scaled_mm.py b/test/registered/kernels/ops/diffusion/test_nvfp4_scaled_mm.py similarity index 100% rename from test/registered/kernels/ops/quantization/test_diffusion_nvfp4_scaled_mm.py rename to test/registered/kernels/ops/diffusion/test_nvfp4_scaled_mm.py diff --git a/test/registered/kernels/ops/dllm/test_gemma4_sampling_cuda.py b/test/registered/kernels/ops/dllm/test_gemma4_sampling_cuda.py deleted file mode 100644 index 30e2d2c1d4..0000000000 --- a/test/registered/kernels/ops/dllm/test_gemma4_sampling_cuda.py +++ /dev/null @@ -1,127 +0,0 @@ -"""CUDA numerical checks for DiffusionGemma's inference optimizations.""" - -import unittest - -import torch - -from sglang.kernels.ops.layernorm.gemma4_fused_ops import gemma_qkv_rmsnorm -from sglang.srt.dllm.algorithm.gemma4_renoise import ( - _compiled_denoiser_statistics, - _denoiser_statistics, - _sample_denoiser, -) -from sglang.srt.runtime_context import get_context, get_parallel -from sglang.test.ci.ci_register import register_cuda_ci - -register_cuda_ci(est_time=35, stage="base-b-kernel-unit", runner_config="1-gpu-large") - - -@unittest.skipUnless(torch.cuda.is_available(), "CUDA is required") -class TestGemma4SamplingCUDA(unittest.TestCase): - def test_full_vocabulary_statistics_and_sampling(self): - generator = torch.Generator(device="cuda").manual_seed(123) - for batch_size in (1, 3): - with self.subTest(batch_size=batch_size): - logits = torch.randn( - batch_size, 16, 262144, device="cuda", generator=generator - ) - temperatures = torch.linspace(0.4, 0.8, batch_size, device="cuda") - expected = _denoiser_statistics(logits, temperatures) - actual = _compiled_denoiser_statistics(logits, temperatures) - for got, want in zip(actual, expected): - torch.testing.assert_close(got, want, rtol=2e-5, atol=2e-6) - a = torch.Generator(device="cuda").manual_seed(42) - b = torch.Generator(device="cuda").manual_seed(42) - probabilities = actual[0].reshape(-1, logits.shape[-1]) - torch.testing.assert_close( - _sample_denoiser(probabilities, a), - torch.multinomial(probabilities, 1, generator=b).squeeze(-1), - ) - torch.testing.assert_close(a.get_state(), b.get_state()) - - def test_expert_activation_matches_checkpoint(self): - from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import ( - fused_experts_impl, - ) - - # The runner uses FP16 or BF16 intermediates, even for FP32 inputs. - for dtype in (torch.float16, torch.bfloat16): - hidden = (torch.arange(-32, 32, device="cuda") / 8).to(dtype) - hidden = hidden.repeat(3, 1) - eye = torch.eye(64, device="cuda", dtype=dtype) - w1 = torch.stack((torch.cat((eye, eye)), torch.cat((-eye, eye)))) - w2 = torch.stack((eye, eye)) - ids = torch.tensor([[0, 1]], device="cuda", dtype=torch.int32).repeat(3, 1) - weights = torch.tensor([[0.25, 0.75]], device="cuda").repeat(3, 1) - for activation, approximate in (("gelu_tanh", "tanh"), ("gelu", "none")): - with ( - self.subTest(dtype=dtype, activation=activation), - get_context().override_server_args( - enable_deterministic_inference=False - ), - get_parallel().override(tp_group=None), - ): - actual = fused_experts_impl( - hidden, - w1, - w2, - weights, - ids, - activation=activation, - filter_expert=False, - ) - x = hidden.float() - left = ( - torch.nn.functional.gelu(x, approximate=approximate) * x - ).to(dtype) - right = ( - torch.nn.functional.gelu(-x, approximate=approximate) * x - ).to(dtype) - expected = (left * 0.25 + right * 0.75).to(dtype) - tolerance = 2 * torch.finfo(dtype).eps - torch.testing.assert_close( - actual, expected, rtol=tolerance, atol=tolerance - ) - - def test_qkv_norm_with_full_and_sliding_head_dimensions(self): - for num_q, num_kv, head_dim in ((8, 4, 256), (8, 1, 512)): - with self.subTest(head_dim=head_dim): - packed = torch.randn( - 16, - (num_q + 2 * num_kv) * head_dim, - device="cuda", - dtype=torch.bfloat16, - ) - q, k, v = packed.split( - [num_q * head_dim, num_kv * head_dim, num_kv * head_dim], dim=-1 - ) - qw = torch.randn(head_dim, device="cuda", dtype=torch.bfloat16) - kw = torch.randn_like(qw) - expected = [] - for x, heads, weight in ( - (q, num_q, qw), - (k, num_kv, kw), - (v, num_kv, 1.0), - ): - x = x.reshape(16, heads, head_dim).float() - normalized = x * torch.rsqrt( - x.square().mean(dim=-1, keepdim=True) + 1e-6 - ) - expected.append((normalized * weight).to(packed.dtype).flatten(1)) - gemma_qkv_rmsnorm( - q, - k, - v, - qw, - kw, - num_q_heads=num_q, - num_kv_heads=num_kv, - head_dim=head_dim, - eps=1e-6, - ) - for got, want in zip((q, k, v), expected): - torch.testing.assert_close(got, want, rtol=8e-3, atol=8e-3) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/kernels/ops/elementwise/test_add3.py b/test/registered/kernels/ops/elementwise/test_add3.py new file mode 100644 index 0000000000..edb78dd38d --- /dev/null +++ b/test/registered/kernels/ops/elementwise/test_add3.py @@ -0,0 +1,26 @@ +"""Bit-exact three-input addition.""" + +import unittest + +import torch + +from sglang.kernels.ops.elementwise import add3 +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=5, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +class TestAdd3(CustomTestCase): + def test_add3_bit_exact(self): + torch.manual_seed(0) + tensors = [ + torch.randn(9, 112, device="cuda", dtype=torch.bfloat16) for _ in range(3) + ] + actual = add3.add3(*tensors, prefetch_bc=True) + expected = (tensors[0] + tensors[1]) + tensors[2] + self.assertTrue(torch.equal(actual, expected)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/attention/test_inkling_row_scale.py b/test/registered/kernels/ops/elementwise/test_row_scale.py similarity index 65% rename from test/registered/kernels/ops/attention/test_inkling_row_scale.py rename to test/registered/kernels/ops/elementwise/test_row_scale.py index 68b5febadc..41bd1aebb8 100644 --- a/test/registered/kernels/ops/attention/test_inkling_row_scale.py +++ b/test/registered/kernels/ops/elementwise/test_row_scale.py @@ -5,14 +5,14 @@ including on the row-strided qkvr-slice layouts.""" import pytest import torch -from sglang.kernels.ops.attention.inkling_row_scale import row_scale_bf16 from sglang.kernels.ops.attention.log_scaling_tau import ( _apply_log_scaling_tau_kernel, ) +from sglang.kernels.ops.elementwise.row_scale import row_scale_bf16 from sglang.test.ci.ci_register import register_cuda_ci -register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") -register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="4-gpu-b200") +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="4-gpu-b200") def _triton_ref(x2d, tau): @@ -44,26 +44,6 @@ def test_row_scale_bitexact(rows, inner, strided): assert torch.equal(out, ref) -@pytest.mark.parametrize("rows", [1, 3, 200, 4096]) -@pytest.mark.parametrize("inner", [8, 256, 16384]) -@pytest.mark.parametrize("strided", [False, True]) -def test_row_compact_bitexact(rows, inner, strided): - """The tau-less compaction flavor (kHasTau=false) must reproduce - .contiguous() exactly on the same strided layouts row_scale handles -- - no other test exercises run_compact.""" - from sglang.kernels.ops.attention.inkling_row_scale import row_compact_bf16 - - torch.manual_seed(rows + inner) - if strided: - packed = torch.randn(rows, inner + 40, device="cuda", dtype=torch.bfloat16) - x = packed[:, 8 : 8 + inner] - else: - x = torch.randn(rows, inner, device="cuda", dtype=torch.bfloat16) - out = row_compact_bf16(x) - assert out.is_contiguous() - assert torch.equal(out, x.contiguous()) - - def test_dispatch_through_apply_log_scaling_tau(): from sglang.kernels.ops.attention.log_scaling_tau import apply_log_scaling_tau diff --git a/test/registered/kernels/ops/gemm/test_dsv4_wo_a.py b/test/registered/kernels/ops/gemm/test_dsv4_wo_a.py new file mode 100644 index 0000000000..8600b72665 --- /dev/null +++ b/test/registered/kernels/ops/gemm/test_dsv4_wo_a.py @@ -0,0 +1,85 @@ +"""MXFP8 epilogues stay bitwise identical to FlashInfer's standalone quantizer.""" + +import sys + +import pytest +import torch +from flashinfer import mxfp8_quantize + +from sglang.kernels.ops.gemm.dsv4_wo_a import ( + _quantize_partial, + _wo_a_reduce, + wo_a_bf16_small_batch, + wo_a_bf16_small_batch_mxfp8, +) +from sglang.srt.runtime_context import get_platform +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available() + or torch.version.cuda is None + or not get_platform().is_blackwell, + reason="the MXFP8 reference quantizer is Blackwell-only", +) + +DEVICE = "cuda" + +HIDDEN = 5120 + +STREAMS = 4 + + +def test_wo_a_partial_quant_matches_quantize(): + torch.manual_seed(0) + for rows in range(2, 9): + for magnitude in (0.0, 1e-37, 1e-7, 1.0, 448.0, 1e10): + partial = torch.randn(8, rows, 2, 1024, device=DEVICE) * magnitude + bf16 = torch.empty(rows, 2048, dtype=torch.bfloat16, device=DEVICE) + _wo_a_reduce[(rows * 8,)](partial, bf16, rows * 2048, num_warps=4) + expected_q, expected_s = mxfp8_quantize(bf16, True, alignment=32) + actual_q, actual_s = _quantize_partial(partial) + torch.testing.assert_close( + actual_q.view(torch.uint8), + expected_q.view(torch.uint8), + rtol=0, + atol=0, + ) + torch.testing.assert_close(actual_s, expected_s, rtol=0, atol=0) + + x = torch.randn(rows, 64, 512, device=DEVICE, dtype=torch.bfloat16)[ + :, :16 + ].view(rows, 2, 4096) + wo_a = torch.randn(2, 1024, 4096, device=DEVICE, dtype=torch.bfloat16) * 0.02 + bf16 = wo_a_bf16_small_batch(x, wo_a).flatten(1) + q, s = wo_a_bf16_small_batch_mxfp8(x, wo_a) + expected_q, expected_s = mxfp8_quantize(bf16, True, alignment=32) + torch.testing.assert_close( + q.view(torch.uint8), expected_q.view(torch.uint8), rtol=0, atol=0 + ) + torch.testing.assert_close(s, expected_s, rtol=0, atol=0) + + partial = torch.randn(8, 6, 2, 1024, device=DEVICE) + _quantize_partial(partial) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + q, s = _quantize_partial(partial) + for _ in range(3): + partial.normal_() + # The replay must regenerate scale padding as well as live rows. + s.fill_(255) + graph.replay() + bf16 = torch.empty(6, 2048, dtype=torch.bfloat16, device=DEVICE) + _wo_a_reduce[(48,)](partial, bf16, 6 * 2048, num_warps=4) + eq, es = mxfp8_quantize(bf16, True, alignment=32) + torch.testing.assert_close( + q.view(torch.uint8), eq.view(torch.uint8), rtol=0, atol=0 + ) + torch.testing.assert_close(s, es, rtol=0, atol=0) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__])) diff --git a/test/registered/kernels/ops/gemm/test_fp8_kernel.py b/test/registered/kernels/ops/gemm/test_fp8_kernel.py new file mode 100644 index 0000000000..3042fc7925 --- /dev/null +++ b/test/registered/kernels/ops/gemm/test_fp8_kernel.py @@ -0,0 +1,46 @@ +import unittest + +import torch + +from sglang.kernels.ops.gemm.fp8_kernel import w8a8_block_fp8_matmul +from sglang.srt.utils import is_cuda, is_xpu +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kernels.fp8 import TestFP8Base + +register_cuda_ci(est_time=5, stage="base-b", runner_config="1-gpu-large") + + +_is_cuda = is_cuda() +_is_xpu = is_xpu() + + +class TestW8A8BlockFP8Matmul(TestFP8Base): + def test_w8a8_block_fp8_matmul(self): + if _is_cuda and torch.cuda.get_device_capability()[0] < 9: + return + elif _is_xpu: + # XPU doesn't provide traditional capability info like CUDA + pass + else: + return + + A, A_quant_gt, A_scale_gt = self._make_A( + M=self.M, K=self.K, group_size=self.group_size, out_dtype=self.quant_type + ) + B, B_quant_gt, B_scale_gt = self._make_B( + K=self.K, N=self.N, group_size=self.group_size, out_dtype=self.quant_type + ) + C_gt = A.to(self.output_type) @ B.to(self.output_type) + C = w8a8_block_fp8_matmul( + A=A_quant_gt, + B=B_quant_gt.T.contiguous(), + As=A_scale_gt, + Bs=B_scale_gt.T.contiguous(), + block_size=[128, 128], + output_dtype=self.output_type, + ) + torch.testing.assert_close(C, C_gt, atol=0.5, rtol=1e-4) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/quantization/test_gptq_marlin.py b/test/registered/kernels/ops/gemm/test_gptq_marlin.py similarity index 97% rename from test/registered/kernels/ops/quantization/test_gptq_marlin.py rename to test/registered/kernels/ops/gemm/test_gptq_marlin.py index eac5bbc845..556ccb897b 100644 --- a/test/registered/kernels/ops/quantization/test_gptq_marlin.py +++ b/test/registered/kernels/ops/gemm/test_gptq_marlin.py @@ -4,7 +4,7 @@ import pytest import torch from sgl_kernel.scalar_type import scalar_types -from sglang.kernels.ops.quantization.gptq_marlin import gptq_marlin_gemm +from sglang.kernels.ops.gemm.gptq_marlin import gptq_marlin_gemm from sglang.srt.layers.quantization.marlin_utils import ( marlin_make_workspace, ) diff --git a/test/registered/kernels/ops/hyperconnection/test_hc_mix_triton.py b/test/registered/kernels/ops/gemm/test_hc_mix.py similarity index 98% rename from test/registered/kernels/ops/hyperconnection/test_hc_mix_triton.py rename to test/registered/kernels/ops/gemm/test_hc_mix.py index f9b8159afd..46021423a4 100644 --- a/test/registered/kernels/ops/hyperconnection/test_hc_mix_triton.py +++ b/test/registered/kernels/ops/gemm/test_hc_mix.py @@ -4,7 +4,7 @@ import pytest import torch import torch.nn.functional as F -from sglang.srt.layers.hc_mix_triton import ( +from sglang.kernels.ops.gemm.hc_mix import ( _FUSED_MIX_MAX_ROWS, fused_hc_mix, fused_hc_mix_supported, diff --git a/test/registered/gemm/test_hopper_bf16_gemv.py b/test/registered/kernels/ops/gemm/test_hopper_bf16_gemv.py similarity index 100% rename from test/registered/gemm/test_hopper_bf16_gemv.py rename to test/registered/kernels/ops/gemm/test_hopper_bf16_gemv.py diff --git a/test/registered/kernels/ops/attention/test_inkling_rel_proj.py b/test/registered/kernels/ops/gemm/test_inkling_rel_proj.py similarity index 97% rename from test/registered/kernels/ops/attention/test_inkling_rel_proj.py rename to test/registered/kernels/ops/gemm/test_inkling_rel_proj.py index 3f90b250da..0c9d781730 100644 --- a/test/registered/kernels/ops/attention/test_inkling_rel_proj.py +++ b/test/registered/kernels/ops/gemm/test_inkling_rel_proj.py @@ -6,7 +6,7 @@ production strided-r layout and contiguous inputs.""" import pytest import torch -from sglang.kernels.ops.attention.inkling_rel_proj import rel_proj_small_t +from sglang.kernels.ops.gemm.inkling_rel_proj import rel_proj_small_t from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") diff --git a/test/registered/gemm/test_linear_bf16_fp32_hpc.py b/test/registered/kernels/ops/gemm/test_linear_bf16_fp32_hpc.py similarity index 98% rename from test/registered/gemm/test_linear_bf16_fp32_hpc.py rename to test/registered/kernels/ops/gemm/test_linear_bf16_fp32_hpc.py index df49c93f2f..4143371c59 100644 --- a/test/registered/gemm/test_linear_bf16_fp32_hpc.py +++ b/test/registered/kernels/ops/gemm/test_linear_bf16_fp32_hpc.py @@ -10,7 +10,7 @@ import unittest import torch -from sglang.kernels.ops.attention.dsv4.gemm import ( +from sglang.kernels.ops.gemm.bf16_fp32 import ( _hpc_gemm_bf16xfp32_available, _linear_bf16_fp32_hpc, hpc_bf16xfp32_gemm_enabled, diff --git a/test/registered/kernels/ops/quantization/test_nvfp4_marlin.py b/test/registered/kernels/ops/gemm/test_nvfp4_marlin.py similarity index 100% rename from test/registered/kernels/ops/quantization/test_nvfp4_marlin.py rename to test/registered/kernels/ops/gemm/test_nvfp4_marlin.py diff --git a/test/registered/kernels/ops/gemm/test_tiny_gemm_variants.py b/test/registered/kernels/ops/gemm/test_tiny_gemm_variants.py new file mode 100644 index 0000000000..70bc7e7712 --- /dev/null +++ b/test/registered/kernels/ops/gemm/test_tiny_gemm_variants.py @@ -0,0 +1,35 @@ +"""Tiny GEMM variant parity.""" + +import unittest + +import torch + +from sglang.kernels.ops.gemm.tiny_gemm import tiny_gemm_bf16 +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=5, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +class TestTinyGemmVariants(CustomTestCase): + def test_tiny_gemm_variants(self): + """Both K3 gate-projection shapes, one per kernel variant: N=144 is the + tiny dimension for the first, K=128 for the second.""" + torch.manual_seed(2) + x = torch.randn(2, 7168, device="cuda", dtype=torch.bfloat16) / 8 + weight = torch.randn(144, 7168, device="cuda", dtype=torch.bfloat16) / 8 + actual = tiny_gemm_bf16(x, weight, out_dtype=torch.float32) + torch.testing.assert_close( + actual.double(), x.double() @ weight.double().t(), rtol=1e-3, atol=1e-3 + ) + + x = torch.randn(7, 128, device="cuda", dtype=torch.bfloat16) / 4 + weight = torch.randn(1536, 128, device="cuda", dtype=torch.bfloat16) / 4 + actual = tiny_gemm_bf16(x, weight) + torch.testing.assert_close( + actual.double(), x.double() @ weight.double().t(), rtol=2e-2, atol=2e-2 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/quant/test_triton_scaled_mm.py b/test/registered/kernels/ops/gemm/test_triton_scaled_mm.py similarity index 98% rename from test/registered/quant/test_triton_scaled_mm.py rename to test/registered/kernels/ops/gemm/test_triton_scaled_mm.py index ab800fcba2..b94ef4402b 100644 --- a/test/registered/quant/test_triton_scaled_mm.py +++ b/test/registered/kernels/ops/gemm/test_triton_scaled_mm.py @@ -4,7 +4,7 @@ from typing import Optional import torch import torch.testing -from sglang.kernels.ops.quantization.fp8_kernel import triton_scaled_mm +from sglang.kernels.ops.gemm.fp8_kernel import triton_scaled_mm from sglang.srt.utils.common import get_device from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/registered/kernels/ops/kimi_k3/test_kimi_k3_prerequisite_ops.py b/test/registered/kernels/ops/kimi_k3/test_kimi_k3_prerequisite_ops.py deleted file mode 100644 index 6f47dc3d6a..0000000000 --- a/test/registered/kernels/ops/kimi_k3/test_kimi_k3_prerequisite_ops.py +++ /dev/null @@ -1,496 +0,0 @@ -"""Representative parity coverage for the lightweight Kimi-K3 prerequisites.""" - -import unittest - -import torch - -from sglang.kernels.ops.attention.concat_mla import concat_mla_absorb_q -from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( - fused_sigmoid_gating_delta_rule_update, -) -from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( - commit_kda_replayssm_spec, -) -from sglang.kernels.ops.attention.set_mla_kv_concat_q import ( - can_use_set_mla_kv_concat_q, - can_use_set_mla_kv_concat_q_fp8, - set_mla_kv_concat_q, - set_mla_kv_concat_q_fp8, -) -from sglang.kernels.ops.attention.utils import concat_mla_absorb_q_general -from sglang.kernels.ops.attention.vision_rope import ( - apply_fused_qk_complex_rope, - apply_fused_qk_complex_rope_inplace, - prepare_fused_qk_complex_rope_inplace, -) -from sglang.kernels.ops.elementwise import add3 -from sglang.kernels.ops.gemm.tiny_gemm import tiny_gemm_bf16 -from sglang.kernels.ops.kvcache.set_mla_kv_buffer import set_mla_kv_buffer -from sglang.kernels.ops.mm.process.image import ( - _normalize_and_patchify_torch, - normalize_and_patchify, -) -from sglang.kernels.ops.moe import moe_route_quant_fused -from sglang.kernels.ops.moe.moe_route_radix import route_radix -from sglang.kernels.ops.moe.moe_topk_sum import moe_topk_sum -from sglang.kernels.ops.quantization.per_token_group_quant import ( - per_token_group_quant, -) -from sglang.kernels.ops.sampling.top_p_renorm_triton import ( - top_p_renorm_probs_triton, -) -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.test_utils import CustomTestCase - -register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="1-gpu-large") - -NUM_EXPERTS = 896 -TOPK = 16 -NOPE_DIM = 512 -ROPE_DIM = 64 -MLA_DIM = NOPE_DIM + ROPE_DIM -MLA_PAGES = 256 - - -def _pack_topk_oracle(topk_ids, topk_weights): - """Pure-torch reference for the packed ids the fused route+quant kernel emits. - - FlashInfer's routed MoE reads one int32 per entry: the expert id in the high - half and the bf16 weight bits in the low half. - """ - weight_bits = ( - topk_weights.to(torch.bfloat16).view(torch.int16).to(torch.int32) & 0xFFFF - ) - return (topk_ids.to(torch.int32) << 16) | weight_bits - - -def _route_oracle( - scores, bias, topk, renormalize, routed_scaling_factor, apply_scale, sorted -): - """Pure-torch fp32 reference for route_radix. - - Deliberately independent of moe_fused_gate: that entry dispatches back to - route_radix whenever scoring is sigmoid with no shared experts, no expert - groups and no softcapping (moe_fused_gate.py, the covered() fast path), which - is exactly the configuration under test. - - Contract, from route_radix.cuh: bias participates in RANKING only and the - emitted weight stays bias-free; NaN is floored so it can never win; ties go to - the lower expert id; renormalize divides by the winners' sum (guarded to 1 when - that sum is non-positive) and only then is routed scaling applied; sorted=True - emits (biased desc, id asc) while sorted=False emits ascending expert id. - """ - s = torch.sigmoid(scores.float()) - biased = s + bias.float() - biased = torch.where(torch.isnan(biased), torch.full_like(biased, -1e30), biased) - # stable + descending: equal biased values keep ascending-id order - ranked = torch.argsort(biased, dim=-1, descending=True, stable=True)[:, :topk] - w = s.gather(1, ranked) - total = w.sum(-1, keepdim=True) - norm = torch.where(total > 0, total, torch.ones_like(total)) - if renormalize: - w = w / norm - if apply_scale: - w = w * routed_scaling_factor - if sorted: - return w, ranked.to(torch.int32) - by_id = ranked.argsort(dim=-1) - return w.gather(1, by_id), ranked.gather(1, by_id).to(torch.int32) - - -def _make_mla_inputs(batch_size, num_heads, seed): - generator = torch.Generator(device="cuda").manual_seed(seed) - - def randn(*shape): - return ( - torch.randn(*shape, generator=generator, device="cuda", dtype=torch.float32) - .mul(0.1) - .to(torch.bfloat16) - ) - - pool = randn(MLA_PAGES, MLA_DIM) - latent = randn(batch_size, MLA_DIM) - query = randn(batch_size, num_heads, MLA_DIM) - loc = ( - torch.randperm(MLA_PAGES - 1, generator=generator, device="cuda")[:batch_size] - + 1 - ).to(torch.int64) - return ( - pool, - loc, - latent[:, :NOPE_DIM], - latent[:, NOPE_DIM:], - query[..., :NOPE_DIM], - query[..., NOPE_DIM:], - ) - - -class TestKimiK3PrerequisiteOps(CustomTestCase): - def test_mla_scatter_concat_bf16_and_fp8(self): - batch_size, num_heads = 64, 8 - pool, loc, k_nope, k_rope, q_nope, q_rope = _make_mla_inputs( - batch_size, num_heads, seed=0 - ) - - if not can_use_set_mla_kv_concat_q(NOPE_DIM * 2, ROPE_DIM * 2): - self.skipTest("fused MLA scatter+concat requires SM90+") - pool_ref = pool.clone() - query = set_mla_kv_concat_q(pool, loc, k_nope, k_rope, q_nope, q_rope) - set_mla_kv_buffer(pool_ref, loc, k_nope, k_rope) - query_ref = concat_mla_absorb_q(q_nope, q_rope) - self.assertTrue(torch.equal(pool, pool_ref)) - self.assertTrue(torch.equal(query, query_ref)) - - if not can_use_set_mla_kv_concat_q_fp8(): - self.skipTest("fused FP8 MLA scatter+concat requires SM90+") - fp8_pool = torch.zeros( - MLA_PAGES, MLA_DIM, device="cuda", dtype=torch.float8_e4m3fn - ) - fp8_ref = fp8_pool.clone() - fp8_query = set_mla_kv_concat_q_fp8( - fp8_pool, loc, k_nope, k_rope, q_nope, q_rope - ) - row = torch.cat([k_nope, k_rope], dim=-1).to(torch.float8_e4m3fn) - fp8_ref[loc] = row - fp8_query_ref = concat_mla_absorb_q_general(q_nope, q_rope).to( - torch.float8_e4m3fn - ) - self.assertTrue( - torch.equal(fp8_pool.view(torch.uint8), fp8_ref.view(torch.uint8)) - ) - self.assertTrue( - torch.equal( - fp8_query.view(torch.uint8), - fp8_query_ref.view(torch.uint8), - ) - ) - - def test_replayssm_ring_fold(self): - batch_size, num_steps = 8, 4 - num_value_heads, num_key_heads = 8, 2 - key_dim = value_dim = 128 - ring_size = 16 - torch.manual_seed(6) - - q = torch.randn( - batch_size, - num_steps, - num_key_heads, - key_dim, - device="cuda", - ) - k = torch.randn_like(q) - v = torch.randn( - batch_size, - num_steps, - num_value_heads, - value_dim, - device="cuda", - ) - a = torch.randn( - batch_size, - num_steps, - num_value_heads, - key_dim, - device="cuda", - ) - b = torch.randn(batch_size, num_steps, num_value_heads, device="cuda") - a_log = torch.randn(num_value_heads, device="cuda") - dt_bias = torch.randn(num_value_heads, key_dim, device="cuda") - slots = torch.arange(1, batch_size + 1, device="cuda", dtype=torch.int32) - slots[-1] = -1 - num_slots = batch_size + 1 - state = torch.randn( - num_slots, - num_value_heads, - value_dim, - key_dim, - device="cuda", - ) - intermediate = torch.zeros( - num_slots, - num_steps, - num_value_heads, - value_dim, - key_dim, - device="cuda", - ) - raw_v = torch.zeros( - num_slots, - num_value_heads, - ring_size, - value_dim, - device="cuda", - ) - raw_k = torch.zeros( - num_slots, - num_key_heads, - ring_size, - key_dim, - device="cuda", - ) - gate = torch.zeros_like(raw_v) - beta = torch.zeros( - num_slots, - num_value_heads, - ring_size, - device="cuda", - ) - - fused_sigmoid_gating_delta_rule_update( - A_log=a_log, - a=a, - dt_bias=dt_bias, - softplus_beta=1.0, - softplus_threshold=20.0, - q=q, - k=k, - v=v, - b=b, - initial_state_source=state, - initial_state_indices=slots, - scale=key_dim**-0.5, - use_qk_l2norm_in_kernel=True, - is_kda=True, - lower_bound=-5.0, - disable_state_update=True, - intermediate_states_buffer=intermediate, - intermediate_state_indices=slots, - cache_steps=num_steps, - cache_ring=True, - replayssm_rawv=raw_v, - replayssm_rawk=raw_k, - replayssm_g=gate, - replayssm_beta=beta, - ) - checkpoint = state.clone() - commit_kda_replayssm_spec( - checkpoint, - raw_v, - raw_k, - gate, - beta, - slots, - torch.full((batch_size,), num_steps, device="cuda", dtype=torch.int32), - max_cache_len=ring_size, - num_k_heads=num_key_heads, - use_qk_l2norm_in_kernel=True, - null_block_id=-1, - ) - for slot in slots[:-1].tolist(): - expected = intermediate[slot, num_steps - 1] - actual = checkpoint[slot] - relative_error = ( - actual - expected - ).abs().max() / expected.abs().max().clamp_min(1e-6) - self.assertLess(relative_error.item(), 1e-3) - - def test_add3_bit_exact(self): - torch.manual_seed(0) - tensors = [ - torch.randn(9, 112, device="cuda", dtype=torch.bfloat16) for _ in range(3) - ] - actual = add3.add3(*tensors, prefetch_bc=True) - expected = (tensors[0] + tensors[1]) + tensors[2] - self.assertTrue(torch.equal(actual, expected)) - - def test_moe_auxiliary_kernels(self): - x = torch.randn(2, TOPK, 7168, device="cuda", dtype=torch.bfloat16) - out = torch.empty(2, 7168, device="cuda", dtype=torch.bfloat16) - self.assertIs(moe_topk_sum(x, out), out) - self.assertTrue(torch.equal(out, x.float().sum(1).to(torch.bfloat16))) - - def test_moe_route_and_quant(self): - torch.manual_seed(1) - scores = torch.randn(8, NUM_EXPERTS, device="cuda", dtype=torch.bfloat16) - bias = torch.randn(NUM_EXPERTS, device="cuda", dtype=torch.float32) - args = (scores, bias, TOPK, True, 2.5, True) - weights, ids = route_radix(*args, sorted=True) - # Oracle, NOT moe_fused_gate: for this exact configuration (sigmoid, no - # shared experts, no expert groups, no softcapping) moe_fused_gate - # dispatches straight back to route_radix, so using it as the reference - # compares the kernel with itself and cannot see a selection, tie-break, - # NaN, renormalize or scaling error. - ref_weights, ref_ids = _route_oracle(*args, sorted=True) - self.assertTrue(torch.equal(ids, ref_ids)) - # rtol is not 1e-6: the kernel computes sigmoid with __fdividef/__expf, - # whose last bits differ from torch's. The old self-comparison could - # afford atol=0; a real oracle cannot. - torch.testing.assert_close(weights, ref_weights, rtol=1e-5, atol=1e-6) - - if not moe_route_quant_fused.available(): - self.skipTest("fused route+quant kernel unavailable") - hidden = torch.randn(8, 3584, device="cuda", dtype=torch.bfloat16) - ref_weights, ref_ids = route_radix(*args, sorted=False) - ref_packed = _pack_topk_oracle(ref_ids, ref_weights) - ref_q, ref_scale = per_token_group_quant( - hidden, group_size=32, scale_ue8m0=True - ) - actual = moe_route_quant_fused.route_quant_fused( - scores, - bias, - hidden, - TOPK, - renormalize=True, - routed_scaling_factor=2.5, - apply_scale=True, - ) - weights, ids, packed, quantized, scale = actual - self.assertTrue(torch.equal(ids, ref_ids)) - self.assertTrue( - torch.equal(weights.view(torch.int32), ref_weights.view(torch.int32)) - ) - self.assertTrue(torch.equal(packed, ref_packed)) - self.assertTrue( - torch.equal(quantized.view(torch.uint8), ref_q.view(torch.uint8)) - ) - torch.testing.assert_close(scale, ref_scale, rtol=0, atol=0) - - def test_route_radix_ties_and_nan(self): - """The cases the self-comparison could not see. - - Exact ties: many experts share one biased value, so the winner set is only - determined by the lowest-id rule. NaN: floored, so a NaN expert must never - be selected while enough finite ones exist. Both run with renormalize and - scaling on and off, since those are applied in a fixed order. - """ - bias = torch.zeros(NUM_EXPERTS, device="cuda", dtype=torch.float32) - - tied = torch.full((4, NUM_EXPERTS), 0.25, device="cuda", dtype=torch.bfloat16) - # a handful of strict winners above the tied plateau, the rest exactly equal - tied[:, 300] = 2.0 - tied[:, 7] = 2.0 - tied[:, 800] = 1.5 - - nan_scores = torch.randn(4, NUM_EXPERTS, device="cuda", dtype=torch.bfloat16) - nan_scores[:, 100] = float("nan") - nan_scores[:, 500] = float("nan") - # make the NaN experts the ones that would otherwise win outright - nan_scores[:, 101] = 5.0 - - for name, scores in (("ties", tied), ("nan", nan_scores)): - for renormalize in (False, True): - for apply_scale in (False, True): - for sorted_ in (False, True): - args = (scores, bias, TOPK, renormalize, 2.5, apply_scale) - ids = route_radix(*args, sorted=sorted_)[1] - ref_ids = _route_oracle(*args, sorted=sorted_)[1] - tag = ( - f"{name} renorm={renormalize} " - f"scale={apply_scale} sorted={sorted_}" - ) - self.assertTrue(torch.equal(ids, ref_ids), msg=tag) - if name == "nan": - self.assertFalse( - bool(((ids == 100) | (ids == 500)).any()), - msg=f"{tag}: a NaN expert was selected", - ) - - def test_tiny_gemm_variants(self): - """Both K3 gate-projection shapes, one per kernel variant: N=144 is the - tiny dimension for the first, K=128 for the second.""" - torch.manual_seed(2) - x = torch.randn(2, 7168, device="cuda", dtype=torch.bfloat16) / 8 - weight = torch.randn(144, 7168, device="cuda", dtype=torch.bfloat16) / 8 - actual = tiny_gemm_bf16(x, weight, out_dtype=torch.float32) - torch.testing.assert_close( - actual.double(), x.double() @ weight.double().t(), rtol=1e-3, atol=1e-3 - ) - - x = torch.randn(7, 128, device="cuda", dtype=torch.bfloat16) / 4 - weight = torch.randn(1536, 128, device="cuda", dtype=torch.bfloat16) / 4 - actual = tiny_gemm_bf16(x, weight) - torch.testing.assert_close( - actual.double(), x.double() @ weight.double().t(), rtol=2e-2, atol=2e-2 - ) - - def test_top_p_renorm(self): - torch.manual_seed(3) - probs = torch.randn(3, 1024, device="cuda").softmax(-1) - top_p = torch.tensor([0.5, 0.8, 0.95], device="cuda") - sorted_probs = probs.sort(-1).values - cutoff = torch.searchsorted( - sorted_probs.cumsum(-1), (1 - top_p).unsqueeze(1) - ).squeeze(1) - cutoff.clamp_(max=probs.shape[1] - 1) - pivot = sorted_probs.gather(1, cutoff[:, None]) - expected = torch.where(probs >= pivot, probs, 0) - expected /= expected.sum(-1, keepdim=True) - torch.testing.assert_close( - top_p_renorm_probs_triton(probs, top_p), - expected, - rtol=2e-6, - atol=1e-8, - ) - - def test_vision_rope(self): - torch.manual_seed(4) - qkv = torch.randn(480, 3, 12, 128, device="cuda", dtype=torch.bfloat16) - q, k, _ = qkv.unbind(1) - angles = torch.randn(480, 64, device="cuda") - freqs = torch.polar(torch.ones_like(angles), angles) - freqs_expanded = freqs.unsqueeze(-2) - - def reference(x): - value = torch.view_as_complex(x.float().view(*x.shape[:-1], -1, 2)) - return torch.view_as_real(value * freqs_expanded).flatten(-2).type_as(x) - - actual_q, actual_k = apply_fused_qk_complex_rope(q, k, freqs) - atol = 2 * torch.finfo(torch.bfloat16).eps - torch.testing.assert_close(actual_q, reference(q), rtol=0, atol=atol) - torch.testing.assert_close(actual_k, reference(k), rtol=0, atol=atol) - - def test_vision_rope_inplace(self): - # VisionAttention hands the applier contiguous q/k, which is what the - # in-place kernel requires; mirror that rather than qkv.unbind views. - for dtype in (torch.bfloat16, torch.float16): - torch.manual_seed(4) - q = torch.randn(480, 12, 128, device="cuda", dtype=dtype) - k = torch.randn(480, 12, 128, device="cuda", dtype=dtype) - angles = torch.randn(480, 64, device="cuda") - freqs = torch.polar(torch.ones_like(angles), angles) - freqs_expanded = freqs.unsqueeze(-2) - - def reference(x): - value = torch.view_as_complex(x.float().view(*x.shape[:-1], -1, 2)) - return torch.view_as_real(value * freqs_expanded).flatten(-2).type_as(x) - - expected_q, expected_k = reference(q), reference(k) - prepared = prepare_fused_qk_complex_rope_inplace(freqs) - actual_q, actual_k = apply_fused_qk_complex_rope_inplace(q, k, prepared) - - atol = 2 * torch.finfo(dtype).eps - torch.testing.assert_close(actual_q, expected_q, rtol=0, atol=atol) - torch.testing.assert_close(actual_k, expected_k, rtol=0, atol=atol) - - def test_vision_rope_inplace_rejects_non_complex_frequencies(self): - with self.assertRaises(ValueError): - prepare_fused_qk_complex_rope_inplace(torch.randn(8, 64, device="cuda")) - - def test_normalize_and_patchify(self): - torch.manual_seed(5) - image = torch.randn(2, 3, 17, 19, device="cuda") - scale = torch.randn(1, 3, 1, 1, device="cuda") - bias = torch.randn(1, 3, 1, 1, device="cuda") - args = (image, scale, bias, 4, 20, 20) - actual = normalize_and_patchify( - args[0], - args[1], - args[2], - patch_size=args[3], - padded_height=args[4], - padded_width=args[5], - ) - expected = _normalize_and_patchify_torch( - args[0], - args[1], - args[2], - patch_size=args[3], - padded_height=args[4], - padded_width=args[5], - ) - torch.testing.assert_close(actual, expected, rtol=1e-2, atol=1e-2) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/kernels/ops/layernorm/test_fused_op_gpu_parity.py b/test/registered/kernels/ops/layernorm/test_fused_op_gpu_parity.py index 71d129f94c..5481df2062 100644 --- a/test/registered/kernels/ops/layernorm/test_fused_op_gpu_parity.py +++ b/test/registered/kernels/ops/layernorm/test_fused_op_gpu_parity.py @@ -11,7 +11,7 @@ import torch from sglang.kernels.spec import KernelBackend from sglang.test.ci.ci_register import register_cuda_ci -register_cuda_ci(est_time=12, stage="extra-a", runner_config="1-gpu-small") +register_cuda_ci(est_time=8, stage="extra-a", runner_config="1-gpu-small") # torch_compile is native under the hood; skip it here (compile time dominates) # -- it is exercised in the CPU lane. @@ -101,22 +101,6 @@ def test_gemma_fused_add_rmsnorm(dtype): _close(r, r_ref, dtype, f"gemma_fused_add {b.value} (residual)") -@pytest.mark.parametrize( - "op_attr", ["_SILU_AND_MUL", "_GELU_AND_MUL", "_GELU_TANH_AND_MUL"] -) -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -@pytest.mark.parametrize("shape", [(1, 8192), (128, 8192)]) -def test_gated_activation(op_attr, dtype, shape): - import sglang.kernels.ops.activation as act - - torch.manual_seed(0) - op = getattr(act, op_attr) - x = torch.randn(shape, dtype=dtype, device="cuda") - ref = op.forward_native(x) - for b in _eligible(op): - _close(op.forward(x, backend=b), ref, dtype, f"{op.op} {b.value}") - - if __name__ == "__main__": import sys diff --git a/test/registered/quant/test_fused_rms_fp8_group_quant.py b/test/registered/kernels/ops/layernorm/test_fused_rms_fp8_group_quant.py similarity index 100% rename from test/registered/quant/test_fused_rms_fp8_group_quant.py rename to test/registered/kernels/ops/layernorm/test_fused_rms_fp8_group_quant.py diff --git a/test/registered/kernels/ops/layernorm/test_gemma4_qkv_norm.py b/test/registered/kernels/ops/layernorm/test_gemma4_qkv_norm.py new file mode 100644 index 0000000000..023f87b2d9 --- /dev/null +++ b/test/registered/kernels/ops/layernorm/test_gemma4_qkv_norm.py @@ -0,0 +1,57 @@ +"""Packed QKV normalization parity for DiffusionGemma.""" + +import unittest + +import torch + +from sglang.kernels.ops.layernorm.gemma4_fused_ops import gemma_qkv_rmsnorm +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +@unittest.skipUnless(torch.cuda.is_available(), "CUDA is required") +class TestGemma4QKVNorm(CustomTestCase): + def test_qkv_norm_with_full_and_sliding_head_dimensions(self): + for num_q, num_kv, head_dim in ((8, 4, 256), (8, 1, 512)): + with self.subTest(head_dim=head_dim): + packed = torch.randn( + 16, + (num_q + 2 * num_kv) * head_dim, + device="cuda", + dtype=torch.bfloat16, + ) + q, k, v = packed.split( + [num_q * head_dim, num_kv * head_dim, num_kv * head_dim], dim=-1 + ) + qw = torch.randn(head_dim, device="cuda", dtype=torch.bfloat16) + kw = torch.randn_like(qw) + expected = [] + for x, heads, weight in ( + (q, num_q, qw), + (k, num_kv, kw), + (v, num_kv, 1.0), + ): + x = x.reshape(16, heads, head_dim).float() + normalized = x * torch.rsqrt( + x.square().mean(dim=-1, keepdim=True) + 1e-6 + ) + expected.append((normalized * weight).to(packed.dtype).flatten(1)) + gemma_qkv_rmsnorm( + q, + k, + v, + qw, + kw, + num_q_heads=num_q, + num_kv_heads=num_kv, + head_dim=head_dim, + eps=1e-6, + ) + for got, want in zip((q, k, v), expected): + torch.testing.assert_close(got, want, rtol=8e-3, atol=8e-3) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/layernorm/test_mxfp8_epilogue.py b/test/registered/kernels/ops/layernorm/test_mxfp8_epilogue.py index 295acb8fb2..0cc156719a 100644 --- a/test/registered/kernels/ops/layernorm/test_mxfp8_epilogue.py +++ b/test/registered/kernels/ops/layernorm/test_mxfp8_epilogue.py @@ -5,19 +5,12 @@ import sys import flashinfer import pytest import torch -from flashinfer import mxfp8_quantize -from sglang.kernels.ops.attention.dsv4.wo_a import ( - _quantize_partial, - _wo_a_reduce, - wo_a_bf16_small_batch, - wo_a_bf16_small_batch_mxfp8, -) from sglang.kernels.ops.layernorm.mxfp8_epilogue import rmsnorm_mxfp8 from sglang.srt.runtime_context import get_platform from sglang.test.ci.ci_register import register_cuda_ci -register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="4-gpu-b200") +register_cuda_ci(est_time=100, stage="base-b-kernel-unit", runner_config="4-gpu-b200") pytestmark = pytest.mark.skipif( not torch.cuda.is_available() @@ -101,53 +94,5 @@ def test_dynamic_graph(m, stride): check(x, w, got) -def test_wo_a_partial_quant_matches_quantize(): - torch.manual_seed(0) - for rows in range(2, 9): - for magnitude in (0.0, 1e-37, 1e-7, 1.0, 448.0, 1e10): - partial = torch.randn(8, rows, 2, 1024, device=DEVICE) * magnitude - bf16 = torch.empty(rows, 2048, dtype=torch.bfloat16, device=DEVICE) - _wo_a_reduce[(rows * 8,)](partial, bf16, rows * 2048, num_warps=4) - expected_q, expected_s = mxfp8_quantize(bf16, True, alignment=32) - actual_q, actual_s = _quantize_partial(partial) - torch.testing.assert_close( - actual_q.view(torch.uint8), - expected_q.view(torch.uint8), - rtol=0, - atol=0, - ) - torch.testing.assert_close(actual_s, expected_s, rtol=0, atol=0) - - x = torch.randn(rows, 64, 512, device=DEVICE, dtype=torch.bfloat16)[ - :, :16 - ].view(rows, 2, 4096) - wo_a = torch.randn(2, 1024, 4096, device=DEVICE, dtype=torch.bfloat16) * 0.02 - bf16 = wo_a_bf16_small_batch(x, wo_a).flatten(1) - q, s = wo_a_bf16_small_batch_mxfp8(x, wo_a) - expected_q, expected_s = mxfp8_quantize(bf16, True, alignment=32) - torch.testing.assert_close( - q.view(torch.uint8), expected_q.view(torch.uint8), rtol=0, atol=0 - ) - torch.testing.assert_close(s, expected_s, rtol=0, atol=0) - - partial = torch.randn(8, 6, 2, 1024, device=DEVICE) - _quantize_partial(partial) - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - q, s = _quantize_partial(partial) - for _ in range(3): - partial.normal_() - # The replay must regenerate scale padding as well as live rows. - s.fill_(255) - graph.replay() - bf16 = torch.empty(6, 2048, dtype=torch.bfloat16, device=DEVICE) - _wo_a_reduce[(48,)](partial, bf16, 6 * 2048, num_warps=4) - eq, es = mxfp8_quantize(bf16, True, alignment=32) - torch.testing.assert_close( - q.view(torch.uint8), eq.view(torch.uint8), rtol=0, atol=0 - ) - torch.testing.assert_close(s, es, rtol=0, atol=0) - - if __name__ == "__main__": sys.exit(pytest.main([__file__])) diff --git a/test/registered/kernels/ops/speculative/test_verify_commit_triton.py b/test/registered/kernels/ops/mamba/test_verify_commit_triton.py similarity index 100% rename from test/registered/kernels/ops/speculative/test_verify_commit_triton.py rename to test/registered/kernels/ops/mamba/test_verify_commit_triton.py diff --git a/test/registered/kernels/ops/memory/test_row_compact.py b/test/registered/kernels/ops/memory/test_row_compact.py new file mode 100644 index 0000000000..4f9aa9bcc9 --- /dev/null +++ b/test/registered/kernels/ops/memory/test_row_compact.py @@ -0,0 +1,38 @@ +"""The vectorized row-scale kernel must be BIT-identical to the triton +apply_log_scaling_tau kernel it replaces (same fp32 multiply + bf16 round), +including on the row-strided qkvr-slice layouts.""" + +import pytest +import torch + +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + + +@pytest.mark.parametrize("rows", [1, 3, 200, 4096]) +@pytest.mark.parametrize("inner", [8, 256, 16384]) +@pytest.mark.parametrize("strided", [False, True]) +def test_row_compact_bitexact(rows, inner, strided): + """The tau-less compaction flavor (kHasTau=false) must reproduce + .contiguous() exactly on the same strided layouts row_scale handles -- + no other test exercises run_compact.""" + from sglang.kernels.ops.memory.row_compact import row_compact_bf16 + + torch.manual_seed(rows + inner) + if strided: + packed = torch.randn(rows, inner + 40, device="cuda", dtype=torch.bfloat16) + x = packed[:, 8 : 8 + inner] + else: + x = torch.randn(rows, inner, device="cuda", dtype=torch.bfloat16) + out = row_compact_bf16(x) + assert out.is_contiguous() + assert torch.equal(out, x.contiguous()) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__])) diff --git a/test/registered/kernels/ops/mm/test_image.py b/test/registered/kernels/ops/mm/test_image.py new file mode 100644 index 0000000000..f1e66754c4 --- /dev/null +++ b/test/registered/kernels/ops/mm/test_image.py @@ -0,0 +1,44 @@ +"""Image normalization and patchification parity.""" + +import unittest + +import torch + +from sglang.kernels.ops.mm.process.image import ( + _normalize_and_patchify_torch, + normalize_and_patchify, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +class TestImagePatchify(CustomTestCase): + def test_normalize_and_patchify(self): + torch.manual_seed(5) + image = torch.randn(2, 3, 17, 19, device="cuda") + scale = torch.randn(1, 3, 1, 1, device="cuda") + bias = torch.randn(1, 3, 1, 1, device="cuda") + args = (image, scale, bias, 4, 20, 20) + actual = normalize_and_patchify( + args[0], + args[1], + args[2], + patch_size=args[3], + padded_height=args[4], + padded_width=args[5], + ) + expected = _normalize_and_patchify_torch( + args[0], + args[1], + args[2], + patch_size=args[3], + padded_height=args[4], + padded_width=args[5], + ) + torch.testing.assert_close(actual, expected, rtol=1e-2, atol=1e-2) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/quant/test_block_int8.py b/test/registered/kernels/ops/moe/test_block_int8.py similarity index 100% rename from test/registered/quant/test_block_int8.py rename to test/registered/kernels/ops/moe/test_block_int8.py diff --git a/test/registered/kernels/ops/moe/test_gemma4_expert_activation.py b/test/registered/kernels/ops/moe/test_gemma4_expert_activation.py new file mode 100644 index 0000000000..92044d7988 --- /dev/null +++ b/test/registered/kernels/ops/moe/test_gemma4_expert_activation.py @@ -0,0 +1,62 @@ +"""Expert activation parity for DiffusionGemma.""" + +import unittest + +import torch + +from sglang.srt.runtime_context import get_context, get_parallel +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +@unittest.skipUnless(torch.cuda.is_available(), "CUDA is required") +class TestGemma4ExpertActivation(CustomTestCase): + def test_expert_activation_matches_checkpoint(self): + from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import ( + fused_experts_impl, + ) + + # The runner uses FP16 or BF16 intermediates, even for FP32 inputs. + for dtype in (torch.float16, torch.bfloat16): + hidden = (torch.arange(-32, 32, device="cuda") / 8).to(dtype) + hidden = hidden.repeat(3, 1) + eye = torch.eye(64, device="cuda", dtype=dtype) + w1 = torch.stack((torch.cat((eye, eye)), torch.cat((-eye, eye)))) + w2 = torch.stack((eye, eye)) + ids = torch.tensor([[0, 1]], device="cuda", dtype=torch.int32).repeat(3, 1) + weights = torch.tensor([[0.25, 0.75]], device="cuda").repeat(3, 1) + for activation, approximate in (("gelu_tanh", "tanh"), ("gelu", "none")): + with ( + self.subTest(dtype=dtype, activation=activation), + get_context().override_server_args( + enable_deterministic_inference=False + ), + get_parallel().override(tp_group=None), + ): + actual = fused_experts_impl( + hidden, + w1, + w2, + weights, + ids, + activation=activation, + filter_expert=False, + ) + x = hidden.float() + left = ( + torch.nn.functional.gelu(x, approximate=approximate) * x + ).to(dtype) + right = ( + torch.nn.functional.gelu(-x, approximate=approximate) * x + ).to(dtype) + expected = (left * 0.25 + right * 0.75).to(dtype) + tolerance = 2 * torch.finfo(dtype).eps + torch.testing.assert_close( + actual, expected, rtol=tolerance, atol=tolerance + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/layernorm/test_gemma4_fused_routing.py b/test/registered/kernels/ops/moe/test_gemma4_fused_routing.py similarity index 97% rename from test/registered/kernels/ops/layernorm/test_gemma4_fused_routing.py rename to test/registered/kernels/ops/moe/test_gemma4_fused_routing.py index f7fa020b51..39ed7c148b 100644 --- a/test/registered/kernels/ops/layernorm/test_gemma4_fused_routing.py +++ b/test/registered/kernels/ops/moe/test_gemma4_fused_routing.py @@ -26,7 +26,7 @@ pytestmark = pytest.mark.skipif( @pytest.fixture(scope="module") def fused_routing(): - from sglang.kernels.ops.layernorm.gemma4_fused_ops import gemma4_fused_routing + from sglang.kernels.ops.moe.gemma4_routing import gemma4_fused_routing return gemma4_fused_routing diff --git a/test/registered/kernels/ops/moe/test_humming_fp8_dispatch.py b/test/registered/kernels/ops/moe/test_humming_fp8_dispatch.py index 7c4d5400be..9f46ba5c55 100644 --- a/test/registered/kernels/ops/moe/test_humming_fp8_dispatch.py +++ b/test/registered/kernels/ops/moe/test_humming_fp8_dispatch.py @@ -6,9 +6,7 @@ from unittest.mock import MagicMock, patch import torch -from sglang.kernels.ops.attention.dsv4.moe import ( - silu_and_mul_masked_post_quant, -) +from sglang.kernels.ops.moe.dsv4 import silu_and_mul_masked_post_quant from sglang.kernels.ops.moe.ep_moe_kernels import moe_permute_with_scale from sglang.srt.layers.moe.moe_runner import humming as humming_runner from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig @@ -196,8 +194,7 @@ class TestHummingFp8Dispatch(CustomTestCase): side_effect=AssertionError("SiLU fused path was not selected"), ) as fallback_activation, patch( - "sglang.kernels.ops.attention.dsv4.moe." - "silu_and_mul_masked_post_quant", + "sglang.kernels.ops.moe.dsv4.silu_and_mul_masked_post_quant", side_effect=run_poisoned_kernel, ) as fused_silu, ): diff --git a/test/registered/quant/test_int8_kernel.py b/test/registered/kernels/ops/moe/test_int8_kernel.py similarity index 100% rename from test/registered/quant/test_int8_kernel.py rename to test/registered/kernels/ops/moe/test_int8_kernel.py diff --git a/test/registered/quant/test_marlin_moe.py b/test/registered/kernels/ops/moe/test_marlin_moe.py similarity index 100% rename from test/registered/quant/test_marlin_moe.py rename to test/registered/kernels/ops/moe/test_marlin_moe.py diff --git a/test/registered/kernels/ops/moe/test_mega_moe_pre_dispatch.py b/test/registered/kernels/ops/moe/test_mega_moe_pre_dispatch.py index 6f9537c6bf..708d4a5e82 100644 --- a/test/registered/kernels/ops/moe/test_mega_moe_pre_dispatch.py +++ b/test/registered/kernels/ops/moe/test_mega_moe_pre_dispatch.py @@ -3,7 +3,7 @@ import sys import pytest import torch -from sglang.kernels.ops.attention.dsv4 import mega_moe_pre_dispatch +from sglang.kernels.ops.moe.dsv4 import mega_moe_pre_dispatch from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="4-gpu-b200") diff --git a/test/registered/kernels/ops/moe/test_route_quant_prerequisites.py b/test/registered/kernels/ops/moe/test_route_quant_prerequisites.py new file mode 100644 index 0000000000..57ae938e73 --- /dev/null +++ b/test/registered/kernels/ops/moe/test_route_quant_prerequisites.py @@ -0,0 +1,164 @@ +"""Expert routing, packed output, and quantization parity.""" + +import unittest + +import torch + +from sglang.kernels.ops.moe import moe_route_quant_fused +from sglang.kernels.ops.moe.moe_route_radix import route_radix +from sglang.kernels.ops.moe.moe_topk_sum import moe_topk_sum +from sglang.kernels.ops.quantization.per_token_group_quant import ( + per_token_group_quant, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=25, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +NUM_EXPERTS = 896 + +TOPK = 16 + + +def _pack_topk_oracle(topk_ids, topk_weights): + """Pure-torch reference for the packed ids the fused route+quant kernel emits. + + FlashInfer's routed MoE reads one int32 per entry: the expert id in the high + half and the bf16 weight bits in the low half. + """ + weight_bits = ( + topk_weights.to(torch.bfloat16).view(torch.int16).to(torch.int32) & 0xFFFF + ) + return (topk_ids.to(torch.int32) << 16) | weight_bits + + +def _route_oracle( + scores, bias, topk, renormalize, routed_scaling_factor, apply_scale, sorted +): + """Pure-torch fp32 reference for route_radix. + + Deliberately independent of moe_fused_gate: that entry dispatches back to + route_radix whenever scoring is sigmoid with no shared experts, no expert + groups and no softcapping (moe_fused_gate.py, the covered() fast path), which + is exactly the configuration under test. + + Contract, from route_radix.cuh: bias participates in RANKING only and the + emitted weight stays bias-free; NaN is floored so it can never win; ties go to + the lower expert id; renormalize divides by the winners' sum (guarded to 1 when + that sum is non-positive) and only then is routed scaling applied; sorted=True + emits (biased desc, id asc) while sorted=False emits ascending expert id. + """ + s = torch.sigmoid(scores.float()) + biased = s + bias.float() + biased = torch.where(torch.isnan(biased), torch.full_like(biased, -1e30), biased) + # stable + descending: equal biased values keep ascending-id order + ranked = torch.argsort(biased, dim=-1, descending=True, stable=True)[:, :topk] + w = s.gather(1, ranked) + total = w.sum(-1, keepdim=True) + norm = torch.where(total > 0, total, torch.ones_like(total)) + if renormalize: + w = w / norm + if apply_scale: + w = w * routed_scaling_factor + if sorted: + return w, ranked.to(torch.int32) + by_id = ranked.argsort(dim=-1) + return w.gather(1, by_id), ranked.gather(1, by_id).to(torch.int32) + + +class TestRouteQuant(CustomTestCase): + def test_moe_auxiliary_kernels(self): + x = torch.randn(2, TOPK, 7168, device="cuda", dtype=torch.bfloat16) + out = torch.empty(2, 7168, device="cuda", dtype=torch.bfloat16) + self.assertIs(moe_topk_sum(x, out), out) + self.assertTrue(torch.equal(out, x.float().sum(1).to(torch.bfloat16))) + + def test_moe_route_and_quant(self): + torch.manual_seed(1) + scores = torch.randn(8, NUM_EXPERTS, device="cuda", dtype=torch.bfloat16) + bias = torch.randn(NUM_EXPERTS, device="cuda", dtype=torch.float32) + args = (scores, bias, TOPK, True, 2.5, True) + weights, ids = route_radix(*args, sorted=True) + # Oracle, NOT moe_fused_gate: for this exact configuration (sigmoid, no + # shared experts, no expert groups, no softcapping) moe_fused_gate + # dispatches straight back to route_radix, so using it as the reference + # compares the kernel with itself and cannot see a selection, tie-break, + # NaN, renormalize or scaling error. + ref_weights, ref_ids = _route_oracle(*args, sorted=True) + self.assertTrue(torch.equal(ids, ref_ids)) + # rtol is not 1e-6: the kernel computes sigmoid with __fdividef/__expf, + # whose last bits differ from torch's. The old self-comparison could + # afford atol=0; a real oracle cannot. + torch.testing.assert_close(weights, ref_weights, rtol=1e-5, atol=1e-6) + + if not moe_route_quant_fused.available(): + self.skipTest("fused route+quant kernel unavailable") + hidden = torch.randn(8, 3584, device="cuda", dtype=torch.bfloat16) + ref_weights, ref_ids = route_radix(*args, sorted=False) + ref_packed = _pack_topk_oracle(ref_ids, ref_weights) + ref_q, ref_scale = per_token_group_quant( + hidden, group_size=32, scale_ue8m0=True + ) + actual = moe_route_quant_fused.route_quant_fused( + scores, + bias, + hidden, + TOPK, + renormalize=True, + routed_scaling_factor=2.5, + apply_scale=True, + ) + weights, ids, packed, quantized, scale = actual + self.assertTrue(torch.equal(ids, ref_ids)) + self.assertTrue( + torch.equal(weights.view(torch.int32), ref_weights.view(torch.int32)) + ) + self.assertTrue(torch.equal(packed, ref_packed)) + self.assertTrue( + torch.equal(quantized.view(torch.uint8), ref_q.view(torch.uint8)) + ) + torch.testing.assert_close(scale, ref_scale, rtol=0, atol=0) + + def test_route_radix_ties_and_nan(self): + """The cases the self-comparison could not see. + + Exact ties: many experts share one biased value, so the winner set is only + determined by the lowest-id rule. NaN: floored, so a NaN expert must never + be selected while enough finite ones exist. Both run with renormalize and + scaling on and off, since those are applied in a fixed order. + """ + bias = torch.zeros(NUM_EXPERTS, device="cuda", dtype=torch.float32) + + tied = torch.full((4, NUM_EXPERTS), 0.25, device="cuda", dtype=torch.bfloat16) + # a handful of strict winners above the tied plateau, the rest exactly equal + tied[:, 300] = 2.0 + tied[:, 7] = 2.0 + tied[:, 800] = 1.5 + + nan_scores = torch.randn(4, NUM_EXPERTS, device="cuda", dtype=torch.bfloat16) + nan_scores[:, 100] = float("nan") + nan_scores[:, 500] = float("nan") + # make the NaN experts the ones that would otherwise win outright + nan_scores[:, 101] = 5.0 + + for name, scores in (("ties", tied), ("nan", nan_scores)): + for renormalize in (False, True): + for apply_scale in (False, True): + for sorted_ in (False, True): + args = (scores, bias, TOPK, renormalize, 2.5, apply_scale) + ids = route_radix(*args, sorted=sorted_)[1] + ref_ids = _route_oracle(*args, sorted=sorted_)[1] + tag = ( + f"{name} renorm={renormalize} " + f"scale={apply_scale} sorted={sorted_}" + ) + self.assertTrue(torch.equal(ids, ref_ids), msg=tag) + if name == "nan": + self.assertFalse( + bool(((ids == 100) | (ids == 500)).any()), + msg=f"{tag}: a NaN expert was selected", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/quant/test_awq_dequant.py b/test/registered/kernels/ops/quantization/test_awq_dequant.py similarity index 100% rename from test/registered/quant/test_awq_dequant.py rename to test/registered/kernels/ops/quantization/test_awq_dequant.py diff --git a/test/registered/e2e/dsv4/test_dsv4_fp8_cast.py b/test/registered/kernels/ops/quantization/test_dsv4_fp8_cast.py similarity index 98% rename from test/registered/e2e/dsv4/test_dsv4_fp8_cast.py rename to test/registered/kernels/ops/quantization/test_dsv4_fp8_cast.py index 42c5095e3f..abca9928ca 100644 --- a/test/registered/e2e/dsv4/test_dsv4_fp8_cast.py +++ b/test/registered/kernels/ops/quantization/test_dsv4_fp8_cast.py @@ -18,7 +18,7 @@ import unittest import torch -from sglang.kernels.ops.attention.dsv4.fp8_cvt import cvt_fp8_e4m3 +from sglang.kernels.ops.quantization.fp8_cvt import cvt_fp8_e4m3 from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype, fp8_max from sglang.srt.utils import is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci diff --git a/test/registered/kernels/ops/quantization/test_fp8_kernel.py b/test/registered/kernels/ops/quantization/test_fp8_kernel.py new file mode 100644 index 0000000000..38afd872fa --- /dev/null +++ b/test/registered/kernels/ops/quantization/test_fp8_kernel.py @@ -0,0 +1,34 @@ +import unittest + +import torch + +from sglang.kernels.ops.quantization.fp8_kernel import per_token_group_quant_fp8 +from sglang.srt.utils import is_cuda +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kernels.fp8 import TestFP8Base + +register_cuda_ci(est_time=4, stage="base-b", runner_config="1-gpu-large") + + +_is_cuda = is_cuda() + + +class TestPerTokenGroupQuantFP8(TestFP8Base): + def test_per_token_group_quant_fp8(self): + if _is_cuda and torch.cuda.get_device_capability()[0] < 9: + return + + A, A_quant_gt, scale_gt = self._make_A( + M=self.M, K=self.K, group_size=self.group_size, out_dtype=self.quant_type + ) + A_quant, scale = per_token_group_quant_fp8( + x=A.to(torch.bfloat16), group_size=self.group_size + ) + torch.testing.assert_close(scale, scale_gt) + diff = (A_quant.to(torch.float16) - A_quant_gt.to(torch.float16)).abs() + diff_count = (diff > 1e-5).count_nonzero() + assert diff_count / diff.numel() < 1e-4 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/quantization/test_per_token_group_quant.py b/test/registered/kernels/ops/quantization/test_per_token_group_quant.py index d7d51271f5..642bbb2737 100644 --- a/test/registered/kernels/ops/quantization/test_per_token_group_quant.py +++ b/test/registered/kernels/ops/quantization/test_per_token_group_quant.py @@ -322,7 +322,7 @@ def test_fp32_silu_post_quant(group_size, hidden, swiglu_limit): round. Reuse the independent UE8M0 oracle, and compare both layouts only on active rows; an empty expert and a partial slab exercise masked counts. """ - from sglang.kernels.ops.attention.dsv4 import ( + from sglang.kernels.ops.moe.dsv4 import ( silu_and_mul_contig_post_quant, silu_and_mul_masked_post_quant, ) diff --git a/test/registered/kernels/ops/moe/test_renorm.py b/test/registered/kernels/ops/sampling/test_renorm.py similarity index 100% rename from test/registered/kernels/ops/moe/test_renorm.py rename to test/registered/kernels/ops/sampling/test_renorm.py diff --git a/test/registered/kernels/ops/sampling/test_top_p_renorm.py b/test/registered/kernels/ops/sampling/test_top_p_renorm.py new file mode 100644 index 0000000000..49a9538a5f --- /dev/null +++ b/test/registered/kernels/ops/sampling/test_top_p_renorm.py @@ -0,0 +1,38 @@ +"""Top-p probability renormalization parity.""" + +import unittest + +import torch + +from sglang.kernels.ops.sampling.top_p_renorm_triton import ( + top_p_renorm_probs_triton, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=5, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +class TestTopPRenorm(CustomTestCase): + def test_top_p_renorm(self): + torch.manual_seed(3) + probs = torch.randn(3, 1024, device="cuda").softmax(-1) + top_p = torch.tensor([0.5, 0.8, 0.95], device="cuda") + sorted_probs = probs.sort(-1).values + cutoff = torch.searchsorted( + sorted_probs.cumsum(-1), (1 - top_p).unsqueeze(1) + ).squeeze(1) + cutoff.clamp_(max=probs.shape[1] - 1) + pivot = sorted_probs.gather(1, cutoff[:, None]) + expected = torch.where(probs >= pivot, probs, 0) + expected /= expected.sum(-1, keepdim=True) + torch.testing.assert_close( + top_p_renorm_probs_triton(probs, top_p), + expected, + rtol=2e-6, + atol=1e-8, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/quant/test_quant_config_parsing.py b/test/registered/unit/configs/test_quant_config_parsing.py similarity index 100% rename from test/registered/quant/test_quant_config_parsing.py rename to test/registered/unit/configs/test_quant_config_parsing.py diff --git a/test/registered/kernels/ops/disaggregation/test_kv_checksum.py b/test/registered/unit/disaggregation/test_kv_checksum.py similarity index 100% rename from test/registered/kernels/ops/disaggregation/test_kv_checksum.py rename to test/registered/unit/disaggregation/test_kv_checksum.py diff --git a/test/registered/unit/dllm/algorithm/test_gemma4_sampling_cuda.py b/test/registered/unit/dllm/algorithm/test_gemma4_sampling_cuda.py new file mode 100644 index 0000000000..2ab6f86022 --- /dev/null +++ b/test/registered/unit/dllm/algorithm/test_gemma4_sampling_cuda.py @@ -0,0 +1,43 @@ +"""DiffusionGemma denoiser statistics and sampling parity.""" + +import unittest + +import torch + +from sglang.srt.dllm.algorithm.gemma4_renoise import ( + _compiled_denoiser_statistics, + _denoiser_statistics, + _sample_denoiser, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +@unittest.skipUnless(torch.cuda.is_available(), "CUDA is required") +class TestGemma4SamplingCUDA(CustomTestCase): + def test_full_vocabulary_statistics_and_sampling(self): + generator = torch.Generator(device="cuda").manual_seed(123) + for batch_size in (1, 3): + with self.subTest(batch_size=batch_size): + logits = torch.randn( + batch_size, 16, 262144, device="cuda", generator=generator + ) + temperatures = torch.linspace(0.4, 0.8, batch_size, device="cuda") + expected = _denoiser_statistics(logits, temperatures) + actual = _compiled_denoiser_statistics(logits, temperatures) + for got, want in zip(actual, expected): + torch.testing.assert_close(got, want, rtol=2e-5, atol=2e-6) + a = torch.Generator(device="cuda").manual_seed(42) + b = torch.Generator(device="cuda").manual_seed(42) + probabilities = actual[0].reshape(-1, logits.shape[-1]) + torch.testing.assert_close( + _sample_denoiser(probabilities, a), + torch.multinomial(probabilities, 1, generator=b).squeeze(-1), + ) + torch.testing.assert_close(a.get_state(), b.get_state()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/layernorm/test_fused_op.py b/test/registered/unit/kernels/test_fused_op.py similarity index 100% rename from test/registered/kernels/ops/layernorm/test_fused_op.py rename to test/registered/unit/kernels/test_fused_op.py diff --git a/test/registered/kernels/ops/layernorm/test_kernels_namespace.py b/test/registered/unit/kernels/test_kernels_namespace.py similarity index 68% rename from test/registered/kernels/ops/layernorm/test_kernels_namespace.py rename to test/registered/unit/kernels/test_kernels_namespace.py index 948307eab3..802c9562cf 100644 --- a/test/registered/kernels/ops/layernorm/test_kernels_namespace.py +++ b/test/registered/unit/kernels/test_kernels_namespace.py @@ -1,8 +1,10 @@ """GPU-free import / registry / selector tests for ``sglang.kernels`` (RFC #29630).""" +import ast import importlib import subprocess import sys +from pathlib import Path import pytest @@ -191,7 +193,6 @@ def test_import_stays_metadata_only(): ("attention.flydsl_kimi_k3_kda_decode_with_f_b", KernelBackend.FLYDSL, "hip"), ("communication.all_reduce_push_res", KernelBackend.JIT, "cuda"), ("communication.gemm_ag_up_proj", KernelBackend.JIT, "cuda"), - ("communication.o_proj_gemm_ar", KernelBackend.JIT, "cuda"), ("communication.reduce_scatter_res", KernelBackend.JIT, "cuda"), ], ) @@ -207,5 +208,98 @@ def test_kimi_k3_model_namespace_is_retired(): assert importlib.util.find_spec("sglang.kernels.ops.kimi_k3") is None +def test_operator_and_test_groups_agree(): + """A new root-level model bundle must not bypass logical op grouping.""" + root = Path(K.__file__).resolve().parents[3] + ops = root / "python/sglang/kernels/ops" + groups = set(K.ops.__all__) + assert {p.name for p in ops.glob("*.py")} == {"__init__.py"} + assert {p.name for p in ops.iterdir() if (p / "__init__.py").is_file()} == groups + for kind in ("ops", "benchmark"): + tests = root / "test/registered/kernels" / kind + actual = {p.relative_to(tests).parts[0] for p in tests.rglob("*.py")} + assert actual <= groups, actual - groups + + +def test_inventory_targets_survive_module_moves(): + """Moving a module without its lazy target otherwise breaks only on first call.""" + root = Path(K.__file__).resolve().parents[3] / "python" + for spec in K.registry.all_specs(): + module, _ = spec.target.split(":", 1) + if not module.startswith("sglang.kernels.ops."): + continue + path = root.joinpath(*module.split(".")) + assert path.with_suffix(".py").is_file() or (path / "__init__.py").is_file(), ( + spec + ) + + +def test_reclassified_public_entry_points_are_inventoried(): + """Public compute functions in these formerly unregistered modules need metadata.""" + root = Path(K.__file__).resolve().parent / "ops" + targets = {spec.target for spec in K.registry.all_specs()} + modules = ( + "attention.minicpm_sala.get_block_table", + "attention.fast_topk", + "attention.dsa.kpool_topk_transform", + "attention.dsa.indexer_k", + "attention.dsv4.wo_a", + "embeddings.qwen4_ngram", + "elementwise.qwen4_gate", + "elementwise.row_scale", + "mamba.qwen4_short_conv", + "mamba.lfm_short_conv", + "gemm.dsv4_wo_a", + "gemm.inkling_rel_proj", + "gemm.hopper_bf16_gemv", + "gemm.sm120_fp8_gemv", + "gemm.fp8_blockwise_gemm", + "gemm.gptq_marlin", + "layernorm.rmsnorm_hf", + "layernorm.grouped_gemma_rmsnorm", + "moe.dsv4", + "moe.gemma4_routing", + "memory.adler32", + "memory.row_compact", + "mm.process.image", + ) + for module in modules: + tree = ast.parse( + root.joinpath(*module.split(".")).with_suffix(".py").read_text() + ) + for node in tree.body: + if not isinstance(node, ast.FunctionDef): + continue + if ( + node.name.startswith(("_", "can_", "is_", "use_")) + or node.name == "make_name" + ): + continue + assert f"sglang.kernels.ops.{module}:{node.name}" in targets + + +@pytest.mark.parametrize( + "op, sm, expected", + [ + ("gemm.hopper_bf16_gemv", (9, 0), True), + ("gemm.hopper_bf16_gemv", (10, 0), False), + ("gemm.fp8_blockwise_scaled_mm", (12, 0), True), + ("gemm.fp8_blockwise_scaled_mm", (12, 1), True), + ("gemm.fp8_blockwise_scaled_mm", (10, 0), False), + ("attention.fused_rope_wo_a_bf16", (9, 0), False), + ("attention.fused_rope_wo_a_bf16", (10, 0), True), + ("attention.fused_rope_wo_a_bf16", (10, 3), True), + ("attention.fused_rope_wo_a_bf16", (12, 0), False), + ], +) +def test_registered_architecture_boundaries(op, sm, expected): + spec = K.registry.get_backend(op, KernelBackend.JIT) + platform = PlatformInfo( + device_type="cuda", cuda_arch_major=sm[0], cuda_arch_minor=sm[1] + ) + assert K.capabilities_satisfied(spec.capabilities, platform) is expected + assert not K.capabilities_satisfied(spec.capabilities, _CPU) + + if __name__ == "__main__": sys.exit(pytest.main([__file__])) diff --git a/test/registered/unit/layers/quantization/test_fp8_blockwise_linear_backends.py b/test/registered/unit/layers/quantization/test_fp8_blockwise_linear_backends.py index 1d7dc98500..56317e09fa 100644 --- a/test/registered/unit/layers/quantization/test_fp8_blockwise_linear_backends.py +++ b/test/registered/unit/layers/quantization/test_fp8_blockwise_linear_backends.py @@ -303,10 +303,7 @@ class TestBlockFp8AsMxfp8Linear(_LinearBackendCheck): def test_mxfp8_view_and_swizzled_input(self): if "flashinfer_cutedsl" not in _block32_backends(): self.skipTest(f"cutedsl not in SM{get_device_sm()} backend set") - from sglang.kernels.ops.attention.dsv4.wo_a import ( - _quantize_partial, - _wo_a_reduce, - ) + from sglang.kernels.ops.gemm.dsv4_wo_a import _quantize_partial, _wo_a_reduce torch.manual_seed(7) with mock.patch.object( diff --git a/test/registered/quant/test_fp8_utils.py b/test/registered/unit/layers/quantization/test_fp8_utils.py similarity index 100% rename from test/registered/quant/test_fp8_utils.py rename to test/registered/unit/layers/quantization/test_fp8_utils.py diff --git a/test/registered/quant/test_is_layer_skipped.py b/test/registered/unit/layers/quantization/test_is_layer_skipped.py similarity index 100% rename from test/registered/quant/test_is_layer_skipped.py rename to test/registered/unit/layers/quantization/test_is_layer_skipped.py diff --git a/test/registered/quant/test_nvfp4_embedding.py b/test/registered/unit/layers/quantization/test_nvfp4_embedding.py similarity index 100% rename from test/registered/quant/test_nvfp4_embedding.py rename to test/registered/unit/layers/quantization/test_nvfp4_embedding.py diff --git a/test/registered/kernels/ops/kvcache/test_kvcacheio_asymmetric.py b/test/registered/unit/mem_cache/pool_host/test_kvcacheio_asymmetric.py similarity index 100% rename from test/registered/kernels/ops/kvcache/test_kvcacheio_asymmetric.py rename to test/registered/unit/mem_cache/pool_host/test_kvcacheio_asymmetric.py diff --git a/test/registered/kernels/ops/kvcache/test_unified_swa_tail_allocation.py b/test/registered/unit/mem_cache/test_unified_swa_tail_allocation.py similarity index 100% rename from test/registered/kernels/ops/kvcache/test_unified_swa_tail_allocation.py rename to test/registered/unit/mem_cache/test_unified_swa_tail_allocation.py diff --git a/test/registered/kernels/ops/cuda_graph/test_cuda_graph_dedup.py b/test/registered/unit/model_executor/runner_backend/test_cuda_graph_dedup.py similarity index 100% rename from test/registered/kernels/ops/cuda_graph/test_cuda_graph_dedup.py rename to test/registered/unit/model_executor/runner_backend/test_cuda_graph_dedup.py diff --git a/test/registered/unit/model_executor/test_compensated_mhc_update_guard.py b/test/registered/unit/model_executor/test_compensated_mhc_update_guard.py index 4fd8864cd3..047b10664c 100644 --- a/test/registered/unit/model_executor/test_compensated_mhc_update_guard.py +++ b/test/registered/unit/model_executor/test_compensated_mhc_update_guard.py @@ -50,7 +50,7 @@ class TestCompensatedMhcUpdateGuard(CustomTestCase): model = torch.nn.Sequential(torch.nn.Linear(1, 1)) model[0]._hc_attn_tf32_parts = model[0]._hc_ffn_tf32_parts = None with patch( - "sglang.kernels.ops.attention.dsv4.gemm.hpc_bf16xfp32_gemm_enabled", + "sglang.kernels.ops.gemm.bf16_fp32.hpc_bf16xfp32_gemm_enabled", return_value=False, ): self.assertIsNone(_unsupported_derived_weight_cache_error(model)) diff --git a/test/registered/unit/model_executor/test_derived_weight_cache.py b/test/registered/unit/model_executor/test_derived_weight_cache.py index aca13a294f..5d2f4bd6ea 100644 --- a/test/registered/unit/model_executor/test_derived_weight_cache.py +++ b/test/registered/unit/model_executor/test_derived_weight_cache.py @@ -49,7 +49,7 @@ class TestDerivedWeightCache(unittest.TestCase): def test_model_without_derived_cache_keeps_updates_enabled(self): model = torch.nn.Module() with patch( - "sglang.kernels.ops.attention.dsv4.gemm.hpc_bf16xfp32_gemm_enabled", + "sglang.kernels.ops.gemm.bf16_fp32.hpc_bf16xfp32_gemm_enabled", return_value=False, ): self.assertIsNone(_unsupported_derived_weight_cache_error(model)) diff --git a/test/registered/kernels/ops/quantization/test_diffusion_modelopt_fp8_scaled_mm.py b/test/registered/unit/multimodal_gen/test_modelopt_fp8_scaled_mm.py similarity index 100% rename from test/registered/kernels/ops/quantization/test_diffusion_modelopt_fp8_scaled_mm.py rename to test/registered/unit/multimodal_gen/test_modelopt_fp8_scaled_mm.py