From 693d33330647c57f82e09ac39a34122fa46b5763 Mon Sep 17 00:00:00 2001 From: Xiaoyu Zhang <1182563586@qq.com> Date: Tue, 20 Oct 2026 08:47:57 +0800 Subject: [PATCH] [Refactor] kernels: Regroup operators by computation and dissolve ops/muse_spark (#45112) Co-authored-by: Claude Opus 5.5 --- .agents/skills/kernel-organization/SKILL.md | 49 +- .../references/fuse-overlap-catalog.md | 16 +- .../scripts/triage_kernel_helpers.py | 10 +- .github/CODEOWNERS | 2 + .github/workflows/_pr-test-check-changes.yml | 2 +- .../bench_cutedsl_kda_decode.py | 4 +- .../bench_gdn_decode.py | 2 +- .../bench_kda_decode.py | 6 +- .../attention/bench_gdn_replayssm_decode.py | 2 +- benchmark/kernels/bench_kda_verify_sweep.py | 6 +- .../deepseek/benchmark_q8kv8_kv_gather.py | 127 +- .../deepseek/benchmark_q8kv8_q_prep.py | 2 +- .../quantization/tuning_mxfp8_native_gfx95.py | 4 +- python/sglang/kernels/README.md | 89 +- .../jit/csrc/moe/expert_pack_mxfp4.cuh | 276 ---- .../kernels/jit/csrc/muse_spark/routing.cuh | 2 +- .../kernels/jit/csrc/muse_spark/row_sum.cuh | 2 +- .../jit/csrc/qprep_bf16_fp8_sm90/kernel.cuh | 2 +- .../sgl_kernel/deepseek_v4/fp4_utils.cuh | 4 +- .../utils}/helion_runtime.py | 26 +- python/sglang/kernels/jit/utils/hipcc.py | 91 ++ python/sglang/kernels/ops/__init__.py | 1 - .../sglang/kernels/ops/activation/__init__.py | 28 +- .../{elementwise => activation}/bias_gelu.py | 0 .../ops/activation/gelu_and_mul_triton.py | 101 ++ ...itu_and_mul.py => kimi_k3_situ_and_mul.py} | 0 .../{moe => activation}/minimax_m3_swiglu.py | 1 + .../ops/activation/silu_and_mul_clamp.py | 91 ++ .../sglang/kernels/ops/activation/softcap.py | 50 - .../ops/activation/swiglu_mxfp8_swizzled.py | 177 +++ .../sglang/kernels/ops/attention/__init__.py | 184 ++- .../aiter_unified_page_table.py} | 0 .../sglang/kernels/ops/attention/attn_res.py | 168 +-- .../kernels/ops/attention/dsa/cp_split.py | 18 +- .../{ => dsa}/cutedsl_fp8_paged_mqa_logits.py | 0 .../attention/dsa/cutedsl_paged_mqa_logits.py | 4 +- .../ops/attention/dsa/dequant_k_cache.py | 120 +- .../{ => dsa}/fused_metadata_copy.py | 0 .../ops/attention/dsa/hip_gfx950/__init__.py | 13 +- .../kernels/ops/attention/dsa/indexer_q.py | 53 + .../kpool_metadata}/__init__.py | 0 .../kpool_metadata}/decode.py | 8 +- .../kpool_metadata}/draft_extend.py | 8 +- .../kpool_metadata}/scan.py | 0 .../kpool_metadata}/verify.py | 12 +- .../{dsa_metadata.py => dsa/metadata.py} | 0 .../ops/attention/dsa/tilelang_kernel.py | 1145 +---------------- .../ops/attention/dsa/transform_index.py | 12 +- .../ops/attention/dsa/triton_kernel.py | 133 -- .../ops/attention/dsa/triton_sparse_mla.py | 2 +- .../kernels/ops/attention/dsv4/__init__.py | 10 +- .../sglang/kernels/ops/attention/dsv4/attn.py | 138 +- .../attn_metadata_kernels.py} | 0 .../ops/attention/dsv4/dequant_k_cache.py | 74 +- .../kernels/ops/attention/dsv4/fp4_indexer.py | 350 +---- .../ops/attention/dsv4/fp4_indexer_hip.py | 214 +-- .../ops/attention/dsv4/fp4_rope_fake_quant.py | 2 +- .../dsv4/{elementwise.py => norm_rope.py} | 42 +- .../ops/attention/dsv4/online_c128_mtp.py | 318 ++--- .../{deepseek_v4_rope.py => dsv4/rope.py} | 2 + .../attention/dsv4/sparse_prefill_kernels.py | 187 +++ .../attention/dsv4/tilelang_fp8_attention.py | 1064 +++++++++++++++ .../triton_sparse_decode}/__init__.py | 2 +- .../triton_mla_kernels_decode_fused.py | 0 .../triton_mla_kernels_decode_optimized.py | 0 .../dsv4/unified_kv_kernels/paged_decode.py | 6 +- .../dsv4/unified_kv_kernels/runtime.py | 171 +-- .../flash_attn/cute/flash_fwd_mla_sm100.py | 789 ------------ .../fused_qk_rope_reshape_and_cache.py} | 0 .../ops/attention/helion/kda_replayssm.py | 2 +- .../ops/{kvcache => attention}/kv_indices.py | 0 .../{fla => linear}/fused_gdn_gating.py | 0 .../fused_kda_conv_recurrent_verify.py | 0 .../fused_recurrent_linear_replayssm.py | 0 .../fused_sigmoid_gating_recurrent.py | 0 .../gdn_replayssm_spec_decode.py | 0 .../gdn_replayssm_spec_fold.py | 0 .../kda_replayssm_spec_decode.py | 0 .../ops/attention/linear/lightning_attn.py | 40 - .../kernels/ops/attention/log_scaling_tau.py | 70 - .../sglang/kernels/ops/attention/metadata.py | 48 + .../minimax_sparse/prefill}/gluon_prefill.py | 0 .../kernels/ops/attention/mla_qk_prep.py | 554 ++++++++ .../muse_spark_qk_norm_rope_store.py} | 14 +- .../muse_spark_qk_norm_rope_store_helion.py} | 2 +- .../ops/attention/qprep_bf16_fp8_sm90.py | 2 +- .../sparse_mla_q8kv8_prefill_sm90.py | 73 ++ .../trtllm_mha_graph_metadata.py | 0 .../trtllm_mha_page_table.py | 0 python/sglang/kernels/ops/attention/utils.py | 317 ----- .../kernels/ops/communication/__init__.py | 46 + .../ops/communication/attn_res_collective.py | 173 +++ .../cutlass_all_gather_gemm.py | 0 .../cutlass_gemm_reduce_scatter.py | 0 .../communication/muse_spark_all_gather.py | 97 ++ .../muse_spark_all_gather_norm_residual.py} | 2 +- .../muse_spark_moe_finalize_all_reduce.py} | 31 +- ..._spark_moe_finalize_all_reduce_cutedsl.py} | 138 +- ...e_spark_moe_finalize_all_reduce_helion.py} | 0 .../sp_collective_gemm.py | 112 +- python/sglang/kernels/ops/diffusion/README.md | 4 +- .../sglang/kernels/ops/diffusion/__init__.py | 10 +- .../indexed_scale_shift_mxfp8_triton.py | 113 ++ .../kernels/ops/elementwise/__init__.py | 25 +- .../kernels/ops/elementwise/elementwise.py | 266 +--- .../mla_output_gate.py | 1 + .../kernels/ops/elementwise/row_scale.py | 74 +- .../{muse_spark => elementwise}/row_sum.py | 4 +- .../ngram_embedding.py | 0 python/sglang/kernels/ops/gemm/__init__.py | 21 +- .../ops/gemm/dsv4_indexer_head_weights_hip.py | 90 ++ python/sglang/kernels/ops/gemm/dsv4_wo_a.py | 2 +- .../kernels/ops/gemm/flashinfer_bmm_fp8.py | 42 + .../hc_mix.py => gemm/hc_mix_splitk.py} | 1 + .../muse_spark_router_gemm.py} | 0 .../mxfp8_aiter_gfx95.py | 0 .../kernels/ops/gemm/mxfp8_amd_gfx95.py | 256 ++++ .../mxfp8_flydsl_gfx95.py | 0 .../mxfp8_gemv_gfx95_configs.json | 0 .../mxfp8_native_amd_gfx95.py | 0 .../nvfp4_gemm_swiglu_nvfp4_quant.py | 0 .../kernels/ops/gemm/router_gate_matvec.py | 131 ++ .../ops/gemm/smallm_fp8_gfx950/__init__.py | 30 +- .../sglang/kernels/ops/kv_canary/__init__.py | 25 + .../ops/kv_canary/scatter_req_token_ids.py | 27 - python/sglang/kernels/ops/kvcache/__init__.py | 24 +- .../sglang/kernels/ops/kvcache/cache_ops.py | 611 --------- .../dsa_fused_store_index_cache.py} | 0 .../dsa_index_k_cache.py} | 0 .../dsv4_c128_cleanup.py} | 0 .../ops/kvcache/dsv4_fp4_index_k_cache.py | 127 ++ .../ops/kvcache/dsv4_fp4_index_k_cache_hip.py | 98 ++ .../dsv4_main_kv_store.py} | 0 .../dsv4_request_window_copy.py} | 0 .../ops/kvcache/dsv4_unified_kv_store.py | 182 +++ .../mxfp8_interleave_sf.py | 0 .../ops/kvcache/mxfp8_quant_store_kv.py | 119 ++ .../kernels/ops/kvcache/shuffle_5d_cache.py | 267 ++++ .../sglang/kernels/ops/layernorm/__init__.py | 37 +- .../_muse_spark_tuning.py} | 20 +- .../kernels/ops/layernorm/fused_rmsnorm.py | 136 ++ .../kernels/ops/layernorm/hc_combine_norm.py | 2 +- python/sglang/kernels/ops/layernorm/mhc.py | 6 +- .../kernels/ops/layernorm/mhc_boundary_hip.py | 4 +- .../muse_spark_norm_gate.py} | 22 +- .../muse_spark_norm_gate_helion.py} | 2 +- .../muse_spark_norm_gate_jit.py} | 0 .../muse_spark_norm_residual.py} | 83 +- .../muse_spark_norm_residual_jit.py} | 2 +- .../muse_spark_rmsnorm_nvfp4_quant.py} | 82 +- .../muse_spark_rmsnorm_nvfp4_quant_helion.py} | 4 +- .../rmsnorm_fake_quant_amd_gfx95.py | 0 .../kernels/ops/layernorm/rmsnorm_mxfp8.py | 67 + python/sglang/kernels/ops/lplb/__init__.py | 28 + python/sglang/kernels/ops/mamba/__init__.py | 10 + .../kda_prefill_track.py} | 0 .../kernels/ops/memory/gpu_tensor_hash.py | 2 +- python/sglang/kernels/ops/moe/__init__.py | 35 +- python/sglang/kernels/ops/moe/dsv4.py | 75 -- .../sglang/kernels/ops/moe/ep_moe_kernels.py | 2 +- .../kernels/ops/moe/expert_pack_mxfp4.py | 30 - ...quant.py => kimi_k3_situ_and_mul_quant.py} | 0 .../minimax_quant_ue8m0.py | 0 ...muse_spark_post_expert_finalize_triton.py} | 0 .../muse_spark_routing.py} | 14 +- .../muse_spark_routing_jit.py} | 4 +- .../kernels/ops/moe/mxfp8_moe_amd_gfx95.py | 2 +- python/sglang/kernels/ops/moe/router.py | 125 -- .../ops/moe/smallm_moe_gfx950/__init__.py | 104 +- .../ops/moe/smallm_router_gfx950/__init__.py | 8 +- .../ops/moe/triton_sigmoid_gate_mul.py | 89 -- .../sglang/kernels/ops/muse_spark/__init__.py | 1 - .../kernels/ops/quantization/__init__.py | 25 +- .../ops/quantization/act_quant_tilelang.py | 122 ++ .../ops/quantization/act_quant_triton.py | 138 ++ .../dsa_quant_k_cache.py} | 54 - .../fp4_fake_quant.py} | 0 .../ops/quantization/fp4_indexer_quant.py | 233 ++++ .../fp8_group_major_quant_ue8m0.py} | 7 +- .../ops/quantization/mxfp8_amd_gfx95.py | 245 +--- .../mxfp8_epilogue.py | 59 - .../kernels/ops/quantization/mxfp8_quant.py | 114 -- .../ops/quantization/mxfp8_swizzled_triton.py | 251 ---- .../rocm_upscale.py} | 0 .../sglang/kernels/ops/sampling/__init__.py | 8 + .../ops/sampling/softcap_row_argmax.py | 56 + .../ops/speculative/dspark/dspark_accept.py | 1 + .../multimodal_gen/runtime/layers/mlp.py | 2 +- .../layers/quantization/modelopt_quant.py | 4 +- .../sglang/srt/arg_groups/deepseek_v4_hook.py | 2 +- python/sglang/srt/arg_groups/hisparse_hook.py | 2 +- .../mp.py => srt/distributed/comm_cleanup.py} | 0 .../fla/fused_sigmoid_gating_recurrent.py | 2 +- .../srt/layers/attention/aiter_backend.py | 18 +- .../srt/layers/attention/aiter_utils.py | 4 +- .../layers/attention/deepseek_v4_backend.py | 41 +- .../deepseek_v4_backend_hip_radix.py | 41 +- .../dsa/dsa_backend_mtp_precompute.py | 2 +- .../srt/layers/attention/dsa/dsa_indexer.py | 14 +- .../layers/attention/dsa/dsa_indexer_kpool.py | 4 +- .../attention/dsa/dsa_metadata_manager.py | 16 +- .../attention/dsa/dsa_prefill_cuda_graph.py | 4 +- .../attention/dsa/gfx950_fused_indexer.py} | 0 .../sglang/srt/layers/attention/dsa/utils.py | 16 +- .../srt/layers/attention/dsa_backend.py | 22 +- .../attention/dsv4/compressor_trtllm.py | 2 +- .../layers/attention/dsv4/compressor_v2.py | 4 +- .../srt/layers/attention/dsv4/dsv41_sparse.py | 8 +- .../attention/dsv4/low_ratio_backend_hip.py | 2 +- .../layers/attention/dsv4/online_c128_mtp.py | 251 ++++ .../attention/dsv4/sparse_prefill_utils.py | 190 +-- .../layers/attention/dsv4/unified_kv_gate.py} | 0 .../attention/dsv4/v41_indexer/scoring.py | 2 +- .../attention/flashattention_backend.py | 4 +- .../srt/layers/attention/flashmla_backend.py | 2 +- .../srt/layers/attention/hip_flash_mla.py | 6 +- .../srt/layers/attention/hpc_ops_backend.py | 4 +- .../attention/hybrid_linear_attn_backend.py | 2 +- .../layers/attention/linear/gdn_backend.py | 12 +- .../layers/attention/linear/kda_backend.py | 4 +- .../attention/linear/kda_prefill_graph.py | 12 +- .../attention/linear/kernels/gdn_cutedsl.py | 21 +- .../attention/linear/kernels/gdn_triton.py | 4 +- .../attention/linear/kernels/kda_cutedsl.py | 17 +- .../linear/kernels/kda_flashinfer.py | 11 +- .../attention/linear/kernels/kda_triton.py | 6 +- .../attention/linear/lightning_backend.py | 42 +- .../attention/minicpm/attention_adapter.py | 2 +- .../minimax_sparse_ops/minimax_sparse.py | 2 +- .../naive/flash_with_topk_idx.py | 84 -- .../minimax_sparse_ops/naive/topk_sparse.py | 88 -- .../attention/nsa/index_buf_accessor.py | 10 - .../srt/layers/attention/nsa/quant_k_cache.py | 10 - .../layers/attention/nsa/tilelang_kernel.py | 10 - .../srt/layers/attention/nsa/triton_kernel.py | 10 - .../layers/attention/trtllm_mha_backend.py | 6 +- .../layers/attention/trtllm_mla_backend.py | 10 +- .../srt/layers/attention/wave_backend.py | 4 +- .../layers/communication/k3_sp_collective.py | 15 +- .../muse_spark_finalize_workspace.py | 139 ++ .../communication/muse_spark_gemm_ag.py} | 76 +- .../sp_collective_gemm_workspace.py | 91 ++ python/sglang/srt/layers/cp/interleave.py | 14 +- python/sglang/srt/layers/engram.py | 2 +- python/sglang/srt/layers/hyperconnection.py | 2 +- python/sglang/srt/layers/layernorm_sp.py | 20 +- python/sglang/srt/layers/logits_processor.py | 2 +- python/sglang/srt/layers/moe/cutlass_moe.py | 2 +- .../sglang/srt/layers/moe/moe_runner/aiter.py | 2 +- .../srt/layers/moe/moe_runner/deep_gemm.py | 10 +- .../moe/moe_runner/triton_utils/fused_moe.py | 4 +- python/sglang/srt/layers/moe/qwen4_decode.py | 2 +- .../layers/muse_spark/muse_spark_bf16_moe.py | 10 +- .../layers/muse_spark/muse_spark_fp4_moe.py | 14 +- python/sglang/srt/layers/n_gram_embedding.py | 2 +- .../srt/layers/quantization/expert_pack.py | 4 +- .../sglang/srt/layers/quantization/fp8_hip.py | 20 +- .../srt/layers/quantization/fp8_utils.py | 41 +- .../srt/layers/quantization/modelopt_quant.py | 2 +- .../srt/layers/rotary_embedding/base.py | 2 +- .../srt/mem_cache/deepseek_v4_memory_pool.py | 35 +- .../srt/mem_cache/dsa_cache_layer_split.py | 4 +- .../srt/mem_cache/dsv41_request_window.py | 2 +- .../sglang/srt/mem_cache/index_key_cache.py | 6 +- .../srt/mem_cache/kv_cache_configurator.py | 2 +- .../srt/mem_cache/kv_index_translator.py | 2 +- python/sglang/srt/mem_cache/memory_pool.py | 18 +- .../srt/mem_cache/page_interleave_pool.py | 8 +- .../srt/mem_cache/unified_radix_cache.py | 2 +- .../forward_batch_deepseek_mha_mixin.py | 2 +- .../ngram_embedding_manager.py | 2 +- .../srt/model_executor/pool_configurator.py | 2 +- .../runner/prefill_cuda_graph_runner.py | 2 +- python/sglang/srt/models/bailing_moe_v3.py | 2 +- .../amd/deepseek_v4_gfx95_dense.py | 6 +- .../attention_forward_methods/forward_mha.py | 10 +- .../forward_mha_rocm.py | 2 +- .../attention_forward_methods/forward_mla.py | 2 +- python/sglang/srt/models/deepseek_v2.py | 4 +- python/sglang/srt/models/deepseek_v4.py | 24 +- .../sglang/srt/models/deepseek_v4_dspark.py | 10 +- python/sglang/srt/models/deepseek_v4_mhc.py | 4 +- .../srt/models/deepseek_v4_replay_tail.py | 4 +- python/sglang/srt/models/grok.py | 4 +- python/sglang/srt/models/hunyuan_v4.py | 10 +- .../sglang/srt/models/inkling_common/attn.py | 6 +- python/sglang/srt/models/kimi_k3.py | 2 +- python/sglang/srt/models/minimax_m3.py | 2 +- python/sglang/srt/models/muse_spark.py | 64 +- python/sglang/srt/models/qwen2_moe.py | 2 +- python/sglang/srt/models/qwen3_next.py | 6 +- python/sglang/srt/models/sarvam_moe.py | 2 +- python/sglang/srt/speculative/spec_utils.py | 6 +- .../sglang/test/kernels/deepseek_v4/common.py | 60 + .../deepseek_v4/dsv41_kv_quant_reference.py | 2 +- .../deepseek_v4/fp4_index_k_reference.py | 40 + .../kernels/deepseek_v4}/fp8_cvt.py | 0 .../kernels/deepseek_v4/k_cache_reference.py | 4 +- .../test/kernels/dsa_k_cache_reference.py | 57 + .../scatter_req_token_ids_reference.py | 30 + .../muse_spark => test/kernels}/triton_sum.py | 2 +- .../attention_methods/dsa_attention.py | 2 +- .../attention/test_trtllm_mla_backend.py | 2 +- .../attention/check_dsa_dequant_k_cache.py | 2 +- .../attention/check_dsa_transform_index.py | 4 +- .../attention/check_dsv4_dequant_k_cache.py | 4 +- .../attention/test_flash_fwd_mla_sm100.py | 800 ++++++++++++ .../attention/test_minimax_msa_fp8_parity.py | 10 +- .../attention/bench_gdn_replayssm_fold.py | 2 +- .../test_dsa_get_k_scale_triton_kernel.py} | 2 +- .../kvcache/test_dsa_index_k_cache.py} | 2 +- .../check_dsa_quant_k_cache.py | 10 +- .../quantization}/test_act_quant_triton.py | 6 +- test/manual/test_kda_target_verify.py | 4 +- .../amd/test_deepseek_v4_pro_fp4_dspark.py | 10 +- .../amd/test_dsv4_hip_bcg_metadata.py | 10 +- .../attention/test_create_kvindices.py | 2 +- .../attention/test_fused_fp8_kv_write.py | 4 +- .../test_gdn_noncontiguous_stride.py | 4 +- .../attention/test_gdn_prefill_cutedsl.py | 2 +- test/registered/attention/test_kda_kernels.py | 6 +- .../attention/test_kda_prefill_cutedsl.py | 2 +- .../test_trtllm_mha_graph_metadata.py | 2 +- .../attention/test_trtllm_mha_page_table.py | 2 +- .../attention/unittests/dsa/README.md | 2 +- .../attention/unittests/dsa/test_dsa.py | 2 +- .../dsv4/test_dsv41_fused_compress.py | 4 +- .../dsv4/test_hip_flash_mla_aiter_sparse.py | 4 +- .../gdn/test_gdn_cutedsl_ring_verify.py | 4 +- .../gdn/test_gdn_replayssm_spec_fold.py | 4 +- .../gdn/test_linear_replayssm_decode.py | 4 +- .../test_dsv4_unified_fp8_compress_store.py | 2 +- .../e2e/dsv4/test_dsv4_unified_fp8_scatter.py | 6 +- .../expert_pack/test_expert_pack_mxfp4.py | 167 --- .../bench_bias_gelu.py | 2 +- .../attention/bench_fp4_index_logits.py | 2 +- .../attention/bench_online_c128_mtp.py | 12 +- .../communication/bench_custom_all_reduce.py | 2 +- .../communication/bench_tp_qknorm.py | 2 +- .../bench_ngram_compute_decode.py | 2 +- .../bench_ngram_update_token_table.py | 2 +- .../test_bias_gelu.py | 2 +- .../activation/test_swiglu_mxfp8_swizzled.py | 120 ++ .../attention/dsv4/test_swapab_attention.py | 2 +- .../ops/attention/dsv4/test_v41_kv_store.py | 18 +- .../test_combine_topk_swa_indices.py | 2 +- .../kernels/ops/attention/test_cutedsl_gdn.py | 2 +- .../attention/test_dense_prefill_indexer.py | 4 +- .../kernels/ops/attention/test_dsa_indexer.py | 4 +- .../test_dsa_kpool_metadata_fusion.py | 6 +- .../ops/attention/test_dsa_metadata.py | 2 +- .../ops/attention/test_dsa_transform_index.py | 16 +- .../attention/test_dsv32_indexer_fusion.py | 4 +- .../test_dsv41_prefill_sparse_indexer.py | 4 +- .../ops/attention/test_dsv4_indexer_quant.py | 6 +- .../ops/attention/test_dsv4_q_rope_store.py | 4 +- .../attention/test_dsv4_trtllm_q_metadata.py | 2 +- .../kernels/ops/attention/test_fp4_indexer.py | 192 +-- .../ops/attention/test_fp4_indexer_hip.py | 20 +- .../test_fused_kda_conv_recurrent_verify.py | 4 +- .../ops/attention/test_fused_metadata_copy.py | 8 +- .../attention/test_fused_verify_triton_gdn.py | 10 +- .../test_inkling_attn_prologue_tau.py | 4 +- .../test_kda_fused_verify_backend.py | 2 +- .../kernels/ops/attention/test_kda_helion.py | 4 +- .../test_kda_mtp_cutedsl_replayssm_ring.py | 6 +- .../ops/attention/test_kda_replayssm_fold.py | 4 +- .../test_kda_replayssm_fold_batched.py | 2 +- .../test_kda_replayssm_ring_fused.py | 4 +- .../test_kda_replayssm_ring_ragged.py | 4 +- ...test_minimax_sparse_flash_with_topk_idx.py | 5 + .../test_minimax_sparse_fp8_attn_gemm.py | 12 +- .../ops/attention/test_minimax_sparse_gqa.py | 4 + .../test_muse_spark_qk_norm_rope_store.py | 71 + .../test_q8kv8_dsa_nope_bf16_prefill.py | 2 +- .../test_q8kv8_sparse_prefill_backend.py | 4 + .../ops/attention/test_qprep_bf16_fp8_sm90.py | 8 +- .../test_sparse_mla_q8kv8_prefill_sm90.py | 4 +- .../attention/test_tilelang_dsa_zero_rope.py | 6 +- .../test_trtllm_kv_norm_rope_store.py | 2 +- .../ops/communication/test_ar_fusion.py | 2 +- .../ops/communication/test_collectives.py | 7 +- .../communication/test_custom_all_reduce.py | 2 +- ...test_muse_spark_moe_finalize_all_reduce.py | 58 + .../ops/communication/test_tp_qknorm.py | 2 +- .../test_indexed_scale_shift_mxfp8.py | 47 + .../test_mla_output_gate.py | 2 +- .../kernels/ops/elementwise/test_row_scale.py | 6 +- .../kernels/ops/elementwise/test_row_sum.py | 31 + .../test_sigmoid_gate_mul.py | 16 +- .../test_sigmoid_gate_mul_broadcast.py | 8 +- .../test_ngram_embedding.py | 2 +- .../test_mxfp8_amd_gfx95.py | 18 +- .../kv_canary/test_scatter_req_token_ids.py | 4 +- .../test_dsv41_request_window_copy.py | 0 .../kvcache/test_dsv4_fp4_index_k_cache.py | 47 + .../test_fused_store_index_cache.py | 4 +- .../ops/layernorm/test_hc_boundary_hip.py | 10 +- ...test_hc_combine.py => test_mhc_combine.py} | 14 +- .../kernels/ops/layernorm/test_mhc_kernels.py | 2 +- .../ops/layernorm/test_muse_spark_norm_ops.py | 118 ++ .../test_muse_spark_rmsnorm_nvfp4_quant.py | 71 + .../ops/layernorm/test_mxfp8_epilogue.py | 2 +- .../kernels/ops/moe/test_minimax_m3_mxfp8.py | 6 +- .../ops/moe/test_minimax_quant_scatter.py | 6 +- .../ops/moe/test_muse_spark_routing.py | 112 ++ .../ops/quantization/test_dsv4_fp8_cast.py | 2 +- .../quantization/test_fp4_indexer_quant.py | 123 ++ .../test_fp8_grid_producers_gfx95.py | 6 +- .../test_fp8_wo_a.py | 6 +- .../ops/quantization/test_mxfp8_swizzled.py | 111 +- .../test_per_token_group_quant.py | 4 +- .../speculative/test_spec_kv_indices_grid.py | 2 +- .../spec/dspark/test_dspark_kernel_parity.py | 4 +- .../dsa/test_dsa_gfx950_fused_workspace.py | 2 +- .../unit/kernels/test_kernels_namespace.py | 103 +- .../test_kpool_preshuffled_cache_layout.py | 2 +- .../unit/layers/moe/test_aiter_runner.py | 2 +- .../test_dsv41_late_layer_tail_readers.py | 2 +- .../layers/test_kpool_stream_scheduling.py | 4 +- .../mem_cache/test_dsv4_unified_fp8_pool.py | 10 +- .../mem_cache/test_unified_cache_linker.py | 4 +- .../mem_cache/test_unified_mla_block_table.py | 2 +- .../model_executor/test_encoder_swa_replay.py | 4 +- .../models/test_deepseek_v4_cp_kv_store.py | 2 +- .../models/test_deepseek_v4_rope_policy.py | 2 +- .../test_deepseek_v4_unified_fp8_q_pair.py | 6 +- .../test_platform_prefill_cp_deprecation.py | 2 +- .../spec/test_ngram_mamba_verify_update.py | 12 +- test/registered/xpu/test_dsa_indexer_xpu.py | 4 +- 430 files changed, 9236 insertions(+), 8447 deletions(-) rename python/sglang/kernels/{ops/muse_spark => jit/utils}/helion_runtime.py (88%) create mode 100644 python/sglang/kernels/jit/utils/hipcc.py rename python/sglang/kernels/ops/{elementwise => activation}/bias_gelu.py (100%) create mode 100644 python/sglang/kernels/ops/activation/gelu_and_mul_triton.py rename python/sglang/kernels/ops/activation/{_jit_situ_and_mul.py => kimi_k3_situ_and_mul.py} (100%) rename python/sglang/kernels/ops/{moe => activation}/minimax_m3_swiglu.py (98%) create mode 100644 python/sglang/kernels/ops/activation/silu_and_mul_clamp.py create mode 100644 python/sglang/kernels/ops/activation/swiglu_mxfp8_swizzled.py rename python/sglang/kernels/ops/{kvcache/aiter_unified_attention.py => attention/aiter_unified_page_table.py} (100%) rename python/sglang/kernels/ops/attention/{ => dsa}/cutedsl_fp8_paged_mqa_logits.py (100%) rename python/sglang/kernels/ops/attention/{ => dsa}/fused_metadata_copy.py (100%) create mode 100644 python/sglang/kernels/ops/attention/dsa/indexer_q.py rename python/sglang/kernels/ops/attention/{dsa_kpool_metadata => dsa/kpool_metadata}/__init__.py (100%) rename python/sglang/kernels/ops/attention/{dsa_kpool_metadata => dsa/kpool_metadata}/decode.py (97%) rename python/sglang/kernels/ops/attention/{dsa_kpool_metadata => dsa/kpool_metadata}/draft_extend.py (97%) rename python/sglang/kernels/ops/attention/{dsa_kpool_metadata => dsa/kpool_metadata}/scan.py (100%) rename python/sglang/kernels/ops/attention/{dsa_kpool_metadata => dsa/kpool_metadata}/verify.py (96%) rename python/sglang/kernels/ops/attention/{dsa_metadata.py => dsa/metadata.py} (100%) rename python/sglang/kernels/ops/attention/{dsv4_attn_metadata_kernels.py => dsv4/attn_metadata_kernels.py} (100%) rename python/sglang/kernels/ops/attention/dsv4/{elementwise.py => norm_rope.py} (88%) rename python/sglang/kernels/ops/attention/{deepseek_v4_rope.py => dsv4/rope.py} (99%) create mode 100644 python/sglang/kernels/ops/attention/dsv4/tilelang_fp8_attention.py rename python/sglang/kernels/ops/attention/{nsa_triton_decode => dsv4/triton_sparse_decode}/__init__.py (96%) rename python/sglang/kernels/ops/attention/{nsa_triton_decode => dsv4/triton_sparse_decode}/triton_mla_kernels_decode_fused.py (100%) rename python/sglang/kernels/ops/attention/{nsa_triton_decode => dsv4/triton_sparse_decode}/triton_mla_kernels_decode_optimized.py (100%) rename python/sglang/kernels/ops/{kvcache/rope_cache.py => attention/fused_qk_rope_reshape_and_cache.py} (100%) rename python/sglang/kernels/ops/{kvcache => attention}/kv_indices.py (100%) rename python/sglang/kernels/ops/attention/{fla => linear}/fused_gdn_gating.py (100%) rename python/sglang/kernels/ops/attention/{fla => linear}/fused_kda_conv_recurrent_verify.py (100%) rename python/sglang/kernels/ops/attention/{fla => linear}/fused_recurrent_linear_replayssm.py (100%) rename python/sglang/kernels/ops/attention/{fla => linear}/fused_sigmoid_gating_recurrent.py (100%) rename python/sglang/kernels/ops/attention/{fla => linear}/gdn_replayssm_spec_decode.py (100%) rename python/sglang/kernels/ops/attention/{fla => linear}/gdn_replayssm_spec_fold.py (100%) rename python/sglang/kernels/ops/attention/{fla => linear}/kda_replayssm_spec_decode.py (100%) delete mode 100644 python/sglang/kernels/ops/attention/log_scaling_tau.py rename python/sglang/{srt/layers/attention/minimax_sparse_ops => kernels/ops/attention/minimax_sparse/prefill}/gluon_prefill.py (100%) create mode 100644 python/sglang/kernels/ops/attention/mla_qk_prep.py rename python/sglang/kernels/ops/{muse_spark/fused_qk_norm.py => attention/muse_spark_qk_norm_rope_store.py} (98%) rename python/sglang/kernels/ops/{muse_spark/helion_qk_norm.py => attention/muse_spark_qk_norm_rope_store_helion.py} (98%) rename python/sglang/kernels/ops/{kvcache => attention}/trtllm_mha_graph_metadata.py (100%) rename python/sglang/kernels/ops/{kvcache => attention}/trtllm_mha_page_table.py (100%) create mode 100644 python/sglang/kernels/ops/communication/attn_res_collective.py rename python/sglang/kernels/ops/{gemm => communication}/cutlass_all_gather_gemm.py (100%) rename python/sglang/kernels/ops/{gemm => communication}/cutlass_gemm_reduce_scatter.py (100%) create mode 100644 python/sglang/kernels/ops/communication/muse_spark_all_gather.py rename python/sglang/kernels/ops/{muse_spark/all_gather_norm_residual.py => communication/muse_spark_all_gather_norm_residual.py} (98%) rename python/sglang/kernels/ops/{muse_spark/fused_post_expert_norm_weighted_sum_trtllm.py => communication/muse_spark_moe_finalize_all_reduce.py} (95%) rename python/sglang/kernels/ops/{muse_spark/fi_trtllm_moe_finalize_with_rmsnorm_cutedsl.py => communication/muse_spark_moe_finalize_all_reduce_cutedsl.py} (88%) rename python/sglang/kernels/ops/{muse_spark/helion_post_expert.py => communication/muse_spark_moe_finalize_all_reduce_helion.py} (100%) rename python/sglang/kernels/ops/{gemm => communication}/sp_collective_gemm.py (76%) create mode 100644 python/sglang/kernels/ops/diffusion/modulate/indexed_scale_shift_mxfp8_triton.py rename python/sglang/kernels/ops/{attention => elementwise}/mla_output_gate.py (94%) rename python/sglang/kernels/ops/{muse_spark => elementwise}/row_sum.py (85%) rename python/sglang/kernels/ops/{speculative => embeddings}/ngram_embedding.py (100%) create mode 100644 python/sglang/kernels/ops/gemm/dsv4_indexer_head_weights_hip.py create mode 100644 python/sglang/kernels/ops/gemm/flashinfer_bmm_fp8.py rename python/sglang/kernels/ops/{elementwise/hc_mix.py => gemm/hc_mix_splitk.py} (97%) rename python/sglang/kernels/ops/{muse_spark/router_gemm.py => gemm/muse_spark_router_gemm.py} (100%) rename python/sglang/kernels/ops/{quantization => gemm}/mxfp8_aiter_gfx95.py (100%) create mode 100644 python/sglang/kernels/ops/gemm/mxfp8_amd_gfx95.py rename python/sglang/kernels/ops/{quantization => gemm}/mxfp8_flydsl_gfx95.py (100%) rename python/sglang/kernels/ops/{quantization => gemm}/mxfp8_gemv_gfx95_configs.json (100%) rename python/sglang/kernels/ops/{quantization => gemm}/mxfp8_native_amd_gfx95.py (100%) rename python/sglang/kernels/ops/{quantization => gemm}/nvfp4_gemm_swiglu_nvfp4_quant.py (100%) create mode 100644 python/sglang/kernels/ops/gemm/router_gate_matvec.py rename python/sglang/kernels/ops/{attention/fused_store_index_cache.py => kvcache/dsa_fused_store_index_cache.py} (100%) rename python/sglang/kernels/ops/{attention/dsa/index_buf_accessor.py => kvcache/dsa_index_k_cache.py} (100%) rename python/sglang/kernels/ops/{attention/dsv4/c128_cleanup.py => kvcache/dsv4_c128_cleanup.py} (100%) create mode 100644 python/sglang/kernels/ops/kvcache/dsv4_fp4_index_k_cache.py create mode 100644 python/sglang/kernels/ops/kvcache/dsv4_fp4_index_k_cache_hip.py rename python/sglang/kernels/ops/{attention/dsv4/index_buf_accessor.py => kvcache/dsv4_main_kv_store.py} (100%) rename python/sglang/kernels/ops/{attention/dsv4/request_window_copy.py => kvcache/dsv4_request_window_copy.py} (100%) create mode 100644 python/sglang/kernels/ops/kvcache/dsv4_unified_kv_store.py rename python/sglang/kernels/ops/{quantization => kvcache}/mxfp8_interleave_sf.py (100%) create mode 100644 python/sglang/kernels/ops/kvcache/mxfp8_quant_store_kv.py create mode 100644 python/sglang/kernels/ops/kvcache/shuffle_5d_cache.py rename python/sglang/kernels/ops/{muse_spark/fused_op_tuning.py => layernorm/_muse_spark_tuning.py} (81%) create mode 100644 python/sglang/kernels/ops/layernorm/fused_rmsnorm.py rename python/sglang/kernels/ops/{muse_spark/fused_norm_gate.py => layernorm/muse_spark_norm_gate.py} (96%) rename python/sglang/kernels/ops/{muse_spark/helion_norm_gate.py => layernorm/muse_spark_norm_gate_helion.py} (98%) rename python/sglang/kernels/ops/{muse_spark/norm_gate.py => layernorm/muse_spark_norm_gate_jit.py} (100%) rename python/sglang/kernels/ops/{muse_spark/fused_norm_residual.py => layernorm/muse_spark_norm_residual.py} (89%) rename python/sglang/kernels/ops/{muse_spark/norm_residual.py => layernorm/muse_spark_norm_residual_jit.py} (98%) rename python/sglang/kernels/ops/{muse_spark/norm_fp4_quant.py => layernorm/muse_spark_rmsnorm_nvfp4_quant.py} (67%) rename python/sglang/kernels/ops/{muse_spark/helion_norm_fp4_quant.py => layernorm/muse_spark_rmsnorm_nvfp4_quant_helion.py} (99%) rename python/sglang/kernels/ops/{quantization => layernorm}/rmsnorm_fake_quant_amd_gfx95.py (100%) create mode 100644 python/sglang/kernels/ops/layernorm/rmsnorm_mxfp8.py rename python/sglang/kernels/ops/{attention/fla/kda_prefill_graph_track.py => mamba/kda_prefill_track.py} (100%) rename python/sglang/kernels/ops/moe/{_jit_situ_and_mul_quant.py => kimi_k3_situ_and_mul_quant.py} (100%) rename python/sglang/kernels/ops/{quantization => moe}/minimax_quant_ue8m0.py (100%) rename python/sglang/kernels/ops/{muse_spark/fused_post_expert_norm_weighted_sum_triton.py => moe/muse_spark_post_expert_finalize_triton.py} (100%) rename python/sglang/kernels/ops/{muse_spark/fused_muse_spark_routing.py => moe/muse_spark_routing.py} (98%) rename python/sglang/kernels/ops/{muse_spark/routing.py => moe/muse_spark_routing_jit.py} (96%) delete mode 100644 python/sglang/kernels/ops/moe/triton_sigmoid_gate_mul.py delete mode 100644 python/sglang/kernels/ops/muse_spark/__init__.py create mode 100644 python/sglang/kernels/ops/quantization/act_quant_tilelang.py create mode 100644 python/sglang/kernels/ops/quantization/act_quant_triton.py rename python/sglang/kernels/ops/{attention/dsa/quant_k_cache.py => quantization/dsa_quant_k_cache.py} (83%) rename python/sglang/kernels/ops/{attention/dsv4/torch_quant.py => quantization/fp4_fake_quant.py} (100%) create mode 100644 python/sglang/kernels/ops/quantization/fp4_indexer_quant.py rename python/sglang/kernels/ops/{attention/dsv4/fp8_wo_a.py => quantization/fp8_group_major_quant_ue8m0.py} (98%) rename python/sglang/kernels/ops/{layernorm => quantization}/mxfp8_epilogue.py (55%) rename python/sglang/kernels/ops/{moe/rocm_moe_utils.py => quantization/rocm_upscale.py} (100%) create mode 100644 python/sglang/kernels/ops/sampling/softcap_row_argmax.py rename python/sglang/{kernels/ops/communication/mp.py => srt/distributed/comm_cleanup.py} (100%) rename python/sglang/{kernels/ops/attention/dsa/hip_gfx950/fused_decode.py => srt/layers/attention/dsa/gfx950_fused_indexer.py} (100%) create mode 100644 python/sglang/srt/layers/attention/dsv4/online_c128_mtp.py rename python/sglang/{kernels/ops/attention/dsv4/unified_kv_kernels/env_gate.py => srt/layers/attention/dsv4/unified_kv_gate.py} (100%) delete mode 100644 python/sglang/srt/layers/attention/minimax_sparse_ops/naive/flash_with_topk_idx.py delete mode 100644 python/sglang/srt/layers/attention/minimax_sparse_ops/naive/topk_sparse.py delete mode 100644 python/sglang/srt/layers/attention/nsa/index_buf_accessor.py delete mode 100644 python/sglang/srt/layers/attention/nsa/quant_k_cache.py delete mode 100644 python/sglang/srt/layers/attention/nsa/tilelang_kernel.py delete mode 100644 python/sglang/srt/layers/attention/nsa/triton_kernel.py create mode 100644 python/sglang/srt/layers/communication/muse_spark_finalize_workspace.py rename python/sglang/{kernels/ops/muse_spark/gemm_ag.py => srt/layers/communication/muse_spark_gemm_ag.py} (54%) create mode 100644 python/sglang/srt/layers/communication/sp_collective_gemm_workspace.py create mode 100644 python/sglang/test/kernels/deepseek_v4/fp4_index_k_reference.py rename python/sglang/{kernels/ops/quantization => test/kernels/deepseek_v4}/fp8_cvt.py (100%) create mode 100644 python/sglang/test/kernels/dsa_k_cache_reference.py create mode 100644 python/sglang/test/kernels/kv_canary/scatter_req_token_ids_reference.py rename python/sglang/{kernels/ops/muse_spark => test/kernels}/triton_sum.py (96%) create mode 100644 test/manual/kernels/attention/test_flash_fwd_mla_sm100.py rename python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_msa_fp8_parity.py => test/manual/kernels/attention/test_minimax_msa_fp8_parity.py (97%) rename test/manual/{layers/attention/dsa/test_get_k_scale_triton_kernel.py => kernels/kvcache/test_dsa_get_k_scale_triton_kernel.py} (98%) rename test/manual/{layers/attention/dsa/test_index_buf_accessor.py => kernels/kvcache/test_dsa_index_k_cache.py} (99%) rename test/manual/kernels/{attention => quantization}/check_dsa_quant_k_cache.py (94%) rename test/manual/{layers/attention/dsa => kernels/quantization}/test_act_quant_triton.py (97%) rename test/registered/kernels/benchmark/{elementwise => activation}/bench_bias_gelu.py (93%) rename test/registered/kernels/benchmark/{speculative => embeddings}/bench_ngram_compute_decode.py (98%) rename test/registered/kernels/benchmark/{speculative => embeddings}/bench_ngram_update_token_table.py (97%) rename test/registered/kernels/ops/{elementwise => activation}/test_bias_gelu.py (95%) create mode 100644 test/registered/kernels/ops/activation/test_swiglu_mxfp8_swizzled.py rename python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_flash_with_topk_idx.py => test/registered/kernels/ops/attention/test_minimax_sparse_flash_with_topk_idx.py (98%) rename python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_fp8_attn_gemm.py => test/registered/kernels/ops/attention/test_minimax_sparse_fp8_attn_gemm.py (96%) rename python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_sparse_gqa.py => test/registered/kernels/ops/attention/test_minimax_sparse_gqa.py (97%) create mode 100644 test/registered/kernels/ops/attention/test_muse_spark_qk_norm_rope_store.py create mode 100644 test/registered/kernels/ops/communication/test_muse_spark_moe_finalize_all_reduce.py create mode 100644 test/registered/kernels/ops/diffusion/test_indexed_scale_shift_mxfp8.py rename test/registered/kernels/ops/{attention => elementwise}/test_mla_output_gate.py (95%) create mode 100644 test/registered/kernels/ops/elementwise/test_row_sum.py rename test/registered/kernels/ops/{moe => elementwise}/test_sigmoid_gate_mul.py (81%) rename test/registered/kernels/ops/{moe => elementwise}/test_sigmoid_gate_mul_broadcast.py (89%) rename test/registered/kernels/ops/{speculative => embeddings}/test_ngram_embedding.py (98%) rename test/registered/kernels/ops/{quantization => gemm}/test_mxfp8_amd_gfx95.py (98%) rename test/registered/kernels/ops/{attention => kvcache}/test_dsv41_request_window_copy.py (100%) create mode 100644 test/registered/kernels/ops/kvcache/test_dsv4_fp4_index_k_cache.py rename test/registered/kernels/ops/{attention => kvcache}/test_fused_store_index_cache.py (99%) rename test/registered/kernels/ops/layernorm/{test_hc_combine.py => test_mhc_combine.py} (82%) create mode 100644 test/registered/kernels/ops/layernorm/test_muse_spark_norm_ops.py create mode 100644 test/registered/kernels/ops/layernorm/test_muse_spark_rmsnorm_nvfp4_quant.py create mode 100644 test/registered/kernels/ops/moe/test_muse_spark_routing.py create mode 100644 test/registered/kernels/ops/quantization/test_fp4_indexer_quant.py rename test/registered/kernels/ops/{attention => quantization}/test_fp8_wo_a.py (97%) diff --git a/.agents/skills/kernel-organization/SKILL.md b/.agents/skills/kernel-organization/SKILL.md index 40081866b4..b2dd9607ef 100644 --- a/.agents/skills/kernel-organization/SKILL.md +++ b/.agents/skills/kernel-organization/SKILL.md @@ -17,25 +17,47 @@ logical grouping and remove the old `_jit_` filename prefix. The `sgl_kernel` wheel retains its own public API and packaging tests. - Group by computation, not model or GPU: GEMM and GEMV in `gemm`, expert routing in `moe`, attention index selection in `attention`, sampling in - `sampling`, normalization in `layernorm`. A quantized GEMM is still a GEMM; - quantization alone belongs in `quantization`. + `sampling`, normalization in `layernorm`. +- Classify a fused operator by its complete contract; do not split a fused + kernel into separate launches just to separate its parts. Split unrelated + public entry points that happen to share a source file. Decided rules: + - Any cross-rank collective or peer-memory access (all-reduce, all-gather, + reduce-scatter, a2a, multicast/NVLS) puts the op in `communication`, + whatever compute it fuses. + - Producer + output quantization stays with the producer (norm + quant in + `layernorm`, activation + quant in `activation`, GEMM with a quant epilogue + in `gemm`, MoE dispatch/permute + quant in `moe`). `quantization` holds only + pure format conversion: quant, dequant, scale-layout conversion, repack. + - RoPE stays in `attention` (standalone RoPE, position ids, QK-norm + RoPE). + - `kvcache` owns ops that read/write KV-cache contents; `attention` owns the + index/shape/schedule tables (page tables, `cu_seqlens`, `kv_indices`, split + metadata) and index selection. A fused compute + store stays with its + compute group. - Model-specific files/subpackages and tuning data are allowed **inside** a logical group. Do not add a model bundle or an implementation file directly under `ops/`. Propose a new logical group only for a distinct responsibility, - and update `ops._GROUPS`, documentation, and tests together. -- Classify a fused operator by its complete contract. Do not split a fused - kernel into separate launches just to separate norm, RoPE, or quantization. - Split unrelated public entry points that happen to share a source file. -- Keep shared CUDA build/runtime infrastructure in `kernels/jit`; operator - wrappers call it from their logical group. Keep process groups, communicator - state, model dispatch, and buffer ownership in `srt`. K3-specific adapters in - `srt/layers/communication/` need not pretend to be generic interfaces. + and update `ops._GROUPS`, the README group list, and tests together + (`test_kernels_namespace.py` checks the README against `_GROUPS` and the + README layout against the directories under `kernels/`). +- Agent-generated implementations live in `kernels/kda_kernels/` (Kernel Design + Agents, `KernelBackend.KDA`) and `kernels/cake_kernels/` (Cake adapters over + FlashInfer). They are backends, not groups: register their public ops in the + logical group with a `KernelSpec` whose target points into them, and do not + reorganize those directories. +- Keep shared build/runtime infrastructure (including the Helion runtime + helper) in `kernels/jit`; operator wrappers call it from their logical group. + Keep process groups, communicator state, workspace classes, model dispatch, + and buffer ownership in `srt` (`srt/layers/communication/` for the K3 and Muse + Spark adapters); kernels must not import srt runtime modules to inspect + model objects, so pass flags in. Adapters there need not pretend to be generic + interfaces. ## Register the public entry point Add lazy `KernelSpec` metadata in the owning group's `__init__.py`, or use the existing `BaseFusedOp` registration when the operation has interchangeable -backends. Group imports must not import GPU implementations or compile kernels. +backends (a `BaseFusedOp` built per layer registers `module:Class.forward_` +targets). Group imports must not import GPU implementations or compile kernels. Use `.` for the op id and `module:callable` for the target. Preserve existing backend dispatch and describe actual device/architecture restrictions @@ -53,8 +75,9 @@ factories, reference helpers, and runtime classes are not separate kernel APIs. runtime module. A CUDA allocation does not make a runtime test a kernel test. - Non-CI smoke scripts and benchmarks: `test/manual/kernels/`; do not leave standalone test/benchmark entry points in production operator modules. -- Shared test helpers: `sglang.test.kernels`. Preserve vendored upstream trees - and AOT wheel packaging boundaries instead of reorganizing them incidentally. +- Shared test helpers and reference oracles: `sglang.test.kernels`. Preserve + vendored upstream trees and AOT wheel packaging boundaries instead of + reorganizing them incidentally. For a move/split, preserve assertions, parametrization, fixtures, platform skips, execution entry points, and CI stages/runners. Apportion existing time estimates diff --git a/.agents/skills/llm-torch-profiler-analysis/references/fuse-overlap-catalog.md b/.agents/skills/llm-torch-profiler-analysis/references/fuse-overlap-catalog.md index a3d4cd394d..1b3a0f3881 100644 --- a/.agents/skills/llm-torch-profiler-analysis/references/fuse-overlap-catalog.md +++ b/.agents/skills/llm-torch-profiler-analysis/references/fuse-overlap-catalog.md @@ -49,15 +49,15 @@ in-flight row as shipped. | FlashInfer unified `allreduce_fusion` | `cross_device_reduce_1stage*`
`all_reduce`
`FusedAddRMSNormKernel`
`rmsnorm*` | `python/sglang/srt/layers/flashinfer_comm_fusion.py`
`python/sglang/srt/layers/layernorm.py::forward_with_allreduce_fusion`
`python/sglang/srt/layers/layer_boundary/residual/add_norm.py::apply_flashinfer_allreduce_fusion` | FlashInfer workspace creation plus `allreduce_fusion(..., pattern=AllReduceFusionPattern.kARResidualRMSNorm, ...)` | First suspect missing / disabled / unsupported FlashInfer allreduce fusion, not a brand new TP fusion idea. | | AITER allreduce fusion | ROCm all-reduce plus RMSNorm still split | `python/sglang/srt/layers/layernorm.py::forward_with_allreduce_fusion`
`python/sglang/srt/distributed/communication_op.py::tensor_model_parallel_fused_allreduce_rmsnorm`
`python/sglang/srt/layers/layer_boundary/residual/add_norm.py::apply_aiter_all_reduce_fusion` | ROCm-side fused TP all-reduce + RMSNorm with fallback to plain all-reduce plus norm | On AMD, rule out existing AITER fusion before proposing a new communication fusion. | | Fused activation-and-mul (`SwiGLU` / `GeGLU`) | `silu_and_mul`
`gelu_and_mul`
`npu_swiglu` | `python/sglang/srt/layers/activation.py` | Single op covers activation plus elementwise multiply across CUDA / CPU / NPU / XPU backends | Treat separate activation + mul on packed MLP outputs as missing existing fusion. | -| Fused dual residual RMSNorm | residual add plus two RMSNorm-like kernels around Grok blocks | `python/sglang/srt/layers/elementwise.py::fused_dual_residual_rmsnorm`
`python/sglang/srt/models/grok.py` | One Triton kernel computes intermediate residual update and next RMSNorm output together | On Grok-like residual layouts, treat split residual + norm as missing existing fusion. | +| Fused dual residual RMSNorm | residual add plus two RMSNorm-like kernels around Grok blocks | `python/sglang/kernels/ops/layernorm/fused_rmsnorm.py::fused_dual_residual_rmsnorm`
`python/sglang/srt/models/grok.py` | One Triton kernel computes intermediate residual update and next RMSNorm output together | On Grok-like residual layouts, treat split residual + norm as missing existing fusion. | | In-place QK RMSNorm | split `q_norm` / `k_norm` kernels | `python/sglang/srt/models/utils.py::apply_qk_norm`
`python/sglang/kernels/ops/layernorm/norm.py::fused_inplace_qknorm` | In-place JIT QK norm plus optional `alt_stream` overlap for K | Check shape, dtype, deterministic mode, and in-place legality before proposing a new QK fuse. | | TorchInductor horizontal Q/K norm combo-kernels | `combo_kernels`
`benchmark_combo_kernel`
`q_norm`
`k_norm`
`split_with_sizes` | `torch._inductor.config.combo_kernels` | TorchInductor can horizontally fuse sibling Q-norm and K-norm kernels in compiled traces, often deleting `split_with_sizes` / `clone` ladders | Treat separate Q/K norm ladders in compile-heavy traces as an existing compiler-fusion family first. | | MiniMax TP fused QK RMSNorm | `MiniMaxM2RMSNormTP`
`rms_sumsq_serial`
`rms_apply_serial`
`forward_qk` | `python/sglang/srt/models/minimax_m2.py` | Triton kernels compute Q / K sumsq together, TP all-reduces shared stats, then apply both RMSNorms together | On MiniMax traces, separate Q norm and K norm are usually a missed model-specific Triton fusion. | | Fused QK RMSNorm + RoPE | `qknorm*` + `rope*` + `rotary*` as separate steps | `python/sglang/kernels/ops/attention/fused_qknorm_rope.py`
`python/sglang/srt/models/qwen3_moe.py` | One JIT kernel applies QK RMSNorm and RoPE in-place on packed QKV | For compatible LLMs, classify split QK norm + RoPE as a missing existing fusion. | -| Fused QK RoPE reshape + KV cache write | `fused_qk_rope_reshape_and_cache*`
RoPE followed by reshape / cache DtoD | `python/sglang/srt/layers/attention/utils.py::fused_qk_rope_reshape_and_cache` | One Triton kernel applies RoPE to Q / K, reshapes cache layout, and writes K / V directly to paged cache | Treat separate RoPE + reshape + cache-write ladders as an existing attention-prep fusion family. | +| Fused QK RoPE reshape + KV cache write | `fused_qk_rope_reshape_and_cache*`
RoPE followed by reshape / cache DtoD | `python/sglang/kernels/ops/attention/fused_qk_rope_reshape_and_cache.py::fused_qk_rope_reshape_and_cache` | One Triton kernel applies RoPE to Q / K, reshapes cache layout, and writes K / V directly to paged cache | Treat separate RoPE + reshape + cache-write ladders as an existing attention-prep fusion family. | | Fused RoPE + KV cache store | `fused_set_kv_buffer`
RoPE followed by KV-store, DtoD, or cache-write kernels | `python/sglang/kernels/ops/attention/rope.py`
`python/sglang/srt/models/utils.py::enable_fused_set_kv_buffer` | Shared entrypoints can route to fused RoPE + KV-store or model-side `fused_set_kv_buffer` fast paths | Compare against the fused cache-store path before proposing a new KV rewrite. | | Fused decode metadata setup | `normal_decode_set_metadata`
`cache_seqlens_int32`
`cu_seqlens_k`
`page_table`
`swa_page_table` | `python/sglang/srt/layers/attention/flashattention_backend.py::normal_decode_set_metadata` | Triton decode path fuses seq-len cast/add, prefix-sum, req-to-token gather, page-table divide, and optional SWA metadata build into 1-2 kernels | If decode exposes multiple tiny metadata kernels before attention, first compare against this existing fused metadata-prep path. | -| NSA fused metadata copy for graph replay | `fused_metadata_copy`
`fused_metadata_copy_multi`
`fused_nsa_cache_seqlens`
`fused_flashmla_metadata` | `python/sglang/kernels/ops/attention/fused_metadata_copy.py` | CUDA graph replay path fuses multiple metadata copies into one kernel or one multi-destination kernel | Treat bursts of tiny metadata-copy kernels around NSA replay as a missed existing replay fusion. | +| NSA fused metadata copy for graph replay | `fused_metadata_copy`
`fused_metadata_copy_multi`
`fused_nsa_cache_seqlens`
`fused_flashmla_metadata` | `python/sglang/kernels/ops/attention/dsa/fused_metadata_copy.py` | CUDA graph replay path fuses multiple metadata copies into one kernel or one multi-destination kernel | Treat bursts of tiny metadata-copy kernels around NSA replay as a missed existing replay fusion. | | DeepSeek MLA fused projection + norm + RoPE | `qkv_proj_with_rope_fused_weight`
`fused_qkv_a_proj_with_mqa`
`forward_absorb_fused_mla_rope*` | `python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_fused_rope_cpu.py`
`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla_fused_rope_rocm.py`
`python/sglang/srt/models/deepseek_v2.py` | CPU / ROCm paths fuse DeepSeek MLA projection packing with q / k norm, RoPE, and cache-oriented MLA prep | For DeepSeek MLA, split proj / norm / rope prep is usually an existing backend-specific fuse that did not fire. | | Fused QK RoPE concat + MLA cache write | `fused_qk_rope_cat_and_cache_mla`
`set_mla_kv_buffer` | `python/sglang/srt/layers/rocm_linear_utils.py`
`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py` | ROCm MLA path can fuse Q / K RoPE packing, concat, and MLA cache write in one backend-specific op | On DeepSeek / MLA traces, separate RoPE-cat-cache steps are not automatically novel. | | Qwen3 decode fused QK norm + 3D mRoPE + KV cache write | `fused_qk_norm_mrope_3d_cache_pts_quant_shuffle`
`mrope`
decode cache write | `python/sglang/srt/models/qwen3.py` | ROCm / AITER decode path fuses QK norm, 3D mRoPE, and paged KV cache write | On Qwen3-style decode, separate norm + mRoPE + cache-store kernels are not a novel opportunity. | @@ -69,18 +69,18 @@ in-flight row as shipped. | Qwen-style shared-expert append into routed top-k output | `_append_shared_to_topk_output`
`fused_append_shared_experts_with_weights`
`num_fused_shared_experts` | `python/sglang/srt/models/qwen2_moe.py`
`python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe_triton_kernels.py` | Qwen-style MoE paths can append shared-expert ids and sigmoid gate weights to routed top-k output in one Triton kernel so the shared experts execute inside the fused MoE path | Treat routed top-k plus shared-expert pad / concat ladders as an existing MoE-prep fusion family first. | | Fused MoE dispatch / permute / combine | token permutation
dispatch / combine
grouped top-k
many small MoE support kernels | `python/sglang/srt/layers/moe/fused_moe_triton/layer.py`
`python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py` | `FusedMoE` plus DeepEP / FlashInfer / FuseEP / standard dispatch backends and `permute_fusion=True` | First ask whether the model is missing an existing `FusedMoE`-style path or backend-specific dispatcher path. | | Fused MoE sum + all-reduce | routed MoE followed by explicit sum-reduce kernels | `python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py`
`python/sglang/srt/layers/moe/fused_moe_triton/fused_moe_triton_kernels.py` | `fuse_sum_all_reduce=True` path in the second MoE GEMM | Before inventing a new MoE reduction fuse, check whether `enable_fused_moe_sum_all_reduce` is simply off or the quant path is incompatible. | -| Fused MoE activation + quant / re-quant | `silu_and_mul_*quant*`
`npu_dequant_swiglu_quant`
`swiglu_quant` | `python/sglang/srt/layers/moe/ep_moe/kernels.py`
`python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py`
`python/sglang/srt/layers/moe/cutlass_w4a8_moe.py`
`python/sglang/srt/hardware_backend/npu/quantization/moe_methods.py` | Quantized MoE backends fuse SwiGLU / SiLU-and-mul with FP8 / FP4 / NPU re-quant before the second expert GEMM | If MoE traces show standalone activation then quant kernels, first check whether the quantized fused path is missing. | +| Fused MoE activation + quant / re-quant | `silu_and_mul_*quant*`
`npu_dequant_swiglu_quant`
`swiglu_quant` | `python/sglang/srt/layers/moe/ep_moe/kernels.py`
`python/sglang/kernels/ops/gemm/nvfp4_gemm_swiglu_nvfp4_quant.py`
`python/sglang/srt/layers/moe/cutlass_w4a8_moe.py`
`python/sglang/srt/hardware_backend/npu/quantization/moe_methods.py` | Quantized MoE backends fuse SwiGLU / SiLU-and-mul with FP8 / FP4 / NPU re-quant before the second expert GEMM | If MoE traces show standalone activation then quant kernels, first check whether the quantized fused path is missing. | | DeepSeek comm-prep fused RMSNorm + quant / flatten-quant | `fused_rms_fp8_group_quant`
`fused_rms_mxfp4_quant`
`fused_flatten_fp8_group_quant`
`fused_flatten_mxfp4_quant` | `python/sglang/srt/layers/layer_boundary/residual/add_norm.py`
`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py`
`python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py` | DeepSeek MLA / MHA ROCm paths fuse RMSNorm or flatten with FP8 / MXFP4 quantization for comm / attention prep | On DeepSeek quant traces, split norm + quant or flatten + quant is an existing family, not a new idea. | | NSA fused top-k transform / page-table build | `fast_topk_transform_fused`
`fast_topk_transform_ragged_fused` | `python/sglang/srt/layers/attention/dsa_backend.py` | NSA can fuse top-k selection with paged / ragged index transform instead of separate top-k plus metadata scatter | If NSA top-k metadata work is split, check `SGLANG_DSA_FUSE_TOPK` and backend support first. | -| NSA fused quantize + indexed K-cache store | `fused_store_index_k_cache`
`act_quant`
`index_k_with_scale_buffer` | `python/sglang/kernels/ops/attention/fused_store_index_cache.py`
`python/sglang/srt/layers/attention/nsa/nsa_indexer.py` | Single JIT kernel quantizes bf16 K to fp8 + scale and writes directly into NSA index cache | Treat split `act_quant` + buffer-store on CUDA as missing an existing fused store path. | +| NSA fused quantize + indexed K-cache store | `fused_store_index_k_cache`
`act_quant`
`index_k_with_scale_buffer` | `python/sglang/kernels/ops/kvcache/dsa_fused_store_index_cache.py`
`python/sglang/srt/layers/attention/nsa/nsa_indexer.py` | Single JIT kernel quantizes bf16 K to fp8 + scale and writes directly into NSA index cache | Treat split `act_quant` + buffer-store on CUDA as missing an existing fused store path. | | Fused sampling temperature + softmax | `fused_temperature_softmax*` | `python/sglang/srt/layers/fused_sampling.py`
`python/sglang/srt/layers/sampler.py` | Triton single-pass / multi-pass kernels fuse temperature scaling and softmax during decode | Separate temp-divide + softmax at decode batch sizes is often a missed existing fusion. | | Fused logit softcap | `fused_softcap`
`final_logit_softcapping` | `python/sglang/srt/layers/elementwise.py`
`python/sglang/srt/layers/logits_processor.py` | Triton kernels fuse cast-to-float and softcap / tanh math for logits or generic elementwise softcapping | Treat exposed cast + softcap ladders as an existing Triton fuse family. | | Linear-attention packed projection reshuffle | `fused_qkvzba_split_reshape_cat*`
`qkvz_proj`
`ba_proj`
`qkvabz_proj`
`fused_qkvbfg_a_proj` | `python/sglang/kernels/ops/attention/triton_gdn_fused_proj.py`
`python/sglang/srt/models/qwen3_next.py`
`python/sglang/srt/models/qwen3_5.py`
`python/sglang/srt/models/kimi_linear.py`
`python/sglang/srt/models/jet_nemotron.py` | GDN / Kimi / Jet-style linear-attn models pack multiple projections, then fuse split / reshape / cat into one kernel | Treat split reshape / transpose / cat ladders as an existing linear-attention fusion family. | -| Fused GDN gating prep | `fused_gdn_gating`
`softplus`
`beta_output` | `python/sglang/srt/layers/attention/fla/fused_gdn_gating.py` | Triton kernel computes GDN gate preparation such as `-exp(A_log) * softplus(...)` and `sigmoid(b)` together | On GDN traces, treat split gate-prep elementwise kernels as missing existing fusion first. | +| Fused GDN gating prep | `fused_gdn_gating`
`softplus`
`beta_output` | `python/sglang/kernels/ops/attention/linear/fused_gdn_gating.py` | Triton kernel computes GDN gate preparation such as `-exp(A_log) * softplus(...)` and `sigmoid(b)` together | On GDN traces, treat split gate-prep elementwise kernels as missing existing fusion first. | | Fused RMSNorm-gated linear-attention output | `FusedRMSNormGated`
`layer_norm_gated_fwd` | `python/sglang/srt/layers/attention/fla/fused_norm_gate.py`
`python/sglang/srt/models/qwen3_next.py`
`python/sglang/srt/models/kimi_linear.py` | One Triton op covers residual-aware (RMS)Norm plus sigmoid / swish gating | If norm and output gate appear as separate kernels in GDN / Kimi-like blocks, first suspect a missing existing fusion. | | Fused gated RMSNorm / LayerNorm | `rms_norm_gated`
`layer_norm_gated` | `python/sglang/srt/layers/attention/mamba/ops/layernorm_gated.py` | Mamba-derived kernels can fuse normalization with the gating branch `z * sigmoid(z)` | Treat split norm and gate post-processing on Mamba-style blocks as an existing fusion family. | | Fused linear-attention chunk KKT + solve_tril | `chunk_gated_delta_rule_fwd_kkt_solve_kernel`
`scaled_dot_kkt`
`solve_tril`
`recompute_w_u` | `python/sglang/srt/layers/attention/fla/chunk_fwd.py`
`python/sglang/srt/layers/attention/fla/kda.py` | GDN / KDA chunk forward fuses `scaled_dot_kkt + solve_tril` in the prefill / intra-chunk path, then finishes `recompute_w_u` as the next step | Treat split KKT + triangular-solve ladders as an existing linear-attention fusion family first. | -| Fused linear-attention recurrent / KDA update | `fused_sigmoid_gating_delta_rule_update`
`fused_recurrent_gated_delta_rule_update`
`fused_kda_gate` | `python/sglang/srt/layers/attention/fla/fused_sigmoid_gating_recurrent.py`
`python/sglang/srt/layers/attention/fla/fused_recurrent.py`
`python/sglang/srt/models/kimi_linear.py`
`python/sglang/srt/models/jet_nemotron.py` | Triton / CuTeDSL kernels fuse gating math, optional QK l2norm, recurrent state update, and output generation | Treat split gating + recurrent-update chains as existing linear-attention fusion, not a novel opportunity. | +| Fused linear-attention recurrent / KDA update | `fused_sigmoid_gating_delta_rule_update`
`fused_recurrent_gated_delta_rule_update`
`fused_kda_gate` | `python/sglang/kernels/ops/attention/linear/fused_sigmoid_gating_recurrent.py`
`python/sglang/kernels/ops/attention/fla/fused_recurrent.py`
`python/sglang/srt/models/kimi_linear.py`
`python/sglang/srt/models/jet_nemotron.py` | Triton / CuTeDSL kernels fuse gating math, optional QK l2norm, recurrent state update, and output generation | Treat split gating + recurrent-update chains as existing linear-attention fusion, not a novel opportunity. | | Fused Mamba state gather/scatter with mask | `fused_mamba_state_scatter_with_mask`
`index_elementwise_kernel` | `python/sglang/srt/layers/attention/mamba/mamba_state_scatter_triton.py` | Triton kernel replaces multiple masked gather / scatter index kernels with one fused update | If Mamba verify/update shows many tiny index kernels, first compare against this existing fused path. | | Staging-buffer fused gather / scatter | `_fused_gather_to_staging_kernel`
`_fused_scatter_from_staging_kernel` | `python/sglang/srt/disaggregation/common/staging_buffer.py` | Triton kernels gather scattered KV slices into contiguous staging memory and scatter them back into KV cache on decode | Treat ladders of tiny gather/scatter/copy kernels in heterogeneous TP staging as missing an existing Triton fusion. | @@ -146,7 +146,7 @@ Stable entries should be folded into the mainline family rows above. | PR `#22005` fused add + RMSNorm + per-token FP8 quant | `fused_add_rmsnorm_per_token_quant`
`per_token_quant_fp8` | `PR #22005`
`python/sglang/kernels/jit/csrc/elementwise/fused_add_rmsnorm_per_token_quant.cuh`
`python/sglang/kernels/jit/fused_add_rmsnorm_per_token_quant.py` | CUDA JIT kernel keeps normed values in registers and emits BF16 + FP8 outputs plus per-token scales | If FP8 online-quant traces show add+norm followed by per-token quant, treat this as an in-flight upstream CUDA fuse family. | | PR `#20667` Qwen3.5 fused QK norm + RoPE + KV cache write | `fused_qk_norm_rope_cache_pts_quant_shuffle`
`fused_qk_norm_mrope_3d_cache_pts_quant_shuffle`
`rotary_dim` | `PR #20667`
`python/sglang/srt/models/qwen3_5.py`
`python/sglang/srt/models/utils.py` | ROCm / AITER path fuses Q / K RMSNorm, partial or 3D RoPE, and direct KV cache write for Qwen3.5 attention | Treat split QK-norm + RoPE + cache-store on Qwen3.5 as a concrete in-flight upstream family, not a novel idea. | | PR `#22392` CUTLASS FP8 GEMM replacing nvjet | `cutlass_scaled_mm`
`fp8_scaled_mm`
`nvjet`
`cudaMemsetAsync` | `PR #22392`
`python/sglang/kernels/aot/python/sgl_kernel/gemm.py`
`python/sglang/srt/layers/quantization/fp8_utils.py` | Runtime replacement swaps nvjet FP8 GEMMs for CUTLASS kernels, removing per-launch memset bubbles and extra output-copy kernels | Treat nvjet GEMM + memset bubble ladders as an in-flight SGLang linear-kernel family before calling them novel. | -| PR `#18612` NVFP4 CUTLASS MoE fused SiLU+Mul+quant | `silu_and_mul_scaled_nvfp4`
`nvfp4 expert quant`
`cutlass moe` | `PR #18612`
`python/sglang/srt/layers/moe/cutlass_w4a8_moe.py`
`python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py` | Fuses MoE activation epilogue and NVFP4 expert quantization before the CUTLASS MoE second GEMM | Treat split SiLU+Mul then NVFP4 expert quant in CUTLASS MoE traces as an in-flight upstream SGLang family. | +| PR `#18612` NVFP4 CUTLASS MoE fused SiLU+Mul+quant | `silu_and_mul_scaled_nvfp4`
`nvfp4 expert quant`
`cutlass moe` | `PR #18612`
`python/sglang/srt/layers/moe/cutlass_w4a8_moe.py`
`python/sglang/kernels/ops/gemm/nvfp4_gemm_swiglu_nvfp4_quant.py` | Fuses MoE activation epilogue and NVFP4 expert quantization before the CUTLASS MoE second GEMM | Treat split SiLU+Mul then NVFP4 expert quant in CUTLASS MoE traces as an in-flight upstream SGLang family. | | PR `#22918` FlashInfer per-token NVFP4 MoE | `per_token_nvfp4`
`trtllm_fp4_block_scale_moe`
`FlashInfer MoE` | `PR #22918`
`python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py` | Adds FlashInfer-backed per-token NVFP4 MoE execution so expert quant/dequant work can move into the fused MoE backend | Treat standalone per-token NVFP4 MoE support kernels as a candidate missing backend-selection path, not an automatically novel kernel idea. | | PR `#22851` NSA top-k backend and FlashInfer / PyTorch top-k split | `nsa topk`
`flashinfer_topk`
`pytorch_topk`
`fast_topk_transform` | `PR #22851`
`python/sglang/srt/layers/attention/dsa_backend.py` | Makes NSA top-k backend selection explicit and aligns fused top-k transform with FlashInfer / PyTorch fallbacks | When NSA top-k dominates decode, first classify it as backend selection or fused-transform eligibility work. | | PR `#24125` GLM5 NSA decode CatArrayBatchedCopy removal | `CatArrayBatchedCopy`
`GLM-5`
`NSA`
`TileLang decode` | `PR #24125`
`python/sglang/srt/layers/attention/dsa_backend.py` | Skips redundant cat/copy work in the GLM5 NSA TileLang decode path | Treat cat/copy bursts in GLM5 NSA decode as a concrete in-flight cleanup opportunity. | diff --git a/.agents/skills/llm-torch-profiler-analysis/scripts/triage_kernel_helpers.py b/.agents/skills/llm-torch-profiler-analysis/scripts/triage_kernel_helpers.py index 5d111ac9d3..700cf79048 100644 --- a/.agents/skills/llm-torch-profiler-analysis/scripts/triage_kernel_helpers.py +++ b/.agents/skills/llm-torch-profiler-analysis/scripts/triage_kernel_helpers.py @@ -482,7 +482,9 @@ FUSION_PATTERN_REGISTRY: Tuple[FusionPatternSpec, ...] = ( ), FusionPatternSpec( pattern="Fused QK RoPE reshape + KV cache write", - candidate_path="python/sglang/srt/layers/attention/utils.py", + candidate_path=( + "python/sglang/kernels/ops/attention/fused_qk_rope_reshape_and_cache.py" + ), active_keywords=("fused_qk_rope_reshape_and_cache",), split_groups=( ("rotary", "rope", "mrope"), @@ -533,7 +535,7 @@ FUSION_PATTERN_REGISTRY: Tuple[FusionPatternSpec, ...] = ( ), FusionPatternSpec( pattern="NSA fused metadata copy for graph replay", - candidate_path="python/sglang/kernels/ops/attention/fused_metadata_copy.py", + candidate_path="python/sglang/kernels/ops/attention/dsa/fused_metadata_copy.py", active_keywords=( "fused_metadata_copy", "fused_metadata_copy_multi", @@ -700,7 +702,7 @@ FUSION_PATTERN_REGISTRY: Tuple[FusionPatternSpec, ...] = ( pattern="Fused MoE activation + quant / re-quant", candidate_path=( "python/sglang/srt/layers/moe/ep_moe/kernels.py" - "
python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py" + "
python/sglang/kernels/ops/gemm/nvfp4_gemm_swiglu_nvfp4_quant.py" "
python/sglang/srt/layers/moe/cutlass_w4a8_moe.py" ), active_keywords=( @@ -761,7 +763,7 @@ FUSION_PATTERN_REGISTRY: Tuple[FusionPatternSpec, ...] = ( FusionPatternSpec( pattern="NSA fused quantize + indexed K-cache store", candidate_path=( - "python/sglang/kernels/ops/attention/fused_store_index_cache.py" + "python/sglang/kernels/ops/kvcache/dsa_fused_store_index_cache.py" "
python/sglang/srt/layers/attention/nsa/nsa_indexer.py" ), active_keywords=("fused_store_index_k_cache",), diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2f7543cb11..1190d19032 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,6 +7,7 @@ /python/sglang/kernels @DarkSharpness @HaiShaw @BBuf @celve @HydraQYH @yuan-luo /python/sglang/kernels/ops/diffusion @yingluosanqian @BBuf @mickqian @triple-mu @niehen6174 /python/sglang/kernels/ops/attention/fla @yizhang2077 @hebiao064 @yuan-luo +/python/sglang/kernels/ops/attention/linear @yizhang2077 @hebiao064 @yuan-luo /python/sglang/kernels/ops/lora @Ying1123 @Fridge003 @lifuhuang @yushengsu-thu @jybsuper /python/sglang/kernels/jit/csrc/lora @Ying1123 @Fridge003 @lifuhuang @yushengsu-thu @jybsuper /python/sglang/multimodal_gen @mickqian @ping1jing2 @HaiShaw @yichiche @AgainstEntropy @BBuf @triple-mu @kevin-mii @niehen6174 @@ -107,6 +108,7 @@ /python/sglang/srt/speculative/cpp_ngram @hnyls2002 @Qiaolin-Yu @kpham-sgl /python/sglang/srt/speculative/frozen_kv_mtp_*.py @hnyls2002 @Qiaolin-Yu @kpham-sgl @pyc96 /python/sglang/kernels/ops/speculative/ngram_*.py @hnyls2002 @Qiaolin-Yu @kpham-sgl +/python/sglang/kernels/ops/embeddings/ngram_embedding.py @hnyls2002 @Qiaolin-Yu @kpham-sgl /python/sglang/kernels/jit/csrc/ngram_corpus @hnyls2002 @Qiaolin-Yu @kpham-sgl /python/sglang/srt/models/laguna.py @Jiminator /python/sglang/srt/configs/laguna.py @Jiminator diff --git a/.github/workflows/_pr-test-check-changes.yml b/.github/workflows/_pr-test-check-changes.yml index fde497c528..5670f4cddb 100644 --- a/.github/workflows/_pr-test-check-changes.yml +++ b/.github/workflows/_pr-test-check-changes.yml @@ -95,7 +95,7 @@ jobs: - "python/setup.py" - "python/sglang/!(multimodal_gen|kernels)/**/!(*.md)" # Diffusion-only kernels and tests are left to multimodal_gen and - # jit_kernel; ops/diffusion/common stays since srt imports it via kda_kernels. + # jit_kernel; ops/diffusion/common stays since kda_kernels (an srt path) imports it. - "python/sglang/kernels/{,ops/,jit/,jit/csrc/}!(*.md)" - "python/sglang/kernels/!(aot|ops|jit)/**/!(*.md)" - "python/sglang/kernels/jit/!(csrc)/**/!(*.md)" diff --git a/benchmark/bench_linear_attention/bench_cutedsl_kda_decode.py b/benchmark/bench_linear_attention/bench_cutedsl_kda_decode.py index f501d54547..c9c70c7f59 100644 --- a/benchmark/bench_linear_attention/bench_cutedsl_kda_decode.py +++ b/benchmark/bench_linear_attention/bench_cutedsl_kda_decode.py @@ -20,10 +20,10 @@ import triton from sglang.kernels.ops.attention.cutedsl_kda import ( cutedsl_fused_sigmoid_gating_kda_update, ) -from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( +from sglang.kernels.ops.attention.fla.kda import chunk_kda +from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, ) -from sglang.kernels.ops.attention.fla.kda import chunk_kda def make_inputs( diff --git a/benchmark/bench_linear_attention/bench_gdn_decode.py b/benchmark/bench_linear_attention/bench_gdn_decode.py index 58bd9d3c31..b89ac1e1ba 100644 --- a/benchmark/bench_linear_attention/bench_gdn_decode.py +++ b/benchmark/bench_linear_attention/bench_gdn_decode.py @@ -32,7 +32,7 @@ import triton from sglang.kernels.ops.attention.fla.fused_recurrent import ( fused_recurrent_gated_delta_rule_packed_decode, ) -from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( +from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, ) diff --git a/benchmark/bench_linear_attention/bench_kda_decode.py b/benchmark/bench_linear_attention/bench_kda_decode.py index f5846a387e..2c0de97932 100644 --- a/benchmark/bench_linear_attention/bench_kda_decode.py +++ b/benchmark/bench_linear_attention/bench_kda_decode.py @@ -26,12 +26,12 @@ import torch from sglang.kernels.ops.attention.fla.fused_recurrent import ( fused_recurrent_kda_packed_decode, ) -from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( - fused_sigmoid_gating_delta_rule_update, -) from sglang.kernels.ops.attention.helion.kda_decode import ( helion_fused_recurrent_kda_packed_decode, ) +from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( + fused_sigmoid_gating_delta_rule_update, +) def make_inputs( diff --git a/benchmark/kernels/attention/bench_gdn_replayssm_decode.py b/benchmark/kernels/attention/bench_gdn_replayssm_decode.py index b4659095a2..74ddc0eb37 100644 --- a/benchmark/kernels/attention/bench_gdn_replayssm_decode.py +++ b/benchmark/kernels/attention/bench_gdn_replayssm_decode.py @@ -37,7 +37,7 @@ import triton from sglang.kernels.ops.attention.fla.fused_recurrent import ( fused_recurrent_gated_delta_rule_packed_decode, ) -from sglang.kernels.ops.attention.fla.fused_recurrent_linear_replayssm import ( +from sglang.kernels.ops.attention.linear.fused_recurrent_linear_replayssm import ( fused_recurrent_gdn_replayssm_decode, ) diff --git a/benchmark/kernels/bench_kda_verify_sweep.py b/benchmark/kernels/bench_kda_verify_sweep.py index 2990acd505..77d6e8c21a 100644 --- a/benchmark/kernels/bench_kda_verify_sweep.py +++ b/benchmark/kernels/bench_kda_verify_sweep.py @@ -22,11 +22,11 @@ import argparse import torch -import sglang.kernels.ops.attention.fla.fused_kda_conv_recurrent_verify as fused_mod -from sglang.kernels.ops.attention.fla.fused_kda_conv_recurrent_verify import ( +import sglang.kernels.ops.attention.linear.fused_kda_conv_recurrent_verify as fused_mod +from sglang.kernels.ops.attention.linear.fused_kda_conv_recurrent_verify import ( fused_kda_conv_gating_verify, ) -from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( +from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, ) from sglang.kernels.ops.mamba.causal_conv1d_triton import causal_conv1d_update diff --git a/benchmark/kernels/deepseek/benchmark_q8kv8_kv_gather.py b/benchmark/kernels/deepseek/benchmark_q8kv8_kv_gather.py index 12a73faf87..fb0e19589e 100755 --- a/benchmark/kernels/deepseek/benchmark_q8kv8_kv_gather.py +++ b/benchmark/kernels/deepseek/benchmark_q8kv8_kv_gather.py @@ -3,7 +3,7 @@ Compares the legacy gather (``gather_dequant_requant_fp8_paged_legacy``: fresh ``torch.zeros`` destination + one program per (token, 128-elem -slice)) against the new gather (``gather_dequant_requant_fp8_paged``: +slice)) against the new gather (``gather_dequant_requant_fp8_paged_dsa``: no pre-zeroing needed, fused pad-row zero-fill, TOKENS_PER_PROG tokens per program with 16B-vectorized access), in three flavors: @@ -43,6 +43,8 @@ import sys from pathlib import Path import torch +import triton +import triton.language as tl _REPO_ROOT = Path(__file__).resolve().parents[3] _MOD_PATH = _REPO_ROOT / "python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py" @@ -54,8 +56,127 @@ _spec = importlib.util.spec_from_file_location("dequant_k_cache", _MOD_PATH) _mod = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(_mod) -gather_new = _mod.gather_dequant_requant_fp8_paged -gather_legacy = _mod.gather_dequant_requant_fp8_paged_legacy +gather_new = _mod.gather_dequant_requant_fp8_paged_dsa +_infer_dsa_dims = _mod._infer_dsa_dims + + +def gather_dequant_requant_fp8_paged_legacy( + quant_k_cache: torch.Tensor, + page_table_1_flattened: torch.Tensor, + group_size: int = 128, + extra_rows: int = 0, +) -> torch.Tensor: + """Legacy (pre-vectorization) gather + dequant + requant. + + Kept as the bit-exactness / performance reference for + ``gather_dequant_requant_fp8_paged_dsa`` (see + ``benchmark/kernels/deepseek/benchmark_q8kv8_kv_gather.py``). Allocates and zero-fills the + full destination each call, then launches one program per + (token, 128-elem slice). + """ + dim_quant = quant_k_cache.shape[-1] + dim_nope, dim_rope = _infer_dsa_dims(dim_quant) + quant_k_cache = quant_k_cache.view((-1, dim_quant)) + + num_tokens = page_table_1_flattened.shape[0] + assert quant_k_cache.dtype == torch.float8_e4m3fn + num_tiles = dim_nope // group_size + out_dim = dim_nope + dim_rope + assert num_tiles * group_size == dim_nope + + total_rows = num_tokens + extra_rows + # Allocate a fresh zero-filled buffer. The extra landing-pad rows at + # the tail must read as zeros (the kernel may over-read past + # num_tokens for masked indices). + output = torch.zeros( + (total_rows, 1, out_dim), + dtype=torch.float8_e4m3fn, + device=quant_k_cache.device, + ) + + num_blocks_per_token = num_tiles + triton.cdiv(dim_rope, group_size) + + input_nope_q = quant_k_cache[:, :dim_nope] + input_nope_s = quant_k_cache[:, dim_nope : dim_nope + num_tiles * 4].view( + torch.float32 + ) + input_rope = quant_k_cache[:, dim_nope + num_tiles * 4 :].view(torch.bfloat16) + + _gather_dequant_requant_fp8_paged_kernel[(num_tokens, num_blocks_per_token)]( + output, + input_nope_q, + input_nope_s, + input_rope, + page_table_1_flattened, + output.stride(0), + input_nope_q.stride(0), + input_nope_s.stride(0), + input_rope.stride(0), + NUM_NOPE_BLOCKS=num_tiles, + GROUP_SIZE=group_size, + DIM_NOPE=dim_nope, + DIM_ROPE=dim_rope, + ) + + return output + + +@triton.jit +def _gather_dequant_requant_fp8_paged_kernel( + output_ptr, + input_nope_q_ptr, + input_nope_s_ptr, + input_rope_ptr, + page_table_1_ptr, + output_stride_0: int, + input_nope_q_stride_0: int, + input_nope_s_stride_0: int, + input_rope_stride_0: int, + NUM_NOPE_BLOCKS: tl.constexpr, + GROUP_SIZE: tl.constexpr, + DIM_NOPE: tl.constexpr, + DIM_ROPE: tl.constexpr, +): + """Fused gather + dequant(per-group) + requant(per-tensor) -> fp8.""" + token_id = tl.program_id(0) + token_id_paged = tl.load(page_table_1_ptr + token_id).to(tl.int32) + raw_block_id = tl.program_id(1) + + if raw_block_id < NUM_NOPE_BLOCKS: + # nope: read fp8, mul group scale -> f32, cast to fp8_e4m3fn + effective_block_id = raw_block_id + offs_q = effective_block_id * GROUP_SIZE + tl.arange(0, GROUP_SIZE) + mask = offs_q < DIM_NOPE + + ptr_q = input_nope_q_ptr + token_id_paged * input_nope_q_stride_0 + offs_q + ptr_s = ( + input_nope_s_ptr + + token_id_paged * input_nope_s_stride_0 + + effective_block_id + ) + + y_q = tl.load(ptr_q, mask=mask, other=0.0).to(tl.float32) + y_s = tl.load(ptr_s) + + # dequant -> f32 -> requant to fp8 + y = (y_q * y_s).to(tl.float8e4nv) + + dst_ptr = output_ptr + token_id * output_stride_0 + offs_q + tl.store(dst_ptr, y, mask=mask) + else: + # rope: read bf16, cast to fp8 + effective_block_id = raw_block_id - NUM_NOPE_BLOCKS + offs = effective_block_id * GROUP_SIZE + tl.arange(0, GROUP_SIZE) + mask = offs < DIM_ROPE + + src_ptr = input_rope_ptr + token_id_paged * input_rope_stride_0 + offs + dst_ptr = output_ptr + token_id * output_stride_0 + DIM_NOPE + offs + + data = tl.load(src_ptr, mask=mask).to(tl.float8e4nv) + tl.store(dst_ptr, data, mask=mask) + + +gather_legacy = gather_dequant_requant_fp8_paged_legacy PAGE_SIZE = 64 DIM_QUANT = 656 # 512 nope fp8 + 16 scale bytes + 128 rope bytes diff --git a/benchmark/kernels/deepseek/benchmark_q8kv8_q_prep.py b/benchmark/kernels/deepseek/benchmark_q8kv8_q_prep.py index 7cc4c84ecd..ad5c63a18c 100755 --- a/benchmark/kernels/deepseek/benchmark_q8kv8_q_prep.py +++ b/benchmark/kernels/deepseek/benchmark_q8kv8_q_prep.py @@ -54,7 +54,7 @@ import argparse import torch -from sglang.kernels.ops.kvcache.cache_ops import ( +from sglang.kernels.ops.attention.mla_qk_prep import ( absorbed_bmm_concat_cast_q_fp8, concat_and_cast_q_fp8_pad, ) diff --git a/benchmark/kernels/quantization/tuning_mxfp8_native_gfx95.py b/benchmark/kernels/quantization/tuning_mxfp8_native_gfx95.py index 7f2c0e68a8..b9ec7e945e 100644 --- a/benchmark/kernels/quantization/tuning_mxfp8_native_gfx95.py +++ b/benchmark/kernels/quantization/tuning_mxfp8_native_gfx95.py @@ -1,5 +1,5 @@ """Tune the gfx950 native MXFP8 dense route for one (N, K) and write its rows into -python/sglang/kernels/ops/quantization/mxfp8_gemv_gfx95_configs.json. +python/sglang/kernels/ops/gemm/mxfp8_gemv_gfx95_configs.json. Two tables: the scaled-MFMA GEMV config per M bucket (1 ... 32) and the dot_scaled tile per M bucket (64 ... 16384). Each candidate is timed by HIP-graph replay rotating over weight @@ -18,7 +18,7 @@ import torch from triton.runtime.errors import OutOfResources from sglang.kernels.jit.utils import empty_sentinel -from sglang.kernels.ops.quantization import mxfp8_native_amd_gfx95 as native +from sglang.kernels.ops.gemm import mxfp8_native_amd_gfx95 as native from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import fp8_grid_quantize GRAPH_CALLS = 20 diff --git a/python/sglang/kernels/README.md b/python/sglang/kernels/README.md index 4be13dafb7..f7c6475d79 100644 --- a/python/sglang/kernels/README.md +++ b/python/sglang/kernels/README.md @@ -18,10 +18,16 @@ sglang/kernels/ registry.py # process-wide KernelRegistry + register_kernel() selector.py # heuristic select_kernel() and cached get_kernel() fused_op.py # BaseFusedOp: per-operator multi-backend contract + kernel_api_logging.py # kernel API crash-debugging decorator + numerics.py # shared numerics helpers for bit-exact Triton kernels ops/ / # one subpackage per operator group (see list below) - jit/ # shared JIT CUDA build/runtime infra: utils/, csrc/, - # include/, __main__ (KERNEL_PATH resolves here) + jit/ # shared JIT build/runtime infra: utils/ (incl. the Helion + # runtime helper), csrc/, include/, __main__ (KERNEL_PATH) + aot/ # source tree of the AOT `sgl_kernel` wheel (own API and + # packaging; not shipped in the sglang wheel) + kda_kernels/ # agent-generated kernel backend (Kernel Design Agents) + cake_kernels/ # agent-generated kernel backend (Cake, shipped by FlashInfer) ``` Operator groups (all populated): `activation`, `attention`, `communication`, @@ -31,7 +37,7 @@ Operator groups (all populated): `activation`, `attention`, `communication`, Place model-specific implementations and tuning data inside the corresponding logical operator group, rather than creating a model-specific top-level group -under `ops/`. +under `ops/` (a model-named file or subpackage inside a group is fine). `lora` is its own group because it has a distinct responsibility: the adapter kernels that only the LoRA runtime in `srt/lora` calls (`dense/` SGMV GEMMs and @@ -51,18 +57,37 @@ that are not CI-registered belong in `test/manual/kernels/`, outside the shipped operator package. Keep vendored upstream trees under their existing maintenance policy. -Classify the public computation, not the model that first used it: expert -routing belongs in `moe`, matrix multiplication (including quantized GEMM and -its tuning data) in `gemm`, and attention index selection in `attention`. -Keep a fused operation intact; an attention kernel that fuses RoPE with a -projection may stay in `attention`. A model name inside a logical group is fine. +Classify an operator by its complete public computation, not by the model +that first used it, and keep a fused operation intact: -Runtime integration that owns process groups, symmetric buffers, or model -dispatch stays in `srt`; the K3 communication adapters live in +- **Collectives win.** Anything that performs a cross-rank collective or + reads/writes peer memory (all-reduce, all-gather, reduce-scatter, all-to-all, + multicast/NVLS) belongs in `communication`, whatever compute it fuses (GEMM, + norm, MoE finalize). +- **Producer + quantization stays with the producer.** Norm + quant is + `layernorm`, activation + quant is `activation`, a GEMM with a quantizing + epilogue is `gemm`, MoE dispatch/permute + quant is `moe`. `quantization` + holds only pure format conversion: quant, dequant, scale-layout conversion, + repack. A quantized GEMM is still a GEMM. +- **RoPE stays in `attention`**: standalone RoPE, position ids, and QK-norm + + RoPE fusions. +- **`kvcache` owns KV-cache contents**: ops that read or write the cache + (store, gather, page copy/move, host transfer, quantize-into-cache, layout + conversion, clearing cache/state rows). **`attention` owns the metadata**: + page tables, `cu_seqlens`, `kv_indices`, split/schedule tables, and attention + index selection. A fused compute + store (e.g. norm + RoPE + quant + store) + stays with its compute group. +- Otherwise: expert routing is `moe`, matrix multiplication (and its tuning + data) is `gemm`, normalization is `layernorm`. + +Runtime integration that owns process groups, symmetric buffers, workspace +classes, or model dispatch stays in `srt`: the K3, Muse Spark and SP +collective-GEMM communication adapters and workspaces live in `srt/layers/communication/`, while their callable kernels live in -`kernels/ops/communication/`. Tests of this runtime state belong with the -runtime subsystem, even when they need a GPU. Kernel inventory/dispatch tests -belong in `test/registered/unit/kernels/`. +`kernels/ops/communication/` and take that state as arguments. Reference/test oracles belong in +`sglang.test.kernels`, not in an operator module. Tests of runtime state belong +with the runtime subsystem, even when they need a GPU. Kernel inventory/dispatch +tests belong in `test/registered/unit/kernels/`. When moving an operator, update its callers, lazy `KernelSpec` targets and op ids, tests, benchmark imports, and tuning-data paths together. Preserve CI @@ -70,6 +95,40 @@ stage/runner registrations and test cases when splitting files. See the [kernel organization skill](../../../.claude/skills/kernel-organization/SKILL.md) for the review checklist. +## Agent-generated kernel backends: `kda_kernels/` and `cake_kernels/` + +These two directories hold implementations produced by kernel-generating agents +and will keep growing; they are implementation homes, not operator groups. + +- `kda_kernels/` — kernels produced or extended by the Kernel Design Agents + (Humanize2) workflow, in CUDA, Triton, or CuTe DSL, with their JIT sources in + `kda_kernels/csrc/`. `KernelBackend.KDA` records that provenance. +- `cake_kernels/` — thin adapters to Cake-generated kernels that FlashInfer + distributes (generated sources and caches stay in FlashInfer); they register + with the provenance of that library (`KernelBackend.FLASHINFER`). + +Their public ops are registered into the logical groups with +`KernelSpec(op=".", backend=..., target="sglang.kernels.kda_kernels.:")` +(or a `cake_kernels` target), and runtime code imports the `ops.` +facade, never these packages. Put a new agent-generated kernel in the matching +directory with its provenance (see `kda_kernels/README.md`). List them from the +registry by target (an agent kernel that is the op's only implementation may +keep its language backend, e.g. `TRITON`): + +```python +import sglang.kernels.ops # populate the registry +from sglang.kernels import KernelBackend, registry + +agent_kernels = [ + spec + for spec in registry.all_specs() + if spec.backend is KernelBackend.KDA + or spec.target.startswith( + ("sglang.kernels.kda_kernels.", "sglang.kernels.cake_kernels.") + ) +] +``` + ## How it works Each `ops.` function is a thin wrapper that forwards to a chosen @@ -146,6 +205,10 @@ module-level instances, so the import surface is unchanged; each instance also registers all of its backends as `KernelSpec`s so the registry inventory and `select_kernel(..., backend=...)` keep working. +An op that is built per layer around the model's own modules (e.g. the Muse +Spark norms) has no module-level instance; its group registers each backend +method as `module:Class.forward_` for the inventory. + What this buys (see the [RFC discussion](https://github.com/sgl-project/sglang/issues/29630#issuecomment-4920387930)): diff --git a/python/sglang/kernels/jit/csrc/moe/expert_pack_mxfp4.cuh b/python/sglang/kernels/jit/csrc/moe/expert_pack_mxfp4.cuh index b360a91522..2959796fe1 100644 --- a/python/sglang/kernels/jit/csrc/moe/expert_pack_mxfp4.cuh +++ b/python/sglang/kernels/jit/csrc/moe/expert_pack_mxfp4.cuh @@ -23,9 +23,6 @@ constexpr int kQuantBlock = 32; constexpr int kBlockBytes = 17; constexpr int kWarpsPerBlock = 4; constexpr int kRowsPerWarp = 4; -constexpr int kMarlinTileK = 16; -constexpr int kMarlinTileN = 64; -constexpr int kMarlinTileWords = 128; __device__ __forceinline__ float fp4_value(uint8_t value) { constexpr float table[16] = { @@ -230,127 +227,6 @@ __global__ void mxfp4_matvec_dual_kernel( } } -__device__ __forceinline__ uint32_t load_raw_word( - const uint8_t* raw, int64_t cache_stride, int slot, int role_offset, int row, int blocks_per_row, int packed_word) { - const int block = packed_word / 4; - const int word_in_block = packed_word & 3; - const int64_t row_bytes = static_cast(blocks_per_row) * kBlockBytes; - const uint8_t* ptr = raw + static_cast(slot) * cache_stride + role_offset + - static_cast(row) * row_bytes + static_cast(block) * kBlockBytes + 1 + - word_in_block * 4; - return static_cast(ptr[0]) | (static_cast(ptr[1]) << 8) | (static_cast(ptr[2]) << 16) | - (static_cast(ptr[3]) << 24); -} - -__device__ __forceinline__ uint8_t load_raw_scale( - const uint8_t* raw, int64_t cache_stride, int slot, int role_offset, int row, int blocks_per_row, int block) { - const int64_t row_bytes = static_cast(blocks_per_row) * kBlockBytes; - const uint8_t* ptr = raw + static_cast(slot) * cache_stride + role_offset + - static_cast(row) * row_bytes + static_cast(block) * kBlockBytes; - return *ptr; -} - -__device__ __forceinline__ uint8_t marlin_scale_perm(int index) { - constexpr int local_perm[4] = {0, 2, 1, 3}; - const int interleaved = (index / 4) * 4 + local_perm[index & 3]; - return static_cast(((interleaved & 7) * 8) + (interleaved >> 3)); -} - -__device__ __forceinline__ uint8_t marlin_nibble(uint32_t word, int value_index) { - return static_cast((word >> ((value_index & 7) * 4)) & 0x0f); -} - -__global__ void mxfp4_marlin_repack_weight_kernel( - const uint8_t* __restrict__ raw, - int64_t raw_stride, - const int32_t* __restrict__ source_slots, - const int32_t* __restrict__ target_slots, - int64_t role_bytes, - int input_size, - int output_size, - bool gate_up, - int32_t* __restrict__ output, - int64_t output_stride) { - const int batch = blockIdx.y; - const int64_t total_words = static_cast(input_size / kMarlinTileK) * (output_size * 2); - const int64_t index = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; - if (batch >= gridDim.y || index >= total_words) return; - - const int64_t tile_span = static_cast(output_size / kMarlinTileN) * kMarlinTileWords; - const int tile_k = static_cast(index / tile_span); - const int64_t tile_rem = index % tile_span; - const int tile_n = static_cast(tile_rem / kMarlinTileWords); - const int local = static_cast(tile_rem % kMarlinTileWords); - const int warp = local & 3; - const int thread = local >> 2; - const int cur_n = warp * 16 + thread / 4; - const int tc_row = (thread & 3) * 2; - constexpr int offsets[4] = {0, 1, 8, 9}; - constexpr int pack_index[8] = {0, 2, 4, 6, 1, 3, 5, 7}; - - const int source_slot = source_slots[batch]; - const int target_slot = target_slots[batch]; - const int rows_per_role = gate_up ? output_size / 2 : output_size; - const int blocks_per_row = input_size / kQuantBlock; - const int role0_offset = 0; - const int role1_offset = static_cast(role_bytes); - const int role2_offset = static_cast(2 * role_bytes); - uint8_t values[8]; -#pragma unroll - for (int i = 0; i < 4; ++i) { - const int value_index = tc_row + offsets[i]; - const int source_row = tile_n * kMarlinTileN + cur_n; - const int role = gate_up && source_row >= rows_per_role ? 1 : (gate_up ? 0 : 2); - const int row = gate_up ? source_row % rows_per_role : source_row; - const int role_offset = role == 0 ? role0_offset : (role == 1 ? role1_offset : role2_offset); - const uint32_t word = - load_raw_word(raw, raw_stride, source_slot, role_offset, row, blocks_per_row, tile_k * 2 + value_index / 8); - values[i] = marlin_nibble(word, value_index); - const int high_source_row = tile_n * kMarlinTileN + cur_n + 8; - const int high_role = gate_up && high_source_row >= rows_per_role ? 1 : (gate_up ? 0 : 2); - const int high_role_offset = high_role == 0 ? role0_offset : (high_role == 1 ? role1_offset : role2_offset); - const int high_row = gate_up ? high_source_row % rows_per_role : high_source_row; - const uint32_t high_word = load_raw_word( - raw, raw_stride, source_slot, high_role_offset, high_row, blocks_per_row, tile_k * 2 + value_index / 8); - values[4 + i] = marlin_nibble(high_word, value_index); - } - - uint32_t packed = 0; -#pragma unroll - for (int i = 0; i < 8; ++i) { - packed |= static_cast(values[pack_index[i]]) << (i * 4); - } - output[static_cast(target_slot) * output_stride + index] = static_cast(packed); -} - -__global__ void mxfp4_marlin_repack_scale_kernel( - const uint8_t* __restrict__ raw, - int64_t raw_stride, - const int32_t* __restrict__ source_slots, - const int32_t* __restrict__ target_slots, - int64_t role_bytes, - int input_size, - int output_size, - bool gate_up, - uint8_t* __restrict__ output, - int64_t output_stride) { - const int batch = blockIdx.y; - const int groups = input_size / kQuantBlock; - const int64_t total = static_cast(groups) * output_size; - const int64_t index = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; - if (batch >= gridDim.y || index >= total) return; - - const int group = static_cast(index / output_size); - const int column = static_cast(index % output_size); - const int source_column = (column / 64) * 64 + marlin_scale_perm(column & 63); - const int rows_per_role = gate_up ? output_size / 2 : output_size; - const int role = gate_up && source_column >= rows_per_role ? 1 : (gate_up ? 0 : 2); - const int row = gate_up ? source_column % rows_per_role : source_column; - const int role_offset = role == 0 ? 0 : (role == 1 ? static_cast(role_bytes) : static_cast(2 * role_bytes)); - const uint8_t value = load_raw_scale(raw, raw_stride, source_slots[batch], role_offset, row, groups, group); - output[static_cast(target_slots[batch]) * output_stride + index] = value; -} - } // namespace // --------------------------------------------------------------------------- @@ -622,156 +498,4 @@ inline void mxfp4_matvec_dual( } } -/** - * \brief Repack raw GGUF MXFP4 objects into contiguous Marlin SoA cache tensors. - * - * One entry of `source_slots` / `target_slots` per object to move: the raw - * matrices at `source_slots[i]` are read and the Marlin-layout weights and - * scales for `target_slots[i]` are written. Slots absent from `target_slots` - * are left untouched. - * - * \param raw Raw MXFP4 slot bank, `[source_slots, 3 * role_bytes]` uint8. - * \param source_slots Row of `raw` to read per object, int32. - * \param target_slots Row of the Marlin tensors to write per object, int32. - * \param role_bytes Byte size of one role (gate, up, or down) per slot. - * \param hidden_size Model hidden size; must be divisible by 32. - * \param intermediate_size Expert intermediate size; must be divisible by 32. - * \param w13 Marlin gate/up weights, int32. - * \param w2 Marlin down weights, int32. - * \param w13_scale Marlin gate/up scales, uint8. - * \param w2_scale Marlin down scales, uint8. - */ -inline void mxfp4_marlin_repack( - tvm::ffi::TensorView raw, - tvm::ffi::TensorView source_slots, - tvm::ffi::TensorView target_slots, - int64_t role_bytes, - int64_t hidden_size, - int64_t intermediate_size, - tvm::ffi::TensorView w13, - tvm::ffi::TensorView w2, - tvm::ffi::TensorView w13_scale, - tvm::ffi::TensorView w2_scale) { - using namespace host; - - // The Marlin layout tiles the n dimension by `kMarlinTileN`, so a size that is - // only a multiple of `kQuantBlock` leaves a partial tile: the scale - // permutation then maps columns outside the role slice and the weight tile - // span collapses to zero. `w2` takes its n from `hidden_size`, which is - // therefore the stricter of the two; `w13` takes `2 * intermediate_size`, so - // the quant-block check already makes it a whole number of tiles. - // Not an OOB read -- memcheck is clean; the partial tile reads a neighbouring - // slot from inside the tensor, and the collapsed span divides by zero on the - // device, which is undefined rather than trapping. Both are silent. - CHECK_HOST(hidden_size > 0 && hidden_size % kMarlinTileN == 0) - << "hidden_size must be divisible by " << kMarlinTileN << ", got " << hidden_size; - CHECK_HOST(intermediate_size > 0 && intermediate_size % kQuantBlock == 0) - << "MXFP4 dimensions must be divisible by 32, got intermediate_size=" << intermediate_size; - - // Every role holds one matrix of `intermediate_size x hidden_size` MXFP4 - // blocks -- transposed for the down projection, which is the same byte count - // -- so the dimensions pin `role_bytes` exactly, the way they do for matvec. - const int64_t expected_role_bytes = intermediate_size * (hidden_size / kQuantBlock) * kBlockBytes; - CHECK_HOST(role_bytes == expected_role_bytes) - << "role byte count does not match matrix dimensions: " << role_bytes << " != " << expected_role_bytes; - - const int64_t w13_n = 2 * intermediate_size; - const int64_t w2_n = hidden_size; - const int64_t w13_k = hidden_size; - const int64_t w2_k = intermediate_size; - const int64_t w13_words = (w13_k / kMarlinTileK) * w13_n * 2; - const int64_t w2_words = (w2_k / kMarlinTileK) * w2_n * 2; - const int64_t w13_scales = (w13_k / kQuantBlock) * w13_n; - const int64_t w2_scales = (w2_k / kQuantBlock) * w2_n; - - auto batch = SymbolicSize{"objects"}; - auto source_capacity = SymbolicSize{"raw_slots"}; - auto target_capacity = SymbolicSize{"marlin_slots"}; - auto device = SymbolicDevice{}; - - TensorMatcher({source_capacity, 3 * role_bytes}) // - .with_dtype() - .with_device(device) - .verify(raw); - TensorMatcher({batch}) // - .with_dtype() - .with_device(device) - .verify(source_slots) - .verify(target_slots); - TensorMatcher({target_capacity, w13_words}) // - .with_dtype() - .with_device(device) - .verify(w13); - TensorMatcher({target_capacity, w2_words}) // - .with_dtype() - .with_device(device) - .verify(w2); - TensorMatcher({target_capacity, w13_scales}) // - .with_dtype() - .with_device(device) - .verify(w13_scale); - TensorMatcher({target_capacity, w2_scales}) // - .with_dtype() - .with_device(device) - .verify(w2_scale); - - const uint32_t objects = static_cast(batch.unwrap()); - if (objects == 0) return; - - constexpr uint32_t kThreads = 256; - const DLDevice dev = device.unwrap(); - const auto* raw_ptr = static_cast(raw.data_ptr()); - const auto* source_ptr = static_cast(source_slots.data_ptr()); - const auto* target_ptr = static_cast(target_slots.data_ptr()); - - LaunchKernel(dim3(div_ceil(static_cast(w13_words), kThreads), objects), kThreads, dev)( - mxfp4_marlin_repack_weight_kernel, - raw_ptr, - raw.stride(0), - source_ptr, - target_ptr, - role_bytes, - static_cast(w13_k), - static_cast(w13_n), - true, - static_cast(w13.data_ptr()), - w13.stride(0)); - LaunchKernel(dim3(div_ceil(static_cast(w2_words), kThreads), objects), kThreads, dev)( - mxfp4_marlin_repack_weight_kernel, - raw_ptr, - raw.stride(0), - source_ptr, - target_ptr, - role_bytes, - static_cast(w2_k), - static_cast(w2_n), - false, - static_cast(w2.data_ptr()), - w2.stride(0)); - LaunchKernel(dim3(div_ceil(static_cast(w13_scales), kThreads), objects), kThreads, dev)( - mxfp4_marlin_repack_scale_kernel, - raw_ptr, - raw.stride(0), - source_ptr, - target_ptr, - role_bytes, - static_cast(w13_k), - static_cast(w13_n), - true, - static_cast(w13_scale.data_ptr()), - w13_scale.stride(0)); - LaunchKernel(dim3(div_ceil(static_cast(w2_scales), kThreads), objects), kThreads, dev)( - mxfp4_marlin_repack_scale_kernel, - raw_ptr, - raw.stride(0), - source_ptr, - target_ptr, - role_bytes, - static_cast(w2_k), - static_cast(w2_n), - false, - static_cast(w2_scale.data_ptr()), - w2_scale.stride(0)); -} - } // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/muse_spark/routing.cuh b/python/sglang/kernels/jit/csrc/muse_spark/routing.cuh index 7f82a4e6f2..8255489435 100644 --- a/python/sglang/kernels/jit/csrc/muse_spark/routing.cuh +++ b/python/sglang/kernels/jit/csrc/muse_spark/routing.cuh @@ -1,7 +1,7 @@ // Fused MuseSpark MoE routing: normalize -> bias -> iterative top-k -> renormalize. // // Drop-in replacement for the Triton `fused_muse_spark_routing_kernel_topk8` -// (python/sglang/kernels/ops/muse_spark/fused_muse_spark_routing.py). One warp owns one token +// (python/sglang/kernels/ops/moe/muse_spark_routing.py). One warp owns one token // row; the whole row lives in registers (kExpertsPerLane per lane), each lane // sorts its slice once, and each of the kTopK rounds pops one head. // diff --git a/python/sglang/kernels/jit/csrc/muse_spark/row_sum.cuh b/python/sglang/kernels/jit/csrc/muse_spark/row_sum.cuh index 8b6f46b652..c74dac9f9b 100644 --- a/python/sglang/kernels/jit/csrc/muse_spark/row_sum.cuh +++ b/python/sglang/kernels/jit/csrc/muse_spark/row_sum.cuh @@ -1,4 +1,4 @@ -// Row sum over [N, K] fp32, bitwise-identical to `sglang.kernels.ops.muse_spark.triton_sum`. +// Row sum over [N, K] fp32, bitwise-identical to `sglang.test.kernels.triton_sum`. // // The point of that Triton kernel is a reduction order fixed by the compiler // rather than by the CUDA runtime version, so this kernel reproduces tl.sum's diff --git a/python/sglang/kernels/jit/csrc/qprep_bf16_fp8_sm90/kernel.cuh b/python/sglang/kernels/jit/csrc/qprep_bf16_fp8_sm90/kernel.cuh index f789b48c66..0b09e9785b 100644 --- a/python/sglang/kernels/jit/csrc/qprep_bf16_fp8_sm90/kernel.cuh +++ b/python/sglang/kernels/jit/csrc/qprep_bf16_fp8_sm90/kernel.cuh @@ -20,7 +20,7 @@ limitations under the License. // out[:, h, 512:576] = fp8_e4m3(q_rope[:, h, :]) // // This is the CUDA replacement for the Triton absorbed_bmm_concat_cast_q_fp8 -// kernel (triton_ops/cache_ops.py). The epilogue keeps the exact rounding +// kernel (attention/mla_qk_prep.py). The epilogue keeps the exact rounding // chain of the Triton variants: fp32 WGMMA accumulate -> bf16 round-to-nearest // (cublas-equivalent output rounding) -> fp8_e4m3 rn/satfinite on store. The // K dimension is consumed as one in-order chain of k=16 WGMMA steps into a diff --git a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/fp4_utils.cuh b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/fp4_utils.cuh index c148521811..5726efed6c 100644 --- a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/fp4_utils.cuh +++ b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/fp4_utils.cuh @@ -17,7 +17,7 @@ namespace deepseek_v4::fp4 { /// Largest finite e2m1 value. constexpr float kMax = 6.0f; -/// `6 * 2^-126`, the amax floor `torch_quant.fake_quant_fp4` clamps to. +/// `6 * 2^-126`, the amax floor `fp4_fake_quant.fake_quant_fp4` clamps to. constexpr float kAmaxFloor = 6.0f * 1.1754943508222875e-38f; /// Elements sharing one ue8m0 scale. constexpr uint32_t kBlockSize = 32; @@ -95,7 +95,7 @@ SGL_DEVICE fp32x2_t block_scale(float amax) { /// \brief Round a pair onto the e2m1 grid and back, through `scale`. /// /// Every e2m1 value is exact in fp16, so the roundtrip is lossless. Adding `0.0f` during -/// scaling clears negative zero to match `torch.sign(0) == 0` in `torch_quant.round_fp4`. +/// scaling clears negative zero to match `torch.sign(0) == 0` in `fp4_fake_quant.round_fp4`. SGL_DEVICE fp32x2_t fake_quant_x2(fp32x2_t x, float scale, float inv_scale) { const fp32x2_t scaled{__fmaf_rn(x.x, inv_scale, 0.0f), __fmaf_rn(x.y, inv_scale, 0.0f)}; #ifdef USE_ROCM diff --git a/python/sglang/kernels/ops/muse_spark/helion_runtime.py b/python/sglang/kernels/jit/utils/helion_runtime.py similarity index 88% rename from python/sglang/kernels/ops/muse_spark/helion_runtime.py rename to python/sglang/kernels/jit/utils/helion_runtime.py index 2125d99461..42db972060 100644 --- a/python/sglang/kernels/ops/muse_spark/helion_runtime.py +++ b/python/sglang/kernels/jit/utils/helion_runtime.py @@ -30,23 +30,20 @@ _compiled: dict[tuple[Any, str], Any] = {} @lru_cache(maxsize=None) -def _warn_unsupported(reason: str) -> bool: +def warn_helion_unsupported(reason: str) -> bool: logger.warning("Muse Spark Helion kernels are disabled: %s", reason) return False @lru_cache(maxsize=None) -def _base_package_version(package: str) -> str | None: +def base_package_version(package: str) -> str | None: try: return version(package).split("+", 1)[0] except PackageNotFoundError: return None -def helion_supported( - op_name: str, - tensor: torch.Tensor, -) -> bool: +def helion_supported(tensor: torch.Tensor) -> bool: """Return whether a selected kernel is validated for this runtime/device.""" if torch.compiler.is_compiling(): return False @@ -57,21 +54,12 @@ def helion_supported( forced_backend = get_fused_op_backend() if forced_backend is not None and forced_backend is not KernelBackend.HELION: return False - helion_version = _base_package_version("helion") + helion_version = base_package_version("helion") if helion_version != "1.4.0": - return _warn_unsupported( + return warn_helion_unsupported( f"Helion 1.4.0 is required, found {helion_version or 'not installed'}" ) - if not tensor.is_cuda: - return False - if op_name == "norm_fp4_quant": - flashinfer_version = _base_package_version("flashinfer-python") - if flashinfer_version != "0.6.17": - return _warn_unsupported( - "NVFP4 quantization requires FlashInfer 0.6.17, found " - f"{flashinfer_version or 'not installed'}" - ) - return True + return tensor.is_cuda def token_count_specialization_key(first_tensor: torch.Tensor, *_: Any) -> int: @@ -167,9 +155,11 @@ def run_pinned_helion( __all__ = [ + "base_package_version", "current_helion_config_bucket", "helion_config_bucket", "helion_supported", "run_pinned_helion", "token_count_specialization_key", + "warn_helion_unsupported", ] diff --git a/python/sglang/kernels/jit/utils/hipcc.py b/python/sglang/kernels/jit/utils/hipcc.py new file mode 100644 index 0000000000..059ab1a33e --- /dev/null +++ b/python/sglang/kernels/jit/utils/hipcc.py @@ -0,0 +1,91 @@ +"""hipcc build and HIP module-launch helpers for the hand-written gfx950 kernels.""" + +from __future__ import annotations + +import ctypes +import os + +import torch + +from sglang.srt.distributed.device_communicators.cuda_wrapper import ( + find_loaded_library, +) + +_hip = None + + +def hip_lib(): + global _hip + if _hip is None: + # Bind to the HIP runtime torch already mapped. ROCm 10 ships a second + # libamdhip64 in _rocm_sdk_devel; an unversioned CDLL("libamdhip64.so") + # picks that one, and every launch on a torch stream then fails with + # hipErrorContextIsDestroyed (709). + torch.cuda.current_device() + _hip = ctypes.CDLL(find_loaded_library("libamdhip64") or "libamdhip64.so") + _hip.hipModuleLoad.restype = ctypes.c_int + _hip.hipModuleLoad.argtypes = [ctypes.c_void_p, ctypes.c_char_p] + _hip.hipModuleGetFunction.restype = ctypes.c_int + _hip.hipModuleGetFunction.argtypes = [ + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_char_p, + ] + _hip.hipModuleLaunchKernel.restype = ctypes.c_int + _hip.hipModuleLaunchKernel.argtypes = ( + [ctypes.c_void_p] + [ctypes.c_uint] * 7 + [ctypes.c_void_p] * 3 + ) + _hip.hipGetErrorString.restype = ctypes.c_char_p + return _hip + + +def check_hip(rc, what): + if rc != 0: + raise RuntimeError( + f"smallm_moe: {what} failed: {rc} ({hip_lib().hipGetErrorString(rc).decode()})" + ) + + +class HipKernel: + def __init__(self, mod, name): + self.fn = ctypes.c_void_p() + check_hip( + hip_lib().hipModuleGetFunction(ctypes.byref(self.fn), mod, name.encode()), + f"hipModuleGetFunction {name}", + ) + + def launch(self, grid, args, stream, block=256, shared=0): + size = ctypes.c_size_t(ctypes.sizeof(args)) + extra = (ctypes.c_void_p * 5)( + 1, + ctypes.cast(ctypes.byref(args), ctypes.c_void_p).value, + 2, + ctypes.cast(ctypes.byref(size), ctypes.c_void_p).value, + 3, + ) + check_hip( + hip_lib().hipModuleLaunchKernel( + self.fn, + grid, + 1, + 1, + block, + 1, + 1, + shared, + ctypes.c_void_p(stream), + None, + ctypes.cast(extra, ctypes.c_void_p), + ), + "hipModuleLaunchKernel", + ) + + +def hipcc_path(): + return os.path.join(os.environ.get("ROCM_PATH", "/opt/rocm"), "bin", "hipcc") + + +def hip_version(): + if torch.version.hip: + return tuple(int(x) for x in torch.version.hip.split("-")[0].split(".")[:3]) + return (0, 0, 0) diff --git a/python/sglang/kernels/ops/__init__.py b/python/sglang/kernels/ops/__init__.py index 78f1c26c56..b27d804ab6 100644 --- a/python/sglang/kernels/ops/__init__.py +++ b/python/sglang/kernels/ops/__init__.py @@ -30,7 +30,6 @@ _GROUPS = ( "memory", "mm", "moe", - "muse_spark", "quantization", "sampling", "speculative", diff --git a/python/sglang/kernels/ops/activation/__init__.py b/python/sglang/kernels/ops/activation/__init__.py index 3128b3d2e6..2f81a27a73 100644 --- a/python/sglang/kernels/ops/activation/__init__.py +++ b/python/sglang/kernels/ops/activation/__init__.py @@ -112,7 +112,7 @@ def situ_and_mul( beta: float, linear_beta: Optional[float], ) -> torch.Tensor: - from ._jit_situ_and_mul import situ_and_mul as impl + from .kimi_k3_situ_and_mul import situ_and_mul as impl return impl(input, out, beta, linear_beta) @@ -150,7 +150,31 @@ register_kernel( KernelSpec( op="activation.situ_and_mul", backend=KernelBackend.JIT, - target="sglang.kernels.ops.activation._jit_situ_and_mul:situ_and_mul", + target="sglang.kernels.ops.activation.kimi_k3_situ_and_mul:situ_and_mul", capabilities=CUDA_ONLY, ) ) + + +# MXFP8-quantizing SwiGLU-OAI producers; registered for inventory. +for _fn in ("swiglu_oai_mxfp8", "swiglu_oai_mxfp8_deepgemm"): + register_kernel( + KernelSpec( + op=f"activation.{_fn}", + backend=KernelBackend.TRITON, + target=f"sglang.kernels.ops.activation.swiglu_mxfp8_swizzled:{_fn}", + ) + ) +del _fn +for _fn in ("silu_and_mul_clamp", "silu_and_mul_contig_post_quant"): + register_kernel( + KernelSpec( + op=f"activation.{_fn}", + backend=KernelBackend.JIT, + target=f"sglang.kernels.ops.activation.silu_and_mul_clamp:{_fn}", + capabilities=frozenset( + {CapabilityRequirement.CUDA, CapabilityRequirement.HIP} + ), + ) + ) +del _fn diff --git a/python/sglang/kernels/ops/elementwise/bias_gelu.py b/python/sglang/kernels/ops/activation/bias_gelu.py similarity index 100% rename from python/sglang/kernels/ops/elementwise/bias_gelu.py rename to python/sglang/kernels/ops/activation/bias_gelu.py diff --git a/python/sglang/kernels/ops/activation/gelu_and_mul_triton.py b/python/sglang/kernels/ops/activation/gelu_and_mul_triton.py new file mode 100644 index 0000000000..da48f0332b --- /dev/null +++ b/python/sglang/kernels/ops/activation/gelu_and_mul_triton.py @@ -0,0 +1,101 @@ +import torch +import triton +import triton.language as tl + +from sglang.srt.utils import is_hip + +_is_hip = is_hip() + + +# gelu on first half of vector +@triton.jit +def gelu_and_mul_kernel( + out_hidden_states_ptr, # (bs, hidden_dim) + out_scales_ptr, # (bs,) + hidden_states_ptr, # (bs, hidden_dim * 2) + quant_max: tl.constexpr, + static_scale: tl.constexpr, + hidden_dim: tl.constexpr, # the output hidden_dim + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(axis=0) + + input_start = pid * hidden_dim * 2 + output_start = pid * hidden_dim + + input1_offs = tl.arange(0, BLOCK_SIZE) + mask = tl.arange(0, BLOCK_SIZE) < hidden_dim # shared for input1, input3, output + input3_offs = hidden_dim + tl.arange(0, BLOCK_SIZE) + output_offs = tl.arange(0, BLOCK_SIZE) + + x1 = tl.load( + hidden_states_ptr + input_start + input1_offs, mask=mask, other=0.0 + ).to(tl.float32) + x3 = tl.load( + hidden_states_ptr + input_start + input3_offs, mask=mask, other=0.0 + ).to(tl.float32) + + # gelu + # cast down before mul to better match training? + gelu_x1 = 0.5 * (1.0 + tl.erf(x1 * 0.7071067811865475)) * x1 + out = x3 * gelu_x1.to(hidden_states_ptr.dtype.element_ty) + + if quant_max is not None: + raise NotImplementedError() + + tl.store(out_hidden_states_ptr + output_start + output_offs, out, mask=mask) + + +def gelu_and_mul_triton( + hidden_states, + scales=None, + quantize=None, # dtype to quantize to + out=None, +): + bs, in_hidden_dim = hidden_states.shape + hidden_dim = in_hidden_dim // 2 + + if out is None: + out_hidden_states = torch.empty( + (bs, hidden_dim), + dtype=quantize or hidden_states.dtype, + device=hidden_states.device, + ) + else: + assert out.shape == (bs, hidden_dim) + assert out.dtype == (quantize or hidden_states.dtype) + out_hidden_states = out + out_scales = None + static_scale = False + if quantize is not None: + if scales is None: + out_scales = torch.empty( + (bs,), dtype=torch.float32, device=hidden_states.device + ) + else: + out_scales = scales + static_scale = True + + max_warps = 16 if _is_hip else 32 + config = { + # 8 ele per thread (not tuned) + "num_warps": max( + min(triton.next_power_of_2(triton.cdiv(hidden_dim, 8 * 32)), max_warps), 4 + ), + } + + gelu_and_mul_kernel[(bs,)]( + out_hidden_states, + out_scales, + hidden_states, + quant_max=torch.finfo(quantize).max if quantize is not None else None, + static_scale=static_scale, + hidden_dim=hidden_dim, + BLOCK_SIZE=triton.next_power_of_2(hidden_dim), + **config, + ) + + if quantize is not None: + return out_hidden_states, out_scales + else: + return out_hidden_states, None diff --git a/python/sglang/kernels/ops/activation/_jit_situ_and_mul.py b/python/sglang/kernels/ops/activation/kimi_k3_situ_and_mul.py similarity index 100% rename from python/sglang/kernels/ops/activation/_jit_situ_and_mul.py rename to python/sglang/kernels/ops/activation/kimi_k3_situ_and_mul.py diff --git a/python/sglang/kernels/ops/moe/minimax_m3_swiglu.py b/python/sglang/kernels/ops/activation/minimax_m3_swiglu.py similarity index 98% rename from python/sglang/kernels/ops/moe/minimax_m3_swiglu.py rename to python/sglang/kernels/ops/activation/minimax_m3_swiglu.py index 4001388b17..15ce3daee9 100644 --- a/python/sglang/kernels/ops/moe/minimax_m3_swiglu.py +++ b/python/sglang/kernels/ops/activation/minimax_m3_swiglu.py @@ -156,6 +156,7 @@ def _swiglu_oai_mxfp8_quant_kernel( ) +# Not swiglu_oai_mxfp8: no bf16 rounding before the quant, and plain [M, I // 32] scales. def swiglu_oai_mxfp8_quant( gate_up: torch.Tensor, alpha: float, diff --git a/python/sglang/kernels/ops/activation/silu_and_mul_clamp.py b/python/sglang/kernels/ops/activation/silu_and_mul_clamp.py new file mode 100644 index 0000000000..df9ae3bace --- /dev/null +++ b/python/sglang/kernels/ops/activation/silu_and_mul_clamp.py @@ -0,0 +1,91 @@ +"""SwiGLU with clamped branches, optionally fused with per-token-group quantization.""" + +from typing import Optional + +import torch + +from sglang.kernels.jit.utils import ( + cache_once, + is_arch_support_pdl, + load_jit, + make_cpp_args, +) +from sglang.srt.utils import is_xpu + +_is_xpu = is_xpu() + + +def make_name(name: str) -> str: + return f"dpsk_v4_{name}" + + +@cache_once +def _jit_silu_mul_quant_contig_module( + quant_group_size: int, + scale_ue8m0: bool, + swizzle: bool, + apply_swiglu_limit: bool, +): + args = make_cpp_args( + quant_group_size, + scale_ue8m0, + swizzle, + is_arch_support_pdl(), + apply_swiglu_limit, + ) + return load_jit( + make_name("silu_mul_quant_contig"), + *args, + cuda_files=["deepseek_v4/silu_and_mul_masked_post_quant.cuh"], + cuda_wrappers=[("run", f"SiluAndMulContigPostQuantKernel<{args}>::run")], + extra_cuda_cflags=["-use_fast_math"], + ) + + +@cache_once +def _jit_silu_and_mul_clamp_module(dtype: torch.dtype): + args = make_cpp_args(dtype, is_arch_support_pdl()) + return load_jit( + make_name("silu_and_mul_clamp"), + *args, + cuda_files=["deepseek_v4/silu_and_mul_masked_post_quant.cuh"], + cuda_wrappers=[("run", f"SiluAndMulClampKernel<{args}>::run")], + extra_cuda_cflags=["-use_fast_math"], + ) + + +def silu_and_mul_clamp( + input: torch.Tensor, + output: torch.Tensor, + swiglu_limit: float, +) -> None: + if _is_xpu: + from sgl_kernel import silu_and_mul_clamp + + silu_and_mul_clamp(input, output, float(swiglu_limit)) + else: + module = _jit_silu_and_mul_clamp_module(input.dtype) + module.run(input, output, float(swiglu_limit)) + + +def silu_and_mul_contig_post_quant( + input: torch.Tensor, + output: torch.Tensor, + output_scale: torch.Tensor, + quant_group_size: int, + scale_ue8m0: bool = False, + transposed: bool = False, + swiglu_limit: Optional[float] = None, + swizzle: bool = False, +) -> None: + apply_swiglu_limit = swiglu_limit is not None + module = _jit_silu_mul_quant_contig_module( + quant_group_size, scale_ue8m0, swizzle, apply_swiglu_limit + ) + module.run( + input, + output, + output_scale, + transposed, + float(swiglu_limit) if apply_swiglu_limit else 0.0, + ) diff --git a/python/sglang/kernels/ops/activation/softcap.py b/python/sglang/kernels/ops/activation/softcap.py index 1040e68aed..46424ca363 100644 --- a/python/sglang/kernels/ops/activation/softcap.py +++ b/python/sglang/kernels/ops/activation/softcap.py @@ -131,53 +131,3 @@ def gathered_softcap_logits(gathered, final_logit_softcapping, output): BLOCK_SIZE=BLOCK_SIZE, ) return output - - -@triton.jit -def _softcap_row_argmax_kernel( - logits_ptr, - out_ptr, - row_stride, - valid_width, - col_offset, - softcapping_value, - BLOCK_SIZE: tl.constexpr, -): - row = tl.program_id(0).to(tl.int64) - best_val = -float("inf") - best_idx = 0 - for start in range(0, valid_width, BLOCK_SIZE): - cols = start + tl.arange(0, BLOCK_SIZE) - mask = cols < valid_width - x = tl.load(logits_ptr + row * row_stride + cols, mask=mask, other=0.0) - x = x / softcapping_value - x = libdevice.tanh(x) - x = x * softcapping_value - x = tl.where(mask, x, -float("inf")) - block_val, block_idx = tl.max( - x, axis=0, return_indices=True, return_indices_tie_break_left=True - ) - take = block_val > best_val - best_idx = tl.where(take, start + block_idx, best_idx) - best_val = tl.where(take, block_val, best_val) - tl.store(out_ptr + row * 2, best_val) - tl.store(out_ptr + row * 2 + 1, (best_idx + col_offset).to(tl.float32)) - - -def softcap_row_argmax(logits, final_logit_softcapping, valid_width, col_offset): - """Per row of a vocab shard: ``(max, first argmax + col_offset)`` of the - softcapped logits over the first ``valid_width`` columns, as fp32 ``[rows, 2]``.""" - assert logits.dtype == torch.float32 and logits.stride(1) == 1 - assert col_offset + logits.shape[1] < 2**24 - out = torch.empty(logits.shape[0], 2, dtype=torch.float32, device=logits.device) - _softcap_row_argmax_kernel[(logits.shape[0],)]( - logits, - out, - logits.stride(0), - valid_width, - col_offset, - final_logit_softcapping, - BLOCK_SIZE=4096, - num_warps=8, - ) - return out diff --git a/python/sglang/kernels/ops/activation/swiglu_mxfp8_swizzled.py b/python/sglang/kernels/ops/activation/swiglu_mxfp8_swizzled.py new file mode 100644 index 0000000000..e1730368ad --- /dev/null +++ b/python/sglang/kernels/ops/activation/swiglu_mxfp8_swizzled.py @@ -0,0 +1,177 @@ +# SPDX-License-Identifier: Apache-2.0 +"""SwiGLU producers of MXFP8: ``silu(gate) * up`` or the clamped SwiGLU-OAI, +quantized in the same pass with the scale layouts of +``ops.quantization.mxfp8_swizzled_triton``. Each quantizes the bf16-rounded +value the unfused activation stores, so it is byte-exact against that kernel +followed by ``mxfp8_quantize_swizzled``. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from sglang.kernels.ops.quantization.mxfp8_swizzled_triton import ( + _E4M3, + _alloc, + _mx_e8m0_from_amax, + _mx_scale_offsets, + can_use_mxfp8_swizzled, +) + + +@triton.jit +def _silu_mul_mxfp8_kernel( + x_ptr, + q_ptr, + s_ptr, + rows, + hidden, + n_groups, + n_col_blocks, + stride_row, + dg_rows, + ALPHA: tl.constexpr, + LIMIT: tl.constexpr, + DEEPGEMM: tl.constexpr, + BLOCK_R: tl.constexpr, + G: tl.constexpr, +): + pid_r = tl.program_id(0) + pid_c = tl.program_id(1) + r = pid_r * BLOCK_R + tl.arange(0, BLOCK_R) + g = pid_c * G + tl.arange(0, G) + c = pid_c * (G * 32) + tl.arange(0, G * 32) + rmask = r < rows + mask = rmask[:, None] & (c < hidden)[None, :] + base = x_ptr + r[:, None].to(tl.int64) * stride_row + gate = tl.load(base + c[None, :], mask=mask, other=0.0).to(tl.float32) + up = tl.load(base + hidden + c[None, :], mask=mask, other=0.0).to(tl.float32) + if ALPHA is None: + act = (gate * tl.sigmoid(gate)).to(tl.bfloat16).to(tl.float32) + prod = (act * up).to(tl.bfloat16).to(tl.float32) + else: + # the op order Inductor emits for swiglu_no_interleaved_with_alpha_and_limit + gate = tl.minimum(gate, LIMIT, propagate_nan=tl.PropagateNan.ALL) + up = tl.maximum(up, -LIMIT, propagate_nan=tl.PropagateNan.ALL) + up = tl.minimum(up, LIMIT, propagate_nan=tl.PropagateNan.ALL) + prod = gate * tl.sigmoid(gate * ALPHA) * (up + 1.0) + prod = prod.to(tl.bfloat16).to(tl.float32) + p3 = tl.reshape(prod, [BLOCK_R, G, 32]) + amax = tl.max(tl.abs(p3), axis=2) + if DEEPGEMM: + amax = tl.maximum(amax, 1e-10) + sbyte, inv = _mx_e8m0_from_amax(amax) + q = tl.reshape(p3 * inv[:, :, None], [BLOCK_R, G * 32]) + tl.store( + q_ptr + r[:, None].to(tl.int64) * hidden + c[None, :], + q.to(tl.float8e4nv), + mask=mask, + ) + valid = rmask[:, None] & (g < n_groups)[None, :] + if DEEPGEMM: + # 4 bytes per int32 along K, MN-major, rows padded to 4 + soff = ( + (g[None, :] // 4).to(tl.int64) * dg_rows * 4 + + r[:, None] * 4 + + g[None, :] % 4 + ) + tl.store(s_ptr + soff, sbyte.to(tl.uint8), mask=valid) + else: + smask = (r < tl.cdiv(rows, 128) * 128)[:, None] & (g < n_col_blocks * 4)[ + None, : + ] + tl.store( + s_ptr + _mx_scale_offsets(r[:, None], g[None, :], n_col_blocks), + tl.where(valid, sbyte, 0).to(tl.uint8), + mask=smask, + ) + + +def can_use_silu_mul_mxfp8(hidden: torch.Tensor) -> bool: + return can_use_mxfp8_swizzled(hidden) and (hidden.shape[-1] // 2) % 32 == 0 + + +def _swiglu_mxfp8( + hidden: torch.Tensor, + alpha: float | None, + limit: float | None, + deepgemm: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: + if not can_use_silu_mul_mxfp8(hidden): + raise ValueError( + "expected a row-major bf16 CUDA [rows, 2 * n] tensor, n % 32 == 0" + ) + rows, twice = hidden.shape + n = twice // 2 + if deepgemm: + if n % 128: + raise ValueError(f"DeepGEMM MXFP8 scales need n % 128 == 0, got {n=}") + q = torch.empty(rows, n, dtype=_E4M3, device=hidden.device) + dg_rows = triton.cdiv(rows, 4) * 4 + base = torch.empty(n // 128, dg_rows, dtype=torch.int32, device=hidden.device) + s = base.T[:rows] + s_bytes = base.view(torch.uint8) + padded_rows = rows + else: + q, s = _alloc(rows, n, hidden.device, zero_scales=False) + s_bytes = s + dg_rows = 0 + padded_rows = triton.cdiv(rows, 128) * 128 + if rows == 0: + return q, s + n_groups = n // 32 + n_col_blocks = triton.cdiv(n_groups, 4) + block_r, g = (4 if rows <= 1024 else 16), 8 + grid = (triton.cdiv(padded_rows, block_r), triton.cdiv(n_groups, g)) + with torch.get_device_module().device(hidden.device): + _silu_mul_mxfp8_kernel[grid]( + hidden, + q, + s_bytes, + rows, + n, + n_groups, + n_col_blocks, + hidden.stride(0), + dg_rows, + ALPHA=alpha, + LIMIT=limit, + DEEPGEMM=deepgemm, + BLOCK_R=block_r, + G=g, + num_warps=4, + ) + return q, s + + +def silu_mul_mxfp8(hidden: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """``hidden [rows, 2n]`` bf16 (gate | up) -> quantized ``silu(gate) * up``.""" + return _swiglu_mxfp8(hidden, None, None) + + +def swiglu_oai_mxfp8( + hidden: torch.Tensor, alpha: float, limit: float +) -> tuple[torch.Tensor, torch.Tensor]: + """``hidden [rows, 2n]`` bf16 (gate | up) -> quantized SwiGLU-OAI, + ``clamp(gate, max=limit) * sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)``.""" + return _swiglu_mxfp8(hidden, alpha, limit) + + +def swiglu_oai_mxfp8_deepgemm( + hidden: torch.Tensor, alpha: float, limit: float +) -> tuple[torch.Tensor, torch.Tensor]: + """Same values as ``swiglu_oai_mxfp8``; scales in DeepGEMM's layout, the + ``sglang_per_token_group_quant_fp8(..., 32, column_major_scales=True, + scale_tma_aligned=True, scale_ue8m0=True)`` output: int32 ``[rows, n / 128]``, + four UE8M0 bytes each, MN-major with rows padded to 4.""" + return _swiglu_mxfp8(hidden, alpha, limit, deepgemm=True) + + +__all__ = [ + "can_use_silu_mul_mxfp8", + "silu_mul_mxfp8", + "swiglu_oai_mxfp8", + "swiglu_oai_mxfp8_deepgemm", +] diff --git a/python/sglang/kernels/ops/attention/__init__.py b/python/sglang/kernels/ops/attention/__init__.py index e982c4ffac..8e5ea819e6 100644 --- a/python/sglang/kernels/ops/attention/__init__.py +++ b/python/sglang/kernels/ops/attention/__init__.py @@ -3,7 +3,7 @@ The Triton kernels migrated here live in this package (``sglang.kernels.ops.attention.``); import them from there. Their ``KernelSpec`` metadata is registered below for inventory (backend = Triton). -KV-cache index/write kernels went to the ``kvcache`` group instead. +KV-cache write kernels went to the ``kvcache`` group instead. """ from __future__ import annotations @@ -33,9 +33,12 @@ _TRITON_KERNELS = [ ("metadata", "get_num_kv_splits_triton"), ("metadata", "prepare_swa_spec_page_table_triton"), ("metadata", "normal_decode_set_metadata"), - ("dsa_metadata", "fused_dsa_decode_metadata"), - ("dsa_metadata", "fused_dsa_target_verify_metadata"), - ("dsa_metadata", "fused_dsa_draft_extend_metadata"), + ("dsa.metadata", "fused_dsa_decode_metadata"), + ("dsa.metadata", "fused_dsa_target_verify_metadata"), + ("dsa.metadata", "fused_dsa_draft_extend_metadata"), + ("dsa.kpool_metadata.decode", "fused_dsa_decode_metadata_kpool"), + ("dsa.kpool_metadata.verify", "fused_dsa_target_verify_metadata_kpool"), + ("dsa.kpool_metadata.draft_extend", "fused_dsa_draft_extend_metadata_kpool"), ("rocm_mla_decode_rope", "decode_attention_fwd_grouped_rope"), ("verify_splitkv", "verify_splitkv_fwd"), ("unified_attention_3d_mtp", "unified_attention_3d_mtp_func"), @@ -44,9 +47,20 @@ _TRITON_KERNELS = [ ("pad", "unpad_draft_extend_output"), ("pad", "seqlens_expand_triton"), ("position", "compute_position_triton"), - ("dsv4_attn_metadata_kernels", "expand_prefill_causally"), - ("dsv4_attn_metadata_kernels", "build_page_table_positions"), - ("dsv4_attn_metadata_kernels", "build_causal_swa_page_indices"), + ("dsv4.attn_metadata_kernels", "expand_prefill_causally"), + ("dsv4.attn_metadata_kernels", "build_page_table_positions"), + ("dsv4.attn_metadata_kernels", "build_causal_swa_page_indices"), + ("kv_indices", "create_flashinfer_kv_indices_triton"), + ("kv_indices", "create_flashmla_kv_indices_triton"), + ("kv_indices", "create_chunked_prefix_cache_kv_indices"), + ("kv_indices", "get_num_kv_index_blocks_flashmla"), + ("kv_indices", "get_num_page_per_block_flashmla"), + ("trtllm_mha_page_table", "build_trtllm_mha_page_table"), + ("trtllm_mha_graph_metadata", "update_trtllm_mha_graph_metadata"), + ("aiter_unified_page_table", "scatter_ragged_to_page_table_kernel"), + ("aiter_unified_page_table", "scatter_req_to_token_to_page_table_kernel"), + ("fused_qk_rope_reshape_and_cache", "fused_qk_rope_reshape_and_cache"), + ("mla_qk_prep", "concat_and_cast_mha_k_triton"), ] for _mod, _fn in _TRITON_KERNELS: register_kernel( @@ -173,9 +187,7 @@ for _mod, _fn in [ ("dsa.transform_index", "transform_index_page_table_decode"), ("dsa.transform_index", "prepare_trtllm_nope_sparse_metadata"), ("dsa.transform_index", "transform_index_page_table_dcp"), - ("dsa.cp_split", "dsa_cp_interleave_q_seqs_kernel"), - ("dsv4.fp4_indexer", "quantize_fp4_indexer_tensor"), - ("dsv4.fp4_indexer", "store_fp4_index_k_cache"), + ("dsa.cp_split", "dsa_cp_interleave_q_seqs"), ]: register_kernel( KernelSpec( @@ -189,15 +201,13 @@ del _mod, _fn # Generic attention kernels migrated in Phase 2.5 (RFC #29630). for _mod, _fn in [ ("utils", "mla_quantize_and_rope_for_fp8"), - ("utils", "launch_reshape_and_cache_flash"), - ("utils", "launch_reshape_and_cache_shuffle_5d"), ("flash_mla_sm120", "flash_mla_with_kvcache_sm120"), ("dcp_kernels", "create_dcp_kv_indices"), ("dcp_kernels", "correct_attn_out"), ("dcp_kernels", "dcp_lse_combine_triton"), ("dcp_kernels", "dcp_pack_a2a_send"), ("pa_page_table", "_build_pa_page_table"), - ("nsa_triton_decode", "triton_sparse_attn_decode"), + ("dsv4.triton_sparse_decode", "triton_sparse_attn_decode"), ]: register_kernel( KernelSpec( @@ -211,7 +221,7 @@ del _mod, _fn # RoPE / QK-norm fusion kernels migrated from srt/layers top-level strays # (RFC #29630, Phase 2.5); registered for inventory. for _mod, _fn in [ - ("deepseek_v4_rope", "precompute_freqs_cis"), + ("dsv4.rope", "precompute_freqs_cis"), ("fused_qk_norm_rope_store", "fused_qk_norm_rope_swa_store"), ("fused_qk_rmsnorm_rope_gate", "fused_qk_gemma_rmsnorm_rope_gate"), ("rotary_triton", "triton_mrope_fused"), @@ -231,25 +241,7 @@ del _mod, _fn # Kernels introduced with Kimi-K3, inventoried by logical operator group. for _mod, _fn, _backend, _device in [ ("attn_res", "attn_res_fused_tma", KernelBackend.JIT, CapabilityRequirement.CUDA), - ( - "attn_res", - "attn_res_fused_pull_rs", - KernelBackend.JIT, - CapabilityRequirement.CUDA, - ), - ( - "attn_res", - "attn_res_fused_direct_ag", - KernelBackend.JIT, - CapabilityRequirement.CUDA, - ), ("attn_res_hip", "attn_res_hip", KernelBackend.TRITON, CapabilityRequirement.HIP), - ( - "mla_output_gate", - "kimi_k3_mla_output_gate", - KernelBackend.JIT, - CapabilityRequirement.CUDA, - ), ( "kda_decode_mtp", "fused_kda_decode_mtp_dspark", @@ -412,3 +404,131 @@ for _fn in ("fp4_index_logits_paged", "finish_paged_indexer_topk"): ) ) del _fn + +# Muse Spark QK norm (+ RoPE, + KV store), built per layer: inventoried through +# its class methods. +for _bk in ("torch", "triton", "helion"): + register_kernel( + KernelSpec( + op="attention.muse_spark_qk_norm_rope_store", + backend=KernelBackend(_bk), + target="sglang.kernels.ops.attention.muse_spark_qk_norm_rope_store:FusedQKNorm." + + ("forward_native" if _bk == "torch" else f"forward_{_bk}"), + capabilities=frozenset() if _bk == "torch" else CUDA_ONLY, + ) + ) +del _bk +for _mod, _fn in [ + ("mla_qk_prep", "concat_and_cast_q_fp8_pad"), + ("mla_qk_prep", "absorbed_bmm_concat_cast_q_fp8"), + ("metadata", "q8kv8_topk_length_from_indices"), + ("minimax_sparse.decode.topk_sparse", "flash_decode_with_gqa_share_sparse"), + ("minimax_sparse.prefill.topk_sparse", "flash_prefill_with_gqa_share_sparse"), +]: + register_kernel( + KernelSpec( + op=f"attention.{_fn}", + backend=KernelBackend.TRITON, + target=f"sglang.kernels.ops.attention.{_mod}:{_fn}", + ) + ) +del _mod, _fn +register_kernel( + KernelSpec( + op="attention.absorbed_bmm_concat_cast_q_fp8", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.attention.qprep_bf16_fp8_sm90:q8kv8_qprep_fwd", + capabilities=frozenset( + {CapabilityRequirement.cuda(min_sm=(9, 0), max_sm=(9, 9))} + ), + ) +) +register_kernel( + KernelSpec( + op="attention.topk_index_reduce", + backend=KernelBackend.TORCH, + target="sglang.kernels.ops.attention.minimax_sparse.common.index:topk_index_reduce", + ) +) +# The context-parallel indexer serves only the gfx950 ROCm top-k contract. +for _fn in ("score_local_blocks", "select_local_candidates", "merge_candidates"): + register_kernel( + KernelSpec( + op=f"attention.{_fn}", + backend=KernelBackend.TRITON, + target=f"sglang.kernels.ops.attention.minimax_sparse.decode.indexer_cp:{_fn}", + capabilities=frozenset({CapabilityRequirement.HIP}), + ) + ) +del _fn +register_kernel( + KernelSpec( + op="attention.gluon_sparse_prefill", + backend=KernelBackend.AITER, + target="sglang.kernels.ops.attention.minimax_sparse.prefill.gluon_prefill:gluon_sparse_prefill", + capabilities=frozenset({CapabilityRequirement.HIP}), + ) +) +register_kernel( + KernelSpec( + op="attention.mtp_verify_attn_fwd_asm", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.attention.vattn_asm_gfx950:mtp_verify_attn_fwd_asm", + capabilities=frozenset({CapabilityRequirement.HIP}), + ) +) +for _mod, _fn in [ + ("helion.kda_decode", "helion_fused_recurrent_kda_packed_decode"), + ("helion.kda_replayssm", "helion_fused_recurrent_kda_replayssm_decode"), + ("helion.kda_prefill", "chunk_kda"), + ("helion.kda_verify", "helion_kda_verify"), +]: + register_kernel( + KernelSpec( + op=f"attention.{_fn}", + backend=KernelBackend.HELION, + target=f"sglang.kernels.ops.attention.{_mod}:{_fn}", + capabilities=CUDA_ONLY, + ) + ) +del _mod, _fn +# tcgen05 + TMEM: datacenter Blackwell only (SM100 / SM103), not SM120. +for _mod, _fn in [ + ("linear.gdn_blackwell", "chunk_gated_delta_rule_cutedsl"), + ("linear.kda_blackwell", "chunk_kda_cutedsl"), + ("linear.kda_nvidia_prefill", "chunk_kda_fwd"), +]: + register_kernel( + KernelSpec( + op=f"attention.{_fn}", + backend=KernelBackend.CUTE_DSL, + target=f"sglang.kernels.ops.attention.{_mod}:{_fn}", + capabilities=frozenset( + {CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))} + ), + ) + ) +del _mod, _fn +register_kernel( + KernelSpec( + op="attention.chunk_kda_fwd", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.attention.linear.kda_ptx_prefill:chunk_kda_fwd", + # Exactly the kda_ptx_prefill.SM_ARCHS build targets (sm_100a, sm_103a). + capabilities=frozenset( + CapabilityRequirement.cuda(min_sm=sm, max_sm=sm) + for sm in ((10, 0), (10, 3)) + ), + ) +) +register_kernel( + KernelSpec( + op="attention.fp8_attention", + backend=KernelBackend.CUTE_DSL, + target="sglang.kernels.ops.attention.fp8_fa_sm120:fp8_attention", + # MmaMXF8Op assembles only for SM120 / SM121. + capabilities=frozenset( + {CapabilityRequirement.cuda(min_sm=(12, 0), max_sm=(12, 1))} + ), + ) +) diff --git a/python/sglang/kernels/ops/kvcache/aiter_unified_attention.py b/python/sglang/kernels/ops/attention/aiter_unified_page_table.py similarity index 100% rename from python/sglang/kernels/ops/kvcache/aiter_unified_attention.py rename to python/sglang/kernels/ops/attention/aiter_unified_page_table.py diff --git a/python/sglang/kernels/ops/attention/attn_res.py b/python/sglang/kernels/ops/attention/attn_res.py index 2074ca4135..9d7431eb24 100644 --- a/python/sglang/kernels/ops/attention/attn_res.py +++ b/python/sglang/kernels/ops/attention/attn_res.py @@ -12,13 +12,10 @@ from sglang.kernels.jit.utils import ( make_cpp_args, override_jit_cuda_arch, ) -from sglang.srt.utils.custom_op import register_custom_op if TYPE_CHECKING: from tvm_ffi.module import Module - from sglang.kernels.ops.communication.all_reduce import Communicator - _DIM: int = 7168 # K3 hidden size, template parameter of the TMA kernel _MAX_BANK_ROWS: int = 8 # K3 has <= 8 snapshots, upper bound of the nvb dispatch tables @@ -86,8 +83,9 @@ _TMA_BEST_CONFIG: dict[int, tuple[int, int, int]] = { def _tuning(nvb: int, num_tokens: int) -> tuple[int, int, int]: """(chunk_rows, occupancy, consumer_regs) for this aggregation point. - Shared by all three entry points below: they run the same kernel template - and differ only in the collective fused onto it.""" + Shared by attn_res_fused_tma and its collective variants in + communication.attn_res_collective: they run the same kernel template and + differ only in the collective fused onto it.""" if not 1 <= nvb <= _MAX_BANK_ROWS: raise ValueError(f"attn_res: nvb must be in [1, {_MAX_BANK_ROWS}], got {nvb}") best = _TMA_BEST_CONFIG[nvb] @@ -98,56 +96,6 @@ def _tuning(nvb: int, num_tokens: int) -> tuple[int, int, int]: return best -_COMM_MAP: dict[int, Communicator] = {} - - -def register_comm(comm: Communicator) -> None: - # One communicator per world_size per process -- see the note in - # communication/all_reduce_residual.py::register_comm. The ops key only on world_size, so an - # overwrite here would hand the old group's callers the new group's peer - # pointers. - prev = _COMM_MAP.get(comm.world_size) - assert prev is None or prev is comm, ( - f"a different communicator is already registered for world_size=" - f"{comm.world_size}" - ) - _COMM_MAP[comm.world_size] = comm - - -@register_custom_op(mutates_args=["out", "prefix_out"]) -def _attn_res_fused_pull_rs_op( - world_size: int, - input: torch.Tensor, - residual: torch.Tensor | None, - bank: torch.Tensor, - cw: torch.Tensor, - ow: torch.Tensor, - out: torch.Tensor, - prefix_out: torch.Tensor, - nvb: int, - eps: float, - input_mc_ptr: int, - max_blocks: int, - chunk_rows: int, - occupancy: int, - consumer_regs: int, -) -> None: - _jit_fused_tma_module(chunk_rows, occupancy, consumer_regs).run_pull_rs( - _COMM_MAP[world_size], - input, - residual, - bank, - cw, - ow, - out, - prefix_out, - nvb, - eps, - input_mc_ptr, - max_blocks, - ) - - def attn_res_fused_tma( prefix_sum: torch.Tensor, bank: torch.Tensor, @@ -186,113 +134,3 @@ def attn_res_fused_tma( _jit_fused_tma_module(*_tuning(nvb, prefix_sum.shape[0])).run( prefix_sum, bank, cw, ow, out, nvb, eps, write_prefix ) - - -@register_custom_op(mutates_args=["bank", "out"]) -def _attn_res_fused_direct_ag_op( - world_size: int, - prefix_sum: torch.Tensor, - bank: torch.Tensor, - cw: torch.Tensor, - ow: torch.Tensor, - out: torch.Tensor, - nvb: int, - eps: float, - output_mc_ptr: int, - max_blocks: int, - write_prefix: bool, - chunk_rows: int, - occupancy: int, - consumer_regs: int, -) -> None: - _jit_fused_tma_module(chunk_rows, occupancy, consumer_regs).run_direct_ag( - _COMM_MAP[world_size], - prefix_sum, - bank, - cw, - ow, - out, - nvb, - eps, - output_mc_ptr, - max_blocks, - write_prefix, - ) - - -def attn_res_fused_direct_ag( - world_size: int, - prefix_sum: torch.Tensor, - bank: torch.Tensor, - cw: torch.Tensor, - ow: torch.Tensor, - out: torch.Tensor, - nvb: int, - eps: float, - *, - output_mc_ptr: int, - max_blocks: int = 128, - write_prefix: bool = False, -) -> torch.Tensor: - """Fuse local attention-residual aggregation with direct multicast AG. - - `out` is the full symmetric [world * local_tokens, H] output. Each rank - aggregates its local token shard and multicast-stores normalized vectors - directly from consumer registers into that rank's slice on every peer. - """ - _attn_res_fused_direct_ag_op( - world_size, - prefix_sum, - bank, - cw, - ow, - out, - nvb, - eps, - output_mc_ptr, - max_blocks, - write_prefix, - *_tuning(nvb, prefix_sum.shape[0]), - ) - return out - - -def attn_res_fused_pull_rs( - world_size: int, - input: torch.Tensor, - residual: torch.Tensor | None, - bank: torch.Tensor, - cw: torch.Tensor, - ow: torch.Tensor, - out: torch.Tensor, - prefix_out: torch.Tensor, - nvb: int, - eps: float, - *, - input_mc_ptr: int, - max_blocks: int = 128, -) -> tuple[torch.Tensor, torch.Tensor]: - """Fused NVLS pull RS + local residual + TMA attention aggregation. - - `input` is the full TP-partial o_proj tensor in multicast symmetric - memory. Every rank reduces only its contiguous token shard, optionally - adds its already-local residual, materializes that prefix in `prefix_out`, - and feeds it directly to the fused attention-residual aggregation/norm. - """ - _attn_res_fused_pull_rs_op( - world_size, - input, - residual, - bank, - cw, - ow, - out, - prefix_out, - nvb, - eps, - input_mc_ptr, - max_blocks, - # prefix_out, not input: the shard this rank actually aggregates. - *_tuning(nvb, prefix_out.shape[0]), - ) - return out, prefix_out diff --git a/python/sglang/kernels/ops/attention/dsa/cp_split.py b/python/sglang/kernels/ops/attention/dsa/cp_split.py index d2b181dff8..d346facfc1 100644 --- a/python/sglang/kernels/ops/attention/dsa/cp_split.py +++ b/python/sglang/kernels/ops/attention/dsa/cp_split.py @@ -3,12 +3,13 @@ Migrated from ``sglang.srt.layers.attention.dsa.utils`` (RFC #29630, Phase 2.5). """ +import torch import triton import triton.language as tl @triton.jit -def dsa_cp_interleave_q_seqs_kernel( +def _dsa_cp_interleave_q_seqs_kernel( in_seqs_ptr, out_seqs_ptr, bs_idx_ptr, @@ -27,3 +28,18 @@ def dsa_cp_interleave_q_seqs_kernel( tl.store(out_seqs_ptr + bs_idx, cur_seq) bs_idx += 1 extra_seq = cur_len - cur_seq * cp_size + + +def dsa_cp_interleave_q_seqs( + extend_seqs: torch.Tensor, num_rank_seqs: int, cp_size: int, cp_rank: int +) -> tuple[torch.Tensor, torch.Tensor]: + """This rank's interleaved share of each request's extend length, as + ``(q_lens, bs_idx)`` over the ``num_rank_seqs`` requests left non-empty.""" + q_lens = torch.empty( + (num_rank_seqs,), device=extend_seqs.device, dtype=extend_seqs.dtype + ) + bs_idx = torch.empty((num_rank_seqs,), device=extend_seqs.device, dtype=torch.int32) + _dsa_cp_interleave_q_seqs_kernel[(1,)]( + extend_seqs, q_lens, bs_idx, len(extend_seqs), cp_size, cp_rank + ) + return q_lens, bs_idx diff --git a/python/sglang/kernels/ops/attention/cutedsl_fp8_paged_mqa_logits.py b/python/sglang/kernels/ops/attention/dsa/cutedsl_fp8_paged_mqa_logits.py similarity index 100% rename from python/sglang/kernels/ops/attention/cutedsl_fp8_paged_mqa_logits.py rename to python/sglang/kernels/ops/attention/dsa/cutedsl_fp8_paged_mqa_logits.py diff --git a/python/sglang/kernels/ops/attention/dsa/cutedsl_paged_mqa_logits.py b/python/sglang/kernels/ops/attention/dsa/cutedsl_paged_mqa_logits.py index 34d9bab382..4df1e7dca5 100644 --- a/python/sglang/kernels/ops/attention/dsa/cutedsl_paged_mqa_logits.py +++ b/python/sglang/kernels/ops/attention/dsa/cutedsl_paged_mqa_logits.py @@ -17,7 +17,9 @@ import cutlass.cute as cute import torch from cutlass.utils import HardwareInfo -from sglang.kernels.ops.attention.cutedsl_fp8_paged_mqa_logits import FP8MQALogitsKernel +from sglang.kernels.ops.attention.dsa.cutedsl_fp8_paged_mqa_logits import ( + FP8MQALogitsKernel, +) from sglang.srt.utils import is_sm100_supported logger = logging.getLogger(__name__) diff --git a/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py b/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py index 3108701f93..829bf75999 100644 --- a/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py +++ b/python/sglang/kernels/ops/attention/dsa/dequant_k_cache.py @@ -141,7 +141,7 @@ def _dequantize_k_cache_fast_kernel( tl.store(dst_ptr, data, mask=mask) -def dequantize_k_cache_paged( +def dequantize_k_cache_paged_dsa( quant_k_cache: torch.Tensor, page_table_1_flattened: torch.Tensor, group_size: int = 128, @@ -262,7 +262,7 @@ def _dequantize_k_cache_paged_kernel( _GATHER_TOKENS_PER_PROG = 4 -def gather_dequant_requant_fp8_paged( +def gather_dequant_requant_fp8_paged_dsa( quant_k_cache: torch.Tensor, page_table_1_flattened: torch.Tensor, group_size: int = 128, @@ -441,122 +441,6 @@ def _gather_dequant_requant_fp8_paged_vec_kernel( tl.store(dst_r, data, mask=row_in_range[:, None]) -def gather_dequant_requant_fp8_paged_legacy( - quant_k_cache: torch.Tensor, - page_table_1_flattened: torch.Tensor, - group_size: int = 128, - extra_rows: int = 0, -) -> torch.Tensor: - """Legacy (pre-vectorization) gather + dequant + requant. - - Kept as the bit-exactness / performance reference for - ``gather_dequant_requant_fp8_paged`` (see - ``benchmark/kernels/deepseek/benchmark_q8kv8_kv_gather.py``). Allocates and zero-fills the - full destination each call, then launches one program per - (token, 128-elem slice). - """ - dim_quant = quant_k_cache.shape[-1] - dim_nope, dim_rope = _infer_dsa_dims(dim_quant) - quant_k_cache = quant_k_cache.view((-1, dim_quant)) - - num_tokens = page_table_1_flattened.shape[0] - assert quant_k_cache.dtype == torch.float8_e4m3fn - num_tiles = dim_nope // group_size - out_dim = dim_nope + dim_rope - assert num_tiles * group_size == dim_nope - - total_rows = num_tokens + extra_rows - # Allocate a fresh zero-filled buffer. The extra landing-pad rows at - # the tail must read as zeros (the kernel may over-read past - # num_tokens for masked indices). - output = torch.zeros( - (total_rows, 1, out_dim), - dtype=torch.float8_e4m3fn, - device=quant_k_cache.device, - ) - - num_blocks_per_token = num_tiles + triton.cdiv(dim_rope, group_size) - - input_nope_q = quant_k_cache[:, :dim_nope] - input_nope_s = quant_k_cache[:, dim_nope : dim_nope + num_tiles * 4].view( - torch.float32 - ) - input_rope = quant_k_cache[:, dim_nope + num_tiles * 4 :].view(torch.bfloat16) - - _gather_dequant_requant_fp8_paged_kernel[(num_tokens, num_blocks_per_token)]( - output, - input_nope_q, - input_nope_s, - input_rope, - page_table_1_flattened, - output.stride(0), - input_nope_q.stride(0), - input_nope_s.stride(0), - input_rope.stride(0), - NUM_NOPE_BLOCKS=num_tiles, - GROUP_SIZE=group_size, - DIM_NOPE=dim_nope, - DIM_ROPE=dim_rope, - ) - - return output - - -@triton.jit -def _gather_dequant_requant_fp8_paged_kernel( - output_ptr, - input_nope_q_ptr, - input_nope_s_ptr, - input_rope_ptr, - page_table_1_ptr, - output_stride_0: int, - input_nope_q_stride_0: int, - input_nope_s_stride_0: int, - input_rope_stride_0: int, - NUM_NOPE_BLOCKS: tl.constexpr, - GROUP_SIZE: tl.constexpr, - DIM_NOPE: tl.constexpr, - DIM_ROPE: tl.constexpr, -): - """Fused gather + dequant(per-group) + requant(per-tensor) -> fp8.""" - token_id = tl.program_id(0) - token_id_paged = tl.load(page_table_1_ptr + token_id).to(tl.int32) - raw_block_id = tl.program_id(1) - - if raw_block_id < NUM_NOPE_BLOCKS: - # nope: read fp8, mul group scale -> f32, cast to fp8_e4m3fn - effective_block_id = raw_block_id - offs_q = effective_block_id * GROUP_SIZE + tl.arange(0, GROUP_SIZE) - mask = offs_q < DIM_NOPE - - ptr_q = input_nope_q_ptr + token_id_paged * input_nope_q_stride_0 + offs_q - ptr_s = ( - input_nope_s_ptr - + token_id_paged * input_nope_s_stride_0 - + effective_block_id - ) - - y_q = tl.load(ptr_q, mask=mask, other=0.0).to(tl.float32) - y_s = tl.load(ptr_s) - - # dequant -> f32 -> requant to fp8 - y = (y_q * y_s).to(tl.float8e4nv) - - dst_ptr = output_ptr + token_id * output_stride_0 + offs_q - tl.store(dst_ptr, y, mask=mask) - else: - # rope: read bf16, cast to fp8 - effective_block_id = raw_block_id - NUM_NOPE_BLOCKS - offs = effective_block_id * GROUP_SIZE + tl.arange(0, GROUP_SIZE) - mask = offs < DIM_ROPE - - src_ptr = input_rope_ptr + token_id_paged * input_rope_stride_0 + offs - dst_ptr = output_ptr + token_id * output_stride_0 + DIM_NOPE + offs - - data = tl.load(src_ptr, mask=mask).to(tl.float8e4nv) - tl.store(dst_ptr, data, mask=mask) - - @triton.jit def _concat_cast_kv_fp8_pad_kernel( out_ptr, diff --git a/python/sglang/kernels/ops/attention/fused_metadata_copy.py b/python/sglang/kernels/ops/attention/dsa/fused_metadata_copy.py similarity index 100% rename from python/sglang/kernels/ops/attention/fused_metadata_copy.py rename to python/sglang/kernels/ops/attention/dsa/fused_metadata_copy.py diff --git a/python/sglang/kernels/ops/attention/dsa/hip_gfx950/__init__.py b/python/sglang/kernels/ops/attention/dsa/hip_gfx950/__init__.py index d7c5c9780e..4aaa997d36 100644 --- a/python/sglang/kernels/ops/attention/dsa/hip_gfx950/__init__.py +++ b/python/sglang/kernels/ops/attention/dsa/hip_gfx950/__init__.py @@ -1,13 +1,4 @@ """gfx950 fused DSA indexer decode path: four raw-HIP kernels replacing the 12-launch ROCm chain. Enabled wherever supported; the gate is -``utils.gfx950_fused_indexer_runtime_ok``.""" - -from sglang.kernels.ops.attention.dsa.hip_gfx950.fused_decode import ( # noqa: F401 - CACHE_TOK_STRIDE, - MAX_ROWS, - PAGE_SIZE, - Gfx950FusedIndexer, - model_shape_supported, - prealloc_workspace, - supported_hardware, -) +``utils.gfx950_fused_indexer_runtime_ok``, the runtime wrapper +``srt.layers.attention.dsa.gfx950_fused_indexer``.""" diff --git a/python/sglang/kernels/ops/attention/dsa/indexer_q.py b/python/sglang/kernels/ops/attention/dsa/indexer_q.py new file mode 100644 index 0000000000..214c9d79d9 --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsa/indexer_q.py @@ -0,0 +1,53 @@ +"""DSA only. Indexer Q kernels (JIT).""" + +from typing import Tuple + +import torch + +from sglang.kernels.jit.utils import ( + cache_once, + is_arch_support_pdl, + load_jit, + make_cpp_args, +) +from sglang.kernels.ops.attention.dsv4.utils import make_name + + +# V3.2 lays q out as [rope | nope] (V4 is [nope | rope]) -> kRopeFirst=true, and +# drops the Hadamard rotation (kHadamard=false). +@cache_once +def _jit_main_q_indexer_rope_first_quant_module(dtype: torch.dtype): + args = make_cpp_args(dtype, is_arch_support_pdl(), True, False) + return load_jit( + make_name("main_q_indexer_rope_first_quant"), + *args, + cuda_files=["deepseek_v4/main_norm_rope.cuh"], + cuda_wrappers=[ + ("forward", f"FusedQIndexerRopeHadamardQuantKernel<{args}>::forward"), + ], + ) + + +def fused_q_indexer_rope_first_quant( + q_input: torch.Tensor, + weight: torch.Tensor, + weight_scale: float, + cos_sin_cache: torch.Tensor, + positions: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + """DeepSeek-V3.2 only. Indexer Q: RoPE on the leading dims + fp8 act-quant. CUDA only.""" + q_fp8 = torch.empty(q_input.shape, dtype=torch.float8_e4m3fn, device=q_input.device) + weights_out = torch.empty( + (*q_input.shape[:-1], 1), dtype=torch.float32, device=q_input.device + ) + module = _jit_main_q_indexer_rope_first_quant_module(q_input.dtype) + module.forward( + q_input, + q_fp8, + weight, + weights_out, + float(weight_scale), + cos_sin_cache, + positions, + ) + return q_fp8, weights_out diff --git a/python/sglang/kernels/ops/attention/dsa_kpool_metadata/__init__.py b/python/sglang/kernels/ops/attention/dsa/kpool_metadata/__init__.py similarity index 100% rename from python/sglang/kernels/ops/attention/dsa_kpool_metadata/__init__.py rename to python/sglang/kernels/ops/attention/dsa/kpool_metadata/__init__.py diff --git a/python/sglang/kernels/ops/attention/dsa_kpool_metadata/decode.py b/python/sglang/kernels/ops/attention/dsa/kpool_metadata/decode.py similarity index 97% rename from python/sglang/kernels/ops/attention/dsa_kpool_metadata/decode.py rename to python/sglang/kernels/ops/attention/dsa/kpool_metadata/decode.py index f9ff0a7ca0..3de6c6db82 100644 --- a/python/sglang/kernels/ops/attention/dsa_kpool_metadata/decode.py +++ b/python/sglang/kernels/ops/attention/dsa/kpool_metadata/decode.py @@ -6,7 +6,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.attention.dsa_kpool_metadata.scan import bounded_scan_num_splits +from sglang.kernels.ops.attention.dsa.kpool_metadata.scan import bounded_scan_num_splits @triton.jit( @@ -17,7 +17,7 @@ from sglang.kernels.ops.attention.dsa_kpool_metadata.scan import bounded_scan_nu "num_splits", ] ) -def _fused_dsa_decode_metadata_kernel( +def _fused_dsa_decode_metadata_kpool_kernel( seq_lens, req_pool_indices, req_to_token, @@ -122,7 +122,7 @@ def _fused_dsa_decode_metadata_kernel( ) -def fused_dsa_decode_metadata( +def fused_dsa_decode_metadata_kpool( seq_lens: torch.Tensor, req_pool_indices: torch.Tensor, req_to_token: torch.Tensor, @@ -194,7 +194,7 @@ def fused_dsa_decode_metadata( num_splits = bounded_scan_num_splits(bs, num_col_blocks) grid = (1 + bs * num_splits,) - _fused_dsa_decode_metadata_kernel[grid]( + _fused_dsa_decode_metadata_kpool_kernel[grid]( seq_lens, req_pool_indices, req_to_token, diff --git a/python/sglang/kernels/ops/attention/dsa_kpool_metadata/draft_extend.py b/python/sglang/kernels/ops/attention/dsa/kpool_metadata/draft_extend.py similarity index 97% rename from python/sglang/kernels/ops/attention/dsa_kpool_metadata/draft_extend.py rename to python/sglang/kernels/ops/attention/dsa/kpool_metadata/draft_extend.py index c60e9e8c13..692219ff9d 100644 --- a/python/sglang/kernels/ops/attention/dsa_kpool_metadata/draft_extend.py +++ b/python/sglang/kernels/ops/attention/dsa/kpool_metadata/draft_extend.py @@ -6,7 +6,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.attention.dsa_kpool_metadata.scan import bounded_scan_num_splits +from sglang.kernels.ops.attention.dsa.kpool_metadata.scan import bounded_scan_num_splits @triton.jit( @@ -18,7 +18,7 @@ from sglang.kernels.ops.attention.dsa_kpool_metadata.scan import bounded_scan_nu "num_splits", ] ) -def _fused_dsa_draft_extend_metadata_kernel( +def _fused_dsa_draft_extend_metadata_kpool_kernel( seq_lens, extend_seq_lens, req_pool_indices, @@ -194,7 +194,7 @@ def _fused_dsa_draft_extend_metadata_kernel( ) -def fused_dsa_draft_extend_metadata( +def fused_dsa_draft_extend_metadata_kpool( seq_lens: torch.Tensor, extend_seq_lens: torch.Tensor, req_pool_indices: torch.Tensor, @@ -270,7 +270,7 @@ def fused_dsa_draft_extend_metadata( num_splits = bounded_scan_num_splits(bs, num_col_blocks) grid = (1 + bs * num_splits,) - _fused_dsa_draft_extend_metadata_kernel[grid]( + _fused_dsa_draft_extend_metadata_kpool_kernel[grid]( seq_lens, extend_seq_lens, req_pool_indices, diff --git a/python/sglang/kernels/ops/attention/dsa_kpool_metadata/scan.py b/python/sglang/kernels/ops/attention/dsa/kpool_metadata/scan.py similarity index 100% rename from python/sglang/kernels/ops/attention/dsa_kpool_metadata/scan.py rename to python/sglang/kernels/ops/attention/dsa/kpool_metadata/scan.py diff --git a/python/sglang/kernels/ops/attention/dsa_kpool_metadata/verify.py b/python/sglang/kernels/ops/attention/dsa/kpool_metadata/verify.py similarity index 96% rename from python/sglang/kernels/ops/attention/dsa_kpool_metadata/verify.py rename to python/sglang/kernels/ops/attention/dsa/kpool_metadata/verify.py index 82f7a8e673..b0c1282613 100644 --- a/python/sglang/kernels/ops/attention/dsa_kpool_metadata/verify.py +++ b/python/sglang/kernels/ops/attention/dsa/kpool_metadata/verify.py @@ -6,7 +6,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.attention.dsa_kpool_metadata.scan import bounded_scan_num_splits +from sglang.kernels.ops.attention.dsa.kpool_metadata.scan import bounded_scan_num_splits @triton.jit( @@ -17,7 +17,7 @@ from sglang.kernels.ops.attention.dsa_kpool_metadata.scan import bounded_scan_nu "num_splits", ] ) -def _fused_dsa_target_verify_metadata_kernel( +def _fused_dsa_target_verify_metadata_kpool_kernel( seq_lens, req_pool_indices, req_to_token, @@ -157,7 +157,7 @@ def _fused_dsa_target_verify_metadata_kernel( ) -def _prep_fused_dsa_target_verify_metadata_launch( +def _prep_fused_dsa_target_verify_metadata_kpool_launch( seq_lens: torch.Tensor, req_pool_indices: torch.Tensor, req_to_token: torch.Tensor, @@ -265,7 +265,7 @@ def _prep_fused_dsa_target_verify_metadata_launch( return grid, args, constexprs -def fused_dsa_target_verify_metadata( +def fused_dsa_target_verify_metadata_kpool( seq_lens: torch.Tensor, req_pool_indices: torch.Tensor, req_to_token: torch.Tensor, @@ -291,7 +291,7 @@ def fused_dsa_target_verify_metadata( dsa_cu_seqlens_k[:1].zero_() return - grid, args, constexprs = _prep_fused_dsa_target_verify_metadata_launch( + grid, args, constexprs = _prep_fused_dsa_target_verify_metadata_kpool_launch( seq_lens, req_pool_indices, req_to_token, @@ -310,4 +310,4 @@ def fused_dsa_target_verify_metadata( paged_mqa_ctx_lens_2d, index_kpool, ) - _fused_dsa_target_verify_metadata_kernel[grid](*args, **constexprs) + _fused_dsa_target_verify_metadata_kpool_kernel[grid](*args, **constexprs) diff --git a/python/sglang/kernels/ops/attention/dsa_metadata.py b/python/sglang/kernels/ops/attention/dsa/metadata.py similarity index 100% rename from python/sglang/kernels/ops/attention/dsa_metadata.py rename to python/sglang/kernels/ops/attention/dsa/metadata.py diff --git a/python/sglang/kernels/ops/attention/dsa/tilelang_kernel.py b/python/sglang/kernels/ops/attention/dsa/tilelang_kernel.py index 9c044d7c4c..4d80f945b1 100644 --- a/python/sglang/kernels/ops/attention/dsa/tilelang_kernel.py +++ b/python/sglang/kernels/ops/attention/dsa/tilelang_kernel.py @@ -1,6 +1,6 @@ import functools from functools import lru_cache -from typing import Any, Optional, Tuple +from typing import Any, Optional import tilelang import tilelang.language as T @@ -55,22 +55,6 @@ INT32 = "int32" UINT8 = "uint8" -def fast_log2_ceil(x): - bits_x = T.reinterpret("uint32", x) - exp_x = (bits_x >> 23) & 0xFF - man_bits = bits_x & ((1 << 23) - 1) - return T.Cast("int32", exp_x - 127 + T.if_then_else(man_bits != 0, 1, 0)) - - -def fast_pow2(x): - bits_x = (x + 127) << 23 - return T.reinterpret("float32", bits_x) - - -def fast_round_scale(amax, fp8_max_inv): - return fast_pow2(fast_log2_ceil(amax * fp8_max_inv)) - - @lru_cache(maxsize=8) def _pick_inner_iter(seq: int, ni: int, cu: int, block_per_cu: int) -> int: """ @@ -88,87 +72,6 @@ def _pick_inner_iter(seq: int, ni: int, cu: int, block_per_cu: int) -> int: return 1 -@tilelang.jit(pass_configs=pass_configs) -def act_quant_kernel( - N, in_dtype=BF16, out_dtype=FP8, scale_dtype=FP32, round_scale=False -): - M = T.symbolic("M") - fp8_min = -224.0 if _is_fp8_fnuz else -448.0 - fp8_max = 224.0 if _is_fp8_fnuz else 448.0 - fp8_max_inv = 1 / fp8_max - num_stages = 0 if round_scale else 2 - blk_m = 32 - group_size = 128 - - @T.prim_func - def act_quant_kernel_( - X: T.Tensor[(M, N), in_dtype], - Y: T.Tensor[(M, N), out_dtype], - S: T.Tensor[(M, T.ceildiv(N, group_size)), scale_dtype], - ): - with T.Kernel(T.ceildiv(M, blk_m), T.ceildiv(N, group_size), threads=128) as ( - pid_m, - pid_n, - ): - x_shared = T.alloc_shared((blk_m, group_size), in_dtype) - x_local = T.alloc_fragment((blk_m, group_size), in_dtype) - amax_local = T.alloc_fragment((blk_m,), scale_dtype) - s_local = T.alloc_fragment((blk_m,), scale_dtype) - y_local = T.alloc_fragment((blk_m, group_size), out_dtype) - y_shared = T.alloc_shared((blk_m, group_size), out_dtype) - - for _ in T.Pipelined(1, num_stages=num_stages): - T.copy(X[pid_m * blk_m, pid_n * group_size], x_shared) - T.copy(x_shared, x_local) - T.reduce_absmax(x_local, amax_local, dim=1) - for i in T.Parallel(blk_m): - amax_local[i] = T.max(amax_local[i], 1e-4) - if round_scale: - s_local[i] = fast_round_scale(amax_local[i], fp8_max_inv) - else: - s_local[i] = amax_local[i] * fp8_max_inv - for i, j in T.Parallel(blk_m, group_size): - y_local[i, j] = T.clamp( - x_local[i, j] / s_local[i], fp8_min, fp8_max - ) - for i in T.Parallel(blk_m): - S[pid_m * blk_m + i, pid_n] = s_local[i] - T.copy(y_local, y_shared) - T.copy(y_shared, Y[pid_m * blk_m, pid_n * group_size]) - - return act_quant_kernel_ - - -def act_quant( - x: torch.Tensor, block_size: int = 128, scale_fmt: Optional[str] = None -) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Quantizes the input tensor `x` using block-wise quantization. - - Args: - x (torch.Tensor): The input tensor to be quantized. Must be contiguous and its last dimension size must be divisible by `block_size`. - block_size (int, optional): The size of the blocks to be used for quantization. Default is 128. - scale_fmt (Optional[str], optional): The format of the scale. Default is None. - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - The quantized tensor with dtype `torch.float8_e4m3fn`. - - A tensor of scaling factors with dtype `torch.float32`. - """ - assert x.is_contiguous(), "Input tensor must be contiguous" - assert x.size(-1) % block_size == 0, ( - f"Last dimension size must be divisible by block_size (block_size={block_size})" - ) - N = x.size(-1) - if _is_fp8_fnuz: - y = torch.empty_like(x, dtype=torch.float8_e4m3fnuz) - else: - y = torch.empty_like(x, dtype=torch.float8_e4m3fn) - s = x.new_empty(*x.size()[:-1], N // block_size, dtype=torch.float32) - kernel = act_quant_kernel(N, round_scale=scale_fmt is not None) - kernel(x.view(-1, N), y.view(-1, N), s.view(-1, N // block_size)) - return y, s - - @tilelang.jit( out_idx=[-1], pass_configs={ @@ -1482,1049 +1385,3 @@ def tilelang_fp8_paged_mqa_logits( kvcache_u8 = kvcache_fp8.view(-1, block_size * (head_dim + 4)) kernel(q_fp8, kvcache_u8, weight, seq_lens, page_table, logits) return logits - - -def _build_fp8_combined_view(k_cache: torch.Tensor) -> Tuple[torch.Tensor, int, int]: - """ - Reinterpret a MODEL1_FP8Sparse KV cache as a contiguous uint32 view. - Input: k_cache (num_blocks, block_size, 1, d_qk) fp8/uint8 - — per-block storage also holds scales + padding past d_qk. - Output: (num_blocks, block_pad_u32) uint32 covering the full block - stride. Same storage ashe input, no copy. - """ - k_u8 = k_cache.view(torch.uint8) if k_cache.dtype != torch.uint8 else k_cache - num_blocks = k_u8.shape[0] - block_size = k_u8.shape[1] - block_pad_u32 = k_u8.stride(0) // 4 - storage = k_u8.untyped_storage() - flat_u32 = torch.empty(0, dtype=torch.uint32, device=k_u8.device).set_( - storage, 0, (storage.nbytes() // 4,), (1,) - ) - k_combined = torch.as_strided( - flat_u32, - size=(num_blocks, block_pad_u32), - stride=(block_pad_u32, 1), - storage_offset=k_u8.storage_offset() // 4, - ) - return k_combined, num_blocks, block_size - - -_TOPK_LEN_SENTINEL_CACHE: dict = {} -_INT32_MAX = 2**30 - - -def _topk_length_sentinel(device: torch.device, batch: int) -> torch.Tensor: - """Cached `(batch,) int32 INT_MAX` tensor used when `topk_length` is None.""" - cur = _TOPK_LEN_SENTINEL_CACHE.get(device) - if cur is None or cur.numel() < batch: - cur = torch.full( - (max(batch, 256),), _INT32_MAX, dtype=torch.int32, device=device - ) - _TOPK_LEN_SENTINEL_CACHE[device] = cur - return cur[:batch] - - -@tilelang.jit( - out_idx=[-2, -1], - pass_configs={ - tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True, - tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True, - tilelang.PassConfigKey.TL_ENABLE_FAST_MATH: True, - }, -) -def dpsk_v4_fp8_partial_kernel( - num_heads: int, - topk_1: int, - block_size_kv_1: int, - topk_2: int = 0, - block_size_kv_2: int = 0, - *, - dim: int = 448, - tail_dim: int = 64, - sm_scale: float = 0.0, - block_I: int = 64, - inner_iter_1: int = 1, - inner_iter_2: int = 0, - num_stages: int = 0, - threads: int = 512, -) -> Any: - """ - Read FP8 K cache directly, dequantise to BF16 in-kernel, do flash-attn - online softmax with split-K. Supports a second cache (`topk_2>0`) and - `attn_sink` is folded later by the combine kernel. - """ - log2e: float = 1.44269504 - if sm_scale <= 0.0: - sm_scale = (1.0 / (dim + tail_dim)) ** 0.5 * log2e - else: - sm_scale = sm_scale * log2e - assert dim == 448 and tail_dim == 64 - assert topk_1 % block_I == 0 - assert (topk_1 // block_I) % inner_iter_1 == 0, ( - f"NI_1={topk_1 // block_I} must be divisible by inner_iter_1={inner_iter_1}" - ) - assert block_size_kv_1 > 0 and (block_size_kv_1 & (block_size_kv_1 - 1)) == 0 - - is_dual = topk_2 > 0 - if is_dual: - assert inner_iter_2 > 0, "dual-cache call requires inner_iter_2 > 0" - assert topk_2 % block_I == 0 - assert (topk_2 // block_I) % inner_iter_2 == 0, ( - f"NI_2={topk_2 // block_I} must be divisible by inner_iter_2={inner_iter_2}" - ) - assert block_size_kv_2 > 0 and (block_size_kv_2 & (block_size_kv_2 - 1)) == 0 - - PACKED_W = dim + 2 * tail_dim - NOPE_TILE = 64 - NUM_TILES = dim // NOPE_TILE - SCALE_W = 8 - PACKED_W4 = PACKED_W // 4 - SCALE_W4 = SCALE_W // 4 - - kv_group = 1 - batch = T.symbolic("batch") - seq_len = T.symbolic("seq_len") - num_blocks_kv_1 = T.symbolic("num_blocks_kv_1") - block_pad_u32_1 = T.symbolic("block_pad_u32_1") - if is_dual: - num_blocks_kv_2 = T.symbolic("num_blocks_kv_2") - block_pad_u32_2 = T.symbolic("block_pad_u32_2") - - head_kv = num_heads // kv_group - D = dim - D_tail = tail_dim - BI = block_I - padded_H = max(tilelang.math.next_power_of_2(head_kv), 16) - if head_kv > 64: - assert head_kv % 64 == 0 - REPLICATE_H = (head_kv + 63) // 64 if head_kv > 64 else 1 - H_per_block = 64 if REPLICATE_H > 1 else padded_H - - NI_1 = topk_1 // BI - n_groups_1 = NI_1 // inner_iter_1 - NI_2 = (topk_2 // BI) if is_dual else 0 - n_groups_2 = (NI_2 // inner_iter_2) if is_dual else 0 - n_groups = n_groups_1 + n_groups_2 - - BS_KV_1 = block_size_kv_1 - NOPE_ROPE_U32_PER_BLOCK_1 = BS_KV_1 * PACKED_W4 - if is_dual: - BS_KV_2 = block_size_kv_2 - NOPE_ROPE_U32_PER_BLOCK_2 = BS_KV_2 * PACKED_W4 - - q_shape = [batch, seq_len, num_heads, D + D_tail] - k1_shape = [num_blocks_kv_1, block_pad_u32_1] - indices1_shape = [batch, seq_len, topk_1] - topk_length_shape = [batch] - partial_o_shape = [batch, seq_len, n_groups, num_heads, D + D_tail] - partial_lse_shape = [batch, seq_len, n_groups, num_heads] - if is_dual: - k2_shape = [num_blocks_kv_2, block_pad_u32_2] - indices2_shape = [batch, seq_len, topk_2] - - accum_dtype = "float" - indices_dtype = INT32 - - if is_dual: - - @T.prim_func - def main( - Q: T.Tensor(q_shape, BF16), # type: ignore - K_combined_1: T.Tensor(k1_shape, "uint32"), # type: ignore - Indices_1: T.Tensor(indices1_shape, indices_dtype), # type: ignore - Topk_length_1: T.Tensor(topk_length_shape, indices_dtype), # type: ignore - K_combined_2: T.Tensor(k2_shape, "uint32"), # type: ignore - Indices_2: T.Tensor(indices2_shape, indices_dtype), # type: ignore - Topk_length_2: T.Tensor(topk_length_shape, indices_dtype), # type: ignore - Partial_O: T.Tensor(partial_o_shape, BF16), # type: ignore - Partial_LSE: T.Tensor(partial_lse_shape, accum_dtype), # type: ignore - ) -> None: - """ - grid: (seq_len * REPLICATE_H * n_groups, batch, 1) - Each block processes `inner_iter_1` (or `inner_iter_2`) consecutive - KV tiles of one phase and writes one (partial_o, partial_lse) entry. - """ - with T.Kernel( - seq_len * REPLICATE_H * n_groups, batch, kv_group, threads=threads - ) as (bx, by, bz): - Q_shared = T.alloc_fragment([H_per_block, D], BF16) - Q_tail_shared = T.alloc_fragment([H_per_block, D_tail], BF16) - K_packed_shared = T.alloc_shared([BI, PACKED_W4], "uint32") - K_scale_shared = T.alloc_shared([BI, SCALE_W4], "uint32") - KV_shared = T.alloc_shared([BI, D], BF16) - K_tail_shared = T.alloc_shared([BI, D_tail], BF16) - S_shared = T.alloc_shared([H_per_block, BI], BF16) - page_idx_shared = T.alloc_shared([BI], INT32) - - mask = T.alloc_fragment([BI], "bool") - scale_byte_local = T.alloc_fragment([BI, NUM_TILES], "uint32") - - acc_o = T.alloc_fragment([H_per_block, D], accum_dtype) - acc_o_tail = T.alloc_fragment([H_per_block, D_tail], accum_dtype) - acc_s = T.alloc_fragment([H_per_block, BI], accum_dtype) - sumexp = T.alloc_fragment([H_per_block], accum_dtype) - sumexp_i = T.alloc_fragment([H_per_block], accum_dtype) - alpha = T.alloc_fragment([H_per_block], accum_dtype) - m_i = T.alloc_fragment([H_per_block], accum_dtype) - m_i_prev = T.alloc_fragment([H_per_block], accum_dtype) - - T.fill(acc_o, 0) - T.fill(acc_o_tail, 0) - T.fill(sumexp, 0) - T.fill(m_i, -(2**30)) - - b_i, g_i = by, bz - # bx encodes (s_i, h_replicate, group_i). - spans_per_seq = REPLICATE_H * n_groups - s_i = bx // spans_per_seq - rest = bx % spans_per_seq - group_i = rest // REPLICATE_H - h_rep = rest % REPLICATE_H - H0 = g_i * padded_H + (0 if REPLICATE_H == 1 else h_rep * 64) - H1 = H0 + H_per_block - - tk_len_1 = Topk_length_1[b_i] - tk_len_2 = Topk_length_2[b_i] - actual_n_groups_1 = T.ceildiv(tk_len_1, BI * inner_iter_1) - actual_n_groups_2 = T.ceildiv(tk_len_2, BI * inner_iter_2) - - if (group_i < n_groups_1) & (group_i < actual_n_groups_1): - # Phase 1 active: SWA cache work + Partial_O write. - T.copy(Q[b_i, s_i, H0:H1, :D], Q_shared) - T.copy(Q[b_i, s_i, H0:H1, D : D + D_tail], Q_tail_shared) - for k_i in T.Pipelined(inner_iter_1, num_stages=num_stages): - iter_i = group_i * inner_iter_1 + k_i - for bi_i in T.Parallel(BI): - pos = iter_i * BI + bi_i - idx = Indices_1[b_i, s_i, pos] - valid = (idx >= 0) & (pos < tk_len_1) - page_idx_shared[bi_i] = T.if_then_else(valid, idx, 0) - mask[bi_i] = valid - - for bi_i, w_i in T.Parallel(BI, PACKED_W4): - page = page_idx_shared[bi_i] - block_id = page // BS_KV_1 - t_in_block = page % BS_KV_1 - K_packed_shared[bi_i, w_i] = K_combined_1[ - block_id, t_in_block * PACKED_W4 + w_i - ] - - for bi_i, w_i in T.Parallel(BI, SCALE_W4): - page = page_idx_shared[bi_i] - block_id = page // BS_KV_1 - t_in_block = page % BS_KV_1 - K_scale_shared[bi_i, w_i] = K_combined_1[ - block_id, - NOPE_ROPE_U32_PER_BLOCK_1 + t_in_block * SCALE_W4 + w_i, - ] - - for bi_i, ti in T.Parallel(BI, NUM_TILES): - word_idx = ti // 4 - byte_in_word = ti % 4 - word = K_scale_shared[bi_i, word_idx] - scale_byte_local[bi_i, ti] = ( - word >> T.Cast("uint32", byte_in_word * 8) - ) & T.uint32(0xFF) - - for bi_i, d_i in T.Parallel(BI, D): - word_idx = d_i // 4 - byte_in_word = d_i % 4 - word = K_packed_shared[bi_i, word_idx] - b_u32 = ( - word >> T.Cast("uint32", byte_in_word * 8) - ) & T.uint32(0xFF) - sign_bf = (b_u32 & T.uint32(0x80)) * T.uint32(0x100) - exp_e4 = (b_u32 & T.uint32(0x78)) >> T.uint32(3) - mant_bf = (b_u32 & T.uint32(0x7)) * T.uint32(0x10) - scale_byte = scale_byte_local[bi_i, d_i // NOPE_TILE] - exp_combined = exp_e4 + scale_byte - T.uint32(7) - bf16_bits = ( - sign_bf | (exp_combined << T.uint32(7)) | mant_bf - ) - KV_shared[bi_i, d_i] = T.reinterpret( - BF16, T.Cast("uint16", bf16_bits) - ) - - for bi_i, j in T.Parallel(BI, D_tail): - abs_off = D + 2 * j - word_idx = abs_off // 4 - word_off = abs_off % 4 - word = K_packed_shared[bi_i, word_idx] - half_u32 = T.if_then_else( - word_off == 0, - word & T.uint32(0xFFFF), - (word >> T.uint32(16)) & T.uint32(0xFFFF), - ) - K_tail_shared[bi_i, j] = T.reinterpret( - BF16, T.Cast("uint16", half_u32) - ) - - for h_i, bi_i in T.Parallel(H_per_block, BI): - acc_s[h_i, bi_i] = T.if_then_else( - mask[bi_i], 0, -T.infinity(acc_s.dtype) - ) - T.gemm( - Q_shared, - KV_shared, - acc_s, - transpose_B=True, - policy=T.GemmWarpPolicy.FullRow, - ) - T.gemm( - Q_tail_shared, - K_tail_shared, - acc_s, - transpose_B=True, - policy=T.GemmWarpPolicy.FullRow, - ) - T.copy(m_i, m_i_prev) - T.reduce_max(acc_s, m_i, dim=1, clear=False) - for h_i in T.Parallel(H_per_block): - m_i[h_i] = T.max(m_i[h_i], m_i_prev[h_i]) - for h_i in T.Parallel(H_per_block): - alpha[h_i] = T.exp2((m_i_prev[h_i] - m_i[h_i]) * sm_scale) - for h_i, bi_i in T.Parallel(H_per_block, BI): - acc_s[h_i, bi_i] = T.exp2( - acc_s[h_i, bi_i] * sm_scale - m_i[h_i] * sm_scale - ) - T.reduce_sum(acc_s, sumexp_i, dim=1) - for h_i in T.Parallel(H_per_block): - sumexp[h_i] = sumexp[h_i] * alpha[h_i] + sumexp_i[h_i] - for h_i, d_i in T.Parallel(H_per_block, D): - acc_o[h_i, d_i] *= alpha[h_i] - for h_i, d_i in T.Parallel(H_per_block, D_tail): - acc_o_tail[h_i, d_i] *= alpha[h_i] - T.copy(acc_s, S_shared) - T.gemm( - S_shared, - KV_shared, - acc_o, - policy=T.GemmWarpPolicy.FullRow, - ) - T.gemm( - S_shared, - K_tail_shared, - acc_o_tail, - policy=T.GemmWarpPolicy.FullRow, - ) - # ---- finalize phase 1 (active) ---- - for h_i, d_i in T.Parallel(H_per_block, D): - acc_o[h_i, d_i] = acc_o[h_i, d_i] / T.if_then_else( - sumexp[h_i] == 0.0, 1.0, sumexp[h_i] - ) - for h_i, d_i in T.Parallel(H_per_block, D_tail): - acc_o_tail[h_i, d_i] = acc_o_tail[h_i, d_i] / T.if_then_else( - sumexp[h_i] == 0.0, 1.0, sumexp[h_i] - ) - for h_i in T.Parallel(H_per_block): - m_i[h_i] = T.if_then_else( - sumexp[h_i] == 0.0, - -(2.0**30), - T.log2(sumexp[h_i]) + m_i[h_i] * sm_scale, - ) - T.copy(acc_o, Partial_O[b_i, s_i, group_i, H0:H1, :D]) - T.copy( - acc_o_tail, - Partial_O[b_i, s_i, group_i, H0:H1, D : D + D_tail], - ) - T.copy(m_i, Partial_LSE[b_i, s_i, group_i, H0:H1]) - elif group_i < n_groups_1: - # Phase 1 skipped: m_i is still the -2^30 - T.copy(m_i, Partial_LSE[b_i, s_i, group_i, H0:H1]) - elif (group_i - n_groups_1) < actual_n_groups_2: - # Phase 2 active: c128 cache work + Partial_O write. - T.copy(Q[b_i, s_i, H0:H1, :D], Q_shared) - T.copy(Q[b_i, s_i, H0:H1, D : D + D_tail], Q_tail_shared) - for k_i in T.Pipelined(inner_iter_2, num_stages=num_stages): - iter_i = (group_i - n_groups_1) * inner_iter_2 + k_i - for bi_i in T.Parallel(BI): - pos = iter_i * BI + bi_i - idx = Indices_2[b_i, s_i, pos] - valid = (idx >= 0) & (pos < tk_len_2) - page_idx_shared[bi_i] = T.if_then_else(valid, idx, 0) - mask[bi_i] = valid - - for bi_i, w_i in T.Parallel(BI, PACKED_W4): - page = page_idx_shared[bi_i] - block_id = page // BS_KV_2 - t_in_block = page % BS_KV_2 - K_packed_shared[bi_i, w_i] = K_combined_2[ - block_id, t_in_block * PACKED_W4 + w_i - ] - - for bi_i, w_i in T.Parallel(BI, SCALE_W4): - page = page_idx_shared[bi_i] - block_id = page // BS_KV_2 - t_in_block = page % BS_KV_2 - K_scale_shared[bi_i, w_i] = K_combined_2[ - block_id, - NOPE_ROPE_U32_PER_BLOCK_2 + t_in_block * SCALE_W4 + w_i, - ] - - for bi_i, ti in T.Parallel(BI, NUM_TILES): - word_idx = ti // 4 - byte_in_word = ti % 4 - word = K_scale_shared[bi_i, word_idx] - scale_byte_local[bi_i, ti] = ( - word >> T.Cast("uint32", byte_in_word * 8) - ) & T.uint32(0xFF) - - for bi_i, d_i in T.Parallel(BI, D): - word_idx = d_i // 4 - byte_in_word = d_i % 4 - word = K_packed_shared[bi_i, word_idx] - b_u32 = ( - word >> T.Cast("uint32", byte_in_word * 8) - ) & T.uint32(0xFF) - sign_bf = (b_u32 & T.uint32(0x80)) * T.uint32(0x100) - exp_e4 = (b_u32 & T.uint32(0x78)) >> T.uint32(3) - mant_bf = (b_u32 & T.uint32(0x7)) * T.uint32(0x10) - scale_byte = scale_byte_local[bi_i, d_i // NOPE_TILE] - exp_combined = exp_e4 + scale_byte - T.uint32(7) - bf16_bits = ( - sign_bf | (exp_combined << T.uint32(7)) | mant_bf - ) - KV_shared[bi_i, d_i] = T.reinterpret( - BF16, T.Cast("uint16", bf16_bits) - ) - - for bi_i, j in T.Parallel(BI, D_tail): - abs_off = D + 2 * j - word_idx = abs_off // 4 - word_off = abs_off % 4 - word = K_packed_shared[bi_i, word_idx] - half_u32 = T.if_then_else( - word_off == 0, - word & T.uint32(0xFFFF), - (word >> T.uint32(16)) & T.uint32(0xFFFF), - ) - K_tail_shared[bi_i, j] = T.reinterpret( - BF16, T.Cast("uint16", half_u32) - ) - - for h_i, bi_i in T.Parallel(H_per_block, BI): - acc_s[h_i, bi_i] = T.if_then_else( - mask[bi_i], 0, -T.infinity(acc_s.dtype) - ) - T.gemm( - Q_shared, - KV_shared, - acc_s, - transpose_B=True, - policy=T.GemmWarpPolicy.FullRow, - ) - T.gemm( - Q_tail_shared, - K_tail_shared, - acc_s, - transpose_B=True, - policy=T.GemmWarpPolicy.FullRow, - ) - T.copy(m_i, m_i_prev) - T.reduce_max(acc_s, m_i, dim=1, clear=False) - for h_i in T.Parallel(H_per_block): - m_i[h_i] = T.max(m_i[h_i], m_i_prev[h_i]) - for h_i in T.Parallel(H_per_block): - alpha[h_i] = T.exp2((m_i_prev[h_i] - m_i[h_i]) * sm_scale) - for h_i, bi_i in T.Parallel(H_per_block, BI): - acc_s[h_i, bi_i] = T.exp2( - acc_s[h_i, bi_i] * sm_scale - m_i[h_i] * sm_scale - ) - T.reduce_sum(acc_s, sumexp_i, dim=1) - for h_i in T.Parallel(H_per_block): - sumexp[h_i] = sumexp[h_i] * alpha[h_i] + sumexp_i[h_i] - for h_i, d_i in T.Parallel(H_per_block, D): - acc_o[h_i, d_i] *= alpha[h_i] - for h_i, d_i in T.Parallel(H_per_block, D_tail): - acc_o_tail[h_i, d_i] *= alpha[h_i] - T.copy(acc_s, S_shared) - T.gemm( - S_shared, - KV_shared, - acc_o, - policy=T.GemmWarpPolicy.FullRow, - ) - T.gemm( - S_shared, - K_tail_shared, - acc_o_tail, - policy=T.GemmWarpPolicy.FullRow, - ) - # ---- finalize phase 2 (active) ---- - for h_i, d_i in T.Parallel(H_per_block, D): - acc_o[h_i, d_i] = acc_o[h_i, d_i] / T.if_then_else( - sumexp[h_i] == 0.0, 1.0, sumexp[h_i] - ) - for h_i, d_i in T.Parallel(H_per_block, D_tail): - acc_o_tail[h_i, d_i] = acc_o_tail[h_i, d_i] / T.if_then_else( - sumexp[h_i] == 0.0, 1.0, sumexp[h_i] - ) - for h_i in T.Parallel(H_per_block): - m_i[h_i] = T.if_then_else( - sumexp[h_i] == 0.0, - -(2.0**30), - T.log2(sumexp[h_i]) + m_i[h_i] * sm_scale, - ) - T.copy(acc_o, Partial_O[b_i, s_i, group_i, H0:H1, :D]) - T.copy( - acc_o_tail, - Partial_O[b_i, s_i, group_i, H0:H1, D : D + D_tail], - ) - T.copy(m_i, Partial_LSE[b_i, s_i, group_i, H0:H1]) - else: - # Phase 2 skipped: m_i is still the -2^30 - T.copy(m_i, Partial_LSE[b_i, s_i, group_i, H0:H1]) - - return main - - @T.prim_func - def main( - Q: T.Tensor(q_shape, BF16), # type: ignore - K_combined_1: T.Tensor(k1_shape, "uint32"), # type: ignore - Indices_1: T.Tensor(indices1_shape, indices_dtype), # type: ignore - Topk_length_1: T.Tensor(topk_length_shape, indices_dtype), # type: ignore - Partial_O: T.Tensor(partial_o_shape, BF16), # type: ignore - Partial_LSE: T.Tensor(partial_lse_shape, accum_dtype), # type: ignore - ) -> None: - """ - grid: (seq_len * REPLICATE_H * n_groups, batch, 1) - Each block processes `inner_iter_1` consecutive KV tiles and writes - one (partial_o, partial_lse) entry. - """ - with T.Kernel( - seq_len * REPLICATE_H * n_groups, batch, kv_group, threads=threads - ) as (bx, by, bz): - Q_shared = T.alloc_fragment([H_per_block, D], BF16) - Q_tail_shared = T.alloc_fragment([H_per_block, D_tail], BF16) - K_packed_shared = T.alloc_shared([BI, PACKED_W4], "uint32") - K_scale_shared = T.alloc_shared([BI, SCALE_W4], "uint32") - KV_shared = T.alloc_shared([BI, D], BF16) - K_tail_shared = T.alloc_shared([BI, D_tail], BF16) - S_shared = T.alloc_shared([H_per_block, BI], BF16) - page_idx_shared = T.alloc_shared([BI], INT32) - - mask = T.alloc_fragment([BI], "bool") - scale_byte_local = T.alloc_fragment([BI, NUM_TILES], "uint32") - - acc_o = T.alloc_fragment([H_per_block, D], accum_dtype) - acc_o_tail = T.alloc_fragment([H_per_block, D_tail], accum_dtype) - acc_s = T.alloc_fragment([H_per_block, BI], accum_dtype) - sumexp = T.alloc_fragment([H_per_block], accum_dtype) - sumexp_i = T.alloc_fragment([H_per_block], accum_dtype) - alpha = T.alloc_fragment([H_per_block], accum_dtype) - m_i = T.alloc_fragment([H_per_block], accum_dtype) - m_i_prev = T.alloc_fragment([H_per_block], accum_dtype) - - T.fill(acc_o, 0) - T.fill(acc_o_tail, 0) - T.fill(sumexp, 0) - T.fill(m_i, -(2**30)) - - b_i, g_i = by, bz - spans_per_seq = REPLICATE_H * n_groups - s_i = bx // spans_per_seq - rest = bx % spans_per_seq - group_i = rest // REPLICATE_H - h_rep = rest % REPLICATE_H - H0 = g_i * padded_H + (0 if REPLICATE_H == 1 else h_rep * 64) - H1 = H0 + H_per_block - - T.copy(Q[b_i, s_i, H0:H1, :D], Q_shared) - T.copy(Q[b_i, s_i, H0:H1, D : D + D_tail], Q_tail_shared) - - tk_len_1 = Topk_length_1[b_i] - - for k_i in T.Pipelined(inner_iter_1, num_stages=num_stages): - iter_i = group_i * inner_iter_1 + k_i - for bi_i in T.Parallel(BI): - pos = iter_i * BI + bi_i - idx = Indices_1[b_i, s_i, pos] - valid = (idx >= 0) & (pos < tk_len_1) - page_idx_shared[bi_i] = T.if_then_else(valid, idx, 0) - mask[bi_i] = valid - - for bi_i, w_i in T.Parallel(BI, PACKED_W4): - page = page_idx_shared[bi_i] - block_id = page // BS_KV_1 - t_in_block = page % BS_KV_1 - K_packed_shared[bi_i, w_i] = K_combined_1[ - block_id, t_in_block * PACKED_W4 + w_i - ] - - for bi_i, w_i in T.Parallel(BI, SCALE_W4): - page = page_idx_shared[bi_i] - block_id = page // BS_KV_1 - t_in_block = page % BS_KV_1 - K_scale_shared[bi_i, w_i] = K_combined_1[ - block_id, - NOPE_ROPE_U32_PER_BLOCK_1 + t_in_block * SCALE_W4 + w_i, - ] - - for bi_i, ti in T.Parallel(BI, NUM_TILES): - word_idx = ti // 4 - byte_in_word = ti % 4 - word = K_scale_shared[bi_i, word_idx] - scale_byte_local[bi_i, ti] = ( - word >> T.Cast("uint32", byte_in_word * 8) - ) & T.uint32(0xFF) - - for bi_i, d_i in T.Parallel(BI, D): - word_idx = d_i // 4 - byte_in_word = d_i % 4 - word = K_packed_shared[bi_i, word_idx] - b_u32 = (word >> T.Cast("uint32", byte_in_word * 8)) & T.uint32( - 0xFF - ) - sign_bf = (b_u32 & T.uint32(0x80)) * T.uint32(0x100) - exp_e4 = (b_u32 & T.uint32(0x78)) >> T.uint32(3) - mant_bf = (b_u32 & T.uint32(0x7)) * T.uint32(0x10) - scale_byte = scale_byte_local[bi_i, d_i // NOPE_TILE] - exp_combined = exp_e4 + scale_byte - T.uint32(7) - bf16_bits = sign_bf | (exp_combined << T.uint32(7)) | mant_bf - KV_shared[bi_i, d_i] = T.reinterpret( - BF16, T.Cast("uint16", bf16_bits) - ) - - for bi_i, j in T.Parallel(BI, D_tail): - abs_off = D + 2 * j - word_idx = abs_off // 4 - word_off = abs_off % 4 - word = K_packed_shared[bi_i, word_idx] - half_u32 = T.if_then_else( - word_off == 0, - word & T.uint32(0xFFFF), - (word >> T.uint32(16)) & T.uint32(0xFFFF), - ) - K_tail_shared[bi_i, j] = T.reinterpret( - BF16, T.Cast("uint16", half_u32) - ) - - for h_i, bi_i in T.Parallel(H_per_block, BI): - acc_s[h_i, bi_i] = T.if_then_else( - mask[bi_i], 0, -T.infinity(acc_s.dtype) - ) - T.gemm( - Q_shared, - KV_shared, - acc_s, - transpose_B=True, - policy=T.GemmWarpPolicy.FullRow, - ) - T.gemm( - Q_tail_shared, - K_tail_shared, - acc_s, - transpose_B=True, - policy=T.GemmWarpPolicy.FullRow, - ) - T.copy(m_i, m_i_prev) - T.reduce_max(acc_s, m_i, dim=1, clear=False) - for h_i in T.Parallel(H_per_block): - m_i[h_i] = T.max(m_i[h_i], m_i_prev[h_i]) - for h_i in T.Parallel(H_per_block): - alpha[h_i] = T.exp2((m_i_prev[h_i] - m_i[h_i]) * sm_scale) - for h_i, bi_i in T.Parallel(H_per_block, BI): - acc_s[h_i, bi_i] = T.exp2( - acc_s[h_i, bi_i] * sm_scale - m_i[h_i] * sm_scale - ) - T.reduce_sum(acc_s, sumexp_i, dim=1) - for h_i in T.Parallel(H_per_block): - sumexp[h_i] = sumexp[h_i] * alpha[h_i] + sumexp_i[h_i] - for h_i, d_i in T.Parallel(H_per_block, D): - acc_o[h_i, d_i] *= alpha[h_i] - for h_i, d_i in T.Parallel(H_per_block, D_tail): - acc_o_tail[h_i, d_i] *= alpha[h_i] - T.copy(acc_s, S_shared) - T.gemm(S_shared, KV_shared, acc_o, policy=T.GemmWarpPolicy.FullRow) - T.gemm( - S_shared, K_tail_shared, acc_o_tail, policy=T.GemmWarpPolicy.FullRow - ) - - for h_i, d_i in T.Parallel(H_per_block, D): - acc_o[h_i, d_i] = acc_o[h_i, d_i] / T.if_then_else( - sumexp[h_i] == 0.0, 1.0, sumexp[h_i] - ) - for h_i, d_i in T.Parallel(H_per_block, D_tail): - acc_o_tail[h_i, d_i] = acc_o_tail[h_i, d_i] / T.if_then_else( - sumexp[h_i] == 0.0, 1.0, sumexp[h_i] - ) - for h_i in T.Parallel(H_per_block): - m_i[h_i] = T.if_then_else( - sumexp[h_i] == 0.0, - -(2.0**30), - T.log2(sumexp[h_i]) + m_i[h_i] * sm_scale, - ) - T.copy(acc_o, Partial_O[b_i, s_i, group_i, H0:H1, :D]) - T.copy(acc_o_tail, Partial_O[b_i, s_i, group_i, H0:H1, D : D + D_tail]) - T.copy(m_i, Partial_LSE[b_i, s_i, group_i, H0:H1]) - - return main - - -@tilelang.jit( - out_idx=[-2, -1], - pass_configs={ - tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True, - tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True, - tilelang.PassConfigKey.TL_ENABLE_FAST_MATH: True, - }, -) -def dpsk_v4_combine_kernel( - num_heads: int, - n_groups_1: int, - n_groups_2: int = 0, - *, - block_I: int = 64, - inner_iter_1: int = 1, - inner_iter_2: int = 1, - dim: int = 448, - tail_dim: int = 64, - head_per_block: int = 16, - threads: int = 256, - use_attn_sink: bool = False, -) -> Any: - """ - Combine `n_groups` flash-attention partials into the final output. - - Inputs: - Partial_O : (batch, seq_len, n_groups, num_heads, dim+tail_dim) bf16 - Partial_LSE : (batch, seq_len, n_groups, num_heads) fp32, log2 form - Topk_length_1: (batch,) int32, actual phase-1 length - Topk_length_2: (batch,) int32, actual phase-2 length (dual only) - Attn_sink : (num_heads,) fp32 - Outputs: - Output : (batch, seq_len, num_heads, dim+tail_dim) bf16 - LSE : (batch, seq_len, num_heads) fp32, natural log - - Each grid block handles `head_per_block` heads of one (batch, seq) row. - """ - log2e: float = 1.44269504 - ln2: float = 0.69314718 - assert num_heads % head_per_block == 0 - - is_dual = n_groups_2 > 0 - n_groups = n_groups_1 + n_groups_2 - - H_per_block = head_per_block - HEAD_BLOCKS = num_heads // H_per_block - DT = dim + tail_dim - - batch = T.symbolic("batch") - seq_len = T.symbolic("seq_len") - - accum_dtype = "float" - - if is_dual: - - @T.prim_func - def main( - Partial_O: T.Tensor([batch, seq_len, n_groups, num_heads, DT], BF16), # type: ignore - Partial_LSE: T.Tensor([batch, seq_len, n_groups, num_heads], accum_dtype), # type: ignore - Topk_length_1: T.Tensor([batch], INT32), # type: ignore - Topk_length_2: T.Tensor([batch], INT32), # type: ignore - Attn_sink: T.Tensor([num_heads], FP32), # type: ignore - Output: T.Tensor([batch, seq_len, num_heads, DT], BF16), # type: ignore - LSE: T.Tensor([batch, seq_len, num_heads], accum_dtype), # type: ignore - ) -> None: - with T.Kernel(seq_len * HEAD_BLOCKS, batch, threads=threads) as ( - bx, - by, - ): - shared_lse = T.alloc_shared([n_groups, H_per_block], accum_dtype) - lse_max = T.alloc_fragment([H_per_block], accum_dtype) - lse_sum = T.alloc_fragment([H_per_block], accum_dtype) - scale = T.alloc_fragment([H_per_block, n_groups], accum_dtype) - acc_o = T.alloc_fragment([H_per_block, DT], accum_dtype) - attn_sink_frag = T.alloc_fragment([H_per_block], accum_dtype) - o_scale_frag = T.alloc_fragment([H_per_block], accum_dtype) - final_lse = T.alloc_fragment([H_per_block], accum_dtype) - - b_i = by - s_i = bx // HEAD_BLOCKS - head_block = bx % HEAD_BLOCKS - H0 = head_block * H_per_block - H1 = H0 + H_per_block - - # Clamp to the captured-shape upper bounds so callers passing - # the INT32_MAX sentinel (= "all valid") still iterate exactly - # n_groups groups, not 33M. - actual_n_groups_1 = T.min( - T.ceildiv(Topk_length_1[b_i], block_I * inner_iter_1), - n_groups_1, - ) - actual_n_groups_2 = T.min( - T.ceildiv(Topk_length_2[b_i], block_I * inner_iter_2), - n_groups - n_groups_1, - ) - actual_n_groups = actual_n_groups_1 + actual_n_groups_2 - - # Pass 1: load only active groups' LSE into compact slots. - for k_c in T.serial(actual_n_groups): - k = T.if_then_else( - k_c < actual_n_groups_1, - k_c, - n_groups_1 + (k_c - actual_n_groups_1), - ) - T.copy(Partial_LSE[b_i, s_i, k, H0:H1], shared_lse[k_c, :]) - - T.fill(lse_max, -(2**30)) - for k_c in T.serial(actual_n_groups): - for h_i in T.Parallel(H_per_block): - lse_max[h_i] = T.max(lse_max[h_i], shared_lse[k_c, h_i]) - T.fill(lse_sum, 0) - for k_c in T.serial(actual_n_groups): - for h_i in T.Parallel(H_per_block): - lse_sum[h_i] = lse_sum[h_i] + T.exp2( - shared_lse[k_c, h_i] - lse_max[h_i] - ) - for k_c in T.serial(actual_n_groups): - for h_i in T.Parallel(H_per_block): - scale[h_i, k_c] = T.exp2( - shared_lse[k_c, h_i] - lse_max[h_i] - T.log2(lse_sum[h_i]) - ) - - T.fill(acc_o, 0) - for k_c in T.serial(actual_n_groups): - k = T.if_then_else( - k_c < actual_n_groups_1, - k_c, - n_groups_1 + (k_c - actual_n_groups_1), - ) - for h_i, d_i in T.Parallel(H_per_block, DT): - acc_o[h_i, d_i] = acc_o[h_i, d_i] + scale[h_i, k_c] * Partial_O[ - b_i, s_i, k, H0 + h_i, d_i - ].astype(accum_dtype) - - for h_i in T.Parallel(H_per_block): - empty = lse_max[h_i] <= -(2**29) - final_lse[h_i] = T.if_then_else( - empty, - T.infinity(accum_dtype), - (lse_max[h_i] + T.log2(lse_sum[h_i])) * ln2, - ) - - if use_attn_sink: - for h_i in T.Parallel(H_per_block): - attn_sink_frag[h_i] = Attn_sink[H0 + h_i] - for h_i in T.Parallel(H_per_block): - empty = lse_max[h_i] <= -(2**29) - o_scale_frag[h_i] = T.if_then_else( - empty, - 0.0, - 1.0 - / ( - 1.0 - + T.exp2((attn_sink_frag[h_i] - final_lse[h_i]) * log2e) - ), - ) - for h_i, d_i in T.Parallel(H_per_block, DT): - acc_o[h_i, d_i] = acc_o[h_i, d_i] * o_scale_frag[h_i] - - T.copy(acc_o, Output[b_i, s_i, H0:H1, :]) - T.copy(final_lse, LSE[b_i, s_i, H0:H1]) - - return main - - @T.prim_func - def main( - Partial_O: T.Tensor([batch, seq_len, n_groups, num_heads, DT], BF16), # type: ignore - Partial_LSE: T.Tensor([batch, seq_len, n_groups, num_heads], accum_dtype), # type: ignore - Attn_sink: T.Tensor([num_heads], FP32), # type: ignore - Output: T.Tensor([batch, seq_len, num_heads, DT], BF16), # type: ignore - LSE: T.Tensor([batch, seq_len, num_heads], accum_dtype), # type: ignore - ) -> None: - with T.Kernel(seq_len * HEAD_BLOCKS, batch, threads=threads) as (bx, by): - shared_lse = T.alloc_shared([n_groups, H_per_block], accum_dtype) - - lse_max = T.alloc_fragment([H_per_block], accum_dtype) - lse_sum = T.alloc_fragment([H_per_block], accum_dtype) - scale = T.alloc_fragment([H_per_block, n_groups], accum_dtype) - acc_o = T.alloc_fragment([H_per_block, DT], accum_dtype) - attn_sink_frag = T.alloc_fragment([H_per_block], accum_dtype) - o_scale_frag = T.alloc_fragment([H_per_block], accum_dtype) - final_lse = T.alloc_fragment([H_per_block], accum_dtype) - - b_i = by - s_i = bx // HEAD_BLOCKS - head_block = bx % HEAD_BLOCKS - H0 = head_block * H_per_block - H1 = H0 + H_per_block - - for k in T.serial(n_groups): - T.copy(Partial_LSE[b_i, s_i, k, H0:H1], shared_lse[k, :]) - - T.fill(lse_max, -(2**30)) - for k in T.serial(n_groups): - for h_i in T.Parallel(H_per_block): - lse_max[h_i] = T.max(lse_max[h_i], shared_lse[k, h_i]) - T.fill(lse_sum, 0) - for k in T.serial(n_groups): - for h_i in T.Parallel(H_per_block): - lse_sum[h_i] = lse_sum[h_i] + T.exp2( - shared_lse[k, h_i] - lse_max[h_i] - ) - for k in T.serial(n_groups): - for h_i in T.Parallel(H_per_block): - scale[h_i, k] = T.exp2( - shared_lse[k, h_i] - lse_max[h_i] - T.log2(lse_sum[h_i]) - ) - - T.fill(acc_o, 0) - for k in T.serial(n_groups): - for h_i, d_i in T.Parallel(H_per_block, DT): - acc_o[h_i, d_i] = acc_o[h_i, d_i] + scale[h_i, k] * Partial_O[ - b_i, s_i, k, H0 + h_i, d_i - ].astype(accum_dtype) - - for h_i in T.Parallel(H_per_block): - empty = lse_max[h_i] <= -(2**29) - final_lse[h_i] = T.if_then_else( - empty, - T.infinity(accum_dtype), - (lse_max[h_i] + T.log2(lse_sum[h_i])) * ln2, - ) - - if use_attn_sink: - for h_i in T.Parallel(H_per_block): - attn_sink_frag[h_i] = Attn_sink[H0 + h_i] - for h_i in T.Parallel(H_per_block): - empty = lse_max[h_i] <= -(2**29) - o_scale_frag[h_i] = T.if_then_else( - empty, - 0.0, - 1.0 - / ( - 1.0 + T.exp2((attn_sink_frag[h_i] - final_lse[h_i]) * log2e) - ), - ) - for h_i, d_i in T.Parallel(H_per_block, DT): - acc_o[h_i, d_i] = acc_o[h_i, d_i] * o_scale_frag[h_i] - - T.copy(acc_o, Output[b_i, s_i, H0:H1, :]) - T.copy(final_lse, LSE[b_i, s_i, H0:H1]) - - return main - - -""" -2-stage attention kernel (partial + combine) over an FP8 KV cache, -with optional second cache (`extra_k_cache`). -""" - - -def dpsk_v4_fp8_attention_fwd( - q: torch.Tensor, - k_cache: torch.Tensor, - block_table: Optional[torch.Tensor], - cache_seqlens: Optional[torch.Tensor], - head_dim_v: int, - tile_scheduler_metadata: Any, - num_splits: None = None, - softmax_scale: Optional[float] = None, - causal: bool = False, - is_fp8_kvcache: bool = False, - indices: Optional[torch.Tensor] = None, - attn_sink: Optional[torch.Tensor] = None, - extra_k_cache: Optional[torch.Tensor] = None, - extra_indices_in_kvcache: Optional[torch.Tensor] = None, - topk_length: Optional[torch.Tensor] = None, - extra_topk_length: Optional[torch.Tensor] = None, -) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Follows the original `flash_mla.flash_mla_with_kvcache` signature. - """ - if _is_gfx95_supported: - block_I, threads, num_stages, block_per_cu, cu = 64, 512, 0, 2, 256 - else: - block_I, threads, num_stages, block_per_cu, cu = 32, 128, 1, 1, 304 - - batch, seq_len, num_heads, _ = q.shape - # Partial grid is (seq_len * REPLICATE_H * n_groups, batch, kv_group); the - # heuristic in _pick_inner_iter assumes `total_blocks = seq * ni / inner_iter`, - # so `seq` must include REPLICATE_H or n_groups doubles for medium batches. - replicate_h = max((num_heads + 63) // 64, 1) - seq = batch * seq_len * replicate_h - - k1, _, bs_kv_1 = _build_fp8_combined_view(k_cache) - topk_1 = indices.shape[-1] - ni_1 = topk_1 // block_I - tk_len_1 = ( - topk_length - if topk_length is not None - else _topk_length_sentinel(q.device, batch) - ) - if attn_sink is None: - attn_sink = torch.full( - (num_heads,), float("-inf"), dtype=torch.float32, device=q.device - ) - - has_extra = extra_k_cache is not None - if not has_extra: - inner_iter_1 = _pick_inner_iter(seq, ni_1, cu, block_per_cu) - inner_iter_2 = 1 - n_groups_1 = ni_1 // inner_iter_1 - n_groups_2 = 0 - partial = dpsk_v4_fp8_partial_kernel( - num_heads, - topk_1, - bs_kv_1, - sm_scale=softmax_scale, - block_I=block_I, - inner_iter_1=inner_iter_1, - num_stages=num_stages, - threads=threads, - ) - partial_o, partial_lse = partial(q, k1, indices, tk_len_1) - else: - k2, _, bs_kv_2 = _build_fp8_combined_view(extra_k_cache) - topk_2 = extra_indices_in_kvcache.shape[-1] - ni_2 = topk_2 // block_I - # Each phase picks its own optimal split-K independently — kernel - # body uses two T.Pipelined loops with separate compile-time iter - # counts, no shared-divisor constraint. - inner_iter_1 = _pick_inner_iter(seq, ni_1, cu, block_per_cu) - inner_iter_2 = _pick_inner_iter(seq, ni_2, cu, block_per_cu) - n_groups_1 = ni_1 // inner_iter_1 - n_groups_2 = ni_2 // inner_iter_2 - tk_len_2 = ( - extra_topk_length - if extra_topk_length is not None - else _topk_length_sentinel(q.device, batch) - ) - partial = dpsk_v4_fp8_partial_kernel( - num_heads, - topk_1, - bs_kv_1, - topk_2, - bs_kv_2, - sm_scale=softmax_scale, - block_I=block_I, - inner_iter_1=inner_iter_1, - inner_iter_2=inner_iter_2, - num_stages=num_stages, - threads=threads, - ) - partial_o, partial_lse = partial( - q, - k1, - indices, - tk_len_1, - k2, - extra_indices_in_kvcache, - tk_len_2, - ) - - combine = dpsk_v4_combine_kernel( - num_heads, - n_groups_1, - n_groups_2, - block_I=block_I, - inner_iter_1=inner_iter_1, - inner_iter_2=inner_iter_2, - head_per_block=4, - threads=256, - use_attn_sink=True, - ) - if has_extra: - return combine(partial_o, partial_lse, tk_len_1, tk_len_2, attn_sink) - return combine(partial_o, partial_lse, attn_sink) diff --git a/python/sglang/kernels/ops/attention/dsa/transform_index.py b/python/sglang/kernels/ops/attention/dsa/transform_index.py index 7cfb4af3f6..81bd8bf282 100644 --- a/python/sglang/kernels/ops/attention/dsa/transform_index.py +++ b/python/sglang/kernels/ops/attention/dsa/transform_index.py @@ -6,14 +6,6 @@ import triton import triton.language as tl -def transform_index_page_table_prefill(**kwargs): - return transform_index_page_table_prefill_fast(**kwargs) - - -def transform_index_page_table_decode(**kwargs): - return transform_index_page_table_decode_fast(**kwargs) - - @triton.jit def transform_index_page_table_dcp_kernel( page_table_ptr, @@ -270,7 +262,7 @@ def transform_index_page_table_prefill_kernel( ) -def transform_index_page_table_decode_fast( +def transform_index_page_table_decode( page_table: torch.Tensor, topk_indices: torch.Tensor, result: Optional[torch.Tensor] = None, @@ -322,7 +314,7 @@ def transform_index_page_table_decode_fast( return result -def transform_index_page_table_prefill_fast( +def transform_index_page_table_prefill( page_table: torch.Tensor, topk_indices: torch.Tensor, extend_lens_cpu: List[int], diff --git a/python/sglang/kernels/ops/attention/dsa/triton_kernel.py b/python/sglang/kernels/ops/attention/dsa/triton_kernel.py index 4ce7a4c1e4..9ac527d597 100644 --- a/python/sglang/kernels/ops/attention/dsa/triton_kernel.py +++ b/python/sglang/kernels/ops/attention/dsa/triton_kernel.py @@ -1,141 +1,8 @@ -from typing import Optional, Tuple - import torch import triton import triton.language as tl -# Triton implementation -@triton.jit -def _act_quant_kernel( - X_ptr, - Y_ptr, - S_ptr, - M, - N, - group_size: tl.constexpr, - round_scale: tl.constexpr, - BLOCK_M: tl.constexpr, - BLOCK_N: tl.constexpr, -): - """ - Triton kernel for activation quantization. - - Each block processes BLOCK_M rows and group_size columns. - """ - # Get block IDs - pid_m = tl.program_id(0) - pid_n = tl.program_id(1) - - # FP8 constants - fp8_min = -448.0 - fp8_max = 448.0 - fp8_max_inv = 1.0 / fp8_max - - # Calculate row and column offsets - row_start = pid_m * BLOCK_M - col_start = pid_n * group_size - - # Create offset arrays - rows = row_start + tl.arange(0, BLOCK_M) - cols = col_start + tl.arange(0, BLOCK_N) - - # Mask for valid rows and columns - row_mask = rows < M - col_mask = cols < N - mask = row_mask[:, None] & col_mask[None, :] - - # Load input data - x_ptrs = X_ptr + rows[:, None] * N + cols[None, :] - x = tl.load(x_ptrs, mask=mask, other=0.0).to(tl.float32) - - # Compute absolute max along columns (group_size dimension) for each row - x_abs = tl.abs(x) - amax = tl.max(x_abs, axis=1) # Shape: (BLOCK_M,) - - # Clamp amax to avoid division by zero - amax = tl.maximum(amax, 1e-4) - - # Compute scale - if round_scale: - # Fast round scale using bit manipulation approximation - # This is a simplified version - the exact bit manipulation is harder in Triton - # Using log2 + ceil + pow2 as approximation - log_val = tl.log2(amax * fp8_max_inv) - log_ceil = tl.ceil(log_val) - scale = tl.exp2(log_ceil) - else: - scale = amax * fp8_max_inv - - # Quantize: y = clamp(x / scale, fp8_min, fp8_max) - scale_broadcast = scale[:, None] - y = x / scale_broadcast - y = tl.minimum(tl.maximum(y, fp8_min), fp8_max) - - # Store quantized output - y_ptrs = Y_ptr + rows[:, None] * N + cols[None, :] - tl.store(y_ptrs, y, mask=mask) - - # Store scales - s_cols = pid_n - s_ptrs = S_ptr + rows * (N // group_size) + s_cols - s_mask = row_mask - tl.store(s_ptrs, scale, mask=s_mask) - - -def act_quant( - x: torch.Tensor, block_size: int = 128, scale_fmt: Optional[str] = None -) -> Tuple[torch.Tensor, torch.Tensor]: - """ - Quantizes the input tensor `x` using block-wise quantization with Triton. - - Args: - x (torch.Tensor): The input tensor to be quantized. Must be contiguous and its last dimension size must be divisible by `block_size`. - block_size (int, optional): The size of the blocks to be used for quantization. Default is 128. - scale_fmt (Optional[str], optional): The format of the scale. Default is None. - Returns: - Tuple[torch.Tensor, torch.Tensor]: A tuple containing: - - The quantized tensor with dtype `torch.float8_e4m3fn`. - - A tensor of scaling factors with dtype `torch.float32`. - """ - assert x.is_contiguous(), "Input tensor must be contiguous" - assert x.size(-1) % block_size == 0, ( - f"Last dimension size must be divisible by block_size (block_size={block_size})" - ) - - # Flatten all dims except last - N = x.size(-1) - x_flat = x.view(-1, N) - M = x_flat.size(0) - - # Allocate output tensors - y = torch.empty_like(x, dtype=torch.float8_e4m3fn) - y_flat = y.view(-1, N) - s = x.new_empty(*x.size()[:-1], N // block_size, dtype=torch.float32) - s_flat = s.view(-1, N // block_size) - - # Launch kernel - BLOCK_M = 32 - BLOCK_N = block_size - grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, block_size)) - round_scale = scale_fmt is not None - - _act_quant_kernel[grid]( - x_flat, - y_flat, - s_flat, - M, - N, - group_size=block_size, - round_scale=round_scale, - BLOCK_M=BLOCK_M, - BLOCK_N=BLOCK_N, - num_stages=0 if round_scale else 2, - ) - - return y, s - - @triton.jit def _get_valid_kv_indices_kernel( page_table_ptr, # [bs, topk] diff --git a/python/sglang/kernels/ops/attention/dsa/triton_sparse_mla.py b/python/sglang/kernels/ops/attention/dsa/triton_sparse_mla.py index 3a41c126f2..221beafbd0 100644 --- a/python/sglang/kernels/ops/attention/dsa/triton_sparse_mla.py +++ b/python/sglang/kernels/ops/attention/dsa/triton_sparse_mla.py @@ -16,7 +16,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.kvcache.cache_ops import q8kv8_topk_length_from_indices +from sglang.kernels.ops.attention.metadata import q8kv8_topk_length_from_indices from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz from sglang.srt.utils import get_device_core_count, is_gfx95_supported diff --git a/python/sglang/kernels/ops/attention/dsv4/__init__.py b/python/sglang/kernels/ops/attention/dsv4/__init__.py index faac4666d1..3b7e3c1da0 100644 --- a/python/sglang/kernels/ops/attention/dsv4/__init__.py +++ b/python/sglang/kernels/ops/attention/dsv4/__init__.py @@ -4,25 +4,21 @@ from .attn import ( fused_store_cache, get_paged_mqa_logits_metadata, - triton_create_paged_compress_data, ) -from .c128_cleanup import clear_unaccepted_c128_draft_states from .compress import ( CompressorDecodePlan, CompressorPrefillPlan, compress_forward, compress_norm_rope_store, ) -from .elementwise import ( +from .norm_rope import ( fused_k_norm_rope_flashmla, fused_norm_rope_inplace, - fused_q_indexer_rope_first_quant, fused_q_indexer_rope_hadamard_fp4_quant, fused_q_indexer_rope_hadamard_quant, fused_q_norm_rope, fused_rope_inplace, ) -from .fp8_wo_a import sglang_per_token_group_quant_fp8_dsv4_wo_a from .topk import ( plan_topk_v2, topk_transform_kpool_ragged_v2, @@ -38,19 +34,15 @@ __all__ = [ "CompressorPrefillPlan", "compress_forward", "compress_norm_rope_store", - "clear_unaccepted_c128_draft_states", "fused_norm_rope_inplace", "fused_store_cache", "fused_rope_inplace", "fused_q_norm_rope", - "fused_q_indexer_rope_first_quant", "fused_q_indexer_rope_hadamard_fp4_quant", "fused_q_indexer_rope_hadamard_quant", "fused_k_norm_rope_flashmla", - "sglang_per_token_group_quant_fp8_dsv4_wo_a", "make_name", "get_paged_mqa_logits_metadata", - "triton_create_paged_compress_data", "topk_transform_paged", "topk_transform_kpool_v2", "topk_transform_kpool_ragged_v2", diff --git a/python/sglang/kernels/ops/attention/dsv4/attn.py b/python/sglang/kernels/ops/attention/dsv4/attn.py index 448e328599..825b4ecb9c 100644 --- a/python/sglang/kernels/ops/attention/dsv4/attn.py +++ b/python/sglang/kernels/ops/attention/dsv4/attn.py @@ -1,8 +1,6 @@ -from typing import Literal, Optional, Tuple, Union +from typing import Literal, Optional, Union import torch -import triton -import triton.language as tl from sglang.kernels.jit.utils import ( cache_once, @@ -121,137 +119,3 @@ def fused_store_cache( if freqs_cis.is_complex(): freqs_cis = torch.view_as_real(freqs_cis).flatten(-2) module.run_rope(input, cache, indices, freqs_cis.contiguous()) - - -@triton.jit -def create_paged_compress_data_kernel( - req_pool_indices_ptr, - seq_lens_ptr, - extend_seq_lens_ptr, - req_to_token_ptr, - full_to_swa_index_mapping_ptr, - out_0_ptr, - out_1_ptr, - batch_size, - stride_req_to_token_0, - stride_req_to_token_1: tl.constexpr, - stride_out_1_0, - stride_out_1_1: tl.constexpr, - compress_ratio: tl.constexpr, - is_overlap: tl.constexpr, - use_req_ring: tl.constexpr, - swa_page_size: tl.constexpr, - ring_size: tl.constexpr, - BLOCK: tl.constexpr, -) -> None: - pid = tl.program_id(0) - offs = pid * BLOCK + tl.arange(0, BLOCK) - mask = offs < batch_size - - rid = tl.load(req_pool_indices_ptr + offs, mask=mask, other=0).to(tl.int32) - seq_len = tl.load(seq_lens_ptr + offs, mask=mask, other=0).to(tl.int32) - extend_len = tl.load(extend_seq_lens_ptr + offs, mask=mask, other=0).to(tl.int32) - prefix_len = seq_len - extend_len - - cr = compress_ratio - write_pos = ((seq_len - 1) // cr) * cr - load_pos = ((prefix_len - 1) // cr) * cr - write_overlap_pos = write_pos - cr - load_overlap_pos = load_pos - cr - v0 = tl.zeros([BLOCK], tl.int32) - v1 = tl.zeros([BLOCK], tl.int32) - v2 = tl.zeros([BLOCK], tl.int32) - v3 = tl.zeros([BLOCK], tl.int32) - - for i in tl.static_range(4): - if i == 0: - pos = load_pos - elif i == 1: - pos = write_pos - elif i == 2: - pos = load_overlap_pos - else: - pos = write_overlap_pos - pos = tl.maximum(pos, 0) - if compress_ratio == 128 or use_req_ring: - state_loc = rid * ring_size + (pos % ring_size) - else: - loc = tl.load( - req_to_token_ptr - + rid.to(tl.int64) * stride_req_to_token_0 - + pos.to(tl.int64) * stride_req_to_token_1, - mask=mask, - other=0, - ).to(tl.int32) - swa_loc = tl.load( - full_to_swa_index_mapping_ptr + loc, mask=mask, other=0 - ).to(tl.int32) - swa_page = swa_loc // swa_page_size - state_loc = swa_page * ring_size + (swa_loc % ring_size) - state_loc = state_loc // cr - if i == 0: - v0 = state_loc - elif i == 1: - v1 = state_loc - elif i == 2: - v2 = state_loc - else: - v3 = state_loc - - tl.store(out_0_ptr + offs, v1, mask=mask) - - if is_overlap: - base = out_1_ptr + offs * stride_out_1_0 - tl.store(base + 0 * stride_out_1_1, v2, mask=mask) - tl.store(base + 1 * stride_out_1_1, v0, mask=mask) - tl.store(base + 2 * stride_out_1_1, v3, mask=mask) - tl.store(base + 3 * stride_out_1_1, write_pos.to(tl.int32), mask=mask) - else: - base = out_1_ptr + offs * stride_out_1_0 - tl.store(base + 0 * stride_out_1_1, v0, mask=mask) - - -def triton_create_paged_compress_data( - *, - compress_ratio: int, - is_overlap: bool, - swa_page_size: int, - ring_size: int, - req_pool_indices: torch.Tensor, - seq_lens: torch.Tensor, - extend_seq_lens: torch.Tensor, - req_to_token: torch.Tensor, - full_to_swa_index_mapping: torch.Tensor, - use_req_ring: bool = False, - block: int = 128, -) -> Tuple[torch.Tensor, torch.Tensor]: - batch_size = req_pool_indices.shape[0] - out_dim = 4 if is_overlap else 1 - device_args: dict = dict(device=req_pool_indices.device, dtype=torch.int32) - out_0 = torch.empty((batch_size,), **device_args) - out_1 = torch.empty((batch_size, out_dim), **device_args) - grid = (triton.cdiv(batch_size, block),) - create_paged_compress_data_kernel[grid]( - req_pool_indices, - seq_lens, - extend_seq_lens, - req_to_token, - full_to_swa_index_mapping, - out_0, - out_1, - batch_size=batch_size, - stride_req_to_token_0=req_to_token.stride(0), - stride_req_to_token_1=req_to_token.stride(1), # type: ignore - stride_out_1_0=out_1.stride(0), - stride_out_1_1=out_1.stride(1), # type: ignore - compress_ratio=compress_ratio, # type: ignore - is_overlap=1 if is_overlap else 0, # type: ignore - use_req_ring=1 if use_req_ring else 0, # type: ignore - swa_page_size=swa_page_size, # type: ignore - ring_size=ring_size, # type: ignore - BLOCK=block, # type: ignore - ) - - if not is_overlap: - out_1.squeeze_(1) - return out_0, out_1 diff --git a/python/sglang/kernels/ops/attention/dsv4_attn_metadata_kernels.py b/python/sglang/kernels/ops/attention/dsv4/attn_metadata_kernels.py similarity index 100% rename from python/sglang/kernels/ops/attention/dsv4_attn_metadata_kernels.py rename to python/sglang/kernels/ops/attention/dsv4/attn_metadata_kernels.py diff --git a/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py b/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py index 7cc9915a62..b4afb2f732 100644 --- a/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py +++ b/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py @@ -11,7 +11,7 @@ from sglang.srt.utils import is_gfx95_supported fp8_dtype = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn -# v4 KV cache layout (see dsv4.index_buf_accessor._set_k_and_s_triton_kernel): +# v4 KV cache layout (see kvcache.dsv4_main_kv_store._set_k_and_s_triton_kernel): # per-token: 448 fp8 nope + 64 bf16 rope (= 576 contiguous bytes) + # 7 ue8m0 scales padded to 8 bytes. # per-page: [token 0..P-1 nope+rope (P*576 bytes)] [token 0..P-1 scale (P*8 bytes)] @@ -24,7 +24,7 @@ NOPE_ROPE_BYTES = DIM_NOPE + DIM_ROPE * 2 # 576 PADDED_SCALE_PER_TOKEN = NUM_SCALE_TILES + 1 # 8 -def dequantize_k_cache_paged( +def dequantize_k_cache_paged_dsv4( quant_k_cache: torch.Tensor, page_table_1_flattened: torch.Tensor, page_size: int, @@ -163,7 +163,7 @@ def dequantize_k_cache_paged_v41( return out -def gather_dequant_requant_fp8_paged( +def gather_dequant_requant_fp8_paged_dsv4( quant_k_cache: torch.Tensor, page_table_1_flattened: torch.Tensor, page_size: int, @@ -229,74 +229,6 @@ def gather_dequant_requant_fp8_paged( return out -def q8kv8_padded_num_heads(num_heads: int) -> int: - """Return a Q-head count supported by the SM90 Q8KV8 kernel.""" - if num_heads <= 0: - raise ValueError(f"num_heads must be positive, got {num_heads}") - if num_heads <= 64: - return 64 - if num_heads <= 128: - return 128 - raise ValueError( - "DeepSeek-V4 Q8KV8 sparse prefill supports at most 128 local " - f"query heads, got {num_heads}" - ) - - -def cast_q_fp8_for_q8kv8_prefill( - q: torch.Tensor, - padded_num_heads: Optional[int] = None, - out: Optional[torch.Tensor] = None, -) -> tuple[torch.Tensor, torch.Tensor]: - """Cast DeepSeek-V4 sparse-prefill Q to the Q8KV8 kernel format. - - The incoming Q is the model-produced BF16/FP16 tensor already shaped as - ``(num_tokens, num_heads, 512)`` after removing the singleton MQA axis. - - The SM90 kernel processes query heads in 64-head blocks. Tensor parallelism - commonly leaves fewer than 64 local heads, so the active heads are copied - into a zero-padded 64/128-head FP8 tensor. - """ - assert q.ndim == 3 - assert q.shape[-1] == DIM_NOPE + DIM_ROPE - - num_tokens, num_heads, head_dim = q.shape - if padded_num_heads is None: - padded_num_heads = q8kv8_padded_num_heads(num_heads) - - if padded_num_heads not in (64, 128) or padded_num_heads < num_heads: - raise ValueError( - f"invalid padded_num_heads={padded_num_heads} for num_heads={num_heads}" - ) - - expected_shape = (num_tokens, padded_num_heads, head_dim) - - if out is None: - q_fp8 = torch.zeros( - expected_shape, - dtype=fp8_dtype, - device=q.device, - ) - else: - if ( - out.shape != expected_shape - or out.dtype != fp8_dtype - or out.device != q.device - ): - raise ValueError( - "Q8KV8 Q output must have shape/dtype/device " - f"{expected_shape}/{fp8_dtype}/{q.device}, got " - f"{tuple(out.shape)}/{out.dtype}/{out.device}" - ) - q_fp8 = out - if padded_num_heads > num_heads: - q_fp8[:, num_heads:].zero_() - - q_fp8[:, :num_heads].copy_(q) - q_scale = torch.ones((), dtype=torch.float32, device=q.device) - return q_fp8, q_scale - - @triton.jit def _dequantize_k_cache_paged_kernel( output_ptr, diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py index 4bcde3084b..5cfe28d295 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer.py @@ -7,8 +7,11 @@ from triton.language.extra import libdevice from sglang.kernels.jit.utils import aligned_new_empty from sglang.kernels.ops.attention.deep_select import get_input_stride_alignment_bytes -from sglang.kernels.ops.attention.dsv4.torch_quant import FP4_AMAX_FLOOR -from sglang.srt.runtime_context import get_platform +from sglang.kernels.ops.quantization.fp4_fake_quant import FP4_AMAX_FLOOR +from sglang.kernels.ops.quantization.fp4_indexer_quant import ( + _ceil_ue8m0_exp, + _fp4_e2m1_code_rne, +) INDEX_HEAD_DIM = 128 # One index-K slot: 64 packed e2m1 bytes and four ue8m0 block exponents. @@ -181,349 +184,6 @@ def finish_flat_indexer_topk( ) -@triton.jit -def _select_group_value(group, v0, v1, v2, v3): - return tl.where( - group == 0, - v0, - tl.where(group == 1, v1, tl.where(group == 2, v2, v3)), - ) - - -@triton.jit -def _ceil_ue8m0_exp(x): - bits = x.to(tl.int32, bitcast=True) - exp = (bits >> 23) & 0xFF - mantissa = bits & 0x7FFFFF - exp += mantissa != 0 - return tl.minimum(tl.maximum(exp, 1), 254) - - -@triton.jit -def _fp4_e2m1_code(x): - ax = tl.minimum(tl.abs(x), 6.0) - idx = (ax > 0.25).to(tl.uint8) - idx += (ax > 0.75).to(tl.uint8) - idx += (ax > 1.25).to(tl.uint8) - idx += (ax > 1.75).to(tl.uint8) - idx += (ax > 2.5).to(tl.uint8) - idx += (ax > 3.5).to(tl.uint8) - idx += (ax > 5.0).to(tl.uint8) - sign = ((x < 0) & (idx != 0)).to(tl.uint8) - return idx | (sign << 3) - - -@triton.jit -def _fp4_e2m1_code_rne(x): - """Round-to-nearest-even e2m1 code, matching the reference rounding.""" - ax = tl.minimum(tl.abs(x), 6.0) - idx = (ax >= 0.25).to(tl.uint8) - idx += (ax >= 0.75).to(tl.uint8) - idx += (ax >= 1.25).to(tl.uint8) - idx += (ax >= 1.75).to(tl.uint8) - idx += (ax >= 2.5).to(tl.uint8) - idx += (ax >= 3.5).to(tl.uint8) - idx += (ax >= 5.0).to(tl.uint8) - # Round-half-to-even: an odd index at an exact boundary drops to the even one. - is_boundary = ( - (ax == 0.25) - | (ax == 0.75) - | (ax == 1.25) - | (ax == 1.75) - | (ax == 2.5) - | (ax == 3.5) - | (ax == 5.0) - ) - idx = tl.where(is_boundary & ((idx & 1) == 1), idx - 1, idx) - sign = ((x < 0) & (idx != 0)).to(tl.uint8) - return idx | (sign << 3) - - -@triton.jit -def quantize_fp4_indexer_row( - values, - v0, - v1, - BLOCK_N: tl.constexpr, - GROUP_N: tl.constexpr, - RNE: tl.constexpr, -): - """One row of quantize_fp4_indexer_tensor: values is the fp32 [BLOCK_N] row, - v0 / v1 its even / odd elements as fp32 [BLOCK_N // 2]. Returns - (packed codes [BLOCK_N // 2], packed ue8m0 exponents), the two stores of the - standalone kernel below.""" - offs = tl.arange(0, BLOCK_N) - abs_values = tl.abs(values) - - amax0 = tl.max(tl.where(offs < GROUP_N, abs_values, 0.0), axis=0) - amax1 = tl.max( - tl.where((GROUP_N <= offs) & (offs < 2 * GROUP_N), abs_values, 0.0), - axis=0, - ) - amax2 = tl.max( - tl.where((2 * GROUP_N <= offs) & (offs < 3 * GROUP_N), abs_values, 0.0), - axis=0, - ) - amax3 = tl.max(tl.where(3 * GROUP_N <= offs, abs_values, 0.0), axis=0) - - sf0 = tl.maximum(amax0 / 6.0, 1.0e-4) - sf1 = tl.maximum(amax1 / 6.0, 1.0e-4) - sf2 = tl.maximum(amax2 / 6.0, 1.0e-4) - sf3 = tl.maximum(amax3 / 6.0, 1.0e-4) - - exp0 = _ceil_ue8m0_exp(sf0) - exp1 = _ceil_ue8m0_exp(sf1) - exp2 = _ceil_ue8m0_exp(sf2) - exp3 = _ceil_ue8m0_exp(sf3) - - packed_sf = exp0 | (exp1 << 8) | (exp2 << 16) | (exp3 << 24) - - pair_offsets = tl.arange(0, BLOCK_N // 2) - offs0 = pair_offsets * 2 - offs1 = offs0 + 1 - group0 = offs0 // GROUP_N - group1 = offs1 // GROUP_N - scale_exp0 = _select_group_value(group0, exp0, exp1, exp2, exp3) - scale_exp1 = _select_group_value(group1, exp0, exp1, exp2, exp3) - scale0 = (scale_exp0 << 23).to(tl.float32, bitcast=True) - scale1 = (scale_exp1 << 23).to(tl.float32, bitcast=True) - - v0 = v0 / scale0 - v1 = v1 / scale1 - if RNE: - code0 = _fp4_e2m1_code_rne(v0) - code1 = _fp4_e2m1_code_rne(v1) - else: - code0 = _fp4_e2m1_code(v0) - code1 = _fp4_e2m1_code(v1) - packed = (code0 & 0x0F) | ((code1 & 0x0F) << 4) - return packed, packed_sf - - -@triton.jit -def _quantize_fp4_indexer_kernel( - x, - x_fp4, - x_sf, - BLOCK_N: tl.constexpr, - GROUP_N: tl.constexpr, - RNE: tl.constexpr, -): - token_id = tl.program_id(0) - offs = tl.arange(0, BLOCK_N) - pair_offsets = tl.arange(0, BLOCK_N // 2) - row = x + token_id * BLOCK_N - values = tl.load(row + offs).to(tl.float32) - v0 = tl.load(row + pair_offsets * 2).to(tl.float32) - v1 = tl.load(row + pair_offsets * 2 + 1).to(tl.float32) - packed, packed_sf = quantize_fp4_indexer_row( - values, v0, v1, BLOCK_N=BLOCK_N, GROUP_N=GROUP_N, RNE=RNE - ) - tl.store(x_sf + token_id, packed_sf) - tl.store(x_fp4 + token_id * (BLOCK_N // 2) + pair_offsets, packed) - - -@triton.jit -def _quantize_fp4_indexer_rows( - x, - x_fp4, - x_sf, - M, - BLOCK_M: tl.constexpr, - BLOCK_N: tl.constexpr, - GROUP_N: tl.constexpr, - RNE: tl.constexpr, -): - tl.static_assert(BLOCK_N == 128 and GROUP_N == 32) - # Each reduction covers one scale group. Keep its values for packing, - # avoiding four masked full-row reductions and a second input load. - group = tl.program_id(0) * BLOCK_M * 4 + tl.arange(0, BLOCK_M * 4) - offs = tl.arange(0, GROUP_N) - values = tl.load( - x + group[:, None].to(tl.int64) * GROUP_N + offs[None, :], - group[:, None] < M * 4, - 0, - ).to(tl.float32) - amax = tl.max(tl.abs(values), axis=1) - exp = _ceil_ue8m0_exp(tl.maximum(amax / 6.0, 1.0e-4)) - scale = (exp << 23).to(tl.float32, bitcast=True) - v0, v1 = tl.split( - tl.reshape(values / scale[:, None], (BLOCK_M * 4, GROUP_N // 2, 2)) - ) - if RNE: - code0 = _fp4_e2m1_code_rne(v0) - code1 = _fp4_e2m1_code_rne(v1) - else: - code0 = _fp4_e2m1_code(v0) - code1 = _fp4_e2m1_code(v1) - packed = (code0 & 0x0F) | ((code1 & 0x0F) << 4) - tl.store( - x_fp4 - + group[:, None].to(tl.int64) * (GROUP_N // 2) - + tl.arange(0, GROUP_N // 2)[None, :], - packed, - group[:, None] < M * 4, - ) - # The four exponents occupy disjoint bytes, so integer sum packs them. - shifts = tl.arange(0, 4) * 8 - packed_sf = tl.sum( - tl.reshape(exp.to(tl.uint32), (BLOCK_M, 4)) << shifts[None, :], axis=1 - ) - token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M) - tl.store(x_sf + token_id, packed_sf.to(tl.int32), token_id < M) - - -@triton.jit -def _store_fp4_index_k_cache_kernel( - k_fp4, - k_sf, - cache, - loc, - page_size: tl.constexpr, - cache_stride: tl.constexpr, - BLOCK: tl.constexpr, -): - token_id = tl.program_id(0) - offsets = tl.arange(0, BLOCK) - cache_loc = tl.load(loc + token_id) - page = cache_loc // page_size - page_offset = cache_loc - page * page_size - - k = tl.load(k_fp4 + token_id * BLOCK + offsets) - tl.store(cache + page * cache_stride + page_offset * BLOCK + offsets, k) - - sf = tl.load(k_sf + token_id) - sf_offsets = tl.arange(0, 4) - sf_bytes = (sf >> (sf_offsets * 8)) & 0xFF - tl.store( - cache + page * cache_stride + page_size * BLOCK + page_offset * 4 + sf_offsets, - sf_bytes, - ) - - -def quantize_fp4_indexer_tensor( - x: torch.Tensor, rne: bool = False -) -> tuple[torch.Tensor, torch.Tensor]: - """Per-32 ue8m0 fp4 quantize. rne=True uses round-to-nearest-even (the dsv41 - reference rounding); the default keeps ``_fp4_e2m1_code``'s thresholds.""" - assert x.shape[-1] == 128 - x = x.contiguous().view(-1, x.shape[-1]) - x_fp4 = torch.empty((x.shape[0], 64), device=x.device, dtype=torch.int8) - x_sf = torch.empty((x.shape[0],), device=x.device, dtype=torch.int32) - if x.shape[0] >= 4096 and get_platform().is_blackwell: - # Independent rows share a CTA to avoid one block per 128 values. - _quantize_fp4_indexer_rows[(triton.cdiv(x.shape[0], 8),)]( - x, - x_fp4, - x_sf, - x.shape[0], - 8, - BLOCK_N=128, - GROUP_N=32, - RNE=rne, - num_warps=4, - ) - elif x.shape[0] > 0: - _quantize_fp4_indexer_kernel[(x.shape[0],)]( - x, - x_fp4, - x_sf, - BLOCK_N=128, - GROUP_N=32, - RNE=rne, - ) - return x_fp4, x_sf - - -def store_fp4_index_k_cache( - input: torch.Tensor, - cache: torch.Tensor, - loc: torch.Tensor, - *, - page_size: int, - rne: bool = False, -) -> None: - assert input.shape[-1] == 128 - k_fp4, k_sf = quantize_fp4_indexer_tensor(input.contiguous(), rne=rne) - n_tokens = input.numel() // input.shape[-1] - assert k_fp4.shape == (n_tokens, 64) - assert k_sf.shape == (n_tokens,) - assert cache.shape[1] == page_size * INDEX_K_SLOT_BYTES - - if n_tokens == 0: - return - _store_fp4_index_k_cache_kernel[(n_tokens,)]( - k_fp4.view(torch.uint8), - k_sf, - cache, - loc, - page_size, - cache.stride(0), - BLOCK=64, - ) - - -@triton.jit -def _gather_fp4_index_k_kernel( - cache, - cache_stride, - slots, - payload, - scales, - n, - page_size: tl.constexpr, - BLOCK_ROWS: tl.constexpr, -): - # int32 view of a page: page_size * 16 payload words, then page_size scale words. - rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS) - valid = rows < n - slot = tl.load(slots + rows, valid, 0).to(tl.int64) - page = slot // page_size - offset = slot - page * page_size - page_base = cache + page * cache_stride - words = tl.arange(0, 16) - payload_words = tl.load( - page_base[:, None] + offset[:, None] * 16 + words[None, :], valid[:, None] - ) - tl.store( - payload + rows[:, None].to(tl.int64) * 16 + words[None, :], - payload_words, - valid[:, None], - ) - tl.store(scales + rows, tl.load(page_base + page_size * 16 + offset, valid), valid) - - -def gather_fp4_index_k( - cache: torch.Tensor, slots: torch.Tensor, *, page_size: int -) -> tuple[torch.Tensor, torch.Tensor]: - """Packed fp4 rows at ``slots`` of the index-K ``cache`` (``[pages, page_size * - 68]`` uint8, ``store_fp4_index_k_cache``'s layout): (payload int8 ``[n, 64]``, - ue8m0 scales packed int32 ``[n]``), the layout ``quantize_fp4_indexer_tensor`` - returns.""" - assert cache.shape[1] == page_size * INDEX_K_SLOT_BYTES and cache.stride(1) == 1 - # The kernel addresses slots as `slots + rows`, so a strided view would be read - # as if its elements were packed. - assert slots.is_contiguous() - words = cache.view(torch.int32) - n = slots.shape[0] - payload = torch.empty((n, 16), device=cache.device, dtype=torch.int32) - scales = torch.empty((n,), device=cache.device, dtype=torch.int32) - if n > 0: - block_rows = 64 - _gather_fp4_index_k_kernel[(triton.cdiv(n, block_rows),)]( - words, - words.stride(0), - slots, - payload, - scales, - n, - page_size, - BLOCK_ROWS=block_rows, - num_warps=4, - ) - return payload.view(torch.int8), scales - - @triton.jit def _index_k_rope_pack_kernel( X, diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py index ac834263bc..21b4deefbd 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py @@ -11,16 +11,17 @@ import triton import triton.language as tl from triton.language.extra import libdevice -from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( - _ceil_ue8m0_exp, - quantize_fp4_indexer_row, - quantize_fp4_indexer_tensor, -) from sglang.kernels.ops.attention.dsv4.fp4_rope_fake_quant import ( FP4_AMAX_FLOOR, rope_tail_fake_quant_fp4_row, ) -from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k +from sglang.kernels.ops.gemm.dsv4_indexer_head_weights_hip import ( + _reduce_scale_bf16_block, +) +from sglang.kernels.ops.quantization.fp4_indexer_quant import ( + _ceil_ue8m0_exp, + quantize_fp4_indexer_row, +) from sglang.srt.environ import envs if TYPE_CHECKING: @@ -811,94 +812,6 @@ def aiter_k_indexer_fp4_cache_write( ) -# FlyDSL split payload / scale layout; bytes equal AITER's fp4 writer up to the sign of zero -@triton.jit -def _store_fp4_index_k_split_kernel( - k_fp4, - k_sf, - payload, - scale, - loc, - page_size: tl.constexpr, - BLOCK: tl.constexpr, - PAGE8: tl.constexpr, -): - token_id = tl.program_id(0) - cache_loc = tl.load(loc + token_id) - page = cache_loc // page_size - page_offset = cache_loc - page * page_size - - # payload [page, 1, 4 chunks, page_size, 16 bytes]: chunk c holds dims [32c, 32c + 32). - offsets = tl.arange(0, BLOCK) - chunk = offsets // 16 - byte = offsets - chunk * 16 - k = tl.load(k_fp4 + token_id * BLOCK + offsets) - sf = tl.load(k_sf + token_id) - sf_offsets = tl.arange(0, 4) - sf_bytes = ((sf >> (sf_offsets * 8)) & 0xFF).to(tl.uint8) - if PAGE8: - # aiter's row-group 8-slot page: payload [t % 4][chunk][t // 4][16 B], scales [chunk][t] - page8 = cache_loc // 8 - t = cache_loc % 8 - tl.store( - payload - + page8 * (8 * BLOCK) - + ((t % 4) * 4 + chunk) * 32 - + (t // 4) * 16 - + byte, - k, - ) - tl.store(scale + page8 * 32 + sf_offsets * 8 + t, sf_bytes) - return - tl.store( - payload - + page * (4 * page_size * 16) - + chunk * (page_size * 16) - + page_offset * 16 - + byte, - k, - ) - # scale [page, 1, 4, page_size]: the slot axis is a 16 x 4 tile transposed (the FlyDSL K ABI) - shuffled = (page_offset % 16) * 4 + page_offset // 16 - tl.store( - scale + page * (4 * page_size) + sf_offsets * page_size + shuffled, sf_bytes - ) - - -def store_fp4_index_k_cache_split( - input: torch.Tensor, - payload: torch.Tensor, - scale: torch.Tensor, - loc: torch.Tensor, - *, - page_size: int, - rne: bool = False, - page8: bool = False, -) -> None: - """Quantize input [n, 128] to fp4 (per-32 ue8m0) and scatter row i to slot - loc[i] of the split FlyDSL K layout (payload [pages, 1, 4, page_size, 16], - scale [pages, 1, 4, page_size] with the slot axis 16 x 4 transposed); with page8 - each page holds eight 8-slot pages of aiter's row-group layout (pack_kv_cache).""" - assert input.shape[-1] == _HEAD_DIM - assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page" - assert payload.shape[1:] == (1, 4, page_size, 16), payload.shape - assert scale.shape[1:] == (1, 4, page_size), scale.shape - k_fp4, k_sf = quantize_fp4_indexer_tensor(input.contiguous(), rne=rne) - n_tokens = k_fp4.shape[0] - if n_tokens == 0: - return - _store_fp4_index_k_split_kernel[(n_tokens,)]( - k_fp4.view(torch.uint8), - k_sf, - payload.view(torch.uint8), - scale, - loc, - page_size, - BLOCK=64, - PAGE8=page8, - ) - - def index_k_page8_views( payload: torch.Tensor, scale: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: @@ -911,36 +824,6 @@ def index_k_page8_views( ) -def read_fp4_index_k_split( - payload: torch.Tensor, - scale: torch.Tensor, - slots: torch.Tensor, - *, - page_size: int, - page8: bool = False, -) -> Tuple[torch.Tensor, torch.Tensor]: - """Inverse of store_fp4_index_k_cache_split: (payload int8 [n, 64], scales - int32 [n] with chunk c's e8m0 byte at bits 8c..8c+7), the layout of - quantize_fp4_indexer_tensor.""" - assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page" - slots = slots.to(torch.int64) - if page8: - kv8, ks8 = index_k_page8_views(payload, scale) - page, t = slots // 8, slots % 8 - kv8 = kv8.view(-1, 4, 4, 2, 16) # [page8, t % 4, chunk, t // 4, 16] - rows = kv8[page, t % 4, :, t // 4, :].reshape(-1, 64).view(torch.int8) - sf = ks8.view(-1, 4, 8)[page, :, t].to(torch.int32) # [n, 4] - packed = sf[:, 0] | (sf[:, 1] << 8) | (sf[:, 2] << 16) | (sf[:, 3] << 24) - return rows, packed - page, off = slots // page_size, slots % page_size - rows = payload.view(torch.uint8)[page, 0, :, off, :] # [n, 4, 16] - rows = rows.reshape(-1, 64).view(torch.int8) - shuffled = (off % 16) * 4 + off // 16 - sf = scale[page, 0, :, shuffled].to(torch.int32) # [n, 4] - packed = sf[:, 0] | (sf[:, 1] << 8) | (sf[:, 2] << 16) | (sf[:, 3] << 24) - return rows, packed - - @triton.jit def _quantize_fp4_query_flydsl_kernel( x, @@ -997,63 +880,6 @@ def pack_fp4_query_flydsl(q: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: return q_fp4, q_scale -@triton.jit -def _reduce_scale_bf16_block( - block, - part_ptr, - out_ptr, - M, - stride_ps, - stride_pm, - stride_om, - scale, - N: tl.constexpr, - SPLIT_K: tl.constexpr, - BLOCK: tl.constexpr, -): - """One BLOCK of _reduce_scale_bf16_kernel (shared with the fused index-Q launch).""" - offs = block * BLOCK + tl.arange(0, BLOCK) - m = offs // N - n = offs % N - mask = m < M - acc = tl.load(part_ptr + m * stride_pm + n, mask=mask, other=0.0) - for s in tl.static_range(1, SPLIT_K): - acc += tl.load( - part_ptr + s * stride_ps + m * stride_pm + n, mask=mask, other=0.0 - ) - # bf16(bf16(sum) * scale): the linear's bf16 output, then the aten multiply rounded to bf16 - w = acc.to(tl.bfloat16).to(tl.float32) * scale - tl.store(out_ptr + m * stride_om + n, w.to(tl.bfloat16), mask=mask) - - -@triton.jit -def _reduce_scale_bf16_kernel( - part_ptr, - out_ptr, - M, - stride_ps, - stride_pm, - stride_om, - scale, - N: tl.constexpr, - SPLIT_K: tl.constexpr, - BLOCK: tl.constexpr, -): - _reduce_scale_bf16_block( - tl.program_id(0), - part_ptr, - out_ptr, - M, - stride_ps, - stride_pm, - stride_om, - scale, - N=N, - SPLIT_K=SPLIT_K, - BLOCK=BLOCK, - ) - - @triton.jit def _fp4_e2m1_code_rne_lean(x): """_fp4_e2m1_code_rne for every fp32 x (checked over all 2**32): within each spacing of the @@ -1347,29 +1173,3 @@ def index_q_rope_pack_weights_flydsl( num_warps=4, ) return q_fp4, q_scale, weights - - -def indexer_head_weights( - x: torch.Tensor, weight: torch.Tensor, scale: float -) -> torch.Tensor: - """bf16(bf16(x @ weight.T) * scale) as a contiguous bf16 [M, N], the layout the - FlyDSL logits kernels take. x bf16 [M, K] and weight bf16 [N, K], with - M <= rocm_gemv_split_k_max_tokens(n=N, k=K, weight_dtype=torch.bfloat16).""" - partials = rocm_router_gemv_split_k(x, weight) - split_k, M, N = partials.shape - out = torch.empty((M, N), dtype=torch.bfloat16, device=x.device) - block = triton.next_power_of_2(M * N) - _reduce_scale_bf16_kernel[(triton.cdiv(M * N, block),)]( - partials, - out, - M, - partials.stride(0), - partials.stride(1), - out.stride(0), - float(scale), - N=N, - SPLIT_K=split_k, - BLOCK=block, - num_warps=4, - ) - return out diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_rope_fake_quant.py b/python/sglang/kernels/ops/attention/dsv4/fp4_rope_fake_quant.py index adc6176e3d..bc103d3dd5 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_rope_fake_quant.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_rope_fake_quant.py @@ -9,7 +9,7 @@ import triton import triton.language as tl from triton.language.extra import libdevice -from sglang.kernels.ops.attention.dsv4.torch_quant import FP4_AMAX_FLOOR +from sglang.kernels.ops.quantization.fp4_fake_quant import FP4_AMAX_FLOOR @triton.jit diff --git a/python/sglang/kernels/ops/attention/dsv4/elementwise.py b/python/sglang/kernels/ops/attention/dsv4/norm_rope.py similarity index 88% rename from python/sglang/kernels/ops/attention/dsv4/elementwise.py rename to python/sglang/kernels/ops/attention/dsv4/norm_rope.py index 8dd7e526dc..24acfae2a8 100644 --- a/python/sglang/kernels/ops/attention/dsv4/elementwise.py +++ b/python/sglang/kernels/ops/attention/dsv4/norm_rope.py @@ -126,21 +126,6 @@ def _jit_main_q_indexer_rope_hadamard_quant_module(dtype: torch.dtype): ) -# V3.2 lays q out as [rope | nope] (V4 is [nope | rope]) -> kRopeFirst=true, and -# drops the Hadamard rotation (kHadamard=false). -@cache_once -def _jit_main_q_indexer_rope_first_quant_module(dtype: torch.dtype): - args = make_cpp_args(dtype, is_arch_support_pdl(), True, False) - return load_jit( - make_name("main_q_indexer_rope_first_quant"), - *args, - cuda_files=["deepseek_v4/main_norm_rope.cuh"], - cuda_wrappers=[ - ("forward", f"FusedQIndexerRopeHadamardQuantKernel<{args}>::forward"), - ], - ) - - @cache_once def _jit_main_q_indexer_rope_hadamard_fp4_quant_module(dtype: torch.dtype): args = make_cpp_args(dtype, is_arch_support_pdl()) @@ -171,7 +156,7 @@ def fused_rope_inplace( inverse: if True, apply inverse rotation (conjugate freqs) """ if _is_hip or _is_xpu: - from sglang.kernels.ops.attention.deepseek_v4_rope import ( + from sglang.kernels.ops.attention.dsv4.rope import ( apply_rotary_emb_triton, ) @@ -283,31 +268,6 @@ def fused_q_indexer_rope_hadamard_quant( return q_fp8, weights_out -def fused_q_indexer_rope_first_quant( - q_input: torch.Tensor, - weight: torch.Tensor, - weight_scale: float, - cos_sin_cache: torch.Tensor, - positions: torch.Tensor, -) -> Tuple[torch.Tensor, torch.Tensor]: - """DeepSeek-V3.2 only. Indexer Q: RoPE on the leading dims + fp8 act-quant. CUDA only.""" - q_fp8 = torch.empty(q_input.shape, dtype=torch.float8_e4m3fn, device=q_input.device) - weights_out = torch.empty( - (*q_input.shape[:-1], 1), dtype=torch.float32, device=q_input.device - ) - module = _jit_main_q_indexer_rope_first_quant_module(q_input.dtype) - module.forward( - q_input, - q_fp8, - weight, - weights_out, - float(weight_scale), - cos_sin_cache, - positions, - ) - return q_fp8, weights_out - - def fused_q_indexer_rope_hadamard_fp4_quant( q_input: torch.Tensor, weight: torch.Tensor, diff --git a/python/sglang/kernels/ops/attention/dsv4/online_c128_mtp.py b/python/sglang/kernels/ops/attention/dsv4/online_c128_mtp.py index 497dcc95e5..c34f6a93b0 100644 --- a/python/sglang/kernels/ops/attention/dsv4/online_c128_mtp.py +++ b/python/sglang/kernels/ops/attention/dsv4/online_c128_mtp.py @@ -1,13 +1,11 @@ from __future__ import annotations -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, List, Optional +from typing import TYPE_CHECKING import torch from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args from sglang.kernels.ops.attention.dsv4.utils import make_name -from sglang.srt.environ import envs if TYPE_CHECKING: from tvm_ffi.module import Module @@ -34,244 +32,84 @@ def _jit_online_c128_mtp_module( ) -@dataclass -class _OnlineC128LayerRuntime: - head_dim: int - main_state: torch.Tensor - state_dtype: torch.dtype - state_slot_offset: int +def online_c128_mtp_write_prefix_states( + kv_score_input: torch.Tensor, + seq_lens: torch.Tensor, + req_pool_indices: torch.Tensor, + req_to_token: torch.Tensor, + ape: torch.Tensor, + state: torch.Tensor, + batch_size: int, + num_verify_tokens: int, + state_slot_stride: int, + *, + head_dim: int, +) -> None: + """Stage the c128 compressor state after each verify token in that token's + draft state slot.""" + _jit_online_c128_mtp_module( + head_dim, seq_lens.dtype, req_pool_indices.dtype, state.dtype + ).write_prefix_states( + kv_score_input, + seq_lens, + req_pool_indices, + req_to_token, + ape, + state, + batch_size, + num_verify_tokens, + state_slot_stride, + ) -@dataclass -class _OnlineC128VerifyContext: - req_pool_indices: torch.Tensor - seq_lens: torch.Tensor +def online_c128_mtp_mark_pending( + seq_lens: torch.Tensor, + req_pool_indices: torch.Tensor, + pending_seq_lens: torch.Tensor, + batch_size: int, + num_req_slots: int, + *, + head_dim: int, + state_dtype: torch.dtype, +) -> None: + """Reset all pending verify lengths, then record ``seq_lens`` for the requests + of this verify batch.""" + _jit_online_c128_mtp_module( + head_dim, seq_lens.dtype, req_pool_indices.dtype, state_dtype + ).mark_pending( + seq_lens, + req_pool_indices, + pending_seq_lens, + batch_size, + num_req_slots, + ) -class OnlineC128MTPController: - def __init__(self, backend: Any): - self.backend = backend - self._verify_ctx: Optional[_OnlineC128VerifyContext] = None - self._layer_runtimes: Optional[List[_OnlineC128LayerRuntime]] = None - - def enabled(self) -> bool: - return ( - envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get() - and envs.SGLANG_EXPERIMENTAL_ONLINE_C128_MTP.get() - and self.backend.mtp_enabled - ) - - def state_slot_offset(self) -> int: - if not self.enabled(): - return 0 - return self.backend.token_to_kv_pool.get_online_c128_mtp_state_slot_offset() - - def begin_verify( - self, - req_pool_indices: torch.Tensor, - seq_lens: torch.Tensor, - ) -> None: - if not self.enabled(): - self.clear() - return - - self._verify_ctx = _OnlineC128VerifyContext( - req_pool_indices=req_pool_indices.detach(), - seq_lens=seq_lens.detach(), - ) - head_dim = self._head_dim() - state_dtype = self._state_dtype() - if head_dim is None or state_dtype is None or self._num_verify_tokens() == 0: - return - token_to_kv_pool = self.backend.token_to_kv_pool - _jit_online_c128_mtp_module( - head_dim, seq_lens.dtype, req_pool_indices.dtype, state_dtype - ).mark_pending( - seq_lens, - req_pool_indices, - token_to_kv_pool.get_online_c128_mtp_pending_seq_lens(), - min(seq_lens.shape[0], req_pool_indices.shape[0]), - token_to_kv_pool.get_online_c128_state_num_req_slots(), - ) - - def clear(self) -> None: - self._verify_ctx = None - - def prepare_forward( - self, - logical_forward_mode, - req_pool_indices: torch.Tensor, - seq_lens: torch.Tensor, - *, - verify_bs: Optional[int] = None, - ) -> int: - if not self.enabled(): - self.clear() - return 0 - if logical_forward_mode is None or logical_forward_mode.is_idle(): - self.clear() - return 0 - - active_req_pool_indices = req_pool_indices - active_seq_lens = seq_lens - if logical_forward_mode.is_target_verify(): - if verify_bs is None: - verify_bs = req_pool_indices.shape[0] - active_req_pool_indices = req_pool_indices[:verify_bs] - active_seq_lens = seq_lens[:verify_bs] - if verify_bs == 0: - self.clear() - return 0 - - self.commit_pending( - req_pool_indices=active_req_pool_indices, - seq_lens=active_seq_lens, - ) - if not logical_forward_mode.is_target_verify(): - return 0 - - self.begin_verify( - req_pool_indices=active_req_pool_indices, - seq_lens=active_seq_lens, - ) - return self.state_slot_offset() - - def write_prefix_states( - self, - layer_id: int, - compressor: Any, - kv_score_input: torch.Tensor, - logical_forward_mode, - ) -> None: - if ( - not self.enabled() - or logical_forward_mode is None - or not logical_forward_mode.is_target_verify() - or compressor.is_in_indexer - or compressor.ratio != 128 - or kv_score_input.numel() == 0 - ): - return - - ctx = self._active_ctx() - num_verify_tokens = self._num_verify_tokens() - if ctx is None or num_verify_tokens == 0: - return - - token_to_kv_pool = self.backend.token_to_kv_pool - head_dim = compressor.head_dim - state_pool = token_to_kv_pool.get_attention_compress_states(layer_id) - state = state_pool.kv_score_buffer.kv_score - total_bs = kv_score_input.numel() // (num_verify_tokens * head_dim * 2) - layer_bs = min(ctx.seq_lens.shape[0], ctx.req_pool_indices.shape[0], total_bs) - if layer_bs <= 0: - return - - _jit_online_c128_mtp_module( - head_dim, ctx.seq_lens.dtype, ctx.req_pool_indices.dtype, state.dtype - ).write_prefix_states( - kv_score_input, - ctx.seq_lens, - ctx.req_pool_indices, - self.backend.req_to_token, - compressor.ape.reshape(128, head_dim), - state, - layer_bs, - num_verify_tokens, - state_pool.online_mtp_state_slot_offset, - ) - - def commit_pending( - self, - req_pool_indices: torch.Tensor, - seq_lens: torch.Tensor, - ) -> None: - if self._verify_ctx is None: - return - if not self.enabled(): - self.clear() - return - if req_pool_indices.numel() == 0 or seq_lens.numel() == 0: - return - - num_verify_tokens = self._num_verify_tokens() - if num_verify_tokens == 0: - self.clear() - return - - backend = self.backend - token_to_kv_pool = backend.token_to_kv_pool - pending_seq_lens = token_to_kv_pool.get_online_c128_mtp_pending_seq_lens() - cur_bs = min(seq_lens.shape[0], req_pool_indices.shape[0]) - - for runtime in self._iter_layer_runtimes(): - _jit_online_c128_mtp_module( - runtime.head_dim, - seq_lens.dtype, - req_pool_indices.dtype, - runtime.state_dtype, - ).commit_pending( - seq_lens, - req_pool_indices, - backend.req_to_token, - pending_seq_lens, - runtime.main_state, - cur_bs, - num_verify_tokens, - runtime.state_slot_offset, - token_to_kv_pool.get_online_c128_state_num_req_slots(), - ) - - self.clear() - - def _num_verify_tokens(self) -> int: - if not self.enabled(): - return 0 - num_verify_tokens = int(self.backend.speculative_num_draft_tokens) - max_draft_tokens = ( - self.backend.token_to_kv_pool.get_online_c128_mtp_max_draft_tokens() - ) - return num_verify_tokens if 0 < num_verify_tokens <= max_draft_tokens else 0 - - def _active_ctx(self) -> Optional[_OnlineC128VerifyContext]: - ctx = self._verify_ctx - if ( - ctx is None - or ctx.seq_lens.numel() == 0 - or ctx.req_pool_indices.numel() == 0 - ): - return None - return ctx - - def _head_dim(self) -> Optional[int]: - for runtime in self._iter_layer_runtimes(): - return runtime.head_dim - return None - - def _state_dtype(self) -> Optional[torch.dtype]: - for runtime in self._iter_layer_runtimes(): - return runtime.state_dtype - return None - - def _iter_layer_runtimes(self): - if self._layer_runtimes is None: - runtimes = [] - token_to_kv_pool = self.backend.token_to_kv_pool - for layer in self.backend.model_runner.model.model.layers: - attn = getattr(layer, "self_attn", None) - compressor = getattr(attn, "compressor", None) - if compressor is None or compressor.ratio != 128: - continue - state_pool = token_to_kv_pool.get_attention_compress_states( - compressor.layer_id - ) - runtimes.append( - _OnlineC128LayerRuntime( - head_dim=compressor.head_dim, - main_state=state_pool.kv_score_buffer.kv_score, - state_dtype=state_pool.kv_score_buffer.kv_score.dtype, - state_slot_offset=state_pool.online_mtp_state_slot_offset, - ) - ) - self._layer_runtimes = runtimes - return iter(self._layer_runtimes) +def online_c128_mtp_commit_pending( + seq_lens: torch.Tensor, + req_pool_indices: torch.Tensor, + req_to_token: torch.Tensor, + pending_seq_lens: torch.Tensor, + state: torch.Tensor, + batch_size: int, + num_verify_tokens: int, + state_slot_stride: int, + num_req_slots: int, + *, + head_dim: int, +) -> None: + """Copy the draft state of each request's accepted prefix into its main c128 + state slot.""" + _jit_online_c128_mtp_module( + head_dim, seq_lens.dtype, req_pool_indices.dtype, state.dtype + ).commit_pending( + seq_lens, + req_pool_indices, + req_to_token, + pending_seq_lens, + state, + batch_size, + num_verify_tokens, + state_slot_stride, + num_req_slots, + ) diff --git a/python/sglang/kernels/ops/attention/deepseek_v4_rope.py b/python/sglang/kernels/ops/attention/dsv4/rope.py similarity index 99% rename from python/sglang/kernels/ops/attention/deepseek_v4_rope.py rename to python/sglang/kernels/ops/attention/dsv4/rope.py index 93d9cb2623..7cc4955be5 100644 --- a/python/sglang/kernels/ops/attention/deepseek_v4_rope.py +++ b/python/sglang/kernels/ops/attention/dsv4/rope.py @@ -519,6 +519,8 @@ def _fused_norm_rope_kernel( ) +# Not bit-exact with the JIT dsv4 fused_norm_rope_inplace (outputs differ by up to +# 1 bf16 ulp), so the two are kept separate. def fused_norm_rope_inplace_triton( kv: torch.Tensor, weight: Optional[torch.Tensor], diff --git a/python/sglang/kernels/ops/attention/dsv4/sparse_prefill_kernels.py b/python/sglang/kernels/ops/attention/dsv4/sparse_prefill_kernels.py index 89e064de85..19813b5ffe 100644 --- a/python/sglang/kernels/ops/attention/dsv4/sparse_prefill_kernels.py +++ b/python/sglang/kernels/ops/attention/dsv4/sparse_prefill_kernels.py @@ -3,9 +3,18 @@ Migrated from ``sglang.srt.layers.attention.dsv4.sparse_prefill_utils`` (RFC #29630, Phase 2.5). """ +from typing import Optional + +import torch import triton import triton.language as tl +from sglang.srt.utils import ceil_align + +# FlashMLA sparse prefill asserts ``params.topk % B_TOPK == 0``. B_TOPK is 64 +# for the h_q=64 kernel and 128 for h_q=128; pad to 128 to satisfy both. +SPARSE_PREFILL_TOPK_ALIGNMENT = 128 + @triton.jit def _build_swa_token_ids_kernel( @@ -139,3 +148,181 @@ def _combine_topk_swa_indices_kernel( ) tl.store(combined_lens_ptr + token, topk_len + swa_len, mask=valid) + + +def combined_topk_width(topk: int, window_size: int) -> int: + """Width of the padded combined_indices last dim that + ``combine_topk_swa_indices`` would produce for these args.""" + return ceil_align(topk + window_size, SPARSE_PREFILL_TOPK_ALIGNMENT) + + +def combine_topk_swa_indices( + topk_indices: torch.Tensor, + query_start_loc: torch.Tensor, + query_pos: torch.Tensor, + seq_lens: torch.Tensor, + gather_lens: torch.Tensor, + compressed_base: torch.Tensor, + swa_base: torch.Tensor, + window_size: int, + compress_ratio: int, + topk: int, + out_indices: Optional[torch.Tensor] = None, + out_lens: Optional[torch.Tensor] = None, + swa_indices: Optional[torch.Tensor] = None, + swa_lengths: Optional[torch.Tensor] = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Combine top-k and SWA indices for flash_mla_sparse_fwd. + + Top-k indices are int32 [num_tokens, K] in each request's compressed region. + Invalid entries within topk_len must be -1; later entries are ignored. + query_start_loc can include a cross-chunk offset; query_pos is absolute. + compressed_base and swa_base address the flat workspace. SWA-only layers use + topk == 0, but compress_ratio must still be positive. + Optional SWA indices are relative to swa_base and preserve replay floors. + + Returns int32 indices [num_tokens, padded_topk_swa] and per-token scanned-prefix + lengths, including -1 entries skipped by attention. Width is padded to 128. + Rows are written whole and -1 padded, so preallocated out_indices / out_lens + are overwritten entirely; rows past the last request are -1 with length 0. + """ + assert topk_indices.dtype == torch.int32 + assert query_start_loc.dtype == torch.int32 + assert query_pos.dtype == torch.int32 + assert seq_lens.dtype == torch.int32 + assert gather_lens.dtype == torch.int32 + assert compressed_base.dtype == torch.int32 + assert swa_base.dtype == torch.int32 + assert compress_ratio >= 1, "compress_ratio must be >= 1 (use topk=0 for SWA-only)" + assert topk_indices.shape[-1] >= topk, ( + f"topk_indices width {topk_indices.shape[-1]} must be >= topk {topk}" + ) + + num_tokens = topk_indices.shape[0] + if swa_indices is not None: + assert swa_indices.shape == (num_tokens, window_size) + assert swa_lengths is not None and swa_lengths.shape == (num_tokens,) + assert swa_indices.dtype == swa_lengths.dtype == torch.int32 + num_reqs = seq_lens.shape[0] + combined_topk = combined_topk_width(topk, window_size) + if out_indices is None: + # The kernel writes whole -1 padded rows. + combined_indices = torch.empty( + (num_tokens, combined_topk), + dtype=torch.int32, + device=topk_indices.device, + ) + else: + assert out_indices.shape == (num_tokens, combined_topk) + assert out_indices.dtype == torch.int32 + combined_indices = out_indices + if out_lens is None: + combined_lens = torch.empty( + num_tokens, dtype=torch.int32, device=topk_indices.device + ) + else: + assert out_lens.shape == (num_tokens,) + assert out_lens.dtype == torch.int32 + combined_lens = out_lens + + if num_tokens == 0: + return combined_indices, combined_lens + block_t = 2 # Measured on B200 with num_warps=2, 16384 tokens, bs 1 to 64. + _combine_topk_swa_indices_kernel[(triton.cdiv(num_tokens, block_t),)]( + combined_indices, + combined_indices.stride(0), + combined_lens, + topk_indices, + topk_indices.stride(0), + query_start_loc, + query_pos, + seq_lens, + gather_lens, + compressed_base, + swa_base, + swa_indices, + 0 if swa_indices is None else swa_indices.stride(0), + swa_lengths, + num_tokens, + num_reqs, + top_k=topk, + EXPLICIT_SWA=swa_indices is not None, + COMPRESS_RATIO=compress_ratio, + WINDOW_SIZE=window_size, + WIDTH=combined_topk, + PADDED_WIDTH=triton.next_power_of_2(combined_topk), + BLOCK_T=block_t, + SEARCH_STEPS=max(num_reqs, 1).bit_length(), + num_warps=2, + ) + return combined_indices, combined_lens + + +def build_swa_token_ids( + seq_lens: torch.Tensor, + extend_seq_lens: torch.Tensor, + req_pool_indices: torch.Tensor, + req_to_token: torch.Tensor, + full_to_swa: torch.Tensor, + swa_window: int, + total_swa: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Build a flat list of physical SWA-cache token IDs covering each + request's positional union of every query's SWA window. + + Per request, the union spans seq positions + ``[max(0, seq_len - extend - W + 1), seq_len)``, of length + ``min(seq_len, extend + W - 1)``. Each position is translated through + ``req_to_token`` (full kv-cache id) and then ``full_to_swa`` (SWA + cache id) to land in the SWA-cache token-id space that + ``dequantize_k_cache_paged_dsv4`` consumes. + + Args: + seq_lens: (num_reqs,) int32, per-request total sequence length. + extend_seq_lens: (num_reqs,) int32, per-request query length. + req_pool_indices: (num_reqs,) int32, per-request row in + ``req_to_token``. + req_to_token: (num_reqs_max, max_seq_len) int32. Full kv-cache id + per (request, seq position). + full_to_swa: (full_pool_size + extra,) int64. Maps full kv id to + SWA-cache id. + swa_window: int. SWA window size. + total_swa: Number of token IDs to allocate, computed from CPU lengths. + + Returns: + swa_token_ids: (total_swa,) int32, flat physical SWA-cache token IDs. + swa_first_pos: (num_reqs,) int32, first seq position covered per req. + swa_gather_lens: (num_reqs,) int32, gather length per request. + swa_offsets: (num_reqs+1,) int32, exclusive cumsum of swa_gather_lens. + """ + assert seq_lens.dtype == torch.int32 + assert extend_seq_lens.dtype == torch.int32 + assert req_pool_indices.dtype == torch.int32 + assert req_to_token.dtype == torch.int32 + assert full_to_swa.dtype == torch.int64 + + num_reqs = seq_lens.shape[0] + device = seq_lens.device + + swa_gather_lens = torch.minimum(seq_lens, extend_seq_lens + (swa_window - 1)).to( + torch.int32 + ) + swa_first_pos = (seq_lens - swa_gather_lens).to(torch.int32) + swa_offsets = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device) + swa_offsets[1:] = torch.cumsum(swa_gather_lens, dim=0).to(torch.int32) + swa_token_ids = torch.empty(total_swa, dtype=torch.int32, device=device) + if total_swa == 0: + return swa_token_ids, swa_first_pos, swa_gather_lens, swa_offsets + + NUM_WORKERS = 128 + _build_swa_token_ids_kernel[(num_reqs, NUM_WORKERS)]( + swa_token_ids, + swa_first_pos, + swa_gather_lens, + swa_offsets, + req_pool_indices, + req_to_token, + req_to_token.stride(0), + full_to_swa, + ) + return swa_token_ids, swa_first_pos, swa_gather_lens, swa_offsets diff --git a/python/sglang/kernels/ops/attention/dsv4/tilelang_fp8_attention.py b/python/sglang/kernels/ops/attention/dsv4/tilelang_fp8_attention.py new file mode 100644 index 0000000000..46f074b2db --- /dev/null +++ b/python/sglang/kernels/ops/attention/dsv4/tilelang_fp8_attention.py @@ -0,0 +1,1064 @@ +"""TileLang FP8 sparse attention for DeepSeek-V4 (``flash_mla_with_kvcache`` +signature, MODEL1_FP8Sparse KV cache).""" + +from typing import Any, Optional, Tuple + +import tilelang +import tilelang.language as T +import torch + +# Importing tilelang_kernel also applies its out_idx adapter patch, which the +# single/dual partial kernel below relies on. +from sglang.kernels.ops.attention.dsa.tilelang_kernel import ( + BF16, + FP32, + INT32, + _is_gfx95_supported, + _pick_inner_iter, +) + + +def _build_fp8_combined_view(k_cache: torch.Tensor) -> Tuple[torch.Tensor, int, int]: + """ + Reinterpret a MODEL1_FP8Sparse KV cache as a contiguous uint32 view. + Input: k_cache (num_blocks, block_size, 1, d_qk) fp8/uint8 + — per-block storage also holds scales + padding past d_qk. + Output: (num_blocks, block_pad_u32) uint32 covering the full block + stride. Same storage ashe input, no copy. + """ + k_u8 = k_cache.view(torch.uint8) if k_cache.dtype != torch.uint8 else k_cache + num_blocks = k_u8.shape[0] + block_size = k_u8.shape[1] + block_pad_u32 = k_u8.stride(0) // 4 + storage = k_u8.untyped_storage() + flat_u32 = torch.empty(0, dtype=torch.uint32, device=k_u8.device).set_( + storage, 0, (storage.nbytes() // 4,), (1,) + ) + k_combined = torch.as_strided( + flat_u32, + size=(num_blocks, block_pad_u32), + stride=(block_pad_u32, 1), + storage_offset=k_u8.storage_offset() // 4, + ) + return k_combined, num_blocks, block_size + + +_TOPK_LEN_SENTINEL_CACHE: dict = {} +_INT32_MAX = 2**30 + + +def _topk_length_sentinel(device: torch.device, batch: int) -> torch.Tensor: + """Cached `(batch,) int32 INT_MAX` tensor used when `topk_length` is None.""" + cur = _TOPK_LEN_SENTINEL_CACHE.get(device) + if cur is None or cur.numel() < batch: + cur = torch.full( + (max(batch, 256),), _INT32_MAX, dtype=torch.int32, device=device + ) + _TOPK_LEN_SENTINEL_CACHE[device] = cur + return cur[:batch] + + +@tilelang.jit( + out_idx=[-2, -1], + pass_configs={ + tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True, + tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True, + tilelang.PassConfigKey.TL_ENABLE_FAST_MATH: True, + }, +) +def dpsk_v4_fp8_partial_kernel( + num_heads: int, + topk_1: int, + block_size_kv_1: int, + topk_2: int = 0, + block_size_kv_2: int = 0, + *, + dim: int = 448, + tail_dim: int = 64, + sm_scale: float = 0.0, + block_I: int = 64, + inner_iter_1: int = 1, + inner_iter_2: int = 0, + num_stages: int = 0, + threads: int = 512, +) -> Any: + """ + Read FP8 K cache directly, dequantise to BF16 in-kernel, do flash-attn + online softmax with split-K. Supports a second cache (`topk_2>0`) and + `attn_sink` is folded later by the combine kernel. + """ + log2e: float = 1.44269504 + if sm_scale <= 0.0: + sm_scale = (1.0 / (dim + tail_dim)) ** 0.5 * log2e + else: + sm_scale = sm_scale * log2e + assert dim == 448 and tail_dim == 64 + assert topk_1 % block_I == 0 + assert (topk_1 // block_I) % inner_iter_1 == 0, ( + f"NI_1={topk_1 // block_I} must be divisible by inner_iter_1={inner_iter_1}" + ) + assert block_size_kv_1 > 0 and (block_size_kv_1 & (block_size_kv_1 - 1)) == 0 + + is_dual = topk_2 > 0 + if is_dual: + assert inner_iter_2 > 0, "dual-cache call requires inner_iter_2 > 0" + assert topk_2 % block_I == 0 + assert (topk_2 // block_I) % inner_iter_2 == 0, ( + f"NI_2={topk_2 // block_I} must be divisible by inner_iter_2={inner_iter_2}" + ) + assert block_size_kv_2 > 0 and (block_size_kv_2 & (block_size_kv_2 - 1)) == 0 + + PACKED_W = dim + 2 * tail_dim + NOPE_TILE = 64 + NUM_TILES = dim // NOPE_TILE + SCALE_W = 8 + PACKED_W4 = PACKED_W // 4 + SCALE_W4 = SCALE_W // 4 + + kv_group = 1 + batch = T.symbolic("batch") + seq_len = T.symbolic("seq_len") + num_blocks_kv_1 = T.symbolic("num_blocks_kv_1") + block_pad_u32_1 = T.symbolic("block_pad_u32_1") + if is_dual: + num_blocks_kv_2 = T.symbolic("num_blocks_kv_2") + block_pad_u32_2 = T.symbolic("block_pad_u32_2") + + head_kv = num_heads // kv_group + D = dim + D_tail = tail_dim + BI = block_I + padded_H = max(tilelang.math.next_power_of_2(head_kv), 16) + if head_kv > 64: + assert head_kv % 64 == 0 + REPLICATE_H = (head_kv + 63) // 64 if head_kv > 64 else 1 + H_per_block = 64 if REPLICATE_H > 1 else padded_H + + NI_1 = topk_1 // BI + n_groups_1 = NI_1 // inner_iter_1 + NI_2 = (topk_2 // BI) if is_dual else 0 + n_groups_2 = (NI_2 // inner_iter_2) if is_dual else 0 + n_groups = n_groups_1 + n_groups_2 + + BS_KV_1 = block_size_kv_1 + NOPE_ROPE_U32_PER_BLOCK_1 = BS_KV_1 * PACKED_W4 + if is_dual: + BS_KV_2 = block_size_kv_2 + NOPE_ROPE_U32_PER_BLOCK_2 = BS_KV_2 * PACKED_W4 + + q_shape = [batch, seq_len, num_heads, D + D_tail] + k1_shape = [num_blocks_kv_1, block_pad_u32_1] + indices1_shape = [batch, seq_len, topk_1] + topk_length_shape = [batch] + partial_o_shape = [batch, seq_len, n_groups, num_heads, D + D_tail] + partial_lse_shape = [batch, seq_len, n_groups, num_heads] + if is_dual: + k2_shape = [num_blocks_kv_2, block_pad_u32_2] + indices2_shape = [batch, seq_len, topk_2] + + accum_dtype = "float" + indices_dtype = INT32 + + if is_dual: + + @T.prim_func + def main( + Q: T.Tensor(q_shape, BF16), # type: ignore + K_combined_1: T.Tensor(k1_shape, "uint32"), # type: ignore + Indices_1: T.Tensor(indices1_shape, indices_dtype), # type: ignore + Topk_length_1: T.Tensor(topk_length_shape, indices_dtype), # type: ignore + K_combined_2: T.Tensor(k2_shape, "uint32"), # type: ignore + Indices_2: T.Tensor(indices2_shape, indices_dtype), # type: ignore + Topk_length_2: T.Tensor(topk_length_shape, indices_dtype), # type: ignore + Partial_O: T.Tensor(partial_o_shape, BF16), # type: ignore + Partial_LSE: T.Tensor(partial_lse_shape, accum_dtype), # type: ignore + ) -> None: + """ + grid: (seq_len * REPLICATE_H * n_groups, batch, 1) + Each block processes `inner_iter_1` (or `inner_iter_2`) consecutive + KV tiles of one phase and writes one (partial_o, partial_lse) entry. + """ + with T.Kernel( + seq_len * REPLICATE_H * n_groups, batch, kv_group, threads=threads + ) as (bx, by, bz): + Q_shared = T.alloc_fragment([H_per_block, D], BF16) + Q_tail_shared = T.alloc_fragment([H_per_block, D_tail], BF16) + K_packed_shared = T.alloc_shared([BI, PACKED_W4], "uint32") + K_scale_shared = T.alloc_shared([BI, SCALE_W4], "uint32") + KV_shared = T.alloc_shared([BI, D], BF16) + K_tail_shared = T.alloc_shared([BI, D_tail], BF16) + S_shared = T.alloc_shared([H_per_block, BI], BF16) + page_idx_shared = T.alloc_shared([BI], INT32) + + mask = T.alloc_fragment([BI], "bool") + scale_byte_local = T.alloc_fragment([BI, NUM_TILES], "uint32") + + acc_o = T.alloc_fragment([H_per_block, D], accum_dtype) + acc_o_tail = T.alloc_fragment([H_per_block, D_tail], accum_dtype) + acc_s = T.alloc_fragment([H_per_block, BI], accum_dtype) + sumexp = T.alloc_fragment([H_per_block], accum_dtype) + sumexp_i = T.alloc_fragment([H_per_block], accum_dtype) + alpha = T.alloc_fragment([H_per_block], accum_dtype) + m_i = T.alloc_fragment([H_per_block], accum_dtype) + m_i_prev = T.alloc_fragment([H_per_block], accum_dtype) + + T.fill(acc_o, 0) + T.fill(acc_o_tail, 0) + T.fill(sumexp, 0) + T.fill(m_i, -(2**30)) + + b_i, g_i = by, bz + # bx encodes (s_i, h_replicate, group_i). + spans_per_seq = REPLICATE_H * n_groups + s_i = bx // spans_per_seq + rest = bx % spans_per_seq + group_i = rest // REPLICATE_H + h_rep = rest % REPLICATE_H + H0 = g_i * padded_H + (0 if REPLICATE_H == 1 else h_rep * 64) + H1 = H0 + H_per_block + + tk_len_1 = Topk_length_1[b_i] + tk_len_2 = Topk_length_2[b_i] + actual_n_groups_1 = T.ceildiv(tk_len_1, BI * inner_iter_1) + actual_n_groups_2 = T.ceildiv(tk_len_2, BI * inner_iter_2) + + if (group_i < n_groups_1) & (group_i < actual_n_groups_1): + # Phase 1 active: SWA cache work + Partial_O write. + T.copy(Q[b_i, s_i, H0:H1, :D], Q_shared) + T.copy(Q[b_i, s_i, H0:H1, D : D + D_tail], Q_tail_shared) + for k_i in T.Pipelined(inner_iter_1, num_stages=num_stages): + iter_i = group_i * inner_iter_1 + k_i + for bi_i in T.Parallel(BI): + pos = iter_i * BI + bi_i + idx = Indices_1[b_i, s_i, pos] + valid = (idx >= 0) & (pos < tk_len_1) + page_idx_shared[bi_i] = T.if_then_else(valid, idx, 0) + mask[bi_i] = valid + + for bi_i, w_i in T.Parallel(BI, PACKED_W4): + page = page_idx_shared[bi_i] + block_id = page // BS_KV_1 + t_in_block = page % BS_KV_1 + K_packed_shared[bi_i, w_i] = K_combined_1[ + block_id, t_in_block * PACKED_W4 + w_i + ] + + for bi_i, w_i in T.Parallel(BI, SCALE_W4): + page = page_idx_shared[bi_i] + block_id = page // BS_KV_1 + t_in_block = page % BS_KV_1 + K_scale_shared[bi_i, w_i] = K_combined_1[ + block_id, + NOPE_ROPE_U32_PER_BLOCK_1 + t_in_block * SCALE_W4 + w_i, + ] + + for bi_i, ti in T.Parallel(BI, NUM_TILES): + word_idx = ti // 4 + byte_in_word = ti % 4 + word = K_scale_shared[bi_i, word_idx] + scale_byte_local[bi_i, ti] = ( + word >> T.Cast("uint32", byte_in_word * 8) + ) & T.uint32(0xFF) + + for bi_i, d_i in T.Parallel(BI, D): + word_idx = d_i // 4 + byte_in_word = d_i % 4 + word = K_packed_shared[bi_i, word_idx] + b_u32 = ( + word >> T.Cast("uint32", byte_in_word * 8) + ) & T.uint32(0xFF) + sign_bf = (b_u32 & T.uint32(0x80)) * T.uint32(0x100) + exp_e4 = (b_u32 & T.uint32(0x78)) >> T.uint32(3) + mant_bf = (b_u32 & T.uint32(0x7)) * T.uint32(0x10) + scale_byte = scale_byte_local[bi_i, d_i // NOPE_TILE] + exp_combined = exp_e4 + scale_byte - T.uint32(7) + bf16_bits = ( + sign_bf | (exp_combined << T.uint32(7)) | mant_bf + ) + KV_shared[bi_i, d_i] = T.reinterpret( + BF16, T.Cast("uint16", bf16_bits) + ) + + for bi_i, j in T.Parallel(BI, D_tail): + abs_off = D + 2 * j + word_idx = abs_off // 4 + word_off = abs_off % 4 + word = K_packed_shared[bi_i, word_idx] + half_u32 = T.if_then_else( + word_off == 0, + word & T.uint32(0xFFFF), + (word >> T.uint32(16)) & T.uint32(0xFFFF), + ) + K_tail_shared[bi_i, j] = T.reinterpret( + BF16, T.Cast("uint16", half_u32) + ) + + for h_i, bi_i in T.Parallel(H_per_block, BI): + acc_s[h_i, bi_i] = T.if_then_else( + mask[bi_i], 0, -T.infinity(acc_s.dtype) + ) + T.gemm( + Q_shared, + KV_shared, + acc_s, + transpose_B=True, + policy=T.GemmWarpPolicy.FullRow, + ) + T.gemm( + Q_tail_shared, + K_tail_shared, + acc_s, + transpose_B=True, + policy=T.GemmWarpPolicy.FullRow, + ) + T.copy(m_i, m_i_prev) + T.reduce_max(acc_s, m_i, dim=1, clear=False) + for h_i in T.Parallel(H_per_block): + m_i[h_i] = T.max(m_i[h_i], m_i_prev[h_i]) + for h_i in T.Parallel(H_per_block): + alpha[h_i] = T.exp2((m_i_prev[h_i] - m_i[h_i]) * sm_scale) + for h_i, bi_i in T.Parallel(H_per_block, BI): + acc_s[h_i, bi_i] = T.exp2( + acc_s[h_i, bi_i] * sm_scale - m_i[h_i] * sm_scale + ) + T.reduce_sum(acc_s, sumexp_i, dim=1) + for h_i in T.Parallel(H_per_block): + sumexp[h_i] = sumexp[h_i] * alpha[h_i] + sumexp_i[h_i] + for h_i, d_i in T.Parallel(H_per_block, D): + acc_o[h_i, d_i] *= alpha[h_i] + for h_i, d_i in T.Parallel(H_per_block, D_tail): + acc_o_tail[h_i, d_i] *= alpha[h_i] + T.copy(acc_s, S_shared) + T.gemm( + S_shared, + KV_shared, + acc_o, + policy=T.GemmWarpPolicy.FullRow, + ) + T.gemm( + S_shared, + K_tail_shared, + acc_o_tail, + policy=T.GemmWarpPolicy.FullRow, + ) + # ---- finalize phase 1 (active) ---- + for h_i, d_i in T.Parallel(H_per_block, D): + acc_o[h_i, d_i] = acc_o[h_i, d_i] / T.if_then_else( + sumexp[h_i] == 0.0, 1.0, sumexp[h_i] + ) + for h_i, d_i in T.Parallel(H_per_block, D_tail): + acc_o_tail[h_i, d_i] = acc_o_tail[h_i, d_i] / T.if_then_else( + sumexp[h_i] == 0.0, 1.0, sumexp[h_i] + ) + for h_i in T.Parallel(H_per_block): + m_i[h_i] = T.if_then_else( + sumexp[h_i] == 0.0, + -(2.0**30), + T.log2(sumexp[h_i]) + m_i[h_i] * sm_scale, + ) + T.copy(acc_o, Partial_O[b_i, s_i, group_i, H0:H1, :D]) + T.copy( + acc_o_tail, + Partial_O[b_i, s_i, group_i, H0:H1, D : D + D_tail], + ) + T.copy(m_i, Partial_LSE[b_i, s_i, group_i, H0:H1]) + elif group_i < n_groups_1: + # Phase 1 skipped: m_i is still the -2^30 + T.copy(m_i, Partial_LSE[b_i, s_i, group_i, H0:H1]) + elif (group_i - n_groups_1) < actual_n_groups_2: + # Phase 2 active: c128 cache work + Partial_O write. + T.copy(Q[b_i, s_i, H0:H1, :D], Q_shared) + T.copy(Q[b_i, s_i, H0:H1, D : D + D_tail], Q_tail_shared) + for k_i in T.Pipelined(inner_iter_2, num_stages=num_stages): + iter_i = (group_i - n_groups_1) * inner_iter_2 + k_i + for bi_i in T.Parallel(BI): + pos = iter_i * BI + bi_i + idx = Indices_2[b_i, s_i, pos] + valid = (idx >= 0) & (pos < tk_len_2) + page_idx_shared[bi_i] = T.if_then_else(valid, idx, 0) + mask[bi_i] = valid + + for bi_i, w_i in T.Parallel(BI, PACKED_W4): + page = page_idx_shared[bi_i] + block_id = page // BS_KV_2 + t_in_block = page % BS_KV_2 + K_packed_shared[bi_i, w_i] = K_combined_2[ + block_id, t_in_block * PACKED_W4 + w_i + ] + + for bi_i, w_i in T.Parallel(BI, SCALE_W4): + page = page_idx_shared[bi_i] + block_id = page // BS_KV_2 + t_in_block = page % BS_KV_2 + K_scale_shared[bi_i, w_i] = K_combined_2[ + block_id, + NOPE_ROPE_U32_PER_BLOCK_2 + t_in_block * SCALE_W4 + w_i, + ] + + for bi_i, ti in T.Parallel(BI, NUM_TILES): + word_idx = ti // 4 + byte_in_word = ti % 4 + word = K_scale_shared[bi_i, word_idx] + scale_byte_local[bi_i, ti] = ( + word >> T.Cast("uint32", byte_in_word * 8) + ) & T.uint32(0xFF) + + for bi_i, d_i in T.Parallel(BI, D): + word_idx = d_i // 4 + byte_in_word = d_i % 4 + word = K_packed_shared[bi_i, word_idx] + b_u32 = ( + word >> T.Cast("uint32", byte_in_word * 8) + ) & T.uint32(0xFF) + sign_bf = (b_u32 & T.uint32(0x80)) * T.uint32(0x100) + exp_e4 = (b_u32 & T.uint32(0x78)) >> T.uint32(3) + mant_bf = (b_u32 & T.uint32(0x7)) * T.uint32(0x10) + scale_byte = scale_byte_local[bi_i, d_i // NOPE_TILE] + exp_combined = exp_e4 + scale_byte - T.uint32(7) + bf16_bits = ( + sign_bf | (exp_combined << T.uint32(7)) | mant_bf + ) + KV_shared[bi_i, d_i] = T.reinterpret( + BF16, T.Cast("uint16", bf16_bits) + ) + + for bi_i, j in T.Parallel(BI, D_tail): + abs_off = D + 2 * j + word_idx = abs_off // 4 + word_off = abs_off % 4 + word = K_packed_shared[bi_i, word_idx] + half_u32 = T.if_then_else( + word_off == 0, + word & T.uint32(0xFFFF), + (word >> T.uint32(16)) & T.uint32(0xFFFF), + ) + K_tail_shared[bi_i, j] = T.reinterpret( + BF16, T.Cast("uint16", half_u32) + ) + + for h_i, bi_i in T.Parallel(H_per_block, BI): + acc_s[h_i, bi_i] = T.if_then_else( + mask[bi_i], 0, -T.infinity(acc_s.dtype) + ) + T.gemm( + Q_shared, + KV_shared, + acc_s, + transpose_B=True, + policy=T.GemmWarpPolicy.FullRow, + ) + T.gemm( + Q_tail_shared, + K_tail_shared, + acc_s, + transpose_B=True, + policy=T.GemmWarpPolicy.FullRow, + ) + T.copy(m_i, m_i_prev) + T.reduce_max(acc_s, m_i, dim=1, clear=False) + for h_i in T.Parallel(H_per_block): + m_i[h_i] = T.max(m_i[h_i], m_i_prev[h_i]) + for h_i in T.Parallel(H_per_block): + alpha[h_i] = T.exp2((m_i_prev[h_i] - m_i[h_i]) * sm_scale) + for h_i, bi_i in T.Parallel(H_per_block, BI): + acc_s[h_i, bi_i] = T.exp2( + acc_s[h_i, bi_i] * sm_scale - m_i[h_i] * sm_scale + ) + T.reduce_sum(acc_s, sumexp_i, dim=1) + for h_i in T.Parallel(H_per_block): + sumexp[h_i] = sumexp[h_i] * alpha[h_i] + sumexp_i[h_i] + for h_i, d_i in T.Parallel(H_per_block, D): + acc_o[h_i, d_i] *= alpha[h_i] + for h_i, d_i in T.Parallel(H_per_block, D_tail): + acc_o_tail[h_i, d_i] *= alpha[h_i] + T.copy(acc_s, S_shared) + T.gemm( + S_shared, + KV_shared, + acc_o, + policy=T.GemmWarpPolicy.FullRow, + ) + T.gemm( + S_shared, + K_tail_shared, + acc_o_tail, + policy=T.GemmWarpPolicy.FullRow, + ) + # ---- finalize phase 2 (active) ---- + for h_i, d_i in T.Parallel(H_per_block, D): + acc_o[h_i, d_i] = acc_o[h_i, d_i] / T.if_then_else( + sumexp[h_i] == 0.0, 1.0, sumexp[h_i] + ) + for h_i, d_i in T.Parallel(H_per_block, D_tail): + acc_o_tail[h_i, d_i] = acc_o_tail[h_i, d_i] / T.if_then_else( + sumexp[h_i] == 0.0, 1.0, sumexp[h_i] + ) + for h_i in T.Parallel(H_per_block): + m_i[h_i] = T.if_then_else( + sumexp[h_i] == 0.0, + -(2.0**30), + T.log2(sumexp[h_i]) + m_i[h_i] * sm_scale, + ) + T.copy(acc_o, Partial_O[b_i, s_i, group_i, H0:H1, :D]) + T.copy( + acc_o_tail, + Partial_O[b_i, s_i, group_i, H0:H1, D : D + D_tail], + ) + T.copy(m_i, Partial_LSE[b_i, s_i, group_i, H0:H1]) + else: + # Phase 2 skipped: m_i is still the -2^30 + T.copy(m_i, Partial_LSE[b_i, s_i, group_i, H0:H1]) + + return main + + @T.prim_func + def main( + Q: T.Tensor(q_shape, BF16), # type: ignore + K_combined_1: T.Tensor(k1_shape, "uint32"), # type: ignore + Indices_1: T.Tensor(indices1_shape, indices_dtype), # type: ignore + Topk_length_1: T.Tensor(topk_length_shape, indices_dtype), # type: ignore + Partial_O: T.Tensor(partial_o_shape, BF16), # type: ignore + Partial_LSE: T.Tensor(partial_lse_shape, accum_dtype), # type: ignore + ) -> None: + """ + grid: (seq_len * REPLICATE_H * n_groups, batch, 1) + Each block processes `inner_iter_1` consecutive KV tiles and writes + one (partial_o, partial_lse) entry. + """ + with T.Kernel( + seq_len * REPLICATE_H * n_groups, batch, kv_group, threads=threads + ) as (bx, by, bz): + Q_shared = T.alloc_fragment([H_per_block, D], BF16) + Q_tail_shared = T.alloc_fragment([H_per_block, D_tail], BF16) + K_packed_shared = T.alloc_shared([BI, PACKED_W4], "uint32") + K_scale_shared = T.alloc_shared([BI, SCALE_W4], "uint32") + KV_shared = T.alloc_shared([BI, D], BF16) + K_tail_shared = T.alloc_shared([BI, D_tail], BF16) + S_shared = T.alloc_shared([H_per_block, BI], BF16) + page_idx_shared = T.alloc_shared([BI], INT32) + + mask = T.alloc_fragment([BI], "bool") + scale_byte_local = T.alloc_fragment([BI, NUM_TILES], "uint32") + + acc_o = T.alloc_fragment([H_per_block, D], accum_dtype) + acc_o_tail = T.alloc_fragment([H_per_block, D_tail], accum_dtype) + acc_s = T.alloc_fragment([H_per_block, BI], accum_dtype) + sumexp = T.alloc_fragment([H_per_block], accum_dtype) + sumexp_i = T.alloc_fragment([H_per_block], accum_dtype) + alpha = T.alloc_fragment([H_per_block], accum_dtype) + m_i = T.alloc_fragment([H_per_block], accum_dtype) + m_i_prev = T.alloc_fragment([H_per_block], accum_dtype) + + T.fill(acc_o, 0) + T.fill(acc_o_tail, 0) + T.fill(sumexp, 0) + T.fill(m_i, -(2**30)) + + b_i, g_i = by, bz + spans_per_seq = REPLICATE_H * n_groups + s_i = bx // spans_per_seq + rest = bx % spans_per_seq + group_i = rest // REPLICATE_H + h_rep = rest % REPLICATE_H + H0 = g_i * padded_H + (0 if REPLICATE_H == 1 else h_rep * 64) + H1 = H0 + H_per_block + + T.copy(Q[b_i, s_i, H0:H1, :D], Q_shared) + T.copy(Q[b_i, s_i, H0:H1, D : D + D_tail], Q_tail_shared) + + tk_len_1 = Topk_length_1[b_i] + + for k_i in T.Pipelined(inner_iter_1, num_stages=num_stages): + iter_i = group_i * inner_iter_1 + k_i + for bi_i in T.Parallel(BI): + pos = iter_i * BI + bi_i + idx = Indices_1[b_i, s_i, pos] + valid = (idx >= 0) & (pos < tk_len_1) + page_idx_shared[bi_i] = T.if_then_else(valid, idx, 0) + mask[bi_i] = valid + + for bi_i, w_i in T.Parallel(BI, PACKED_W4): + page = page_idx_shared[bi_i] + block_id = page // BS_KV_1 + t_in_block = page % BS_KV_1 + K_packed_shared[bi_i, w_i] = K_combined_1[ + block_id, t_in_block * PACKED_W4 + w_i + ] + + for bi_i, w_i in T.Parallel(BI, SCALE_W4): + page = page_idx_shared[bi_i] + block_id = page // BS_KV_1 + t_in_block = page % BS_KV_1 + K_scale_shared[bi_i, w_i] = K_combined_1[ + block_id, + NOPE_ROPE_U32_PER_BLOCK_1 + t_in_block * SCALE_W4 + w_i, + ] + + for bi_i, ti in T.Parallel(BI, NUM_TILES): + word_idx = ti // 4 + byte_in_word = ti % 4 + word = K_scale_shared[bi_i, word_idx] + scale_byte_local[bi_i, ti] = ( + word >> T.Cast("uint32", byte_in_word * 8) + ) & T.uint32(0xFF) + + for bi_i, d_i in T.Parallel(BI, D): + word_idx = d_i // 4 + byte_in_word = d_i % 4 + word = K_packed_shared[bi_i, word_idx] + b_u32 = (word >> T.Cast("uint32", byte_in_word * 8)) & T.uint32( + 0xFF + ) + sign_bf = (b_u32 & T.uint32(0x80)) * T.uint32(0x100) + exp_e4 = (b_u32 & T.uint32(0x78)) >> T.uint32(3) + mant_bf = (b_u32 & T.uint32(0x7)) * T.uint32(0x10) + scale_byte = scale_byte_local[bi_i, d_i // NOPE_TILE] + exp_combined = exp_e4 + scale_byte - T.uint32(7) + bf16_bits = sign_bf | (exp_combined << T.uint32(7)) | mant_bf + KV_shared[bi_i, d_i] = T.reinterpret( + BF16, T.Cast("uint16", bf16_bits) + ) + + for bi_i, j in T.Parallel(BI, D_tail): + abs_off = D + 2 * j + word_idx = abs_off // 4 + word_off = abs_off % 4 + word = K_packed_shared[bi_i, word_idx] + half_u32 = T.if_then_else( + word_off == 0, + word & T.uint32(0xFFFF), + (word >> T.uint32(16)) & T.uint32(0xFFFF), + ) + K_tail_shared[bi_i, j] = T.reinterpret( + BF16, T.Cast("uint16", half_u32) + ) + + for h_i, bi_i in T.Parallel(H_per_block, BI): + acc_s[h_i, bi_i] = T.if_then_else( + mask[bi_i], 0, -T.infinity(acc_s.dtype) + ) + T.gemm( + Q_shared, + KV_shared, + acc_s, + transpose_B=True, + policy=T.GemmWarpPolicy.FullRow, + ) + T.gemm( + Q_tail_shared, + K_tail_shared, + acc_s, + transpose_B=True, + policy=T.GemmWarpPolicy.FullRow, + ) + T.copy(m_i, m_i_prev) + T.reduce_max(acc_s, m_i, dim=1, clear=False) + for h_i in T.Parallel(H_per_block): + m_i[h_i] = T.max(m_i[h_i], m_i_prev[h_i]) + for h_i in T.Parallel(H_per_block): + alpha[h_i] = T.exp2((m_i_prev[h_i] - m_i[h_i]) * sm_scale) + for h_i, bi_i in T.Parallel(H_per_block, BI): + acc_s[h_i, bi_i] = T.exp2( + acc_s[h_i, bi_i] * sm_scale - m_i[h_i] * sm_scale + ) + T.reduce_sum(acc_s, sumexp_i, dim=1) + for h_i in T.Parallel(H_per_block): + sumexp[h_i] = sumexp[h_i] * alpha[h_i] + sumexp_i[h_i] + for h_i, d_i in T.Parallel(H_per_block, D): + acc_o[h_i, d_i] *= alpha[h_i] + for h_i, d_i in T.Parallel(H_per_block, D_tail): + acc_o_tail[h_i, d_i] *= alpha[h_i] + T.copy(acc_s, S_shared) + T.gemm(S_shared, KV_shared, acc_o, policy=T.GemmWarpPolicy.FullRow) + T.gemm( + S_shared, K_tail_shared, acc_o_tail, policy=T.GemmWarpPolicy.FullRow + ) + + for h_i, d_i in T.Parallel(H_per_block, D): + acc_o[h_i, d_i] = acc_o[h_i, d_i] / T.if_then_else( + sumexp[h_i] == 0.0, 1.0, sumexp[h_i] + ) + for h_i, d_i in T.Parallel(H_per_block, D_tail): + acc_o_tail[h_i, d_i] = acc_o_tail[h_i, d_i] / T.if_then_else( + sumexp[h_i] == 0.0, 1.0, sumexp[h_i] + ) + for h_i in T.Parallel(H_per_block): + m_i[h_i] = T.if_then_else( + sumexp[h_i] == 0.0, + -(2.0**30), + T.log2(sumexp[h_i]) + m_i[h_i] * sm_scale, + ) + T.copy(acc_o, Partial_O[b_i, s_i, group_i, H0:H1, :D]) + T.copy(acc_o_tail, Partial_O[b_i, s_i, group_i, H0:H1, D : D + D_tail]) + T.copy(m_i, Partial_LSE[b_i, s_i, group_i, H0:H1]) + + return main + + +@tilelang.jit( + out_idx=[-2, -1], + pass_configs={ + tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True, + tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True, + tilelang.PassConfigKey.TL_ENABLE_FAST_MATH: True, + }, +) +def dpsk_v4_combine_kernel( + num_heads: int, + n_groups_1: int, + n_groups_2: int = 0, + *, + block_I: int = 64, + inner_iter_1: int = 1, + inner_iter_2: int = 1, + dim: int = 448, + tail_dim: int = 64, + head_per_block: int = 16, + threads: int = 256, + use_attn_sink: bool = False, +) -> Any: + """ + Combine `n_groups` flash-attention partials into the final output. + + Inputs: + Partial_O : (batch, seq_len, n_groups, num_heads, dim+tail_dim) bf16 + Partial_LSE : (batch, seq_len, n_groups, num_heads) fp32, log2 form + Topk_length_1: (batch,) int32, actual phase-1 length + Topk_length_2: (batch,) int32, actual phase-2 length (dual only) + Attn_sink : (num_heads,) fp32 + Outputs: + Output : (batch, seq_len, num_heads, dim+tail_dim) bf16 + LSE : (batch, seq_len, num_heads) fp32, natural log + + Each grid block handles `head_per_block` heads of one (batch, seq) row. + """ + log2e: float = 1.44269504 + ln2: float = 0.69314718 + assert num_heads % head_per_block == 0 + + is_dual = n_groups_2 > 0 + n_groups = n_groups_1 + n_groups_2 + + H_per_block = head_per_block + HEAD_BLOCKS = num_heads // H_per_block + DT = dim + tail_dim + + batch = T.symbolic("batch") + seq_len = T.symbolic("seq_len") + + accum_dtype = "float" + + if is_dual: + + @T.prim_func + def main( + Partial_O: T.Tensor([batch, seq_len, n_groups, num_heads, DT], BF16), # type: ignore + Partial_LSE: T.Tensor([batch, seq_len, n_groups, num_heads], accum_dtype), # type: ignore + Topk_length_1: T.Tensor([batch], INT32), # type: ignore + Topk_length_2: T.Tensor([batch], INT32), # type: ignore + Attn_sink: T.Tensor([num_heads], FP32), # type: ignore + Output: T.Tensor([batch, seq_len, num_heads, DT], BF16), # type: ignore + LSE: T.Tensor([batch, seq_len, num_heads], accum_dtype), # type: ignore + ) -> None: + with T.Kernel(seq_len * HEAD_BLOCKS, batch, threads=threads) as ( + bx, + by, + ): + shared_lse = T.alloc_shared([n_groups, H_per_block], accum_dtype) + lse_max = T.alloc_fragment([H_per_block], accum_dtype) + lse_sum = T.alloc_fragment([H_per_block], accum_dtype) + scale = T.alloc_fragment([H_per_block, n_groups], accum_dtype) + acc_o = T.alloc_fragment([H_per_block, DT], accum_dtype) + attn_sink_frag = T.alloc_fragment([H_per_block], accum_dtype) + o_scale_frag = T.alloc_fragment([H_per_block], accum_dtype) + final_lse = T.alloc_fragment([H_per_block], accum_dtype) + + b_i = by + s_i = bx // HEAD_BLOCKS + head_block = bx % HEAD_BLOCKS + H0 = head_block * H_per_block + H1 = H0 + H_per_block + + # Clamp to the captured-shape upper bounds so callers passing + # the INT32_MAX sentinel (= "all valid") still iterate exactly + # n_groups groups, not 33M. + actual_n_groups_1 = T.min( + T.ceildiv(Topk_length_1[b_i], block_I * inner_iter_1), + n_groups_1, + ) + actual_n_groups_2 = T.min( + T.ceildiv(Topk_length_2[b_i], block_I * inner_iter_2), + n_groups - n_groups_1, + ) + actual_n_groups = actual_n_groups_1 + actual_n_groups_2 + + # Pass 1: load only active groups' LSE into compact slots. + for k_c in T.serial(actual_n_groups): + k = T.if_then_else( + k_c < actual_n_groups_1, + k_c, + n_groups_1 + (k_c - actual_n_groups_1), + ) + T.copy(Partial_LSE[b_i, s_i, k, H0:H1], shared_lse[k_c, :]) + + T.fill(lse_max, -(2**30)) + for k_c in T.serial(actual_n_groups): + for h_i in T.Parallel(H_per_block): + lse_max[h_i] = T.max(lse_max[h_i], shared_lse[k_c, h_i]) + T.fill(lse_sum, 0) + for k_c in T.serial(actual_n_groups): + for h_i in T.Parallel(H_per_block): + lse_sum[h_i] = lse_sum[h_i] + T.exp2( + shared_lse[k_c, h_i] - lse_max[h_i] + ) + for k_c in T.serial(actual_n_groups): + for h_i in T.Parallel(H_per_block): + scale[h_i, k_c] = T.exp2( + shared_lse[k_c, h_i] - lse_max[h_i] - T.log2(lse_sum[h_i]) + ) + + T.fill(acc_o, 0) + for k_c in T.serial(actual_n_groups): + k = T.if_then_else( + k_c < actual_n_groups_1, + k_c, + n_groups_1 + (k_c - actual_n_groups_1), + ) + for h_i, d_i in T.Parallel(H_per_block, DT): + acc_o[h_i, d_i] = acc_o[h_i, d_i] + scale[h_i, k_c] * Partial_O[ + b_i, s_i, k, H0 + h_i, d_i + ].astype(accum_dtype) + + for h_i in T.Parallel(H_per_block): + empty = lse_max[h_i] <= -(2**29) + final_lse[h_i] = T.if_then_else( + empty, + T.infinity(accum_dtype), + (lse_max[h_i] + T.log2(lse_sum[h_i])) * ln2, + ) + + if use_attn_sink: + for h_i in T.Parallel(H_per_block): + attn_sink_frag[h_i] = Attn_sink[H0 + h_i] + for h_i in T.Parallel(H_per_block): + empty = lse_max[h_i] <= -(2**29) + o_scale_frag[h_i] = T.if_then_else( + empty, + 0.0, + 1.0 + / ( + 1.0 + + T.exp2((attn_sink_frag[h_i] - final_lse[h_i]) * log2e) + ), + ) + for h_i, d_i in T.Parallel(H_per_block, DT): + acc_o[h_i, d_i] = acc_o[h_i, d_i] * o_scale_frag[h_i] + + T.copy(acc_o, Output[b_i, s_i, H0:H1, :]) + T.copy(final_lse, LSE[b_i, s_i, H0:H1]) + + return main + + @T.prim_func + def main( + Partial_O: T.Tensor([batch, seq_len, n_groups, num_heads, DT], BF16), # type: ignore + Partial_LSE: T.Tensor([batch, seq_len, n_groups, num_heads], accum_dtype), # type: ignore + Attn_sink: T.Tensor([num_heads], FP32), # type: ignore + Output: T.Tensor([batch, seq_len, num_heads, DT], BF16), # type: ignore + LSE: T.Tensor([batch, seq_len, num_heads], accum_dtype), # type: ignore + ) -> None: + with T.Kernel(seq_len * HEAD_BLOCKS, batch, threads=threads) as (bx, by): + shared_lse = T.alloc_shared([n_groups, H_per_block], accum_dtype) + + lse_max = T.alloc_fragment([H_per_block], accum_dtype) + lse_sum = T.alloc_fragment([H_per_block], accum_dtype) + scale = T.alloc_fragment([H_per_block, n_groups], accum_dtype) + acc_o = T.alloc_fragment([H_per_block, DT], accum_dtype) + attn_sink_frag = T.alloc_fragment([H_per_block], accum_dtype) + o_scale_frag = T.alloc_fragment([H_per_block], accum_dtype) + final_lse = T.alloc_fragment([H_per_block], accum_dtype) + + b_i = by + s_i = bx // HEAD_BLOCKS + head_block = bx % HEAD_BLOCKS + H0 = head_block * H_per_block + H1 = H0 + H_per_block + + for k in T.serial(n_groups): + T.copy(Partial_LSE[b_i, s_i, k, H0:H1], shared_lse[k, :]) + + T.fill(lse_max, -(2**30)) + for k in T.serial(n_groups): + for h_i in T.Parallel(H_per_block): + lse_max[h_i] = T.max(lse_max[h_i], shared_lse[k, h_i]) + T.fill(lse_sum, 0) + for k in T.serial(n_groups): + for h_i in T.Parallel(H_per_block): + lse_sum[h_i] = lse_sum[h_i] + T.exp2( + shared_lse[k, h_i] - lse_max[h_i] + ) + for k in T.serial(n_groups): + for h_i in T.Parallel(H_per_block): + scale[h_i, k] = T.exp2( + shared_lse[k, h_i] - lse_max[h_i] - T.log2(lse_sum[h_i]) + ) + + T.fill(acc_o, 0) + for k in T.serial(n_groups): + for h_i, d_i in T.Parallel(H_per_block, DT): + acc_o[h_i, d_i] = acc_o[h_i, d_i] + scale[h_i, k] * Partial_O[ + b_i, s_i, k, H0 + h_i, d_i + ].astype(accum_dtype) + + for h_i in T.Parallel(H_per_block): + empty = lse_max[h_i] <= -(2**29) + final_lse[h_i] = T.if_then_else( + empty, + T.infinity(accum_dtype), + (lse_max[h_i] + T.log2(lse_sum[h_i])) * ln2, + ) + + if use_attn_sink: + for h_i in T.Parallel(H_per_block): + attn_sink_frag[h_i] = Attn_sink[H0 + h_i] + for h_i in T.Parallel(H_per_block): + empty = lse_max[h_i] <= -(2**29) + o_scale_frag[h_i] = T.if_then_else( + empty, + 0.0, + 1.0 + / ( + 1.0 + T.exp2((attn_sink_frag[h_i] - final_lse[h_i]) * log2e) + ), + ) + for h_i, d_i in T.Parallel(H_per_block, DT): + acc_o[h_i, d_i] = acc_o[h_i, d_i] * o_scale_frag[h_i] + + T.copy(acc_o, Output[b_i, s_i, H0:H1, :]) + T.copy(final_lse, LSE[b_i, s_i, H0:H1]) + + return main + + +""" +2-stage attention kernel (partial + combine) over an FP8 KV cache, +with optional second cache (`extra_k_cache`). +""" + + +def dpsk_v4_fp8_attention_fwd( + q: torch.Tensor, + k_cache: torch.Tensor, + block_table: Optional[torch.Tensor], + cache_seqlens: Optional[torch.Tensor], + head_dim_v: int, + tile_scheduler_metadata: Any, + num_splits: None = None, + softmax_scale: Optional[float] = None, + causal: bool = False, + is_fp8_kvcache: bool = False, + indices: Optional[torch.Tensor] = None, + attn_sink: Optional[torch.Tensor] = None, + extra_k_cache: Optional[torch.Tensor] = None, + extra_indices_in_kvcache: Optional[torch.Tensor] = None, + topk_length: Optional[torch.Tensor] = None, + extra_topk_length: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Follows the original `flash_mla.flash_mla_with_kvcache` signature. + """ + if _is_gfx95_supported: + block_I, threads, num_stages, block_per_cu, cu = 64, 512, 0, 2, 256 + else: + block_I, threads, num_stages, block_per_cu, cu = 32, 128, 1, 1, 304 + + batch, seq_len, num_heads, _ = q.shape + # Partial grid is (seq_len * REPLICATE_H * n_groups, batch, kv_group); the + # heuristic in _pick_inner_iter assumes `total_blocks = seq * ni / inner_iter`, + # so `seq` must include REPLICATE_H or n_groups doubles for medium batches. + replicate_h = max((num_heads + 63) // 64, 1) + seq = batch * seq_len * replicate_h + + k1, _, bs_kv_1 = _build_fp8_combined_view(k_cache) + topk_1 = indices.shape[-1] + ni_1 = topk_1 // block_I + tk_len_1 = ( + topk_length + if topk_length is not None + else _topk_length_sentinel(q.device, batch) + ) + if attn_sink is None: + attn_sink = torch.full( + (num_heads,), float("-inf"), dtype=torch.float32, device=q.device + ) + + has_extra = extra_k_cache is not None + if not has_extra: + inner_iter_1 = _pick_inner_iter(seq, ni_1, cu, block_per_cu) + inner_iter_2 = 1 + n_groups_1 = ni_1 // inner_iter_1 + n_groups_2 = 0 + partial = dpsk_v4_fp8_partial_kernel( + num_heads, + topk_1, + bs_kv_1, + sm_scale=softmax_scale, + block_I=block_I, + inner_iter_1=inner_iter_1, + num_stages=num_stages, + threads=threads, + ) + partial_o, partial_lse = partial(q, k1, indices, tk_len_1) + else: + k2, _, bs_kv_2 = _build_fp8_combined_view(extra_k_cache) + topk_2 = extra_indices_in_kvcache.shape[-1] + ni_2 = topk_2 // block_I + # Each phase picks its own optimal split-K independently — kernel + # body uses two T.Pipelined loops with separate compile-time iter + # counts, no shared-divisor constraint. + inner_iter_1 = _pick_inner_iter(seq, ni_1, cu, block_per_cu) + inner_iter_2 = _pick_inner_iter(seq, ni_2, cu, block_per_cu) + n_groups_1 = ni_1 // inner_iter_1 + n_groups_2 = ni_2 // inner_iter_2 + tk_len_2 = ( + extra_topk_length + if extra_topk_length is not None + else _topk_length_sentinel(q.device, batch) + ) + partial = dpsk_v4_fp8_partial_kernel( + num_heads, + topk_1, + bs_kv_1, + topk_2, + bs_kv_2, + sm_scale=softmax_scale, + block_I=block_I, + inner_iter_1=inner_iter_1, + inner_iter_2=inner_iter_2, + num_stages=num_stages, + threads=threads, + ) + partial_o, partial_lse = partial( + q, + k1, + indices, + tk_len_1, + k2, + extra_indices_in_kvcache, + tk_len_2, + ) + + combine = dpsk_v4_combine_kernel( + num_heads, + n_groups_1, + n_groups_2, + block_I=block_I, + inner_iter_1=inner_iter_1, + inner_iter_2=inner_iter_2, + head_per_block=4, + threads=256, + use_attn_sink=True, + ) + if has_extra: + return combine(partial_o, partial_lse, tk_len_1, tk_len_2, attn_sink) + return combine(partial_o, partial_lse, attn_sink) diff --git a/python/sglang/kernels/ops/attention/nsa_triton_decode/__init__.py b/python/sglang/kernels/ops/attention/dsv4/triton_sparse_decode/__init__.py similarity index 96% rename from python/sglang/kernels/ops/attention/nsa_triton_decode/__init__.py rename to python/sglang/kernels/ops/attention/dsv4/triton_sparse_decode/__init__.py index f3e8bfc39d..cda2343fd3 100644 --- a/python/sglang/kernels/ops/attention/nsa_triton_decode/__init__.py +++ b/python/sglang/kernels/ops/attention/dsv4/triton_sparse_decode/__init__.py @@ -9,7 +9,7 @@ from typing import Optional, Tuple import torch -from sglang.kernels.ops.attention.nsa_triton_decode.triton_mla_kernels_decode_optimized import ( +from sglang.kernels.ops.attention.dsv4.triton_sparse_decode.triton_mla_kernels_decode_optimized import ( triton_sparse_attn_decode, ) diff --git a/python/sglang/kernels/ops/attention/nsa_triton_decode/triton_mla_kernels_decode_fused.py b/python/sglang/kernels/ops/attention/dsv4/triton_sparse_decode/triton_mla_kernels_decode_fused.py similarity index 100% rename from python/sglang/kernels/ops/attention/nsa_triton_decode/triton_mla_kernels_decode_fused.py rename to python/sglang/kernels/ops/attention/dsv4/triton_sparse_decode/triton_mla_kernels_decode_fused.py diff --git a/python/sglang/kernels/ops/attention/nsa_triton_decode/triton_mla_kernels_decode_optimized.py b/python/sglang/kernels/ops/attention/dsv4/triton_sparse_decode/triton_mla_kernels_decode_optimized.py similarity index 100% rename from python/sglang/kernels/ops/attention/nsa_triton_decode/triton_mla_kernels_decode_optimized.py rename to python/sglang/kernels/ops/attention/dsv4/triton_sparse_decode/triton_mla_kernels_decode_optimized.py diff --git a/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/paged_decode.py b/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/paged_decode.py index 6fa3bc8382..f0a7380bac 100644 --- a/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/paged_decode.py +++ b/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/paged_decode.py @@ -86,7 +86,7 @@ _TARGET_WG_PER_CU = 1.5 if _is_hip else 2.0 # halving from bf16→fp8 = 2× saving — net ~46% read bandwidth reduction). _FP8_GROUP_SIZE = 64 # Match the KV-cache write format: fnuz only on AMD gfx942; Other platform e.g. -# gfx950 store e4m3fn. See quant_k_cache.py / indexer.py FP8_DTYPE. +# gfx950 store e4m3fn. See quantization/dsa_quant_k_cache.py / indexer.py FP8_DTYPE. _FP8_DTYPE = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn @@ -120,7 +120,7 @@ _FP8_DTYPE = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn _HCA_KV_SPLITS = envs.SGLANG_MLA_HCA_KV_SPLITS.get() -def _kv_splits_for_stream(compress_ratio: int) -> int | None: +def kv_splits_for_stream(compress_ratio: int) -> int | None: """-> kv_splits override for a decode stream, or None to keep the heuristic. Only the unclamped stream is overridden. SWA (ratio 0) is a 128-entry @@ -951,7 +951,7 @@ def sparse_attn_v4_paged_decode( will be dequantized in-kernel using 1xGROUP_SIZE (default 64) block scales. ``kv_splits`` overrides `_kv_splits_heuristic`. The caller knows the layer's - ``compress_ratio`` and the heuristic does not; see `_kv_splits_for_stream`. + ``compress_ratio`` and the heuristic does not; see `kv_splits_for_stream`. """ if _is_gfx1250_supported: # aiter ships only on ROCm, and this module is imported by a CPU-registered diff --git a/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py b/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py index 53e12a5a00..71de495e23 100644 --- a/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py +++ b/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/runtime.py @@ -1,8 +1,7 @@ """Runtime glue for the unified_kv backend. Builds unified_kv-style flat ``kv_indices`` / ``kv_indptr`` from SGLang's already-computed -DSV4 metadata, scatters SWA K into the ``unified_kv`` ring, and dispatches the -vendored paged decode/prefill kernels. +DSV4 metadata and dispatches the vendored paged decode/prefill kernels. unified_kv[L] layout (page_size 1, row-major): - rows ``[0, swa_pages)`` = SWA ring (``state_slot * win + pos % win``); @@ -53,174 +52,6 @@ from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.paged_prefill import ( logger = logging.getLogger(__name__) -# --------------------------------------------------------------------------- -# SWA ring scatter -# --------------------------------------------------------------------------- -@triton.jit -def _swa_scatter_kernel( - kv_ptr, # [T, D] bf16 - state_slot_ptr, # [T] int - positions_ptr, # [T] int - final_pos_ptr, # [T] int - unified_ptr, # [pages, D] bf16 - n_rows, - ring_stride, # SWA ring per-slot stride - win: tl.constexpr, - D: tl.constexpr, - HAS_FINAL: tl.constexpr, - BLOCK_D: tl.constexpr, -): - row = tl.program_id(0) - if row >= n_rows: - return - pos = tl.load(positions_ptr + row) - if HAS_FINAL: - fp = tl.load(final_pos_ptr + row) - if pos <= fp - win: - return - s = tl.load(state_slot_ptr + row) - loc = s * ring_stride + (pos % ring_stride) - offs = tl.arange(0, BLOCK_D) - mask = offs < D - vals = tl.load(kv_ptr + row * D + offs, mask=mask, other=0.0) - tl.store(unified_ptr + loc * D + offs, vals, mask=mask) - - -def store_swa_into_unified( - *, - kv: torch.Tensor, # [T, head_dim] bf16, or [T, nope_row_bytes] packed fp8 - state_slot: torch.Tensor, # [T] int - positions: torch.Tensor, # [T] int - unified_kv: torch.Tensor, # [pages, ...] same dtype and row width as kv - win: int, # SWA attention window length - ring_stride: int, # SWA ring stride - final_pos: Optional[torch.Tensor] = None, # [T] req's last position - kv_rope: Optional[torch.Tensor] = None, # [T, rope_dim] bf16, fp8 layout only - unified_kv_rope: Optional[torch.Tensor] = None, # [pages, rope_dim] bf16 -) -> None: - """Scatter SWA K into ring row ``state_slot * ring_stride + pos % ring_stride``. - - Under the fp8 layout the latent is split over two pools, so ``kv`` carries the - already-packed nope row (DSV4_FP8_NOPE_ROW_BYTES wide: values + E8M0 scales + - pad; nothing is quantized here) and ``kv_rope`` the bf16 rope half. That width - is a byte count that happens to equal the bf16 head_dim in elements -- the two - are not the same thing. The scatter itself takes the row width off the tensor; - only the pair check reads the constant. - - Both scatters recompute the row index from the same ``state_slot`` / - ``positions``, so the two pools stay in lockstep with each other and with the - bf16 layout. What a caller can still get wrong is passing two pools that aren't - a pair, so the pair goes through ``check_two_pool_pair`` before the first launch - -- shared with the fused store, which has the same coupling. - """ - n_rows = kv.shape[0] - if n_rows == 0: - return - - has_final = final_pos is not None - fp_arg = final_pos if has_final else positions - assert kv.is_contiguous() and kv.dtype == unified_kv.dtype - assert state_slot.is_contiguous() and positions.is_contiguous() - assert fp_arg.is_contiguous() - two_pool = kv_rope is not None - assert two_pool == (unified_kv_rope is not None), ( - "kv_rope and unified_kv_rope come together" - ) - if two_pool: - assert kv_rope.is_contiguous(), ( - f"kv_rope must be contiguous, got strides {kv_rope.stride()}" - ) - assert kv_rope.shape[0] == n_rows, ( - f"kv_rope holds {kv_rope.shape[0]} rows, kv holds {n_rows}" - ) - check_two_pool_pair( - unified_kv, - unified_kv_rope, - rope_width=kv_rope.shape[1], - rope_dtype=kv_rope.dtype, - ) - - def _scatter(src: torch.Tensor, dst: torch.Tensor) -> None: - D = src.shape[1] - assert dst.shape[1] == D, f"row width {D} does not fit pool {dst.shape[1]}" - _swa_scatter_kernel[(n_rows,)]( - src, - state_slot, - positions, - fp_arg, - dst, - n_rows, - ring_stride, - win=win, - D=D, - HAS_FINAL=has_final, - BLOCK_D=triton.next_power_of_2(D), - num_warps=8, - ) - - if kv.element_size() == 1: - # single-byte rows (any fp8 variant) are a pure byte move, and the E8M0 - # scale bytes aren't floats -- uint8 avoids a triton convert for `other=` - assert unified_kv.is_contiguous() - _scatter(kv.view(torch.uint8), unified_kv.view(torch.uint8)) - else: - _scatter(kv, unified_kv) - - if two_pool: - _scatter(kv_rope, unified_kv_rope) - - -@triton.jit -def _scatter_loc_kernel( - kv_ptr, # [T, D] bf16 - loc_ptr, # [T] int (unified row index; <0 => skip) - unified_ptr, # [pages, D] bf16 - n_rows, - D: tl.constexpr, - BLOCK_D: tl.constexpr, -): - row = tl.program_id(0) - if row >= n_rows: - return - loc = tl.load(loc_ptr + row).to(tl.int64) - if loc < 0: - return - offs = tl.arange(0, BLOCK_D) - mask = offs < D - vals = tl.load(kv_ptr + row * D + offs, mask=mask, other=0.0) - tl.store(unified_ptr + loc * D + offs, vals, mask=mask) - - -def scatter_bf16_into_unified( - *, - kv: torch.Tensor, # [T, head_dim] bf16 (already norm+rope'd) - loc: torch.Tensor, # [T] int32/int64 unified ring row; <0 => skip - unified_kv: torch.Tensor, # [pages, head_dim] bf16 -) -> None: - """Scatter already-norm+rope'd bf16 K into ``unified_kv[loc]`` (skip loc < 0). - - Companion to ``store_swa_into_unified`` for callers that already hold the - precomputed ring row index (the DSpark draft: ``get_unified_swa_loc`` for the - draft forward, or the commit-inject layout for target-hidden injection) and - need per-row commit masking expressed as ``loc == -1``. - """ - n_rows, D = kv.shape - if n_rows == 0: - return - assert kv.is_contiguous() and kv.dtype == unified_kv.dtype - assert loc.is_contiguous() - assert unified_kv.is_contiguous() - _scatter_loc_kernel[(n_rows,)]( - kv, - loc, - unified_kv, - n_rows, - D=D, - BLOCK_D=triton.next_power_of_2(D), - num_warps=8, - ) - - # --------------------------------------------------------------------------- # Ragged indptr helper (shared by the decode streams + prefill builders) # --------------------------------------------------------------------------- diff --git a/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_mla_sm100.py b/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_mla_sm100.py index 93c86cef3e..29042a6cdc 100644 --- a/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_mla_sm100.py +++ b/python/sglang/kernels/ops/attention/flash_attn/cute/flash_fwd_mla_sm100.py @@ -1,7 +1,6 @@ # Copyright (c) 2026, Colfax International. import math -import time from functools import partial from typing import Callable, Optional @@ -10,21 +9,15 @@ import cutlass import cutlass.cute as cute import cutlass.pipeline as pipeline import cutlass.utils.blackwell_helpers as sm100_utils -import torch -import torch.utils.benchmark as benchmark from cutlass import Boolean, Float32, Int32, Int64, Uint32, const_expr from cutlass.cute import FastDivmodDivisor from cutlass.cute.nvgpu import cpasync, tcgen05 -from cutlass.cute.runtime import from_dlpack from cutlass.utils import ClcDynamicPersistentTileScheduler from quack import copy_utils import sglang.kernels.ops.attention.flash_attn.cute.blackwell_helpers as fa_sm100_utils from sglang.kernels.ops.attention.flash_attn.cute import utils as fa_utils from sglang.kernels.ops.attention.flash_attn.cute.block_info import BlockInfo -from sglang.kernels.ops.attention.flash_attn.cute.cute_dsl_utils import ( - dump_kernel_attributes, -) from sglang.kernels.ops.attention.flash_attn.cute.fa_logging import fa_log, fa_printf from sglang.kernels.ops.attention.flash_attn.cute.mask import AttentionMask from sglang.kernels.ops.attention.flash_attn.cute.named_barrier import ( @@ -37,7 +30,6 @@ from sglang.kernels.ops.attention.flash_attn.cute.pack_gqa import ( from sglang.kernels.ops.attention.flash_attn.cute.paged_kv import PagedKVManager from sglang.kernels.ops.attention.flash_attn.cute.seqlen_info import SeqlenInfoQK from sglang.kernels.ops.attention.flash_attn.cute.softmax import SoftmaxSm100 -from sglang.kernels.ops.attention.flash_attn.cute.testing import attention_ref from sglang.kernels.ops.attention.flash_attn.cute.tile_scheduler import ( ClcState, ParamsBase, @@ -3154,784 +3146,3 @@ class FlashAttentionMLAForwardSm100: ) cute.copy(thr_tmem_store, tOrO_t2r_frg, tOtO_r2t_cur) cute.arch.fence_view_async_tmem_store() - - -def test_mla_kernel( - seqlen_q=2048, - seqlen_k=2048, - topk_length=2048, - nheads=1, - batch=1, - iter=0, - compile_cache=dict(), - validate=True, - seed=0, - gather_kv=True, - pack_gqa=False, - is_causal=False, - varlen_q=False, - varlen_k=False, - disable_bitmask=False, - has_qk=True, - store_P=False, -): - torch.manual_seed(seed) - hdim = 64 - hdimv = 512 - softmax_scale = 1.0 / math.sqrt(hdim + hdimv) if has_qk else 1.0 / math.sqrt(hdimv) - - nheads_kv = 1 - qhead_per_kvhead = nheads - seqlen_k_rounded = (seqlen_k + 128 - 1) // 128 * 128 - P_k_length = seqlen_k_rounded if not gather_kv else topk_length - - torch_stream = torch.cuda.current_stream() - stream = cuda.CUstream(torch_stream.cuda_stream) - - compile_key = ( - is_causal, - gather_kv, - topk_length if gather_kv else None, - pack_gqa, - qhead_per_kvhead, - nheads_kv, - varlen_q, - varlen_k, - disable_bitmask, - has_qk, - ) - if compile_key not in compile_cache: - total_q_dummy = batch * seqlen_q - total_k_dummy = batch * seqlen_k - - if varlen_q: - Q = torch.randn( - total_q_dummy, nheads, hdim, dtype=torch.bfloat16, device="cuda" - ) - Qv = torch.randn( - total_q_dummy, nheads, hdimv, dtype=torch.bfloat16, device="cuda" - ) - O = torch.empty( - total_q_dummy, nheads, hdimv, dtype=torch.bfloat16, device="cuda" - ) - P = torch.empty( - total_q_dummy, nheads, P_k_length, dtype=torch.bfloat16, device="cuda" - ) - lse = torch.empty(total_q_dummy, nheads, dtype=torch.float32, device="cuda") - row_max = torch.empty( - total_q_dummy, - nheads, - P_k_length // 128, - dtype=torch.float32, - device="cuda", - ) - index_topk = ( - torch.rand(total_q_dummy, topk_length, device="cuda") - .argsort(dim=-1) - .to(torch.int32) - ) - cu_seqlens_q_dummy = torch.arange( - 0, (batch + 1) * seqlen_q, seqlen_q, dtype=torch.int32, device="cuda" - ) - else: - Q = torch.randn( - batch, seqlen_q, nheads, hdim, dtype=torch.bfloat16, device="cuda" - ) - Qv = torch.randn( - batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda" - ) - O = torch.empty( - batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda" - ) - P = torch.empty( - batch, seqlen_q, nheads, P_k_length, dtype=torch.bfloat16, device="cuda" - ) - lse = torch.empty( - batch, seqlen_q, nheads, dtype=torch.float32, device="cuda" - ) - row_max = torch.empty( - batch, - seqlen_q, - nheads, - P_k_length // 128, - dtype=torch.float32, - device="cuda", - ) - index_topk = ( - torch.rand(batch, seqlen_q, topk_length, device="cuda") - .argsort(dim=-1) - .to(torch.int32) - ) - - if varlen_k: - K = torch.randn( - total_k_dummy, nheads_kv, hdim, dtype=torch.bfloat16, device="cuda" - ) - V = torch.randn( - total_k_dummy, nheads_kv, hdimv, dtype=torch.bfloat16, device="cuda" - ) - cu_seqlens_k_dummy = torch.arange( - 0, (batch + 1) * seqlen_k, seqlen_k, dtype=torch.int32, device="cuda" - ) - else: - K = torch.randn( - batch, seqlen_k, nheads_kv, hdim, dtype=torch.bfloat16, device="cuda" - ) - V = torch.randn( - batch, seqlen_k, nheads_kv, hdimv, dtype=torch.bfloat16, device="cuda" - ) - - mQ = from_dlpack(Q, assumed_align=16).mark_layout_dynamic( - leading_dim=Q.ndim - 1 - ) - mQv = from_dlpack(Qv, assumed_align=16).mark_layout_dynamic( - leading_dim=Qv.ndim - 1 - ) - mK = from_dlpack(K, assumed_align=16).mark_layout_dynamic( - leading_dim=K.ndim - 1 - ) - mV = from_dlpack(V, assumed_align=16).mark_layout_dynamic( - leading_dim=V.ndim - 1 - ) - mO = from_dlpack(O, assumed_align=16).mark_layout_dynamic( - leading_dim=O.ndim - 1 - ) - mP = from_dlpack(P, assumed_align=16).mark_layout_dynamic( - leading_dim=P.ndim - 1 - ) - mLSE = from_dlpack(lse, assumed_align=4).mark_layout_dynamic( - leading_dim=lse.ndim - 1 - ) - mRowMax = from_dlpack(row_max, assumed_align=4).mark_layout_dynamic( - leading_dim=row_max.ndim - 1 - ) - if gather_kv: - mIndexTopk = from_dlpack(index_topk, assumed_align=16).mark_layout_dynamic( - leading_dim=index_topk.ndim - 1 - ) - else: - mIndexTopk = None - - compile_kwargs = dict(mIndexTopk=mIndexTopk) - if varlen_q: - compile_kwargs["mCuSeqlensQ"] = from_dlpack( - cu_seqlens_q_dummy, assumed_align=4 - ) - if varlen_k: - compile_kwargs["mCuSeqlensK"] = from_dlpack( - cu_seqlens_k_dummy, assumed_align=4 - ) - - if not has_qk: - mQ = mK = None - - if store_P is False: - mP = mRowMax = None - - kernel = cute.compile( - FlashAttentionMLAForwardSm100( - is_causal=is_causal, - use_cpasync_load_KV=gather_kv, - topk_length=topk_length if gather_kv else 2048, - is_topk_gather=gather_kv, - pack_gqa=pack_gqa, - qhead_per_kvhead=qhead_per_kvhead, - nheads_kv=nheads_kv, - is_varlen_q=varlen_q, - disable_bitmask=disable_bitmask, - has_qk=has_qk, - ), - mQ, - mQv, - mK, - mV, - mO, - mLSE, - softmax_scale, - mP, - mRowMax, - **compile_kwargs, - stream=stream, - options="--keep-ptx --keep-cubin --generate-line-info", - ) - dump_kernel_attributes(kernel) - compile_cache[compile_key] = kernel - - # ================================================================ - # ---- Generate variable seqlens for this run ---- - if varlen_q: - torch.manual_seed(seed + 1000) - # When causal without varlen_k, every per-batch seqlen_q must not exceed seqlen_k. - max_seqlen_q = seqlen_k if (is_causal and not varlen_k) else seqlen_q - seqlens_q = torch.randint(1, max_seqlen_q + 1, (batch,), dtype=torch.int32) - cu_seqlens_q = torch.zeros(batch + 1, dtype=torch.int32, device="cuda") - cu_seqlens_q[1:] = seqlens_q.cumsum(0).to(torch.int32).cuda() - total_q = cu_seqlens_q[-1].item() - else: - seqlens_q = torch.full((batch,), seqlen_q, dtype=torch.int32) - total_q = None # unused - - if varlen_k: - torch.manual_seed(seed + 2000) - # Each batch item must have at least topk_length keys so topk gather is valid. - min_seqlen_k = topk_length if gather_kv else 1 - seqlens_k = torch.randint( - min_seqlen_k, seqlen_k + 1, (batch,), dtype=torch.int32 - ) - # When causal, every batch item needs seqlens_k[b] >= seqlens_q[b]. - if is_causal: - seqlens_k = torch.maximum(seqlens_k, seqlens_q) - cu_seqlens_k = torch.zeros(batch + 1, dtype=torch.int32, device="cuda") - cu_seqlens_k[1:] = seqlens_k.cumsum(0).to(torch.int32).cuda() - total_k = cu_seqlens_k[-1].item() - else: - seqlens_k = torch.full((batch,), seqlen_k, dtype=torch.int32) - total_k = None # unused - - torch.manual_seed(seed) # restore main seed before drawing actual tensors - - # ---- Allocate Q / Qv / O / lse ---- - if varlen_q: - Q = torch.randn(total_q, nheads, hdim, dtype=torch.bfloat16, device="cuda") - Qv = torch.randn(total_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda") - O = torch.empty(total_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda") - P = torch.empty( - total_q, nheads, P_k_length, dtype=torch.bfloat16, device="cuda" - ) - lse = torch.empty(total_q, nheads, dtype=torch.float32, device="cuda") - row_max = torch.empty( - total_q_dummy, P_k_length // 128, nheads, dtype=torch.float32, device="cuda" - ) - else: - Q = torch.randn( - batch, seqlen_q, nheads, hdim, dtype=torch.bfloat16, device="cuda" - ) - Qv = torch.randn( - batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda" - ) - O = torch.empty( - batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda" - ) - P = torch.empty( - batch, seqlen_q, nheads, P_k_length, dtype=torch.bfloat16, device="cuda" - ) - lse = torch.empty(batch, seqlen_q, nheads, dtype=torch.float32, device="cuda") - row_max = torch.empty( - batch, - seqlen_q, - P_k_length // 128, - nheads, - dtype=torch.float32, - device="cuda", - ) - - # ---- Allocate K / V ---- - if varlen_k: - K = torch.randn(total_k, nheads_kv, hdim, dtype=torch.bfloat16, device="cuda") - V = torch.randn(total_k, nheads_kv, hdimv, dtype=torch.bfloat16, device="cuda") - else: - K = torch.randn( - batch, seqlen_k, nheads_kv, hdim, dtype=torch.bfloat16, device="cuda" - ) - V = torch.randn( - batch, seqlen_k, nheads_kv, hdimv, dtype=torch.bfloat16, device="cuda" - ) - - # ---- Generate index_topk with per-batch valid ranges when varlen_k ---- - # index_topk shape: (total_q, topk_length) if varlen_q else (batch, seqlen_q, topk_length) - if gather_kv: - topk_parts = [] - for b in range(batch): - sl_q_b = seqlens_q[b].item() - sl_k_b = seqlens_k[b].item() - # Draw topk_length unique indices from [0, sl_k_b) for each query in this batch item. - topk_b = ( - torch.rand(sl_q_b, sl_k_b, device="cuda") - .argsort(dim=-1)[..., :topk_length] - .to(torch.int32) - ) # (sl_q_b, topk_length), all < sl_k_b - topk_parts.append(topk_b) - - if varlen_q: - index_topk = torch.cat(topk_parts, dim=0) # (total_q, topk_length) - else: - index_topk = torch.stack( - topk_parts, dim=0 - ) # (batch, seqlen_q, topk_length) - else: - index_topk = None - - # ---- Reference computation (per-batch loop covers all four varlen combos) ---- - O_ref_list, O_pt_list, lse_ref_list, lse_pt_list = [], [], [], [] - for b in range(batch): - qs = cu_seqlens_q[b].item() if varlen_q else b * seqlen_q - qe = cu_seqlens_q[b + 1].item() if varlen_q else (b + 1) * seqlen_q - ks = cu_seqlens_k[b].item() if varlen_k else b * seqlen_k - ke = cu_seqlens_k[b + 1].item() if varlen_k else (b + 1) * seqlen_k - - Q_b = ( - Q[qs:qe].unsqueeze(0) if varlen_q else Q[b : b + 1] - ) # (1, sl_q, nheads, hdim) - Qv_b = ( - Qv[qs:qe].unsqueeze(0) if varlen_q else Qv[b : b + 1] - ) # (1, sl_q, nheads, hdimv) - K_b = ( - K[ks:ke].unsqueeze(0) if varlen_k else K[b : b + 1] - ) # (1, sl_k, nheads_kv, hdim) - V_b = ( - V[ks:ke].unsqueeze(0) if varlen_k else V[b : b + 1] - ) # (1, sl_k, nheads_kv, hdimv) - if gather_kv: - topk_b = ( - index_topk[qs:qe].unsqueeze(0) if varlen_q else index_topk[b : b + 1] - ) - else: - topk_b = None - - O_b, _, lse_b = attention_ref( - Q_b if has_qk else None, - K_b if has_qk else None, - V_b, - qv=Qv_b, - causal=is_causal, - return_lse=True, - gather_kv_indices=topk_b, - ) - O_pt_b, _, lse_pt_b = attention_ref( - Q_b if has_qk else None, - K_b if has_qk else None, - V_b, - qv=Qv_b, - causal=is_causal, - upcast=False, - reorder_ops=True, - return_lse=True, - gather_kv_indices=topk_b, - ) - O_ref_list.append(O_b.squeeze(0)) - O_pt_list.append(O_pt_b.squeeze(0)) - lse_ref_list.append(lse_b.squeeze(0)) - lse_pt_list.append(lse_pt_b.squeeze(0)) - - cat_dim_o = 0 if (varlen_q) else 0 # always 0: leading token/batch dim - cat_dim_lse = -1 if (varlen_q) else -1 # always last: token dim - - if varlen_q: - O_ref = torch.cat(O_ref_list, dim=0) # (total_q, nheads, hdimv) - O_pt = torch.cat(O_pt_list, dim=0) - lse_ref = torch.cat(lse_ref_list, dim=-1) # (nheads, total_q) - lse_pt = torch.cat(lse_pt_list, dim=-1) - else: - O_ref = torch.stack(O_ref_list, dim=0) # (batch, seqlen_q, nheads, hdimv) - O_pt = torch.stack(O_pt_list, dim=0) - lse_ref = torch.stack(lse_ref_list, dim=0) # (batch, nheads, seqlen_q) - lse_pt = torch.stack(lse_pt_list, dim=0) - - rtol = 2 - atol = 2 * (O_ref + 0.3 - 0.3 - O_ref).abs().max().item() - - # ---- CuTe tensor wrappers ---- - mQ = from_dlpack(Q, assumed_align=16).mark_layout_dynamic(leading_dim=Q.ndim - 1) - mQv = from_dlpack(Qv, assumed_align=16).mark_layout_dynamic(leading_dim=Qv.ndim - 1) - mK = from_dlpack(K, assumed_align=16).mark_layout_dynamic(leading_dim=K.ndim - 1) - mV = from_dlpack(V, assumed_align=16).mark_layout_dynamic(leading_dim=V.ndim - 1) - mO = from_dlpack(O, assumed_align=16).mark_layout_dynamic(leading_dim=O.ndim - 1) - mP = from_dlpack(P, assumed_align=16).mark_layout_dynamic(leading_dim=P.ndim - 1) - mLSE = from_dlpack(lse, assumed_align=4).mark_layout_dynamic( - leading_dim=lse.ndim - 1 - ) - mRowMax = from_dlpack(row_max, assumed_align=4).mark_layout_dynamic( - leading_dim=row_max.ndim - 1 - ) - if index_topk is not None: - mIndexTopk = from_dlpack(index_topk, assumed_align=16).mark_layout_dynamic( - leading_dim=index_topk.ndim - 1 - ) - else: - mIndexTopk = None - - run_kwargs = dict(mIndexTopk=mIndexTopk) - if varlen_q: - run_kwargs["mCuSeqlensQ"] = from_dlpack(cu_seqlens_q, assumed_align=4) - if varlen_k: - run_kwargs["mCuSeqlensK"] = from_dlpack(cu_seqlens_k, assumed_align=4) - - if not has_qk: - mQ = mK = None - - if store_P is False: - mP = mRowMax = None - - # ---- Run kernel ---- - compile_cache[compile_key]( - mQ, - mQv, - mK, - mV, - mO, - mLSE, - softmax_scale, - mP, - mRowMax, - **run_kwargs, - stream=stream, - ) - - O_ref_max = O_ref.abs().max().item() - O_max = O.abs().max().item() - print(f"Pytorch O max = {O_ref_max} and our O max = {O_max}") - print(f"Pytorch max O diff: {(O_pt - O_ref).abs().max().item()}") - print(f"Pytorch mean O diff: {(O_pt - O_ref).abs().mean().item()}") - print(f"Max abs diff O, O_ref: {(O - O_ref).abs().max().item()}") - print(f"Mean abs diff O, O_ref: {(O - O_ref).abs().mean().item()}") - - lse = lse.transpose(-1, -2) - lse_ref_max = lse_ref.abs().max().item() - lse_max = lse.abs().max().item() - print(f"Pytorch LSE max = {lse_ref_max} and our LSE max = {lse_max}") - print(f"Pytorch LSE max diff: {(lse_pt - lse_ref).abs().max().item()}") - print(f"Pytorch LSE mean diff: {(lse_pt - lse_ref).abs().mean().item()}") - print(f"Max abs diff LSE: {(lse - lse_ref).abs().max().item()}") - print(f"Mean abs diff LSE: {(lse - lse_ref).abs().mean().item()}") - - if validate: - assert (O - O_ref).abs().max().item() <= rtol * ( - O_pt - O_ref - ).abs().max().item() + atol - varlen_tag = "" - if varlen_q: - varlen_tag += f", total_q:{total_q}" - if varlen_k: - varlen_tag += f", total_k:{total_k}" - print( - f"batch:{batch:3d}, nheads:{nheads:3d}, seqlen_q:{seqlen_q:5d}, seqlen_k:{seqlen_k:5d}" - f"{varlen_tag}, iter:{iter:2d} PASSED" - ) - else: - print(mO) - print( - f"batch:{batch:3d}, nheads:{nheads:3d}, seqlen_q:{seqlen_q:5d}, seqlen_k:{seqlen_k:5d}" - f", iter:{iter:2d} RUN (NOT TESTING CORRECTNESS)" - ) - - return None - - -def timeit(fn, *args, **kwargs): - # Synchronize before timing - torch.cuda.synchronize() - - # Warmup - for _ in range(10): - fn(*args, **kwargs) - - # Benchmark using PyTorch's Timer - t = benchmark.Timer( - stmt="fn(*args, **kwargs)", globals={"fn": fn, "args": args, "kwargs": kwargs} - ) - - # Time it multiple runs - measurement = t.timeit(20) # 20 repeats - avg_time = measurement.mean # Average time in seconds - - time.sleep(1) - - return avg_time - - -def benchmark_mla_kernel( - batch=1, - seqlen_q=2048, - seqlen_k=2048, - topk_length=2048, - nheads=128, - hdim=64, - hdimv=512, - compile_cache=dict(), - gather_kv=True, - is_causal=False, - disable_bitmask=False, - store_P=False, -): - assert hdim == 64, "hdim must be 64" - assert hdimv == 512, "hdimv must be 512" - - qhead_per_kvhead = nheads - nheads_kv = 1 - pack_gqa = True - softmax_scale = 1.0 / math.sqrt(hdim + hdimv) - seqlen_k_rounded = (seqlen_k + 128 - 1) // 128 * 128 - P_k_length = seqlen_k_rounded if not gather_kv else topk_length - - torch_stream = torch.cuda.current_stream() - stream = cuda.CUstream(torch_stream.cuda_stream) - - compile_key = ( - is_causal, - gather_kv, - topk_length if gather_kv else None, - pack_gqa, - qhead_per_kvhead, - nheads_kv, - disable_bitmask, - ) - if compile_key not in compile_cache: - Q = torch.randn( - batch, seqlen_q, nheads, hdim, dtype=torch.bfloat16, device="cuda" - ) - Qv = torch.randn( - batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda" - ) - K = torch.randn( - batch, seqlen_k, nheads_kv, hdim, dtype=torch.bfloat16, device="cuda" - ) - V = torch.randn( - batch, seqlen_k, nheads_kv, hdimv, dtype=torch.bfloat16, device="cuda" - ) - O = torch.empty( - batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda" - ) - P = torch.empty( - batch, seqlen_q, nheads, P_k_length, dtype=torch.bfloat16, device="cuda" - ) - index_topk = ( - torch.rand(batch, seqlen_q, topk_length, device="cuda") - .argsort(dim=-1) - .to(torch.int32) - ) - - mQ = from_dlpack(Q, assumed_align=16).mark_layout_dynamic( - leading_dim=Q.ndim - 1 - ) - mQv = from_dlpack(Qv, assumed_align=16).mark_layout_dynamic( - leading_dim=Qv.ndim - 1 - ) - mK = from_dlpack(K, assumed_align=16).mark_layout_dynamic( - leading_dim=K.ndim - 1 - ) - mV = from_dlpack(V, assumed_align=16).mark_layout_dynamic( - leading_dim=V.ndim - 1 - ) - mO = from_dlpack(O, assumed_align=16).mark_layout_dynamic( - leading_dim=O.ndim - 1 - ) - mP = from_dlpack(P, assumed_align=16).mark_layout_dynamic( - leading_dim=P.ndim - 1 - ) - if gather_kv: - mIndexTopk = from_dlpack(index_topk, assumed_align=16).mark_layout_dynamic( - leading_dim=index_topk.ndim - 1 - ) - else: - mIndexTopk = None - - mLSE = None - - if store_P is False: - mP = None - - kernel = cute.compile( - FlashAttentionMLAForwardSm100( - is_causal=is_causal, - use_cpasync_load_KV=gather_kv, - topk_length=topk_length if gather_kv else 2048, - is_topk_gather=gather_kv, - pack_gqa=pack_gqa, - qhead_per_kvhead=qhead_per_kvhead, - nheads_kv=nheads_kv, - disable_bitmask=disable_bitmask, - ), - mQ, - mQv, - mK, - mV, - mO, - mLSE, - softmax_scale, - mP=mP, - mIndexTopk=mIndexTopk, - stream=stream, - ) - compile_cache[compile_key] = kernel - - Q = torch.randn(batch, seqlen_q, nheads, hdim, dtype=torch.bfloat16, device="cuda") - Qv = torch.randn( - batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda" - ) - K = torch.randn( - batch, seqlen_k, nheads_kv, hdim, dtype=torch.bfloat16, device="cuda" - ) - V = torch.randn( - batch, seqlen_k, nheads_kv, hdimv, dtype=torch.bfloat16, device="cuda" - ) - O = torch.empty(batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda") - P = torch.empty( - batch, seqlen_q, nheads, P_k_length, dtype=torch.bfloat16, device="cuda" - ) - - index_topk = ( - torch.rand(batch, seqlen_q, topk_length, device="cuda") - .argsort(dim=-1) - .to(torch.int32) - ) - - mQ = from_dlpack(Q, assumed_align=16).mark_layout_dynamic(leading_dim=Q.ndim - 1) - mQv = from_dlpack(Qv, assumed_align=16).mark_layout_dynamic(leading_dim=Qv.ndim - 1) - mK = from_dlpack(K, assumed_align=16).mark_layout_dynamic(leading_dim=K.ndim - 1) - mV = from_dlpack(V, assumed_align=16).mark_layout_dynamic(leading_dim=V.ndim - 1) - mO = from_dlpack(O, assumed_align=16).mark_layout_dynamic(leading_dim=O.ndim - 1) - mP = from_dlpack(P, assumed_align=16).mark_layout_dynamic(leading_dim=P.ndim - 1) - if gather_kv: - mIndexTopk = from_dlpack(index_topk, assumed_align=16).mark_layout_dynamic( - leading_dim=index_topk.ndim - 1 - ) - else: - mIndexTopk = None - mLSE = None - - if store_P is False: - mP = None - - exec_time_in_s = timeit( - compile_cache[compile_key], - mQ, - mQv, - mK, - mV, - mO, - mLSE, - softmax_scale, - mP=mP, - mIndexTopk=mIndexTopk, - stream=stream, - ) - - seqlen_k_eff = topk_length if gather_kv else seqlen_k - - FLOPs = 2 * batch * nheads * seqlen_q * seqlen_k_eff * (hdim + 2 * hdimv) - if is_causal and not gather_kv: - FLOPs /= 2 - - TFLOPS = FLOPs / exec_time_in_s / 1e12 - - q_bytes = 2 * batch * nheads * seqlen_q * hdim - qv_bytes = 2 * batch * nheads * seqlen_q * hdimv - k_bytes = 2 * batch * nheads_kv * seqlen_k_eff * hdim - v_bytes = 2 * batch * nheads_kv * seqlen_k_eff * hdimv - o_bytes = 2 * batch * nheads * seqlen_q * hdimv - total_bytes = q_bytes + qv_bytes + k_bytes + v_bytes + o_bytes - TBs = total_bytes / exec_time_in_s / 1e12 - - print( - f"batch: {batch}, seqlen_q: {seqlen_q}, seqlen_k: {seqlen_k}, nheads: {nheads}, -> {exec_time_in_s * 1e3:.2f} ms, {TFLOPS:.2f} TFLOPS, {TBs:.2f} TBs" - ) - - -if __name__ == "__main__": - run_test = True - run_benchmark = True - gather_kv = True - is_causal = False - pack_gqa = True - topk_length = 2048 - varlen_q = False - varlen_k = False - disable_bitmask = False - validate = True - has_qk = True - - if run_test: - if not gather_kv: - seqlen_q_test_values = range(1, 4002, 400) - seqlen_k_test_values = range(1, 4002, 400) - else: - seqlen_q_test_values = range(1, 1001, 200) - seqlen_k_test_values = range(topk_length, 9001, 2000) - seqlen_q_test_values = [4096] - seqlen_k_test_values = [4096] - nheads_test_values = [128] - batch_test_values = [1] - test_configs = [ - ( - batch, - nheads, - seqlen_q, - seqlen_k, - ) - for batch in batch_test_values - for nheads in nheads_test_values - for seqlen_q in seqlen_q_test_values - for seqlen_k in seqlen_k_test_values - ] - iters_per_config = 1 - compile_cache = dict() - print("=" * 40) - print("Testing MLA Kernel") - print("=" * 40) - for config in test_configs: - batch, nheads, seqlen_q, seqlen_k = config - # if is_causal and seqlen_k < seqlen_q: - # continue - for iter in range(iters_per_config): - test_mla_kernel( - seqlen_q=seqlen_q, - seqlen_k=seqlen_k, - topk_length=topk_length, - nheads=nheads, - batch=batch, - iter=iter, - compile_cache=compile_cache, - validate=validate, - seed=0, - gather_kv=gather_kv, - pack_gqa=pack_gqa, - is_causal=is_causal, - varlen_q=varlen_q, - varlen_k=varlen_k, - disable_bitmask=disable_bitmask, - has_qk=has_qk, - ) - if run_benchmark: - if gather_kv: - seqlen_q_benchmark_values = [1] - seqlen_k_benchmark_values = [8192 * 2] - nheads_benchmark_values = [128] - batch_benchmark_values = [128] - else: - seqlen_q_benchmark_values = [1] - seqlen_k_benchmark_values = [8192] - nheads_benchmark_values = [128] - batch_benchmark_values = [128] - # seqlen_q_benchmark_values = [4096] - # seqlen_k_benchmark_values = [4096] - # nheads_benchmark_values = [16] - # batch_benchmark_values = [8] - benchmark_configs = [ - ( - batch, - nheads, - seqlen_q, - seqlen_k, - ) - for batch in batch_benchmark_values - for nheads in nheads_benchmark_values - for seqlen_q in seqlen_q_benchmark_values - for seqlen_k in seqlen_k_benchmark_values - ] - compile_cache = dict() - print("=" * 40) - print("Benchmarking MLA Kernel") - print("=" * 40) - for config in benchmark_configs: - batch, nheads, seqlen_q, seqlen_k = config - benchmark_mla_kernel( - batch=batch, - seqlen_q=seqlen_q, - seqlen_k=seqlen_k, - topk_length=topk_length, - nheads=nheads, - gather_kv=gather_kv, - is_causal=is_causal, - disable_bitmask=disable_bitmask, - compile_cache=compile_cache, - ) diff --git a/python/sglang/kernels/ops/kvcache/rope_cache.py b/python/sglang/kernels/ops/attention/fused_qk_rope_reshape_and_cache.py similarity index 100% rename from python/sglang/kernels/ops/kvcache/rope_cache.py rename to python/sglang/kernels/ops/attention/fused_qk_rope_reshape_and_cache.py diff --git a/python/sglang/kernels/ops/attention/helion/kda_replayssm.py b/python/sglang/kernels/ops/attention/helion/kda_replayssm.py index 3323c2c6c1..0e0dd1dc88 100644 --- a/python/sglang/kernels/ops/attention/helion/kda_replayssm.py +++ b/python/sglang/kernels/ops/attention/helion/kda_replayssm.py @@ -2,7 +2,7 @@ The public :func:`helion_fused_recurrent_kda_replayssm_decode` mirrors ``fused_recurrent_linear_replayssm_decode(..., is_kda=True)`` from -``sglang.kernels.ops.attention.fla.fused_recurrent_linear_replayssm``. That +``sglang.kernels.ops.attention.linear.fused_recurrent_linear_replayssm``. That Triton kernel is gate-generic (GDN scalar gate / KDA per-K gate); this module implements the KDA path only, and additionally supports the bounded gate, which the Triton ReplaySSM kernel does not expose. diff --git a/python/sglang/kernels/ops/kvcache/kv_indices.py b/python/sglang/kernels/ops/attention/kv_indices.py similarity index 100% rename from python/sglang/kernels/ops/kvcache/kv_indices.py rename to python/sglang/kernels/ops/attention/kv_indices.py diff --git a/python/sglang/kernels/ops/attention/fla/fused_gdn_gating.py b/python/sglang/kernels/ops/attention/linear/fused_gdn_gating.py similarity index 100% rename from python/sglang/kernels/ops/attention/fla/fused_gdn_gating.py rename to python/sglang/kernels/ops/attention/linear/fused_gdn_gating.py diff --git a/python/sglang/kernels/ops/attention/fla/fused_kda_conv_recurrent_verify.py b/python/sglang/kernels/ops/attention/linear/fused_kda_conv_recurrent_verify.py similarity index 100% rename from python/sglang/kernels/ops/attention/fla/fused_kda_conv_recurrent_verify.py rename to python/sglang/kernels/ops/attention/linear/fused_kda_conv_recurrent_verify.py diff --git a/python/sglang/kernels/ops/attention/fla/fused_recurrent_linear_replayssm.py b/python/sglang/kernels/ops/attention/linear/fused_recurrent_linear_replayssm.py similarity index 100% rename from python/sglang/kernels/ops/attention/fla/fused_recurrent_linear_replayssm.py rename to python/sglang/kernels/ops/attention/linear/fused_recurrent_linear_replayssm.py diff --git a/python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py b/python/sglang/kernels/ops/attention/linear/fused_sigmoid_gating_recurrent.py similarity index 100% rename from python/sglang/kernels/ops/attention/fla/fused_sigmoid_gating_recurrent.py rename to python/sglang/kernels/ops/attention/linear/fused_sigmoid_gating_recurrent.py diff --git a/python/sglang/kernels/ops/attention/fla/gdn_replayssm_spec_decode.py b/python/sglang/kernels/ops/attention/linear/gdn_replayssm_spec_decode.py similarity index 100% rename from python/sglang/kernels/ops/attention/fla/gdn_replayssm_spec_decode.py rename to python/sglang/kernels/ops/attention/linear/gdn_replayssm_spec_decode.py diff --git a/python/sglang/kernels/ops/attention/fla/gdn_replayssm_spec_fold.py b/python/sglang/kernels/ops/attention/linear/gdn_replayssm_spec_fold.py similarity index 100% rename from python/sglang/kernels/ops/attention/fla/gdn_replayssm_spec_fold.py rename to python/sglang/kernels/ops/attention/linear/gdn_replayssm_spec_fold.py diff --git a/python/sglang/kernels/ops/attention/fla/kda_replayssm_spec_decode.py b/python/sglang/kernels/ops/attention/linear/kda_replayssm_spec_decode.py similarity index 100% rename from python/sglang/kernels/ops/attention/fla/kda_replayssm_spec_decode.py rename to python/sglang/kernels/ops/attention/linear/kda_replayssm_spec_decode.py diff --git a/python/sglang/kernels/ops/attention/linear/lightning_attn.py b/python/sglang/kernels/ops/attention/linear/lightning_attn.py index 1a968dd88b..9f35728cac 100644 --- a/python/sglang/kernels/ops/attention/linear/lightning_attn.py +++ b/python/sglang/kernels/ops/attention/linear/lightning_attn.py @@ -724,43 +724,3 @@ def linear_decode_forward_triton( # Reshape output and return output = rearrange(output, "b h n d -> b n (h d)") return output.squeeze(1).contiguous() - - -class BailingLinearKernel: - """ - Linear attention kernel implementation for Bailing models. - - This class is adapted from MiniMaxText01LinearKernel in vllm: - https://github.com/vllm-project/vllm/blob/a9138e85b14047e06300685b48e3485b995425fb/vllm/model_executor/models/minimax_text_01.py#L289 - - The implementation maintains the same functionality while being renamed to - match our Bailing model naming convention. - """ - - @staticmethod - def jit_linear_forward_prefix( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - kv_caches: torch.Tensor, - slope_rate: torch.Tensor, - block_size: int, - layer_idx: int = None, - **kwargs, - ) -> torch.Tensor: - - slope_rate = slope_rate.to(torch.float32) - should_pad_dim = q.dim() == 3 - if should_pad_dim: - q = q.unsqueeze(0) - k = k.unsqueeze(0) - v = v.unsqueeze(0) - b, h, n, d = q.shape - e = d - kv_history = kv_caches.reshape(1, h, d, e).contiguous() - output, kv_history = lightning_attention( - q, k, v, slope_rate, block_size=block_size, kv_history=kv_history - ) - kv_caches.copy_(kv_history[:, :, -1, :, :].reshape(h, d, e)) - assert output.shape[0] == 1, "batch size must be 1" - return output.squeeze(0).transpose(0, 1).reshape([n, h * d]).contiguous() diff --git a/python/sglang/kernels/ops/attention/log_scaling_tau.py b/python/sglang/kernels/ops/attention/log_scaling_tau.py deleted file mode 100644 index 3884d732b8..0000000000 --- a/python/sglang/kernels/ops/attention/log_scaling_tau.py +++ /dev/null @@ -1,70 +0,0 @@ -import torch -import triton -import triton.language as tl - - -@triton.jit -def _apply_log_scaling_tau_kernel( - x_ptr, - tau_ptr, # [rows] fp32 (flattened per-row scale) - out_ptr, # [rows, inner] contiguous, same dtype as x - x_row_stride, - inner, - total, - BLOCK: tl.constexpr, -): - pid = tl.program_id(0) - offs = pid.to(tl.int64) * BLOCK + tl.arange(0, BLOCK) - mask = offs < total - row = offs // inner - col = offs % inner - x = tl.load(x_ptr + row * x_row_stride + col, mask=mask).to(tl.float32) - tau = tl.load(tau_ptr + row, mask=mask) - y = x * tau - tl.store(out_ptr + offs, y.to(out_ptr.dtype.element_ty), mask=mask) - - -def apply_log_scaling_tau(x: torch.Tensor, tau: torch.Tensor) -> torch.Tensor: - """out = (x.float() * tau).to(x.dtype) with tau broadcast per leading row, - fused into one launch. x may carry a leading-dim stride (the q slice of the - fused qkvr output); its trailing dims must be contiguous. No dynamo: the - torch.compile'd predecessor's call sites spanned enough rank / - dispatch-key / 0-1 specialization variants (target + de-tied MTP heads) to - exceed the recompile limit, which crashed (fullgraph) or wedged capture - (raised limit).""" - rows = x.shape[0] - inner = x.numel() // rows if rows else 0 - inner_contiguous = x.stride(-1) == 1 and ( - x.dim() == 2 or x.stride(-2) == x.shape[-1] * x.stride(-1) - ) - if rows == 0 or inner == 0 or not inner_contiguous: - return (x.float() * tau).to(x.dtype) - - if ( - x.is_cuda - and x.dtype == torch.bfloat16 - and inner % 8 == 0 - and x.data_ptr() % 16 == 0 - and (x.stride(0) * 2) % 16 == 0 - ): - # Vectorized JIT kernel (16B loads, one row divide per vector) -- - # bit-identical output (same fp32-mul + bf16-round), ~2-3x the - # scalar triton kernel below at every size. - from sglang.kernels.ops.elementwise.row_scale import row_scale_bf16 - - x2d = torch.as_strided(x, (rows, inner), (x.stride(0), 1)) - return row_scale_bf16(x2d, tau.reshape(rows).float()).view(x.shape) - - out = torch.empty(x.shape, dtype=x.dtype, device=x.device) - total = rows * inner - BLOCK = 1024 - _apply_log_scaling_tau_kernel[(triton.cdiv(total, BLOCK),)]( - x, - tau.reshape(rows).to(torch.float32), - out, - x.stride(0), - inner, - total, - BLOCK=BLOCK, - ) - return out diff --git a/python/sglang/kernels/ops/attention/metadata.py b/python/sglang/kernels/ops/attention/metadata.py index 2985807899..ba4866571c 100644 --- a/python/sglang/kernels/ops/attention/metadata.py +++ b/python/sglang/kernels/ops/attention/metadata.py @@ -590,3 +590,51 @@ def normal_decode_set_metadata( num_warps=block_cols // 32, num_stages=3, ) + + +@triton.jit +def q8kv8_topk_length_backscan_kernel( + indices_ptr, + out_ptr, + stride_row, + topk, + BLOCK: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + base = indices_ptr + row * stride_row + off = topk + length = 1 + found = 0 + while (found == 0) & (off > 0): + off -= BLOCK + idx = off + tl.arange(0, BLOCK) + vals = tl.load(base + idx) + pos = tl.max(tl.where(vals >= 0, idx, -1), axis=0) + found = tl.where(pos >= 0, 1, found) + length = tl.where(pos >= 0, pos + 1, length) + tl.store(out_ptr + row, length) + + +def q8kv8_topk_length_from_indices(indices: torch.Tensor) -> torch.Tensor: + """Per-row valid-topk count = last non-negative position + 1 (min 1). + + ``indices``: [s_q, topk] int32 topk output whose pad slots are -1. + Backward block scan per row: the loop exits at the first block holding a + valid entry, so the cost is proportional to the trailing pad run — one + block (~topk/4 elements) for rows with a full topk, which dominate long + contexts. Semantics match the unfused ``(indices >= 0) * ramp).amax`` + derivation exactly, including all-pad rows (length 1: one pad-only block + keeps the kernel on its clamp+mask path, contributing zero). + """ + s_q, topk = indices.shape + assert indices.dtype == torch.int32 and indices.stride(1) == 1 + out = torch.empty(s_q, dtype=torch.int32, device=indices.device) + block = 512 if topk % 512 == 0 else (256 if topk % 256 == 0 else 128) + q8kv8_topk_length_backscan_kernel[(s_q,)]( + indices, + out, + indices.stride(0), + topk, + BLOCK=block, + ) + return out diff --git a/python/sglang/srt/layers/attention/minimax_sparse_ops/gluon_prefill.py b/python/sglang/kernels/ops/attention/minimax_sparse/prefill/gluon_prefill.py similarity index 100% rename from python/sglang/srt/layers/attention/minimax_sparse_ops/gluon_prefill.py rename to python/sglang/kernels/ops/attention/minimax_sparse/prefill/gluon_prefill.py diff --git a/python/sglang/kernels/ops/attention/mla_qk_prep.py b/python/sglang/kernels/ops/attention/mla_qk_prep.py new file mode 100644 index 0000000000..e2002a2fc4 --- /dev/null +++ b/python/sglang/kernels/ops/attention/mla_qk_prep.py @@ -0,0 +1,554 @@ +import torch +import triton +import triton.language as tl + + +@triton.jit +def concat_and_cast_mha_k_kernel( + k_ptr, + k_nope_ptr, + k_rope_ptr, + head_cnt: tl.constexpr, + k_stride0: tl.constexpr, + k_stride1: tl.constexpr, + nope_stride0: tl.constexpr, + nope_stride1: tl.constexpr, + rope_stride0: tl.constexpr, + nope_dim: tl.constexpr, + rope_dim: tl.constexpr, +): + pid_loc = tl.program_id(0) + head_range = tl.arange(0, head_cnt) + + k_head_ptr = k_ptr + pid_loc * k_stride0 + head_range[:, None] * k_stride1 + + nope_offs = tl.arange(0, nope_dim) + + src_nope_ptr = ( + k_nope_ptr + + pid_loc * nope_stride0 + + head_range[:, None] * nope_stride1 + + nope_offs[None, :] + ) + dst_nope_ptr = k_head_ptr + nope_offs[None, :] + + src_nope = tl.load(src_nope_ptr) + tl.store(dst_nope_ptr, src_nope) + + rope_offs = tl.arange(0, rope_dim) + src_rope_ptr = k_rope_ptr + pid_loc * rope_stride0 + rope_offs[None, :] + dst_rope_ptr = k_head_ptr + nope_dim + rope_offs[None, :] + src_rope = tl.load(src_rope_ptr) + tl.store(dst_rope_ptr, src_rope) + + +@triton.jit +def concat_and_cast_mha_k_pad_kernel( + k_ptr, + k_nope_ptr, + k_rope_ptr, + head_cnt: tl.constexpr, + k_stride0: tl.constexpr, + k_stride1: tl.constexpr, + nope_stride0: tl.constexpr, + nope_stride1: tl.constexpr, + rope_stride0: tl.constexpr, + nope_dim: tl.constexpr, + rope_dim: tl.constexpr, + HEAD_BLOCK: tl.constexpr, +): + """ + concat_and_cast_mha_k_kernel for a head count that is not a power of two. + tl.arange needs a power-of-two extent, so walk HEAD_BLOCK == next_power_of_2(head_cnt) + lanes and mask the tail. + """ + pid_loc = tl.program_id(0) + head_range = tl.arange(0, HEAD_BLOCK) + head_mask = head_range < head_cnt + + k_head_ptr = k_ptr + pid_loc * k_stride0 + head_range[:, None] * k_stride1 + + nope_offs = tl.arange(0, nope_dim) + + src_nope_ptr = ( + k_nope_ptr + + pid_loc * nope_stride0 + + head_range[:, None] * nope_stride1 + + nope_offs[None, :] + ) + dst_nope_ptr = k_head_ptr + nope_offs[None, :] + + nope_mask = tl.broadcast_to(head_mask[:, None], (HEAD_BLOCK, nope_dim)) + src_nope = tl.load(src_nope_ptr, mask=nope_mask, other=0) + tl.store(dst_nope_ptr, src_nope, mask=nope_mask) + + rope_offs = tl.arange(0, rope_dim) + src_rope_ptr = k_rope_ptr + pid_loc * rope_stride0 + rope_offs[None, :] + dst_rope_ptr = k_head_ptr + nope_dim + rope_offs[None, :] + rope_mask = tl.broadcast_to(head_mask[:, None], (HEAD_BLOCK, rope_dim)) + src_rope = tl.load(src_rope_ptr) + tl.store( + dst_rope_ptr, + tl.broadcast_to(src_rope, (HEAD_BLOCK, rope_dim)), + mask=rope_mask, + ) + + +def concat_and_cast_mha_k_triton( + k: torch.Tensor, + k_nope: torch.Tensor, + k_rope: torch.Tensor, +): + # The source data type will be implicitly converted to the target data type. + assert len(k.shape) == 3 and len(k_nope.shape) == 3 and len(k_rope.shape) == 3, ( + f"shape should be 3d, but got {k.shape=}, {k_nope.shape=}, {k_rope.shape=}" + ) + assert k.shape[0] == k_nope.shape[0] and k.shape[0] == k_rope.shape[0], ( + f"invalid shape, got {k.shape=}, {k_nope.shape=}, {k_rope.shape=}" + ) + assert k.shape[1] == k_nope.shape[1] and 1 == k_rope.shape[1], ( + f"invalid shape, got {k.shape=}, {k_nope.shape=}, {k_rope.shape=}" + ) + assert k.shape[-1] == k_nope.shape[-1] + k_rope.shape[-1], ( + f"invalid shape, got {k.shape=}, {k_nope.shape=}, {k_rope.shape=}" + ) + + nope_dim = k_nope.shape[-1] + rope_dim = k_rope.shape[-1] + grid = (k.shape[0],) + + head_cnt = k.shape[1] + if head_cnt != triton.next_power_of_2(head_cnt): + # Not the power of two concat_and_cast_mha_k_kernel's tl.arange needs; + # Kimi-K3's 96 heads give 12 at TP8. + concat_and_cast_mha_k_pad_kernel[grid]( + k, + k_nope, + k_rope, + head_cnt, + k.stride(0), + k.stride(1), + k_nope.stride(0), + k_nope.stride(1), + k_rope.stride(0), + nope_dim, + rope_dim, + HEAD_BLOCK=triton.next_power_of_2(head_cnt), + ) + return + + concat_and_cast_mha_k_kernel[grid]( + k, + k_nope, + k_rope, + k.shape[1], + k.stride(0), + k.stride(1), + k_nope.stride(0), + k_nope.stride(1), + k_rope.stride(0), + nope_dim, + rope_dim, + ) + + +@triton.jit +def concat_and_cast_q_fp8_pad_kernel( + qpad_ptr, # [num_tokens, pad_heads, NOPE+ROPE] fp8 (dst; only [:, :H, :] written) + q_nope_ptr, # [num_tokens, H, NOPE] bf16 + q_rope_ptr, # [num_tokens, H, ROPE] bf16 + qpad_s0, + qpad_s1, + nope_s0, + nope_s1, + rope_s0, + rope_s1, + H: tl.constexpr, + NOPE: tl.constexpr, + ROPE: tl.constexpr, +): + # One program per token: write the H active heads of the padded fp8 q buffer, + # fusing the bf16->fp8 cast (on store) with the nope/rope concat. Bit-exact vs the + # two strided copy_() it replaces; ~3.7x faster because copy_ into the + # 64-head-padded buffer is strided (~4.5x off memory-bound). Strides are passed in, + # so q_nope/q_rope may be views of a [T, H, NOPE+ROPE] q (head-stride != last-dim). + pid = tl.program_id(0) + hr = tl.arange(0, H) + qpad_head = qpad_ptr + pid * qpad_s0 + hr[:, None] * qpad_s1 + no = tl.arange(0, NOPE) + src_n = tl.load(q_nope_ptr + pid * nope_s0 + hr[:, None] * nope_s1 + no[None, :]) + tl.store(qpad_head + no[None, :], src_n) + # NoPE models pass ROPE=0; tl.arange rejects empty ranges, so skip statically. + if ROPE > 0: + ro = tl.arange(0, ROPE) + src_r = tl.load( + q_rope_ptr + pid * rope_s0 + hr[:, None] * rope_s1 + ro[None, :] + ) + tl.store(qpad_head + NOPE + ro[None, :], src_r) + + +def concat_and_cast_q_fp8_pad(q_fp8_pad, q_nope, q_rope, num_heads): + """fused bf16->fp8 concat-cast of q_nope/q_rope into the active + [:, :num_heads, :] slice of the padded fp8 q buffer. Bit-exact replacement for the + two strided converting copy_() in the Q8KV8 prefill q-prep, ~3.7x faster. Requires + num_heads / nope_dim / rope_dim to be powers of two (always true for DeepSeek: 128 + heads / any TP, 512 nope, 64 rope); rope_dim may also be 0 for NoPE models.""" + num_tokens = q_nope.shape[0] + nope_dim = q_nope.shape[-1] + rope_dim = q_rope.shape[-1] + concat_and_cast_q_fp8_pad_kernel[(num_tokens,)]( + q_fp8_pad, + q_nope, + q_rope, + q_fp8_pad.stride(0), + q_fp8_pad.stride(1), + q_nope.stride(0), + q_nope.stride(1), + q_rope.stride(0), + q_rope.stride(1), + H=num_heads, + NOPE=nope_dim, + ROPE=rope_dim, + ) + + +@triton.jit +def absorbed_bmm_concat_cast_q_fp8_kernel( + qout_ptr, # [num_tokens, pad_heads, N+ROPE] fp8 (dst; only [:, :H, :] written) + a_ptr, # q_nope (pre-absorb) [num_tokens, H, K] bf16 + b_ptr, # w_kc [H, K, N] bf16 (any strides; typically N-major) + rope_ptr, # q_rope (post-rope) [num_tokens, H, ROPE] bf16 + T, # num_tokens (runtime; masked) + qout_s0, + qout_s1, + a_s0, + a_s1, + b_s0, + b_s1, + b_s2, + rope_s0, + rope_s1, + K: tl.constexpr, + N: tl.constexpr, + ROPE: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + K_MODE: tl.constexpr, +): + # One program per (token-block, head): q_out[m, h, :N] = fp8(bf16(fp32( + # q_nope[m, h, :K] @ w_kc[h, :K, :N]))) and q_out[m, h, N:] = + # fp8(q_rope[m, h, :ROPE]). This makes q "born fp8": the absorbed bmm, + # the nope/rope concat, and the bf16->fp8 cast collapse into one kernel, + # so neither the bf16 q_nope_out ([H, T, N], written by cublas and re-read + # by the concat-cast) nor the standalone concat-cast launch exist anymore. + # + # K handling (K_MODE selects the codegen for the nope-gemm K dimension; + # every mode keeps the same fp32-accumulator -> bf16 -> fp8 epilogue): + # 0 "single": BLOCK_K == K. Preload the whole [BLOCK_M, K] a-tile once, + # one tl.dot per N-block — identical codegen to the original + # power-of-2-only kernel (DeepSeek K=128). For non-power-of-2 K this + # only compiles if the Triton build allows non-power-of-2 tl.arange + # (Triton <= 3.5.x does NOT: "arange's range must be a power of 2"). + # 1 "loop": split-K loop, K % BLOCK_K == 0, BLOCK_K power of 2 >= 16 + # (e.g. K=192 with BLOCK_K=64 -> 3 iterations). The a-tile is + # re-loaded per (N-block, K-block); slices are L1/L2-resident after + # the first N-block, but the load/dot interleave costs bandwidth + # (measured ~1372 GB/s vs ~2x that for mode 0 at K=128). + # 2 "two_dot": K = BLOCK_K + (K - BLOCK_K), both power-of-2 halves + # (192 = 128 + 64). Both a-tiles preload once before the N-loop; + # each N-block issues two chained tl.dot into one fp32 accumulator. + # No K-loop, no a re-reads — the direct generalization of mode 0. + # 3 "three_dot": K = 3 * BLOCK_K (192 = 3 x 64). Same as mode 2 with + # three preloaded a-tiles / three chained tl.dot per N-block; the + # hoisted-loads analogue of mode 1 (identical fp32 add order). + # 4 "pad": BLOCK_K = next_pow2(K) > K, k-masked loads (zero fill). + # Single tl.dot per N-block; the padded zeros are exact fp32 + # additive identities so the result matches a K-wide single dot, + # at the cost of BLOCK_K/K (e.g. 256/192 = 1.33x) extra MMA work. + # + # Rounding contract: the fp32 accumulator is rounded to bf16 first (the + # same output rounding stage as the cublas bf16 bmm) and then converted + # bf16->fp8 by the same implicit-store conversion the fused concat-cast + # kernel uses. The split-K accumulator stays fp32 across all K-blocks, + # so the rounding stages are identical in both layouts. The rope half is + # a bit-exact copy of that kernel (loads the post-rope bf16, converts on + # store). The nope half is NOT guaranteed bit-exact vs the default path: + # tl.dot accumulates fp32 in a different order than cublas, so last-ulp + # fp32 differences can occasionally flip the bf16 (and hence fp8) + # rounding. + pid_m = tl.program_id(0) + h = tl.program_id(1) + offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + m_mask = offs_m < T + # token-row offsets in int64: T * row-stride can exceed int32 (e.g. 128 + # heads x 576 dims x tens of thousands of tokens). + offs_m64 = offs_m.to(tl.int64) + qout_head = qout_ptr + offs_m64[:, None] * qout_s0 + h * qout_s1 + a_row = a_ptr + offs_m64[:, None] * a_s0 + h * a_s1 + b_head = b_ptr + h * b_s0 + if K_MODE == 0: + # single-dot path (original kernel): BLOCK_K == K, preload a once. + offs_k = tl.arange(0, BLOCK_K) + a = tl.load(a_row + offs_k[None, :], mask=m_mask[:, None], other=0.0) + for nb in tl.static_range(N // BLOCK_N): + offs_n = nb * BLOCK_N + tl.arange(0, BLOCK_N) + b = tl.load(b_head + offs_k[:, None] * b_s1 + offs_n[None, :] * b_s2) + acc = tl.dot(a, b) # fp32 accumulator + val = acc.to(tl.bfloat16) # cublas-equivalent bf16 output rounding + # implicit bf16 -> fp8 conversion on store (same as the concat-cast) + tl.store(qout_head + offs_n[None, :], val, mask=m_mask[:, None]) + elif K_MODE == 2: + # two-dot preload: K split as BLOCK_K + (K - BLOCK_K), no K-loop. + offs_k0 = tl.arange(0, BLOCK_K) + offs_k1 = BLOCK_K + tl.arange(0, K - BLOCK_K) + a0 = tl.load(a_row + offs_k0[None, :], mask=m_mask[:, None], other=0.0) + a1 = tl.load(a_row + offs_k1[None, :], mask=m_mask[:, None], other=0.0) + for nb in tl.static_range(N // BLOCK_N): + offs_n = nb * BLOCK_N + tl.arange(0, BLOCK_N) + b0 = tl.load(b_head + offs_k0[:, None] * b_s1 + offs_n[None, :] * b_s2) + b1 = tl.load(b_head + offs_k1[:, None] * b_s1 + offs_n[None, :] * b_s2) + acc = tl.dot(a0, b0) # fp32 accumulator + acc = tl.dot(a1, b1, acc) # chained: stays fp32 across both dots + val = acc.to(tl.bfloat16) # cublas-equivalent bf16 output rounding + # implicit bf16 -> fp8 conversion on store (same as the concat-cast) + tl.store(qout_head + offs_n[None, :], val, mask=m_mask[:, None]) + elif K_MODE == 3: + # three-dot preload: K = 3 * BLOCK_K, a-tiles hoisted out of the N-loop. + offs_k0 = tl.arange(0, BLOCK_K) + offs_k1 = BLOCK_K + offs_k0 + offs_k2 = 2 * BLOCK_K + offs_k0 + a0 = tl.load(a_row + offs_k0[None, :], mask=m_mask[:, None], other=0.0) + a1 = tl.load(a_row + offs_k1[None, :], mask=m_mask[:, None], other=0.0) + a2 = tl.load(a_row + offs_k2[None, :], mask=m_mask[:, None], other=0.0) + for nb in tl.static_range(N // BLOCK_N): + offs_n = nb * BLOCK_N + tl.arange(0, BLOCK_N) + b0 = tl.load(b_head + offs_k0[:, None] * b_s1 + offs_n[None, :] * b_s2) + b1 = tl.load(b_head + offs_k1[:, None] * b_s1 + offs_n[None, :] * b_s2) + b2 = tl.load(b_head + offs_k2[:, None] * b_s1 + offs_n[None, :] * b_s2) + acc = tl.dot(a0, b0) # fp32 accumulator + acc = tl.dot(a1, b1, acc) + acc = tl.dot(a2, b2, acc) # same fp32 add order as the K_MODE=1 loop + val = acc.to(tl.bfloat16) # cublas-equivalent bf16 output rounding + # implicit bf16 -> fp8 conversion on store (same as the concat-cast) + tl.store(qout_head + offs_n[None, :], val, mask=m_mask[:, None]) + elif K_MODE == 4: + # padded single dot: BLOCK_K = next_pow2(K), zero-fill the k tail. + offs_k = tl.arange(0, BLOCK_K) + k_mask = offs_k < K + a = tl.load( + a_row + offs_k[None, :], + mask=m_mask[:, None] & k_mask[None, :], + other=0.0, + ) + for nb in tl.static_range(N // BLOCK_N): + offs_n = nb * BLOCK_N + tl.arange(0, BLOCK_N) + b = tl.load( + b_head + offs_k[:, None] * b_s1 + offs_n[None, :] * b_s2, + mask=k_mask[:, None], + other=0.0, + ) + acc = tl.dot(a, b) # fp32 accumulator (padded zeros add exactly 0) + val = acc.to(tl.bfloat16) # cublas-equivalent bf16 output rounding + # implicit bf16 -> fp8 conversion on store (same as the concat-cast) + tl.store(qout_head + offs_n[None, :], val, mask=m_mask[:, None]) + else: + # K_MODE == 1: split-K loop (K % BLOCK_K == 0, e.g. K=192, BLOCK_K=64). + for nb in tl.static_range(N // BLOCK_N): + offs_n = nb * BLOCK_N + tl.arange(0, BLOCK_N) + acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for kb in tl.static_range(K // BLOCK_K): + offs_k = kb * BLOCK_K + tl.arange(0, BLOCK_K) + a = tl.load(a_row + offs_k[None, :], mask=m_mask[:, None], other=0.0) + b = tl.load(b_head + offs_k[:, None] * b_s1 + offs_n[None, :] * b_s2) + acc = tl.dot(a, b, acc) # fp32 accumulator across K-blocks + val = acc.to(tl.bfloat16) # cublas-equivalent bf16 output rounding + # implicit bf16 -> fp8 conversion on store (same as the concat-cast) + tl.store(qout_head + offs_n[None, :], val, mask=m_mask[:, None]) + offs_r = tl.arange(0, ROPE) + r = tl.load( + rope_ptr + offs_m64[:, None] * rope_s0 + h * rope_s1 + offs_r[None, :], + mask=m_mask[:, None], + other=0.0, + ) + tl.store(qout_head + N + offs_r[None, :], r, mask=m_mask[:, None]) + + +# Non-power-of-2-K variant used by variant="auto" (power-of-2 K always takes +# the single-dot fast path). Set to the winner of the K=192 A/B in +# benchmark/kernels/deepseek/benchmark_q8kv8_q_prep.py; "loop" = the pre-A/B split-K behavior. +_AUTO_NONPOW2_VARIANT = "two_dot" + + +def _qprep_env_variant(): + from sglang.srt.environ import envs + + return envs.SGLANG_OPT_Q8KV8_QPREP_VARIANT.get() + + +# Resolved once at import (matches the module-constant style above). "auto" +# keeps the per-K dispatch; "cuda" routes every shape to the hand-written +# SM90 WGMMA kernel (bitwise-identical to two_dot; 1.16-1.38x faster). +_ENV_QPREP_VARIANT = None + + +def absorbed_bmm_concat_cast_q_fp8( + q_fp8_pad: "torch.Tensor", + q_nope: "torch.Tensor", + w_kc: "torch.Tensor", + q_rope: "torch.Tensor", + num_heads: int, + block_m: int = 128, + block_n: int = 64, + variant: str = "auto", + block_k: int = 0, + num_warps: int = 8, + num_stages: int = 0, +): + """Fused absorbed-q bmm + nope/rope concat + bf16->fp8 cast ("born fp8" q). + + Replaces ``torch.bmm(q_nope.transpose(0, 1), w_kc).transpose(0, 1)`` + followed by ``concat_and_cast_q_fp8_pad`` on the Q8KV8 sparse-prefill + path, writing the active ``[:, :num_heads, :]`` slice of the padded fp8 q + buffer directly. Inputs: + + * ``q_fp8_pad``: [num_tokens, pad_heads, N + ROPE] fp8_e4m3 destination. + * ``q_nope``: [num_tokens, H, K] bf16 pre-absorb q (strided views OK). + * ``w_kc``: [H, K, N] bf16 absorbed weight (any strides). + * ``q_rope``: [num_tokens, H, ROPE] bf16 post-rope q (strided views OK). + + The rope half is bit-exact vs ``concat_and_cast_q_fp8_pad``. The nope + half keeps the same rounding stages (fp32 accum -> bf16 -> fp8) but a + different fp32 accumulation order than cublas, so it is near- but not + guaranteed bit-exact; keep this path behind + ``SGLANG_ENABLE_DSA_Q8KV8_BORN_FP8_Q``. + + K (``qk_nope_head_dim``) supports any multiple of 16 in [16, 256]. + Power-of-2 K (DeepSeek 128) always takes the preload-once + single-``tl.dot`` path. For other K (GLM 192), ``variant`` selects the + K-dimension codegen (every variant keeps the identical fp32 -> bf16 -> + fp8 epilogue): + + * ``"auto"``: the current production choice (see + ``_AUTO_NONPOW2_VARIANT``). + * ``"loop"``: split-K accumulator loop, ``BLOCK_K`` = ``block_k`` or the + largest power-of-2 divisor of K capped at 128 (192 -> 64 x 3). + * ``"two_dot"``: preload a as two power-of-2 tiles (192 = 128 + 64), two + chained ``tl.dot`` per N-block, no K-loop. + * ``"three_dot"``: preload a as three K/3 tiles (192 = 3 x 64), three + chained ``tl.dot`` per N-block; same fp32 add order as ``"loop"``. + * ``"pad"``: single ``tl.dot`` with ``BLOCK_K`` = next_pow2(K) (192 -> + 256) and zero-masked k tails. + * ``"single_k"``: single ``tl.dot`` with ``BLOCK_K`` == K. Only + compiles if the Triton build supports non-power-of-2 ``tl.arange`` + (Triton <= 3.5.x raises "arange's range must be a power of 2"). + ``block_m`` / ``block_n`` / ``num_warps`` / ``num_stages`` are tuning + knobs for the microbench sweep (0 = Triton default for ``num_stages``). + """ + num_tokens, _, k_dim = q_nope.shape + n_dim = w_kc.shape[-1] + rope_dim = q_rope.shape[-1] + assert q_fp8_pad.dtype == torch.float8_e4m3fn + assert q_nope.dtype == torch.bfloat16 and w_kc.dtype == torch.bfloat16 + assert q_rope.dtype == torch.bfloat16 + assert q_nope.shape[1] == num_heads and q_rope.shape[1] == num_heads + assert w_kc.shape[0] == num_heads and w_kc.shape[1] == k_dim + assert q_fp8_pad.shape[0] >= num_tokens and q_fp8_pad.shape[1] >= num_heads + assert q_fp8_pad.shape[2] == n_dim + rope_dim + # tl.arange / tl.dot constraints + assert k_dim % 16 == 0 and 16 <= k_dim <= 256, ( + "K must be a multiple of 16 in [16, 256]" + ) + assert (rope_dim & (rope_dim - 1)) == 0, "ROPE must be a power of two" + assert n_dim % block_n == 0, "N must be a multiple of block_n" + assert q_nope.stride(2) == 1 and q_rope.stride(2) == 1 + assert q_fp8_pad.stride(2) == 1 + # Env override for production dispatch (SGLANG_OPT_Q8KV8_QPREP_VARIANT): + # "auto" (default) keeps the per-K Triton dispatch; "cuda" routes every + # shape to the WGMMA kernel below. + global _ENV_QPREP_VARIANT + if _ENV_QPREP_VARIANT is None: + _ENV_QPREP_VARIANT = _qprep_env_variant() + if variant == "auto" and _ENV_QPREP_VARIANT != "auto": + variant = _ENV_QPREP_VARIANT + # Hand-written SM90 WGMMA kernel (opt-in only; "auto" never routes here). + # Same fp32 -> bf16 -> fp8 epilogue; bitwise identical to "two_dot" on + # SM90. Requires K in {128, 192} and the production N-major w_kc layout + # (see the wrapper's asserts); the Triton variants remain the + # general-strides fallback. + _valid = ("auto", "cuda", "loop", "two_dot", "three_dot", "pad", "single_k") + if variant not in _valid: + raise ValueError( + f"unknown q-prep variant {variant!r} " + f"(SGLANG_OPT_Q8KV8_QPREP_VARIANT); valid: {_valid}" + ) + if variant == "cuda": + from sglang.kernels.ops.attention.qprep_bf16_fp8_sm90 import q8kv8_qprep_fwd + + q8kv8_qprep_fwd(q_fp8_pad, q_nope, w_kc, q_rope, num_heads) + return + # Resolve (K_MODE, BLOCK_K) from the variant; see the kernel's K-handling + # comment for what each mode compiles to. + if k_dim & (k_dim - 1) == 0: + # power-of-2 K: every variant collapses to the single-dot fast path. + k_mode, blk_k = 0, k_dim + else: + v = _AUTO_NONPOW2_VARIANT if variant == "auto" else variant + if v == "loop": + # Largest power-of-2 divisor of K, capped at 128 (K % 16 == 0 + # makes this >= 16), unless the caller pinned block_k. + blk_k = block_k or min(k_dim & -k_dim, 128) + assert k_dim % blk_k == 0 and blk_k & (blk_k - 1) == 0 and blk_k >= 16, ( + "loop needs BLOCK_K a power-of-2 divisor of K >= 16" + ) + k_mode = 1 + elif v == "two_dot": + blk_k = 1 << (k_dim.bit_length() - 1) # largest power of 2 < K + k1 = k_dim - blk_k + assert k1 & (k1 - 1) == 0 and k1 >= 16, ( + "two_dot needs K = pow2 + pow2 with both halves >= 16" + ) + k_mode = 2 + elif v == "three_dot": + blk_k = k_dim // 3 + assert k_dim % 3 == 0 and blk_k & (blk_k - 1) == 0 and blk_k >= 16, ( + "three_dot needs K = 3 * pow2 with pow2 >= 16" + ) + k_mode = 3 + elif v == "pad": + blk_k = 1 << k_dim.bit_length() # next power of 2 above K + k_mode = 4 + elif v == "single_k": + # Non-power-of-2 BLOCK_K == K: compiles only on Triton builds + # that allow non-power-of-2 tl.arange (not 3.5.x). + blk_k = k_dim + k_mode = 0 + else: + raise ValueError(f"unknown absorbed-bmm K variant: {variant!r}") + extra = {"num_stages": num_stages} if num_stages else {} + grid = (triton.cdiv(num_tokens, block_m), num_heads) + absorbed_bmm_concat_cast_q_fp8_kernel[grid]( + q_fp8_pad, + q_nope, + w_kc, + q_rope, + num_tokens, + q_fp8_pad.stride(0), + q_fp8_pad.stride(1), + q_nope.stride(0), + q_nope.stride(1), + w_kc.stride(0), + w_kc.stride(1), + w_kc.stride(2), + q_rope.stride(0), + q_rope.stride(1), + K=k_dim, + N=n_dim, + ROPE=rope_dim, + BLOCK_M=block_m, + BLOCK_N=block_n, + BLOCK_K=blk_k, + K_MODE=k_mode, + num_warps=num_warps, + **extra, + ) diff --git a/python/sglang/kernels/ops/muse_spark/fused_qk_norm.py b/python/sglang/kernels/ops/attention/muse_spark_qk_norm_rope_store.py similarity index 98% rename from python/sglang/kernels/ops/muse_spark/fused_qk_norm.py rename to python/sglang/kernels/ops/attention/muse_spark_qk_norm_rope_store.py index 966c847e24..235eecae1b 100644 --- a/python/sglang/kernels/ops/muse_spark/fused_qk_norm.py +++ b/python/sglang/kernels/ops/attention/muse_spark_qk_norm_rope_store.py @@ -143,7 +143,7 @@ def _triton_config(n_rows: int) -> tuple[int, int]: class FusedQKNorm(BaseFusedOp): """Per-head RMS norm of the Q and K slices of a QKV projection output.""" - op = "muse_spark.qk_norm" + op = "attention.muse_spark_qk_norm_rope_store" priority = (KernelBackend.TRITON, KernelBackend.HELION) capabilities = { KernelBackend.HELION: {CapabilityRequirement.CUDA}, @@ -375,10 +375,10 @@ class FusedQKNorm(BaseFusedOp): cos_sin_cache: Optional[torch.Tensor] = None, kv_store: Optional[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = None, ) -> bool: - from sglang.kernels.ops.muse_spark.helion_runtime import helion_supported + from sglang.kernels.jit.utils.helion_runtime import helion_supported return ( - helion_supported("qk_norm", qkv) + helion_supported(qkv) and (kv_store is None or kv_store[0].dtype == qkv.dtype) and qkv.shape[0] > 0 and qkv.dtype == torch.bfloat16 @@ -405,13 +405,13 @@ class FusedQKNorm(BaseFusedOp): ) -> tuple[torch.Tensor, torch.Tensor]: """Run Helion behind the existing fused operation boundary.""" - from sglang.kernels.ops.muse_spark.helion_qk_norm import ( + from sglang.kernels.jit.utils.helion_runtime import ( + run_pinned_helion, + ) + from sglang.kernels.ops.attention.muse_spark_qk_norm_rope_store_helion import ( QK_NORM_CONFIGS, _qk_norm_helion_kernel, ) - from sglang.kernels.ops.muse_spark.helion_runtime import ( - run_pinned_helion, - ) if kv_store is None: k_cache = v_cache = cache_loc = None diff --git a/python/sglang/kernels/ops/muse_spark/helion_qk_norm.py b/python/sglang/kernels/ops/attention/muse_spark_qk_norm_rope_store_helion.py similarity index 98% rename from python/sglang/kernels/ops/muse_spark/helion_qk_norm.py rename to python/sglang/kernels/ops/attention/muse_spark_qk_norm_rope_store_helion.py index fc8c269da7..1a2e39e214 100644 --- a/python/sglang/kernels/ops/muse_spark/helion_qk_norm.py +++ b/python/sglang/kernels/ops/attention/muse_spark_qk_norm_rope_store_helion.py @@ -6,7 +6,7 @@ import helion import helion.language as hl import torch -from sglang.kernels.ops.muse_spark.helion_runtime import ( +from sglang.kernels.jit.utils.helion_runtime import ( token_count_specialization_key, ) diff --git a/python/sglang/kernels/ops/attention/qprep_bf16_fp8_sm90.py b/python/sglang/kernels/ops/attention/qprep_bf16_fp8_sm90.py index 66a7b8589f..636766639a 100644 --- a/python/sglang/kernels/ops/attention/qprep_bf16_fp8_sm90.py +++ b/python/sglang/kernels/ops/attention/qprep_bf16_fp8_sm90.py @@ -3,7 +3,7 @@ Fuses the per-head absorbed-q bmm (q_nope [T, H, K] bf16 x w_kc [H, K, N] bf16, fp32 accumulate), the nope/rope concat, and the bf16 -> fp8_e4m3 cast into one hand-written WGMMA kernel. CUDA replacement for the Triton -``absorbed_bmm_concat_cast_q_fp8`` (triton_ops/cache_ops.py) with the +``absorbed_bmm_concat_cast_q_fp8`` (attention/mla_qk_prep.py) with the identical fp32 -> bf16 -> fp8 epilogue rounding chain; the rope half is bit-exact vs ``concat_and_cast_q_fp8_pad``. """ diff --git a/python/sglang/kernels/ops/attention/sparse_mla_q8kv8_prefill_sm90.py b/python/sglang/kernels/ops/attention/sparse_mla_q8kv8_prefill_sm90.py index 312febb67e..9d64b5e1d0 100644 --- a/python/sglang/kernels/ops/attention/sparse_mla_q8kv8_prefill_sm90.py +++ b/python/sglang/kernels/ops/attention/sparse_mla_q8kv8_prefill_sm90.py @@ -12,6 +12,11 @@ import torch from sglang.kernels.jit.utils import cache_once, load_jit from sglang.kernels.kernel_api_logging import debug_kernel_api +from sglang.kernels.ops.attention.dsv4.dequant_k_cache import ( + DIM_NOPE, + DIM_ROPE, + fp8_dtype, +) from sglang.srt.utils.custom_op import register_custom_op if TYPE_CHECKING: @@ -524,3 +529,71 @@ def sparse_mla_q8kv8_prefill_fwd( ) return out, max_logits, lse + + +def q8kv8_padded_num_heads(num_heads: int) -> int: + """Return a Q-head count supported by the SM90 Q8KV8 kernel.""" + if num_heads <= 0: + raise ValueError(f"num_heads must be positive, got {num_heads}") + if num_heads <= 64: + return 64 + if num_heads <= 128: + return 128 + raise ValueError( + "DeepSeek-V4 Q8KV8 sparse prefill supports at most 128 local " + f"query heads, got {num_heads}" + ) + + +def cast_q_fp8_for_q8kv8_prefill( + q: torch.Tensor, + padded_num_heads: Optional[int] = None, + out: Optional[torch.Tensor] = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Cast DeepSeek-V4 sparse-prefill Q to the Q8KV8 kernel format. + + The incoming Q is the model-produced BF16/FP16 tensor already shaped as + ``(num_tokens, num_heads, 512)`` after removing the singleton MQA axis. + + The SM90 kernel processes query heads in 64-head blocks. Tensor parallelism + commonly leaves fewer than 64 local heads, so the active heads are copied + into a zero-padded 64/128-head FP8 tensor. + """ + assert q.ndim == 3 + assert q.shape[-1] == DIM_NOPE + DIM_ROPE + + num_tokens, num_heads, head_dim = q.shape + if padded_num_heads is None: + padded_num_heads = q8kv8_padded_num_heads(num_heads) + + if padded_num_heads not in (64, 128) or padded_num_heads < num_heads: + raise ValueError( + f"invalid padded_num_heads={padded_num_heads} for num_heads={num_heads}" + ) + + expected_shape = (num_tokens, padded_num_heads, head_dim) + + if out is None: + q_fp8 = torch.zeros( + expected_shape, + dtype=fp8_dtype, + device=q.device, + ) + else: + if ( + out.shape != expected_shape + or out.dtype != fp8_dtype + or out.device != q.device + ): + raise ValueError( + "Q8KV8 Q output must have shape/dtype/device " + f"{expected_shape}/{fp8_dtype}/{q.device}, got " + f"{tuple(out.shape)}/{out.dtype}/{out.device}" + ) + q_fp8 = out + if padded_num_heads > num_heads: + q_fp8[:, num_heads:].zero_() + + q_fp8[:, :num_heads].copy_(q) + q_scale = torch.ones((), dtype=torch.float32, device=q.device) + return q_fp8, q_scale diff --git a/python/sglang/kernels/ops/kvcache/trtllm_mha_graph_metadata.py b/python/sglang/kernels/ops/attention/trtllm_mha_graph_metadata.py similarity index 100% rename from python/sglang/kernels/ops/kvcache/trtllm_mha_graph_metadata.py rename to python/sglang/kernels/ops/attention/trtllm_mha_graph_metadata.py diff --git a/python/sglang/kernels/ops/kvcache/trtllm_mha_page_table.py b/python/sglang/kernels/ops/attention/trtllm_mha_page_table.py similarity index 100% rename from python/sglang/kernels/ops/kvcache/trtllm_mha_page_table.py rename to python/sglang/kernels/ops/attention/trtllm_mha_page_table.py diff --git a/python/sglang/kernels/ops/attention/utils.py b/python/sglang/kernels/ops/attention/utils.py index 2bede6eb97..fe54e7fd60 100644 --- a/python/sglang/kernels/ops/attention/utils.py +++ b/python/sglang/kernels/ops/attention/utils.py @@ -1,59 +1,6 @@ import torch -import triton -import triton.language as tl from sglang.kernels.jit.utils import is_arch_support_pdl -from sglang.kernels.ops.attention.pad import ( - pad_sequence_with_mask as pad_sequence_with_mask, -) -from sglang.kernels.ops.attention.pad import ( - pad_sequence_with_mask_kernel as pad_sequence_with_mask_kernel, -) -from sglang.kernels.ops.attention.pad import ( - seqlens_expand_kernel as seqlens_expand_kernel, -) -from sglang.kernels.ops.attention.pad import ( - seqlens_expand_triton as seqlens_expand_triton, -) -from sglang.kernels.ops.kvcache.cache_ops import ( - concat_and_cast_mha_k_kernel as concat_and_cast_mha_k_kernel, -) -from sglang.kernels.ops.kvcache.cache_ops import ( - concat_and_cast_mha_k_triton as concat_and_cast_mha_k_triton, -) -from sglang.kernels.ops.kvcache.cache_ops import ( - concat_and_cast_q_fp8_pad as concat_and_cast_q_fp8_pad, -) -from sglang.kernels.ops.kvcache.cache_ops import ( - concat_and_cast_q_fp8_pad_kernel as concat_and_cast_q_fp8_pad_kernel, -) -from sglang.kernels.ops.kvcache.cache_ops import ( - launch_reshape_and_cache_flash as launch_reshape_and_cache_flash, -) -from sglang.kernels.ops.kvcache.cache_ops import ( - q8kv8_topk_length_from_indices as q8kv8_topk_length_from_indices, -) -from sglang.kernels.ops.kvcache.cache_ops import ( - reshape_and_cache_flash as reshape_and_cache_flash, -) -from sglang.kernels.ops.kvcache.kv_indices import ( - create_flashinfer_kv_indices_triton as create_flashinfer_kv_indices_triton, -) -from sglang.kernels.ops.kvcache.kv_indices import ( - create_flashmla_kv_indices_triton as create_flashmla_kv_indices_triton, -) -from sglang.kernels.ops.kvcache.kv_indices import ( - get_num_kv_index_blocks_flashmla as get_num_kv_index_blocks_flashmla, -) -from sglang.kernels.ops.kvcache.kv_indices import ( - get_num_page_per_block_flashmla as get_num_page_per_block_flashmla, -) -from sglang.kernels.ops.kvcache.kv_indices import ( - kv_indices_num_token_blocks as kv_indices_num_token_blocks, -) -from sglang.kernels.ops.kvcache.rope_cache import ( - fused_qk_rope_reshape_and_cache as fused_qk_rope_reshape_and_cache, -) from sglang.srt.utils import is_cuda _is_cuda = is_cuda() @@ -214,270 +161,6 @@ def concat_mla_absorb_q_general(q_nope, q_rope): return torch.cat([q_nope, q_rope], dim=-1) -@triton.jit -def reshape_and_cache_shuffle_5d( - key_ptr, - value_ptr, - key_cache_ptr, - value_cache_ptr, - slot_mapping_ptr, - swa_slot_mapping_ptr, - key_stride_token, - value_stride_token, - num_heads, - head_size, - block_size, - X: tl.constexpr, - HEAD_BLOCK: tl.constexpr, - BLOCK_D: tl.constexpr, - HAS_SWA: tl.constexpr, -): - """Scatter per-token (num_tokens, num_heads, head_size) K/V into the - SHUFFLE 5D "vectorized" KV cache layout used by aiter CK - `mha_batch_prefill_func` and aiter `pa_decode_gluon`. - - K cache shape: (num_blocks, num_heads, head_size // X, block_size, X) - V cache shape: (num_blocks, num_heads, block_size // X, head_size, X) - where X = 16 // element_size (=8 for bf16/fp16, =16 for fp8). - block_size must be divisible by X, and head_size must be divisible by X. - - Each program handles one token and a HEAD_BLOCK-wide slice of heads. - """ - token_idx = tl.program_id(0) - head_block_idx = tl.program_id(1) - - slot_idx = tl.load(slot_mapping_ptr + token_idx) - if HAS_SWA: - slot_idx = tl.load(swa_slot_mapping_ptr + slot_idx) - if slot_idx < 0: - return - - block_idx = slot_idx // block_size - slot_in_page = slot_idx % block_size - page_outer = slot_in_page // X - page_inner = slot_in_page % X - - head_idx = head_block_idx * HEAD_BLOCK + tl.arange(0, HEAD_BLOCK) - head_mask = head_idx < num_heads - d = tl.arange(0, BLOCK_D) - d_mask = d < head_size - d_outer = d // X - d_inner = d % X - - src_off = token_idx * key_stride_token + head_idx[:, None] * head_size + d[None, :] - src_mask = head_mask[:, None] & d_mask[None, :] - k = tl.load(key_ptr + src_off, mask=src_mask) - src_off_v = ( - token_idx * value_stride_token + head_idx[:, None] * head_size + d[None, :] - ) - v = tl.load(value_ptr + src_off_v, mask=src_mask) - - layer_stride = num_heads * head_size * block_size - head_stride = head_size * block_size - - k_tgt = ( - block_idx * layer_stride - + head_idx[:, None] * head_stride - + d_outer[None, :] * block_size * X - + slot_in_page * X - + d_inner[None, :] - ) - tl.store(key_cache_ptr + k_tgt, k, mask=src_mask) - - v_tgt = ( - block_idx * layer_stride - + head_idx[:, None] * head_stride - + page_outer * head_size * X - + d[None, :] * X - + page_inner - ) - tl.store(value_cache_ptr + v_tgt, v, mask=src_mask) - - -def launch_reshape_and_cache_shuffle_5d( - key: torch.Tensor, - value: torch.Tensor, - key_cache: torch.Tensor, - value_cache: torch.Tensor, - slot_mapping: torch.Tensor, - swa_slot_mapping=None, -): - """Launcher for reshape_and_cache_shuffle_5d. - - Args: - key/value: (num_tokens, num_heads, head_size) source tensors - key_cache: (num_blocks, num_heads, head_size//X, block_size, X) - value_cache: (num_blocks, num_heads, block_size//X, head_size, X) - slot_mapping: per-token destination slot in [0, num_blocks*block_size) - """ - num_tokens, num_heads, head_size = key.shape - assert value.shape == key.shape, "K/V must share token-major shape" - assert key_cache.dim() == 5 and value_cache.dim() == 5 - num_blocks, kc_H, kc_D_over_X, block_size, X = key_cache.shape - assert kc_H == num_heads and kc_D_over_X * X == head_size - vb_blocks, vc_H, vc_page_over_X, vc_D, vc_X = value_cache.shape - assert ( - vc_H == num_heads - and vc_page_over_X * X == block_size - and vc_D == head_size - and vc_X == X - ) - assert block_size % X == 0 and head_size % X == 0 - - HEAD_BLOCK = min(4, triton.next_power_of_2(num_heads)) - BLOCK_D = triton.next_power_of_2(head_size) - grid = (num_tokens, triton.cdiv(num_heads, HEAD_BLOCK)) - - reshape_and_cache_shuffle_5d[grid]( - key, - value, - key_cache, - value_cache, - slot_mapping, - swa_slot_mapping if swa_slot_mapping is not None else slot_mapping, - key.stride(0), - value.stride(0), - num_heads, - head_size, - block_size, - X=X, - HEAD_BLOCK=HEAD_BLOCK, - BLOCK_D=BLOCK_D, - HAS_SWA=(swa_slot_mapping is not None), - ) - - -@triton.jit -def gather_shuffle_5d_to_linear( - key_cache_ptr, - value_cache_ptr, - key_out_ptr, # (T, num_heads, head_size), store dtype - value_out_ptr, # (T, num_heads, head_size), store dtype - slot_mapping_ptr, # (T,) absolute pool slot id per token - key_out_stride_token, - value_out_stride_token, - num_heads, - head_size, - block_size, - X: tl.constexpr, - HEAD_BLOCK: tl.constexpr, - BLOCK_D: tl.constexpr, -): - """Inverse of :func:`reshape_and_cache_shuffle_5d`. - - Gather one token's K/V from the SHUFFLE 5D paged cache into the - canonical (T, H, D) layout that aiter's ``mha_batch_prefill_func`` - expects in LINEAR mode. Source addressing is identical to the - writer kernel so any bit-exact round-trip is guaranteed. - """ - token_idx = tl.program_id(0) - head_block_idx = tl.program_id(1) - - slot_idx = tl.load(slot_mapping_ptr + token_idx) - - block_idx = slot_idx // block_size - slot_in_page = slot_idx % block_size - page_outer = slot_in_page // X - page_inner = slot_in_page % X - - head_idx = head_block_idx * HEAD_BLOCK + tl.arange(0, HEAD_BLOCK) - head_mask = head_idx < num_heads - d = tl.arange(0, BLOCK_D) - d_mask = d < head_size - d_outer = d // X - d_inner = d % X - - layer_stride = num_heads * head_size * block_size - head_stride = head_size * block_size - - src_mask = head_mask[:, None] & d_mask[None, :] - k_src = ( - block_idx * layer_stride - + head_idx[:, None] * head_stride - + d_outer[None, :] * block_size * X - + slot_in_page * X - + d_inner[None, :] - ) - k = tl.load(key_cache_ptr + k_src, mask=src_mask) - v_src = ( - block_idx * layer_stride - + head_idx[:, None] * head_stride - + page_outer * head_size * X - + d[None, :] * X - + page_inner - ) - v = tl.load(value_cache_ptr + v_src, mask=src_mask) - - dst_k = ( - token_idx * key_out_stride_token + head_idx[:, None] * head_size + d[None, :] - ) - tl.store(key_out_ptr + dst_k, k, mask=src_mask) - dst_v = ( - token_idx * value_out_stride_token + head_idx[:, None] * head_size + d[None, :] - ) - tl.store(value_out_ptr + dst_v, v, mask=src_mask) - - -def launch_gather_shuffle_5d_to_linear( - key_cache: torch.Tensor, - value_cache: torch.Tensor, - slot_mapping: torch.Tensor, -): - """Inverse of :func:`launch_reshape_and_cache_shuffle_5d`. - - Returns ``(key_out, value_out)`` each shaped - ``(T, num_heads, head_size)`` in ``key_cache.dtype`` / - ``value_cache.dtype``. The caller is responsible for passing the - right per-tensor descales downstream when ``store_dtype`` is fp8. - - Args: - key_cache: (num_blocks, num_heads, head_size // X, block_size, X) - value_cache: (num_blocks, num_heads, block_size // X, head_size, X) - slot_mapping: (T,) per-token absolute slot id in - ``[0, num_blocks * block_size)`` - """ - assert key_cache.dim() == 5 and value_cache.dim() == 5 - num_blocks, num_heads, kc_D_over_X, block_size, X = key_cache.shape - vc_blocks, vc_H, vc_page_over_X, vc_D, vc_X = value_cache.shape - assert vc_blocks == num_blocks and vc_H == num_heads - assert vc_page_over_X * X == block_size and vc_X == X - head_size = kc_D_over_X * X - assert vc_D == head_size - - num_tokens = slot_mapping.numel() - key_out = torch.empty( - (num_tokens, num_heads, head_size), - dtype=key_cache.dtype, - device=key_cache.device, - ) - value_out = torch.empty( - (num_tokens, num_heads, head_size), - dtype=value_cache.dtype, - device=value_cache.device, - ) - - HEAD_BLOCK = min(4, triton.next_power_of_2(num_heads)) - BLOCK_D = triton.next_power_of_2(head_size) - grid = (num_tokens, triton.cdiv(num_heads, HEAD_BLOCK)) - - gather_shuffle_5d_to_linear[grid]( - key_cache, - value_cache, - key_out, - value_out, - slot_mapping, - key_out.stride(0), - value_out.stride(0), - num_heads, - head_size, - block_size, - X=X, - HEAD_BLOCK=HEAD_BLOCK, - BLOCK_D=BLOCK_D, - ) - return key_out, value_out - - def assert_buffer_fits(used: int, capacity: int, what: str, **context) -> None: """Safety guard: a preallocated cuda-graph buffer must hold the runtime write. diff --git a/python/sglang/kernels/ops/communication/__init__.py b/python/sglang/kernels/ops/communication/__init__.py index a2fb10cbda..cb2b2411e9 100644 --- a/python/sglang/kernels/ops/communication/__init__.py +++ b/python/sglang/kernels/ops/communication/__init__.py @@ -37,6 +37,8 @@ for _mod, _fn in [ ("sp_collective", "reduce_scatter_pull"), ("sp_collective", "all_gather"), ("sp_collective", "all_gather_direct"), + ("attn_res_collective", "attn_res_fused_pull_rs"), + ("attn_res_collective", "attn_res_fused_direct_ag"), ]: register_kernel( KernelSpec( @@ -47,3 +49,47 @@ for _mod, _fn in [ ) ) del _mod, _fn + +# Muse Spark collectives. The MoE finalize op (with its optional fused TP +# all-reduce) is built per layer: inventoried through its class methods. +register_kernel( + KernelSpec( + op="communication.muse_spark_all_gather", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.communication.muse_spark_all_gather:all_gather", + capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}), + ) +) +register_kernel( + KernelSpec( + op="communication.muse_spark_all_gather_norm_residual", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.communication.muse_spark_all_gather_norm_residual:all_gather_norm_residual", + capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(10, 0))}), + ) +) +for _bk in ("torch", "cute_dsl", "helion"): + register_kernel( + KernelSpec( + op="communication.muse_spark_moe_finalize_all_reduce", + backend=KernelBackend(_bk), + target="sglang.kernels.ops.communication.muse_spark_moe_finalize_all_reduce:FusedPostExpertNormWeightedSumTRTLLM." + + ("forward_native" if _bk == "torch" else f"forward_{_bk}"), + capabilities=frozenset() if _bk == "torch" else CUDA_ONLY, + ) + ) +del _bk + +# Blackwell sequence-parallel GEMM collectives (tcgen05 CuTeDSL kernels). +for _fn in ("all_gather_matmul", "matmul_reduce_scatter"): + register_kernel( + KernelSpec( + op=f"communication.{_fn}", + backend=KernelBackend.CUTE_DSL, + target=f"sglang.kernels.ops.communication.sp_collective_gemm:{_fn}", + capabilities=frozenset( + {CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))} + ), + ) + ) +del _fn diff --git a/python/sglang/kernels/ops/communication/attn_res_collective.py b/python/sglang/kernels/ops/communication/attn_res_collective.py new file mode 100644 index 0000000000..6e7486ce2e --- /dev/null +++ b/python/sglang/kernels/ops/communication/attn_res_collective.py @@ -0,0 +1,173 @@ +"""Kimi-K3 attention-residual aggregation fused with an NVLS pull reduce-scatter +or a direct multicast all-gather (the collective variants of attn_res_fused_tma).""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from sglang.kernels.ops.attention.attn_res import _jit_fused_tma_module, _tuning +from sglang.srt.utils.custom_op import register_custom_op + +if TYPE_CHECKING: + from sglang.kernels.ops.communication.all_reduce import Communicator + +_COMM_MAP: dict[int, Communicator] = {} + + +def register_comm(comm: Communicator) -> None: + # One communicator per world_size per process -- see the note in + # communication/all_reduce_residual.py::register_comm. The ops key only on world_size, so an + # overwrite here would hand the old group's callers the new group's peer + # pointers. + prev = _COMM_MAP.get(comm.world_size) + assert prev is None or prev is comm, ( + f"a different communicator is already registered for world_size=" + f"{comm.world_size}" + ) + _COMM_MAP[comm.world_size] = comm + + +@register_custom_op(mutates_args=["out", "prefix_out"]) +def _attn_res_fused_pull_rs_op( + world_size: int, + input: torch.Tensor, + residual: torch.Tensor | None, + bank: torch.Tensor, + cw: torch.Tensor, + ow: torch.Tensor, + out: torch.Tensor, + prefix_out: torch.Tensor, + nvb: int, + eps: float, + input_mc_ptr: int, + max_blocks: int, + chunk_rows: int, + occupancy: int, + consumer_regs: int, +) -> None: + _jit_fused_tma_module(chunk_rows, occupancy, consumer_regs).run_pull_rs( + _COMM_MAP[world_size], + input, + residual, + bank, + cw, + ow, + out, + prefix_out, + nvb, + eps, + input_mc_ptr, + max_blocks, + ) + + +@register_custom_op(mutates_args=["bank", "out"]) +def _attn_res_fused_direct_ag_op( + world_size: int, + prefix_sum: torch.Tensor, + bank: torch.Tensor, + cw: torch.Tensor, + ow: torch.Tensor, + out: torch.Tensor, + nvb: int, + eps: float, + output_mc_ptr: int, + max_blocks: int, + write_prefix: bool, + chunk_rows: int, + occupancy: int, + consumer_regs: int, +) -> None: + _jit_fused_tma_module(chunk_rows, occupancy, consumer_regs).run_direct_ag( + _COMM_MAP[world_size], + prefix_sum, + bank, + cw, + ow, + out, + nvb, + eps, + output_mc_ptr, + max_blocks, + write_prefix, + ) + + +def attn_res_fused_direct_ag( + world_size: int, + prefix_sum: torch.Tensor, + bank: torch.Tensor, + cw: torch.Tensor, + ow: torch.Tensor, + out: torch.Tensor, + nvb: int, + eps: float, + *, + output_mc_ptr: int, + max_blocks: int = 128, + write_prefix: bool = False, +) -> torch.Tensor: + """Fuse local attention-residual aggregation with direct multicast AG. + + `out` is the full symmetric [world * local_tokens, H] output. Each rank + aggregates its local token shard and multicast-stores normalized vectors + directly from consumer registers into that rank's slice on every peer. + """ + _attn_res_fused_direct_ag_op( + world_size, + prefix_sum, + bank, + cw, + ow, + out, + nvb, + eps, + output_mc_ptr, + max_blocks, + write_prefix, + *_tuning(nvb, prefix_sum.shape[0]), + ) + return out + + +def attn_res_fused_pull_rs( + world_size: int, + input: torch.Tensor, + residual: torch.Tensor | None, + bank: torch.Tensor, + cw: torch.Tensor, + ow: torch.Tensor, + out: torch.Tensor, + prefix_out: torch.Tensor, + nvb: int, + eps: float, + *, + input_mc_ptr: int, + max_blocks: int = 128, +) -> tuple[torch.Tensor, torch.Tensor]: + """Fused NVLS pull RS + local residual + TMA attention aggregation. + + `input` is the full TP-partial o_proj tensor in multicast symmetric + memory. Every rank reduces only its contiguous token shard, optionally + adds its already-local residual, materializes that prefix in `prefix_out`, + and feeds it directly to the fused attention-residual aggregation/norm. + """ + _attn_res_fused_pull_rs_op( + world_size, + input, + residual, + bank, + cw, + ow, + out, + prefix_out, + nvb, + eps, + input_mc_ptr, + max_blocks, + # prefix_out, not input: the shard this rank actually aggregates. + *_tuning(nvb, prefix_out.shape[0]), + ) + return out, prefix_out diff --git a/python/sglang/kernels/ops/gemm/cutlass_all_gather_gemm.py b/python/sglang/kernels/ops/communication/cutlass_all_gather_gemm.py similarity index 100% rename from python/sglang/kernels/ops/gemm/cutlass_all_gather_gemm.py rename to python/sglang/kernels/ops/communication/cutlass_all_gather_gemm.py diff --git a/python/sglang/kernels/ops/gemm/cutlass_gemm_reduce_scatter.py b/python/sglang/kernels/ops/communication/cutlass_gemm_reduce_scatter.py similarity index 100% rename from python/sglang/kernels/ops/gemm/cutlass_gemm_reduce_scatter.py rename to python/sglang/kernels/ops/communication/cutlass_gemm_reduce_scatter.py diff --git a/python/sglang/kernels/ops/communication/muse_spark_all_gather.py b/python/sglang/kernels/ops/communication/muse_spark_all_gather.py new file mode 100644 index 0000000000..c9eaf738f2 --- /dev/null +++ b/python/sglang/kernels/ops/communication/muse_spark_all_gather.py @@ -0,0 +1,97 @@ +"""Multicast all-gather of bf16 column slices through the CustomAllReduceV2 push workspace. + +``out[:, rank slice] = local`` (+ b) (+ c) for every rank's ``[M, N / world]`` +slice: one kernel multicast-stores the slice into every peer's push workspace +and Lamport-spins the full rows together. Any TP world size; needs the plane's +multicast mapping. The column-parallel projection that produces the slices +lives in ``sglang.srt.layers.communication.muse_spark_gemm_ag``. + +Unlike ``communication.gemm_ag`` (Kimi-K3), whose producer fuses the GEMV for +one fixed TP8 shape, this gathers slices an external GEMM already computed. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +import torch + +from sglang.kernels.jit.utils import ( + cache_once, + is_arch_support_pdl, + load_jit, + make_cpp_args, +) + +if TYPE_CHECKING: + from tvm_ffi.module import Module + + from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( + CustomAllReduceV2, + ) + +SMALL_PROJECTION_MAX_TOKENS = 8 +MAX_TOKENS = 512 +_VEC = 8 + + +@cache_once +def _jit_module(n: int, world_size: int) -> Module: + args = make_cpp_args(n, world_size, is_arch_support_pdl()) + return load_jit( + "muse_spark_all_gather", + *args, + cuda_files=["muse_spark/all_gather.cuh"], + cuda_wrappers=[("run", f"MuseSparkAllGather<{args}>::run")], + extra_cuda_cflags=["-O3"], + ) + + +def gemm_ag_supported( + comm: Optional[CustomAllReduceV2], x: torch.Tensor, weight: torch.Tensor +) -> bool: + from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( + CustomAllReduceV2, + ) + + if ( + not isinstance(comm, CustomAllReduceV2) + or comm.disabled + or not comm.has_multicast + or comm.world_size < 2 + ): + return False + if x.dtype != torch.bfloat16 or weight.dtype != torch.bfloat16 or x.dim() != 2: + return False + m, k = x.shape + n = weight.shape[0] + return ( + 1 <= m <= MAX_TOKENS + and ( + m <= SMALL_PROJECTION_MAX_TOKENS + or ( + comm.world_size in (4, 8) + and n in (4096, 8192) + and torch.cuda.get_device_capability(x.device) == (10, 3) + ) + ) + and weight.shape[1] == k + and n % (comm.world_size * _VEC) == 0 + and m * n * 2 <= comm.max_push_size + and comm.config.num_push_blocks >= 2 + and weight.is_contiguous() + ) + + +def all_gather( + comm: CustomAllReduceV2, + local: torch.Tensor, + out: torch.Tensor, + b: Optional[torch.Tensor] = None, + c: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """``out[:, rank slice] = local`` gathered from every rank, (+ b) (+ c).""" + _jit_module(out.shape[1], comm.world_size).run( + comm.obj, local.contiguous(), b, c, out + ) + return out diff --git a/python/sglang/kernels/ops/muse_spark/all_gather_norm_residual.py b/python/sglang/kernels/ops/communication/muse_spark_all_gather_norm_residual.py similarity index 98% rename from python/sglang/kernels/ops/muse_spark/all_gather_norm_residual.py rename to python/sglang/kernels/ops/communication/muse_spark_all_gather_norm_residual.py index 6ab884f857..abfdf4d3c5 100644 --- a/python/sglang/kernels/ops/muse_spark/all_gather_norm_residual.py +++ b/python/sglang/kernels/ops/communication/muse_spark_all_gather_norm_residual.py @@ -12,7 +12,7 @@ from sglang.kernels.jit.utils import ( load_jit, make_cpp_args, ) -from sglang.kernels.ops.muse_spark.norm_residual import ( +from sglang.kernels.ops.layernorm.muse_spark_norm_residual_jit import ( norm_residual_launch_config, ) diff --git a/python/sglang/kernels/ops/muse_spark/fused_post_expert_norm_weighted_sum_trtllm.py b/python/sglang/kernels/ops/communication/muse_spark_moe_finalize_all_reduce.py similarity index 95% rename from python/sglang/kernels/ops/muse_spark/fused_post_expert_norm_weighted_sum_trtllm.py rename to python/sglang/kernels/ops/communication/muse_spark_moe_finalize_all_reduce.py index c4e48699a9..a42d6f7956 100644 --- a/python/sglang/kernels/ops/muse_spark/fused_post_expert_norm_weighted_sum_trtllm.py +++ b/python/sglang/kernels/ops/communication/muse_spark_moe_finalize_all_reduce.py @@ -14,20 +14,23 @@ Backends: from __future__ import annotations import logging +from typing import TYPE_CHECKING import torch from sglang.kernels.fused_op import BaseFusedOp from sglang.kernels.jit.utils import is_arch_support_pdl -from sglang.kernels.ops.muse_spark import ( - fi_trtllm_moe_finalize_with_rmsnorm_cutedsl, -) -from sglang.kernels.ops.muse_spark.fi_trtllm_moe_finalize_with_rmsnorm_cutedsl import ( - FinalizePushWorkspace, +from sglang.kernels.ops.communication import ( + muse_spark_moe_finalize_all_reduce_cutedsl, ) from sglang.kernels.spec import CapabilityRequirement, KernelBackend from sglang.srt.runtime_context import get_forward +if TYPE_CHECKING: + from sglang.srt.layers.communication.muse_spark_finalize_workspace import ( + FinalizePushWorkspace, + ) + logger = logging.getLogger(__name__) _missing_cutedsl_warning_logged = False @@ -103,7 +106,7 @@ class FusedPostExpertNormWeightedSumTRTLLM(BaseFusedOp): from token count for H=4096; other shapes use a single CTA. """ - op = "muse_spark.post_expert_finalize" + op = "communication.muse_spark_moe_finalize_all_reduce" priority = (KernelBackend.CUTE_DSL, KernelBackend.HELION) capabilities = { KernelBackend.CUTE_DSL: {CapabilityRequirement.CUDA}, @@ -132,7 +135,7 @@ class FusedPostExpertNormWeightedSumTRTLLM(BaseFusedOp): return self._helion_eligible(*args, **kwargs) if backend is not KernelBackend.CUTE_DSL: return True - if not fi_trtllm_moe_finalize_with_rmsnorm_cutedsl.HAS_CUTEDSL: + if not muse_spark_moe_finalize_all_reduce_cutedsl.HAS_CUTEDSL: global _missing_cutedsl_warning_logged if not _missing_cutedsl_warning_logged: _missing_cutedsl_warning_logged = True @@ -249,7 +252,7 @@ class FusedPostExpertNormWeightedSumTRTLLM(BaseFusedOp): from cutlass import Int32 from cutlass.cute.runtime import make_ptr - kernel_module = fi_trtllm_moe_finalize_with_rmsnorm_cutedsl + kernel_module = muse_spark_moe_finalize_all_reduce_cutedsl assert gemm2_output.is_cuda cutlass_dtype = kernel_module._torch_to_cutlass_dtype[gemm2_output.dtype] @@ -354,10 +357,10 @@ class FusedPostExpertNormWeightedSumTRTLLM(BaseFusedOp): *, ar_workspace: FinalizePushWorkspace | None = None, ) -> bool: - from sglang.kernels.ops.muse_spark.helion_runtime import helion_supported + from sglang.kernels.jit.utils.helion_runtime import helion_supported return ( - helion_supported("post_expert_finalize", output) + helion_supported(output) and output.shape[0] > 0 and self.topk == 8 and self.hidden_dim == 4096 @@ -385,13 +388,13 @@ class FusedPostExpertNormWeightedSumTRTLLM(BaseFusedOp): ) -> None: """Run Helion finalize, then a separate AR when delegated by the model.""" - from sglang.kernels.ops.muse_spark.helion_post_expert import ( + from sglang.kernels.jit.utils.helion_runtime import ( + run_pinned_helion, + ) + from sglang.kernels.ops.communication.muse_spark_moe_finalize_all_reduce_helion import ( POST_EXPERT_FINALIZE_CONFIGS, _post_expert_finalize_helion_kernel, ) - from sglang.kernels.ops.muse_spark.helion_runtime import ( - run_pinned_helion, - ) run_pinned_helion( _post_expert_finalize_helion_kernel, diff --git a/python/sglang/kernels/ops/muse_spark/fi_trtllm_moe_finalize_with_rmsnorm_cutedsl.py b/python/sglang/kernels/ops/communication/muse_spark_moe_finalize_all_reduce_cutedsl.py similarity index 88% rename from python/sglang/kernels/ops/muse_spark/fi_trtllm_moe_finalize_with_rmsnorm_cutedsl.py rename to python/sglang/kernels/ops/communication/muse_spark_moe_finalize_all_reduce_cutedsl.py index 2f3e610420..55babc1f4b 100644 --- a/python/sglang/kernels/ops/muse_spark/fi_trtllm_moe_finalize_with_rmsnorm_cutedsl.py +++ b/python/sglang/kernels/ops/communication/muse_spark_moe_finalize_all_reduce_cutedsl.py @@ -24,146 +24,14 @@ Algorithm for each token (1 block or DSMEM cluster per token): Convert the FP32 accumulator to the output dtype once. The public API is ``FusedPostExpertNormWeightedSumTRTLLM`` in -``fused_post_expert_norm_weighted_sum_trtllm.py``. This module exposes -the CuTeDSL kernel, compilation cache, and borrowed TP push workspace. +``muse_spark_moe_finalize_all_reduce.py``. This module exposes +the CuTeDSL kernel and its compilation cache; the borrowed TP push workspace +is ``sglang.srt.layers.communication.muse_spark_finalize_workspace``. """ import functools import torch -import torch.distributed as dist - - -class FinalizePushWorkspace: - """Borrow TP push buffers before capture; dependency-ordered launches only.""" - - def __init__( - self, - communicator, - device_group: dist.ProcessGroup, - hidden_dim=4096, - *, - multicast=False, - ): - self.communicator = communicator # Keep the allocation owner alive. - self.group = device_group - self.rank = communicator.rank - self.world_size = communicator.world_size - self.device = communicator.device - self.sm_count = torch.cuda.get_device_properties( - self.device - ).multi_processor_count - self.hidden_dim = hidden_dim - self.slot_bytes = communicator.max_push_size - self.peers = communicator._push_workspaces - self.counter = communicator._push_counter - self.multicast = multicast - self.multicast_ptr = communicator._push_multicast_ptr - config = ( - self.controller_count, - self.slot_bytes, - self.sm_count, - hidden_dim, - multicast, - ) - configs = [None] * self.world_size - dist.all_gather_object(configs, config, group=communicator.group) - if any(peer != config for peer in configs): - raise ValueError( - "Finalize AR workspace configuration must match across ranks" - ) - if multicast and not communicator.has_multicast: - raise ValueError( - "Finalize multicast requires a multicast-capable push workspace" - ) - if not 0 < self.controller_count <= self.sm_count: - raise ValueError("Unsupported finalize AR controller budget") - if hidden_dim <= 0 or hidden_dim % 256 or self.slot_bytes < hidden_dim * 2: - raise ValueError( - "Finalize push requires H divisible by 256 and sufficient buffer capacity" - ) - self.max_tokens = min(384, self.slot_bytes // (hidden_dim * 2)) - self.fallback_logged = False - self.fusion_logged = False - - def uses_one_shot_push(self, num_tokens: int) -> bool: - return self.communicator.uses_one_shot_push(num_tokens * self.hidden_dim * 2) - - def can_fuse(self, tp_group, num_tokens: int) -> bool: - """Use custom AR storage while respecting explicit algorithm overrides. - - Fusion has a different crossover from a standalone AR launch. Allow - push through 384 tokens when its allocated slot fits, even if ordinary - CUDA-graph AR would select pull above its 288-token crossover (H=4096), - but not where the standalone AR would pick 2shot_push. - No standalone AR thresholds or buffer allocations are changed. - """ - if ( - torch.compiler.is_compiling() - or self.communicator is not tp_group.ca_comm - or self.communicator.disabled - or not 0 < num_tokens <= self.max_tokens - ): - return False - if self.world_size > 4 and not self.uses_one_shot_push(num_tokens): - return False - if self.communicator.uses_two_shot_push(num_tokens * self.hidden_dim * 2): - return False - output = ( - self.peers[self.rank] - .view(torch.bfloat16) - .view(-1)[: num_tokens * self.hidden_dim] - .view(num_tokens, self.hidden_dim) - ) - return ( - not tp_group._can_use_flashinfer_allreduce(output) - and tp_group._resolve_outplace_all_reduce_method(output) == "ca" - and ( - self.communicator.override_algo is None - or self.uses_one_shot_push(num_tokens) - ) - ) - - @property - def controller_count(self) -> int: - return self.counter.numel() - - def validate(self, output: torch.Tensor, cluster_size: int = 1) -> None: - if ( - output.device != self.device - or output.dtype != torch.bfloat16 - or output.ndim != 2 - or output.shape[1] != self.hidden_dim - or output.shape[0] > self.max_tokens - or not output.is_contiguous() - or output.data_ptr() % 16 - ): - raise ValueError("Unsupported output for finalize push workspace") - if cluster_size not in (1, 2, 4, 8, 16) or cluster_size > self.sm_count: - raise ValueError("Unsupported cluster size") - - if output.shape[0] == 0: - raise ValueError("Finalize push requires M>0") - if cluster_size > 1 and output.shape[0] * cluster_size > self.controller_count: - raise ValueError( - "Single-token clusters require M * C <= controller_count; use C=1 otherwise" - ) - - def arguments(self) -> tuple: - import cutlass - from cutlass.cute.runtime import make_ptr - - return ( - tuple(make_ptr(cutlass.Uint32, p.data_ptr()) for p in self.peers), - make_ptr(cutlass.Uint32, self.counter.data_ptr()), - cutlass.Int32(self.rank), - cutlass.Int32(self.slot_bytes // 4), - make_ptr(cutlass.Uint32, self.peers[self.rank].data_ptr()), - make_ptr( - cutlass.Uint32, self.multicast_ptr or self.peers[self.rank].data_ptr() - ), - ) - try: import cuda.bindings.driver as cuda diff --git a/python/sglang/kernels/ops/muse_spark/helion_post_expert.py b/python/sglang/kernels/ops/communication/muse_spark_moe_finalize_all_reduce_helion.py similarity index 100% rename from python/sglang/kernels/ops/muse_spark/helion_post_expert.py rename to python/sglang/kernels/ops/communication/muse_spark_moe_finalize_all_reduce_helion.py diff --git a/python/sglang/kernels/ops/gemm/sp_collective_gemm.py b/python/sglang/kernels/ops/communication/sp_collective_gemm.py similarity index 76% rename from python/sglang/kernels/ops/gemm/sp_collective_gemm.py rename to python/sglang/kernels/ops/communication/sp_collective_gemm.py index ff9e7b1e78..b28786a5d4 100644 --- a/python/sglang/kernels/ops/gemm/sp_collective_gemm.py +++ b/python/sglang/kernels/ops/communication/sp_collective_gemm.py @@ -3,13 +3,25 @@ Torch symmetric memory is used only to allocate/register NVLink-accessible buffers. AG dispatches between a tile-gated CUTLASS GEMM and per-shard cuBLAS GEMMs; RS uses a fused CUTLASS kernel. + +The symmetric-memory workspaces are owned by +``sglang.srt.layers.communication.sp_collective_gemm_workspace``: each entry +point takes its getter as ``workspace`` and calls it with the tile-padded shape +``(kind, group, rows, width, x, row_align, flag_tile_mn)``. """ from __future__ import annotations +from typing import TYPE_CHECKING, Callable + import torch import torch.distributed as dist +if TYPE_CHECKING: + from sglang.srt.layers.communication.sp_collective_gemm_workspace import ( + SPCollectiveGemmWorkspace, + ) + _TILE_M = 64 _TILE_N = 128 _AG_TILES = ( @@ -18,7 +30,6 @@ _AG_TILES = ( (64, 128, False, (1, 1)), ) _RS_TILES = ((128, 256, False, (1, 1)), (64, 128, False, (1, 1))) -_workspaces = {} _compiled = {} @@ -93,86 +104,10 @@ def _check_inputs(x, weight, group): return world, dist.get_rank(group) -class _Workspace: - def __init__( - self, - kind, - group, - rows, - width, - dtype, - device, - row_align=_TILE_M, - flag_tile_mn=(_TILE_M, _TILE_N), - ): - import cutlass.torch as cutlass_torch - import torch.distributed._symmetric_memory as symm_mem - - # Power of two >= row_align, so capacity stays a whole number of M tiles. - self.capacity = max(row_align, 1 << (rows - 1).bit_length()) - self.world = dist.get_world_size(group) - self.sms = torch.cuda.get_device_properties(device).multi_processor_count - self.storage = symm_mem.empty( - (self.capacity, width), dtype=dtype, device=device - ) - self.handle = symm_mem.rendezvous(self.storage, group) - if kind == "ag": - self.gathered = torch.empty( - (self.capacity * self.world, width), dtype=dtype, device=device - ) - self.flags = torch.zeros(self.world, dtype=torch.int32, device=device) - self.copy_stream = torch.cuda.Stream(device=device) - else: - if not self.handle.multicast_ptr: - raise RuntimeError( - "CUTLASS GEMM-RS requires NVSwitch multicast mapping" - ) - self.multicast = cutlass_torch.as_tensor( - self.handle.multicast_ptr, self.storage.shape, dtype - ) - num_flags = ( - self.capacity // flag_tile_mn[0] * (width // flag_tile_mn[1]) - + self.sms - + 8 - ) - self.flags = symm_mem.empty((num_flags,), dtype=torch.int32, device=device) - self.flags.zero_() - self.flag_handle = symm_mem.rendezvous(self.flags, group) - if not self.flag_handle.multicast_ptr: - raise RuntimeError("CUTLASS GEMM-RS requires multicast flag storage") - self.flag_multicast = cutlass_torch.as_tensor( - self.flag_handle.multicast_ptr, self.flags.shape, self.flags.dtype - ) - self.peer_flags = tuple( - self.flag_handle.get_buffer(r, self.flags.shape, self.flags.dtype) - for r in range(self.world) - ) - self.handle.barrier(channel=0) - - -def _workspace( - kind, group, rows, width, x, row_align=_TILE_M, flag_tile_mn=(_TILE_M, _TILE_N) -): - key = (kind, group, width, x.dtype, x.device) - ws = _workspaces.get(key) - if ws is None or rows > ws.capacity: - if torch.cuda.is_current_stream_capturing(): - raise RuntimeError("Warm up CUTLASS SP workspace before CUDA graph capture") - if ws is not None: - # Peer mappings must stay alive until all ranks stop reading them. - ws.handle.barrier(channel=0) - torch.cuda.current_stream().synchronize() - ws = _Workspace( - kind, group, rows, width, x.dtype, x.device, row_align, flag_tile_mn - ) - _workspaces[key] = ws - return ws - - def _all_gather_matmul_cublas( local: torch.Tensor, weight: torch.Tensor, - ws: _Workspace, + ws: SPCollectiveGemmWorkspace, rank: int, rows: int, out: torch.Tensor, @@ -236,7 +171,14 @@ def _all_gather_matmul_cublas( main.wait_stream(ws.copy_stream) -def all_gather_matmul(x, weight, group, *, backend: str = "auto"): +def all_gather_matmul( + x, + weight, + group, + *, + workspace: Callable[..., SPCollectiveGemmWorkspace], + backend: str = "auto", +): """all_gather(x, dim=0) @ weight.T, with shape-based backend selection. Backend overrides must agree across ranks and are fixed at graph capture. @@ -245,7 +187,7 @@ def all_gather_matmul(x, weight, group, *, backend: str = "auto"): import cutlass import cutlass.cute as cute - from sglang.kernels.ops.gemm.cutlass_all_gather_gemm import ( + from sglang.kernels.ops.communication.cutlass_all_gather_gemm import ( PersistentDenseGemmKernel, ReleaseFlagSet, ) @@ -262,7 +204,7 @@ def all_gather_matmul(x, weight, group, *, backend: str = "auto"): mma_m, mma_n, use_2cta, cluster = _ag_tile(n) local = _pad_shards(x.contiguous(), 1, mma_m) padded = local.shape[0] - ws = _workspace("ag", group, padded, k, x, mma_m) + ws = workspace("ag", group, padded, k, x, mma_m) gathered = ws.gathered[: padded * world] if backend == "cublas": out = x.new_empty((rows * world, n)) @@ -328,13 +270,15 @@ def all_gather_matmul(x, weight, group, *, backend: str = "auto"): return _unpad_shards(out, rows, world) -def matmul_reduce_scatter(x, weight, group): +def matmul_reduce_scatter( + x, weight, group, *, workspace: Callable[..., SPCollectiveGemmWorkspace] +): """reduce_scatter(x @ weight.T, SUM, dim=0), one fused GEMM-RS kernel.""" import cuda.bindings.driver as cuda import cutlass import cutlass.cute as cute - from sglang.kernels.ops.gemm.cutlass_gemm_reduce_scatter import ( + from sglang.kernels.ops.communication.cutlass_gemm_reduce_scatter import ( Sm100PersistentDenseGemmReduceScatterLDMCKernel, ) @@ -349,7 +293,7 @@ def matmul_reduce_scatter(x, weight, group): cta_m = mma_m // (2 if use_2cta else 1) padded_input = _pad_shards(x.contiguous(), world, cta_m) m = padded_input.shape[0] - ws = _workspace("rs", group, m, n, x, cta_m, (cta_m, mma_n)) + ws = workspace("rs", group, m, n, x, cta_m, (cta_m, mma_n)) out = x.new_empty((m // world, n)) args = dict( a=_tensor(padded_input), diff --git a/python/sglang/kernels/ops/diffusion/README.md b/python/sglang/kernels/ops/diffusion/README.md index 3c183c9806..64b569cae9 100644 --- a/python/sglang/kernels/ops/diffusion/README.md +++ b/python/sglang/kernels/ops/diffusion/README.md @@ -206,7 +206,9 @@ tensor copy at each residual site. ### MXFP8 producers (online `mxfp8`, cuBLASLt block-scaled GEMM on SM100) -Defined in `sglang.kernels.ops.quantization.mxfp8_swizzled_triton` and re-exported here. +Defined in `sglang.kernels.ops.quantization.mxfp8_swizzled_triton` (`silu_mul_mxfp8` in +`sglang.kernels.ops.activation.swiglu_mxfp8_swizzled`, `indexed_scale_shift_mxfp8_` in +`modulate/indexed_scale_shift_mxfp8_triton.py`) and re-exported here. | Entry point | Backend | Contract | |---|---|---| diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index bad9abdef9..2f60b0c617 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -504,13 +504,13 @@ _SPECS: tuple[tuple[str, KernelBackend, str, frozenset], ...] = ( ( "diffusion.silu_mul_mxfp8", KernelBackend.TRITON, - "sglang.kernels.ops.quantization.mxfp8_swizzled_triton:silu_mul_mxfp8", + "sglang.kernels.ops.activation.swiglu_mxfp8_swizzled:silu_mul_mxfp8", _CUDA, ), ( "diffusion.indexed_scale_shift_mxfp8_", KernelBackend.TRITON, - "sglang.kernels.ops.quantization.mxfp8_swizzled_triton:indexed_scale_shift_mxfp8_", + "modulate.indexed_scale_shift_mxfp8_triton:indexed_scale_shift_mxfp8_", _CUDA, ), ( @@ -786,10 +786,10 @@ _EXPORTS: dict[str, str] = { "vdn_gather_linear_state": "attention.vdn_linear_branch_triton", "vdn_linear_epilogue": "attention.vdn_linear_branch_triton", "can_use_mxfp8_swizzled": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton", - "can_use_silu_mul_mxfp8": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton", - "indexed_scale_shift_mxfp8_": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton", + "can_use_silu_mul_mxfp8": "sglang.kernels.ops.activation.swiglu_mxfp8_swizzled", + "indexed_scale_shift_mxfp8_": "modulate.indexed_scale_shift_mxfp8_triton", "mxfp8_quantize_swizzled": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton", - "silu_mul_mxfp8": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton", + "silu_mul_mxfp8": "sglang.kernels.ops.activation.swiglu_mxfp8_swizzled", "can_use_usp_merge_heads": "layout.usp_relayout_jit", "usp_merge_heads": "layout.usp_relayout_jit", "build_inv_indices": "layout.varlen_pack_pad_triton", diff --git a/python/sglang/kernels/ops/diffusion/modulate/indexed_scale_shift_mxfp8_triton.py b/python/sglang/kernels/ops/diffusion/modulate/indexed_scale_shift_mxfp8_triton.py new file mode 100644 index 0000000000..47e73f7252 --- /dev/null +++ b/python/sglang/kernels/ops/diffusion/modulate/indexed_scale_shift_mxfp8_triton.py @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: Apache-2.0 +"""``x * (1 + scale[idx]) + shift[idx]`` fused with MXFP8 quantization, scales in +the cuBLASLt ``SWIZZLE_32_4_4`` layout of ``ops.quantization.mxfp8_swizzled_triton``.""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from sglang.kernels.numerics import round_bf16_to_fp32 +from sglang.kernels.ops.quantization.mxfp8_swizzled_triton import ( + _alloc, + _mx_e8m0_from_amax, + _mx_scale_offsets, + can_use_mxfp8_swizzled, +) + + +@triton.jit +def _indexed_scale_shift_mxfp8_kernel( + x_ptr, + q_ptr, + s_ptr, + shift_ptr, + scale_ptr, + indices_ptr, + hidden_size, + n_groups, + n_col_blocks, + stride_x_row, + stride_shift_row, + stride_scale_row, + stride_indices, + STORE_BF16: tl.constexpr, + BLOCK_N: tl.constexpr, +): + row = tl.program_id(0) + columns = tl.arange(0, BLOCK_N) + mask = columns < hidden_size + index = tl.load(indices_ptr + row * stride_indices) + xrow = x_ptr + row.to(tl.int64) * stride_x_row + x = tl.load(xrow + columns, mask=mask, other=0.0).to(tl.float32) + shift = tl.load( + shift_ptr + index * stride_shift_row + columns, mask=mask, other=0.0 + ).to(tl.float32) + scale = tl.load( + scale_ptr + index * stride_scale_row + columns, mask=mask, other=0.0 + ).to(tl.float32) + # the rounding points of _indexed_scale_shift_bf16_kernel + one_plus_scale = round_bf16_to_fp32(1.0 + scale) + scaled = round_bf16_to_fp32(x * one_plus_scale) + out = round_bf16_to_fp32(scaled + shift) + if STORE_BF16: + tl.store(xrow + columns, out, mask=mask) + v3 = tl.reshape(out, [BLOCK_N // 32, 32]) + amax = tl.max(tl.abs(v3), axis=1) + sbyte, inv = _mx_e8m0_from_amax(amax) + q = tl.reshape(v3 * inv[:, None], [BLOCK_N]) + tl.store( + q_ptr + row.to(tl.int64) * hidden_size + columns, + q.to(tl.float8e4nv), + mask=mask, + ) + g = tl.arange(0, BLOCK_N // 32) + tl.store( + s_ptr + _mx_scale_offsets(row, g, n_col_blocks), + sbyte.to(tl.uint8), + mask=g < n_groups, + ) + + +def indexed_scale_shift_mxfp8_( + x: torch.Tensor, + shift: torch.Tensor, + scale: torch.Tensor, + indices: torch.Tensor, + *, + keep_bf16: bool, +) -> tuple[torch.Tensor | None, torch.Tensor, torch.Tensor]: + """``x * (1 + scale[idx]) + shift[idx]`` -> ``(x | None, fp8, scales)``; with + ``keep_bf16`` the bf16 result is also written into ``x`` and returned.""" + if not can_use_mxfp8_swizzled(x): + raise ValueError( + "expected a row-major bf16 CUDA [rows, hidden] tensor, hidden % 32 == 0" + ) + rows, hidden_size = x.shape + q, s = _alloc(rows, hidden_size, x.device) + if rows == 0: + return (x if keep_bf16 else None), q, s + n_groups = hidden_size // 32 + n_col_blocks = triton.cdiv(n_groups, 4) + block_n = triton.next_power_of_2(hidden_size) + with torch.get_device_module().device(x.device): + _indexed_scale_shift_mxfp8_kernel[(rows,)]( + x, + q, + s, + shift, + scale, + indices, + hidden_size, + n_groups, + n_col_blocks, + x.stride(0), + shift.stride(0), + scale.stride(0), + indices.stride(0), + STORE_BF16=keep_bf16, + BLOCK_N=block_n, + num_warps=8, + ) + return (x if keep_bf16 else None), q, s diff --git a/python/sglang/kernels/ops/elementwise/__init__.py b/python/sglang/kernels/ops/elementwise/__init__.py index 4de164ba29..da2e40a304 100644 --- a/python/sglang/kernels/ops/elementwise/__init__.py +++ b/python/sglang/kernels/ops/elementwise/__init__.py @@ -2,7 +2,7 @@ Home for cross-cutting pointwise kernels that do not belong to a single functional group: the fused-pointwise Triton collection (``elementwise``: -sigmoid-mul, gated-activation and fused-rmsnorm variants shared across models) +sigmoid-mul variants shared across models) and the ``add_constant`` JIT reference kernel used by the developer guide. Individual functions register (or are imported) under the functional op id they logically belong to. @@ -55,3 +55,26 @@ register_kernel( capabilities=CUDA_ONLY, ) ) +register_kernel( + KernelSpec( + op="elementwise.row_sum", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.elementwise.row_sum:row_sum", + capabilities=CUDA_ONLY, + ) +) +register_kernel( + KernelSpec( + op="elementwise.apply_log_scaling_tau", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.elementwise.row_scale:apply_log_scaling_tau", + ) +) +register_kernel( + KernelSpec( + op="elementwise.kimi_k3_mla_output_gate", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.elementwise.mla_output_gate:kimi_k3_mla_output_gate", + capabilities=CUDA_ONLY, + ) +) diff --git a/python/sglang/kernels/ops/elementwise/elementwise.py b/python/sglang/kernels/ops/elementwise/elementwise.py index de793ccb06..0ed8914b67 100644 --- a/python/sglang/kernels/ops/elementwise/elementwise.py +++ b/python/sglang/kernels/ops/elementwise/elementwise.py @@ -8,229 +8,6 @@ from sglang.srt.utils import is_hip _is_hip = is_hip() -@triton.jit -def fused_dual_residual_rmsnorm_kernel( - output_ptr, - mid_ptr, - activ_ptr, - residual_ptr, - weight1_ptr, - weight2_ptr, - eps: tl.constexpr, - hidden_dim: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - pid = tl.program_id(axis=0) - input_start = pid * hidden_dim - - offsets = tl.arange(0, BLOCK_SIZE) - mask = offsets < hidden_dim - - a_ = tl.load(activ_ptr + input_start + offsets, mask=mask, other=0.0) - a = a_.to(tl.float32) - rms = tl.sqrt(tl.sum(a * a, axis=0) / hidden_dim + eps) - - r = tl.load(residual_ptr + input_start + offsets, mask=mask, other=0.0) - w1_ = tl.load(weight1_ptr + offsets, mask=mask, other=0.0) - w1 = w1_.to(tl.float32) - - a2r = r + (a / rms * w1).to(r.dtype) - tl.store( - mid_ptr + input_start + offsets, - a2r, - mask=mask, - ) - - a2r = a2r.to(tl.float32) - rms2 = tl.sqrt(tl.sum(a2r * a2r, axis=0) / hidden_dim + eps) - - w2_ = tl.load(weight2_ptr + offsets, mask=mask, other=0.0) - w2 = w2_.to(tl.float32) - - tl.store( - output_ptr + input_start + offsets, - a2r / rms2 * w2, # implicitly casts to output dtype here - mask=mask, - ) - - -def fused_dual_residual_rmsnorm(x, residual, weight1, weight2, eps): - assert len(x.shape) == 2 - assert x.shape == residual.shape and x.dtype == residual.dtype, ( - f"{x.shape=} {residual.shape=} {x.dtype=} {residual.dtype=}" - ) - output, mid = torch.empty_like(x), torch.empty_like(x) - bs, hidden_dim = x.shape - max_warps = 16 if _is_hip else 32 - config = { - "BLOCK_SIZE": triton.next_power_of_2(hidden_dim), - "num_warps": max( - min(triton.next_power_of_2(triton.cdiv(hidden_dim, 256)), max_warps), 4 - ), - } - - fused_dual_residual_rmsnorm_kernel[(bs,)]( - output, - mid, - x, - residual, - weight1, - weight2, - eps=eps, - hidden_dim=hidden_dim, - **config, - ) - - return output, mid - - -# Not layernorm.rmsnorm_fp32: dividing by sqrt(var + eps) rounds differently from rsqrt. -@triton.jit -def fused_rmsnorm_kernel( - output_ptr, - activ_ptr, - weight_ptr, - eps: tl.constexpr, - hidden_dim: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - pid = tl.program_id(axis=0).to(tl.int64) - input_start = pid * hidden_dim - - offsets = tl.arange(0, BLOCK_SIZE) - mask = offsets < hidden_dim - - a_ = tl.load(activ_ptr + input_start + offsets, mask=mask, other=0.0) - a = a_.to(tl.float32) - rms = tl.sqrt(tl.sum(a * a, axis=0) / hidden_dim + eps) - - w1_ = tl.load(weight_ptr + offsets, mask=mask, other=0.0) - w1 = w1_.to(tl.float32) - - a_rms = a / rms * w1 - - tl.store( - output_ptr + input_start + offsets, - a_rms, # implicitly casts to output dtype here - mask=mask, - ) - - -def fused_rmsnorm(x, weight, eps, inplace=False): - assert len(x.shape) == 2 - if inplace: - output = x - else: - output = torch.empty_like(x) - bs, hidden_dim = x.shape - max_warps = 16 if _is_hip else 32 - config = { - "BLOCK_SIZE": triton.next_power_of_2(hidden_dim), - "num_warps": max( - min(triton.next_power_of_2(triton.cdiv(hidden_dim, 256)), max_warps), 4 - ), - } - - fused_rmsnorm_kernel[(bs,)]( - output, x, weight, eps=eps, hidden_dim=hidden_dim, **config - ) - return output - - -# gelu on first half of vector -@triton.jit -def gelu_and_mul_kernel( - out_hidden_states_ptr, # (bs, hidden_dim) - out_scales_ptr, # (bs,) - hidden_states_ptr, # (bs, hidden_dim * 2) - quant_max: tl.constexpr, - static_scale: tl.constexpr, - hidden_dim: tl.constexpr, # the output hidden_dim - BLOCK_SIZE: tl.constexpr, -): - pid = tl.program_id(axis=0) - - input_start = pid * hidden_dim * 2 - output_start = pid * hidden_dim - - input1_offs = tl.arange(0, BLOCK_SIZE) - mask = tl.arange(0, BLOCK_SIZE) < hidden_dim # shared for input1, input3, output - input3_offs = hidden_dim + tl.arange(0, BLOCK_SIZE) - output_offs = tl.arange(0, BLOCK_SIZE) - - x1 = tl.load( - hidden_states_ptr + input_start + input1_offs, mask=mask, other=0.0 - ).to(tl.float32) - x3 = tl.load( - hidden_states_ptr + input_start + input3_offs, mask=mask, other=0.0 - ).to(tl.float32) - - # gelu - # cast down before mul to better match training? - gelu_x1 = 0.5 * (1.0 + tl.erf(x1 * 0.7071067811865475)) * x1 - out = x3 * gelu_x1.to(hidden_states_ptr.dtype.element_ty) - - if quant_max is not None: - raise NotImplementedError() - - tl.store(out_hidden_states_ptr + output_start + output_offs, out, mask=mask) - - -def gelu_and_mul_triton( - hidden_states, - scales=None, - quantize=None, # dtype to quantize to - out=None, -): - bs, in_hidden_dim = hidden_states.shape - hidden_dim = in_hidden_dim // 2 - - if out is None: - out_hidden_states = torch.empty( - (bs, hidden_dim), - dtype=quantize or hidden_states.dtype, - device=hidden_states.device, - ) - else: - assert out.shape == (bs, hidden_dim) - assert out.dtype == (quantize or hidden_states.dtype) - out_hidden_states = out - out_scales = None - static_scale = False - if quantize is not None: - if scales is None: - out_scales = torch.empty( - (bs,), dtype=torch.float32, device=hidden_states.device - ) - else: - out_scales = scales - static_scale = True - - max_warps = 16 if _is_hip else 32 - config = { - # 8 ele per thread (not tuned) - "num_warps": max( - min(triton.next_power_of_2(triton.cdiv(hidden_dim, 8 * 32)), max_warps), 4 - ), - } - - gelu_and_mul_kernel[(bs,)]( - out_hidden_states, - out_scales, - hidden_states, - quant_max=torch.finfo(quantize).max if quantize is not None else None, - static_scale=static_scale, - hidden_dim=hidden_dim, - BLOCK_SIZE=triton.next_power_of_2(hidden_dim), - **config, - ) - - if quantize is not None: - return out_hidden_states, out_scales - else: - return out_hidden_states, None - - @triton.jit def _fused_sigmoid_mul_kernel( output_ptr, @@ -333,6 +110,49 @@ def fused_sigmoid_mul( return (q, s) if quant else out +@triton.jit +def _sigmoid_gate_mul_broadcast_kernel( + out_ptr, + gate_ptr, + x_ptr, + hidden_dim: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + row = tl.program_id(0) + g = tl.load(gate_ptr + row).to(tl.float32) + g = tl.sigmoid(g) + + offs = tl.arange(0, BLOCK_SIZE) + mask = offs < hidden_dim + x = tl.load(x_ptr + row * hidden_dim + offs, mask=mask).to(tl.float32) + out = x * g + tl.store( + out_ptr + row * hidden_dim + offs, + out.to(x_ptr.dtype.element_ty), + mask=mask, + ) + + +def sigmoid_gate_mul_broadcast(x: torch.Tensor, gate: torch.Tensor) -> torch.Tensor: + """Compute ``x * sigmoid(gate)`` where gate is (N, 1) and x is (N, D).""" + bs, hidden_dim = x.shape + out = torch.empty_like(x) + BLOCK_SIZE = triton.next_power_of_2(hidden_dim) + max_warps = 16 if _is_hip else 32 + num_warps = max( + min(triton.next_power_of_2(triton.cdiv(hidden_dim, 8 * 32)), max_warps), 4 + ) + _sigmoid_gate_mul_broadcast_kernel[(bs,)]( + out, + gate, + x, + hidden_dim=hidden_dim, + BLOCK_SIZE=BLOCK_SIZE, + num_warps=num_warps, + ) + return out + + @triton.jit def _fused_gate_sigmoid_mul_add_kernel( hidden_states_ptr, # [num_tokens, hidden_dim] diff --git a/python/sglang/kernels/ops/attention/mla_output_gate.py b/python/sglang/kernels/ops/elementwise/mla_output_gate.py similarity index 94% rename from python/sglang/kernels/ops/attention/mla_output_gate.py rename to python/sglang/kernels/ops/elementwise/mla_output_gate.py index 9b0ca7c2a7..ff6005c645 100644 --- a/python/sglang/kernels/ops/attention/mla_output_gate.py +++ b/python/sglang/kernels/ops/elementwise/mla_output_gate.py @@ -48,6 +48,7 @@ def covered(x: torch.Tensor, gate: torch.Tensor) -> bool: ) +# Not fused_sigmoid_mul: this rounds sigmoid(gate) to bf16 before the multiply. def kimi_k3_mla_output_gate(x: torch.Tensor, gate: torch.Tensor) -> torch.Tensor: """out = bf16(x * bf16(sigmoid(gate))); double rounding matches the unfused torch.sigmoid + mul pair bit-for-bit. Caller checks covered().""" diff --git a/python/sglang/kernels/ops/elementwise/row_scale.py b/python/sglang/kernels/ops/elementwise/row_scale.py index ec8420c0e5..21000c7de9 100644 --- a/python/sglang/kernels/ops/elementwise/row_scale.py +++ b/python/sglang/kernels/ops/elementwise/row_scale.py @@ -1,13 +1,14 @@ -"""CUDA-JIT vectorized per-row scale (the apply_log_scaling_tau contract): -``out[row, :] = bf16(fp32(x[row, :]) * tau[row])``. See -csrc/tml/inkling_row_scale.cuh; the scalar triton kernel remains the fallback -for non-bf16 / unaligned inputs.""" +"""Per-row scale ``out[row, :] = bf16(fp32(x[row, :]) * tau[row])``: the CUDA-JIT +vectorized kernel (csrc/inkling/inkling_row_scale.cuh) and the scalar Triton +kernel that remains the fallback for non-bf16 / unaligned inputs.""" from __future__ import annotations from typing import TYPE_CHECKING import torch +import triton +import triton.language as tl from sglang.kernels.jit.utils import ( cache_once, @@ -45,3 +46,68 @@ def row_scale_bf16( module = _jit_row_scale_module(is_arch_support_pdl()) module.run(x, tau, out) return out + + +@triton.jit +def _apply_log_scaling_tau_kernel( + x_ptr, + tau_ptr, # [rows] fp32 (flattened per-row scale) + out_ptr, # [rows, inner] contiguous, same dtype as x + x_row_stride, + inner, + total, + BLOCK: tl.constexpr, +): + pid = tl.program_id(0) + offs = pid.to(tl.int64) * BLOCK + tl.arange(0, BLOCK) + mask = offs < total + row = offs // inner + col = offs % inner + x = tl.load(x_ptr + row * x_row_stride + col, mask=mask).to(tl.float32) + tau = tl.load(tau_ptr + row, mask=mask) + y = x * tau + tl.store(out_ptr + offs, y.to(out_ptr.dtype.element_ty), mask=mask) + + +def apply_log_scaling_tau(x: torch.Tensor, tau: torch.Tensor) -> torch.Tensor: + """out = (x.float() * tau).to(x.dtype) with tau broadcast per leading row, + fused into one launch. x may carry a leading-dim stride (the q slice of the + fused qkvr output); its trailing dims must be contiguous. No dynamo: the + torch.compile'd predecessor's call sites spanned enough rank / + dispatch-key / 0-1 specialization variants (target + de-tied MTP heads) to + exceed the recompile limit, which crashed (fullgraph) or wedged capture + (raised limit).""" + rows = x.shape[0] + inner = x.numel() // rows if rows else 0 + inner_contiguous = x.stride(-1) == 1 and ( + x.dim() == 2 or x.stride(-2) == x.shape[-1] * x.stride(-1) + ) + if rows == 0 or inner == 0 or not inner_contiguous: + return (x.float() * tau).to(x.dtype) + + if ( + x.is_cuda + and x.dtype == torch.bfloat16 + and inner % 8 == 0 + and x.data_ptr() % 16 == 0 + and (x.stride(0) * 2) % 16 == 0 + ): + # Vectorized JIT kernel (16B loads, one row divide per vector) -- + # bit-identical output (same fp32-mul + bf16-round), ~2-3x the + # scalar triton kernel below at every size. + x2d = torch.as_strided(x, (rows, inner), (x.stride(0), 1)) + return row_scale_bf16(x2d, tau.reshape(rows).float()).view(x.shape) + + out = torch.empty(x.shape, dtype=x.dtype, device=x.device) + total = rows * inner + BLOCK = 1024 + _apply_log_scaling_tau_kernel[(triton.cdiv(total, BLOCK),)]( + x, + tau.reshape(rows).to(torch.float32), + out, + x.stride(0), + inner, + total, + BLOCK=BLOCK, + ) + return out diff --git a/python/sglang/kernels/ops/muse_spark/row_sum.py b/python/sglang/kernels/ops/elementwise/row_sum.py similarity index 85% rename from python/sglang/kernels/ops/muse_spark/row_sum.py rename to python/sglang/kernels/ops/elementwise/row_sum.py index 41b7bcd0b6..1c49eac2f5 100644 --- a/python/sglang/kernels/ops/muse_spark/row_sum.py +++ b/python/sglang/kernels/ops/elementwise/row_sum.py @@ -1,7 +1,9 @@ -"""Row sum over [N, K] fp32, bitwise-identical to ``sglang.kernels.ops.muse_spark.triton_sum``. +"""Row sum over [N, K] fp32, bitwise-identical to ``sglang.test.kernels.triton_sum``. Same contract: the reduction order is fixed by the kernel (tl.sum's pairwise tree), not by the CUDA runtime version, so results are stable across toolkits. +The match does not hold when K is a multiple of 16: Triton specializes such a K +and splits each row across lanes. The router top-k it serves is 8. """ from __future__ import annotations diff --git a/python/sglang/kernels/ops/speculative/ngram_embedding.py b/python/sglang/kernels/ops/embeddings/ngram_embedding.py similarity index 100% rename from python/sglang/kernels/ops/speculative/ngram_embedding.py rename to python/sglang/kernels/ops/embeddings/ngram_embedding.py diff --git a/python/sglang/kernels/ops/gemm/__init__.py b/python/sglang/kernels/ops/gemm/__init__.py index 515d4c943e..804f0d2641 100644 --- a/python/sglang/kernels/ops/gemm/__init__.py +++ b/python/sglang/kernels/ops/gemm/__init__.py @@ -164,11 +164,22 @@ register_kernel( capabilities=_SM10X, ) ) +register_kernel( + KernelSpec( + op="gemm.nvfp4_gemm_swiglu_nvfp4_quant", + backend=KernelBackend.CUTE_DSL, + target=( + "sglang.kernels.ops.gemm.nvfp4_gemm_swiglu_nvfp4_quant" + ":nvfp4_gemm_swiglu_nvfp4_quant" + ), + capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(10, 0))}), + ) +) register_kernel( KernelSpec( op="gemm.bmm_fp8", backend=KernelBackend.FLASHINFER, - target="sglang.srt.layers.quantization.fp8_utils:bmm_fp8", + target="sglang.kernels.ops.gemm.flashinfer_bmm_fp8:bmm_fp8", capabilities=CUDA_ONLY, ) ) @@ -492,3 +503,11 @@ register_kernel( target="sglang.kernels.ops.gemm.int8_kernel:w8a8_block_int8_matmul", ) ) +register_kernel( + KernelSpec( + op="gemm.muse_spark_router_gemm", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.gemm.muse_spark_router_gemm:router_gemm", + capabilities=CUDA_ONLY, + ) +) diff --git a/python/sglang/kernels/ops/gemm/dsv4_indexer_head_weights_hip.py b/python/sglang/kernels/ops/gemm/dsv4_indexer_head_weights_hip.py new file mode 100644 index 0000000000..22abf5b83b --- /dev/null +++ b/python/sglang/kernels/ops/gemm/dsv4_indexer_head_weights_hip.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k + + +@triton.jit +def _reduce_scale_bf16_block( + block, + part_ptr, + out_ptr, + M, + stride_ps, + stride_pm, + stride_om, + scale, + N: tl.constexpr, + SPLIT_K: tl.constexpr, + BLOCK: tl.constexpr, +): + """One BLOCK of _reduce_scale_bf16_kernel (shared with the fused index-Q launch).""" + offs = block * BLOCK + tl.arange(0, BLOCK) + m = offs // N + n = offs % N + mask = m < M + acc = tl.load(part_ptr + m * stride_pm + n, mask=mask, other=0.0) + for s in tl.static_range(1, SPLIT_K): + acc += tl.load( + part_ptr + s * stride_ps + m * stride_pm + n, mask=mask, other=0.0 + ) + # bf16(bf16(sum) * scale): the linear's bf16 output, then the aten multiply rounded to bf16 + w = acc.to(tl.bfloat16).to(tl.float32) * scale + tl.store(out_ptr + m * stride_om + n, w.to(tl.bfloat16), mask=mask) + + +@triton.jit +def _reduce_scale_bf16_kernel( + part_ptr, + out_ptr, + M, + stride_ps, + stride_pm, + stride_om, + scale, + N: tl.constexpr, + SPLIT_K: tl.constexpr, + BLOCK: tl.constexpr, +): + _reduce_scale_bf16_block( + tl.program_id(0), + part_ptr, + out_ptr, + M, + stride_ps, + stride_pm, + stride_om, + scale, + N=N, + SPLIT_K=SPLIT_K, + BLOCK=BLOCK, + ) + + +def indexer_head_weights( + x: torch.Tensor, weight: torch.Tensor, scale: float +) -> torch.Tensor: + """bf16(bf16(x @ weight.T) * scale) as a contiguous bf16 [M, N], the layout the + FlyDSL logits kernels take. x bf16 [M, K] and weight bf16 [N, K], with + M <= rocm_gemv_split_k_max_tokens(n=N, k=K, weight_dtype=torch.bfloat16).""" + partials = rocm_router_gemv_split_k(x, weight) + split_k, M, N = partials.shape + out = torch.empty((M, N), dtype=torch.bfloat16, device=x.device) + block = triton.next_power_of_2(M * N) + _reduce_scale_bf16_kernel[(triton.cdiv(M * N, block),)]( + partials, + out, + M, + partials.stride(0), + partials.stride(1), + out.stride(0), + float(scale), + N=N, + SPLIT_K=split_k, + BLOCK=block, + num_warps=4, + ) + return out diff --git a/python/sglang/kernels/ops/gemm/dsv4_wo_a.py b/python/sglang/kernels/ops/gemm/dsv4_wo_a.py index 8ed5c910b1..80733a710d 100644 --- a/python/sglang/kernels/ops/gemm/dsv4_wo_a.py +++ b/python/sglang/kernels/ops/gemm/dsv4_wo_a.py @@ -4,7 +4,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.layernorm.mxfp8_epilogue import ue8m0_scale +from sglang.kernels.ops.quantization.mxfp8_epilogue import ue8m0_scale @triton.jit diff --git a/python/sglang/kernels/ops/gemm/flashinfer_bmm_fp8.py b/python/sglang/kernels/ops/gemm/flashinfer_bmm_fp8.py new file mode 100644 index 0000000000..530ae15838 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/flashinfer_bmm_fp8.py @@ -0,0 +1,42 @@ +"""Batched (3D) per-tensor-scale FP8 matmul through FlashInfer's cuBLAS backend.""" + +from typing import Optional + +import torch + +from sglang.srt.utils.custom_op import register_custom_op +from sglang.utils import LazyImport + +# Resolved on first call: CUDA builds without FlashInfer must still import +# this module. +_raw_bmm_fp8_batched = LazyImport("flashinfer", "bmm_fp8") + + +@register_custom_op(op_name="flashinfer_bmm_fp8_batched", mutates_args=["out"]) +def _bmm_fp8_batched_op( + A: torch.Tensor, + B: torch.Tensor, + out: torch.Tensor, + A_scale: torch.Tensor, + B_scale: torch.Tensor, +) -> None: + _raw_bmm_fp8_batched(A, B, A_scale, B_scale, out.dtype, out) + + +def bmm_fp8( + A: torch.Tensor, + B: torch.Tensor, + A_scale: torch.Tensor, + B_scale: torch.Tensor, + dtype: torch.dtype, + out: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """Batched (3D) per-tensor-scale FP8 matmul, via flashinfer's cuBLAS backend.""" + if out is None: + out = torch.empty( + (A.shape[0], A.shape[1], B.shape[2]), + device=A.device, + dtype=dtype, + ) + _bmm_fp8_batched_op(A, B, out, A_scale, B_scale) + return out diff --git a/python/sglang/kernels/ops/elementwise/hc_mix.py b/python/sglang/kernels/ops/gemm/hc_mix_splitk.py similarity index 97% rename from python/sglang/kernels/ops/elementwise/hc_mix.py rename to python/sglang/kernels/ops/gemm/hc_mix_splitk.py index cc6054f0d8..0f2ff807b9 100644 --- a/python/sglang/kernels/ops/elementwise/hc_mix.py +++ b/python/sglang/kernels/ops/gemm/hc_mix_splitk.py @@ -58,6 +58,7 @@ def _get_tactic(m: int, n: int, k: int): return tactic +# Not fused_hc_mix: fixed split-K order vs its device-scope atomics, so the bits differ. def hc_mix( hyper_input_normed: torch.Tensor, w_down: torch.Tensor, diff --git a/python/sglang/kernels/ops/muse_spark/router_gemm.py b/python/sglang/kernels/ops/gemm/muse_spark_router_gemm.py similarity index 100% rename from python/sglang/kernels/ops/muse_spark/router_gemm.py rename to python/sglang/kernels/ops/gemm/muse_spark_router_gemm.py diff --git a/python/sglang/kernels/ops/quantization/mxfp8_aiter_gfx95.py b/python/sglang/kernels/ops/gemm/mxfp8_aiter_gfx95.py similarity index 100% rename from python/sglang/kernels/ops/quantization/mxfp8_aiter_gfx95.py rename to python/sglang/kernels/ops/gemm/mxfp8_aiter_gfx95.py diff --git a/python/sglang/kernels/ops/gemm/mxfp8_amd_gfx95.py b/python/sglang/kernels/ops/gemm/mxfp8_amd_gfx95.py new file mode 100644 index 0000000000..9156597215 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/mxfp8_amd_gfx95.py @@ -0,0 +1,256 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Dense MXFP8 linears for AMD CDNA4 (gfx950). + + * dense GEMM via Triton ``tl.dot_scaled`` (consumes FP8 E4M3 weights + E8M0 + block scales directly, no dequant-to-BF16), lowering to the CDNA4 native MX + matrix-core ops; ``K % 128 != 0`` falls back to dequant + ``F.linear``. + * the bf16-dequant linear for shapes the native route does not tile. + +Replaces the FlyDSL ``v_mfma_scale_f32_32x32x64`` dense path with a single +Triton ``dot_scaled`` GEMM: no load-time weight reformat (fp8 + E8M0 are +consumed as-is) and the activation is MXFP8-quantized in one fused pass. +""" + +from __future__ import annotations + +from typing import Optional + +import torch +import torch.nn.functional as F +import triton +import triton.language as tl + +from sglang.kernels.jit.utils import cache_once +from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( + MXFP8_VALUE_DTYPE, + dequant_mxfp8_to_bf16, + fake_quant_fp8_activation, + mxfp8_e4m3_quantize, +) + + +# --------------------------------------------------------------------------- # +# Dense MXFP8 linear via Triton tl.dot_scaled (CDNA4 native microscaling) +# --------------------------------------------------------------------------- # +@triton.jit +def _mxfp8_linear_kernel( + x_ptr, + xs_ptr, + w_ptr, + ws_ptr, + out_ptr, + M, + N, + K, + stride_xm, + stride_xk, + stride_xsm, + stride_xsk, + stride_wn, + stride_wk, + stride_wsn, + stride_wsk, + stride_om, + stride_on, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, +): + pid_m = tl.program_id(0) + pid_n = tl.program_id(1) + offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) + offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + offs_k = tl.arange(0, BLOCK_K) + offs_sk = tl.arange(0, BLOCK_K // 32) + m_mask = offs_m < M + n_mask = offs_n < N + + x_ptrs = x_ptr + offs_m[:, None] * stride_xm + offs_k[None, :] * stride_xk + xs_ptrs = xs_ptr + offs_m[:, None] * stride_xsm + offs_sk[None, :] * stride_xsk + w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk + ws_ptrs = ws_ptr + offs_n[:, None] * stride_wsn + offs_sk[None, :] * stride_wsk + + acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for _ in range(0, tl.cdiv(K, BLOCK_K)): + x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0) + w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0) + xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=0) + ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=0) + acc += tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3") + x_ptrs += BLOCK_K * stride_xk + w_ptrs += BLOCK_K * stride_wk + xs_ptrs += (BLOCK_K // 32) * stride_xsk + ws_ptrs += (BLOCK_K // 32) * stride_wsk + + o_ptrs = out_ptr + offs_m[:, None] * stride_om + offs_n[None, :] * stride_on + tl.store( + o_ptrs, + acc.to(out_ptr.dtype.element_ty), + mask=m_mask[:, None] & n_mask[None, :], + ) + + +def _run_mxfp8_linear_kernel( + x_q: torch.Tensor, # [M, K] fp8 e4m3 + x_scale: torch.Tensor, # [M, K//32] uint8 (E8M0) + w: torch.Tensor, # [N, K] fp8 e4m3 + w_scale: torch.Tensor, # [N, K//32] uint8 (E8M0) + out_dtype: torch.dtype, +) -> torch.Tensor: + M, K = x_q.shape + N = w.shape[0] + out = torch.empty((M, N), dtype=out_dtype, device=x_q.device) + BLOCK_M, BLOCK_K = 64, 128 + if M <= 512 and (K >= 4096 or (N == 6144 and K in (2048, 3072))): + BLOCK_N, num_warps = 64, 4 + else: + BLOCK_N, num_warps = 128, 8 + grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, BLOCK_N)) + _mxfp8_linear_kernel[grid]( + x_q, + x_scale, + w, + w_scale, + out, + M, + N, + K, + x_q.stride(0), + x_q.stride(1), + x_scale.stride(0), + x_scale.stride(1), + w.stride(0), + w.stride(1), + w_scale.stride(0), + w_scale.stride(1), + out.stride(0), + out.stride(1), + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + BLOCK_K=BLOCK_K, + num_warps=num_warps, + ) + return out + + +def _mxfp8_dot_scaled_linear( + x: torch.Tensor, # [M, K] bf16/fp16 + w: torch.Tensor, # [N, K] fp8 e4m3 + w_scale: torch.Tensor, # [N, K//32] uint8 (E8M0) +) -> torch.Tensor: + """bf16/fp16 input -> per-token MXFP8 quant -> dot_scaled GEMM.""" + x_q, x_scale = mxfp8_e4m3_quantize(x) + return _run_mxfp8_linear_kernel(x_q, x_scale, w, w_scale, x.dtype) + + +def dot_scaled_mxfp8_blockscaled_linear( + input: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + input_scale: Optional[torch.Tensor] = None, + bias: Optional[torch.Tensor] = None, + output_dtype: Optional[torch.dtype] = None, +) -> torch.Tensor: + """Native dense MXFP8 linear (CDNA4 ``tl.dot_scaled``). + + Consumes FP8 E4M3 ``weight`` + canonical 2D UE8M0 ``weight_scale`` [N, K//32] + directly. Activations are MXFP8-quantized per token inside the kernel path. + Drop-in for the SGLang ``w8a8_mxfp8_linear`` callable signature. + """ + assert weight.dtype == torch.float8_e4m3fn, "MXFP8 weight must be FP8 E4M3." + assert weight_scale.dtype == torch.uint8, "MXFP8 weight_scale must be UE8M0 uint8." + assert weight_scale.dim() == 2, ( + "dot_scaled MXFP8 linear expects canonical 2D [N, K//32] weight scales, " + f"got {weight_scale.dim()}D." + ) + + input_2d = input.view(-1, input.shape[-1]).contiguous() + output_shape = [*input.shape[:-1], weight.shape[0]] + if output_dtype is None: + output_dtype = ( + input_2d.dtype + if input_2d.dtype in (torch.float16, torch.bfloat16, torch.float32) + else torch.bfloat16 + ) + + m, k = input_2d.shape + n, k_w = weight.shape + assert k == k_w, f"{k=} does not match {k_w=}" + + if k % 128 == 0: + if input_scale is None: + # Quantize the bf16/fp16 activations per token inside the path. + x_q, x_scale = mxfp8_e4m3_quantize(input_2d) + kernel_out_dtype = input_2d.dtype + else: + # Activations already MXFP8-quantized by a fused upstream op. + assert input_2d.dtype == MXFP8_VALUE_DTYPE, ( + "pre-quantized input must be FP8 E4M3 when input_scale is given." + ) + assert input_scale.dtype == torch.uint8 and input_scale.shape == ( + m, + k // 32, + ), "input_scale must be UE8M0 uint8 [M, K//32]." + x_q, x_scale = input_2d, input_scale + kernel_out_dtype = output_dtype + out = _run_mxfp8_linear_kernel( + x_q, x_scale, weight, weight_scale, kernel_out_dtype + ) + else: + # dot_scaled tiling needs K % 128 == 0; dequantize fallback otherwise. + w_bf16 = dequant_mxfp8_to_bf16(weight, weight_scale) + if input_scale is not None: + input_2d = dequant_mxfp8_to_bf16(input_2d, input_scale) + out = F.linear(input_2d.to(w_bf16.dtype), w_bf16).to(output_dtype) + + if bias is not None: + out = out + bias + return out.to(output_dtype).view(*output_shape) + + +# at one or two rows aiter's wvSpltK (fixed-order split-K) beats hipBLASLt's small-M kernels +_SKINNY_GEMM_MAX_TOKENS = 2 + + +@cache_once +def _skinny_cu_count() -> int: + try: + from aiter.jit.utils.chip_info import get_cu_num + except ImportError: + return 0 + return int(get_cu_num()) + + +# both operands are exact in bf16, so only the fp32 sum order differs from the MXFP8 route +def bf16_dequant_blockscaled_linear( + input: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor] = None, + input_on_fp8_grid: bool = False, +) -> torch.Tensor: + """weight is the dequantized bf16 weight (dequant_block_fp8_weight_to_bf16). + input_on_fp8_grid says a fused upstream kernel (rmsnorm_fake_quant_fp8) + already put input on the fp8 grid, so the fake-quant here is skipped.""" + assert weight.dtype == torch.bfloat16, weight.dtype + input_2d = input.view(-1, input.shape[-1]) + if input_on_fp8_grid: + x = input_2d.to(torch.bfloat16) + else: + x = fake_quant_fp8_activation(input_2d.to(torch.bfloat16)) + cu_count = _skinny_cu_count() + if ( + bias is None + and x.shape[0] <= _SKINNY_GEMM_MAX_TOKENS + and cu_count > 0 + and weight.is_contiguous() + and weight.shape[1] % 8 == 0 + ): + import aiter + + out = torch.empty( + (x.shape[0], weight.shape[0]), dtype=torch.bfloat16, device=x.device + ) + aiter.wvSpltK(weight, x, out, x.shape[0], cu_count) + else: + out = F.linear(x, weight, bias) + return out.view(*input.shape[:-1], weight.shape[0]) diff --git a/python/sglang/kernels/ops/quantization/mxfp8_flydsl_gfx95.py b/python/sglang/kernels/ops/gemm/mxfp8_flydsl_gfx95.py similarity index 100% rename from python/sglang/kernels/ops/quantization/mxfp8_flydsl_gfx95.py rename to python/sglang/kernels/ops/gemm/mxfp8_flydsl_gfx95.py diff --git a/python/sglang/kernels/ops/quantization/mxfp8_gemv_gfx95_configs.json b/python/sglang/kernels/ops/gemm/mxfp8_gemv_gfx95_configs.json similarity index 100% rename from python/sglang/kernels/ops/quantization/mxfp8_gemv_gfx95_configs.json rename to python/sglang/kernels/ops/gemm/mxfp8_gemv_gfx95_configs.json diff --git a/python/sglang/kernels/ops/quantization/mxfp8_native_amd_gfx95.py b/python/sglang/kernels/ops/gemm/mxfp8_native_amd_gfx95.py similarity index 100% rename from python/sglang/kernels/ops/quantization/mxfp8_native_amd_gfx95.py rename to python/sglang/kernels/ops/gemm/mxfp8_native_amd_gfx95.py diff --git a/python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py b/python/sglang/kernels/ops/gemm/nvfp4_gemm_swiglu_nvfp4_quant.py similarity index 100% rename from python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py rename to python/sglang/kernels/ops/gemm/nvfp4_gemm_swiglu_nvfp4_quant.py diff --git a/python/sglang/kernels/ops/gemm/router_gate_matvec.py b/python/sglang/kernels/ops/gemm/router_gate_matvec.py new file mode 100644 index 0000000000..ea4251b580 --- /dev/null +++ b/python/sglang/kernels/ops/gemm/router_gate_matvec.py @@ -0,0 +1,131 @@ +"""Small-M router-gate logits as a single Triton matvec launch.""" + +import torch +import triton +import triton.language as tl + +from sglang.kernels.jit.utils import is_arch_support_pdl + + +@triton.jit +def router_gate_matvec_kernel( + x_ptr, # (M, K) bf16/fp16/fp32, row-major + w_ptr, # (E, K) fp32/bf16/fp16, k-major + out_ptr, # (M, E) fp32 + K, + E, + stride_xm, + stride_we, + BLOCK_E: tl.constexpr, + BLOCK_K: tl.constexpr, + USE_GDC: tl.constexpr = False, +): + """Router-gate logits as a single matvec launch, fp32 accumulation for + any float weight dtype. BLOCK_K covers the whole K in one masked load + (single iteration for K <= BLOCK_K): a cold gate weight then costs one + HBM round trip per CTA instead of a serial dependent-load chain, which + is what dominates in the real model where ~94MB/layer of expert traffic + flushes L2 between gate calls. + """ + pid_m = tl.program_id(0) + pid_e = tl.program_id(1) + e_offs = pid_e * BLOCK_E + tl.arange(0, BLOCK_E) + e_mask = e_offs < E + + # First K tile, weight load ahead of the PDL wait (see docstring). + k_offs = tl.arange(0, BLOCK_K) + k_mask = k_offs < K + w = tl.load( + w_ptr + e_offs[:, None] * stride_we + k_offs[None, :], + mask=e_mask[:, None] & k_mask[None, :], + other=0.0, + ).to(tl.float32) + if USE_GDC: + tl.extra.cuda.gdc_wait() + tl.extra.cuda.gdc_launch_dependents() + x = tl.load(x_ptr + pid_m * stride_xm + k_offs, mask=k_mask, other=0.0).to( + tl.float32 + ) + acc = tl.sum(w * x[None, :], axis=1) + + for k0 in range(BLOCK_K, K, BLOCK_K): + k_offs = k0 + tl.arange(0, BLOCK_K) + k_mask = k_offs < K + x = tl.load(x_ptr + pid_m * stride_xm + k_offs, mask=k_mask, other=0.0).to( + tl.float32 + ) + w = tl.load( + w_ptr + e_offs[:, None] * stride_we + k_offs[None, :], + mask=e_mask[:, None] & k_mask[None, :], + other=0.0, + ).to(tl.float32) + acc += tl.sum(w * x[None, :], axis=1) + + tl.store( + out_ptr + pid_m * E + e_offs, acc.to(out_ptr.dtype.element_ty), mask=e_mask + ) + + +# Cold-cache tuned on H20-3e (41 rotating gate weights so each call misses L2, +# like the real model); expected to carry to B200 (more SMs favor the wide +# grid even more) — re-tune with benchmark/kernels/bench_router_gate_matvec.py. +ROUTER_GATE_MATVEC_BLOCK_E = 4 +ROUTER_GATE_MATVEC_NUM_WARPS = 8 +# Beyond this M the per-M re-reads of the gate weight outgrow the library +# GEMM (cold H20-3e: bf16 wins to M=12, fp32 to M=8; cap at the lower). +ROUTER_GATE_MATVEC_MAX_M = 8 + + +def router_gate_matvec( + hidden_states: torch.Tensor, weight: torch.Tensor +) -> torch.Tensor: + """Small-M router-gate logits: one triton launch replacing the library + path — for fp32 gate weights the eager upcast + fp32 GEMM + splitKreduce + triple, for bf16 the F.linear GEMV. Returns fp32 (M, E) logits with fp32 + accumulation (deterministic; for fp32 weights 0 top-8 routing flips over + 30104 random draws vs the fp32 reference; for bf16 weights this is + slightly MORE precise than the library GEMV, so near-tie logits can + round-trip differently — same order as the bf16-vs-fp32 gate change). + + Cold-cache (41 rotating weights, in-graph, H20-3e, E=513, K=2560), us/call: + + M lib bf16 matvec bf16 lib fp32 chain matvec fp32 + 1 4.2 4.3 6.4 6.2 + 2 13.8 4.5 14.6 6.4 + 4 14.0 5.4 20.7 10.4 + 8 14.6 10.4 20.1 18.1 + 16 14.6 18.9 (lib) 20.9 30.9 (lib) + + Callers must gate on M <= ROUTER_GATE_MATVEC_MAX_M; prefill-sized M + keeps the library GEMM.""" + assert ( + weight.dtype + in ( + torch.float32, + torch.bfloat16, + torch.float16, + ) + and weight.is_contiguous() + ) + M, K = hidden_states.shape + E = weight.shape[0] + out = torch.empty((M, E), dtype=torch.float32, device=hidden_states.device) + block_e = ROUTER_GATE_MATVEC_BLOCK_E + # Single k-iteration whenever K fits one block: no serial dependent-load + # chain on a cold weight. + block_k = min(4096, triton.next_power_of_2(K)) + pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {} + router_gate_matvec_kernel[(M, triton.cdiv(E, block_e))]( + hidden_states, + weight, + out, + K, + E, + hidden_states.stride(0), + weight.stride(0), + BLOCK_E=block_e, + BLOCK_K=block_k, + num_warps=ROUTER_GATE_MATVEC_NUM_WARPS, + **pdl_kwargs, + ) + return out diff --git a/python/sglang/kernels/ops/gemm/smallm_fp8_gfx950/__init__.py b/python/sglang/kernels/ops/gemm/smallm_fp8_gfx950/__init__.py index 87a1d7c27f..eb4742a623 100644 --- a/python/sglang/kernels/ops/gemm/smallm_fp8_gfx950/__init__.py +++ b/python/sglang/kernels/ops/gemm/smallm_fp8_gfx950/__init__.py @@ -12,12 +12,12 @@ import tempfile import torch -from sglang.kernels.ops.moe.smallm_moe_gfx950 import ( - _check, - _hip_lib, - _hip_version, - _hipcc, - _Kernel, +from sglang.kernels.jit.utils.hipcc import ( + HipKernel, + check_hip, + hip_lib, + hip_version, + hipcc_path, ) from sglang.srt.environ import envs @@ -53,13 +53,23 @@ def smallm_fp8_gemm_enabled() -> bool: if _mod is None: _mod = False arch = torch.cuda.get_device_properties(0).gcnArchName.split(":")[0] - if arch == "gfx950" and _hip_version() >= (7, 2, 0): + if arch == "gfx950" and hip_version() >= (7, 2, 0): co = os.path.join(tempfile.mkdtemp(prefix="sglang_smallm_fp8_"), "k.co") - cmd = [_hipcc(), "--genco", "--offload-arch=gfx950", "-O3", "-o", co, _SRC] + cmd = [ + hipcc_path(), + "--genco", + "--offload-arch=gfx950", + "-O3", + "-o", + co, + _SRC, + ] try: subprocess.run(cmd, check=True, capture_output=True) mod = ctypes.c_void_p() - _check(_hip_lib().hipModuleLoad(ctypes.byref(mod), co.encode()), "load") + check_hip( + hip_lib().hipModuleLoad(ctypes.byref(mod), co.encode()), "load" + ) _mod = mod except (OSError, subprocess.CalledProcessError, RuntimeError) as e: logger.warning("smallm_fp8_gemm disabled: %s", e) @@ -98,7 +108,7 @@ def smallm_fp8_gemm(XQ, WQ, x_scale, w_scale, dtype): kernel = _kernels.get((mt, *cfg)) if kernel is None: name = f"smallm_fp8_gemm_m{mt}_n{nt}_s{steps}_w{waves}_x{xlds}" - kernel = _kernels[(mt, *cfg)] = _Kernel(_mod, name) + kernel = _kernels[(mt, *cfg)] = HipKernel(_mod, name) out = torch.empty(M, N, dtype=torch.bfloat16, device=XQ.device) xs = x_scale.view(M, -1)[:, 0] ptrs = (XQ, WQ, xs, w_scale.contiguous(), out) diff --git a/python/sglang/kernels/ops/kv_canary/__init__.py b/python/sglang/kernels/ops/kv_canary/__init__.py index e69de29bb2..f766a91622 100644 --- a/python/sglang/kernels/ops/kv_canary/__init__.py +++ b/python/sglang/kernels/ops/kv_canary/__init__.py @@ -0,0 +1,25 @@ +"""KV-cache canary kernels: plan, write and verify canary slots, and scatter +each request's expected token ids into its pool row.""" + +from sglang.kernels.registry import register_kernel +from sglang.kernels.spec import KernelBackend, KernelSpec + +# The launchers fall back to their torch references off CUDA, so no device pin. +for _mod, _fn, _backend in ( + ("plan.api", "launch_canary_plan_kernels", KernelBackend.JIT), + ("write", "launch_canary_write_kernel", KernelBackend.JIT), + ("verify", "launch_canary_verify_kernel", KernelBackend.JIT), + ( + "scatter_req_token_ids", + "launch_scatter_req_token_ids_kernel", + KernelBackend.TRITON, + ), +): + register_kernel( + KernelSpec( + op=f"kv_canary.{_fn}", + backend=_backend, + target=f"sglang.kernels.ops.kv_canary.{_mod}:{_fn}", + ) + ) +del _mod, _fn, _backend diff --git a/python/sglang/kernels/ops/kv_canary/scatter_req_token_ids.py b/python/sglang/kernels/ops/kv_canary/scatter_req_token_ids.py index 34f829c87c..fef3da8695 100644 --- a/python/sglang/kernels/ops/kv_canary/scatter_req_token_ids.py +++ b/python/sglang/kernels/ops/kv_canary/scatter_req_token_ids.py @@ -164,30 +164,3 @@ def _scatter_req_token_ids_kernel( tl.int32 ) # [TOKEN_BLOCK] int32 tl.store(pool_out_ptr + rp * pool_stride0 + pos, val, mask=write_mask) - - -def scatter_req_token_ids_torch_reference( - *, - flat_in: torch.Tensor, - offsets: torch.Tensor, - req_pool_indices: torch.Tensor, - pool_out: torch.Tensor, -) -> None: - """Plain-PyTorch reference for :func:`launch_scatter_req_token_ids_kernel`.""" - bs = int(req_pool_indices.shape[0]) - offsets_host = offsets.detach().cpu().tolist() - req_pool_indices_host = req_pool_indices.detach().cpu().tolist() - flat_host = flat_in.detach().cpu() - pool_max_context_len = int(pool_out.shape[1]) - - for r in range(bs): - start = int(offsets_host[r]) - end = int(offsets_host[r + 1]) - if end <= start: - continue - rp = int(req_pool_indices_host[r]) - seg = flat_host[start:end].to(torch.int32) - write_len = min(int(seg.shape[0]), pool_max_context_len) - if write_len <= 0: - continue - pool_out[rp, :write_len] = seg[:write_len].to(pool_out.device) diff --git a/python/sglang/kernels/ops/kvcache/__init__.py b/python/sglang/kernels/ops/kvcache/__init__.py index 454870cae0..10e56ebe5f 100644 --- a/python/sglang/kernels/ops/kvcache/__init__.py +++ b/python/sglang/kernels/ops/kvcache/__init__.py @@ -54,21 +54,11 @@ __all__ = ["reshape_and_cache_flash"] # Other Triton kernels migrated into this group (from attention/mem_cache # triton_ops); registered for inventory. Import them from their modules. _TRITON_KERNELS = [ - ("cache_ops", "concat_and_cast_mha_k_triton"), - ("cache_ops", "launch_reshape_and_cache_flash"), + ("shuffle_5d_cache", "launch_reshape_and_cache_shuffle_5d"), + ("shuffle_5d_cache", "launch_gather_shuffle_5d_to_linear"), ("pd_dcp_gather", "copy_mla_rows_into_pack"), ("kv_read_table", "build_kv_read_table"), ("kv_read_table", "build_kv_read_table_packed"), - ("kv_indices", "create_flashinfer_kv_indices_triton"), - ("kv_indices", "create_flashmla_kv_indices_triton"), - ("kv_indices", "create_chunked_prefix_cache_kv_indices"), - ("kv_indices", "get_num_kv_index_blocks_flashmla"), - ("kv_indices", "get_num_page_per_block_flashmla"), - ("rope_cache", "fused_qk_rope_reshape_and_cache"), - ("trtllm_mha_page_table", "build_trtllm_mha_page_table"), - ("trtllm_mha_graph_metadata", "update_trtllm_mha_graph_metadata"), - ("aiter_unified_attention", "scatter_ragged_to_page_table_kernel"), - ("aiter_unified_attention", "scatter_req_to_token_to_page_table_kernel"), ("cache_move", "set_kv_buffer_prefix_valid_tiled"), ("cache_move", "copy_all_layer_kv_cache_tiled"), ("mla_buffer", "set_mla_kv_buffer_triton"), @@ -83,3 +73,13 @@ for _mod, _fn in _TRITON_KERNELS: ) ) del _mod, _fn + +register_kernel( + KernelSpec( + op="kvcache.store_fp4_index_k_cache", + backend=KernelBackend.TRITON, + target=( + "sglang.kernels.ops.kvcache.dsv4_fp4_index_k_cache:store_fp4_index_k_cache" + ), + ) +) diff --git a/python/sglang/kernels/ops/kvcache/cache_ops.py b/python/sglang/kernels/ops/kvcache/cache_ops.py index 803b15ccc8..c8496856a6 100644 --- a/python/sglang/kernels/ops/kvcache/cache_ops.py +++ b/python/sglang/kernels/ops/kvcache/cache_ops.py @@ -1,157 +1,7 @@ -import torch import triton import triton.language as tl -@triton.jit -def concat_and_cast_mha_k_kernel( - k_ptr, - k_nope_ptr, - k_rope_ptr, - head_cnt: tl.constexpr, - k_stride0: tl.constexpr, - k_stride1: tl.constexpr, - nope_stride0: tl.constexpr, - nope_stride1: tl.constexpr, - rope_stride0: tl.constexpr, - nope_dim: tl.constexpr, - rope_dim: tl.constexpr, -): - pid_loc = tl.program_id(0) - head_range = tl.arange(0, head_cnt) - - k_head_ptr = k_ptr + pid_loc * k_stride0 + head_range[:, None] * k_stride1 - - nope_offs = tl.arange(0, nope_dim) - - src_nope_ptr = ( - k_nope_ptr - + pid_loc * nope_stride0 - + head_range[:, None] * nope_stride1 - + nope_offs[None, :] - ) - dst_nope_ptr = k_head_ptr + nope_offs[None, :] - - src_nope = tl.load(src_nope_ptr) - tl.store(dst_nope_ptr, src_nope) - - rope_offs = tl.arange(0, rope_dim) - src_rope_ptr = k_rope_ptr + pid_loc * rope_stride0 + rope_offs[None, :] - dst_rope_ptr = k_head_ptr + nope_dim + rope_offs[None, :] - src_rope = tl.load(src_rope_ptr) - tl.store(dst_rope_ptr, src_rope) - - -@triton.jit -def concat_and_cast_mha_k_pad_kernel( - k_ptr, - k_nope_ptr, - k_rope_ptr, - head_cnt: tl.constexpr, - k_stride0: tl.constexpr, - k_stride1: tl.constexpr, - nope_stride0: tl.constexpr, - nope_stride1: tl.constexpr, - rope_stride0: tl.constexpr, - nope_dim: tl.constexpr, - rope_dim: tl.constexpr, - HEAD_BLOCK: tl.constexpr, -): - """ - concat_and_cast_mha_k_kernel for a head count that is not a power of two. - tl.arange needs a power-of-two extent, so walk HEAD_BLOCK == next_power_of_2(head_cnt) - lanes and mask the tail. - """ - pid_loc = tl.program_id(0) - head_range = tl.arange(0, HEAD_BLOCK) - head_mask = head_range < head_cnt - - k_head_ptr = k_ptr + pid_loc * k_stride0 + head_range[:, None] * k_stride1 - - nope_offs = tl.arange(0, nope_dim) - - src_nope_ptr = ( - k_nope_ptr - + pid_loc * nope_stride0 - + head_range[:, None] * nope_stride1 - + nope_offs[None, :] - ) - dst_nope_ptr = k_head_ptr + nope_offs[None, :] - - nope_mask = tl.broadcast_to(head_mask[:, None], (HEAD_BLOCK, nope_dim)) - src_nope = tl.load(src_nope_ptr, mask=nope_mask, other=0) - tl.store(dst_nope_ptr, src_nope, mask=nope_mask) - - rope_offs = tl.arange(0, rope_dim) - src_rope_ptr = k_rope_ptr + pid_loc * rope_stride0 + rope_offs[None, :] - dst_rope_ptr = k_head_ptr + nope_dim + rope_offs[None, :] - rope_mask = tl.broadcast_to(head_mask[:, None], (HEAD_BLOCK, rope_dim)) - src_rope = tl.load(src_rope_ptr) - tl.store( - dst_rope_ptr, - tl.broadcast_to(src_rope, (HEAD_BLOCK, rope_dim)), - mask=rope_mask, - ) - - -def concat_and_cast_mha_k_triton( - k: torch.Tensor, - k_nope: torch.Tensor, - k_rope: torch.Tensor, -): - # The source data type will be implicitly converted to the target data type. - assert len(k.shape) == 3 and len(k_nope.shape) == 3 and len(k_rope.shape) == 3, ( - f"shape should be 3d, but got {k.shape=}, {k_nope.shape=}, {k_rope.shape=}" - ) - assert k.shape[0] == k_nope.shape[0] and k.shape[0] == k_rope.shape[0], ( - f"invalid shape, got {k.shape=}, {k_nope.shape=}, {k_rope.shape=}" - ) - assert k.shape[1] == k_nope.shape[1] and 1 == k_rope.shape[1], ( - f"invalid shape, got {k.shape=}, {k_nope.shape=}, {k_rope.shape=}" - ) - assert k.shape[-1] == k_nope.shape[-1] + k_rope.shape[-1], ( - f"invalid shape, got {k.shape=}, {k_nope.shape=}, {k_rope.shape=}" - ) - - nope_dim = k_nope.shape[-1] - rope_dim = k_rope.shape[-1] - grid = (k.shape[0],) - - head_cnt = k.shape[1] - if head_cnt != triton.next_power_of_2(head_cnt): - # Not the power of two concat_and_cast_mha_k_kernel's tl.arange needs; - # Kimi-K3's 96 heads give 12 at TP8. - concat_and_cast_mha_k_pad_kernel[grid]( - k, - k_nope, - k_rope, - head_cnt, - k.stride(0), - k.stride(1), - k_nope.stride(0), - k_nope.stride(1), - k_rope.stride(0), - nope_dim, - rope_dim, - HEAD_BLOCK=triton.next_power_of_2(head_cnt), - ) - return - - concat_and_cast_mha_k_kernel[grid]( - k, - k_nope, - k_rope, - k.shape[1], - k.stride(0), - k.stride(1), - k_nope.stride(0), - k_nope.stride(1), - k_rope.stride(0), - nope_dim, - rope_dim, - ) - - @triton.jit def reshape_and_cache_flash( key_ptr, @@ -362,464 +212,3 @@ def launch_reshape_and_cache_flash( HAS_SWA=(swa_slot_mapping is not None), USE_SCALE=(k_scale is not None), ) - - -@triton.jit -def concat_and_cast_q_fp8_pad_kernel( - qpad_ptr, # [num_tokens, pad_heads, NOPE+ROPE] fp8 (dst; only [:, :H, :] written) - q_nope_ptr, # [num_tokens, H, NOPE] bf16 - q_rope_ptr, # [num_tokens, H, ROPE] bf16 - qpad_s0, - qpad_s1, - nope_s0, - nope_s1, - rope_s0, - rope_s1, - H: tl.constexpr, - NOPE: tl.constexpr, - ROPE: tl.constexpr, -): - # One program per token: write the H active heads of the padded fp8 q buffer, - # fusing the bf16->fp8 cast (on store) with the nope/rope concat. Bit-exact vs the - # two strided copy_() it replaces; ~3.7x faster because copy_ into the - # 64-head-padded buffer is strided (~4.5x off memory-bound). Strides are passed in, - # so q_nope/q_rope may be views of a [T, H, NOPE+ROPE] q (head-stride != last-dim). - pid = tl.program_id(0) - hr = tl.arange(0, H) - qpad_head = qpad_ptr + pid * qpad_s0 + hr[:, None] * qpad_s1 - no = tl.arange(0, NOPE) - src_n = tl.load(q_nope_ptr + pid * nope_s0 + hr[:, None] * nope_s1 + no[None, :]) - tl.store(qpad_head + no[None, :], src_n) - # NoPE models pass ROPE=0; tl.arange rejects empty ranges, so skip statically. - if ROPE > 0: - ro = tl.arange(0, ROPE) - src_r = tl.load( - q_rope_ptr + pid * rope_s0 + hr[:, None] * rope_s1 + ro[None, :] - ) - tl.store(qpad_head + NOPE + ro[None, :], src_r) - - -def concat_and_cast_q_fp8_pad(q_fp8_pad, q_nope, q_rope, num_heads): - """fused bf16->fp8 concat-cast of q_nope/q_rope into the active - [:, :num_heads, :] slice of the padded fp8 q buffer. Bit-exact replacement for the - two strided converting copy_() in the Q8KV8 prefill q-prep, ~3.7x faster. Requires - num_heads / nope_dim / rope_dim to be powers of two (always true for DeepSeek: 128 - heads / any TP, 512 nope, 64 rope); rope_dim may also be 0 for NoPE models.""" - num_tokens = q_nope.shape[0] - nope_dim = q_nope.shape[-1] - rope_dim = q_rope.shape[-1] - concat_and_cast_q_fp8_pad_kernel[(num_tokens,)]( - q_fp8_pad, - q_nope, - q_rope, - q_fp8_pad.stride(0), - q_fp8_pad.stride(1), - q_nope.stride(0), - q_nope.stride(1), - q_rope.stride(0), - q_rope.stride(1), - H=num_heads, - NOPE=nope_dim, - ROPE=rope_dim, - ) - - -@triton.jit -def absorbed_bmm_concat_cast_q_fp8_kernel( - qout_ptr, # [num_tokens, pad_heads, N+ROPE] fp8 (dst; only [:, :H, :] written) - a_ptr, # q_nope (pre-absorb) [num_tokens, H, K] bf16 - b_ptr, # w_kc [H, K, N] bf16 (any strides; typically N-major) - rope_ptr, # q_rope (post-rope) [num_tokens, H, ROPE] bf16 - T, # num_tokens (runtime; masked) - qout_s0, - qout_s1, - a_s0, - a_s1, - b_s0, - b_s1, - b_s2, - rope_s0, - rope_s1, - K: tl.constexpr, - N: tl.constexpr, - ROPE: tl.constexpr, - BLOCK_M: tl.constexpr, - BLOCK_N: tl.constexpr, - BLOCK_K: tl.constexpr, - K_MODE: tl.constexpr, -): - # One program per (token-block, head): q_out[m, h, :N] = fp8(bf16(fp32( - # q_nope[m, h, :K] @ w_kc[h, :K, :N]))) and q_out[m, h, N:] = - # fp8(q_rope[m, h, :ROPE]). This makes q "born fp8": the absorbed bmm, - # the nope/rope concat, and the bf16->fp8 cast collapse into one kernel, - # so neither the bf16 q_nope_out ([H, T, N], written by cublas and re-read - # by the concat-cast) nor the standalone concat-cast launch exist anymore. - # - # K handling (K_MODE selects the codegen for the nope-gemm K dimension; - # every mode keeps the same fp32-accumulator -> bf16 -> fp8 epilogue): - # 0 "single": BLOCK_K == K. Preload the whole [BLOCK_M, K] a-tile once, - # one tl.dot per N-block — identical codegen to the original - # power-of-2-only kernel (DeepSeek K=128). For non-power-of-2 K this - # only compiles if the Triton build allows non-power-of-2 tl.arange - # (Triton <= 3.5.x does NOT: "arange's range must be a power of 2"). - # 1 "loop": split-K loop, K % BLOCK_K == 0, BLOCK_K power of 2 >= 16 - # (e.g. K=192 with BLOCK_K=64 -> 3 iterations). The a-tile is - # re-loaded per (N-block, K-block); slices are L1/L2-resident after - # the first N-block, but the load/dot interleave costs bandwidth - # (measured ~1372 GB/s vs ~2x that for mode 0 at K=128). - # 2 "two_dot": K = BLOCK_K + (K - BLOCK_K), both power-of-2 halves - # (192 = 128 + 64). Both a-tiles preload once before the N-loop; - # each N-block issues two chained tl.dot into one fp32 accumulator. - # No K-loop, no a re-reads — the direct generalization of mode 0. - # 3 "three_dot": K = 3 * BLOCK_K (192 = 3 x 64). Same as mode 2 with - # three preloaded a-tiles / three chained tl.dot per N-block; the - # hoisted-loads analogue of mode 1 (identical fp32 add order). - # 4 "pad": BLOCK_K = next_pow2(K) > K, k-masked loads (zero fill). - # Single tl.dot per N-block; the padded zeros are exact fp32 - # additive identities so the result matches a K-wide single dot, - # at the cost of BLOCK_K/K (e.g. 256/192 = 1.33x) extra MMA work. - # - # Rounding contract: the fp32 accumulator is rounded to bf16 first (the - # same output rounding stage as the cublas bf16 bmm) and then converted - # bf16->fp8 by the same implicit-store conversion the fused concat-cast - # kernel uses. The split-K accumulator stays fp32 across all K-blocks, - # so the rounding stages are identical in both layouts. The rope half is - # a bit-exact copy of that kernel (loads the post-rope bf16, converts on - # store). The nope half is NOT guaranteed bit-exact vs the default path: - # tl.dot accumulates fp32 in a different order than cublas, so last-ulp - # fp32 differences can occasionally flip the bf16 (and hence fp8) - # rounding. - pid_m = tl.program_id(0) - h = tl.program_id(1) - offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) - m_mask = offs_m < T - # token-row offsets in int64: T * row-stride can exceed int32 (e.g. 128 - # heads x 576 dims x tens of thousands of tokens). - offs_m64 = offs_m.to(tl.int64) - qout_head = qout_ptr + offs_m64[:, None] * qout_s0 + h * qout_s1 - a_row = a_ptr + offs_m64[:, None] * a_s0 + h * a_s1 - b_head = b_ptr + h * b_s0 - if K_MODE == 0: - # single-dot path (original kernel): BLOCK_K == K, preload a once. - offs_k = tl.arange(0, BLOCK_K) - a = tl.load(a_row + offs_k[None, :], mask=m_mask[:, None], other=0.0) - for nb in tl.static_range(N // BLOCK_N): - offs_n = nb * BLOCK_N + tl.arange(0, BLOCK_N) - b = tl.load(b_head + offs_k[:, None] * b_s1 + offs_n[None, :] * b_s2) - acc = tl.dot(a, b) # fp32 accumulator - val = acc.to(tl.bfloat16) # cublas-equivalent bf16 output rounding - # implicit bf16 -> fp8 conversion on store (same as the concat-cast) - tl.store(qout_head + offs_n[None, :], val, mask=m_mask[:, None]) - elif K_MODE == 2: - # two-dot preload: K split as BLOCK_K + (K - BLOCK_K), no K-loop. - offs_k0 = tl.arange(0, BLOCK_K) - offs_k1 = BLOCK_K + tl.arange(0, K - BLOCK_K) - a0 = tl.load(a_row + offs_k0[None, :], mask=m_mask[:, None], other=0.0) - a1 = tl.load(a_row + offs_k1[None, :], mask=m_mask[:, None], other=0.0) - for nb in tl.static_range(N // BLOCK_N): - offs_n = nb * BLOCK_N + tl.arange(0, BLOCK_N) - b0 = tl.load(b_head + offs_k0[:, None] * b_s1 + offs_n[None, :] * b_s2) - b1 = tl.load(b_head + offs_k1[:, None] * b_s1 + offs_n[None, :] * b_s2) - acc = tl.dot(a0, b0) # fp32 accumulator - acc = tl.dot(a1, b1, acc) # chained: stays fp32 across both dots - val = acc.to(tl.bfloat16) # cublas-equivalent bf16 output rounding - # implicit bf16 -> fp8 conversion on store (same as the concat-cast) - tl.store(qout_head + offs_n[None, :], val, mask=m_mask[:, None]) - elif K_MODE == 3: - # three-dot preload: K = 3 * BLOCK_K, a-tiles hoisted out of the N-loop. - offs_k0 = tl.arange(0, BLOCK_K) - offs_k1 = BLOCK_K + offs_k0 - offs_k2 = 2 * BLOCK_K + offs_k0 - a0 = tl.load(a_row + offs_k0[None, :], mask=m_mask[:, None], other=0.0) - a1 = tl.load(a_row + offs_k1[None, :], mask=m_mask[:, None], other=0.0) - a2 = tl.load(a_row + offs_k2[None, :], mask=m_mask[:, None], other=0.0) - for nb in tl.static_range(N // BLOCK_N): - offs_n = nb * BLOCK_N + tl.arange(0, BLOCK_N) - b0 = tl.load(b_head + offs_k0[:, None] * b_s1 + offs_n[None, :] * b_s2) - b1 = tl.load(b_head + offs_k1[:, None] * b_s1 + offs_n[None, :] * b_s2) - b2 = tl.load(b_head + offs_k2[:, None] * b_s1 + offs_n[None, :] * b_s2) - acc = tl.dot(a0, b0) # fp32 accumulator - acc = tl.dot(a1, b1, acc) - acc = tl.dot(a2, b2, acc) # same fp32 add order as the K_MODE=1 loop - val = acc.to(tl.bfloat16) # cublas-equivalent bf16 output rounding - # implicit bf16 -> fp8 conversion on store (same as the concat-cast) - tl.store(qout_head + offs_n[None, :], val, mask=m_mask[:, None]) - elif K_MODE == 4: - # padded single dot: BLOCK_K = next_pow2(K), zero-fill the k tail. - offs_k = tl.arange(0, BLOCK_K) - k_mask = offs_k < K - a = tl.load( - a_row + offs_k[None, :], - mask=m_mask[:, None] & k_mask[None, :], - other=0.0, - ) - for nb in tl.static_range(N // BLOCK_N): - offs_n = nb * BLOCK_N + tl.arange(0, BLOCK_N) - b = tl.load( - b_head + offs_k[:, None] * b_s1 + offs_n[None, :] * b_s2, - mask=k_mask[:, None], - other=0.0, - ) - acc = tl.dot(a, b) # fp32 accumulator (padded zeros add exactly 0) - val = acc.to(tl.bfloat16) # cublas-equivalent bf16 output rounding - # implicit bf16 -> fp8 conversion on store (same as the concat-cast) - tl.store(qout_head + offs_n[None, :], val, mask=m_mask[:, None]) - else: - # K_MODE == 1: split-K loop (K % BLOCK_K == 0, e.g. K=192, BLOCK_K=64). - for nb in tl.static_range(N // BLOCK_N): - offs_n = nb * BLOCK_N + tl.arange(0, BLOCK_N) - acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) - for kb in tl.static_range(K // BLOCK_K): - offs_k = kb * BLOCK_K + tl.arange(0, BLOCK_K) - a = tl.load(a_row + offs_k[None, :], mask=m_mask[:, None], other=0.0) - b = tl.load(b_head + offs_k[:, None] * b_s1 + offs_n[None, :] * b_s2) - acc = tl.dot(a, b, acc) # fp32 accumulator across K-blocks - val = acc.to(tl.bfloat16) # cublas-equivalent bf16 output rounding - # implicit bf16 -> fp8 conversion on store (same as the concat-cast) - tl.store(qout_head + offs_n[None, :], val, mask=m_mask[:, None]) - offs_r = tl.arange(0, ROPE) - r = tl.load( - rope_ptr + offs_m64[:, None] * rope_s0 + h * rope_s1 + offs_r[None, :], - mask=m_mask[:, None], - other=0.0, - ) - tl.store(qout_head + N + offs_r[None, :], r, mask=m_mask[:, None]) - - -# Non-power-of-2-K variant used by variant="auto" (power-of-2 K always takes -# the single-dot fast path). Set to the winner of the K=192 A/B in -# benchmark/kernels/deepseek/benchmark_q8kv8_q_prep.py; "loop" = the pre-A/B split-K behavior. -_AUTO_NONPOW2_VARIANT = "two_dot" - - -def _qprep_env_variant(): - from sglang.srt.environ import envs - - return envs.SGLANG_OPT_Q8KV8_QPREP_VARIANT.get() - - -# Resolved once at import (matches the module-constant style above). "auto" -# keeps the per-K dispatch; "cuda" routes every shape to the hand-written -# SM90 WGMMA kernel (bitwise-identical to two_dot; 1.16-1.38x faster). -_ENV_QPREP_VARIANT = None - - -def absorbed_bmm_concat_cast_q_fp8( - q_fp8_pad: "torch.Tensor", - q_nope: "torch.Tensor", - w_kc: "torch.Tensor", - q_rope: "torch.Tensor", - num_heads: int, - block_m: int = 128, - block_n: int = 64, - variant: str = "auto", - block_k: int = 0, - num_warps: int = 8, - num_stages: int = 0, -): - """Fused absorbed-q bmm + nope/rope concat + bf16->fp8 cast ("born fp8" q). - - Replaces ``torch.bmm(q_nope.transpose(0, 1), w_kc).transpose(0, 1)`` - followed by ``concat_and_cast_q_fp8_pad`` on the Q8KV8 sparse-prefill - path, writing the active ``[:, :num_heads, :]`` slice of the padded fp8 q - buffer directly. Inputs: - - * ``q_fp8_pad``: [num_tokens, pad_heads, N + ROPE] fp8_e4m3 destination. - * ``q_nope``: [num_tokens, H, K] bf16 pre-absorb q (strided views OK). - * ``w_kc``: [H, K, N] bf16 absorbed weight (any strides). - * ``q_rope``: [num_tokens, H, ROPE] bf16 post-rope q (strided views OK). - - The rope half is bit-exact vs ``concat_and_cast_q_fp8_pad``. The nope - half keeps the same rounding stages (fp32 accum -> bf16 -> fp8) but a - different fp32 accumulation order than cublas, so it is near- but not - guaranteed bit-exact; keep this path behind - ``SGLANG_ENABLE_DSA_Q8KV8_BORN_FP8_Q``. - - K (``qk_nope_head_dim``) supports any multiple of 16 in [16, 256]. - Power-of-2 K (DeepSeek 128) always takes the preload-once - single-``tl.dot`` path. For other K (GLM 192), ``variant`` selects the - K-dimension codegen (every variant keeps the identical fp32 -> bf16 -> - fp8 epilogue): - - * ``"auto"``: the current production choice (see - ``_AUTO_NONPOW2_VARIANT``). - * ``"loop"``: split-K accumulator loop, ``BLOCK_K`` = ``block_k`` or the - largest power-of-2 divisor of K capped at 128 (192 -> 64 x 3). - * ``"two_dot"``: preload a as two power-of-2 tiles (192 = 128 + 64), two - chained ``tl.dot`` per N-block, no K-loop. - * ``"three_dot"``: preload a as three K/3 tiles (192 = 3 x 64), three - chained ``tl.dot`` per N-block; same fp32 add order as ``"loop"``. - * ``"pad"``: single ``tl.dot`` with ``BLOCK_K`` = next_pow2(K) (192 -> - 256) and zero-masked k tails. - * ``"single_k"``: single ``tl.dot`` with ``BLOCK_K`` == K. Only - compiles if the Triton build supports non-power-of-2 ``tl.arange`` - (Triton <= 3.5.x raises "arange's range must be a power of 2"). - ``block_m`` / ``block_n`` / ``num_warps`` / ``num_stages`` are tuning - knobs for the microbench sweep (0 = Triton default for ``num_stages``). - """ - num_tokens, _, k_dim = q_nope.shape - n_dim = w_kc.shape[-1] - rope_dim = q_rope.shape[-1] - assert q_fp8_pad.dtype == torch.float8_e4m3fn - assert q_nope.dtype == torch.bfloat16 and w_kc.dtype == torch.bfloat16 - assert q_rope.dtype == torch.bfloat16 - assert q_nope.shape[1] == num_heads and q_rope.shape[1] == num_heads - assert w_kc.shape[0] == num_heads and w_kc.shape[1] == k_dim - assert q_fp8_pad.shape[0] >= num_tokens and q_fp8_pad.shape[1] >= num_heads - assert q_fp8_pad.shape[2] == n_dim + rope_dim - # tl.arange / tl.dot constraints - assert k_dim % 16 == 0 and 16 <= k_dim <= 256, ( - "K must be a multiple of 16 in [16, 256]" - ) - assert (rope_dim & (rope_dim - 1)) == 0, "ROPE must be a power of two" - assert n_dim % block_n == 0, "N must be a multiple of block_n" - assert q_nope.stride(2) == 1 and q_rope.stride(2) == 1 - assert q_fp8_pad.stride(2) == 1 - # Env override for production dispatch (SGLANG_OPT_Q8KV8_QPREP_VARIANT): - # "auto" (default) keeps the per-K Triton dispatch; "cuda" routes every - # shape to the WGMMA kernel below. - global _ENV_QPREP_VARIANT - if _ENV_QPREP_VARIANT is None: - _ENV_QPREP_VARIANT = _qprep_env_variant() - if variant == "auto" and _ENV_QPREP_VARIANT != "auto": - variant = _ENV_QPREP_VARIANT - # Hand-written SM90 WGMMA kernel (opt-in only; "auto" never routes here). - # Same fp32 -> bf16 -> fp8 epilogue; bitwise identical to "two_dot" on - # SM90. Requires K in {128, 192} and the production N-major w_kc layout - # (see the wrapper's asserts); the Triton variants remain the - # general-strides fallback. - _valid = ("auto", "cuda", "loop", "two_dot", "three_dot", "pad", "single_k") - if variant not in _valid: - raise ValueError( - f"unknown q-prep variant {variant!r} " - f"(SGLANG_OPT_Q8KV8_QPREP_VARIANT); valid: {_valid}" - ) - if variant == "cuda": - from sglang.kernels.ops.attention.qprep_bf16_fp8_sm90 import q8kv8_qprep_fwd - - q8kv8_qprep_fwd(q_fp8_pad, q_nope, w_kc, q_rope, num_heads) - return - # Resolve (K_MODE, BLOCK_K) from the variant; see the kernel's K-handling - # comment for what each mode compiles to. - if k_dim & (k_dim - 1) == 0: - # power-of-2 K: every variant collapses to the single-dot fast path. - k_mode, blk_k = 0, k_dim - else: - v = _AUTO_NONPOW2_VARIANT if variant == "auto" else variant - if v == "loop": - # Largest power-of-2 divisor of K, capped at 128 (K % 16 == 0 - # makes this >= 16), unless the caller pinned block_k. - blk_k = block_k or min(k_dim & -k_dim, 128) - assert k_dim % blk_k == 0 and blk_k & (blk_k - 1) == 0 and blk_k >= 16, ( - "loop needs BLOCK_K a power-of-2 divisor of K >= 16" - ) - k_mode = 1 - elif v == "two_dot": - blk_k = 1 << (k_dim.bit_length() - 1) # largest power of 2 < K - k1 = k_dim - blk_k - assert k1 & (k1 - 1) == 0 and k1 >= 16, ( - "two_dot needs K = pow2 + pow2 with both halves >= 16" - ) - k_mode = 2 - elif v == "three_dot": - blk_k = k_dim // 3 - assert k_dim % 3 == 0 and blk_k & (blk_k - 1) == 0 and blk_k >= 16, ( - "three_dot needs K = 3 * pow2 with pow2 >= 16" - ) - k_mode = 3 - elif v == "pad": - blk_k = 1 << k_dim.bit_length() # next power of 2 above K - k_mode = 4 - elif v == "single_k": - # Non-power-of-2 BLOCK_K == K: compiles only on Triton builds - # that allow non-power-of-2 tl.arange (not 3.5.x). - blk_k = k_dim - k_mode = 0 - else: - raise ValueError(f"unknown absorbed-bmm K variant: {variant!r}") - extra = {"num_stages": num_stages} if num_stages else {} - grid = (triton.cdiv(num_tokens, block_m), num_heads) - absorbed_bmm_concat_cast_q_fp8_kernel[grid]( - q_fp8_pad, - q_nope, - w_kc, - q_rope, - num_tokens, - q_fp8_pad.stride(0), - q_fp8_pad.stride(1), - q_nope.stride(0), - q_nope.stride(1), - w_kc.stride(0), - w_kc.stride(1), - w_kc.stride(2), - q_rope.stride(0), - q_rope.stride(1), - K=k_dim, - N=n_dim, - ROPE=rope_dim, - BLOCK_M=block_m, - BLOCK_N=block_n, - BLOCK_K=blk_k, - K_MODE=k_mode, - num_warps=num_warps, - **extra, - ) - - -@triton.jit -def q8kv8_topk_length_backscan_kernel( - indices_ptr, - out_ptr, - stride_row, - topk, - BLOCK: tl.constexpr, -): - row = tl.program_id(0).to(tl.int64) - base = indices_ptr + row * stride_row - off = topk - length = 1 - found = 0 - while (found == 0) & (off > 0): - off -= BLOCK - idx = off + tl.arange(0, BLOCK) - vals = tl.load(base + idx) - pos = tl.max(tl.where(vals >= 0, idx, -1), axis=0) - found = tl.where(pos >= 0, 1, found) - length = tl.where(pos >= 0, pos + 1, length) - tl.store(out_ptr + row, length) - - -def q8kv8_topk_length_from_indices(indices: torch.Tensor) -> torch.Tensor: - """Per-row valid-topk count = last non-negative position + 1 (min 1). - - ``indices``: [s_q, topk] int32 topk output whose pad slots are -1. - Backward block scan per row: the loop exits at the first block holding a - valid entry, so the cost is proportional to the trailing pad run — one - block (~topk/4 elements) for rows with a full topk, which dominate long - contexts. Semantics match the unfused ``(indices >= 0) * ramp).amax`` - derivation exactly, including all-pad rows (length 1: one pad-only block - keeps the kernel on its clamp+mask path, contributing zero). - """ - s_q, topk = indices.shape - assert indices.dtype == torch.int32 and indices.stride(1) == 1 - out = torch.empty(s_q, dtype=torch.int32, device=indices.device) - block = 512 if topk % 512 == 0 else (256 if topk % 256 == 0 else 128) - q8kv8_topk_length_backscan_kernel[(s_q,)]( - indices, - out, - indices.stride(0), - topk, - BLOCK=block, - ) - return out - - -# --------------------------------------------------------------------------- -# Decode Context Parallel (DCP) helpers. -# -# Not part of upstream main (PR #26000 centralized the other Triton utility -# kernels into triton_ops/*). These three live here because they are DCP-only: -# - create_triton_kv_indices_for_dcp_triton: per-rank local KV indices -# - get_dcp_lens: per-rank visible KV length -# - cp_lse_ag_out_rs: merge DCP partial attention via natural-log LSE -# --------------------------------------------------------------------------- diff --git a/python/sglang/kernels/ops/attention/fused_store_index_cache.py b/python/sglang/kernels/ops/kvcache/dsa_fused_store_index_cache.py similarity index 100% rename from python/sglang/kernels/ops/attention/fused_store_index_cache.py rename to python/sglang/kernels/ops/kvcache/dsa_fused_store_index_cache.py diff --git a/python/sglang/kernels/ops/attention/dsa/index_buf_accessor.py b/python/sglang/kernels/ops/kvcache/dsa_index_k_cache.py similarity index 100% rename from python/sglang/kernels/ops/attention/dsa/index_buf_accessor.py rename to python/sglang/kernels/ops/kvcache/dsa_index_k_cache.py diff --git a/python/sglang/kernels/ops/attention/dsv4/c128_cleanup.py b/python/sglang/kernels/ops/kvcache/dsv4_c128_cleanup.py similarity index 100% rename from python/sglang/kernels/ops/attention/dsv4/c128_cleanup.py rename to python/sglang/kernels/ops/kvcache/dsv4_c128_cleanup.py diff --git a/python/sglang/kernels/ops/kvcache/dsv4_fp4_index_k_cache.py b/python/sglang/kernels/ops/kvcache/dsv4_fp4_index_k_cache.py new file mode 100644 index 0000000000..a2bc9a338f --- /dev/null +++ b/python/sglang/kernels/ops/kvcache/dsv4_fp4_index_k_cache.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from sglang.kernels.ops.attention.dsv4.fp4_indexer import INDEX_K_SLOT_BYTES +from sglang.kernels.ops.quantization.fp4_indexer_quant import ( + quantize_fp4_indexer_tensor, +) + + +@triton.jit +def _store_fp4_index_k_cache_kernel( + k_fp4, + k_sf, + cache, + loc, + page_size: tl.constexpr, + cache_stride: tl.constexpr, + BLOCK: tl.constexpr, +): + token_id = tl.program_id(0) + offsets = tl.arange(0, BLOCK) + cache_loc = tl.load(loc + token_id) + page = cache_loc // page_size + page_offset = cache_loc - page * page_size + + k = tl.load(k_fp4 + token_id * BLOCK + offsets) + tl.store(cache + page * cache_stride + page_offset * BLOCK + offsets, k) + + sf = tl.load(k_sf + token_id) + sf_offsets = tl.arange(0, 4) + sf_bytes = (sf >> (sf_offsets * 8)) & 0xFF + tl.store( + cache + page * cache_stride + page_size * BLOCK + page_offset * 4 + sf_offsets, + sf_bytes, + ) + + +def store_fp4_index_k_cache( + input: torch.Tensor, + cache: torch.Tensor, + loc: torch.Tensor, + *, + page_size: int, + rne: bool = False, +) -> None: + assert input.shape[-1] == 128 + k_fp4, k_sf = quantize_fp4_indexer_tensor(input.contiguous(), rne=rne) + n_tokens = input.numel() // input.shape[-1] + assert k_fp4.shape == (n_tokens, 64) + assert k_sf.shape == (n_tokens,) + assert cache.shape[1] == page_size * INDEX_K_SLOT_BYTES + + if n_tokens == 0: + return + _store_fp4_index_k_cache_kernel[(n_tokens,)]( + k_fp4.view(torch.uint8), + k_sf, + cache, + loc, + page_size, + cache.stride(0), + BLOCK=64, + ) + + +@triton.jit +def _gather_fp4_index_k_kernel( + cache, + cache_stride, + slots, + payload, + scales, + n, + page_size: tl.constexpr, + BLOCK_ROWS: tl.constexpr, +): + # int32 view of a page: page_size * 16 payload words, then page_size scale words. + rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS) + valid = rows < n + slot = tl.load(slots + rows, valid, 0).to(tl.int64) + page = slot // page_size + offset = slot - page * page_size + page_base = cache + page * cache_stride + words = tl.arange(0, 16) + payload_words = tl.load( + page_base[:, None] + offset[:, None] * 16 + words[None, :], valid[:, None] + ) + tl.store( + payload + rows[:, None].to(tl.int64) * 16 + words[None, :], + payload_words, + valid[:, None], + ) + tl.store(scales + rows, tl.load(page_base + page_size * 16 + offset, valid), valid) + + +def gather_fp4_index_k( + cache: torch.Tensor, slots: torch.Tensor, *, page_size: int +) -> tuple[torch.Tensor, torch.Tensor]: + """Packed fp4 rows at ``slots`` of the index-K ``cache`` (``[pages, page_size * + 68]`` uint8, ``store_fp4_index_k_cache``'s layout): (payload int8 ``[n, 64]``, + ue8m0 scales packed int32 ``[n]``), the layout ``quantize_fp4_indexer_tensor`` + returns.""" + assert cache.shape[1] == page_size * INDEX_K_SLOT_BYTES and cache.stride(1) == 1 + # The kernel addresses slots as `slots + rows`, so a strided view would be read + # as if its elements were packed. + assert slots.is_contiguous() + words = cache.view(torch.int32) + n = slots.shape[0] + payload = torch.empty((n, 16), device=cache.device, dtype=torch.int32) + scales = torch.empty((n,), device=cache.device, dtype=torch.int32) + if n > 0: + block_rows = 64 + _gather_fp4_index_k_kernel[(triton.cdiv(n, block_rows),)]( + words, + words.stride(0), + slots, + payload, + scales, + n, + page_size, + BLOCK_ROWS=block_rows, + num_warps=4, + ) + return payload.view(torch.int8), scales diff --git a/python/sglang/kernels/ops/kvcache/dsv4_fp4_index_k_cache_hip.py b/python/sglang/kernels/ops/kvcache/dsv4_fp4_index_k_cache_hip.py new file mode 100644 index 0000000000..53bb58969e --- /dev/null +++ b/python/sglang/kernels/ops/kvcache/dsv4_fp4_index_k_cache_hip.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import _HEAD_DIM, _KV_BLOCK_SIZE +from sglang.kernels.ops.quantization.fp4_indexer_quant import ( + quantize_fp4_indexer_tensor, +) + + +# FlyDSL split payload / scale layout; bytes equal AITER's fp4 writer up to the sign of zero +@triton.jit +def _store_fp4_index_k_split_kernel( + k_fp4, + k_sf, + payload, + scale, + loc, + page_size: tl.constexpr, + BLOCK: tl.constexpr, + PAGE8: tl.constexpr, +): + token_id = tl.program_id(0) + cache_loc = tl.load(loc + token_id) + page = cache_loc // page_size + page_offset = cache_loc - page * page_size + + # payload [page, 1, 4 chunks, page_size, 16 bytes]: chunk c holds dims [32c, 32c + 32). + offsets = tl.arange(0, BLOCK) + chunk = offsets // 16 + byte = offsets - chunk * 16 + k = tl.load(k_fp4 + token_id * BLOCK + offsets) + sf = tl.load(k_sf + token_id) + sf_offsets = tl.arange(0, 4) + sf_bytes = ((sf >> (sf_offsets * 8)) & 0xFF).to(tl.uint8) + if PAGE8: + # aiter's row-group 8-slot page: payload [t % 4][chunk][t // 4][16 B], scales [chunk][t] + page8 = cache_loc // 8 + t = cache_loc % 8 + tl.store( + payload + + page8 * (8 * BLOCK) + + ((t % 4) * 4 + chunk) * 32 + + (t // 4) * 16 + + byte, + k, + ) + tl.store(scale + page8 * 32 + sf_offsets * 8 + t, sf_bytes) + return + tl.store( + payload + + page * (4 * page_size * 16) + + chunk * (page_size * 16) + + page_offset * 16 + + byte, + k, + ) + # scale [page, 1, 4, page_size]: the slot axis is a 16 x 4 tile transposed (the FlyDSL K ABI) + shuffled = (page_offset % 16) * 4 + page_offset // 16 + tl.store( + scale + page * (4 * page_size) + sf_offsets * page_size + shuffled, sf_bytes + ) + + +def store_fp4_index_k_cache_split( + input: torch.Tensor, + payload: torch.Tensor, + scale: torch.Tensor, + loc: torch.Tensor, + *, + page_size: int, + rne: bool = False, + page8: bool = False, +) -> None: + """Quantize input [n, 128] to fp4 (per-32 ue8m0) and scatter row i to slot + loc[i] of the split FlyDSL K layout (payload [pages, 1, 4, page_size, 16], + scale [pages, 1, 4, page_size] with the slot axis 16 x 4 transposed); with page8 + each page holds eight 8-slot pages of aiter's row-group layout (pack_kv_cache).""" + assert input.shape[-1] == _HEAD_DIM + assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page" + assert payload.shape[1:] == (1, 4, page_size, 16), payload.shape + assert scale.shape[1:] == (1, 4, page_size), scale.shape + k_fp4, k_sf = quantize_fp4_indexer_tensor(input.contiguous(), rne=rne) + n_tokens = k_fp4.shape[0] + if n_tokens == 0: + return + _store_fp4_index_k_split_kernel[(n_tokens,)]( + k_fp4.view(torch.uint8), + k_sf, + payload.view(torch.uint8), + scale, + loc, + page_size, + BLOCK=64, + PAGE8=page8, + ) diff --git a/python/sglang/kernels/ops/attention/dsv4/index_buf_accessor.py b/python/sglang/kernels/ops/kvcache/dsv4_main_kv_store.py similarity index 100% rename from python/sglang/kernels/ops/attention/dsv4/index_buf_accessor.py rename to python/sglang/kernels/ops/kvcache/dsv4_main_kv_store.py diff --git a/python/sglang/kernels/ops/attention/dsv4/request_window_copy.py b/python/sglang/kernels/ops/kvcache/dsv4_request_window_copy.py similarity index 100% rename from python/sglang/kernels/ops/attention/dsv4/request_window_copy.py rename to python/sglang/kernels/ops/kvcache/dsv4_request_window_copy.py diff --git a/python/sglang/kernels/ops/kvcache/dsv4_unified_kv_store.py b/python/sglang/kernels/ops/kvcache/dsv4_unified_kv_store.py new file mode 100644 index 0000000000..6e666ec0bd --- /dev/null +++ b/python/sglang/kernels/ops/kvcache/dsv4_unified_kv_store.py @@ -0,0 +1,182 @@ +"""Scatter DSV4 SWA K rows into the unified_kv pool (``unified_kv_triton``). + +unified_kv[L] rows ``[0, swa_pages)`` are the SWA ring +(``state_slot * win + pos % win``). +""" + +from __future__ import annotations + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.layout import ( + check_two_pool_pair, +) + + +@triton.jit +def _swa_scatter_kernel( + kv_ptr, # [T, D] bf16 + state_slot_ptr, # [T] int + positions_ptr, # [T] int + final_pos_ptr, # [T] int + unified_ptr, # [pages, D] bf16 + n_rows, + ring_stride, # SWA ring per-slot stride + win: tl.constexpr, + D: tl.constexpr, + HAS_FINAL: tl.constexpr, + BLOCK_D: tl.constexpr, +): + row = tl.program_id(0) + if row >= n_rows: + return + pos = tl.load(positions_ptr + row) + if HAS_FINAL: + fp = tl.load(final_pos_ptr + row) + if pos <= fp - win: + return + s = tl.load(state_slot_ptr + row) + loc = s * ring_stride + (pos % ring_stride) + offs = tl.arange(0, BLOCK_D) + mask = offs < D + vals = tl.load(kv_ptr + row * D + offs, mask=mask, other=0.0) + tl.store(unified_ptr + loc * D + offs, vals, mask=mask) + + +def store_swa_into_unified( + *, + kv: torch.Tensor, # [T, head_dim] bf16, or [T, nope_row_bytes] packed fp8 + state_slot: torch.Tensor, # [T] int + positions: torch.Tensor, # [T] int + unified_kv: torch.Tensor, # [pages, ...] same dtype and row width as kv + win: int, # SWA attention window length + ring_stride: int, # SWA ring stride + final_pos: Optional[torch.Tensor] = None, # [T] req's last position + kv_rope: Optional[torch.Tensor] = None, # [T, rope_dim] bf16, fp8 layout only + unified_kv_rope: Optional[torch.Tensor] = None, # [pages, rope_dim] bf16 +) -> None: + """Scatter SWA K into ring row ``state_slot * ring_stride + pos % ring_stride``. + + Under the fp8 layout the latent is split over two pools, so ``kv`` carries the + already-packed nope row (DSV4_FP8_NOPE_ROW_BYTES wide: values + E8M0 scales + + pad; nothing is quantized here) and ``kv_rope`` the bf16 rope half. That width + is a byte count that happens to equal the bf16 head_dim in elements -- the two + are not the same thing. The scatter itself takes the row width off the tensor; + only the pair check reads the constant. + + Both scatters recompute the row index from the same ``state_slot`` / + ``positions``, so the two pools stay in lockstep with each other and with the + bf16 layout. What a caller can still get wrong is passing two pools that aren't + a pair, so the pair goes through ``check_two_pool_pair`` before the first launch + -- shared with the fused store, which has the same coupling. + """ + n_rows = kv.shape[0] + if n_rows == 0: + return + + has_final = final_pos is not None + fp_arg = final_pos if has_final else positions + assert kv.is_contiguous() and kv.dtype == unified_kv.dtype + assert state_slot.is_contiguous() and positions.is_contiguous() + assert fp_arg.is_contiguous() + two_pool = kv_rope is not None + assert two_pool == (unified_kv_rope is not None), ( + "kv_rope and unified_kv_rope come together" + ) + if two_pool: + assert kv_rope.is_contiguous(), ( + f"kv_rope must be contiguous, got strides {kv_rope.stride()}" + ) + assert kv_rope.shape[0] == n_rows, ( + f"kv_rope holds {kv_rope.shape[0]} rows, kv holds {n_rows}" + ) + check_two_pool_pair( + unified_kv, + unified_kv_rope, + rope_width=kv_rope.shape[1], + rope_dtype=kv_rope.dtype, + ) + + def _scatter(src: torch.Tensor, dst: torch.Tensor) -> None: + D = src.shape[1] + assert dst.shape[1] == D, f"row width {D} does not fit pool {dst.shape[1]}" + _swa_scatter_kernel[(n_rows,)]( + src, + state_slot, + positions, + fp_arg, + dst, + n_rows, + ring_stride, + win=win, + D=D, + HAS_FINAL=has_final, + BLOCK_D=triton.next_power_of_2(D), + num_warps=8, + ) + + if kv.element_size() == 1: + # single-byte rows (any fp8 variant) are a pure byte move, and the E8M0 + # scale bytes aren't floats -- uint8 avoids a triton convert for `other=` + assert unified_kv.is_contiguous() + _scatter(kv.view(torch.uint8), unified_kv.view(torch.uint8)) + else: + _scatter(kv, unified_kv) + + if two_pool: + _scatter(kv_rope, unified_kv_rope) + + +@triton.jit +def _scatter_loc_kernel( + kv_ptr, # [T, D] bf16 + loc_ptr, # [T] int (unified row index; <0 => skip) + unified_ptr, # [pages, D] bf16 + n_rows, + D: tl.constexpr, + BLOCK_D: tl.constexpr, +): + row = tl.program_id(0) + if row >= n_rows: + return + loc = tl.load(loc_ptr + row).to(tl.int64) + if loc < 0: + return + offs = tl.arange(0, BLOCK_D) + mask = offs < D + vals = tl.load(kv_ptr + row * D + offs, mask=mask, other=0.0) + tl.store(unified_ptr + loc * D + offs, vals, mask=mask) + + +def scatter_bf16_into_unified( + *, + kv: torch.Tensor, # [T, head_dim] bf16 (already norm+rope'd) + loc: torch.Tensor, # [T] int32/int64 unified ring row; <0 => skip + unified_kv: torch.Tensor, # [pages, head_dim] bf16 +) -> None: + """Scatter already-norm+rope'd bf16 K into ``unified_kv[loc]`` (skip loc < 0). + + Companion to ``store_swa_into_unified`` for callers that already hold the + precomputed ring row index (the DSpark draft: ``get_unified_swa_loc`` for the + draft forward, or the commit-inject layout for target-hidden injection) and + need per-row commit masking expressed as ``loc == -1``. + """ + n_rows, D = kv.shape + if n_rows == 0: + return + assert kv.is_contiguous() and kv.dtype == unified_kv.dtype + assert loc.is_contiguous() + assert unified_kv.is_contiguous() + _scatter_loc_kernel[(n_rows,)]( + kv, + loc, + unified_kv, + n_rows, + D=D, + BLOCK_D=triton.next_power_of_2(D), + num_warps=8, + ) diff --git a/python/sglang/kernels/ops/quantization/mxfp8_interleave_sf.py b/python/sglang/kernels/ops/kvcache/mxfp8_interleave_sf.py similarity index 100% rename from python/sglang/kernels/ops/quantization/mxfp8_interleave_sf.py rename to python/sglang/kernels/ops/kvcache/mxfp8_interleave_sf.py diff --git a/python/sglang/kernels/ops/kvcache/mxfp8_quant_store_kv.py b/python/sglang/kernels/ops/kvcache/mxfp8_quant_store_kv.py new file mode 100644 index 0000000000..e981edb553 --- /dev/null +++ b/python/sglang/kernels/ops/kvcache/mxfp8_quant_store_kv.py @@ -0,0 +1,119 @@ +"""Quantize bf16 K/V rows to MXFP8 and scatter them into the paged KV cache.""" + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _mxfp8_quant_store_qkv_kernel( + q_ptr, + k_ptr, + v_ptr, + loc_ptr, + q8_ptr, + sfq_ptr, + kc_ptr, + vc_ptr, + sfk_ptr, + sfv_ptr, + sqt, + sqh, + skt, + skh, + svt, + svh, + NQ: tl.constexpr, + NKV: tl.constexpr, + D: tl.constexpr, + PAGE: tl.constexpr, +): + t = tl.program_id(0) + r = tl.program_id(1) + SF: tl.constexpr = D // 32 + blk = tl.arange(0, SF) + off2 = blk[:, None] * 32 + tl.arange(0, 32)[None, :] + + if r < NQ: + src = q_ptr + t * sqt + r * sqh + elif r < NQ + NKV: + src = k_ptr + t * skt + (r - NQ) * skh + else: + src = v_ptr + t * svt + (r - NQ - NKV) * svh + + x = tl.load(src + off2).to(tl.float32) + amax = tl.maximum(tl.max(tl.abs(x), axis=1), 1e-30) + scale_biased = tl.ceil(tl.log2(amax / 448.0)) + 127.0 + scale_biased = tl.minimum(tl.maximum(scale_biased, 0.0), 254.0) + descale = tl.exp2(scale_biased - 127.0) + xq = tl.clamp(x / descale[:, None], -448.0, 448.0).to(q8_ptr.dtype.element_ty) + sf = scale_biased.to(tl.uint8) + + if r < NQ: + tl.store(q8_ptr + (t * NQ + r) * D + off2, xq) + tl.store(sfq_ptr + (t * NQ + r) * SF + blk, sf) + else: + myloc = tl.load(loc_ptr + t).to(tl.int64) + # Slot 0 is the reserved CUDA-graph padding sink; skip writes to it. + if myloc == 0: + return + if r < NQ + NKV: + h = r - NQ + cache = kc_ptr + sfb = sfk_ptr + else: + h = r - NQ - NKV + cache = vc_ptr + sfb = sfv_ptr + tl.store(cache + (myloc * NKV + h) * D + off2, xq) + # BlockScaledBasicChunk byte layout (see mxfp8_interleave_sf.py): + # a page block is (NKV, 32, PAGE//32, SF) bytes. + po = myloc % PAGE + base = ( + ((myloc // PAGE) * NKV + h) * (32 * (PAGE // 32) * SF) + + (po % 32) * ((PAGE // 32) * SF) + + (po // 32) * SF + ) + tl.store(sfb + base + blk, sf) + + +def quant_store_kv_mxfp8( + k: torch.Tensor, + v: torch.Tensor, + loc: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + sfk: torch.Tensor, + sfv: torch.Tensor, + page_size: int = 128, +) -> None: + """One kernel quantizes bf16 K/V rows, scatters the fp8 payload into the paged + cache at `loc`, and writes UE8M0 scales in the interleaved FA4 layout.""" + T, NKV, D = k.shape + assert D % 32 == 0 and page_size % 32 == 0 + assert k.stride(2) == 1 and v.stride(2) == 1 + assert k_cache.is_contiguous() and v_cache.is_contiguous() + if T == 0: + return + _mxfp8_quant_store_qkv_kernel[(T, 2 * NKV)]( + k, # unused q_ptr (same elem dtype) + k, + v, + loc, + k_cache, # unused q8_ptr, supplies the fp8 store dtype + sfk.view(torch.uint8), # unused sfq_ptr (uint8, matches the dead store) + k_cache, + v_cache, + sfk.view(torch.uint8), + sfv.view(torch.uint8), + 0, + 0, + k.stride(0), + k.stride(1), + v.stride(0), + v.stride(1), + NQ=0, + NKV=NKV, + D=D, + PAGE=page_size, + ) diff --git a/python/sglang/kernels/ops/kvcache/shuffle_5d_cache.py b/python/sglang/kernels/ops/kvcache/shuffle_5d_cache.py new file mode 100644 index 0000000000..44f9dbc8f2 --- /dev/null +++ b/python/sglang/kernels/ops/kvcache/shuffle_5d_cache.py @@ -0,0 +1,267 @@ +import torch +import triton +import triton.language as tl + + +@triton.jit +def reshape_and_cache_shuffle_5d( + key_ptr, + value_ptr, + key_cache_ptr, + value_cache_ptr, + slot_mapping_ptr, + swa_slot_mapping_ptr, + key_stride_token, + value_stride_token, + num_heads, + head_size, + block_size, + X: tl.constexpr, + HEAD_BLOCK: tl.constexpr, + BLOCK_D: tl.constexpr, + HAS_SWA: tl.constexpr, +): + """Scatter per-token (num_tokens, num_heads, head_size) K/V into the + SHUFFLE 5D "vectorized" KV cache layout used by aiter CK + `mha_batch_prefill_func` and aiter `pa_decode_gluon`. + + K cache shape: (num_blocks, num_heads, head_size // X, block_size, X) + V cache shape: (num_blocks, num_heads, block_size // X, head_size, X) + where X = 16 // element_size (=8 for bf16/fp16, =16 for fp8). + block_size must be divisible by X, and head_size must be divisible by X. + + Each program handles one token and a HEAD_BLOCK-wide slice of heads. + """ + token_idx = tl.program_id(0) + head_block_idx = tl.program_id(1) + + slot_idx = tl.load(slot_mapping_ptr + token_idx) + if HAS_SWA: + slot_idx = tl.load(swa_slot_mapping_ptr + slot_idx) + if slot_idx < 0: + return + + block_idx = slot_idx // block_size + slot_in_page = slot_idx % block_size + page_outer = slot_in_page // X + page_inner = slot_in_page % X + + head_idx = head_block_idx * HEAD_BLOCK + tl.arange(0, HEAD_BLOCK) + head_mask = head_idx < num_heads + d = tl.arange(0, BLOCK_D) + d_mask = d < head_size + d_outer = d // X + d_inner = d % X + + src_off = token_idx * key_stride_token + head_idx[:, None] * head_size + d[None, :] + src_mask = head_mask[:, None] & d_mask[None, :] + k = tl.load(key_ptr + src_off, mask=src_mask) + src_off_v = ( + token_idx * value_stride_token + head_idx[:, None] * head_size + d[None, :] + ) + v = tl.load(value_ptr + src_off_v, mask=src_mask) + + layer_stride = num_heads * head_size * block_size + head_stride = head_size * block_size + + k_tgt = ( + block_idx * layer_stride + + head_idx[:, None] * head_stride + + d_outer[None, :] * block_size * X + + slot_in_page * X + + d_inner[None, :] + ) + tl.store(key_cache_ptr + k_tgt, k, mask=src_mask) + + v_tgt = ( + block_idx * layer_stride + + head_idx[:, None] * head_stride + + page_outer * head_size * X + + d[None, :] * X + + page_inner + ) + tl.store(value_cache_ptr + v_tgt, v, mask=src_mask) + + +def launch_reshape_and_cache_shuffle_5d( + key: torch.Tensor, + value: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + swa_slot_mapping=None, +): + """Launcher for reshape_and_cache_shuffle_5d. + + Args: + key/value: (num_tokens, num_heads, head_size) source tensors + key_cache: (num_blocks, num_heads, head_size//X, block_size, X) + value_cache: (num_blocks, num_heads, block_size//X, head_size, X) + slot_mapping: per-token destination slot in [0, num_blocks*block_size) + """ + num_tokens, num_heads, head_size = key.shape + assert value.shape == key.shape, "K/V must share token-major shape" + assert key_cache.dim() == 5 and value_cache.dim() == 5 + num_blocks, kc_H, kc_D_over_X, block_size, X = key_cache.shape + assert kc_H == num_heads and kc_D_over_X * X == head_size + vb_blocks, vc_H, vc_page_over_X, vc_D, vc_X = value_cache.shape + assert ( + vc_H == num_heads + and vc_page_over_X * X == block_size + and vc_D == head_size + and vc_X == X + ) + assert block_size % X == 0 and head_size % X == 0 + + HEAD_BLOCK = min(4, triton.next_power_of_2(num_heads)) + BLOCK_D = triton.next_power_of_2(head_size) + grid = (num_tokens, triton.cdiv(num_heads, HEAD_BLOCK)) + + reshape_and_cache_shuffle_5d[grid]( + key, + value, + key_cache, + value_cache, + slot_mapping, + swa_slot_mapping if swa_slot_mapping is not None else slot_mapping, + key.stride(0), + value.stride(0), + num_heads, + head_size, + block_size, + X=X, + HEAD_BLOCK=HEAD_BLOCK, + BLOCK_D=BLOCK_D, + HAS_SWA=(swa_slot_mapping is not None), + ) + + +@triton.jit +def gather_shuffle_5d_to_linear( + key_cache_ptr, + value_cache_ptr, + key_out_ptr, # (T, num_heads, head_size), store dtype + value_out_ptr, # (T, num_heads, head_size), store dtype + slot_mapping_ptr, # (T,) absolute pool slot id per token + key_out_stride_token, + value_out_stride_token, + num_heads, + head_size, + block_size, + X: tl.constexpr, + HEAD_BLOCK: tl.constexpr, + BLOCK_D: tl.constexpr, +): + """Inverse of :func:`reshape_and_cache_shuffle_5d`. + + Gather one token's K/V from the SHUFFLE 5D paged cache into the + canonical (T, H, D) layout that aiter's ``mha_batch_prefill_func`` + expects in LINEAR mode. Source addressing is identical to the + writer kernel so any bit-exact round-trip is guaranteed. + """ + token_idx = tl.program_id(0) + head_block_idx = tl.program_id(1) + + slot_idx = tl.load(slot_mapping_ptr + token_idx) + + block_idx = slot_idx // block_size + slot_in_page = slot_idx % block_size + page_outer = slot_in_page // X + page_inner = slot_in_page % X + + head_idx = head_block_idx * HEAD_BLOCK + tl.arange(0, HEAD_BLOCK) + head_mask = head_idx < num_heads + d = tl.arange(0, BLOCK_D) + d_mask = d < head_size + d_outer = d // X + d_inner = d % X + + layer_stride = num_heads * head_size * block_size + head_stride = head_size * block_size + + src_mask = head_mask[:, None] & d_mask[None, :] + k_src = ( + block_idx * layer_stride + + head_idx[:, None] * head_stride + + d_outer[None, :] * block_size * X + + slot_in_page * X + + d_inner[None, :] + ) + k = tl.load(key_cache_ptr + k_src, mask=src_mask) + v_src = ( + block_idx * layer_stride + + head_idx[:, None] * head_stride + + page_outer * head_size * X + + d[None, :] * X + + page_inner + ) + v = tl.load(value_cache_ptr + v_src, mask=src_mask) + + dst_k = ( + token_idx * key_out_stride_token + head_idx[:, None] * head_size + d[None, :] + ) + tl.store(key_out_ptr + dst_k, k, mask=src_mask) + dst_v = ( + token_idx * value_out_stride_token + head_idx[:, None] * head_size + d[None, :] + ) + tl.store(value_out_ptr + dst_v, v, mask=src_mask) + + +def launch_gather_shuffle_5d_to_linear( + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, +): + """Inverse of :func:`launch_reshape_and_cache_shuffle_5d`. + + Returns ``(key_out, value_out)`` each shaped + ``(T, num_heads, head_size)`` in ``key_cache.dtype`` / + ``value_cache.dtype``. The caller is responsible for passing the + right per-tensor descales downstream when ``store_dtype`` is fp8. + + Args: + key_cache: (num_blocks, num_heads, head_size // X, block_size, X) + value_cache: (num_blocks, num_heads, block_size // X, head_size, X) + slot_mapping: (T,) per-token absolute slot id in + ``[0, num_blocks * block_size)`` + """ + assert key_cache.dim() == 5 and value_cache.dim() == 5 + num_blocks, num_heads, kc_D_over_X, block_size, X = key_cache.shape + vc_blocks, vc_H, vc_page_over_X, vc_D, vc_X = value_cache.shape + assert vc_blocks == num_blocks and vc_H == num_heads + assert vc_page_over_X * X == block_size and vc_X == X + head_size = kc_D_over_X * X + assert vc_D == head_size + + num_tokens = slot_mapping.numel() + key_out = torch.empty( + (num_tokens, num_heads, head_size), + dtype=key_cache.dtype, + device=key_cache.device, + ) + value_out = torch.empty( + (num_tokens, num_heads, head_size), + dtype=value_cache.dtype, + device=value_cache.device, + ) + + HEAD_BLOCK = min(4, triton.next_power_of_2(num_heads)) + BLOCK_D = triton.next_power_of_2(head_size) + grid = (num_tokens, triton.cdiv(num_heads, HEAD_BLOCK)) + + gather_shuffle_5d_to_linear[grid]( + key_cache, + value_cache, + key_out, + value_out, + slot_mapping, + key_out.stride(0), + value_out.stride(0), + num_heads, + head_size, + block_size, + X=X, + HEAD_BLOCK=HEAD_BLOCK, + BLOCK_D=BLOCK_D, + ) + return key_out, value_out diff --git a/python/sglang/kernels/ops/layernorm/__init__.py b/python/sglang/kernels/ops/layernorm/__init__.py index 433bfbedf3..f9d04b0b83 100644 --- a/python/sglang/kernels/ops/layernorm/__init__.py +++ b/python/sglang/kernels/ops/layernorm/__init__.py @@ -31,7 +31,7 @@ _HC_NORM_KERNELS = [ ("hc_combine_norm", "hc_combine_norm"), ("mhc_post_split_h", "mhc_post_split_h"), ("hc_combine_norm", "hc_combine_norm_mxfp8"), - ("mxfp8_epilogue", "rmsnorm_mxfp8"), + ("rmsnorm_mxfp8", "rmsnorm_mxfp8"), ] for _mod, _fn in _HC_NORM_KERNELS: register_kernel( @@ -43,14 +43,12 @@ for _mod, _fn in _HC_NORM_KERNELS: ) del _mod, _fn -# The fused-rmsnorm variants physically live in the shared fused-pointwise -# collection (sglang.kernels.ops.elementwise.elementwise) but stay layernorm ops. for _fn in ("fused_dual_residual_rmsnorm", "fused_rmsnorm"): register_kernel( KernelSpec( op=f"layernorm.{_fn}", backend=KernelBackend.TRITON, - target=f"sglang.kernels.ops.elementwise.elementwise:{_fn}", + target=f"sglang.kernels.ops.layernorm.fused_rmsnorm:{_fn}", ) ) del _fn @@ -84,3 +82,34 @@ register_kernel( ), ) ) + +# Muse Spark fused norms. The per-layer ops are inventoried through their class +# methods; the NVFP4 quant op through its module-level instance. +for _op, _target, _backends in ( + ( + "muse_spark_norm_gate", + "muse_spark_norm_gate:FusedRMSNormGate", + ("torch", "jit", "helion", "triton"), + ), + ( + "muse_spark_norm_residual", + "muse_spark_norm_residual:FusedPostNormResidualPreNorm", + ("torch", "jit", "triton"), + ), + ( + "muse_spark_rmsnorm_nvfp4_quant", + "muse_spark_rmsnorm_nvfp4_quant:_FUSED_NORM_FP4_QUANT", + ("torch", "jit", "helion"), + ), +): + for _bk in _backends: + register_kernel( + KernelSpec( + op=f"layernorm.{_op}", + backend=KernelBackend(_bk), + target=f"sglang.kernels.ops.layernorm.{_target}." + + ("forward_native" if _bk == "torch" else f"forward_{_bk}"), + capabilities=frozenset() if _bk == "torch" else CUDA_ONLY, + ) + ) +del _op, _target, _backends, _bk diff --git a/python/sglang/kernels/ops/muse_spark/fused_op_tuning.py b/python/sglang/kernels/ops/layernorm/_muse_spark_tuning.py similarity index 81% rename from python/sglang/kernels/ops/muse_spark/fused_op_tuning.py rename to python/sglang/kernels/ops/layernorm/_muse_spark_tuning.py index bffe399178..4fd1d18a1c 100644 --- a/python/sglang/kernels/ops/muse_spark/fused_op_tuning.py +++ b/python/sglang/kernels/ops/layernorm/_muse_spark_tuning.py @@ -1,13 +1,13 @@ -"""Hardware- and shape-keyed launch configs for the fused Triton ops. +"""Hardware- and shape-keyed launch configs for the Muse Spark Triton norm ops. Mirrors the FlashInfer autotune-config pattern (a JSON per GPU, keyed on shape): -``autotune_configs/fused_triton_ops/.json`` holds, per op, an ordered -list of rules ``{"when": {...}, : , ...}`` (first match wins). -Each op passes a portable default to :func:`resolve`, used whenever no rule matches. +``configs/muse_spark/.json`` next to this module holds, per op, an +ordered list of rules ``{"when": {...}, : , ...}`` (first match +wins). Each op passes a portable default to :func:`resolve`, used whenever no +rule matches (including when no file exists for the GPU). ```` is the normalized device name on CUDA (``NVIDIA_GB300``). -``when`` keys -are ```` (equality) or ``_{lt,le,ge,gt,eq}``. +``when`` keys are ```` (equality) or ``_{lt,le,ge,gt,eq}``. """ from __future__ import annotations @@ -20,13 +20,7 @@ from typing import Callable import torch -# Shares the curated-config tree with the FlashInfer autotune configs in srt/. -_CONFIG_DIR = ( - Path(__file__).resolve().parents[3] - / "srt" - / "autotune_configs" - / "fused_triton_ops" -) +_CONFIG_DIR = Path(__file__).resolve().parent / "configs" / "muse_spark" _METADATA_KEY = "_metadata" _COMPARATORS = { diff --git a/python/sglang/kernels/ops/layernorm/fused_rmsnorm.py b/python/sglang/kernels/ops/layernorm/fused_rmsnorm.py new file mode 100644 index 0000000000..522e1d9a3b --- /dev/null +++ b/python/sglang/kernels/ops/layernorm/fused_rmsnorm.py @@ -0,0 +1,136 @@ +import torch +import triton +import triton.language as tl + +from sglang.srt.utils import is_hip + +_is_hip = is_hip() + + +@triton.jit +def fused_dual_residual_rmsnorm_kernel( + output_ptr, + mid_ptr, + activ_ptr, + residual_ptr, + weight1_ptr, + weight2_ptr, + eps: tl.constexpr, + hidden_dim: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(axis=0) + input_start = pid * hidden_dim + + offsets = tl.arange(0, BLOCK_SIZE) + mask = offsets < hidden_dim + + a_ = tl.load(activ_ptr + input_start + offsets, mask=mask, other=0.0) + a = a_.to(tl.float32) + rms = tl.sqrt(tl.sum(a * a, axis=0) / hidden_dim + eps) + + r = tl.load(residual_ptr + input_start + offsets, mask=mask, other=0.0) + w1_ = tl.load(weight1_ptr + offsets, mask=mask, other=0.0) + w1 = w1_.to(tl.float32) + + a2r = r + (a / rms * w1).to(r.dtype) + tl.store( + mid_ptr + input_start + offsets, + a2r, + mask=mask, + ) + + a2r = a2r.to(tl.float32) + rms2 = tl.sqrt(tl.sum(a2r * a2r, axis=0) / hidden_dim + eps) + + w2_ = tl.load(weight2_ptr + offsets, mask=mask, other=0.0) + w2 = w2_.to(tl.float32) + + tl.store( + output_ptr + input_start + offsets, + a2r / rms2 * w2, # implicitly casts to output dtype here + mask=mask, + ) + + +def fused_dual_residual_rmsnorm(x, residual, weight1, weight2, eps): + assert len(x.shape) == 2 + assert x.shape == residual.shape and x.dtype == residual.dtype, ( + f"{x.shape=} {residual.shape=} {x.dtype=} {residual.dtype=}" + ) + output, mid = torch.empty_like(x), torch.empty_like(x) + bs, hidden_dim = x.shape + max_warps = 16 if _is_hip else 32 + config = { + "BLOCK_SIZE": triton.next_power_of_2(hidden_dim), + "num_warps": max( + min(triton.next_power_of_2(triton.cdiv(hidden_dim, 256)), max_warps), 4 + ), + } + + fused_dual_residual_rmsnorm_kernel[(bs,)]( + output, + mid, + x, + residual, + weight1, + weight2, + eps=eps, + hidden_dim=hidden_dim, + **config, + ) + + return output, mid + + +# Not layernorm.rmsnorm_fp32: dividing by sqrt(var + eps) rounds differently from rsqrt. +@triton.jit +def fused_rmsnorm_kernel( + output_ptr, + activ_ptr, + weight_ptr, + eps: tl.constexpr, + hidden_dim: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(axis=0).to(tl.int64) + input_start = pid * hidden_dim + + offsets = tl.arange(0, BLOCK_SIZE) + mask = offsets < hidden_dim + + a_ = tl.load(activ_ptr + input_start + offsets, mask=mask, other=0.0) + a = a_.to(tl.float32) + rms = tl.sqrt(tl.sum(a * a, axis=0) / hidden_dim + eps) + + w1_ = tl.load(weight_ptr + offsets, mask=mask, other=0.0) + w1 = w1_.to(tl.float32) + + a_rms = a / rms * w1 + + tl.store( + output_ptr + input_start + offsets, + a_rms, # implicitly casts to output dtype here + mask=mask, + ) + + +def fused_rmsnorm(x, weight, eps, inplace=False): + assert len(x.shape) == 2 + if inplace: + output = x + else: + output = torch.empty_like(x) + bs, hidden_dim = x.shape + max_warps = 16 if _is_hip else 32 + config = { + "BLOCK_SIZE": triton.next_power_of_2(hidden_dim), + "num_warps": max( + min(triton.next_power_of_2(triton.cdiv(hidden_dim, 256)), max_warps), 4 + ), + } + + fused_rmsnorm_kernel[(bs,)]( + output, x, weight, eps=eps, hidden_dim=hidden_dim, **config + ) + return output diff --git a/python/sglang/kernels/ops/layernorm/hc_combine_norm.py b/python/sglang/kernels/ops/layernorm/hc_combine_norm.py index b18e87260e..0759ed1163 100644 --- a/python/sglang/kernels/ops/layernorm/hc_combine_norm.py +++ b/python/sglang/kernels/ops/layernorm/hc_combine_norm.py @@ -4,7 +4,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.layernorm.mxfp8_epilogue import mxfp8_epilogue +from sglang.kernels.ops.quantization.mxfp8_epilogue import mxfp8_epilogue @triton.jit diff --git a/python/sglang/kernels/ops/layernorm/mhc.py b/python/sglang/kernels/ops/layernorm/mhc.py index 9ae2485876..d65725f165 100644 --- a/python/sglang/kernels/ops/layernorm/mhc.py +++ b/python/sglang/kernels/ops/layernorm/mhc.py @@ -2155,7 +2155,7 @@ def npu_hc_pre( @triton.jit -def _hc_combine_kernel( +def _mhc_combine_kernel( x_ptr, pre_ptr, y_ptr, @@ -2710,7 +2710,7 @@ def hc_mix_stats_sinkhorn_deepgemm( return pre, post, comb -def hc_combine( +def mhc_combine( x_flat: torch.Tensor, pre: torch.Tensor, hc: int, out_dtype: torch.dtype ) -> torch.Tensor: """Fused y[m, h] = sum_k pre[m, k] * x_flat[m, k*H + h].""" @@ -2718,7 +2718,7 @@ def hc_combine( h = x_flat.shape[1] // hc y = torch.empty((m, h), dtype=out_dtype, device=x_flat.device) block_h = 1024 - _hc_combine_kernel[(m, triton.cdiv(h, block_h))]( + _mhc_combine_kernel[(m, triton.cdiv(h, block_h))]( x_flat, pre, y, diff --git a/python/sglang/kernels/ops/layernorm/mhc_boundary_hip.py b/python/sglang/kernels/ops/layernorm/mhc_boundary_hip.py index 33469f915f..6706c8c907 100644 --- a/python/sglang/kernels/ops/layernorm/mhc_boundary_hip.py +++ b/python/sglang/kernels/ops/layernorm/mhc_boundary_hip.py @@ -8,7 +8,7 @@ import triton import triton.language as tl from sglang.kernels.jit.utils import cache_once, load_jit -from sglang.kernels.ops.quantization.rmsnorm_fake_quant_amd_gfx95 import ( +from sglang.kernels.ops.layernorm.rmsnorm_fake_quant_amd_gfx95 import ( Fp8GridActivation, Mxfp8Activation, rmsnorm_fake_quant_row, @@ -488,7 +488,7 @@ def _hc_boundary_partial_kernel( ): """Grid (cdiv(M, BLOCK_M), H // BLOCK_K), one hidden slice of every copy per program. HAS_POST: res_out[k] = post[k]*x + sum_j comb[j,k]*res[j] in aiter::mhc_post's order, read back as bf16 - for the statistics; HAS_COMBINE: y = sum_k pre_prev[k] * copy_k in _hc_combine_kernel's order.""" + for the statistics; HAS_COMBINE: y = sum_k pre_prev[k] * copy_k in _mhc_combine_kernel's order.""" tl.static_assert(HC == 4, "the weight tiles are prefetched by name") pid_m = tl.program_id(0) pid_t = tl.program_id(1) diff --git a/python/sglang/kernels/ops/muse_spark/fused_norm_gate.py b/python/sglang/kernels/ops/layernorm/muse_spark_norm_gate.py similarity index 96% rename from python/sglang/kernels/ops/muse_spark/fused_norm_gate.py rename to python/sglang/kernels/ops/layernorm/muse_spark_norm_gate.py index ec89ab820b..ea63251be7 100644 --- a/python/sglang/kernels/ops/muse_spark/fused_norm_gate.py +++ b/python/sglang/kernels/ops/layernorm/muse_spark_norm_gate.py @@ -15,7 +15,7 @@ import triton import triton.language as tl from sglang.kernels.fused_op import BaseFusedOp -from sglang.kernels.ops.muse_spark import fused_op_tuning +from sglang.kernels.ops.layernorm import _muse_spark_tuning from sglang.kernels.spec import CapabilityRequirement, KernelBackend @@ -110,7 +110,7 @@ def _rms_norm_gate_config( op_name: str = "rms_norm_gate", ) -> tuple[int, int, dict]: """Resolve launch parameters for the exact norm-gate input layout.""" - cfg = fused_op_tuning.resolve( + cfg = _muse_spark_tuning.resolve( op_name, {"block_m": 8, "num_warps": 4}, device_index=device_index, @@ -123,7 +123,7 @@ def _rms_norm_gate_config( has_gate=has_gate, has_weight=has_weight, ) - extra = fused_op_tuning.split_extra(cfg, ("block_m", "num_warps")) + extra = _muse_spark_tuning.split_extra(cfg, ("block_m", "num_warps")) return cfg["block_m"], cfg["num_warps"], extra @@ -155,7 +155,7 @@ class FusedRMSNormGate(BaseFusedOp): rank-grouped gates to the same zero-copy grouped view before execution. """ - op = "muse_spark.rms_norm_gate" + op = "layernorm.muse_spark_norm_gate" priority = (KernelBackend.JIT, KernelBackend.HELION, KernelBackend.TRITON) capabilities = { KernelBackend.HELION: {CapabilityRequirement.CUDA}, @@ -229,7 +229,7 @@ class FusedRMSNormGate(BaseFusedOp): if x.numel() == 0: return self.forward_native(x, g) - from sglang.kernels.ops.muse_spark.norm_gate import ( + from sglang.kernels.ops.layernorm.muse_spark_norm_gate_jit import ( rms_norm_gate as _jit_rms_norm_gate, ) @@ -248,9 +248,9 @@ class FusedRMSNormGate(BaseFusedOp): x: torch.Tensor, g: torch.Tensor = None, ) -> bool: - from sglang.kernels.ops.muse_spark.helion_runtime import helion_supported + from sglang.kernels.jit.utils.helion_runtime import helion_supported - if not helion_supported("rms_norm_gate", x): + if not helion_supported(x): return False self._ensure_cached_weight() return ( @@ -278,13 +278,13 @@ class FusedRMSNormGate(BaseFusedOp): ) -> torch.Tensor: """Run Helion behind the existing fused operation boundary.""" from sglang.kernels.jit.utils import is_arch_support_pdl - from sglang.kernels.ops.muse_spark.helion_norm_gate import ( + from sglang.kernels.jit.utils.helion_runtime import ( + run_pinned_helion, + ) + from sglang.kernels.ops.layernorm.muse_spark_norm_gate_helion import ( RMS_NORM_GATE_CONFIGS, _rms_norm_gate_helion_kernel, ) - from sglang.kernels.ops.muse_spark.helion_runtime import ( - run_pinned_helion, - ) return run_pinned_helion( _rms_norm_gate_helion_kernel, diff --git a/python/sglang/kernels/ops/muse_spark/helion_norm_gate.py b/python/sglang/kernels/ops/layernorm/muse_spark_norm_gate_helion.py similarity index 98% rename from python/sglang/kernels/ops/muse_spark/helion_norm_gate.py rename to python/sglang/kernels/ops/layernorm/muse_spark_norm_gate_helion.py index 9e0bee8baf..019d794a8e 100644 --- a/python/sglang/kernels/ops/muse_spark/helion_norm_gate.py +++ b/python/sglang/kernels/ops/layernorm/muse_spark_norm_gate_helion.py @@ -6,7 +6,7 @@ import helion import helion.language as hl import torch -from sglang.kernels.ops.muse_spark.helion_runtime import ( +from sglang.kernels.jit.utils.helion_runtime import ( token_count_specialization_key, ) diff --git a/python/sglang/kernels/ops/muse_spark/norm_gate.py b/python/sglang/kernels/ops/layernorm/muse_spark_norm_gate_jit.py similarity index 100% rename from python/sglang/kernels/ops/muse_spark/norm_gate.py rename to python/sglang/kernels/ops/layernorm/muse_spark_norm_gate_jit.py diff --git a/python/sglang/kernels/ops/muse_spark/fused_norm_residual.py b/python/sglang/kernels/ops/layernorm/muse_spark_norm_residual.py similarity index 89% rename from python/sglang/kernels/ops/muse_spark/fused_norm_residual.py rename to python/sglang/kernels/ops/layernorm/muse_spark_norm_residual.py index 2d011d61f1..8d0b813897 100644 --- a/python/sglang/kernels/ops/muse_spark/fused_norm_residual.py +++ b/python/sglang/kernels/ops/layernorm/muse_spark_norm_residual.py @@ -19,15 +19,8 @@ import triton import triton.language as tl from sglang.kernels.fused_op import BaseFusedOp, get_fused_op_backend -from sglang.kernels.ops.muse_spark import fused_op_tuning -from sglang.kernels.ops.muse_spark.all_gather_norm_residual import ( - all_gather_norm_residual, -) +from sglang.kernels.ops.layernorm import _muse_spark_tuning from sglang.kernels.spec import CapabilityRequirement, KernelBackend -from sglang.srt.layers.muse_spark.muse_spark_layers import ( - MuseSpark1xNormPreservingResidualConnection, - MuseSpark1xRMSNormWithInputScale, -) logger = logging.getLogger(__name__) @@ -39,7 +32,7 @@ def _fused_norm_residual_config( post_norm_is_input_scale: bool, ) -> tuple[int, int, dict]: """Resolve launch parameters for the exact residual-norm shape.""" - cfg = fused_op_tuning.resolve( + cfg = _muse_spark_tuning.resolve( "fused_norm_residual", {"num_warps": 8, "num_stages": 1}, hidden_size=hidden_size, @@ -47,12 +40,12 @@ def _fused_norm_residual_config( num_tokens=num_tokens, post_norm_is_input_scale=post_norm_is_input_scale, ) - extra = fused_op_tuning.split_extra(cfg, ("num_warps", "num_stages")) + extra = _muse_spark_tuning.split_extra(cfg, ("num_warps", "num_stages")) return cfg["num_warps"], cfg["num_stages"], extra def _jit_launch_config_ok(hidden_size: int) -> bool: - from sglang.kernels.ops.muse_spark.norm_residual import _launch_config + from sglang.kernels.ops.layernorm.muse_spark_norm_residual_jit import _launch_config try: _launch_config(hidden_size, 1) @@ -167,6 +160,7 @@ class FusedPostNormResidualPreNorm(BaseFusedOp): - ``forward_jit``: the JIT CUDA kernel. """ + op = "layernorm.muse_spark_norm_residual" priority = (KernelBackend.JIT, KernelBackend.TRITON) capabilities = { KernelBackend.JIT: {CapabilityRequirement.CUDA}, @@ -175,23 +169,27 @@ class FusedPostNormResidualPreNorm(BaseFusedOp): }, } - def __init__(self, post_norm, residual_add, pre_norm): + def __init__( + self, + post_norm, + residual_add, + pre_norm, + *, + post_norm_is_input_scale: bool, + use_residual_gate: bool, + ): + """``post_norm_is_input_scale``: the post-norm scales its input by the + weight before normalizing. ``use_residual_gate``: ``residual_add`` is the + norm-preserving gated connection (``_alpha``/``_beta``/``gate``), not a + plain add. Both select kernel template specializations.""" super().__init__() # Store module references (weights are owned by the decoder layer module tree) object.__setattr__(self, "_post_norm", post_norm) object.__setattr__(self, "_residual_add", residual_add) object.__setattr__(self, "_pre_norm", pre_norm) - # Introspection flags for CUDA C kernel template specialization - self._post_norm_is_input_scale = isinstance( - post_norm, MuseSpark1xRMSNormWithInputScale - ) - self._use_residual_gate = isinstance( - residual_add, MuseSpark1xNormPreservingResidualConnection - ) - self._pre_norm_is_input_scale = isinstance( - pre_norm, MuseSpark1xRMSNormWithInputScale - ) + self._post_norm_is_input_scale = post_norm_is_input_scale + self._use_residual_gate = use_residual_gate self._hidden_size = post_norm.hidden_size def backend_eligible( @@ -447,24 +445,33 @@ class FusedPostNormResidualPreNorm(BaseFusedOp): pre_norm_dtype=pre_norm_dtype, ) - from sglang.kernels.ops.muse_spark.norm_residual import ( + from sglang.kernels.ops.layernorm.muse_spark_norm_residual_jit import ( fused_norm_residual as _jit_fused_norm_residual, ) - self._ensure_cached_weights(branch_dtype) - return _jit_fused_norm_residual( branch, stream, - self._post_norm.eps, - self._pre_norm.eps, + **self.jit_kernel_kwargs(branch_dtype), + want_router=router_dtype is not None, + pre_norm_bf16=pre_norm_dtype is torch.bfloat16, + ) + + def jit_kernel_kwargs(self, branch_dtype: torch.dtype = torch.bfloat16) -> dict: + """Norm and residual-gate arguments of the JIT kernels for this boundary. + + Shared with ``communication.muse_spark_all_gather_norm_residual``, which + applies the same boundary to the TP all-gather of bf16 column slices. + """ + self._ensure_cached_weights(branch_dtype) + return dict( + post_eps=self._post_norm.eps, + pre_eps=self._pre_norm.eps, post_norm_is_input_scale=self._post_norm_is_input_scale, post_weight=self._cc_post_weight if self._post_norm.has_weight else None, pre_weight=self._cc_pre_weight if self._pre_norm.has_weight else None, alpha=self._cc_alpha if self._use_residual_gate else None, beta=self._cc_beta if self._use_residual_gate else None, - want_router=router_dtype is not None, - pre_norm_bf16=pre_norm_dtype is torch.bfloat16, ) def jit_selected(self) -> bool: @@ -479,21 +486,3 @@ class FusedPostNormResidualPreNorm(BaseFusedOp): and BaseFusedOp.backend_eligible(self, KernelBackend.JIT) and _jit_launch_config_ok(self._hidden_size) ) - - def forward_with_ag( - self, stream: torch.Tensor, local: torch.Tensor, comm - ) -> tuple[torch.Tensor, torch.Tensor]: - """Apply the boundary to the TP all-gather of bf16 column slices ``local``.""" - self._ensure_cached_weights(torch.bfloat16) - return all_gather_norm_residual( - comm, - local, - stream, - self._post_norm.eps, - self._pre_norm.eps, - post_norm_is_input_scale=self._post_norm_is_input_scale, - post_weight=self._cc_post_weight if self._post_norm.has_weight else None, - pre_weight=self._cc_pre_weight if self._pre_norm.has_weight else None, - alpha=self._cc_alpha if self._use_residual_gate else None, - beta=self._cc_beta if self._use_residual_gate else None, - ) diff --git a/python/sglang/kernels/ops/muse_spark/norm_residual.py b/python/sglang/kernels/ops/layernorm/muse_spark_norm_residual_jit.py similarity index 98% rename from python/sglang/kernels/ops/muse_spark/norm_residual.py rename to python/sglang/kernels/ops/layernorm/muse_spark_norm_residual_jit.py index 293e97a782..d843094dc2 100644 --- a/python/sglang/kernels/ops/muse_spark/norm_residual.py +++ b/python/sglang/kernels/ops/layernorm/muse_spark_norm_residual_jit.py @@ -1,7 +1,7 @@ """Fused post-norm + residual-add + pre-norm for MuseSpark v1.2. CUDA replacement for ``_fused_norm_residual_triton_kernel`` in -``sglang.kernels.ops.muse_spark.fused_norm_residual``. It dispatches between the original +``sglang.kernels.ops.layernorm.muse_spark_norm_residual``. It dispatches between the original one-CTA kernel and the measured multi-CTA cluster kernel from the row count. The bf16 round-trips reproduced here are load-bearing for parity with the torch diff --git a/python/sglang/kernels/ops/muse_spark/norm_fp4_quant.py b/python/sglang/kernels/ops/layernorm/muse_spark_rmsnorm_nvfp4_quant.py similarity index 67% rename from python/sglang/kernels/ops/muse_spark/norm_fp4_quant.py rename to python/sglang/kernels/ops/layernorm/muse_spark_rmsnorm_nvfp4_quant.py index ce49886941..8f7e5e4a46 100644 --- a/python/sglang/kernels/ops/muse_spark/norm_fp4_quant.py +++ b/python/sglang/kernels/ops/layernorm/muse_spark_rmsnorm_nvfp4_quant.py @@ -103,10 +103,52 @@ def _fused_norm_fp4_quant_jit( return fp4, sf, pts, normed +def _e2m1_codes(x: torch.Tensor) -> torch.Tensor: + """Round-to-nearest-even, saturating E2M1 nibble (sign in bit 3) of fp32 ``x``.""" + ax = x.abs() + # Midpoints between 0, .5, 1, 1.5, 2, 3, 4, 6; ties go to the even code. + magnitude = ( + (ax > 0.25).to(torch.uint8) + + (ax >= 0.75) + + (ax > 1.25) + + (ax >= 1.75) + + (ax > 2.5) + + (ax >= 3.5) + + (ax > 5.0) + ) + return magnitude | (torch.signbit(x).to(torch.uint8) << 3) + + +def _nvfp4_quantize_per_token_native( + rows: torch.Tensor, global_scale_inv: float +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """``(fp4, sf, per_token_scale)`` of bf16 ``rows`` [M, K], as the kernel lays them out.""" + m, hidden_size = rows.shape + r = rows.float() + amax = r.abs().amax(-1) + nonzero = amax != 0 + per_token_scale = torch.where(nonzero, amax * global_scale_inv, 0.0) + encode_scale = torch.where( + nonzero, 1.0 / per_token_scale, torch.finfo(torch.float32).max + ) + + blocks = r.view(m, hidden_size // _BLOCK, _BLOCK) + block_max = blocks.abs().amax(-1) + scale = encode_scale[:, None] * (block_max * (1.0 / 6.0)) + sf = scale.clamp(max=torch.finfo(torch.float8_e4m3fn).max).to(torch.float8_e4m3fn) + sf_f32 = sf.float() + out_scale = torch.where( + sf_f32 == 0, 0.0, 1.0 / (sf_f32 * (1.0 / encode_scale[:, None])) + ) + codes = _e2m1_codes(blocks * out_scale[..., None]).view(m, hidden_size) + fp4 = codes[:, 0::2] | (codes[:, 1::2] << 4) + return fp4, sf.view(torch.uint8), per_token_scale + + class _FusedNormFP4Quant(BaseFusedOp): """Backend dispatcher for the Muse Spark fused norm/quant boundary.""" - op = "muse_spark.norm_fp4_quant" + op = "layernorm.muse_spark_rmsnorm_nvfp4_quant" priority = (KernelBackend.JIT, KernelBackend.HELION) capabilities = { KernelBackend.HELION: {CapabilityRequirement.CUDA}, @@ -137,11 +179,22 @@ class _FusedNormFP4Quant(BaseFusedOp): if backend is not KernelBackend.HELION: return True - from sglang.kernels.ops.muse_spark.helion_runtime import helion_supported + from sglang.kernels.jit.utils.helion_runtime import ( + base_package_version, + helion_supported, + warn_helion_unsupported, + ) + if not helion_supported(x): + return False + flashinfer_version = base_package_version("flashinfer-python") + if flashinfer_version != "0.6.17": + return warn_helion_unsupported( + "NVFP4 quantization requires FlashInfer 0.6.17, found " + f"{flashinfer_version or 'not installed'}" + ) return ( - helion_supported("norm_fp4_quant", x) - and x.shape[0] > 0 + x.shape[0] > 0 and x.dtype == torch.bfloat16 and x.dim() == 2 and x.is_contiguous() @@ -163,7 +216,24 @@ class _FusedNormFP4Quant(BaseFusedOp): quantize: bool = True, want_normed: bool = False, ): - raise NotImplementedError("norm_fp4_quant has no device-independent backend") + """Torch reference for the JIT kernel's outputs. + + Same formula and rounding points as the kernel, but IEEE reciprocals + where it uses ``rcp.approx``, so a value on a rounding boundary can land + one fp8/fp4 code apart. + """ + assert quantize or want_normed + xf = x.float() + normed = xf * torch.rsqrt(xf.square().mean(-1, keepdim=True) + eps) + if weight is not None: + normed = normed * weight.float() + normed = normed.to(torch.bfloat16) + if not quantize: + return None, None, None, normed + fp4, sf, per_token_scale = _nvfp4_quantize_per_token_native( + normed, global_scale_inv + ) + return fp4, sf, per_token_scale, normed if want_normed else None def forward_jit( self, @@ -194,7 +264,7 @@ class _FusedNormFP4Quant(BaseFusedOp): quantize: bool = True, want_normed: bool = False, ): - from sglang.kernels.ops.muse_spark.helion_norm_fp4_quant import ( + from sglang.kernels.ops.layernorm.muse_spark_rmsnorm_nvfp4_quant_helion import ( fused_norm_fp4_quant_helion, ) diff --git a/python/sglang/kernels/ops/muse_spark/helion_norm_fp4_quant.py b/python/sglang/kernels/ops/layernorm/muse_spark_rmsnorm_nvfp4_quant_helion.py similarity index 99% rename from python/sglang/kernels/ops/muse_spark/helion_norm_fp4_quant.py rename to python/sglang/kernels/ops/layernorm/muse_spark_rmsnorm_nvfp4_quant_helion.py index 3a3d248340..ce90c582e9 100644 --- a/python/sglang/kernels/ops/muse_spark/helion_norm_fp4_quant.py +++ b/python/sglang/kernels/ops/layernorm/muse_spark_rmsnorm_nvfp4_quant_helion.py @@ -8,7 +8,7 @@ import torch from helion.autotuner import BooleanFragment from sglang.kernels.jit.utils import is_arch_support_pdl -from sglang.kernels.ops.muse_spark.helion_runtime import ( +from sglang.kernels.jit.utils.helion_runtime import ( token_count_specialization_key, ) @@ -680,7 +680,7 @@ def fused_norm_fp4_quant_helion( assert weight.dtype == torch.bfloat16 and weight.shape == (x.shape[1],) weight = weight.contiguous() - from sglang.kernels.ops.muse_spark.helion_runtime import run_pinned_helion + from sglang.kernels.jit.utils.helion_runtime import run_pinned_helion fp4, sf, per_token_scale = run_pinned_helion( _norm_fp4_quant_helion_kernel, diff --git a/python/sglang/kernels/ops/quantization/rmsnorm_fake_quant_amd_gfx95.py b/python/sglang/kernels/ops/layernorm/rmsnorm_fake_quant_amd_gfx95.py similarity index 100% rename from python/sglang/kernels/ops/quantization/rmsnorm_fake_quant_amd_gfx95.py rename to python/sglang/kernels/ops/layernorm/rmsnorm_fake_quant_amd_gfx95.py diff --git a/python/sglang/kernels/ops/layernorm/rmsnorm_mxfp8.py b/python/sglang/kernels/ops/layernorm/rmsnorm_mxfp8.py new file mode 100644 index 0000000000..3b076df318 --- /dev/null +++ b/python/sglang/kernels/ops/layernorm/rmsnorm_mxfp8.py @@ -0,0 +1,67 @@ +"""RMSNorm with the MXFP8 epilogue of ``ops.quantization.mxfp8_epilogue``.""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from sglang.kernels.ops.quantization.mxfp8_epilogue import mxfp8_epilogue + + +@triton.jit +def _rmsnorm_mxfp8_kernel( + X, + W, + Y, + Q, + S, + SX: tl.constexpr, + M: tl.constexpr, + K: tl.constexpr, + EPS: tl.constexpr, + BLOCK: tl.constexpr, + GROUPS: tl.constexpr, +): + row = tl.program_id(0) + if row < M: + h = tl.arange(0, BLOCK) + v = tl.load(X + row * SX + h, h < K, 0).to(tl.float32) + weight = tl.load(W + h, h < K, 0).to(tl.float32) + inv = tl.rsqrt(tl.sum(v * v, 0) / K + EPS) + y = (v * inv * weight).to(tl.bfloat16) + tl.store(Y + row * K + h, y, h < K) + mxfp8_epilogue(y, row, Q, S, K, BLOCK, GROUPS, 0, GROUPS) + else: + # Padding scale entries are disjoint from the live rows above. + off = (row - M) * 512 + tl.arange(0, 512) + pad_row = ((off // 4) % 4) * 32 + ((off // 16) % 32) + tl.store(S + off, 0, (off < GROUPS * 128) & (pad_row >= M)) + + +def rmsnorm_mxfp8( + x: torch.Tensor, weight: torch.Tensor, eps: float +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """RMSNorm returning ``(y_bf16, y_q, y_sf)`` with the same MXFP8 epilogue.""" + m, k = x.shape + assert 0 < m <= 8 and k % 32 == 0 + assert x.dtype == weight.dtype == torch.bfloat16 and x.stride(1) == 1 + y = torch.empty_like(x, memory_format=torch.contiguous_format) + q = torch.empty((m, k), dtype=torch.float8_e4m3fn, device=x.device) + s = torch.empty((k // 32) * 128, dtype=torch.uint8, device=x.device) + _rmsnorm_mxfp8_kernel[(m + triton.cdiv(s.numel(), 512),)]( + x, + weight, + y, + q, + s, + SX=x.stride(0), + M=m, + K=k, + EPS=eps, + BLOCK=triton.next_power_of_2(k), + GROUPS=k // 32, + num_warps=8, + enable_fp_fusion=False, + ) + return y, q, s diff --git a/python/sglang/kernels/ops/lplb/__init__.py b/python/sglang/kernels/ops/lplb/__init__.py index 2c6f32ed9d..d08ac943d2 100644 --- a/python/sglang/kernels/ops/lplb/__init__.py +++ b/python/sglang/kernels/ops/lplb/__init__.py @@ -1 +1,29 @@ """LPLB (LP-based Load Balancer) JIT kernels for expert parallelism.""" + +from sglang.kernels.registry import register_kernel +from sglang.kernels.spec import ( + CUDA_ONLY, + CapabilityRequirement, + KernelBackend, + KernelSpec, +) + +# Mirrors the SM >= 9.0 check in lplb.torch_solver. +register_kernel( + KernelSpec( + op="lplb.solve_ipm", + backend=KernelBackend.JIT, + target="sglang.kernels.ops.lplb.cuda_solver:solve_ipm", + capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}), + ) +) +for _fn in ("prep_lp_inputs", "extract_log2phy_prob", "dispatch_probability"): + register_kernel( + KernelSpec( + op=f"lplb.{_fn}", + backend=KernelBackend.JIT, + target=f"sglang.kernels.ops.lplb.cuda_solver:{_fn}", + capabilities=CUDA_ONLY, + ) + ) +del _fn diff --git a/python/sglang/kernels/ops/mamba/__init__.py b/python/sglang/kernels/ops/mamba/__init__.py index a7649e0148..6466f6fc3b 100644 --- a/python/sglang/kernels/ops/mamba/__init__.py +++ b/python/sglang/kernels/ops/mamba/__init__.py @@ -144,3 +144,13 @@ register_kernel( capabilities=CUDA_ONLY, ) ) +for _fn in ("kda_track_conv_window", "kda_track_ssm_state"): + register_kernel( + KernelSpec( + op=f"mamba.{_fn}", + backend=KernelBackend.TRITON, + target=f"sglang.kernels.ops.mamba.kda_prefill_track:{_fn}", + capabilities=CUDA_ONLY, + ) + ) +del _fn diff --git a/python/sglang/kernels/ops/attention/fla/kda_prefill_graph_track.py b/python/sglang/kernels/ops/mamba/kda_prefill_track.py similarity index 100% rename from python/sglang/kernels/ops/attention/fla/kda_prefill_graph_track.py rename to python/sglang/kernels/ops/mamba/kda_prefill_track.py diff --git a/python/sglang/kernels/ops/memory/gpu_tensor_hash.py b/python/sglang/kernels/ops/memory/gpu_tensor_hash.py index 0622928e5c..d76fafc33f 100644 --- a/python/sglang/kernels/ops/memory/gpu_tensor_hash.py +++ b/python/sglang/kernels/ops/memory/gpu_tensor_hash.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Logits processing.""" +"""Content hash of a GPU tensor: per-tile 32-bit word hashes reduced to one 64-bit value.""" import torch import triton diff --git a/python/sglang/kernels/ops/moe/__init__.py b/python/sglang/kernels/ops/moe/__init__.py index c09ecb74fc..a825e84ff2 100644 --- a/python/sglang/kernels/ops/moe/__init__.py +++ b/python/sglang/kernels/ops/moe/__init__.py @@ -129,7 +129,7 @@ def situ_and_mul_masked_post_quant( transposed: bool = False, swizzle: bool = False, ) -> None: - from ._jit_situ_and_mul_quant import situ_and_mul_masked_post_quant as impl + from .kimi_k3_situ_and_mul_quant import situ_and_mul_masked_post_quant as impl return impl( input, @@ -159,8 +159,6 @@ _PHASE25_TRITON_KERNELS = [ ("fused_moe_triton_kernels", "fused_moe_kernel"), ("fused_moe_triton_kernels", "fused_moe_kernel_gptq_awq"), ("mxfp8_moe_amd_gfx95", "fused_experts_mxfp8"), - ("rocm_moe_utils", "upscale"), - ("rocm_moe_utils", "upscale_mxfp4"), ("router", "fused_moe_router_shim"), ("deepep_waterfill_kernels", "materialize_waterfill_dispatch_fused"), ("fill_padded_rows", "_fill_padded_rows"), @@ -206,7 +204,7 @@ register_kernel( KernelSpec( op="moe.situ_and_mul_masked_post_quant", backend=KernelBackend.JIT, - target="sglang.kernels.ops.moe._jit_situ_and_mul_quant:situ_and_mul_masked_post_quant", + target="sglang.kernels.ops.moe.kimi_k3_situ_and_mul_quant:situ_and_mul_masked_post_quant", capabilities=CUDA_ONLY, ) ) @@ -252,14 +250,6 @@ register_kernel( capabilities=frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP}), ) ) -register_kernel( - KernelSpec( - op="moe.silu_and_mul_clamp", - backend=KernelBackend.JIT, - target="sglang.kernels.ops.moe.dsv4:silu_and_mul_clamp", - capabilities=frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP}), - ) -) register_kernel( KernelSpec( op="moe.silu_and_mul_masked_post_quant", @@ -268,11 +258,24 @@ register_kernel( capabilities=frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP}), ) ) + +# Muse Spark routing, built per layer: inventoried through its class methods. +for _bk in ("torch", "jit", "triton"): + register_kernel( + KernelSpec( + op="moe.muse_spark_routing", + backend=KernelBackend(_bk), + target="sglang.kernels.ops.moe.muse_spark_routing:FusedMuseSparkRouting." + + ("forward_native" if _bk == "torch" else f"forward_{_bk}"), + capabilities=frozenset() if _bk == "torch" else CUDA_ONLY, + ) + ) +del _bk register_kernel( KernelSpec( - op="moe.silu_and_mul_contig_post_quant", - backend=KernelBackend.JIT, - target="sglang.kernels.ops.moe.dsv4:silu_and_mul_contig_post_quant", - capabilities=frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP}), + op="moe.muse_spark_post_expert_finalize", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.moe.muse_spark_post_expert_finalize_triton:fused_post_expert_norm_weighted_sum", + capabilities=CUDA_ONLY, ) ) diff --git a/python/sglang/kernels/ops/moe/dsv4.py b/python/sglang/kernels/ops/moe/dsv4.py index 7c9fa71f77..d4d62535fc 100644 --- a/python/sglang/kernels/ops/moe/dsv4.py +++ b/python/sglang/kernels/ops/moe/dsv4.py @@ -9,9 +9,6 @@ from sglang.kernels.jit.utils import ( load_jit, make_cpp_args, ) -from sglang.srt.utils import is_xpu - -_is_xpu = is_xpu() def make_name(name: str) -> str: @@ -72,41 +69,6 @@ def _jit_silu_mul_quant_varlen_module( ) -@cache_once -def _jit_silu_mul_quant_contig_module( - quant_group_size: int, - scale_ue8m0: bool, - swizzle: bool, - apply_swiglu_limit: bool, -): - args = make_cpp_args( - quant_group_size, - scale_ue8m0, - swizzle, - is_arch_support_pdl(), - apply_swiglu_limit, - ) - return load_jit( - make_name("silu_mul_quant_contig"), - *args, - cuda_files=["deepseek_v4/silu_and_mul_masked_post_quant.cuh"], - cuda_wrappers=[("run", f"SiluAndMulContigPostQuantKernel<{args}>::run")], - extra_cuda_cflags=["-use_fast_math"], - ) - - -@cache_once -def _jit_silu_and_mul_clamp_module(dtype: torch.dtype): - args = make_cpp_args(dtype, is_arch_support_pdl()) - return load_jit( - make_name("silu_and_mul_clamp"), - *args, - cuda_files=["deepseek_v4/silu_and_mul_masked_post_quant.cuh"], - cuda_wrappers=[("run", f"SiluAndMulClampKernel<{args}>::run")], - extra_cuda_cflags=["-use_fast_math"], - ) - - def mask_topk_ids(topk_ids: torch.Tensor, num_token_non_padded: torch.Tensor): return _jit_mask_topk_module().run(topk_ids, num_token_non_padded) @@ -185,20 +147,6 @@ def mega_moe_pre_dispatch( ) -def silu_and_mul_clamp( - input: torch.Tensor, - output: torch.Tensor, - swiglu_limit: float, -) -> None: - if _is_xpu: - from sgl_kernel import silu_and_mul_clamp - - silu_and_mul_clamp(input, output, float(swiglu_limit)) - else: - module = _jit_silu_and_mul_clamp_module(input.dtype) - module.run(input, output, float(swiglu_limit)) - - def silu_and_mul_masked_post_quant( input: torch.Tensor, output: torch.Tensor, @@ -224,26 +172,3 @@ def silu_and_mul_masked_post_quant( transposed, float(swiglu_limit) if apply_swiglu_limit else 0.0, ) - - -def silu_and_mul_contig_post_quant( - input: torch.Tensor, - output: torch.Tensor, - output_scale: torch.Tensor, - quant_group_size: int, - scale_ue8m0: bool = False, - transposed: bool = False, - swiglu_limit: Optional[float] = None, - swizzle: bool = False, -) -> None: - apply_swiglu_limit = swiglu_limit is not None - module = _jit_silu_mul_quant_contig_module( - quant_group_size, scale_ue8m0, swizzle, apply_swiglu_limit - ) - module.run( - input, - output, - output_scale, - transposed, - float(swiglu_limit) if apply_swiglu_limit else 0.0, - ) diff --git a/python/sglang/kernels/ops/moe/ep_moe_kernels.py b/python/sglang/kernels/ops/moe/ep_moe_kernels.py index dec42b8167..1f4587fd97 100644 --- a/python/sglang/kernels/ops/moe/ep_moe_kernels.py +++ b/python/sglang/kernels/ops/moe/ep_moe_kernels.py @@ -1669,7 +1669,7 @@ def moe_ep_deepgemm_preprocess( from sglang.srt.layers import deep_gemm_wrapper if is_fp8 and (use_mxfp8 or deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0): - from sglang.kernels.ops.quantization.minimax_quant_ue8m0 import ( + from sglang.kernels.ops.moe.minimax_quant_ue8m0 import ( per_token_quant_fp8_ue8m0_scatter, ) diff --git a/python/sglang/kernels/ops/moe/expert_pack_mxfp4.py b/python/sglang/kernels/ops/moe/expert_pack_mxfp4.py index 5139f52752..68c1119195 100644 --- a/python/sglang/kernels/ops/moe/expert_pack_mxfp4.py +++ b/python/sglang/kernels/ops/moe/expert_pack_mxfp4.py @@ -29,7 +29,6 @@ def _extension() -> Module: cuda_wrappers=[ ("mxfp4_matvec", "mxfp4_matvec"), ("mxfp4_matvec_dual", "mxfp4_matvec_dual"), - ("mxfp4_marlin_repack", "mxfp4_marlin_repack"), ], extra_cuda_cflags=["--use_fast_math"], ) @@ -100,32 +99,3 @@ def prewarm_mxfp4_extension() -> None: """Build and load the extension before the server accepts requests.""" _extension() - - -def mxfp4_marlin_repack( - raw: torch.Tensor, - source_slots: torch.Tensor, - target_slots: torch.Tensor, - *, - role_bytes: int, - hidden_size: int, - intermediate_size: int, - w13: torch.Tensor, - w2: torch.Tensor, - w13_scale: torch.Tensor, - w2_scale: torch.Tensor, -) -> None: - """Repack raw GGUF objects into contiguous Marlin SoA cache tensors.""" - - _extension().mxfp4_marlin_repack( - raw, - source_slots, - target_slots, - role_bytes, - hidden_size, - intermediate_size, - w13, - w2, - w13_scale, - w2_scale, - ) diff --git a/python/sglang/kernels/ops/moe/_jit_situ_and_mul_quant.py b/python/sglang/kernels/ops/moe/kimi_k3_situ_and_mul_quant.py similarity index 100% rename from python/sglang/kernels/ops/moe/_jit_situ_and_mul_quant.py rename to python/sglang/kernels/ops/moe/kimi_k3_situ_and_mul_quant.py diff --git a/python/sglang/kernels/ops/quantization/minimax_quant_ue8m0.py b/python/sglang/kernels/ops/moe/minimax_quant_ue8m0.py similarity index 100% rename from python/sglang/kernels/ops/quantization/minimax_quant_ue8m0.py rename to python/sglang/kernels/ops/moe/minimax_quant_ue8m0.py diff --git a/python/sglang/kernels/ops/muse_spark/fused_post_expert_norm_weighted_sum_triton.py b/python/sglang/kernels/ops/moe/muse_spark_post_expert_finalize_triton.py similarity index 100% rename from python/sglang/kernels/ops/muse_spark/fused_post_expert_norm_weighted_sum_triton.py rename to python/sglang/kernels/ops/moe/muse_spark_post_expert_finalize_triton.py diff --git a/python/sglang/kernels/ops/muse_spark/fused_muse_spark_routing.py b/python/sglang/kernels/ops/moe/muse_spark_routing.py similarity index 98% rename from python/sglang/kernels/ops/muse_spark/fused_muse_spark_routing.py rename to python/sglang/kernels/ops/moe/muse_spark_routing.py index 1f36ce28c3..3bdfc60efa 100644 --- a/python/sglang/kernels/ops/muse_spark/fused_muse_spark_routing.py +++ b/python/sglang/kernels/ops/moe/muse_spark_routing.py @@ -13,6 +13,11 @@ operations in a single pass: 7. Return results The kernel is CUDA graph compatible by accepting pre-allocated output buffers. + +Not ``moe.moe_fused_gate``: that router has no logit multiplier or EP +local-expert map, adds the bias before softmax, zeroes padded-row weights, and +rounds weights as ``score / (sum + eps) * scale`` instead of +``score * (scale / (sum + eps))``. """ from __future__ import annotations @@ -686,6 +691,7 @@ class FusedMuseSparkRouting(BaseFusedOp): 5. Renormalize: scores / (sum(scores) + eps) * route_scale """ + op = "moe.muse_spark_routing" priority = (KernelBackend.JIT, KernelBackend.TRITON) capabilities = { KernelBackend.JIT: {CapabilityRequirement.CUDA}, @@ -766,8 +772,8 @@ class FusedMuseSparkRouting(BaseFusedOp): ): """Fused JIT CUDA kernel implementation.""" assert renormalize, "MuseSpark routing requires renormalize=True" - from sglang.kernels.ops.muse_spark.routing import ( - fused_muse_spark_routing as _jit_fused_muse_spark_routing, + from sglang.kernels.ops.moe.muse_spark_routing_jit import ( + fused_muse_spark_routing_jit, ) # See forward_triton: the SoL buffer is written by the router itself. @@ -782,7 +788,7 @@ class FusedMuseSparkRouting(BaseFusedOp): if want_packed else None ) - topk_weights, topk_ids, packed = _jit_fused_muse_spark_routing( + topk_weights, topk_ids, packed = fused_muse_spark_routing_jit( gating, self.expert_bias, topk, @@ -835,7 +841,7 @@ class FusedMuseSparkRouting(BaseFusedOp): backends differ only in how they produce the raw scores and indices. """ if envs.SGLANG_USE_CUSTOM_ROUTER_SUM.get(): - from sglang.kernels.ops.muse_spark.row_sum import row_sum + from sglang.kernels.ops.elementwise.row_sum import row_sum score_sum = row_sum(topk_scores, dim=-1, keepdim=True) else: diff --git a/python/sglang/kernels/ops/muse_spark/routing.py b/python/sglang/kernels/ops/moe/muse_spark_routing_jit.py similarity index 96% rename from python/sglang/kernels/ops/muse_spark/routing.py rename to python/sglang/kernels/ops/moe/muse_spark_routing_jit.py index bd33be60f5..a7d1bc0c9c 100644 --- a/python/sglang/kernels/ops/muse_spark/routing.py +++ b/python/sglang/kernels/ops/moe/muse_spark_routing_jit.py @@ -1,7 +1,7 @@ """Fused MuseSpark MoE routing (CUDA JIT). Drop-in replacement for the Triton ``fused_muse_spark_routing`` topk8 path in -``sglang.kernels.ops.muse_spark.fused_muse_spark_routing``: sigmoid/softmax normalization, +``sglang.kernels.ops.moe.muse_spark_routing``: sigmoid/softmax normalization, bias-scored iterative top-k, and renormalization by the selected unbiased score sum, fused into one kernel with one warp per token. @@ -57,7 +57,7 @@ def _experts_per_lane(num_experts: int) -> int: return -(-per_lane // 4) * 4 -def fused_muse_spark_routing( +def fused_muse_spark_routing_jit( gating_output: torch.Tensor, # [num_tokens, num_experts] fp32 bias: torch.Tensor, # [num_experts] fp32 topk: int, diff --git a/python/sglang/kernels/ops/moe/mxfp8_moe_amd_gfx95.py b/python/sglang/kernels/ops/moe/mxfp8_moe_amd_gfx95.py index 6ff509d3e9..e42a3c9823 100644 --- a/python/sglang/kernels/ops/moe/mxfp8_moe_amd_gfx95.py +++ b/python/sglang/kernels/ops/moe/mxfp8_moe_amd_gfx95.py @@ -262,7 +262,7 @@ def fused_moe_mxfp8_native( ) -> torch.Tensor: # Lazy import: the jit_kernel package pulls in Triton at first use; importing # at call time avoids any import-time cycle with the moe runner package. - from sglang.kernels.ops.moe.minimax_m3_swiglu import ( + from sglang.kernels.ops.activation.minimax_m3_swiglu import ( swiglu_oai_mxfp8_quant, swiglu_oai_split, ) diff --git a/python/sglang/kernels/ops/moe/router.py b/python/sglang/kernels/ops/moe/router.py index d6a0b2d338..bce31503b4 100644 --- a/python/sglang/kernels/ops/moe/router.py +++ b/python/sglang/kernels/ops/moe/router.py @@ -4,7 +4,6 @@ import torch import triton import triton.language as tl -from sglang.kernels.jit.utils import is_arch_support_pdl from sglang.srt.utils import is_hip _is_hip = is_hip() @@ -386,127 +385,3 @@ def fused_moe_router_shim( moe_softcapping=moe_softcapping, correction_bias=correction_bias, ) - - -@triton.jit -def router_gate_matvec_kernel( - x_ptr, # (M, K) bf16/fp16/fp32, row-major - w_ptr, # (E, K) fp32/bf16/fp16, k-major - out_ptr, # (M, E) fp32 - K, - E, - stride_xm, - stride_we, - BLOCK_E: tl.constexpr, - BLOCK_K: tl.constexpr, - USE_GDC: tl.constexpr = False, -): - """Router-gate logits as a single matvec launch, fp32 accumulation for - any float weight dtype. BLOCK_K covers the whole K in one masked load - (single iteration for K <= BLOCK_K): a cold gate weight then costs one - HBM round trip per CTA instead of a serial dependent-load chain, which - is what dominates in the real model where ~94MB/layer of expert traffic - flushes L2 between gate calls. - """ - pid_m = tl.program_id(0) - pid_e = tl.program_id(1) - e_offs = pid_e * BLOCK_E + tl.arange(0, BLOCK_E) - e_mask = e_offs < E - - # First K tile, weight load ahead of the PDL wait (see docstring). - k_offs = tl.arange(0, BLOCK_K) - k_mask = k_offs < K - w = tl.load( - w_ptr + e_offs[:, None] * stride_we + k_offs[None, :], - mask=e_mask[:, None] & k_mask[None, :], - other=0.0, - ).to(tl.float32) - if USE_GDC: - tl.extra.cuda.gdc_wait() - tl.extra.cuda.gdc_launch_dependents() - x = tl.load(x_ptr + pid_m * stride_xm + k_offs, mask=k_mask, other=0.0).to( - tl.float32 - ) - acc = tl.sum(w * x[None, :], axis=1) - - for k0 in range(BLOCK_K, K, BLOCK_K): - k_offs = k0 + tl.arange(0, BLOCK_K) - k_mask = k_offs < K - x = tl.load(x_ptr + pid_m * stride_xm + k_offs, mask=k_mask, other=0.0).to( - tl.float32 - ) - w = tl.load( - w_ptr + e_offs[:, None] * stride_we + k_offs[None, :], - mask=e_mask[:, None] & k_mask[None, :], - other=0.0, - ).to(tl.float32) - acc += tl.sum(w * x[None, :], axis=1) - - tl.store( - out_ptr + pid_m * E + e_offs, acc.to(out_ptr.dtype.element_ty), mask=e_mask - ) - - -# Cold-cache tuned on H20-3e (41 rotating gate weights so each call misses L2, -# like the real model); expected to carry to B200 (more SMs favor the wide -# grid even more) — re-tune with benchmark/kernels/bench_router_gate_matvec.py. -ROUTER_GATE_MATVEC_BLOCK_E = 4 -ROUTER_GATE_MATVEC_NUM_WARPS = 8 -# Beyond this M the per-M re-reads of the gate weight outgrow the library -# GEMM (cold H20-3e: bf16 wins to M=12, fp32 to M=8; cap at the lower). -ROUTER_GATE_MATVEC_MAX_M = 8 - - -def router_gate_matvec( - hidden_states: torch.Tensor, weight: torch.Tensor -) -> torch.Tensor: - """Small-M router-gate logits: one triton launch replacing the library - path — for fp32 gate weights the eager upcast + fp32 GEMM + splitKreduce - triple, for bf16 the F.linear GEMV. Returns fp32 (M, E) logits with fp32 - accumulation (deterministic; for fp32 weights 0 top-8 routing flips over - 30104 random draws vs the fp32 reference; for bf16 weights this is - slightly MORE precise than the library GEMV, so near-tie logits can - round-trip differently — same order as the bf16-vs-fp32 gate change). - - Cold-cache (41 rotating weights, in-graph, H20-3e, E=513, K=2560), us/call: - - M lib bf16 matvec bf16 lib fp32 chain matvec fp32 - 1 4.2 4.3 6.4 6.2 - 2 13.8 4.5 14.6 6.4 - 4 14.0 5.4 20.7 10.4 - 8 14.6 10.4 20.1 18.1 - 16 14.6 18.9 (lib) 20.9 30.9 (lib) - - Callers must gate on M <= ROUTER_GATE_MATVEC_MAX_M; prefill-sized M - keeps the library GEMM.""" - assert ( - weight.dtype - in ( - torch.float32, - torch.bfloat16, - torch.float16, - ) - and weight.is_contiguous() - ) - M, K = hidden_states.shape - E = weight.shape[0] - out = torch.empty((M, E), dtype=torch.float32, device=hidden_states.device) - block_e = ROUTER_GATE_MATVEC_BLOCK_E - # Single k-iteration whenever K fits one block: no serial dependent-load - # chain on a cold weight. - block_k = min(4096, triton.next_power_of_2(K)) - pdl_kwargs = {"USE_GDC": True, "launch_pdl": True} if is_arch_support_pdl() else {} - router_gate_matvec_kernel[(M, triton.cdiv(E, block_e))]( - hidden_states, - weight, - out, - K, - E, - hidden_states.stride(0), - weight.stride(0), - BLOCK_E=block_e, - BLOCK_K=block_k, - num_warps=ROUTER_GATE_MATVEC_NUM_WARPS, - **pdl_kwargs, - ) - return out diff --git a/python/sglang/kernels/ops/moe/smallm_moe_gfx950/__init__.py b/python/sglang/kernels/ops/moe/smallm_moe_gfx950/__init__.py index 97268acf9d..ec85fda951 100644 --- a/python/sglang/kernels/ops/moe/smallm_moe_gfx950/__init__.py +++ b/python/sglang/kernels/ops/moe/smallm_moe_gfx950/__init__.py @@ -23,8 +23,12 @@ import tempfile import torch -from sglang.srt.distributed.device_communicators.cuda_wrapper import ( - find_loaded_library, +from sglang.kernels.jit.utils.hipcc import ( + HipKernel, + check_hip, + hip_lib, + hip_version, + hipcc_path, ) from sglang.srt.environ import envs @@ -38,7 +42,6 @@ MAX_TOK = 64 # kernel list capacity (tokens per expert) and workspace size MAX_TOK_DISPATCH = {256: 40} MAX_TOK_DISPATCH_FP8 = {256: 20, 128: 48} # crossover vs aiter's per_128x128 asm/CK -_hip = None _kernels: dict = {} _available = None _disabled_reason = None @@ -81,83 +84,6 @@ assert ctypes.sizeof(Args) == 104, ctypes.sizeof( ) # must match static_assert(sizeof(Args) == 104) in smallm_moe.hip -def _hip_lib(): - global _hip - if _hip is None: - # Bind to the HIP runtime torch already mapped. ROCm 10 ships a second - # libamdhip64 in _rocm_sdk_devel; an unversioned CDLL("libamdhip64.so") - # picks that one, and every launch on a torch stream then fails with - # hipErrorContextIsDestroyed (709). - torch.cuda.current_device() - _hip = ctypes.CDLL(find_loaded_library("libamdhip64") or "libamdhip64.so") - _hip.hipModuleLoad.restype = ctypes.c_int - _hip.hipModuleLoad.argtypes = [ctypes.c_void_p, ctypes.c_char_p] - _hip.hipModuleGetFunction.restype = ctypes.c_int - _hip.hipModuleGetFunction.argtypes = [ - ctypes.c_void_p, - ctypes.c_void_p, - ctypes.c_char_p, - ] - _hip.hipModuleLaunchKernel.restype = ctypes.c_int - _hip.hipModuleLaunchKernel.argtypes = ( - [ctypes.c_void_p] + [ctypes.c_uint] * 7 + [ctypes.c_void_p] * 3 - ) - _hip.hipGetErrorString.restype = ctypes.c_char_p - return _hip - - -def _check(rc, what): - if rc != 0: - raise RuntimeError( - f"smallm_moe: {what} failed: {rc} ({_hip_lib().hipGetErrorString(rc).decode()})" - ) - - -class _Kernel: - def __init__(self, mod, name): - self.fn = ctypes.c_void_p() - _check( - _hip_lib().hipModuleGetFunction(ctypes.byref(self.fn), mod, name.encode()), - f"hipModuleGetFunction {name}", - ) - - def launch(self, grid, args, stream, block=256, shared=0): - size = ctypes.c_size_t(ctypes.sizeof(args)) - extra = (ctypes.c_void_p * 5)( - 1, - ctypes.cast(ctypes.byref(args), ctypes.c_void_p).value, - 2, - ctypes.cast(ctypes.byref(size), ctypes.c_void_p).value, - 3, - ) - _check( - _hip_lib().hipModuleLaunchKernel( - self.fn, - grid, - 1, - 1, - block, - 1, - 1, - shared, - ctypes.c_void_p(stream), - None, - ctypes.cast(extra, ctypes.c_void_p), - ), - "hipModuleLaunchKernel", - ) - - -def _hipcc(): - return os.path.join(os.environ.get("ROCM_PATH", "/opt/rocm"), "bin", "hipcc") - - -def _hip_version(): - if torch.version.hip: - return tuple(int(x) for x in torch.version.hip.split("-")[0].split(".")[:3]) - return (0, 0, 0) - - def smallm_moe_enabled() -> bool: """True unless SGLANG_ROCM_SMALLM_MOE=0, when the device is exactly gfx950 on ROCm >= 7.2 (the kernel uses v_cvt_scalef32_pk_bf16_fp4 / v_dot2_f32_bf16 and needs that toolchain), hipcc is present, and no build/load @@ -176,15 +102,15 @@ def smallm_moe_enabled() -> bool: arch = torch.cuda.get_device_properties(0).gcnArchName.split(":")[0] _available = ( arch == "gfx950" - and _hip_version() >= (7, 2, 0) - and os.path.exists(_hipcc()) + and hip_version() >= (7, 2, 0) + and os.path.exists(hipcc_path()) ) if not _available: logger.info( "smallm_moe not enabled (arch=%r, hip=%s, hipcc=%s); aiter fused_moe stays in use", arch, - _hip_version(), - os.path.exists(_hipcc()), + hip_version(), + os.path.exists(hipcc_path()), ) return _available and _disabled_reason is None @@ -209,7 +135,7 @@ def _get_kernels(inter: int, pre: str): try: subprocess.run( [ - _hipcc(), + hipcc_path(), "--genco", "--offload-arch=gfx950", "-O3", @@ -226,8 +152,8 @@ def _get_kernels(inter: int, pre: str): _disable(f"hipcc failed: {detail.strip()[-400:]}") mod = ctypes.c_void_p() try: - _check( - _hip_lib().hipModuleLoad(ctypes.byref(mod), co.encode()), + check_hip( + hip_lib().hipModuleLoad(ctypes.byref(mod), co.encode()), "hipModuleLoad", ) except RuntimeError as e: @@ -237,9 +163,9 @@ def _get_kernels(inter: int, pre: str): mod = _kernels["_mod"] try: kk = { - "p1": _Kernel(mod, f"smallm{pre}_p1_i{inter}_c16"), + "p1": HipKernel(mod, f"smallm{pre}_p1_i{inter}_c16"), "p2": { - sl: _Kernel(mod, f"smallm{pre}_p2_i{inter}_s{sl}") + sl: HipKernel(mod, f"smallm{pre}_p2_i{inter}_s{sl}") for sl in (10, 11) }, } diff --git a/python/sglang/kernels/ops/moe/smallm_router_gfx950/__init__.py b/python/sglang/kernels/ops/moe/smallm_router_gfx950/__init__.py index 1a360f28e5..2a533d7313 100644 --- a/python/sglang/kernels/ops/moe/smallm_router_gfx950/__init__.py +++ b/python/sglang/kernels/ops/moe/smallm_router_gfx950/__init__.py @@ -10,7 +10,7 @@ import tempfile import torch -from sglang.kernels.ops.moe.smallm_moe_gfx950 import _check, _hip_lib, _hipcc, _Kernel +from sglang.kernels.jit.utils.hipcc import HipKernel, check_hip, hip_lib, hipcc_path from sglang.srt.environ import envs from sglang.srt.utils import is_gfx95_supported @@ -62,12 +62,12 @@ def smallm_router(x, gate_weight, shared_gate_weight, scale=1.0): ) if not _kernels: co = os.path.join(tempfile.mkdtemp(prefix="sglang_smallm_router_"), "r.co") - cmd = [_hipcc(), "--genco", "--offload-arch=gfx950", "-O3", "-o", co, _SRC] + cmd = [hipcc_path(), "--genco", "--offload-arch=gfx950", "-O3", "-o", co, _SRC] try: subprocess.run(cmd, check=True, capture_output=True) mod = ctypes.c_void_p() - _check(_hip_lib().hipModuleLoad(ctypes.byref(mod), co.encode()), "load") - _kernels.update({r: _Kernel(mod, f"smallm_router_r{r}") for r in (1, 2)}) + check_hip(hip_lib().hipModuleLoad(ctypes.byref(mod), co.encode()), "load") + _kernels.update({r: HipKernel(mod, f"smallm_router_r{r}") for r in (1, 2)}) except (OSError, subprocess.CalledProcessError, RuntimeError) as e: logger.warning("smallm_router disabled: %s", e) _disabled = True diff --git a/python/sglang/kernels/ops/moe/triton_sigmoid_gate_mul.py b/python/sglang/kernels/ops/moe/triton_sigmoid_gate_mul.py deleted file mode 100644 index 861ba3ec6a..0000000000 --- a/python/sglang/kernels/ops/moe/triton_sigmoid_gate_mul.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Fused sigmoid-gate-multiply Triton kernels. - -Two variants: -- ``sigmoid_gate_mul``: element-wise ``x * sigmoid(gate)`` when x and gate - have identical shapes. -- ``sigmoid_gate_mul_broadcast``: broadcast ``x * sigmoid(gate)`` when gate - is ``(N, 1)`` and x is ``(N, D)``. -""" - -from __future__ import annotations - -import torch -import triton -import triton.language as tl - -from sglang.srt.utils import is_hip - -_is_hip = is_hip() - - -@triton.jit -def _sigmoid_gate_mul_kernel( - x_ptr, - gate_ptr, - out_ptr, - n_elements, - BLOCK_SIZE: tl.constexpr, -): - pid = tl.program_id(0) - offsets = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - mask = offsets < n_elements - x = tl.load(x_ptr + offsets, mask=mask).to(tl.float32) - g = tl.load(gate_ptr + offsets, mask=mask).to(tl.float32) - out = x * tl.sigmoid(g) - tl.store(out_ptr + offsets, out.to(x_ptr.dtype.element_ty), mask=mask) - - -def sigmoid_gate_mul(x: torch.Tensor, gate: torch.Tensor) -> torch.Tensor: - """Compute ``x * sigmoid(gate)`` in a single fused kernel (same-shape).""" - assert x.shape == gate.shape, f"shape mismatch: {x.shape=} {gate.shape=}" - assert x.is_contiguous() and gate.is_contiguous(), "inputs must be contiguous" - out = torch.empty_like(x) - n = x.numel() - grid = lambda meta: (triton.cdiv(n, meta["BLOCK_SIZE"]),) - _sigmoid_gate_mul_kernel[grid](x, gate, out, n, BLOCK_SIZE=1024) - return out - - -@triton.jit -def _sigmoid_gate_mul_broadcast_kernel( - out_ptr, - gate_ptr, - x_ptr, - hidden_dim: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - row = tl.program_id(0) - g = tl.load(gate_ptr + row).to(tl.float32) - g = tl.sigmoid(g) - - offs = tl.arange(0, BLOCK_SIZE) - mask = offs < hidden_dim - x = tl.load(x_ptr + row * hidden_dim + offs, mask=mask).to(tl.float32) - out = x * g - tl.store( - out_ptr + row * hidden_dim + offs, - out.to(x_ptr.dtype.element_ty), - mask=mask, - ) - - -def sigmoid_gate_mul_broadcast(x: torch.Tensor, gate: torch.Tensor) -> torch.Tensor: - """Compute ``x * sigmoid(gate)`` where gate is (N, 1) and x is (N, D).""" - bs, hidden_dim = x.shape - out = torch.empty_like(x) - BLOCK_SIZE = triton.next_power_of_2(hidden_dim) - max_warps = 16 if _is_hip else 32 - num_warps = max( - min(triton.next_power_of_2(triton.cdiv(hidden_dim, 8 * 32)), max_warps), 4 - ) - _sigmoid_gate_mul_broadcast_kernel[(bs,)]( - out, - gate, - x, - hidden_dim=hidden_dim, - BLOCK_SIZE=BLOCK_SIZE, - num_warps=num_warps, - ) - return out diff --git a/python/sglang/kernels/ops/muse_spark/__init__.py b/python/sglang/kernels/ops/muse_spark/__init__.py deleted file mode 100644 index d1090fc76e..0000000000 --- a/python/sglang/kernels/ops/muse_spark/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Model-specific fused operations used by SGLang runtimes.""" diff --git a/python/sglang/kernels/ops/quantization/__init__.py b/python/sglang/kernels/ops/quantization/__init__.py index 67a88fa4b0..67dd4d56b0 100644 --- a/python/sglang/kernels/ops/quantization/__init__.py +++ b/python/sglang/kernels/ops/quantization/__init__.py @@ -8,7 +8,6 @@ from sglang.kernels.registry import register_kernel from sglang.kernels.selector import get_kernel from sglang.kernels.spec import ( CUDA_ONLY, - CapabilityRequirement, KernelBackend, KernelSpec, ) @@ -109,9 +108,9 @@ _TRITON_KERNELS = [ ("int8_kernel", "per_token_quant_int8"), ("int8_kernel", "per_token_group_quant_int8"), ("awq_triton", "awq_dequantize_triton"), + ("rocm_upscale", "upscale"), + ("rocm_upscale", "upscale_mxfp4"), ("mxfp8_amd_gfx95", "mxfp8_e4m3_quantize"), - ("mxfp8_swizzled_triton", "swiglu_oai_mxfp8"), - ("mxfp8_swizzled_triton", "swiglu_oai_mxfp8_deepgemm"), ] for _mod, _fn in _TRITON_KERNELS: register_kernel( @@ -125,23 +124,21 @@ del _mod, _fn register_kernel( KernelSpec( - op="quantization.nvfp4_gemm_swiglu_nvfp4_quant", - backend=KernelBackend.CUTE_DSL, + op="quantization.quantize_fp4_indexer_tensor", + backend=KernelBackend.TRITON, target=( - "sglang.kernels.ops.quantization.nvfp4_gemm_swiglu_nvfp4_quant" - ":nvfp4_gemm_swiglu_nvfp4_quant" + "sglang.kernels.ops.quantization.fp4_indexer_quant" + ":quantize_fp4_indexer_tensor" ), - capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(10, 0))}), ) ) - -# Public entry points inventoried by logical operator group (RFC #29630). +# act_quant also has a TileLang implementation (act_quant_tilelang, used on HIP); +# KernelBackend has no TileLang provenance, so only the Triton one is inventoried. register_kernel( KernelSpec( - op="quantization.cvt_fp8_e4m3", - backend=KernelBackend.JIT, - target="sglang.kernels.ops.quantization.fp8_cvt:cvt_fp8_e4m3", - capabilities=CUDA_ONLY, + op="quantization.act_quant", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.quantization.act_quant_triton:act_quant", ) ) diff --git a/python/sglang/kernels/ops/quantization/act_quant_tilelang.py b/python/sglang/kernels/ops/quantization/act_quant_tilelang.py new file mode 100644 index 0000000000..62fbb9a743 --- /dev/null +++ b/python/sglang/kernels/ops/quantization/act_quant_tilelang.py @@ -0,0 +1,122 @@ +from typing import Optional, Tuple + +import tilelang +import tilelang.language as T +import torch + +from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz + +tilelang.set_log_level("WARNING") + +pass_configs = { + tilelang.PassConfigKey.TL_DISABLE_WARP_SPECIALIZED: True, + tilelang.PassConfigKey.TL_DISABLE_TMA_LOWER: True, +} +# TL_DISABLE_FAST_MATH has deprecated in v0.1.7.post1 tilelang +if hasattr(tilelang.PassConfigKey, "TL_DISABLE_FAST_MATH"): + pass_configs[tilelang.PassConfigKey.TL_DISABLE_FAST_MATH] = True +elif hasattr(tilelang.PassConfigKey, "TL_ENABLE_FAST_MATH"): + pass_configs[tilelang.PassConfigKey.TL_ENABLE_FAST_MATH] = False + +_is_fp8_fnuz = is_fp8_fnuz() + +BF16 = "bfloat16" +FP8 = "float8_e4m3fnuz" if _is_fp8_fnuz else "float8_e4m3fn" +FP32 = "float32" + + +def fast_log2_ceil(x): + bits_x = T.reinterpret("uint32", x) + exp_x = (bits_x >> 23) & 0xFF + man_bits = bits_x & ((1 << 23) - 1) + return T.Cast("int32", exp_x - 127 + T.if_then_else(man_bits != 0, 1, 0)) + + +def fast_pow2(x): + bits_x = (x + 127) << 23 + return T.reinterpret("float32", bits_x) + + +def fast_round_scale(amax, fp8_max_inv): + return fast_pow2(fast_log2_ceil(amax * fp8_max_inv)) + + +@tilelang.jit(pass_configs=pass_configs) +def act_quant_kernel( + N, in_dtype=BF16, out_dtype=FP8, scale_dtype=FP32, round_scale=False +): + M = T.symbolic("M") + fp8_min = -224.0 if _is_fp8_fnuz else -448.0 + fp8_max = 224.0 if _is_fp8_fnuz else 448.0 + fp8_max_inv = 1 / fp8_max + num_stages = 0 if round_scale else 2 + blk_m = 32 + group_size = 128 + + @T.prim_func + def act_quant_kernel_( + X: T.Tensor[(M, N), in_dtype], + Y: T.Tensor[(M, N), out_dtype], + S: T.Tensor[(M, T.ceildiv(N, group_size)), scale_dtype], + ): + with T.Kernel(T.ceildiv(M, blk_m), T.ceildiv(N, group_size), threads=128) as ( + pid_m, + pid_n, + ): + x_shared = T.alloc_shared((blk_m, group_size), in_dtype) + x_local = T.alloc_fragment((blk_m, group_size), in_dtype) + amax_local = T.alloc_fragment((blk_m,), scale_dtype) + s_local = T.alloc_fragment((blk_m,), scale_dtype) + y_local = T.alloc_fragment((blk_m, group_size), out_dtype) + y_shared = T.alloc_shared((blk_m, group_size), out_dtype) + + for _ in T.Pipelined(1, num_stages=num_stages): + T.copy(X[pid_m * blk_m, pid_n * group_size], x_shared) + T.copy(x_shared, x_local) + T.reduce_absmax(x_local, amax_local, dim=1) + for i in T.Parallel(blk_m): + amax_local[i] = T.max(amax_local[i], 1e-4) + if round_scale: + s_local[i] = fast_round_scale(amax_local[i], fp8_max_inv) + else: + s_local[i] = amax_local[i] * fp8_max_inv + for i, j in T.Parallel(blk_m, group_size): + y_local[i, j] = T.clamp( + x_local[i, j] / s_local[i], fp8_min, fp8_max + ) + for i in T.Parallel(blk_m): + S[pid_m * blk_m + i, pid_n] = s_local[i] + T.copy(y_local, y_shared) + T.copy(y_shared, Y[pid_m * blk_m, pid_n * group_size]) + + return act_quant_kernel_ + + +def act_quant( + x: torch.Tensor, block_size: int = 128, scale_fmt: Optional[str] = None +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Quantizes the input tensor `x` using block-wise quantization. + + Args: + x (torch.Tensor): The input tensor to be quantized. Must be contiguous and its last dimension size must be divisible by `block_size`. + block_size (int, optional): The size of the blocks to be used for quantization. Default is 128. + scale_fmt (Optional[str], optional): The format of the scale. Default is None. + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - The quantized tensor with dtype `torch.float8_e4m3fn`. + - A tensor of scaling factors with dtype `torch.float32`. + """ + assert x.is_contiguous(), "Input tensor must be contiguous" + assert x.size(-1) % block_size == 0, ( + f"Last dimension size must be divisible by block_size (block_size={block_size})" + ) + N = x.size(-1) + if _is_fp8_fnuz: + y = torch.empty_like(x, dtype=torch.float8_e4m3fnuz) + else: + y = torch.empty_like(x, dtype=torch.float8_e4m3fn) + s = x.new_empty(*x.size()[:-1], N // block_size, dtype=torch.float32) + kernel = act_quant_kernel(N, round_scale=scale_fmt is not None) + kernel(x.view(-1, N), y.view(-1, N), s.view(-1, N // block_size)) + return y, s diff --git a/python/sglang/kernels/ops/quantization/act_quant_triton.py b/python/sglang/kernels/ops/quantization/act_quant_triton.py new file mode 100644 index 0000000000..0736fc2ec1 --- /dev/null +++ b/python/sglang/kernels/ops/quantization/act_quant_triton.py @@ -0,0 +1,138 @@ +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + + +# Triton implementation +@triton.jit +def _act_quant_kernel( + X_ptr, + Y_ptr, + S_ptr, + M, + N, + group_size: tl.constexpr, + round_scale: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, +): + """ + Triton kernel for activation quantization. + + Each block processes BLOCK_M rows and group_size columns. + """ + # Get block IDs + pid_m = tl.program_id(0) + pid_n = tl.program_id(1) + + # FP8 constants + fp8_min = -448.0 + fp8_max = 448.0 + fp8_max_inv = 1.0 / fp8_max + + # Calculate row and column offsets + row_start = pid_m * BLOCK_M + col_start = pid_n * group_size + + # Create offset arrays + rows = row_start + tl.arange(0, BLOCK_M) + cols = col_start + tl.arange(0, BLOCK_N) + + # Mask for valid rows and columns + row_mask = rows < M + col_mask = cols < N + mask = row_mask[:, None] & col_mask[None, :] + + # Load input data + x_ptrs = X_ptr + rows[:, None] * N + cols[None, :] + x = tl.load(x_ptrs, mask=mask, other=0.0).to(tl.float32) + + # Compute absolute max along columns (group_size dimension) for each row + x_abs = tl.abs(x) + amax = tl.max(x_abs, axis=1) # Shape: (BLOCK_M,) + + # Clamp amax to avoid division by zero + amax = tl.maximum(amax, 1e-4) + + # Compute scale + if round_scale: + # Fast round scale using bit manipulation approximation + # This is a simplified version - the exact bit manipulation is harder in Triton + # Using log2 + ceil + pow2 as approximation + log_val = tl.log2(amax * fp8_max_inv) + log_ceil = tl.ceil(log_val) + scale = tl.exp2(log_ceil) + else: + scale = amax * fp8_max_inv + + # Quantize: y = clamp(x / scale, fp8_min, fp8_max) + scale_broadcast = scale[:, None] + y = x / scale_broadcast + y = tl.minimum(tl.maximum(y, fp8_min), fp8_max) + + # Store quantized output + y_ptrs = Y_ptr + rows[:, None] * N + cols[None, :] + tl.store(y_ptrs, y, mask=mask) + + # Store scales + s_cols = pid_n + s_ptrs = S_ptr + rows * (N // group_size) + s_cols + s_mask = row_mask + tl.store(s_ptrs, scale, mask=s_mask) + + +# Not bit-exact with per_token_group_quant (1e-4 vs 1e-10 amax floor, x / scale vs +# x * (448 / amax)), so it is not merged into that op. +def act_quant( + x: torch.Tensor, block_size: int = 128, scale_fmt: Optional[str] = None +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Quantizes the input tensor `x` using block-wise quantization with Triton. + + Args: + x (torch.Tensor): The input tensor to be quantized. Must be contiguous and its last dimension size must be divisible by `block_size`. + block_size (int, optional): The size of the blocks to be used for quantization. Default is 128. + scale_fmt (Optional[str], optional): The format of the scale. Default is None. + Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - The quantized tensor with dtype `torch.float8_e4m3fn`. + - A tensor of scaling factors with dtype `torch.float32`. + """ + assert x.is_contiguous(), "Input tensor must be contiguous" + assert x.size(-1) % block_size == 0, ( + f"Last dimension size must be divisible by block_size (block_size={block_size})" + ) + + # Flatten all dims except last + N = x.size(-1) + x_flat = x.view(-1, N) + M = x_flat.size(0) + + # Allocate output tensors + y = torch.empty_like(x, dtype=torch.float8_e4m3fn) + y_flat = y.view(-1, N) + s = x.new_empty(*x.size()[:-1], N // block_size, dtype=torch.float32) + s_flat = s.view(-1, N // block_size) + + # Launch kernel + BLOCK_M = 32 + BLOCK_N = block_size + grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, block_size)) + round_scale = scale_fmt is not None + + _act_quant_kernel[grid]( + x_flat, + y_flat, + s_flat, + M, + N, + group_size=block_size, + round_scale=round_scale, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + num_stages=0 if round_scale else 2, + ) + + return y, s diff --git a/python/sglang/kernels/ops/attention/dsa/quant_k_cache.py b/python/sglang/kernels/ops/quantization/dsa_quant_k_cache.py similarity index 83% rename from python/sglang/kernels/ops/attention/dsa/quant_k_cache.py rename to python/sglang/kernels/ops/quantization/dsa_quant_k_cache.py index 67c9a93bda..1e7bf23b45 100644 --- a/python/sglang/kernels/ops/attention/dsa/quant_k_cache.py +++ b/python/sglang/kernels/ops/quantization/dsa_quant_k_cache.py @@ -65,60 +65,6 @@ def quantize_k_cache_separate( ) -# Copied from original -def _quantize_k_cache_ref( - input_k_cache: torch.Tensor, # (num_blocks, block_size, h_k, d) - dv: int = 512, - tile_size: int = 128, -) -> torch.Tensor: - """ - Quantize the k-cache - Return a tensor with shape (num_blocks, block_size, h_k, dv + 4(dv/tile_size) + t(d-dv)) of dtype uint8_t, where t = input_k_cache.element_size() - For more detail about the layout of K/V, please refer to comments in flash_mla_interface.py or README.md - """ - assert dv % tile_size == 0 - num_tiles = dv // tile_size - num_blocks, block_size, h_k, d = input_k_cache.shape - assert h_k == 1 - input_k_cache = input_k_cache.squeeze(2) # [num_blocks, block_size, d] - input_elem_size = input_k_cache.element_size() - - result = torch.empty( - (num_blocks, block_size, dv + num_tiles * 4 + input_elem_size * (d - dv)), - dtype=torch.float8_e4m3fn, - device=input_k_cache.device, - ) - result_k_nope_part = result[..., :dv] - result_k_scale_factor = result[..., dv : dv + num_tiles * 4].view(torch.float32) - result_k_rope_part = result[..., dv + num_tiles * 4 :].view(input_k_cache.dtype) - result_k_rope_part[:] = input_k_cache[..., dv:] - - for tile_idx in range(0, num_tiles): - cur_scale_factors_inv = ( - torch.abs( - input_k_cache[..., tile_idx * tile_size : (tile_idx + 1) * tile_size] - ) - .max(dim=-1) - .values - / 448.0 - ) # [num_blocks, block_size] - result_k_scale_factor[:, :, tile_idx] = cur_scale_factors_inv - - cur_scale_factors_inv.unsqueeze_(-1) # [num_blocks, block_size, 1] - cur_quantized_nope = ( - input_k_cache[ - ..., tile_idx * tile_size : (tile_idx + 1) * tile_size - ].float() - / cur_scale_factors_inv.float() - ).to(torch.float8_e4m3fn) - result_k_nope_part[..., tile_idx * tile_size : (tile_idx + 1) * tile_size] = ( - cur_quantized_nope - ) - - result = result.view(num_blocks, block_size, 1, -1) - return result - - def _quantize_k_cache_fast_wrapped( input_k_cache: torch.Tensor, dv: int | None = None, diff --git a/python/sglang/kernels/ops/attention/dsv4/torch_quant.py b/python/sglang/kernels/ops/quantization/fp4_fake_quant.py similarity index 100% rename from python/sglang/kernels/ops/attention/dsv4/torch_quant.py rename to python/sglang/kernels/ops/quantization/fp4_fake_quant.py diff --git a/python/sglang/kernels/ops/quantization/fp4_indexer_quant.py b/python/sglang/kernels/ops/quantization/fp4_indexer_quant.py new file mode 100644 index 0000000000..a43680ca67 --- /dev/null +++ b/python/sglang/kernels/ops/quantization/fp4_indexer_quant.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from sglang.srt.runtime_context import get_platform + + +@triton.jit +def _select_group_value(group, v0, v1, v2, v3): + return tl.where( + group == 0, + v0, + tl.where(group == 1, v1, tl.where(group == 2, v2, v3)), + ) + + +@triton.jit +def _ceil_ue8m0_exp(x): + bits = x.to(tl.int32, bitcast=True) + exp = (bits >> 23) & 0xFF + mantissa = bits & 0x7FFFFF + exp += mantissa != 0 + return tl.minimum(tl.maximum(exp, 1), 254) + + +@triton.jit +def _fp4_e2m1_code(x): + ax = tl.minimum(tl.abs(x), 6.0) + idx = (ax > 0.25).to(tl.uint8) + idx += (ax > 0.75).to(tl.uint8) + idx += (ax > 1.25).to(tl.uint8) + idx += (ax > 1.75).to(tl.uint8) + idx += (ax > 2.5).to(tl.uint8) + idx += (ax > 3.5).to(tl.uint8) + idx += (ax > 5.0).to(tl.uint8) + sign = ((x < 0) & (idx != 0)).to(tl.uint8) + return idx | (sign << 3) + + +@triton.jit +def _fp4_e2m1_code_rne(x): + """Round-to-nearest-even e2m1 code, matching the reference rounding.""" + ax = tl.minimum(tl.abs(x), 6.0) + idx = (ax >= 0.25).to(tl.uint8) + idx += (ax >= 0.75).to(tl.uint8) + idx += (ax >= 1.25).to(tl.uint8) + idx += (ax >= 1.75).to(tl.uint8) + idx += (ax >= 2.5).to(tl.uint8) + idx += (ax >= 3.5).to(tl.uint8) + idx += (ax >= 5.0).to(tl.uint8) + # Round-half-to-even: an odd index at an exact boundary drops to the even one. + is_boundary = ( + (ax == 0.25) + | (ax == 0.75) + | (ax == 1.25) + | (ax == 1.75) + | (ax == 2.5) + | (ax == 3.5) + | (ax == 5.0) + ) + idx = tl.where(is_boundary & ((idx & 1) == 1), idx - 1, idx) + sign = ((x < 0) & (idx != 0)).to(tl.uint8) + return idx | (sign << 3) + + +@triton.jit +def quantize_fp4_indexer_row( + values, + v0, + v1, + BLOCK_N: tl.constexpr, + GROUP_N: tl.constexpr, + RNE: tl.constexpr, +): + """One row of quantize_fp4_indexer_tensor: values is the fp32 [BLOCK_N] row, + v0 / v1 its even / odd elements as fp32 [BLOCK_N // 2]. Returns + (packed codes [BLOCK_N // 2], packed ue8m0 exponents), the two stores of the + standalone kernel below.""" + offs = tl.arange(0, BLOCK_N) + abs_values = tl.abs(values) + + amax0 = tl.max(tl.where(offs < GROUP_N, abs_values, 0.0), axis=0) + amax1 = tl.max( + tl.where((GROUP_N <= offs) & (offs < 2 * GROUP_N), abs_values, 0.0), + axis=0, + ) + amax2 = tl.max( + tl.where((2 * GROUP_N <= offs) & (offs < 3 * GROUP_N), abs_values, 0.0), + axis=0, + ) + amax3 = tl.max(tl.where(3 * GROUP_N <= offs, abs_values, 0.0), axis=0) + + sf0 = tl.maximum(amax0 / 6.0, 1.0e-4) + sf1 = tl.maximum(amax1 / 6.0, 1.0e-4) + sf2 = tl.maximum(amax2 / 6.0, 1.0e-4) + sf3 = tl.maximum(amax3 / 6.0, 1.0e-4) + + exp0 = _ceil_ue8m0_exp(sf0) + exp1 = _ceil_ue8m0_exp(sf1) + exp2 = _ceil_ue8m0_exp(sf2) + exp3 = _ceil_ue8m0_exp(sf3) + + packed_sf = exp0 | (exp1 << 8) | (exp2 << 16) | (exp3 << 24) + + pair_offsets = tl.arange(0, BLOCK_N // 2) + offs0 = pair_offsets * 2 + offs1 = offs0 + 1 + group0 = offs0 // GROUP_N + group1 = offs1 // GROUP_N + scale_exp0 = _select_group_value(group0, exp0, exp1, exp2, exp3) + scale_exp1 = _select_group_value(group1, exp0, exp1, exp2, exp3) + scale0 = (scale_exp0 << 23).to(tl.float32, bitcast=True) + scale1 = (scale_exp1 << 23).to(tl.float32, bitcast=True) + + v0 = v0 / scale0 + v1 = v1 / scale1 + if RNE: + code0 = _fp4_e2m1_code_rne(v0) + code1 = _fp4_e2m1_code_rne(v1) + else: + code0 = _fp4_e2m1_code(v0) + code1 = _fp4_e2m1_code(v1) + packed = (code0 & 0x0F) | ((code1 & 0x0F) << 4) + return packed, packed_sf + + +@triton.jit +def _quantize_fp4_indexer_kernel( + x, + x_fp4, + x_sf, + BLOCK_N: tl.constexpr, + GROUP_N: tl.constexpr, + RNE: tl.constexpr, +): + token_id = tl.program_id(0) + offs = tl.arange(0, BLOCK_N) + pair_offsets = tl.arange(0, BLOCK_N // 2) + row = x + token_id * BLOCK_N + values = tl.load(row + offs).to(tl.float32) + v0 = tl.load(row + pair_offsets * 2).to(tl.float32) + v1 = tl.load(row + pair_offsets * 2 + 1).to(tl.float32) + packed, packed_sf = quantize_fp4_indexer_row( + values, v0, v1, BLOCK_N=BLOCK_N, GROUP_N=GROUP_N, RNE=RNE + ) + tl.store(x_sf + token_id, packed_sf) + tl.store(x_fp4 + token_id * (BLOCK_N // 2) + pair_offsets, packed) + + +@triton.jit +def _quantize_fp4_indexer_rows( + x, + x_fp4, + x_sf, + M, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + GROUP_N: tl.constexpr, + RNE: tl.constexpr, +): + tl.static_assert(BLOCK_N == 128 and GROUP_N == 32) + # Each reduction covers one scale group. Keep its values for packing, + # avoiding four masked full-row reductions and a second input load. + group = tl.program_id(0) * BLOCK_M * 4 + tl.arange(0, BLOCK_M * 4) + offs = tl.arange(0, GROUP_N) + values = tl.load( + x + group[:, None].to(tl.int64) * GROUP_N + offs[None, :], + group[:, None] < M * 4, + 0, + ).to(tl.float32) + amax = tl.max(tl.abs(values), axis=1) + exp = _ceil_ue8m0_exp(tl.maximum(amax / 6.0, 1.0e-4)) + scale = (exp << 23).to(tl.float32, bitcast=True) + v0, v1 = tl.split( + tl.reshape(values / scale[:, None], (BLOCK_M * 4, GROUP_N // 2, 2)) + ) + if RNE: + code0 = _fp4_e2m1_code_rne(v0) + code1 = _fp4_e2m1_code_rne(v1) + else: + code0 = _fp4_e2m1_code(v0) + code1 = _fp4_e2m1_code(v1) + packed = (code0 & 0x0F) | ((code1 & 0x0F) << 4) + tl.store( + x_fp4 + + group[:, None].to(tl.int64) * (GROUP_N // 2) + + tl.arange(0, GROUP_N // 2)[None, :], + packed, + group[:, None] < M * 4, + ) + # The four exponents occupy disjoint bytes, so integer sum packs them. + shifts = tl.arange(0, 4) * 8 + packed_sf = tl.sum( + tl.reshape(exp.to(tl.uint32), (BLOCK_M, 4)) << shifts[None, :], axis=1 + ) + token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M) + tl.store(x_sf + token_id, packed_sf.to(tl.int32), token_id < M) + + +def quantize_fp4_indexer_tensor( + x: torch.Tensor, rne: bool = False +) -> tuple[torch.Tensor, torch.Tensor]: + """Per-32 ue8m0 fp4 quantize. rne=True uses round-to-nearest-even (the dsv41 + reference rounding); the default keeps ``_fp4_e2m1_code``'s thresholds.""" + assert x.shape[-1] == 128 + x = x.contiguous().view(-1, x.shape[-1]) + x_fp4 = torch.empty((x.shape[0], 64), device=x.device, dtype=torch.int8) + x_sf = torch.empty((x.shape[0],), device=x.device, dtype=torch.int32) + if x.shape[0] >= 4096 and get_platform().is_blackwell: + # Independent rows share a CTA to avoid one block per 128 values. + _quantize_fp4_indexer_rows[(triton.cdiv(x.shape[0], 8),)]( + x, + x_fp4, + x_sf, + x.shape[0], + 8, + BLOCK_N=128, + GROUP_N=32, + RNE=rne, + num_warps=4, + ) + elif x.shape[0] > 0: + _quantize_fp4_indexer_kernel[(x.shape[0],)]( + x, + x_fp4, + x_sf, + BLOCK_N=128, + GROUP_N=32, + RNE=rne, + ) + return x_fp4, x_sf diff --git a/python/sglang/kernels/ops/attention/dsv4/fp8_wo_a.py b/python/sglang/kernels/ops/quantization/fp8_group_major_quant_ue8m0.py similarity index 98% rename from python/sglang/kernels/ops/attention/dsv4/fp8_wo_a.py rename to python/sglang/kernels/ops/quantization/fp8_group_major_quant_ue8m0.py index a170c58abe..f524635f0d 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp8_wo_a.py +++ b/python/sglang/kernels/ops/quantization/fp8_group_major_quant_ue8m0.py @@ -13,11 +13,14 @@ from sglang.kernels.jit.utils import ( from sglang.kernels.kernel_api_logging import debug_kernel_api from sglang.srt.utils.custom_op import register_custom_op -from .utils import make_name - if TYPE_CHECKING: from tvm_ffi.module import Module + +def make_name(name: str) -> str: + return f"dpsk_v4_{name}" + + _GROUP_SIZE = 128 _UE8M0_SCALES_PER_PACK = 4 diff --git a/python/sglang/kernels/ops/quantization/mxfp8_amd_gfx95.py b/python/sglang/kernels/ops/quantization/mxfp8_amd_gfx95.py index 6bbb5267da..95e64fdf99 100644 --- a/python/sglang/kernels/ops/quantization/mxfp8_amd_gfx95.py +++ b/python/sglang/kernels/ops/quantization/mxfp8_amd_gfx95.py @@ -1,17 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 -"""Native MXFP8 (1x32 block, E8M0 scale) ops for AMD CDNA4 (gfx950). +"""Native MXFP8 (1x32 block, E8M0 scale) quantization for AMD CDNA4 (gfx950). - * per-token MXFP8 activation quant (single fused Triton pass) - * dense GEMM via Triton ``tl.dot_scaled`` (consumes FP8 E4M3 weights + E8M0 - block scales directly, no dequant-to-BF16), lowering to the CDNA4 native MX - matrix-core ops; ``K % 128 != 0`` falls back to dequant + ``F.linear``. - * the fp8 grid of the gfx950 native route (``fp8_grid_quant`` and the activation - wrappers the fused producers emit), and the bf16-dequant linear for shapes it does - not tile. - -Replaces the FlyDSL ``v_mfma_scale_f32_32x32x64`` dense path with a single -Triton ``dot_scaled`` GEMM: no load-time weight reformat (fp8 + E8M0 are -consumed as-is) and the activation is MXFP8-quantized in one fused pass. +* per-token MXFP8 activation quant (single fused Triton pass) +* the fp8 grid of the gfx950 native route (``fp8_grid_quant`` and the activation + wrappers the fused producers emit). """ from __future__ import annotations @@ -20,11 +12,9 @@ from typing import Optional, Sequence, Tuple import msgspec import torch -import torch.nn.functional as F import triton import triton.language as tl -from sglang.kernels.jit.utils import cache_once from sglang.kernels.ops.quantization.mxfp8_quant import _mxfp8_quant_kernel # MXFP8 constants (OCP microscaling: 1x32 block, E8M0 shared scale). @@ -134,185 +124,6 @@ def dequant_mxfp8_to_bf16(x: torch.Tensor, scales: torch.Tensor) -> torch.Tensor return dequantized.to(torch.bfloat16) -# --------------------------------------------------------------------------- # -# Dense MXFP8 linear via Triton tl.dot_scaled (CDNA4 native microscaling) -# --------------------------------------------------------------------------- # -@triton.jit -def _mxfp8_linear_kernel( - x_ptr, - xs_ptr, - w_ptr, - ws_ptr, - out_ptr, - M, - N, - K, - stride_xm, - stride_xk, - stride_xsm, - stride_xsk, - stride_wn, - stride_wk, - stride_wsn, - stride_wsk, - stride_om, - stride_on, - BLOCK_M: tl.constexpr, - BLOCK_N: tl.constexpr, - BLOCK_K: tl.constexpr, -): - pid_m = tl.program_id(0) - pid_n = tl.program_id(1) - offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) - offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) - offs_k = tl.arange(0, BLOCK_K) - offs_sk = tl.arange(0, BLOCK_K // 32) - m_mask = offs_m < M - n_mask = offs_n < N - - x_ptrs = x_ptr + offs_m[:, None] * stride_xm + offs_k[None, :] * stride_xk - xs_ptrs = xs_ptr + offs_m[:, None] * stride_xsm + offs_sk[None, :] * stride_xsk - w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk - ws_ptrs = ws_ptr + offs_n[:, None] * stride_wsn + offs_sk[None, :] * stride_wsk - - acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) - for _ in range(0, tl.cdiv(K, BLOCK_K)): - x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0) - w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0) - xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=0) - ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=0) - acc += tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3") - x_ptrs += BLOCK_K * stride_xk - w_ptrs += BLOCK_K * stride_wk - xs_ptrs += (BLOCK_K // 32) * stride_xsk - ws_ptrs += (BLOCK_K // 32) * stride_wsk - - o_ptrs = out_ptr + offs_m[:, None] * stride_om + offs_n[None, :] * stride_on - tl.store( - o_ptrs, - acc.to(out_ptr.dtype.element_ty), - mask=m_mask[:, None] & n_mask[None, :], - ) - - -def _run_mxfp8_linear_kernel( - x_q: torch.Tensor, # [M, K] fp8 e4m3 - x_scale: torch.Tensor, # [M, K//32] uint8 (E8M0) - w: torch.Tensor, # [N, K] fp8 e4m3 - w_scale: torch.Tensor, # [N, K//32] uint8 (E8M0) - out_dtype: torch.dtype, -) -> torch.Tensor: - M, K = x_q.shape - N = w.shape[0] - out = torch.empty((M, N), dtype=out_dtype, device=x_q.device) - BLOCK_M, BLOCK_K = 64, 128 - if M <= 512 and (K >= 4096 or (N == 6144 and K in (2048, 3072))): - BLOCK_N, num_warps = 64, 4 - else: - BLOCK_N, num_warps = 128, 8 - grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(N, BLOCK_N)) - _mxfp8_linear_kernel[grid]( - x_q, - x_scale, - w, - w_scale, - out, - M, - N, - K, - x_q.stride(0), - x_q.stride(1), - x_scale.stride(0), - x_scale.stride(1), - w.stride(0), - w.stride(1), - w_scale.stride(0), - w_scale.stride(1), - out.stride(0), - out.stride(1), - BLOCK_M=BLOCK_M, - BLOCK_N=BLOCK_N, - BLOCK_K=BLOCK_K, - num_warps=num_warps, - ) - return out - - -def _mxfp8_dot_scaled_linear( - x: torch.Tensor, # [M, K] bf16/fp16 - w: torch.Tensor, # [N, K] fp8 e4m3 - w_scale: torch.Tensor, # [N, K//32] uint8 (E8M0) -) -> torch.Tensor: - """bf16/fp16 input -> per-token MXFP8 quant -> dot_scaled GEMM.""" - x_q, x_scale = mxfp8_e4m3_quantize(x) - return _run_mxfp8_linear_kernel(x_q, x_scale, w, w_scale, x.dtype) - - -def dot_scaled_mxfp8_blockscaled_linear( - input: torch.Tensor, - weight: torch.Tensor, - weight_scale: torch.Tensor, - input_scale: Optional[torch.Tensor] = None, - bias: Optional[torch.Tensor] = None, - output_dtype: Optional[torch.dtype] = None, -) -> torch.Tensor: - """Native dense MXFP8 linear (CDNA4 ``tl.dot_scaled``). - - Consumes FP8 E4M3 ``weight`` + canonical 2D UE8M0 ``weight_scale`` [N, K//32] - directly. Activations are MXFP8-quantized per token inside the kernel path. - Drop-in for the SGLang ``w8a8_mxfp8_linear`` callable signature. - """ - assert weight.dtype == torch.float8_e4m3fn, "MXFP8 weight must be FP8 E4M3." - assert weight_scale.dtype == torch.uint8, "MXFP8 weight_scale must be UE8M0 uint8." - assert weight_scale.dim() == 2, ( - "dot_scaled MXFP8 linear expects canonical 2D [N, K//32] weight scales, " - f"got {weight_scale.dim()}D." - ) - - input_2d = input.view(-1, input.shape[-1]).contiguous() - output_shape = [*input.shape[:-1], weight.shape[0]] - if output_dtype is None: - output_dtype = ( - input_2d.dtype - if input_2d.dtype in (torch.float16, torch.bfloat16, torch.float32) - else torch.bfloat16 - ) - - m, k = input_2d.shape - n, k_w = weight.shape - assert k == k_w, f"{k=} does not match {k_w=}" - - if k % 128 == 0: - if input_scale is None: - # Quantize the bf16/fp16 activations per token inside the path. - x_q, x_scale = mxfp8_e4m3_quantize(input_2d) - kernel_out_dtype = input_2d.dtype - else: - # Activations already MXFP8-quantized by a fused upstream op. - assert input_2d.dtype == MXFP8_VALUE_DTYPE, ( - "pre-quantized input must be FP8 E4M3 when input_scale is given." - ) - assert input_scale.dtype == torch.uint8 and input_scale.shape == ( - m, - k // 32, - ), "input_scale must be UE8M0 uint8 [M, K//32]." - x_q, x_scale = input_2d, input_scale - kernel_out_dtype = output_dtype - out = _run_mxfp8_linear_kernel( - x_q, x_scale, weight, weight_scale, kernel_out_dtype - ) - else: - # dot_scaled tiling needs K % 128 == 0; dequantize fallback otherwise. - w_bf16 = dequant_mxfp8_to_bf16(weight, weight_scale) - if input_scale is not None: - input_2d = dequant_mxfp8_to_bf16(input_2d, input_scale) - out = F.linear(input_2d.to(w_bf16.dtype), w_bf16).to(output_dtype) - - if bias is not None: - out = out + bias - return out.to(output_dtype).view(*output_shape) - - # CUDA's per_token_group_quant amax floor (LOCAL_ABSMAX_ABS); mxfp8_gemv_gfx95.cuh hard-codes it too _FP8_GRID_AMAX_FLOOR = tl.constexpr(1e-10) @@ -466,51 +277,3 @@ def dequant_block_fp8_weight_to_bf16( scale = weight_scale.float().repeat_interleave(block_n, dim=0)[:n] scale = scale.repeat_interleave(block_k, dim=1)[:, :k] return (weight.float() * scale).to(torch.bfloat16) - - -# at one or two rows aiter's wvSpltK (fixed-order split-K) beats hipBLASLt's small-M kernels -_SKINNY_GEMM_MAX_TOKENS = 2 - - -@cache_once -def _skinny_cu_count() -> int: - try: - from aiter.jit.utils.chip_info import get_cu_num - except ImportError: - return 0 - return int(get_cu_num()) - - -# both operands are exact in bf16, so only the fp32 sum order differs from the MXFP8 route -def bf16_dequant_blockscaled_linear( - input: torch.Tensor, - weight: torch.Tensor, - bias: Optional[torch.Tensor] = None, - input_on_fp8_grid: bool = False, -) -> torch.Tensor: - """weight is the dequantized bf16 weight (dequant_block_fp8_weight_to_bf16). - input_on_fp8_grid says a fused upstream kernel (rmsnorm_fake_quant_fp8) - already put input on the fp8 grid, so the fake-quant here is skipped.""" - assert weight.dtype == torch.bfloat16, weight.dtype - input_2d = input.view(-1, input.shape[-1]) - if input_on_fp8_grid: - x = input_2d.to(torch.bfloat16) - else: - x = fake_quant_fp8_activation(input_2d.to(torch.bfloat16)) - cu_count = _skinny_cu_count() - if ( - bias is None - and x.shape[0] <= _SKINNY_GEMM_MAX_TOKENS - and cu_count > 0 - and weight.is_contiguous() - and weight.shape[1] % 8 == 0 - ): - import aiter - - out = torch.empty( - (x.shape[0], weight.shape[0]), dtype=torch.bfloat16, device=x.device - ) - aiter.wvSpltK(weight, x, out, x.shape[0], cu_count) - else: - out = F.linear(x, weight, bias) - return out.view(*input.shape[:-1], weight.shape[0]) diff --git a/python/sglang/kernels/ops/layernorm/mxfp8_epilogue.py b/python/sglang/kernels/ops/quantization/mxfp8_epilogue.py similarity index 55% rename from python/sglang/kernels/ops/layernorm/mxfp8_epilogue.py rename to python/sglang/kernels/ops/quantization/mxfp8_epilogue.py index f4267d49ff..f7c5a89b32 100644 --- a/python/sglang/kernels/ops/layernorm/mxfp8_epilogue.py +++ b/python/sglang/kernels/ops/quantization/mxfp8_epilogue.py @@ -15,7 +15,6 @@ over a row count padded to a multiple of 128, with the UE8M0 conversion from __future__ import annotations -import torch import triton import triton.language as tl @@ -51,61 +50,3 @@ def mxfp8_epilogue( tl.store(Q + row * K + idx, q, gmask[:, None]) off = (g // 4) * 512 + ((row % 32) * 4 + ((row // 32) % 4)) * 4 + (g % 4) tl.store(S + off, sf.to(tl.uint8), gmask) - - -@triton.jit -def _rmsnorm_mxfp8_kernel( - X, - W, - Y, - Q, - S, - SX: tl.constexpr, - M: tl.constexpr, - K: tl.constexpr, - EPS: tl.constexpr, - BLOCK: tl.constexpr, - GROUPS: tl.constexpr, -): - row = tl.program_id(0) - if row < M: - h = tl.arange(0, BLOCK) - v = tl.load(X + row * SX + h, h < K, 0).to(tl.float32) - weight = tl.load(W + h, h < K, 0).to(tl.float32) - inv = tl.rsqrt(tl.sum(v * v, 0) / K + EPS) - y = (v * inv * weight).to(tl.bfloat16) - tl.store(Y + row * K + h, y, h < K) - mxfp8_epilogue(y, row, Q, S, K, BLOCK, GROUPS, 0, GROUPS) - else: - # Padding scale entries are disjoint from the live rows above. - off = (row - M) * 512 + tl.arange(0, 512) - pad_row = ((off // 4) % 4) * 32 + ((off // 16) % 32) - tl.store(S + off, 0, (off < GROUPS * 128) & (pad_row >= M)) - - -def rmsnorm_mxfp8( - x: torch.Tensor, weight: torch.Tensor, eps: float -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """RMSNorm returning ``(y_bf16, y_q, y_sf)`` with the same MXFP8 epilogue.""" - m, k = x.shape - assert 0 < m <= 8 and k % 32 == 0 - assert x.dtype == weight.dtype == torch.bfloat16 and x.stride(1) == 1 - y = torch.empty_like(x, memory_format=torch.contiguous_format) - q = torch.empty((m, k), dtype=torch.float8_e4m3fn, device=x.device) - s = torch.empty((k // 32) * 128, dtype=torch.uint8, device=x.device) - _rmsnorm_mxfp8_kernel[(m + triton.cdiv(s.numel(), 512),)]( - x, - weight, - y, - q, - s, - SX=x.stride(0), - M=m, - K=k, - EPS=eps, - BLOCK=triton.next_power_of_2(k), - GROUPS=k // 32, - num_warps=8, - enable_fp_fusion=False, - ) - return y, q, s diff --git a/python/sglang/kernels/ops/quantization/mxfp8_quant.py b/python/sglang/kernels/ops/quantization/mxfp8_quant.py index a21916ce37..d672dac551 100644 --- a/python/sglang/kernels/ops/quantization/mxfp8_quant.py +++ b/python/sglang/kernels/ops/quantization/mxfp8_quant.py @@ -92,117 +92,3 @@ def to_mxfp8(x: torch.Tensor) -> MXFP8Tensor: data=xq.view(orig_shape), scale=scales.view(*orig_shape[:-1], K // MXFP8_BLOCK_SIZE), ) - - -@triton.jit -def _mxfp8_quant_store_qkv_kernel( - q_ptr, - k_ptr, - v_ptr, - loc_ptr, - q8_ptr, - sfq_ptr, - kc_ptr, - vc_ptr, - sfk_ptr, - sfv_ptr, - sqt, - sqh, - skt, - skh, - svt, - svh, - NQ: tl.constexpr, - NKV: tl.constexpr, - D: tl.constexpr, - PAGE: tl.constexpr, -): - t = tl.program_id(0) - r = tl.program_id(1) - SF: tl.constexpr = D // 32 - blk = tl.arange(0, SF) - off2 = blk[:, None] * 32 + tl.arange(0, 32)[None, :] - - if r < NQ: - src = q_ptr + t * sqt + r * sqh - elif r < NQ + NKV: - src = k_ptr + t * skt + (r - NQ) * skh - else: - src = v_ptr + t * svt + (r - NQ - NKV) * svh - - x = tl.load(src + off2).to(tl.float32) - amax = tl.maximum(tl.max(tl.abs(x), axis=1), 1e-30) - scale_biased = tl.ceil(tl.log2(amax / 448.0)) + 127.0 - scale_biased = tl.minimum(tl.maximum(scale_biased, 0.0), 254.0) - descale = tl.exp2(scale_biased - 127.0) - xq = tl.clamp(x / descale[:, None], -448.0, 448.0).to(q8_ptr.dtype.element_ty) - sf = scale_biased.to(tl.uint8) - - if r < NQ: - tl.store(q8_ptr + (t * NQ + r) * D + off2, xq) - tl.store(sfq_ptr + (t * NQ + r) * SF + blk, sf) - else: - myloc = tl.load(loc_ptr + t).to(tl.int64) - # Slot 0 is the reserved CUDA-graph padding sink; skip writes to it. - if myloc == 0: - return - if r < NQ + NKV: - h = r - NQ - cache = kc_ptr - sfb = sfk_ptr - else: - h = r - NQ - NKV - cache = vc_ptr - sfb = sfv_ptr - tl.store(cache + (myloc * NKV + h) * D + off2, xq) - # BlockScaledBasicChunk byte layout (see mxfp8_interleave_sf.py): - # a page block is (NKV, 32, PAGE//32, SF) bytes. - po = myloc % PAGE - base = ( - ((myloc // PAGE) * NKV + h) * (32 * (PAGE // 32) * SF) - + (po % 32) * ((PAGE // 32) * SF) - + (po // 32) * SF - ) - tl.store(sfb + base + blk, sf) - - -def quant_store_kv_mxfp8( - k: torch.Tensor, - v: torch.Tensor, - loc: torch.Tensor, - k_cache: torch.Tensor, - v_cache: torch.Tensor, - sfk: torch.Tensor, - sfv: torch.Tensor, - page_size: int = 128, -) -> None: - """One kernel quantizes bf16 K/V rows, scatters the fp8 payload into the paged - cache at `loc`, and writes UE8M0 scales in the interleaved FA4 layout.""" - T, NKV, D = k.shape - assert D % 32 == 0 and page_size % 32 == 0 - assert k.stride(2) == 1 and v.stride(2) == 1 - assert k_cache.is_contiguous() and v_cache.is_contiguous() - if T == 0: - return - _mxfp8_quant_store_qkv_kernel[(T, 2 * NKV)]( - k, # unused q_ptr (same elem dtype) - k, - v, - loc, - k_cache, # unused q8_ptr, supplies the fp8 store dtype - sfk.view(torch.uint8), # unused sfq_ptr (uint8, matches the dead store) - k_cache, - v_cache, - sfk.view(torch.uint8), - sfv.view(torch.uint8), - 0, - 0, - k.stride(0), - k.stride(1), - v.stride(0), - v.stride(1), - NQ=0, - NKV=NKV, - D=D, - PAGE=page_size, - ) diff --git a/python/sglang/kernels/ops/quantization/mxfp8_swizzled_triton.py b/python/sglang/kernels/ops/quantization/mxfp8_swizzled_triton.py index 0dc789233d..55d6441d24 100644 --- a/python/sglang/kernels/ops/quantization/mxfp8_swizzled_triton.py +++ b/python/sglang/kernels/ops/quantization/mxfp8_swizzled_triton.py @@ -18,8 +18,6 @@ import torch import triton import triton.language as tl -from sglang.kernels.numerics import round_bf16_to_fp32 - _E4M3 = torch.float8_e4m3fn @@ -86,127 +84,6 @@ def _mxfp8_quant_kernel( ) -@triton.jit -def _silu_mul_mxfp8_kernel( - x_ptr, - q_ptr, - s_ptr, - rows, - hidden, - n_groups, - n_col_blocks, - stride_row, - dg_rows, - ALPHA: tl.constexpr, - LIMIT: tl.constexpr, - DEEPGEMM: tl.constexpr, - BLOCK_R: tl.constexpr, - G: tl.constexpr, -): - pid_r = tl.program_id(0) - pid_c = tl.program_id(1) - r = pid_r * BLOCK_R + tl.arange(0, BLOCK_R) - g = pid_c * G + tl.arange(0, G) - c = pid_c * (G * 32) + tl.arange(0, G * 32) - rmask = r < rows - mask = rmask[:, None] & (c < hidden)[None, :] - base = x_ptr + r[:, None].to(tl.int64) * stride_row - gate = tl.load(base + c[None, :], mask=mask, other=0.0).to(tl.float32) - up = tl.load(base + hidden + c[None, :], mask=mask, other=0.0).to(tl.float32) - if ALPHA is None: - act = (gate * tl.sigmoid(gate)).to(tl.bfloat16).to(tl.float32) - prod = (act * up).to(tl.bfloat16).to(tl.float32) - else: - # the op order Inductor emits for swiglu_no_interleaved_with_alpha_and_limit - gate = tl.minimum(gate, LIMIT, propagate_nan=tl.PropagateNan.ALL) - up = tl.maximum(up, -LIMIT, propagate_nan=tl.PropagateNan.ALL) - up = tl.minimum(up, LIMIT, propagate_nan=tl.PropagateNan.ALL) - prod = gate * tl.sigmoid(gate * ALPHA) * (up + 1.0) - prod = prod.to(tl.bfloat16).to(tl.float32) - p3 = tl.reshape(prod, [BLOCK_R, G, 32]) - amax = tl.max(tl.abs(p3), axis=2) - if DEEPGEMM: - amax = tl.maximum(amax, 1e-10) - sbyte, inv = _mx_e8m0_from_amax(amax) - q = tl.reshape(p3 * inv[:, :, None], [BLOCK_R, G * 32]) - tl.store( - q_ptr + r[:, None].to(tl.int64) * hidden + c[None, :], - q.to(tl.float8e4nv), - mask=mask, - ) - valid = rmask[:, None] & (g < n_groups)[None, :] - if DEEPGEMM: - # 4 bytes per int32 along K, MN-major, rows padded to 4 - soff = ( - (g[None, :] // 4).to(tl.int64) * dg_rows * 4 - + r[:, None] * 4 - + g[None, :] % 4 - ) - tl.store(s_ptr + soff, sbyte.to(tl.uint8), mask=valid) - else: - smask = (r < tl.cdiv(rows, 128) * 128)[:, None] & (g < n_col_blocks * 4)[ - None, : - ] - tl.store( - s_ptr + _mx_scale_offsets(r[:, None], g[None, :], n_col_blocks), - tl.where(valid, sbyte, 0).to(tl.uint8), - mask=smask, - ) - - -@triton.jit -def _indexed_scale_shift_mxfp8_kernel( - x_ptr, - q_ptr, - s_ptr, - shift_ptr, - scale_ptr, - indices_ptr, - hidden_size, - n_groups, - n_col_blocks, - stride_x_row, - stride_shift_row, - stride_scale_row, - stride_indices, - STORE_BF16: tl.constexpr, - BLOCK_N: tl.constexpr, -): - row = tl.program_id(0) - columns = tl.arange(0, BLOCK_N) - mask = columns < hidden_size - index = tl.load(indices_ptr + row * stride_indices) - xrow = x_ptr + row.to(tl.int64) * stride_x_row - x = tl.load(xrow + columns, mask=mask, other=0.0).to(tl.float32) - shift = tl.load( - shift_ptr + index * stride_shift_row + columns, mask=mask, other=0.0 - ).to(tl.float32) - scale = tl.load( - scale_ptr + index * stride_scale_row + columns, mask=mask, other=0.0 - ).to(tl.float32) - # the rounding points of _indexed_scale_shift_bf16_kernel - one_plus_scale = round_bf16_to_fp32(1.0 + scale) - scaled = round_bf16_to_fp32(x * one_plus_scale) - out = round_bf16_to_fp32(scaled + shift) - if STORE_BF16: - tl.store(xrow + columns, out, mask=mask) - v3 = tl.reshape(out, [BLOCK_N // 32, 32]) - amax = tl.max(tl.abs(v3), axis=1) - sbyte, inv = _mx_e8m0_from_amax(amax) - q = tl.reshape(v3 * inv[:, None], [BLOCK_N]) - tl.store( - q_ptr + row.to(tl.int64) * hidden_size + columns, - q.to(tl.float8e4nv), - mask=mask, - ) - g = tl.arange(0, BLOCK_N // 32) - tl.store( - s_ptr + _mx_scale_offsets(row, g, n_col_blocks), - sbyte.to(tl.uint8), - mask=g < n_groups, - ) - - def can_use_mxfp8_swizzled(x: torch.Tensor) -> bool: """Row-major bf16 CUDA 2D tensor with K % 32 == 0, outside torch.compile.""" return ( @@ -257,135 +134,7 @@ def mxfp8_quantize_swizzled(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor return q, s -def can_use_silu_mul_mxfp8(hidden: torch.Tensor) -> bool: - return can_use_mxfp8_swizzled(hidden) and (hidden.shape[-1] // 2) % 32 == 0 - - -def _swiglu_mxfp8( - hidden: torch.Tensor, - alpha: float | None, - limit: float | None, - deepgemm: bool = False, -) -> tuple[torch.Tensor, torch.Tensor]: - if not can_use_silu_mul_mxfp8(hidden): - raise ValueError( - "expected a row-major bf16 CUDA [rows, 2 * n] tensor, n % 32 == 0" - ) - rows, twice = hidden.shape - n = twice // 2 - if deepgemm: - if n % 128: - raise ValueError(f"DeepGEMM MXFP8 scales need n % 128 == 0, got {n=}") - q = torch.empty(rows, n, dtype=_E4M3, device=hidden.device) - dg_rows = triton.cdiv(rows, 4) * 4 - base = torch.empty(n // 128, dg_rows, dtype=torch.int32, device=hidden.device) - s = base.T[:rows] - s_bytes = base.view(torch.uint8) - padded_rows = rows - else: - q, s = _alloc(rows, n, hidden.device, zero_scales=False) - s_bytes = s - dg_rows = 0 - padded_rows = triton.cdiv(rows, 128) * 128 - if rows == 0: - return q, s - n_groups = n // 32 - n_col_blocks = triton.cdiv(n_groups, 4) - block_r, g = (4 if rows <= 1024 else 16), 8 - grid = (triton.cdiv(padded_rows, block_r), triton.cdiv(n_groups, g)) - with torch.get_device_module().device(hidden.device): - _silu_mul_mxfp8_kernel[grid]( - hidden, - q, - s_bytes, - rows, - n, - n_groups, - n_col_blocks, - hidden.stride(0), - dg_rows, - ALPHA=alpha, - LIMIT=limit, - DEEPGEMM=deepgemm, - BLOCK_R=block_r, - G=g, - num_warps=4, - ) - return q, s - - -def silu_mul_mxfp8(hidden: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: - """``hidden [rows, 2n]`` bf16 (gate | up) -> quantized ``silu(gate) * up``.""" - return _swiglu_mxfp8(hidden, None, None) - - -def swiglu_oai_mxfp8( - hidden: torch.Tensor, alpha: float, limit: float -) -> tuple[torch.Tensor, torch.Tensor]: - """``hidden [rows, 2n]`` bf16 (gate | up) -> quantized SwiGLU-OAI, - ``clamp(gate, max=limit) * sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)``.""" - return _swiglu_mxfp8(hidden, alpha, limit) - - -def swiglu_oai_mxfp8_deepgemm( - hidden: torch.Tensor, alpha: float, limit: float -) -> tuple[torch.Tensor, torch.Tensor]: - """Same values as ``swiglu_oai_mxfp8``; scales in DeepGEMM's layout, the - ``sglang_per_token_group_quant_fp8(..., 32, column_major_scales=True, - scale_tma_aligned=True, scale_ue8m0=True)`` output: int32 ``[rows, n / 128]``, - four UE8M0 bytes each, MN-major with rows padded to 4.""" - return _swiglu_mxfp8(hidden, alpha, limit, deepgemm=True) - - -def indexed_scale_shift_mxfp8_( - x: torch.Tensor, - shift: torch.Tensor, - scale: torch.Tensor, - indices: torch.Tensor, - *, - keep_bf16: bool, -) -> tuple[torch.Tensor | None, torch.Tensor, torch.Tensor]: - """``x * (1 + scale[idx]) + shift[idx]`` -> ``(x | None, fp8, scales)``; with - ``keep_bf16`` the bf16 result is also written into ``x`` and returned.""" - if not can_use_mxfp8_swizzled(x): - raise ValueError( - "expected a row-major bf16 CUDA [rows, hidden] tensor, hidden % 32 == 0" - ) - rows, hidden_size = x.shape - q, s = _alloc(rows, hidden_size, x.device) - if rows == 0: - return (x if keep_bf16 else None), q, s - n_groups = hidden_size // 32 - n_col_blocks = triton.cdiv(n_groups, 4) - block_n = triton.next_power_of_2(hidden_size) - with torch.get_device_module().device(x.device): - _indexed_scale_shift_mxfp8_kernel[(rows,)]( - x, - q, - s, - shift, - scale, - indices, - hidden_size, - n_groups, - n_col_blocks, - x.stride(0), - shift.stride(0), - scale.stride(0), - indices.stride(0), - STORE_BF16=keep_bf16, - BLOCK_N=block_n, - num_warps=8, - ) - return (x if keep_bf16 else None), q, s - - __all__ = [ "can_use_mxfp8_swizzled", - "can_use_silu_mul_mxfp8", - "indexed_scale_shift_mxfp8_", "mxfp8_quantize_swizzled", - "silu_mul_mxfp8", - "swiglu_oai_mxfp8", - "swiglu_oai_mxfp8_deepgemm", ] diff --git a/python/sglang/kernels/ops/moe/rocm_moe_utils.py b/python/sglang/kernels/ops/quantization/rocm_upscale.py similarity index 100% rename from python/sglang/kernels/ops/moe/rocm_moe_utils.py rename to python/sglang/kernels/ops/quantization/rocm_upscale.py diff --git a/python/sglang/kernels/ops/sampling/__init__.py b/python/sglang/kernels/ops/sampling/__init__.py index f94ffc9880..d0dee0cfd3 100644 --- a/python/sglang/kernels/ops/sampling/__init__.py +++ b/python/sglang/kernels/ops/sampling/__init__.py @@ -78,3 +78,11 @@ register_kernel( capabilities=CUDA_ONLY, ) ) +register_kernel( + KernelSpec( + op="sampling.softcap_row_argmax", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.sampling.softcap_row_argmax:softcap_row_argmax", + capabilities=CUDA_ONLY, + ) +) diff --git a/python/sglang/kernels/ops/sampling/softcap_row_argmax.py b/python/sglang/kernels/ops/sampling/softcap_row_argmax.py new file mode 100644 index 0000000000..8663c9584e --- /dev/null +++ b/python/sglang/kernels/ops/sampling/softcap_row_argmax.py @@ -0,0 +1,56 @@ +"""Greedy token selection over softcapped logits of a vocab shard.""" + +import torch +import triton +import triton.language as tl +from triton.language.extra import libdevice + + +@triton.jit +def _softcap_row_argmax_kernel( + logits_ptr, + out_ptr, + row_stride, + valid_width, + col_offset, + softcapping_value, + BLOCK_SIZE: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + best_val = -float("inf") + best_idx = 0 + for start in range(0, valid_width, BLOCK_SIZE): + cols = start + tl.arange(0, BLOCK_SIZE) + mask = cols < valid_width + x = tl.load(logits_ptr + row * row_stride + cols, mask=mask, other=0.0) + x = x / softcapping_value + x = libdevice.tanh(x) + x = x * softcapping_value + x = tl.where(mask, x, -float("inf")) + block_val, block_idx = tl.max( + x, axis=0, return_indices=True, return_indices_tie_break_left=True + ) + take = block_val > best_val + best_idx = tl.where(take, start + block_idx, best_idx) + best_val = tl.where(take, block_val, best_val) + tl.store(out_ptr + row * 2, best_val) + tl.store(out_ptr + row * 2 + 1, (best_idx + col_offset).to(tl.float32)) + + +def softcap_row_argmax(logits, final_logit_softcapping, valid_width, col_offset): + """Per row of a vocab shard: ``(max, first argmax + col_offset)`` of the + softcapped logits over the first ``valid_width`` columns, as fp32 ``[rows, 2]``.""" + assert logits.dtype == torch.float32 and logits.stride(1) == 1 + assert col_offset + logits.shape[1] < 2**24 + out = torch.empty(logits.shape[0], 2, dtype=torch.float32, device=logits.device) + _softcap_row_argmax_kernel[(logits.shape[0],)]( + logits, + out, + logits.stride(0), + valid_width, + col_offset, + final_logit_softcapping, + BLOCK_SIZE=4096, + num_warps=8, + ) + return out diff --git a/python/sglang/kernels/ops/speculative/dspark/dspark_accept.py b/python/sglang/kernels/ops/speculative/dspark/dspark_accept.py index 47dc383aa6..991caf2dd9 100644 --- a/python/sglang/kernels/ops/speculative/dspark/dspark_accept.py +++ b/python/sglang/kernels/ops/speculative/dspark/dspark_accept.py @@ -272,6 +272,7 @@ except ImportError: _flashinfer_softmax = None +# Not sampling.softmax: the FlashInfer and Triton paths are not bitwise torch.softmax(x / t). class SoftmaxTemp: @classmethod def execute(cls, *args, **kwargs) -> torch.Tensor: diff --git a/python/sglang/multimodal_gen/runtime/layers/mlp.py b/python/sglang/multimodal_gen/runtime/layers/mlp.py index d582d0cc83..e1268643f7 100644 --- a/python/sglang/multimodal_gen/runtime/layers/mlp.py +++ b/python/sglang/multimodal_gen/runtime/layers/mlp.py @@ -77,7 +77,7 @@ class MLP(nn.Module): and x.is_cuda and x.dtype in (torch.float16, torch.bfloat16) ): - from sglang.kernels.ops.elementwise.bias_gelu import bias_gelu_tanh + from sglang.kernels.ops.activation.bias_gelu import bias_gelu_tanh return bias_gelu_tanh(x, bias) return self.act(x + bias) diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py b/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py index d5d1c93be4..7014c896e5 100755 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py @@ -101,7 +101,7 @@ def _prepare_nvfp4_swiglu_fusion_weights( weight: torch.Tensor, scales: torch.Tensor, ) -> None: - from sglang.kernels.ops.quantization.nvfp4_gemm_swiglu_nvfp4_quant import ( + from sglang.kernels.ops.gemm.nvfp4_gemm_swiglu_nvfp4_quant import ( interleave_linear_and_gate, swizzle_blockscale_2d, ) @@ -972,7 +972,7 @@ def apply_nvfp4_gemm_swiglu_quant( if fp4_quantize is None: raise RuntimeError("No FP4 quantization kernel available. Install flashinfer.") - from sglang.kernels.ops.quantization.nvfp4_gemm_swiglu_nvfp4_quant import ( + from sglang.kernels.ops.gemm.nvfp4_gemm_swiglu_nvfp4_quant import ( nvfp4_gemm_swiglu_nvfp4_quant, ) diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index 6a1f66d862..a0076764ac 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -250,7 +250,7 @@ def validate_deepseek_v41_encoder_replay_budget(server_args: ServerArgs) -> None def validate_deepseek_v41_features(server_args: ServerArgs) -> None: - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_triton, ) diff --git a/python/sglang/srt/arg_groups/hisparse_hook.py b/python/sglang/srt/arg_groups/hisparse_hook.py index 38ddd24ca1..dcca33a444 100644 --- a/python/sglang/srt/arg_groups/hisparse_hook.py +++ b/python/sglang/srt/arg_groups/hisparse_hook.py @@ -133,7 +133,7 @@ def validate_hisparse(server_args: ServerArgs) -> None: # In unified-KV mode c4_kv_pool is None, so DeepSeekV4HiSparseTokenToKVPoolAllocator # cannot attach and pool init dies with a cryptic AssertionError. Fail fast # at startup with a clear message instead. Remove once unified-KV HiSparse lands. - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_triton, ) diff --git a/python/sglang/kernels/ops/communication/mp.py b/python/sglang/srt/distributed/comm_cleanup.py similarity index 100% rename from python/sglang/kernels/ops/communication/mp.py rename to python/sglang/srt/distributed/comm_cleanup.py diff --git a/python/sglang/srt/hardware_backend/xpu/kernels/fla/fused_sigmoid_gating_recurrent.py b/python/sglang/srt/hardware_backend/xpu/kernels/fla/fused_sigmoid_gating_recurrent.py index 8568b874ff..a46d7d87bb 100644 --- a/python/sglang/srt/hardware_backend/xpu/kernels/fla/fused_sigmoid_gating_recurrent.py +++ b/python/sglang/srt/hardware_backend/xpu/kernels/fla/fused_sigmoid_gating_recurrent.py @@ -3,7 +3,7 @@ from typing import Optional import torch import triton -from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( +from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update_kernel, ) diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index a6da8049ab..fddd5272b6 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -15,17 +15,17 @@ from typing import TYPE_CHECKING, Optional import torch import triton -from sglang.kernels.ops.attention.utils import ( - assert_buffer_fits, +from sglang.kernels.ops.attention.aiter_unified_page_table import ( + scatter_ragged_to_page_table_kernel, + scatter_req_to_token_to_page_table_kernel, +) +from sglang.kernels.ops.attention.kv_indices import ( create_flashinfer_kv_indices_triton, create_flashmla_kv_indices_triton, get_num_kv_index_blocks_flashmla, kv_indices_num_token_blocks, ) -from sglang.kernels.ops.kvcache.aiter_unified_attention import ( - scatter_ragged_to_page_table_kernel, - scatter_req_to_token_to_page_table_kernel, -) +from sglang.kernels.ops.attention.utils import assert_buffer_fits from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.dcp import update_local_kv_lens_for_dcp from sglang.srt.layers.dcp.planner import plan_dcp_decode_metadata @@ -77,10 +77,8 @@ from sglang.kernels.ops.attention.dcp_kernels import ( create_mla_kv_page_table_for_dcp, ) from sglang.kernels.ops.attention.merge_state import merge_state_triton -from sglang.kernels.ops.attention.utils import ( - launch_reshape_and_cache_flash, - pad_sequence_with_mask, -) +from sglang.kernels.ops.attention.pad import pad_sequence_with_mask +from sglang.kernels.ops.kvcache.cache_ops import launch_reshape_and_cache_flash from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype, scaled_fp8_quant from sglang.srt.configs.model_config import AttentionArch from sglang.srt.environ import envs diff --git a/python/sglang/srt/layers/attention/aiter_utils.py b/python/sglang/srt/layers/attention/aiter_utils.py index 9d57584cca..4429f83614 100644 --- a/python/sglang/srt/layers/attention/aiter_utils.py +++ b/python/sglang/srt/layers/attention/aiter_utils.py @@ -33,7 +33,9 @@ except ImportError: # pragma: no cover - import-time guard mirrors aiter_backen pa_decode_gluon = None get_recommended_splits = None -from sglang.kernels.ops.attention.utils import launch_gather_shuffle_5d_to_linear +from sglang.kernels.ops.kvcache.shuffle_5d_cache import ( + launch_gather_shuffle_5d_to_linear, +) from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype if TYPE_CHECKING: diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index 75c8bb74c2..e00ea4a681 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -19,15 +19,20 @@ import msgspec import torch import torch.nn.functional as F +from sglang.kernels.ops.attention.dsv4.attn_metadata_kernels import ( + BuildCausalSwaPageIndices, + BuildPageTablePositions, + ExpandPrefillCausally, + late_layer_tail_layout, + late_layer_tail_lens, +) from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import ( can_use_swapab_attention, ) from sglang.kernels.ops.attention.dsv4.dequant_k_cache import ( - cast_q_fp8_for_q8kv8_prefill, - dequantize_k_cache_paged, + dequantize_k_cache_paged_dsv4, fp8_dtype, - gather_dequant_requant_fp8_paged, - q8kv8_padded_num_heads, + gather_dequant_requant_fp8_paged_dsv4, ) from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout from sglang.kernels.ops.attention.dsv4.metadata_kernel import ( @@ -36,14 +41,6 @@ from sglang.kernels.ops.attention.dsv4.metadata_kernel import ( from sglang.kernels.ops.attention.dsv4.metadata_kernel import ( init_compression_metadata as _init_compression_metadata_triton, ) -from sglang.kernels.ops.attention.dsv4.online_c128_mtp import OnlineC128MTPController -from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import ( - BuildCausalSwaPageIndices, - BuildPageTablePositions, - ExpandPrefillCausally, - late_layer_tail_layout, - late_layer_tail_lens, -) from sglang.kernels.ops.speculative.dspark.dspark_attn_metadata import ( BuildBlockSeqLensCausal, BuildDsparkSwaPageIndices, @@ -75,6 +72,7 @@ from sglang.srt.layers.attention.dsv4.metadata import ( expand_index_page_table, maybe_copy_inplace, ) +from sglang.srt.layers.attention.dsv4.online_c128_mtp import OnlineC128MTPController from sglang.srt.layers.attention.dsv4.sparse_prefill_utils import ( SparsePrefillChunkCache, SparsePrefillWorkspace, @@ -4247,14 +4245,14 @@ class DeepseekV4AttnBackend( swa_slice = workspace[n_compressed:] if compressed_slice is not None: - dequantize_k_cache_paged( + dequantize_k_cache_paged_dsv4( extra_k_cache, flat_token_ids, page_size=extra_page_size, out=compressed_slice, layout=token_to_kv_pool.get_extra_key_layout(layer_id), ) - dequantize_k_cache_paged( + dequantize_k_cache_paged_dsv4( token_to_kv_pool.get_swa_key_buffer_radix(layer_id), cache.swa_token_ids, page_size=cache.swa_page_size, @@ -4291,6 +4289,11 @@ class DeepseekV4AttnBackend( attn_sink: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]: """Pad TP-local heads to the SM90 kernel's 64-head CTA granularity.""" + from sglang.kernels.ops.attention.sparse_mla_q8kv8_prefill_sm90 import ( + cast_q_fp8_for_q8kv8_prefill, + q8kv8_padded_num_heads, + ) + num_tokens, num_heads, head_dim = q.shape padded_heads = q8kv8_padded_num_heads(num_heads) @@ -4420,14 +4423,14 @@ class DeepseekV4AttnBackend( # The Q8KV8 gather reads the 584-byte V4 layout only (its kernel is SM90). assert token_to_kv_pool.get_swa_key_layout() is KVLayout.V4 if compressed_slice is not None: - gather_dequant_requant_fp8_paged( + gather_dequant_requant_fp8_paged_dsv4( extra_k_cache, flat_token_ids, page_size=extra_page_size, out=compressed_slice, ) - gather_dequant_requant_fp8_paged( + gather_dequant_requant_fp8_paged_dsv4( token_to_kv_pool.get_swa_key_buffer_radix(layer_id), cache.swa_token_ids, page_size=cache.swa_page_size, @@ -4547,12 +4550,12 @@ class DeepseekV4AttnBackend( ) build_pages = BuildPageTablePositions.execute if small_metadata: + from sglang.kernels.ops.attention.dsv4.attn_metadata_kernels import ( + build_page_table_positions_small, + ) from sglang.kernels.ops.attention.dsv4.metadata_kernel import ( build_low_ratio_metadata, ) - from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import ( - build_page_table_positions_small, - ) build_pages = build_page_table_positions_small prep = build_pages( diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py index ee6082a94f..74597972ae 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend_hip_radix.py @@ -28,6 +28,10 @@ from sglang.kernels.ops.attention.dsv4.attn_glue_hip import ( sparse_buffers, widen_pair_i64, ) +from sglang.kernels.ops.attention.dsv4.attn_metadata_kernels import ( + BuildCausalSwaPageIndices, + late_layer_tail_layout, +) from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import ( CandidateBlocks, slice_candidate_blocks, @@ -44,7 +48,9 @@ from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import ( from sglang.kernels.ops.attention.dsv4.decode_attention_sm100 import ( SOFTMAX_SCALE as SWAPAB_SOFTMAX_SCALE, ) -from sglang.kernels.ops.attention.dsv4.dequant_k_cache import dequantize_k_cache_paged +from sglang.kernels.ops.attention.dsv4.dequant_k_cache import ( + dequantize_k_cache_paged_dsv4, +) from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout from sglang.kernels.ops.attention.dsv4.metadata_kernel import ( init_compression_metadata as _init_compression_metadata_triton, @@ -55,10 +61,6 @@ from sglang.kernels.ops.attention.dsv4.opus_sparse_prefill_hip import ( opus_sparse_prefill, ) from sglang.kernels.ops.attention.dsv4.swapab_gluon_hip import swapab_attention -from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import ( - BuildCausalSwaPageIndices, - late_layer_tail_layout, -) from sglang.srt.environ import envs from sglang.srt.layers.attention import deepseek_v4_backend as _cuda_backend from sglang.srt.layers.attention.base_attn_backend import AttentionBackend @@ -973,7 +975,7 @@ class DeepseekV4HipRadixBackend( ) if self.enable_decoder_swa_bounded_replay: # both read the store target off the full batch, not the tail metadata - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_triton, ) @@ -1185,7 +1187,7 @@ class DeepseekV4HipRadixBackend( assert cp_metadata is not None padded_num_tokens = sum(cp_metadata.per_rank_actual_token) - from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import ( + from sglang.kernels.ops.attention.dsv4.attn_metadata_kernels import ( ExpandPrefillCausally, ) @@ -1430,7 +1432,7 @@ class DeepseekV4HipRadixBackend( """Whether the V4.1 DSpark draft block takes the block-window metadata; DSv4's draft keeps the causal verify indices, and unified_kv addresses its ring by (slot, position) and keeps its own path.""" - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_triton, ) @@ -2420,7 +2422,7 @@ class DeepseekV4HipRadixBackend( self._refresh_fp4_prefill_workspace(forward_batch) assert isinstance(self.forward_metadata, DSV4Metadata) return self.forward_metadata - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_triton, ) @@ -2568,7 +2570,7 @@ class DeepseekV4HipRadixBackend( """fp8 unified_kv asm decode: per-stream KV bound for the split plan and, at init (outside capture, before warmup), the constant split table and the HCA persistent workspace.""" - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_fp8, ) @@ -2603,7 +2605,7 @@ class DeepseekV4HipRadixBackend( ) -> None: # state_slot maps each query token to its request slot; # target-verify repeats request slots for the draft tokens. - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_fp8, is_unified_kv_triton, ) @@ -2737,7 +2739,7 @@ class DeepseekV4HipRadixBackend( num_tokens: int, exact_num_tokens: bool = True, ) -> None: - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_triton, ) @@ -2805,6 +2807,7 @@ class DeepseekV4HipRadixBackend( wo_a_quant_freqs (fp8 decode): return the inverse-RoPE'd wo_a mxfp8 operands instead of the attention output.""" from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime + from sglang.kernels.ops.kvcache import dsv4_unified_kv_store pool = self.token_to_kv_pool layer_id = layer.layer_id @@ -2837,7 +2840,7 @@ class DeepseekV4HipRadixBackend( if save_kv_cache: # Only verify reaches this under fp8; plain decode's rows are # written by the fused kernel itself, which leaves kv None. - runtime.store_swa_into_unified( + dsv4_unified_kv_store.store_swa_into_unified( kv=kv, state_slot=state_slot, positions=positions, @@ -2951,7 +2954,7 @@ class DeepseekV4HipRadixBackend( ), ) from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.paged_decode import ( - _kv_splits_for_stream, + kv_splits_for_stream, ) return runtime.decode( @@ -2963,7 +2966,7 @@ class DeepseekV4HipRadixBackend( softmax_scale=self.softmax_scale, # Only this call site knows compress_ratio, and it is the one # thing that separates the ragged stream from the clamped ones. - kv_splits=_kv_splits_for_stream(compress_ratio), + kv_splits=kv_splits_for_stream(compress_ratio), ) # prefill / extend @@ -3049,7 +3052,7 @@ class DeepseekV4HipRadixBackend( n_real = kv.shape[0] if _cp_active else state_slot_full.shape[0] # the store launches one program per kv row and never checks these lengths assert min(state_slot_full.shape[0], positions_full.shape[0]) >= n_real - runtime.store_swa_into_unified( + dsv4_unified_kv_store.store_swa_into_unified( kv=kv[:n_real], state_slot=state_slot_full[:n_real], positions=positions_full[:n_real], @@ -3218,7 +3221,7 @@ class DeepseekV4HipRadixBackend( cache = state.cache num_swa_rows = cache.swa_token_ids.shape[0] swa_rows = self.opus_prefill_workspace.get(num_swa_rows) - dequantize_k_cache_paged( + dequantize_k_cache_paged_dsv4( pool.get_swa_key_buffer_radix(layer_id), cache.swa_token_ids, page_size=cache.swa_page_size, @@ -3267,7 +3270,7 @@ class DeepseekV4HipRadixBackend( ) num_rows = gather.flat_token_ids.shape[0] rows = self.opus_compressed_workspaces[compress_ratio].get(num_rows) - dequantize_k_cache_paged( + dequantize_k_cache_paged_dsv4( c_cache, gather.flat_token_ids, page_size=c_page_size, @@ -3321,7 +3324,7 @@ class DeepseekV4HipRadixBackend( token_to_kv_pool = self.token_to_kv_pool assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_triton, ) diff --git a/python/sglang/srt/layers/attention/dsa/dsa_backend_mtp_precompute.py b/python/sglang/srt/layers/attention/dsa/dsa_backend_mtp_precompute.py index e967b0854e..817aa9f038 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_backend_mtp_precompute.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_backend_mtp_precompute.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Optional import torch -from sglang.kernels.ops.attention.utils import seqlens_expand_triton +from sglang.kernels.ops.attention.pad import seqlens_expand_triton from sglang.srt.layers.attention.dsa.utils import compute_dsa_seqlens from sglang.srt.utils import is_cuda, is_hip diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py index 91350f04df..e808efcfb1 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py @@ -14,7 +14,7 @@ from sglang.kernels.ops.attention.dsa.fp8_fused_writer_hip import ( fused_fp8_writer_geometry_supported, prepare_aiter_rope_caches, ) -from sglang.kernels.ops.attention.fused_store_index_cache import ( +from sglang.kernels.ops.kvcache.dsa_fused_store_index_cache import ( can_use_dsa_fused_store, fused_store_index_k_cache, ) @@ -183,7 +183,9 @@ if _is_cuda: fused_k_indexer_norm_rope, fused_k_indexer_norm_rope_store, ) - from sglang.kernels.ops.attention.dsv4 import fused_q_indexer_rope_first_quant + from sglang.kernels.ops.attention.dsa.indexer_q import ( + fused_q_indexer_rope_first_quant, + ) @register_custom_op(mutates_args=["topk_indices"]) @register_split_op() @@ -484,7 +486,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp): # Off rather than fatal: without the fused path both phases share a format. self.use_gfx950_fused_indexer = hadamard_preserved(self) if self.use_gfx950_fused_indexer: - from sglang.kernels.ops.attention.dsa.hip_gfx950 import ( + from sglang.srt.layers.attention.dsa.gfx950_fused_indexer import ( Gfx950FusedIndexer, prealloc_workspace, ) @@ -703,7 +705,7 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp): """Per-call half of the gate plus the four launches. Returns the (rows, index_topk) physical-slot tensor, or None to mean this call is not served here -- forward_cuda then continues into the standard path unchanged.""" - from sglang.kernels.ops.attention.dsa.hip_gfx950 import ( + from sglang.srt.layers.attention.dsa.gfx950_fused_indexer import ( CACHE_TOK_STRIDE, MAX_ROWS, PAGE_SIZE, @@ -1914,9 +1916,9 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp): return maybe_capture_indexer_topk(layer_id, topk_result) if _is_hip and not _is_gfx1250_supported: - from sglang.kernels.ops.attention.dsa.tilelang_kernel import act_quant + from sglang.kernels.ops.quantization.act_quant_tilelang import act_quant elif not _is_npu: - from sglang.kernels.ops.attention.dsa.triton_kernel import act_quant + from sglang.kernels.ops.quantization.act_quant_triton import act_quant if TYPE_CHECKING: assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool) diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py index 0a25452985..8609d5d1c2 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer_kpool.py @@ -1707,9 +1707,9 @@ class IndexerKPool(MultiPlatformOp): return_indices: bool = True, ) -> Optional[torch.Tensor]: if is_hip(): - from sglang.kernels.ops.attention.dsa.tilelang_kernel import act_quant + from sglang.kernels.ops.quantization.act_quant_tilelang import act_quant elif not is_npu(): - from sglang.kernels.ops.attention.dsa.triton_kernel import act_quant + from sglang.kernels.ops.quantization.act_quant_triton import act_quant if TYPE_CHECKING: assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool) diff --git a/python/sglang/srt/layers/attention/dsa/dsa_metadata_manager.py b/python/sglang/srt/layers/attention/dsa/dsa_metadata_manager.py index 1a84ccb509..98cce48d58 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_metadata_manager.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_metadata_manager.py @@ -6,7 +6,7 @@ import logging from functools import partial from typing import TYPE_CHECKING -from sglang.kernels.ops.attention.dsa_metadata import ( +from sglang.kernels.ops.attention.dsa.metadata import ( fused_dsa_decode_metadata, fused_dsa_draft_extend_metadata, fused_dsa_target_verify_metadata, @@ -50,14 +50,14 @@ class DSAMetadataManagementMixin: self._fused_verify_metadata = fused_dsa_target_verify_metadata self._fused_draft_extend_metadata = fused_dsa_draft_extend_metadata if self.experimental_kpool_metadata_fusion: - from sglang.kernels.ops.attention.dsa_kpool_metadata.decode import ( - fused_dsa_decode_metadata as decode, + from sglang.kernels.ops.attention.dsa.kpool_metadata.decode import ( + fused_dsa_decode_metadata_kpool as decode, ) - from sglang.kernels.ops.attention.dsa_kpool_metadata.draft_extend import ( - fused_dsa_draft_extend_metadata as draft_extend, + from sglang.kernels.ops.attention.dsa.kpool_metadata.draft_extend import ( + fused_dsa_draft_extend_metadata_kpool as draft_extend, ) - from sglang.kernels.ops.attention.dsa_kpool_metadata.verify import ( - fused_dsa_target_verify_metadata as verify, + from sglang.kernels.ops.attention.dsa.kpool_metadata.verify import ( + fused_dsa_target_verify_metadata_kpool as verify, ) self._fused_decode_metadata = partial( @@ -84,7 +84,7 @@ class DSAMetadataManagementMixin: # Use fused CUDA kernel for all copy operations if not _is_hip: try: - from sglang.kernels.ops.attention.fused_metadata_copy import ( + from sglang.kernels.ops.attention.dsa.fused_metadata_copy import ( fused_metadata_copy_cuda, ) diff --git a/python/sglang/srt/layers/attention/dsa/dsa_prefill_cuda_graph.py b/python/sglang/srt/layers/attention/dsa/dsa_prefill_cuda_graph.py index 9fcbf04ac2..851b411df6 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_prefill_cuda_graph.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_prefill_cuda_graph.py @@ -62,9 +62,9 @@ def pcg_dsa_indexer_prefill_split( "Internal error: DSA graph dispatch is only supported on CUDA and ROCm" ) if _is_hip and not _is_gfx1250_supported: - from sglang.kernels.ops.attention.dsa.tilelang_kernel import act_quant + from sglang.kernels.ops.quantization.act_quant_tilelang import act_quant else: - from sglang.kernels.ops.attention.dsa.triton_kernel import act_quant + from sglang.kernels.ops.quantization.act_quant_triton import act_quant if x_s is not None: x = (x, x_s) if x_bf16 is None else (x, x_s, x_bf16) diff --git a/python/sglang/kernels/ops/attention/dsa/hip_gfx950/fused_decode.py b/python/sglang/srt/layers/attention/dsa/gfx950_fused_indexer.py similarity index 100% rename from python/sglang/kernels/ops/attention/dsa/hip_gfx950/fused_decode.py rename to python/sglang/srt/layers/attention/dsa/gfx950_fused_indexer.py diff --git a/python/sglang/srt/layers/attention/dsa/utils.py b/python/sglang/srt/layers/attention/dsa/utils.py index 859d60e65c..4b0c4a8ecc 100644 --- a/python/sglang/srt/layers/attention/dsa/utils.py +++ b/python/sglang/srt/layers/attention/dsa/utils.py @@ -73,9 +73,9 @@ def gfx950_fused_indexer_runtime_ok() -> bool: """Whether this runtime can serve the gfx950 fused indexer: aiter with preshuffled paged MQA, and kernels that build. - Reached only on gfx950, since fused_decode.supported_hardware() is evaluated - first. Every decline here is therefore a configuration or toolchain error; - it is logged, as a warning when the path was asked for by name.""" + Reached only on gfx950, since gfx950_fused_indexer.supported_hardware() is + evaluated first. Every decline here is therefore a configuration or toolchain + error; it is logged, as a warning when the path was asked for by name.""" from sglang.srt.runtime_context import get_exec requested = get_exec().kernel.enable_dsa_fused_indexer @@ -91,9 +91,9 @@ def gfx950_fused_indexer_runtime_ok() -> bool: log("gfx950 fused DSA indexer disabled: %s", reason) return False - # No hardware term here: fused_decode.supported_hardware() is the hardware - # half of the gate and runs first, so anything reaching this point is - # already on gfx950. What is left is what a deployment can get wrong. + # No hardware term here: gfx950_fused_indexer.supported_hardware() is the + # hardware half of the gate and runs first, so anything reaching this point + # is already on gfx950. What is left is what a deployment can get wrong. if not get_bool_env_var("SGLANG_USE_AITER"): return _refuse("SGLANG_USE_AITER is not set") if not aiter_can_use_preshuffle_paged_mqa(): @@ -110,7 +110,9 @@ def gfx950_fused_indexer_runtime_ok() -> bool: def gfx950_model_shape_supported(**kwargs) -> bool: """Static per-model half of the gate: shapes and dtypes that cannot change after load.""" - from sglang.kernels.ops.attention.dsa.hip_gfx950 import model_shape_supported + from sglang.srt.layers.attention.dsa.gfx950_fused_indexer import ( + model_shape_supported, + ) return model_shape_supported(**kwargs) diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py index f98fbc8440..650923ed4b 100644 --- a/python/sglang/srt/layers/attention/dsa_backend.py +++ b/python/sglang/srt/layers/attention/dsa_backend.py @@ -28,24 +28,24 @@ from sglang.srt.runtime_context import ( logger = logging.getLogger(__name__) from sglang.kernels.ops.attention.dsa.dequant_k_cache import ( concat_cast_kv_fp8_pad, - dequantize_k_cache_paged, + dequantize_k_cache_paged_dsa, gather_cast_kv_fp8_pad_paged, - gather_dequant_requant_fp8_paged, + gather_dequant_requant_fp8_paged_dsa, ) -from sglang.kernels.ops.attention.dsa.quant_k_cache import quantize_k_cache from sglang.kernels.ops.attention.dsa.transform_index import ( prepare_trtllm_nope_sparse_metadata, transform_index_page_table_decode, transform_index_page_table_prefill, ) +from sglang.kernels.ops.attention.metadata import q8kv8_topk_length_from_indices +from sglang.kernels.ops.attention.mla_qk_prep import concat_and_cast_q_fp8_pad +from sglang.kernels.ops.attention.pad import seqlens_expand_triton from sglang.kernels.ops.attention.utils import ( concat_mla_absorb_q_general, mla_quantize_and_rope_for_fp8, mla_quantize_for_fp8_no_rope, - q8kv8_topk_length_from_indices, - seqlens_expand_triton, ) -from sglang.kernels.ops.kvcache.cache_ops import concat_and_cast_q_fp8_pad +from sglang.kernels.ops.quantization.dsa_quant_k_cache import quantize_k_cache from sglang.srt.configs.model_config import ( get_dsa_index_kpool, ) @@ -2414,7 +2414,7 @@ class DeepseekSparseAttnBackend( self.forward_metadata.page_table_1_flattened ) assert page_table_1_flattened is not None - kv_cache = dequantize_k_cache_paged( + kv_cache = dequantize_k_cache_paged_dsa( kv_cache, page_table_1_flattened ) else: @@ -3007,7 +3007,7 @@ class DeepseekSparseAttnBackend( * non-prefix extend: ``kv_bf16`` (the gathered bf16 KV) is cast into a zero-padded fp8 buffer. * prefix extend on an fp8 pool (656 B/token: nope_fp8 + per-group - scales + rope_bf16): ``gather_dequant_requant_fp8_paged`` gathers, + scales + rope_bf16): ``gather_dequant_requant_fp8_paged_dsa`` gathers, dequantizes per group and requantizes to per-tensor fp8 in one fused Triton kernel. * prefix extend on a bf16 pool: ``gather_cast_kv_fp8_pad_paged`` @@ -3130,7 +3130,7 @@ class DeepseekSparseAttnBackend( ) self._q8kv8_kv_buf = kv_buf if paged_kv_cache is not None and self.dsa_kv_cache_store_fp8: - kv_padded = gather_dequant_requant_fp8_paged( + kv_padded = gather_dequant_requant_fp8_paged_dsa( paged_kv_cache, page_table_1_flattened, extra_rows=topk, @@ -4026,7 +4026,7 @@ class DeepseekSparseAttnBackend( ): total_kv_tokens = forward_batch.seq_lens_sum total_q_tokens = forward_batch.extend_num_tokens - # Heuristic based on benchmarking flashmla_kv vs flashmla_sparse + dequantize_k_cache_paged + # Heuristic based on benchmarking flashmla_kv vs flashmla_sparse + dequantize_k_cache_paged_dsa if total_kv_tokens < total_q_tokens * 512: self.dsa_prefill_impl = "flashmla_sparse" return @@ -4186,7 +4186,7 @@ class DeepseekSparseAttnMultiStepBackend: # This is 3x faster than calling the single-backend copy 3 times if self.speculative_num_steps > 3: try: - from sglang.kernels.ops.attention.fused_metadata_copy import ( + from sglang.kernels.ops.attention.dsa.fused_metadata_copy import ( fused_metadata_copy_multi_cuda, ) diff --git a/python/sglang/srt/layers/attention/dsv4/compressor_trtllm.py b/python/sglang/srt/layers/attention/dsv4/compressor_trtllm.py index ecdcd2c464..0a47eca599 100644 --- a/python/sglang/srt/layers/attention/dsv4/compressor_trtllm.py +++ b/python/sglang/srt/layers/attention/dsv4/compressor_trtllm.py @@ -49,7 +49,7 @@ def forward_compress_uniform_fp8( layer_id: int, ) -> None: """Compress, normalize, apply RoPE, and store as uniform e4m3.""" - from sglang.kernels.ops.attention.deepseek_v4_rope import ( + from sglang.kernels.ops.attention.dsv4.rope import ( fused_norm_rope_inplace_triton, ) from sglang.srt.layers.attention.dsv4.compressor_v2 import ( diff --git a/python/sglang/srt/layers/attention/dsv4/compressor_v2.py b/python/sglang/srt/layers/attention/dsv4/compressor_v2.py index 3afb09f3ff..529d30335a 100644 --- a/python/sglang/srt/layers/attention/dsv4/compressor_v2.py +++ b/python/sglang/srt/layers/attention/dsv4/compressor_v2.py @@ -167,7 +167,7 @@ class CompressorBackendMixin: kv_score_input = compressor.compute_kv_score(x, forward_batch) state_pool = compressor.get_state_pool(self) - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_fp8, is_unified_kv_triton, ) @@ -272,7 +272,7 @@ class CompressorBackendMixin: layer_id: int, ) -> None: """HIP-specific forward path using PyTorch/Triton fallbacks.""" - from sglang.kernels.ops.attention.deepseek_v4_rope import ( + from sglang.kernels.ops.attention.dsv4.rope import ( fused_norm_rope_inplace_triton, ) from sglang.srt.layers.attention.nsa.nsa_indexer import rotate_activation diff --git a/python/sglang/srt/layers/attention/dsv4/dsv41_sparse.py b/python/sglang/srt/layers/attention/dsv4/dsv41_sparse.py index 7a32a5a107..6b08bd9dbd 100644 --- a/python/sglang/srt/layers/attention/dsv4/dsv41_sparse.py +++ b/python/sglang/srt/layers/attention/dsv4/dsv41_sparse.py @@ -11,13 +11,13 @@ from typing import Optional, Tuple import torch from torch import nn -from sglang.kernels.ops.attention.dsv4.torch_quant import ( - fake_quant_compressed_kv, - fake_quant_fp4, -) from sglang.kernels.ops.gemm.bf16_fp32 import linear_bf16_fp32 from sglang.kernels.ops.gemm.router_gemv_hip import rocm_gemv_split_k_max_tokens from sglang.kernels.ops.layernorm.rmsnorm_fp32 import rmsnorm_fp32 +from sglang.kernels.ops.quantization.fp4_fake_quant import ( + fake_quant_compressed_kv, + fake_quant_fp4, +) from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.utils import add_prefix, is_gfx95_supported diff --git a/python/sglang/srt/layers/attention/dsv4/low_ratio_backend_hip.py b/python/sglang/srt/layers/attention/dsv4/low_ratio_backend_hip.py index ac85618f3f..9634b76a8f 100644 --- a/python/sglang/srt/layers/attention/dsv4/low_ratio_backend_hip.py +++ b/python/sglang/srt/layers/attention/dsv4/low_ratio_backend_hip.py @@ -41,7 +41,6 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( aiter_has_fp4_rowgroup, index_q_rope_pack_flydsl, index_q_rope_pack_weights_flydsl, - indexer_head_weights, logits_rows_per_chunk, pack_fp4_query_flydsl, prepare_fp4_decode_workspace, @@ -55,6 +54,7 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope_hip import ( index_k_norm_rope_pack_store_split, ) +from sglang.kernels.ops.gemm.dsv4_indexer_head_weights_hip import indexer_head_weights from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k from sglang.srt.environ import envs from sglang.srt.layers.attention.deepseek_v4_backend import ( diff --git a/python/sglang/srt/layers/attention/dsv4/online_c128_mtp.py b/python/sglang/srt/layers/attention/dsv4/online_c128_mtp.py new file mode 100644 index 0000000000..762b2c941a --- /dev/null +++ b/python/sglang/srt/layers/attention/dsv4/online_c128_mtp.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, List, Optional + +import torch + +from sglang.kernels.ops.attention.dsv4.online_c128_mtp import ( + online_c128_mtp_commit_pending, + online_c128_mtp_mark_pending, + online_c128_mtp_write_prefix_states, +) +from sglang.srt.environ import envs + + +@dataclass +class _OnlineC128LayerRuntime: + head_dim: int + main_state: torch.Tensor + state_dtype: torch.dtype + state_slot_offset: int + + +@dataclass +class _OnlineC128VerifyContext: + req_pool_indices: torch.Tensor + seq_lens: torch.Tensor + + +class OnlineC128MTPController: + def __init__(self, backend: Any): + self.backend = backend + self._verify_ctx: Optional[_OnlineC128VerifyContext] = None + self._layer_runtimes: Optional[List[_OnlineC128LayerRuntime]] = None + + def enabled(self) -> bool: + return ( + envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get() + and envs.SGLANG_EXPERIMENTAL_ONLINE_C128_MTP.get() + and self.backend.mtp_enabled + ) + + def state_slot_offset(self) -> int: + if not self.enabled(): + return 0 + return self.backend.token_to_kv_pool.get_online_c128_mtp_state_slot_offset() + + def begin_verify( + self, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + ) -> None: + if not self.enabled(): + self.clear() + return + + self._verify_ctx = _OnlineC128VerifyContext( + req_pool_indices=req_pool_indices.detach(), + seq_lens=seq_lens.detach(), + ) + head_dim = self._head_dim() + state_dtype = self._state_dtype() + if head_dim is None or state_dtype is None or self._num_verify_tokens() == 0: + return + token_to_kv_pool = self.backend.token_to_kv_pool + online_c128_mtp_mark_pending( + seq_lens, + req_pool_indices, + token_to_kv_pool.get_online_c128_mtp_pending_seq_lens(), + min(seq_lens.shape[0], req_pool_indices.shape[0]), + token_to_kv_pool.get_online_c128_state_num_req_slots(), + head_dim=head_dim, + state_dtype=state_dtype, + ) + + def clear(self) -> None: + self._verify_ctx = None + + def prepare_forward( + self, + logical_forward_mode, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + *, + verify_bs: Optional[int] = None, + ) -> int: + if not self.enabled(): + self.clear() + return 0 + if logical_forward_mode is None or logical_forward_mode.is_idle(): + self.clear() + return 0 + + active_req_pool_indices = req_pool_indices + active_seq_lens = seq_lens + if logical_forward_mode.is_target_verify(): + if verify_bs is None: + verify_bs = req_pool_indices.shape[0] + active_req_pool_indices = req_pool_indices[:verify_bs] + active_seq_lens = seq_lens[:verify_bs] + if verify_bs == 0: + self.clear() + return 0 + + self.commit_pending( + req_pool_indices=active_req_pool_indices, + seq_lens=active_seq_lens, + ) + if not logical_forward_mode.is_target_verify(): + return 0 + + self.begin_verify( + req_pool_indices=active_req_pool_indices, + seq_lens=active_seq_lens, + ) + return self.state_slot_offset() + + def write_prefix_states( + self, + layer_id: int, + compressor: Any, + kv_score_input: torch.Tensor, + logical_forward_mode, + ) -> None: + if ( + not self.enabled() + or logical_forward_mode is None + or not logical_forward_mode.is_target_verify() + or compressor.is_in_indexer + or compressor.ratio != 128 + or kv_score_input.numel() == 0 + ): + return + + ctx = self._active_ctx() + num_verify_tokens = self._num_verify_tokens() + if ctx is None or num_verify_tokens == 0: + return + + token_to_kv_pool = self.backend.token_to_kv_pool + head_dim = compressor.head_dim + state_pool = token_to_kv_pool.get_attention_compress_states(layer_id) + state = state_pool.kv_score_buffer.kv_score + total_bs = kv_score_input.numel() // (num_verify_tokens * head_dim * 2) + layer_bs = min(ctx.seq_lens.shape[0], ctx.req_pool_indices.shape[0], total_bs) + if layer_bs <= 0: + return + + online_c128_mtp_write_prefix_states( + kv_score_input, + ctx.seq_lens, + ctx.req_pool_indices, + self.backend.req_to_token, + compressor.ape.reshape(128, head_dim), + state, + layer_bs, + num_verify_tokens, + state_pool.online_mtp_state_slot_offset, + head_dim=head_dim, + ) + + def commit_pending( + self, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + ) -> None: + if self._verify_ctx is None: + return + if not self.enabled(): + self.clear() + return + if req_pool_indices.numel() == 0 or seq_lens.numel() == 0: + return + + num_verify_tokens = self._num_verify_tokens() + if num_verify_tokens == 0: + self.clear() + return + + backend = self.backend + token_to_kv_pool = backend.token_to_kv_pool + pending_seq_lens = token_to_kv_pool.get_online_c128_mtp_pending_seq_lens() + cur_bs = min(seq_lens.shape[0], req_pool_indices.shape[0]) + + for runtime in self._iter_layer_runtimes(): + online_c128_mtp_commit_pending( + seq_lens, + req_pool_indices, + backend.req_to_token, + pending_seq_lens, + runtime.main_state, + cur_bs, + num_verify_tokens, + runtime.state_slot_offset, + token_to_kv_pool.get_online_c128_state_num_req_slots(), + head_dim=runtime.head_dim, + ) + + self.clear() + + def _num_verify_tokens(self) -> int: + if not self.enabled(): + return 0 + num_verify_tokens = int(self.backend.speculative_num_draft_tokens) + max_draft_tokens = ( + self.backend.token_to_kv_pool.get_online_c128_mtp_max_draft_tokens() + ) + return num_verify_tokens if 0 < num_verify_tokens <= max_draft_tokens else 0 + + def _active_ctx(self) -> Optional[_OnlineC128VerifyContext]: + ctx = self._verify_ctx + if ( + ctx is None + or ctx.seq_lens.numel() == 0 + or ctx.req_pool_indices.numel() == 0 + ): + return None + return ctx + + def _head_dim(self) -> Optional[int]: + for runtime in self._iter_layer_runtimes(): + return runtime.head_dim + return None + + def _state_dtype(self) -> Optional[torch.dtype]: + for runtime in self._iter_layer_runtimes(): + return runtime.state_dtype + return None + + def _iter_layer_runtimes(self): + if self._layer_runtimes is None: + runtimes = [] + token_to_kv_pool = self.backend.token_to_kv_pool + for layer in self.backend.model_runner.model.model.layers: + attn = getattr(layer, "self_attn", None) + compressor = getattr(attn, "compressor", None) + if compressor is None or compressor.ratio != 128: + continue + state_pool = token_to_kv_pool.get_attention_compress_states( + compressor.layer_id + ) + runtimes.append( + _OnlineC128LayerRuntime( + head_dim=compressor.head_dim, + main_state=state_pool.kv_score_buffer.kv_score, + state_dtype=state_pool.kv_score_buffer.kv_score.dtype, + state_slot_offset=state_pool.online_mtp_state_slot_offset, + ) + ) + self._layer_runtimes = runtimes + return iter(self._layer_runtimes) diff --git a/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py b/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py index 241d2ba09e..5478a8290d 100644 --- a/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py +++ b/python/sglang/srt/layers/attention/dsv4/sparse_prefill_utils.py @@ -37,24 +37,20 @@ from dataclasses import dataclass, field from typing import Dict, Optional import torch -import triton from sglang.kernels.ops.attention.dsv4.dequant_k_cache import DIM_NOPE, DIM_ROPE -from sglang.srt.utils import ceil_align from sglang.srt.utils.common import async_h2d -# FlashMLA sparse prefill asserts ``params.topk % B_TOPK == 0``. B_TOPK is 64 -# for the h_q=64 kernel and 128 for h_q=128; pad to 128 to satisfy both. -SPARSE_PREFILL_TOPK_ALIGNMENT = 128 -# Bf16 workspace per-token width, matching ``dequantize_k_cache_paged``'s +# Bf16 workspace per-token width, matching ``dequantize_k_cache_paged_dsv4``'s # output: 448 fp8 nope (dequanted) + 64 bf16 rope = 512. WORKSPACE_DIM = DIM_NOPE + DIM_ROPE DSV4_Q8KV8_PREFILL_ENV = "SGLANG_DSV4_Q8KV8_PREFILL" DSV4_Q8KV8_PREFILL_LOG_ENV = "SGLANG_DSV4_Q8KV8_PREFILL_LOG" from sglang.kernels.ops.attention.dsv4.sparse_prefill_kernels import ( - _build_swa_token_ids_kernel, - _combine_topk_swa_indices_kernel, + build_swa_token_ids, + combine_topk_swa_indices, + combined_topk_width, ) @@ -106,184 +102,6 @@ class SparsePrefillWorkspace: return self._buffer[:num_tokens] -def combined_topk_width(topk: int, window_size: int) -> int: - """Width of the padded combined_indices last dim that - ``combine_topk_swa_indices`` would produce for these args.""" - return ceil_align(topk + window_size, SPARSE_PREFILL_TOPK_ALIGNMENT) - - -def combine_topk_swa_indices( - topk_indices: torch.Tensor, - query_start_loc: torch.Tensor, - query_pos: torch.Tensor, - seq_lens: torch.Tensor, - gather_lens: torch.Tensor, - compressed_base: torch.Tensor, - swa_base: torch.Tensor, - window_size: int, - compress_ratio: int, - topk: int, - out_indices: Optional[torch.Tensor] = None, - out_lens: Optional[torch.Tensor] = None, - swa_indices: Optional[torch.Tensor] = None, - swa_lengths: Optional[torch.Tensor] = None, -) -> tuple[torch.Tensor, torch.Tensor]: - """Combine top-k and SWA indices for flash_mla_sparse_fwd. - - Top-k indices are int32 [num_tokens, K] in each request's compressed region. - Invalid entries within topk_len must be -1; later entries are ignored. - query_start_loc can include a cross-chunk offset; query_pos is absolute. - compressed_base and swa_base address the flat workspace. SWA-only layers use - topk == 0, but compress_ratio must still be positive. - Optional SWA indices are relative to swa_base and preserve replay floors. - - Returns int32 indices [num_tokens, padded_topk_swa] and per-token scanned-prefix - lengths, including -1 entries skipped by attention. Width is padded to 128. - Rows are written whole and -1 padded, so preallocated out_indices / out_lens - are overwritten entirely; rows past the last request are -1 with length 0. - """ - assert topk_indices.dtype == torch.int32 - assert query_start_loc.dtype == torch.int32 - assert query_pos.dtype == torch.int32 - assert seq_lens.dtype == torch.int32 - assert gather_lens.dtype == torch.int32 - assert compressed_base.dtype == torch.int32 - assert swa_base.dtype == torch.int32 - assert compress_ratio >= 1, "compress_ratio must be >= 1 (use topk=0 for SWA-only)" - assert topk_indices.shape[-1] >= topk, ( - f"topk_indices width {topk_indices.shape[-1]} must be >= topk {topk}" - ) - - num_tokens = topk_indices.shape[0] - if swa_indices is not None: - assert swa_indices.shape == (num_tokens, window_size) - assert swa_lengths is not None and swa_lengths.shape == (num_tokens,) - assert swa_indices.dtype == swa_lengths.dtype == torch.int32 - num_reqs = seq_lens.shape[0] - combined_topk = combined_topk_width(topk, window_size) - if out_indices is None: - # The kernel writes whole -1 padded rows. - combined_indices = torch.empty( - (num_tokens, combined_topk), - dtype=torch.int32, - device=topk_indices.device, - ) - else: - assert out_indices.shape == (num_tokens, combined_topk) - assert out_indices.dtype == torch.int32 - combined_indices = out_indices - if out_lens is None: - combined_lens = torch.empty( - num_tokens, dtype=torch.int32, device=topk_indices.device - ) - else: - assert out_lens.shape == (num_tokens,) - assert out_lens.dtype == torch.int32 - combined_lens = out_lens - - if num_tokens == 0: - return combined_indices, combined_lens - block_t = 2 # Measured on B200 with num_warps=2, 16384 tokens, bs 1 to 64. - _combine_topk_swa_indices_kernel[(triton.cdiv(num_tokens, block_t),)]( - combined_indices, - combined_indices.stride(0), - combined_lens, - topk_indices, - topk_indices.stride(0), - query_start_loc, - query_pos, - seq_lens, - gather_lens, - compressed_base, - swa_base, - swa_indices, - 0 if swa_indices is None else swa_indices.stride(0), - swa_lengths, - num_tokens, - num_reqs, - top_k=topk, - EXPLICIT_SWA=swa_indices is not None, - COMPRESS_RATIO=compress_ratio, - WINDOW_SIZE=window_size, - WIDTH=combined_topk, - PADDED_WIDTH=triton.next_power_of_2(combined_topk), - BLOCK_T=block_t, - SEARCH_STEPS=max(num_reqs, 1).bit_length(), - num_warps=2, - ) - return combined_indices, combined_lens - - -def build_swa_token_ids( - seq_lens: torch.Tensor, - extend_seq_lens: torch.Tensor, - req_pool_indices: torch.Tensor, - req_to_token: torch.Tensor, - full_to_swa: torch.Tensor, - swa_window: int, - total_swa: int, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """Build a flat list of physical SWA-cache token IDs covering each - request's positional union of every query's SWA window. - - Per request, the union spans seq positions - ``[max(0, seq_len - extend - W + 1), seq_len)``, of length - ``min(seq_len, extend + W - 1)``. Each position is translated through - ``req_to_token`` (full kv-cache id) and then ``full_to_swa`` (SWA - cache id) to land in the SWA-cache token-id space that - ``dequantize_k_cache_paged`` consumes. - - Args: - seq_lens: (num_reqs,) int32, per-request total sequence length. - extend_seq_lens: (num_reqs,) int32, per-request query length. - req_pool_indices: (num_reqs,) int32, per-request row in - ``req_to_token``. - req_to_token: (num_reqs_max, max_seq_len) int32. Full kv-cache id - per (request, seq position). - full_to_swa: (full_pool_size + extra,) int64. Maps full kv id to - SWA-cache id. - swa_window: int. SWA window size. - total_swa: Number of token IDs to allocate, computed from CPU lengths. - - Returns: - swa_token_ids: (total_swa,) int32, flat physical SWA-cache token IDs. - swa_first_pos: (num_reqs,) int32, first seq position covered per req. - swa_gather_lens: (num_reqs,) int32, gather length per request. - swa_offsets: (num_reqs+1,) int32, exclusive cumsum of swa_gather_lens. - """ - assert seq_lens.dtype == torch.int32 - assert extend_seq_lens.dtype == torch.int32 - assert req_pool_indices.dtype == torch.int32 - assert req_to_token.dtype == torch.int32 - assert full_to_swa.dtype == torch.int64 - - num_reqs = seq_lens.shape[0] - device = seq_lens.device - - swa_gather_lens = torch.minimum(seq_lens, extend_seq_lens + (swa_window - 1)).to( - torch.int32 - ) - swa_first_pos = (seq_lens - swa_gather_lens).to(torch.int32) - swa_offsets = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device) - swa_offsets[1:] = torch.cumsum(swa_gather_lens, dim=0).to(torch.int32) - swa_token_ids = torch.empty(total_swa, dtype=torch.int32, device=device) - if total_swa == 0: - return swa_token_ids, swa_first_pos, swa_gather_lens, swa_offsets - - NUM_WORKERS = 128 - _build_swa_token_ids_kernel[(num_reqs, NUM_WORKERS)]( - swa_token_ids, - swa_first_pos, - swa_gather_lens, - swa_offsets, - req_pool_indices, - req_to_token, - req_to_token.stride(0), - full_to_swa, - ) - return swa_token_ids, swa_first_pos, swa_gather_lens, swa_offsets - - @dataclass class CompressedGather: """Positional layout of one compressed cache inside the workspace.""" diff --git a/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/env_gate.py b/python/sglang/srt/layers/attention/dsv4/unified_kv_gate.py similarity index 100% rename from python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/env_gate.py rename to python/sglang/srt/layers/attention/dsv4/unified_kv_gate.py diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py index 4423b2033c..a6ddca3f92 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/scoring.py @@ -88,7 +88,7 @@ class DeepGEMMPrefillData(msgspec.Struct, frozen=True): def quantize_index_q(q: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: - from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( + from sglang.kernels.ops.quantization.fp4_indexer_quant import ( quantize_fp4_indexer_tensor, ) diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index 7177b153eb..0b4ecf1884 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -15,10 +15,10 @@ from sglang.kernels.ops.attention.suffix_attention_merge import ( can_use_fused_suffix_attention_merge, merge_suffix_attention_in_place, ) -from sglang.kernels.ops.attention.utils import assert_buffer_fits -from sglang.kernels.ops.kvcache.trtllm_mha_page_table import ( +from sglang.kernels.ops.attention.trtllm_mha_page_table import ( build_trtllm_mha_page_table, ) +from sglang.kernels.ops.attention.utils import assert_buffer_fits from sglang.srt.configs.model_config import AttentionArch from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.attention.kv_shard_hooks import ( diff --git a/python/sglang/srt/layers/attention/flashmla_backend.py b/python/sglang/srt/layers/attention/flashmla_backend.py index daa7c83e32..4dbc30a2bd 100644 --- a/python/sglang/srt/layers/attention/flashmla_backend.py +++ b/python/sglang/srt/layers/attention/flashmla_backend.py @@ -12,7 +12,7 @@ import torch import triton from sgl_kernel.flash_mla import flash_mla_with_kvcache, get_mla_metadata -from sglang.kernels.ops.attention.utils import ( +from sglang.kernels.ops.attention.kv_indices import ( create_flashmla_kv_indices_triton, get_num_kv_index_blocks_flashmla, ) diff --git a/python/sglang/srt/layers/attention/hip_flash_mla.py b/python/sglang/srt/layers/attention/hip_flash_mla.py index 432b13a9a1..e6342d6982 100644 --- a/python/sglang/srt/layers/attention/hip_flash_mla.py +++ b/python/sglang/srt/layers/attention/hip_flash_mla.py @@ -156,7 +156,7 @@ def _apply_inverse_rope( ) -> None: """The model's standalone inverse RoPE on out [n, h, d] (last 64 dims of every head), for the paths that did not fold it into their combine.""" - from sglang.kernels.ops.attention.dsv4.elementwise import fused_rope_inplace + from sglang.kernels.ops.attention.dsv4.norm_rope import fused_rope_inplace freqs_real, positions = inv_rope n = out.shape[0] @@ -221,14 +221,14 @@ def flash_mla_with_kvcache_entrypoint(backend: str, **kwargs): return flash_mla_with_kvcache_torch(**kwargs) if backend == "tilelang": - from sglang.kernels.ops.attention.dsa.tilelang_kernel import ( + from sglang.kernels.ops.attention.dsv4.tilelang_fp8_attention import ( dpsk_v4_fp8_attention_fwd, ) return dpsk_v4_fp8_attention_fwd(**kwargs) if backend == "triton": - from sglang.kernels.ops.attention.nsa_triton_decode import ( + from sglang.kernels.ops.attention.dsv4.triton_sparse_decode import ( triton_fp8_attention_fwd, ) diff --git a/python/sglang/srt/layers/attention/hpc_ops_backend.py b/python/sglang/srt/layers/attention/hpc_ops_backend.py index e9a3a8483f..af37169c88 100644 --- a/python/sglang/srt/layers/attention/hpc_ops_backend.py +++ b/python/sglang/srt/layers/attention/hpc_ops_backend.py @@ -34,10 +34,10 @@ from typing import TYPE_CHECKING, Optional import msgspec import torch -from sglang.kernels.ops.kvcache.trtllm_mha_graph_metadata import ( +from sglang.kernels.ops.attention.trtllm_mha_graph_metadata import ( update_trtllm_mha_graph_metadata, ) -from sglang.kernels.ops.kvcache.trtllm_mha_page_table import ( +from sglang.kernels.ops.attention.trtllm_mha_page_table import ( build_trtllm_mha_page_table, ) from sglang.srt.layers.attention.base_attn_backend import AttentionBackend diff --git a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py index 80a4d1f35e..89121f76f6 100644 --- a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py @@ -1560,7 +1560,7 @@ class HybridLinearAttnBackend(AttentionBackend): # accept_lens == last_correct_step_indices + 1. mamba_pool = req_pool.mamba_pool if getattr(mamba_pool, "replayssm_is_kda", False): - from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( + from sglang.kernels.ops.attention.linear.kda_replayssm_spec_decode import ( commit_kda_replayssm_after_verify, ) diff --git a/python/sglang/srt/layers/attention/linear/gdn_backend.py b/python/sglang/srt/layers/attention/linear/gdn_backend.py index 16f92f4ff3..71d94aeab3 100644 --- a/python/sglang/srt/layers/attention/linear/gdn_backend.py +++ b/python/sglang/srt/layers/attention/linear/gdn_backend.py @@ -3,7 +3,7 @@ from typing import Optional, Tuple, Union import msgspec import torch -from sglang.kernels.ops.attention.fla.fused_gdn_gating import fused_gdn_gating +from sglang.kernels.ops.attention.linear.fused_gdn_gating import fused_gdn_gating from sglang.kernels.ops.mamba.causal_conv1d_triton import ( CAUSAL_CONV1D_FWD_BLOCK_M, causal_conv1d_fn, @@ -313,15 +313,15 @@ class GDNKernelDispatcher: ) cutedsl_kernel = CuteDSLGDNKernel() - # The CuteDSL prefill kernel only exists on SM100+ (Blackwell). - # On SM90 (Hopper) fall back to Triton so users can pick + # The CuteDSL prefill kernel only exists on SM10x. Elsewhere + # (SM90, SM120) fall back to Triton so users can pick # `cutedsl` uniformly across hardware. if cutedsl_kernel.supports_prefill: self.extend_kernel = cutedsl_kernel else: rank0_log( "CuTe DSL GDN prefill is not supported on this GPU " - "(requires SM100+). Falling back to Triton for prefill." + "(requires SM10x). Falling back to Triton for prefill." ) self.extend_kernel = triton_kernel elif prefill_backend.is_flashinfer(): @@ -1378,7 +1378,7 @@ class GDNAttnBackend(MambaAttnBackendBase): into ``temporal``. Uses the vendored CuTe DSL MTP kernel when the dispatcher selected the FlashInfer bf16-state verify, else the Triton recurrent kernel (both store the same raw window).""" - from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( + from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, ) @@ -1468,7 +1468,7 @@ class GDNAttnBackend(MambaAttnBackendBase): ([slots, HV, K, V]) is consumed directly as the kernel's [slots, HV, V, K] checkpoint. """ - from sglang.kernels.ops.attention.fla.gdn_replayssm_spec_decode import ( + from sglang.kernels.ops.attention.linear.gdn_replayssm_spec_decode import ( gdn_replayssm_spec_decode, ) diff --git a/python/sglang/srt/layers/attention/linear/kda_backend.py b/python/sglang/srt/layers/attention/linear/kda_backend.py index 0632858f2f..29cb57b9aa 100644 --- a/python/sglang/srt/layers/attention/linear/kda_backend.py +++ b/python/sglang/srt/layers/attention/linear/kda_backend.py @@ -224,7 +224,7 @@ class KDAKernelDispatcher: # SM100 chunk prefill pipeline. self.extend_kernel = cutedsl_kernel else: - # CuTe DSL prefill kernels need SM100 (Blackwell); on older GPUs + # CuTe DSL prefill kernels need SM10x (tcgen05); elsewhere # fall back to the Triton chunk kernel. self.extend_kernel = triton_kernel rank0_log( @@ -531,7 +531,7 @@ class KDAAttnBackend(MambaAttnBackendBase): and verify_backend.is_triton() and self.kernel_dispatcher.verify_kernel.supports_fused_chain_verify ): - from sglang.kernels.ops.attention.fla.fused_kda_conv_recurrent_verify import ( + from sglang.kernels.ops.attention.linear.fused_kda_conv_recurrent_verify import ( fused_kda_conv_gating_verify, ) diff --git a/python/sglang/srt/layers/attention/linear/kda_prefill_graph.py b/python/sglang/srt/layers/attention/linear/kda_prefill_graph.py index 636d6d9c77..f1eb37360b 100644 --- a/python/sglang/srt/layers/attention/linear/kda_prefill_graph.py +++ b/python/sglang/srt/layers/attention/linear/kda_prefill_graph.py @@ -33,17 +33,17 @@ import numpy as np import torch import triton -from sglang.kernels.ops.attention.fla.kda_prefill_graph_track import ( - TRACK_CHUNK_STATE, - TRACK_FINAL_STATE, - kda_track_conv_window, - kda_track_ssm_state, -) from sglang.kernels.ops.mamba.causal_conv1d_triton import ( CAUSAL_CONV1D_FWD_BLOCK_M, PAD_SLOT_ID, causal_conv1d_fn, ) +from sglang.kernels.ops.mamba.kda_prefill_track import ( + TRACK_CHUNK_STATE, + TRACK_FINAL_STATE, + kda_track_conv_window, + kda_track_ssm_state, +) from sglang.srt.layers.attention.mamba.prefill_track_metadata import ( build_prefill_track_plan, ) diff --git a/python/sglang/srt/layers/attention/linear/kernels/gdn_cutedsl.py b/python/sglang/srt/layers/attention/linear/kernels/gdn_cutedsl.py index 46f865d021..088e6f1dcd 100644 --- a/python/sglang/srt/layers/attention/linear/kernels/gdn_cutedsl.py +++ b/python/sglang/srt/layers/attention/linear/kernels/gdn_cutedsl.py @@ -4,7 +4,7 @@ Decode path uses the existing ``cutedsl_fused_sigmoid_gating_delta_rule_update`` (works on SM90+). Prefill (extend) path uses the ported vLLM SM100 chunkwise kernel -(``chunk_gated_delta_rule_cutedsl``). Requires SM100+ and ``head_k_dim == 128``. +(``chunk_gated_delta_rule_cutedsl``). Requires SM10x and ``head_k_dim == 128``. """ import logging @@ -22,12 +22,15 @@ from sglang.srt.layers.attention.linear.kernels.kernel_backend import ( logger = logging.getLogger(__name__) -def _is_blackwell() -> bool: - """True iff running on SM100+ (Blackwell) where the ported kernel is valid.""" +def _is_sm10x() -> bool: + """True iff running on SM10x, the only arch with the tcgen05/TMEM the kernel uses. + + SM120 (desktop Blackwell) lacks tcgen05 despite its larger capability number. + """ if not torch.cuda.is_available(): return False major, _ = torch.cuda.get_device_capability() - return major >= 10 + return major == 10 class CuteDSLGDNKernel(LinearAttnKernelBase): @@ -35,15 +38,15 @@ class CuteDSLGDNKernel(LinearAttnKernelBase): Decode: ``cutedsl_fused_sigmoid_gating_delta_rule_update`` (SM90+). Extend (prefill): chunkwise ``chunk_gated_delta_rule_cutedsl`` - (SM100+ only, ``head_k_dim`` must be 128). On SM90 the prefill path is + (SM10x only, ``head_k_dim`` must be 128). Elsewhere the prefill path is unsupported; callers should query :attr:`supports_prefill` and fall back to another backend (e.g. Triton). """ def __init__(self): - # The Blackwell extend kernel uses tcgen05/TMA-bulk-swizzle features - # that don't exist on SM90. The decode kernel does work on SM90+. - self.supports_prefill = _is_blackwell() + # The extend kernel uses tcgen05/TMA-bulk-swizzle features that exist + # only on SM10x. The decode kernel does work on SM90+. + self.supports_prefill = _is_sm10x() # Heavy CuteDSL imports are deferred to extend() so SM90 boxes can # still construct the kernel just for decode. @@ -61,7 +64,7 @@ class CuteDSLGDNKernel(LinearAttnKernelBase): else -1 ) raise RuntimeError( - f"CuTe DSL GDN prefill requires SM100+ (Blackwell); got SM{major}." + f"CuTe DSL GDN prefill requires SM10x (tcgen05); got SM{major}." ) if head_k_dim != 128: raise RuntimeError( diff --git a/python/sglang/srt/layers/attention/linear/kernels/gdn_triton.py b/python/sglang/srt/layers/attention/linear/kernels/gdn_triton.py index 0e5c0e10ca..1908031013 100644 --- a/python/sglang/srt/layers/attention/linear/kernels/gdn_triton.py +++ b/python/sglang/srt/layers/attention/linear/kernels/gdn_triton.py @@ -10,10 +10,10 @@ if not is_cpu(): from sglang.kernels.ops.attention.fla.fused_recurrent import ( fused_recurrent_gated_delta_rule_packed_decode, ) - from sglang.kernels.ops.attention.fla.fused_recurrent_linear_replayssm import ( + from sglang.kernels.ops.attention.linear.fused_recurrent_linear_replayssm import ( fused_recurrent_gdn_replayssm_decode, ) - from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( + from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, ) diff --git a/python/sglang/srt/layers/attention/linear/kernels/kda_cutedsl.py b/python/sglang/srt/layers/attention/linear/kernels/kda_cutedsl.py index b2ff3643d4..83a671afc2 100644 --- a/python/sglang/srt/layers/attention/linear/kernels/kda_cutedsl.py +++ b/python/sglang/srt/layers/attention/linear/kernels/kda_cutedsl.py @@ -13,27 +13,30 @@ from sglang.srt.layers.attention.linear.kernels.kernel_backend import ( logger = logging.getLogger(__name__) -def _is_blackwell() -> bool: - """True iff running on SM100+ (Blackwell), where the chunk prefill kernels run.""" +def _is_sm10x() -> bool: + """True iff on SM10x, the only arch with the tcgen05/TMEM the chunk prefill uses. + + SM120 (desktop Blackwell) lacks tcgen05 despite its larger capability number. + """ if not torch.cuda.is_available(): return False major, _ = torch.cuda.get_device_capability() - return major >= 10 + return major == 10 class CuteDSLKDAKernel(LinearAttnKernelBase): """CuTe DSL kernel for KDA. Decode: ``cutedsl_fused_sigmoid_gating_kda_update`` (SM90+). - Extend (prefill): SM100 chunk pipeline ``chunk_kda_cutedsl`` (SM100+ only, - ``head_k_dim`` must be 128). On SM90 the prefill path is unsupported; callers + Extend (prefill): SM100 chunk pipeline ``chunk_kda_cutedsl`` (SM10x only, + ``head_k_dim`` must be 128). Elsewhere the prefill path is unsupported; callers query :attr:`supports_prefill` and fall back to Triton. """ supports_safe_gate: bool = False def __init__(self): - self.supports_prefill = _is_blackwell() + self.supports_prefill = _is_sm10x() self._extend_fn: Optional[callable] = None self._l2norm_fn: Optional[callable] = None @@ -47,7 +50,7 @@ class CuteDSLKDAKernel(LinearAttnKernelBase): else -1 ) raise RuntimeError( - f"CuTe DSL KDA prefill requires SM100+ (Blackwell); got SM{major}." + f"CuTe DSL KDA prefill requires SM10x (tcgen05); got SM{major}." ) if head_k_dim != 128: raise RuntimeError( diff --git a/python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer.py b/python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer.py index cb1e6bdcbc..ec448783a3 100644 --- a/python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer.py +++ b/python/sglang/srt/layers/attention/linear/kernels/kda_flashinfer.py @@ -1,7 +1,7 @@ """FlashInfer KDA decode/verify wrapper. -Wraps ``flashinfer.kda_decode.recurrent_kda`` (SM100 / Blackwell). KDA prefill -uses a separate adapter in ``kda_flashinfer_prefill``. +Wraps ``flashinfer.kda_decode.recurrent_kda`` (SM100+). KDA prefill uses a +separate SM100/SM103-only adapter in ``kda_flashinfer_prefill``. Contract with the Triton KDA reference: - raw per-K gate ``a`` is activated in-kernel as @@ -44,7 +44,8 @@ def _get_flashinfer_kda_kernel(): from flashinfer.kda_decode import recurrent_kda _flashinfer_recurrent_kda = recurrent_kda - # recurrent_kda is SM100-only (CuTe DSL, Blackwell). + # SM100/SM103 get frozen fast paths; the generic CuTe DSL path + # also runs on SM120. _flashinfer_kda_available = ( is_cuda() and torch.cuda.get_device_capability()[0] >= 10 ) @@ -85,14 +86,14 @@ def build_fused_accept_indices( class FlashInferKDAKernel(LinearAttnKernelBase): - """FlashInfer KDA kernel: SM100 decode + MTP (target_verify), topk=1.""" + """FlashInfer KDA kernel: decode + MTP (target_verify), topk=1.""" def __init__(self): available, self._recurrent_kda = _get_flashinfer_kda_kernel() if not available or self._recurrent_kda is None: raise RuntimeError( "FlashInfer KDA kernel (recurrent_kda) is not available. " - "Requires SM100 (Blackwell) and a FlashInfer build with KDA support." + "Requires SM100+ and a FlashInfer build with KDA support." ) # Cache the per-layer constant gate-param prep (A_log/dt_bias reshape+cast), # keyed by tensor identity. Layer params are persistent weights so id() is diff --git a/python/sglang/srt/layers/attention/linear/kernels/kda_triton.py b/python/sglang/srt/layers/attention/linear/kernels/kda_triton.py index c035de4ac9..a4aa5e24c9 100644 --- a/python/sglang/srt/layers/attention/linear/kernels/kda_triton.py +++ b/python/sglang/srt/layers/attention/linear/kernels/kda_triton.py @@ -11,13 +11,13 @@ if not is_cpu(): from sglang.kernels.ops.attention.fla.fused_recurrent import ( fused_recurrent_kda_packed_decode, ) - from sglang.kernels.ops.attention.fla.fused_recurrent_linear_replayssm import ( + from sglang.kernels.ops.attention.fla.kda import chunk_kda + from sglang.kernels.ops.attention.linear.fused_recurrent_linear_replayssm import ( fused_recurrent_linear_replayssm_decode, ) - from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( + from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, ) - from sglang.kernels.ops.attention.fla.kda import chunk_kda class TritonKDAKernel(LinearAttnKernelBase): diff --git a/python/sglang/srt/layers/attention/linear/lightning_backend.py b/python/sglang/srt/layers/attention/linear/lightning_backend.py index 306953d1d3..cffa50d8fa 100644 --- a/python/sglang/srt/layers/attention/linear/lightning_backend.py +++ b/python/sglang/srt/layers/attention/linear/lightning_backend.py @@ -4,7 +4,7 @@ import math import torch from sglang.kernels.ops.attention.linear.lightning_attn import ( - BailingLinearKernel, + lightning_attention, linear_decode_forward_triton, ) from sglang.kernels.ops.attention.linear.seg_la import SegLaMeta, seg_la_fwd @@ -23,6 +23,46 @@ from sglang.srt.runtime_context import ( logger = logging.getLogger(__name__) +class BailingLinearKernel: + """ + Linear attention kernel implementation for Bailing models. + + This class is adapted from MiniMaxText01LinearKernel in vllm: + https://github.com/vllm-project/vllm/blob/a9138e85b14047e06300685b48e3485b995425fb/vllm/model_executor/models/minimax_text_01.py#L289 + + The implementation maintains the same functionality while being renamed to + match our Bailing model naming convention. + """ + + @staticmethod + def jit_linear_forward_prefix( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + kv_caches: torch.Tensor, + slope_rate: torch.Tensor, + block_size: int, + layer_idx: int = None, + **kwargs, + ) -> torch.Tensor: + + slope_rate = slope_rate.to(torch.float32) + should_pad_dim = q.dim() == 3 + if should_pad_dim: + q = q.unsqueeze(0) + k = k.unsqueeze(0) + v = v.unsqueeze(0) + b, h, n, d = q.shape + e = d + kv_history = kv_caches.reshape(1, h, d, e).contiguous() + output, kv_history = lightning_attention( + q, k, v, slope_rate, block_size=block_size, kv_history=kv_history + ) + kv_caches.copy_(kv_history[:, :, -1, :, :].reshape(h, d, e)) + assert output.shape[0] == 1, "batch size must be 1" + return output.squeeze(0).transpose(0, 1).reshape([n, h * d]).contiguous() + + class LightningAttentionBackend(MambaAttnBackendBase): """ Note about the init: diff --git a/python/sglang/srt/layers/attention/minicpm/attention_adapter.py b/python/sglang/srt/layers/attention/minicpm/attention_adapter.py index 9162513882..06a4742b9a 100644 --- a/python/sglang/srt/layers/attention/minicpm/attention_adapter.py +++ b/python/sglang/srt/layers/attention/minicpm/attention_adapter.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Optional import torch from sglang.kernels.ops.attention.flash_attention import flash_attn_with_kvcache -from sglang.kernels.ops.attention.utils import create_flashinfer_kv_indices_triton +from sglang.kernels.ops.attention.kv_indices import create_flashinfer_kv_indices_triton from sglang.srt.layers.attention.flashattention_backend import FlashAttentionBackend from sglang.srt.layers.attention.minicpm.sparse_utils import ( MiniCPMSparseMetadata, diff --git a/python/sglang/srt/layers/attention/minimax_sparse_ops/minimax_sparse.py b/python/sglang/srt/layers/attention/minimax_sparse_ops/minimax_sparse.py index 5900952999..65639bdf82 100644 --- a/python/sglang/srt/layers/attention/minimax_sparse_ops/minimax_sparse.py +++ b/python/sglang/srt/layers/attention/minimax_sparse_ops/minimax_sparse.py @@ -178,7 +178,7 @@ def minimax_sparse_prefill( # the scratch gather reads pool slots without the HiSparse remap and loc_mapping is None ): - from .gluon_prefill import ( + from sglang.kernels.ops.attention.minimax_sparse.prefill.gluon_prefill import ( GluonPrefillUnavailableError, can_use_gluon_prefill, gluon_sparse_prefill, diff --git a/python/sglang/srt/layers/attention/minimax_sparse_ops/naive/flash_with_topk_idx.py b/python/sglang/srt/layers/attention/minimax_sparse_ops/naive/flash_with_topk_idx.py deleted file mode 100644 index d473cef752..0000000000 --- a/python/sglang/srt/layers/attention/minimax_sparse_ops/naive/flash_with_topk_idx.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2025 XunhaoLai. All rights reserved. - -from typing import Optional - -import torch -from einops import einsum, rearrange - - -def naive_flash_decode_with_topk_idx( - q: torch.Tensor, # [batch_size, num_heads, head_dim] - sink: Optional[torch.Tensor], # [num_heads, head_dim] - kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_heads, head_dim] - seq_lens: torch.Tensor, # [batch_size, ] - max_seqlen: int, - slot_ids: torch.Tensor, # [batch_size, ] - block_size: int, - topk: int, - sm_scale: Optional[float] = None, - init_blocks: int = 0, - local_blocks: int = 0, -): - assert kv_cache.shape[2] % block_size == 0, ( - "max cache len must be divisible by block size" - ) - if sm_scale is None: - sm_scale = q.shape[-1] ** -0.5 - original_dtype = q.dtype - batch_size = q.shape[0] - num_q_heads = q.shape[1] - num_kv_heads = kv_cache.shape[3] - q = rearrange(q.float(), "b (h g) d -> b h g d", h=num_kv_heads) - kv_cache_float = kv_cache.float() - qk = ( - einsum(q, kv_cache_float[slot_ids, 0, ...], "b h g d, b n h d -> b h g n") - * sm_scale - ) - mask = torch.arange(kv_cache.shape[2], device=q.device) < seq_lens[:, None] - qk = qk.masked_fill(~mask[:, None, None, :], float("-inf")) - # get score - score = qk.clone().to(torch.float32) - score = rearrange(score, "b h g (n s) -> b (h g) n s", s=block_size) - score = score.max(dim=-1).values # [batch_size, num_q_heads, num_blocks] - # post-process score for init_blocks and local_blocks - INIT_SCORE = 1e30 - LOCAL_SCORE = 1e29 - if init_blocks > 0: - score[:, :, :init_blocks] = INIT_SCORE - if local_blocks > 0: - num_blocks_per_batch = (seq_lens + block_size - 1) // block_size - for b in range(batch_size): - num_blks = num_blocks_per_batch[b].item() - local_start = max(0, num_blks - local_blocks) - score[b, :, local_start:num_blks] = LOCAL_SCORE - # compute topk indices per (batch, head) - # score shape: [batch_size, num_q_heads, num_blocks] - topk_idx = torch.full( - (num_q_heads, batch_size, topk), - fill_value=-1, - device=score.device, - dtype=torch.int32, - ) - num_blocks_per_batch = (seq_lens + block_size - 1) // block_size - for b in range(batch_size): - num_blks = num_blocks_per_batch[b].item() - actual_topk = min(topk, num_blks) - for h in range(num_q_heads): - # get topk indices for this (batch, head) - _, indices = torch.topk(score[b, h, :num_blks], k=actual_topk, dim=-1) - topk_idx[h, b, :actual_topk] = indices.to(torch.int32) - # compute attention output with sink - if sink is not None: - # sink: [num_q_heads, head_dim] -> reshape to match q: [b, h, g, d] - sink_reshaped = rearrange(sink.float(), "(h g) d -> h g d", h=num_kv_heads) - qsink = ( - einsum(q, sink_reshaped, "b h g d, h g d -> b h g") * sm_scale - ) # [b, h, g] - qk_with_sink = torch.cat([qsink[..., None], qk], dim=-1) # [b, h, g, n+1] - attn = qk_with_sink.softmax(dim=-1, dtype=torch.float32) - attn = attn[..., 1:] # remove sink score - else: - attn = qk.softmax(dim=-1, dtype=torch.float32) - o = einsum(attn, kv_cache_float[slot_ids, 1, ...], "b h g n, b n h d -> b h g d") - o = rearrange(o, "b h g d -> b (h g) d", h=num_kv_heads) - return o.to(original_dtype), topk_idx diff --git a/python/sglang/srt/layers/attention/minimax_sparse_ops/naive/topk_sparse.py b/python/sglang/srt/layers/attention/minimax_sparse_ops/naive/topk_sparse.py deleted file mode 100644 index 6fefac38e0..0000000000 --- a/python/sglang/srt/layers/attention/minimax_sparse_ops/naive/topk_sparse.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2025 XunhaoLai. All rights reserved. - -from typing import Optional - -import torch - - -def naive_flash_decode_with_gqa_share_sparse( - q: torch.Tensor, # [batch_size, num_q_heads, head_dim] - sink: Optional[torch.Tensor], # [num_q_heads, head_dim] - kv_cache: torch.Tensor, # [max_slots, 2, max_len, num_kv_heads, head_dim] - seq_lens: torch.Tensor, # [batch_size, ] - slot_ids: torch.Tensor, # [batch_size, ] - block_size: int, - topk_idx: torch.Tensor, # [num_kv_heads, batch_size, topk] - sm_scale: Optional[float] = None, -) -> torch.Tensor: - """ - Naive implementation of sparse attention with GQA group sharing. - Each GQA group (all q heads sharing the same kv head) uses the same topk_idx. - """ - if sm_scale is None: - sm_scale = q.shape[-1] ** -0.5 - - original_dtype = q.dtype - batch_size, num_q_heads, head_dim = q.shape - max_slots, _, max_kv_len, num_kv_heads, _ = kv_cache.shape - gqa_group_size = num_q_heads // num_kv_heads - # Output tensor - o = torch.zeros(batch_size, num_q_heads, head_dim, dtype=q.dtype, device=q.device) - - for b in range(batch_size): - seq_len = seq_lens[b].item() - sid = slot_ids[b].item() % max_slots - - for kh in range(num_kv_heads): - # Get topk indices for this (kv_head, batch) - block_indices = topk_idx[kh, b, :].tolist() - - # Collect selected K and V blocks - selected_k_blocks = [] - selected_v_blocks = [] - for block_idx in block_indices: - if block_idx < 0: - continue # Invalid index - start = block_idx * block_size - end = min(start + block_size, seq_len) - if start >= seq_len: - continue - # K: [block_len, head_dim] - k_block = kv_cache[sid, 0, start:end, kh, :] - # V: [block_len, head_dim] - v_block = kv_cache[sid, 1, start:end, kh, :] - selected_k_blocks.append(k_block) - selected_v_blocks.append(v_block) - - if len(selected_k_blocks) == 0: - continue - - # Concatenate selected blocks: [total_selected_len, head_dim] - k_selected = torch.cat(selected_k_blocks, dim=0) - v_selected = torch.cat(selected_v_blocks, dim=0) - - # Compute attention for all q heads in this GQA group - for g in range(gqa_group_size): - qh = kh * gqa_group_size + g - # q_vec: [head_dim] - q_vec = q[b, qh, :].float() - - # Compute attention scores: [total_selected_len] - scores = torch.matmul(q_vec, k_selected.float().T) * sm_scale - - # Add sink to softmax normalization if present - if sink is not None: - sink_vec = sink[qh, :].float() - qsink = torch.dot(q_vec, sink_vec) * sm_scale - # Concatenate sink score with regular scores for softmax - scores_with_sink = torch.cat([qsink.unsqueeze(0), scores], dim=0) - attn_weights_with_sink = torch.softmax(scores_with_sink, dim=-1) - # Remove sink weight (only used for normalization) - attn_weights = attn_weights_with_sink[1:] - else: - attn_weights = torch.softmax(scores, dim=-1) - - # Compute output: [head_dim] - o[b, qh, :] = torch.matmul(attn_weights.to(original_dtype), v_selected) - - return o diff --git a/python/sglang/srt/layers/attention/nsa/index_buf_accessor.py b/python/sglang/srt/layers/attention/nsa/index_buf_accessor.py deleted file mode 100644 index 72b92ec44f..0000000000 --- a/python/sglang/srt/layers/attention/nsa/index_buf_accessor.py +++ /dev/null @@ -1,10 +0,0 @@ -# [Deprecated] Re-export shim for backward compatibility. Use dsa.index_buf_accessor instead. -import warnings - -warnings.warn( - "sglang.srt.layers.attention.nsa.index_buf_accessor is deprecated; " - "use sglang.kernels.ops.attention.dsa.index_buf_accessor instead.", - DeprecationWarning, - stacklevel=2, -) -from sglang.kernels.ops.attention.dsa.index_buf_accessor import * # noqa: F401, F403 diff --git a/python/sglang/srt/layers/attention/nsa/quant_k_cache.py b/python/sglang/srt/layers/attention/nsa/quant_k_cache.py deleted file mode 100644 index 1b3a2274cb..0000000000 --- a/python/sglang/srt/layers/attention/nsa/quant_k_cache.py +++ /dev/null @@ -1,10 +0,0 @@ -# [Deprecated] Re-export shim for backward compatibility. Use dsa.quant_k_cache instead. -import warnings - -warnings.warn( - "sglang.srt.layers.attention.nsa.quant_k_cache is deprecated; " - "use sglang.kernels.ops.attention.dsa.quant_k_cache instead.", - DeprecationWarning, - stacklevel=2, -) -from sglang.kernels.ops.attention.dsa.quant_k_cache import * # noqa: F401, F403 diff --git a/python/sglang/srt/layers/attention/nsa/tilelang_kernel.py b/python/sglang/srt/layers/attention/nsa/tilelang_kernel.py deleted file mode 100644 index 3fb456f0c3..0000000000 --- a/python/sglang/srt/layers/attention/nsa/tilelang_kernel.py +++ /dev/null @@ -1,10 +0,0 @@ -# [Deprecated] Re-export shim for backward compatibility. Use dsa.tilelang_kernel instead. -import warnings - -warnings.warn( - "sglang.srt.layers.attention.nsa.tilelang_kernel is deprecated; " - "use sglang.kernels.ops.attention.dsa.tilelang_kernel instead.", - DeprecationWarning, - stacklevel=2, -) -from sglang.kernels.ops.attention.dsa.tilelang_kernel import * # noqa: F401, F403 diff --git a/python/sglang/srt/layers/attention/nsa/triton_kernel.py b/python/sglang/srt/layers/attention/nsa/triton_kernel.py deleted file mode 100644 index aa54fa9502..0000000000 --- a/python/sglang/srt/layers/attention/nsa/triton_kernel.py +++ /dev/null @@ -1,10 +0,0 @@ -# [Deprecated] Re-export shim for backward compatibility. Use dsa.triton_kernel instead. -import warnings - -warnings.warn( - "sglang.srt.layers.attention.nsa.triton_kernel is deprecated; " - "use sglang.kernels.ops.attention.dsa.triton_kernel instead.", - DeprecationWarning, - stacklevel=2, -) -from sglang.kernels.ops.attention.dsa.triton_kernel import * # noqa: F401, F403 diff --git a/python/sglang/srt/layers/attention/trtllm_mha_backend.py b/python/sglang/srt/layers/attention/trtllm_mha_backend.py index f5b37798ad..98e124bc35 100644 --- a/python/sglang/srt/layers/attention/trtllm_mha_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mha_backend.py @@ -18,15 +18,15 @@ from typing import TYPE_CHECKING, Optional import torch -from sglang.kernels.ops.attention.utils import canonicalize_stride -from sglang.kernels.ops.kvcache.trtllm_mha_graph_metadata import ( +from sglang.kernels.ops.attention.trtllm_mha_graph_metadata import ( Q_MODE_NONE, Q_MODE_STRIDED, update_trtllm_mha_graph_metadata, ) -from sglang.kernels.ops.kvcache.trtllm_mha_page_table import ( +from sglang.kernels.ops.attention.trtllm_mha_page_table import ( build_trtllm_mha_page_table, ) +from sglang.kernels.ops.attention.utils import canonicalize_stride from sglang.srt.environ import envs from sglang.srt.layers.attention.base_attn_backend import SharedReadEnds from sglang.srt.layers.attention.flashinfer_backend import ( diff --git a/python/sglang/srt/layers/attention/trtllm_mla_backend.py b/python/sglang/srt/layers/attention/trtllm_mla_backend.py index 6bc304811a..44989bc726 100755 --- a/python/sglang/srt/layers/attention/trtllm_mla_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mla_backend.py @@ -17,6 +17,11 @@ from sglang.kernels.ops.attention.fixup_zero_kv import ( _jit_fixup_module, fixup_zero_kv_rows, ) +from sglang.kernels.ops.attention.kv_indices import ( + create_flashmla_kv_indices_triton, + get_num_kv_index_blocks_flashmla, + get_num_page_per_block_flashmla, +) from sglang.kernels.ops.attention.pad import ( pad_draft_extend_query as pad_draft_extend_query_triton, ) @@ -42,11 +47,6 @@ from sglang.kernels.ops.attention.utils import ( mla_quantize_and_rope_for_fp8, mla_quantize_without_rope_for_fp8, ) -from sglang.kernels.ops.kvcache.kv_indices import ( - create_flashmla_kv_indices_triton, - get_num_kv_index_blocks_flashmla, - get_num_page_per_block_flashmla, -) from sglang.kernels.ops.quantization.fp8_kernel import scaled_fp8_quant from sglang.srt.environ import envs from sglang.srt.layers.attention.flashinfer_mla_backend import ( diff --git a/python/sglang/srt/layers/attention/wave_backend.py b/python/sglang/srt/layers/attention/wave_backend.py index 1a5fb755aa..7a38523b2c 100644 --- a/python/sglang/srt/layers/attention/wave_backend.py +++ b/python/sglang/srt/layers/attention/wave_backend.py @@ -7,10 +7,10 @@ from typing import TYPE_CHECKING, Optional import torch import triton -from sglang.kernels.ops.attention.metadata import get_num_kv_splits_triton -from sglang.kernels.ops.kvcache.kv_indices import ( +from sglang.kernels.ops.attention.kv_indices import ( create_flashinfer_kv_indices_triton, ) +from sglang.kernels.ops.attention.metadata import get_num_kv_splits_triton from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.runtime_context import ( diff --git a/python/sglang/srt/layers/communication/k3_sp_collective.py b/python/sglang/srt/layers/communication/k3_sp_collective.py index b46a9824ff..9787640072 100644 --- a/python/sglang/srt/layers/communication/k3_sp_collective.py +++ b/python/sglang/srt/layers/communication/k3_sp_collective.py @@ -98,8 +98,7 @@ def _init_state() -> Optional[_State]: (logger.warning if explicit else logger.info)(message) return None - from sglang.kernels.ops.attention import attn_res - from sglang.kernels.ops.communication import sp_collective + from sglang.kernels.ops.communication import attn_res_collective, sp_collective # Refuse to enable without a checked-in table for this exact device. if ( @@ -119,7 +118,7 @@ def _init_state() -> Optional[_State]: return None sp_collective.register_comm(comm.obj) - attn_res.register_comm(comm.obj) + attn_res_collective.register_comm(comm.obj) _STATE = _State(group, comm) logger.info( "K3 SP collective enabled (TP%d, fused RS residual + AG)", @@ -318,8 +317,7 @@ def reduce_scatter_attn_res( or ow.shape != (_HIDDEN_SIZE,) ): return None - from sglang.kernels.ops.attention import attn_res - from sglang.kernels.ops.communication import sp_collective + from sglang.kernels.ops.communication import attn_res_collective, sp_collective dispatch = sp_collective.get_fusion_dispatch( "reduce_scatter_attn_res", @@ -338,7 +336,7 @@ def reduce_scatter_attn_res( (local_tokens, tensor.shape[1]), dtype=tensor.dtype, device=tensor.device ) prefix = torch.empty_like(normed) - attn_res.attn_res_fused_pull_rs( + attn_res_collective.attn_res_fused_pull_rs( state.group.world_size, tensor, residual, @@ -433,8 +431,7 @@ def attn_res_all_gather( or ow.shape != (_HIDDEN_SIZE,) ): return None - from sglang.kernels.ops.attention import attn_res - from sglang.kernels.ops.communication import sp_collective + from sglang.kernels.ops.communication import attn_res_collective, sp_collective global_tokens = prefix.shape[0] * state.group.world_size dispatch = sp_collective.get_fusion_dispatch( @@ -449,7 +446,7 @@ def attn_res_all_gather( output = _symm_buffer( state, _ATTN_RES_AG, global_tokens, prefix.shape[1], prefix.dtype ) - attn_res.attn_res_fused_direct_ag( + attn_res_collective.attn_res_fused_direct_ag( state.group.world_size, prefix, bank, diff --git a/python/sglang/srt/layers/communication/muse_spark_finalize_workspace.py b/python/sglang/srt/layers/communication/muse_spark_finalize_workspace.py new file mode 100644 index 0000000000..60f67482aa --- /dev/null +++ b/python/sglang/srt/layers/communication/muse_spark_finalize_workspace.py @@ -0,0 +1,139 @@ +"""Borrowed CustomAllReduceV2 push buffers for the fused Muse Spark MoE finalize + all-reduce. + +The kernel is ``FusedPostExpertNormWeightedSumTRTLLM`` in +``sglang.kernels.ops.communication.muse_spark_moe_finalize_all_reduce``. +""" + +import torch +import torch.distributed as dist + + +class FinalizePushWorkspace: + """Borrow TP push buffers before capture; dependency-ordered launches only.""" + + def __init__( + self, + communicator, + device_group: dist.ProcessGroup, + hidden_dim=4096, + *, + multicast=False, + ): + self.communicator = communicator # Keep the allocation owner alive. + self.group = device_group + self.rank = communicator.rank + self.world_size = communicator.world_size + self.device = communicator.device + self.sm_count = torch.cuda.get_device_properties( + self.device + ).multi_processor_count + self.hidden_dim = hidden_dim + self.slot_bytes = communicator.max_push_size + self.peers = communicator._push_workspaces + self.counter = communicator._push_counter + self.multicast = multicast + self.multicast_ptr = communicator._push_multicast_ptr + config = ( + self.controller_count, + self.slot_bytes, + self.sm_count, + hidden_dim, + multicast, + ) + configs = [None] * self.world_size + dist.all_gather_object(configs, config, group=communicator.group) + if any(peer != config for peer in configs): + raise ValueError( + "Finalize AR workspace configuration must match across ranks" + ) + if multicast and not communicator.has_multicast: + raise ValueError( + "Finalize multicast requires a multicast-capable push workspace" + ) + if not 0 < self.controller_count <= self.sm_count: + raise ValueError("Unsupported finalize AR controller budget") + if hidden_dim <= 0 or hidden_dim % 256 or self.slot_bytes < hidden_dim * 2: + raise ValueError( + "Finalize push requires H divisible by 256 and sufficient buffer capacity" + ) + self.max_tokens = min(384, self.slot_bytes // (hidden_dim * 2)) + self.fallback_logged = False + self.fusion_logged = False + + def uses_one_shot_push(self, num_tokens: int) -> bool: + return self.communicator.uses_one_shot_push(num_tokens * self.hidden_dim * 2) + + def can_fuse(self, tp_group, num_tokens: int) -> bool: + """Use custom AR storage while respecting explicit algorithm overrides. + + Fusion has a different crossover from a standalone AR launch. Allow + push through 384 tokens when its allocated slot fits, even if ordinary + CUDA-graph AR would select pull above its 288-token crossover (H=4096), + but not where the standalone AR would pick 2shot_push. + No standalone AR thresholds or buffer allocations are changed. + """ + if ( + torch.compiler.is_compiling() + or self.communicator is not tp_group.ca_comm + or self.communicator.disabled + or not 0 < num_tokens <= self.max_tokens + ): + return False + if self.world_size > 4 and not self.uses_one_shot_push(num_tokens): + return False + if self.communicator.uses_two_shot_push(num_tokens * self.hidden_dim * 2): + return False + output = ( + self.peers[self.rank] + .view(torch.bfloat16) + .view(-1)[: num_tokens * self.hidden_dim] + .view(num_tokens, self.hidden_dim) + ) + return ( + not tp_group._can_use_flashinfer_allreduce(output) + and tp_group._resolve_outplace_all_reduce_method(output) == "ca" + and ( + self.communicator.override_algo is None + or self.uses_one_shot_push(num_tokens) + ) + ) + + @property + def controller_count(self) -> int: + return self.counter.numel() + + def validate(self, output: torch.Tensor, cluster_size: int = 1) -> None: + if ( + output.device != self.device + or output.dtype != torch.bfloat16 + or output.ndim != 2 + or output.shape[1] != self.hidden_dim + or output.shape[0] > self.max_tokens + or not output.is_contiguous() + or output.data_ptr() % 16 + ): + raise ValueError("Unsupported output for finalize push workspace") + if cluster_size not in (1, 2, 4, 8, 16) or cluster_size > self.sm_count: + raise ValueError("Unsupported cluster size") + + if output.shape[0] == 0: + raise ValueError("Finalize push requires M>0") + if cluster_size > 1 and output.shape[0] * cluster_size > self.controller_count: + raise ValueError( + "Single-token clusters require M * C <= controller_count; use C=1 otherwise" + ) + + def arguments(self) -> tuple: + import cutlass + from cutlass.cute.runtime import make_ptr + + return ( + tuple(make_ptr(cutlass.Uint32, p.data_ptr()) for p in self.peers), + make_ptr(cutlass.Uint32, self.counter.data_ptr()), + cutlass.Int32(self.rank), + cutlass.Int32(self.slot_bytes // 4), + make_ptr(cutlass.Uint32, self.peers[self.rank].data_ptr()), + make_ptr( + cutlass.Uint32, self.multicast_ptr or self.peers[self.rank].data_ptr() + ), + ) diff --git a/python/sglang/kernels/ops/muse_spark/gemm_ag.py b/python/sglang/srt/layers/communication/muse_spark_gemm_ag.py similarity index 54% rename from python/sglang/kernels/ops/muse_spark/gemm_ag.py rename to python/sglang/srt/layers/communication/muse_spark_gemm_ag.py index d406cd657e..be859b407b 100644 --- a/python/sglang/kernels/ops/muse_spark/gemm_ag.py +++ b/python/sglang/srt/layers/communication/muse_spark_gemm_ag.py @@ -14,72 +14,16 @@ from typing import TYPE_CHECKING, Optional import torch -from sglang.kernels.jit.utils import ( - cache_once, - is_arch_support_pdl, - load_jit, - make_cpp_args, +from sglang.kernels.ops.communication.muse_spark_all_gather import ( + MAX_TOKENS, + all_gather, ) if TYPE_CHECKING: - from tvm_ffi.module import Module - from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( CustomAllReduceV2, ) -SMALL_PROJECTION_MAX_TOKENS = 8 -MAX_TOKENS = 512 -_VEC = 8 - - -@cache_once -def _jit_module(n: int, world_size: int) -> Module: - args = make_cpp_args(n, world_size, is_arch_support_pdl()) - return load_jit( - "muse_spark_all_gather", - *args, - cuda_files=["muse_spark/all_gather.cuh"], - cuda_wrappers=[("run", f"MuseSparkAllGather<{args}>::run")], - extra_cuda_cflags=["-O3"], - ) - - -def gemm_ag_supported( - comm: Optional[CustomAllReduceV2], x: torch.Tensor, weight: torch.Tensor -) -> bool: - from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( - CustomAllReduceV2, - ) - - if ( - not isinstance(comm, CustomAllReduceV2) - or comm.disabled - or not comm.has_multicast - or comm.world_size < 2 - ): - return False - if x.dtype != torch.bfloat16 or weight.dtype != torch.bfloat16 or x.dim() != 2: - return False - m, k = x.shape - n = weight.shape[0] - return ( - 1 <= m <= MAX_TOKENS - and ( - m <= SMALL_PROJECTION_MAX_TOKENS - or ( - comm.world_size in (4, 8) - and n in (4096, 8192) - and torch.cuda.get_device_capability(x.device) == (10, 3) - ) - ) - and weight.shape[1] == k - and n % (comm.world_size * _VEC) == 0 - and m * n * 2 <= comm.max_push_size - and comm.config.num_push_blocks >= 2 - and weight.is_contiguous() - ) - def reserve_projection_workspace(comm: Optional[CustomAllReduceV2], n: int) -> None: """Reserve shared push storage during model construction, before capture. @@ -101,20 +45,6 @@ def reserve_projection_workspace(comm: Optional[CustomAllReduceV2], n: int) -> N comm.reserve_push_capacity(MAX_TOKENS * n * 2) -def all_gather( - comm: CustomAllReduceV2, - local: torch.Tensor, - out: torch.Tensor, - b: Optional[torch.Tensor] = None, - c: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """``out[:, rank slice] = local`` gathered from every rank, (+ b) (+ c).""" - _jit_module(out.shape[1], comm.world_size).run( - comm.obj, local.contiguous(), b, c, out - ) - return out - - def gemm_local( comm: CustomAllReduceV2, x: torch.Tensor, weight: torch.Tensor ) -> torch.Tensor: diff --git a/python/sglang/srt/layers/communication/sp_collective_gemm_workspace.py b/python/sglang/srt/layers/communication/sp_collective_gemm_workspace.py new file mode 100644 index 0000000000..ed365c1b3c --- /dev/null +++ b/python/sglang/srt/layers/communication/sp_collective_gemm_workspace.py @@ -0,0 +1,91 @@ +"""Symmetric-memory workspaces of the SP collective GEMMs. + +The kernels are ``all_gather_matmul`` / ``matmul_reduce_scatter`` in +``sglang.kernels.ops.communication.sp_collective_gemm``; pass +:func:`sp_collective_gemm_workspace` as their ``workspace``. One workspace per +``(kind, group, width, dtype, device)`` lives for the process and grows by +reallocation outside CUDA-graph capture. +""" + +import torch +import torch.distributed as dist + +from sglang.kernels.ops.communication.sp_collective_gemm import _TILE_M, _TILE_N + +_workspaces = {} + + +class SPCollectiveGemmWorkspace: + def __init__( + self, + kind, + group, + rows, + width, + dtype, + device, + row_align=_TILE_M, + flag_tile_mn=(_TILE_M, _TILE_N), + ): + import cutlass.torch as cutlass_torch + import torch.distributed._symmetric_memory as symm_mem + + # Power of two >= row_align, so capacity stays a whole number of M tiles. + self.capacity = max(row_align, 1 << (rows - 1).bit_length()) + self.world = dist.get_world_size(group) + self.sms = torch.cuda.get_device_properties(device).multi_processor_count + self.storage = symm_mem.empty( + (self.capacity, width), dtype=dtype, device=device + ) + self.handle = symm_mem.rendezvous(self.storage, group) + if kind == "ag": + self.gathered = torch.empty( + (self.capacity * self.world, width), dtype=dtype, device=device + ) + self.flags = torch.zeros(self.world, dtype=torch.int32, device=device) + self.copy_stream = torch.cuda.Stream(device=device) + else: + if not self.handle.multicast_ptr: + raise RuntimeError( + "CUTLASS GEMM-RS requires NVSwitch multicast mapping" + ) + self.multicast = cutlass_torch.as_tensor( + self.handle.multicast_ptr, self.storage.shape, dtype + ) + num_flags = ( + self.capacity // flag_tile_mn[0] * (width // flag_tile_mn[1]) + + self.sms + + 8 + ) + self.flags = symm_mem.empty((num_flags,), dtype=torch.int32, device=device) + self.flags.zero_() + self.flag_handle = symm_mem.rendezvous(self.flags, group) + if not self.flag_handle.multicast_ptr: + raise RuntimeError("CUTLASS GEMM-RS requires multicast flag storage") + self.flag_multicast = cutlass_torch.as_tensor( + self.flag_handle.multicast_ptr, self.flags.shape, self.flags.dtype + ) + self.peer_flags = tuple( + self.flag_handle.get_buffer(r, self.flags.shape, self.flags.dtype) + for r in range(self.world) + ) + self.handle.barrier(channel=0) + + +def sp_collective_gemm_workspace( + kind, group, rows, width, x, row_align=_TILE_M, flag_tile_mn=(_TILE_M, _TILE_N) +): + key = (kind, group, width, x.dtype, x.device) + ws = _workspaces.get(key) + if ws is None or rows > ws.capacity: + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError("Warm up CUTLASS SP workspace before CUDA graph capture") + if ws is not None: + # Peer mappings must stay alive until all ranks stop reading them. + ws.handle.barrier(channel=0) + torch.cuda.current_stream().synchronize() + ws = SPCollectiveGemmWorkspace( + kind, group, rows, width, x.dtype, x.device, row_align, flag_tile_mn + ) + _workspaces[key] = ws + return ws diff --git a/python/sglang/srt/layers/cp/interleave.py b/python/sglang/srt/layers/cp/interleave.py index 0f1a7e1ca9..1cf64e5381 100644 --- a/python/sglang/srt/layers/cp/interleave.py +++ b/python/sglang/srt/layers/cp/interleave.py @@ -231,9 +231,7 @@ class InterleaveCPStrategy(ContextParallelStrategy): extend_seqs: Any, ): """Build device outputs in the shared kernel to keep the split graph-safe.""" - from sglang.kernels.ops.attention.dsa.cp_split import ( - dsa_cp_interleave_q_seqs_kernel, - ) + from sglang.kernels.ops.attention.dsa.cp_split import dsa_cp_interleave_q_seqs cp_size = self.cp_size cp_rank = self.cp_rank @@ -248,14 +246,8 @@ class InterleaveCPStrategy(ContextParallelStrategy): bs_idx_cpu = [i for i, q_len in enumerate(q_lens_cpu) if q_len > 0] q_lens_cpu = [q_len for q_len in q_lens_cpu if q_len > 0] - q_lens = torch.empty( - (len(bs_idx_cpu),), device=extend_seqs.device, dtype=extend_seqs.dtype - ) - bs_idx = torch.empty( - (len(bs_idx_cpu),), device=extend_seqs.device, dtype=torch.int32 - ) - dsa_cp_interleave_q_seqs_kernel[(1,)]( - extend_seqs, q_lens, bs_idx, len(extend_seqs), cp_size, cp_rank + q_lens, bs_idx = dsa_cp_interleave_q_seqs( + extend_seqs, len(bs_idx_cpu), cp_size, cp_rank ) return q_lens_cpu, q_lens, bs_idx_cpu, bs_idx diff --git a/python/sglang/srt/layers/engram.py b/python/sglang/srt/layers/engram.py index 453d63811d..f8f28ab0d1 100644 --- a/python/sglang/srt/layers/engram.py +++ b/python/sglang/srt/layers/engram.py @@ -22,7 +22,6 @@ import numpy as np import torch from torch import nn -from sglang.kernels.ops.attention.dsv4.torch_quant import FP8_BLOCK_SIZE from sglang.kernels.ops.embeddings.engram_gate import fused_engram_gate from sglang.kernels.ops.embeddings.engram_gather import engram_gather from sglang.kernels.ops.embeddings.engram_hash import ( @@ -34,6 +33,7 @@ from sglang.kernels.ops.embeddings.engram_hash import ( engram_hash_ids, engram_hash_ids_and_commit, ) +from sglang.kernels.ops.quantization.fp4_fake_quant import FP8_BLOCK_SIZE from sglang.srt.distributed import tensor_model_parallel_all_reduce from sglang.srt.distributed.device_communicators.cuda_wrapper import ( find_loaded_library, diff --git a/python/sglang/srt/layers/hyperconnection.py b/python/sglang/srt/layers/hyperconnection.py index 7329cb5740..a84ca5c7e4 100644 --- a/python/sglang/srt/layers/hyperconnection.py +++ b/python/sglang/srt/layers/hyperconnection.py @@ -261,7 +261,7 @@ class GatedResidual(HyperConnectionBase): and hyper_input_normed.dtype in (torch.bfloat16, torch.float16) and hyper_input_normed.shape[0] <= 24 ): - from sglang.kernels.ops.elementwise.hc_mix import ( + from sglang.kernels.ops.gemm.hc_mix_splitk import ( hc_mix, permute_pad_up_weight, ) diff --git a/python/sglang/srt/layers/layernorm_sp.py b/python/sglang/srt/layers/layernorm_sp.py index 186e857cf5..132715e40f 100644 --- a/python/sglang/srt/layers/layernorm_sp.py +++ b/python/sglang/srt/layers/layernorm_sp.py @@ -216,12 +216,18 @@ def column_parallel_g_matmul( num_tokens = sp_num_tokens() if sp_fused_matmul_eligible(linear): if getattr(linear, "_sp_use_cutlass", False): - from sglang.kernels.ops.gemm.sp_collective_gemm import all_gather_matmul + from sglang.kernels.ops.communication.sp_collective_gemm import ( + all_gather_matmul, + ) + from sglang.srt.layers.communication.sp_collective_gemm_workspace import ( + sp_collective_gemm_workspace, + ) return all_gather_matmul( input_parallel.contiguous(), linear.weight, get_parallel().tp_group.device_group, + workspace=sp_collective_gemm_workspace, )[:num_tokens] group_name = get_parallel().tp_group.device_group.group_name _, mm_outputs = torch.ops.symm_mem.fused_all_gather_matmul( @@ -252,10 +258,18 @@ def row_parallel_gbar_matmul(linear, input_: torch.Tensor, bias) -> torch.Tensor x = torch.nn.functional.pad(x, (0, 0, 0, padded - num_tokens)) if sp_fused_matmul_eligible(linear): if getattr(linear, "_sp_use_cutlass", False): - from sglang.kernels.ops.gemm.sp_collective_gemm import matmul_reduce_scatter + from sglang.kernels.ops.communication.sp_collective_gemm import ( + matmul_reduce_scatter, + ) + from sglang.srt.layers.communication.sp_collective_gemm_workspace import ( + sp_collective_gemm_workspace, + ) return matmul_reduce_scatter( - x, linear.weight, get_parallel().tp_group.device_group + x, + linear.weight, + get_parallel().tp_group.device_group, + workspace=sp_collective_gemm_workspace, ) group_name = get_parallel().tp_group.device_group.group_name return torch.ops.symm_mem.fused_matmul_reduce_scatter( diff --git a/python/sglang/srt/layers/logits_processor.py b/python/sglang/srt/layers/logits_processor.py index 19fd6e8ebf..26c64dfee0 100644 --- a/python/sglang/srt/layers/logits_processor.py +++ b/python/sglang/srt/layers/logits_processor.py @@ -30,9 +30,9 @@ from sglang.kernels.ops.activation.softcap import ( softcap_inplace_logits as fused_softcap, ) from sglang.kernels.ops.activation.softcap import ( - softcap_row_argmax, softcap_to_float32_logits, ) +from sglang.kernels.ops.sampling.softcap_row_argmax import softcap_row_argmax from sglang.srt.beam_search.logits_capture import BeamLogitsCapture from sglang.srt.distributed.device_communicators import triton_symm_mem_ag from sglang.srt.environ import envs diff --git a/python/sglang/srt/layers/moe/cutlass_moe.py b/python/sglang/srt/layers/moe/cutlass_moe.py index 918c51c99b..b9c9e37634 100755 --- a/python/sglang/srt/layers/moe/cutlass_moe.py +++ b/python/sglang/srt/layers/moe/cutlass_moe.py @@ -275,7 +275,7 @@ def cutlass_fused_experts_fp8( if swiglu_limit is None: silu_and_mul(c1, intermediate) else: - from sglang.kernels.ops.moe.dsv4 import silu_and_mul_clamp + from sglang.kernels.ops.activation.silu_and_mul_clamp import silu_and_mul_clamp silu_and_mul_clamp(c1, intermediate, swiglu_limit) diff --git a/python/sglang/srt/layers/moe/moe_runner/aiter.py b/python/sglang/srt/layers/moe/moe_runner/aiter.py index f432629dfe..9dabbfa41c 100644 --- a/python/sglang/srt/layers/moe/moe_runner/aiter.py +++ b/python/sglang/srt/layers/moe/moe_runner/aiter.py @@ -413,7 +413,7 @@ def _pre_permute_deepep_to_aiter( quant_type = quant_info.quant_type if is_mori: - from sglang.kernels.ops.moe.rocm_moe_utils import upscale, upscale_mxfp4 + from sglang.kernels.ops.quantization.rocm_upscale import upscale, upscale_mxfp4 a1_scale = dispatch_output.hidden_states_scale num_local_tokens = dispatch_output.num_recv_tokens_per_expert diff --git a/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py b/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py index 446f8ee1d8..377317d807 100644 --- a/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py +++ b/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py @@ -8,10 +8,8 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.moe.dsv4 import ( - silu_and_mul_clamp, - silu_and_mul_masked_post_quant, -) +from sglang.kernels.ops.activation.silu_and_mul_clamp import silu_and_mul_clamp +from sglang.kernels.ops.moe.dsv4 import silu_and_mul_masked_post_quant from sglang.kernels.ops.moe.triton_pad_expert_counts import pad_expert_counts from sglang.kernels.ops.quantization.per_token_group_quant import per_token_group_quant @@ -402,7 +400,9 @@ class DeepGemmRunnerCore(MoeRunnerCore): quant_info: DeepGemmMoeQuantInfo, running_state: dict, ) -> torch.Tensor: - from sglang.kernels.ops.moe.dsv4 import silu_and_mul_contig_post_quant + from sglang.kernels.ops.activation.silu_and_mul_clamp import ( + silu_and_mul_contig_post_quant, + ) from sglang.kernels.ops.moe.ep_moe_kernels import tma_align_input_scale from sglang.kernels.ops.quantization.fp8_kernel import ( create_per_token_group_quant_fp8_output_scale, diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py index 3e735b605a..ab355838e1 100644 --- a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py @@ -773,7 +773,9 @@ def _fused_moe_kernel_sequence( if not filter_expert: if swiglu_limit_for_silu_and_mul_clamp is not None: - from sglang.kernels.ops.moe.dsv4 import silu_and_mul_clamp + from sglang.kernels.ops.activation.silu_and_mul_clamp import ( + silu_and_mul_clamp, + ) silu_and_mul_clamp( intermediate_cache1.view(-1, N), diff --git a/python/sglang/srt/layers/moe/qwen4_decode.py b/python/sglang/srt/layers/moe/qwen4_decode.py index 56124d9186..7c297188a2 100644 --- a/python/sglang/srt/layers/moe/qwen4_decode.py +++ b/python/sglang/srt/layers/moe/qwen4_decode.py @@ -10,8 +10,8 @@ import torch from sglang.kernels.ops.communication.all_reduce_fusion import ( moe_finalize_shared_gate_all_reduce, ) -from sglang.kernels.ops.communication.mp import register_comm_cleanup from sglang.kernels.ops.moe.shared_expert_gate import shared_expert_gate +from sglang.srt.distributed.comm_cleanup import register_comm_cleanup from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( CustomAllReduceV2, ) diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_bf16_moe.py b/python/sglang/srt/layers/muse_spark/muse_spark_bf16_moe.py index b8bca06017..24f1e1ccd1 100644 --- a/python/sglang/srt/layers/muse_spark/muse_spark_bf16_moe.py +++ b/python/sglang/srt/layers/muse_spark/muse_spark_bf16_moe.py @@ -16,12 +16,12 @@ import torch import torch.nn.functional as F from torch import nn -from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_triton import ( - fused_post_expert_norm_weighted_sum, -) -from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_trtllm import ( +from sglang.kernels.ops.communication.muse_spark_moe_finalize_all_reduce import ( FusedPostExpertNormWeightedSumTRTLLM, ) +from sglang.kernels.ops.moe.muse_spark_post_expert_finalize_triton import ( + fused_post_expert_norm_weighted_sum, +) from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig from sglang.srt.layers.muse_spark.muse_spark_fp4_moe import ( finalize_with_post_expert_norm, @@ -31,7 +31,7 @@ from sglang.srt.utils.common import next_power_of_2, set_weight_attrs logger = logging.getLogger(__name__) if TYPE_CHECKING: - from sglang.kernels.ops.muse_spark.fi_trtllm_moe_finalize_with_rmsnorm_cutedsl import ( + from sglang.srt.layers.communication.muse_spark_finalize_workspace import ( FinalizePushWorkspace, ) from sglang.srt.layers.moe.token_dispatcher.flashinfer import ( diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py b/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py index ae6705d84a..f4a564ac23 100644 --- a/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py +++ b/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py @@ -19,13 +19,15 @@ from typing import TYPE_CHECKING, Optional import torch from torch import nn -from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_triton import ( - fused_post_expert_norm_weighted_sum, -) -from sglang.kernels.ops.muse_spark.fused_post_expert_norm_weighted_sum_trtllm import ( +from sglang.kernels.ops.communication.muse_spark_moe_finalize_all_reduce import ( FusedPostExpertNormWeightedSumTRTLLM, ) -from sglang.kernels.ops.muse_spark.norm_fp4_quant import fused_norm_fp4_quant +from sglang.kernels.ops.layernorm.muse_spark_rmsnorm_nvfp4_quant import ( + fused_norm_fp4_quant, +) +from sglang.kernels.ops.moe.muse_spark_post_expert_finalize_triton import ( + fused_post_expert_norm_weighted_sum, +) from sglang.srt.layers.moe import MoeRunnerBackend, get_moe_runner_backend from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig from sglang.srt.layers.muse_spark.muse_spark_nvfp4_moe import ( @@ -35,7 +37,7 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.utils.common import get_device_capability, is_cuda if TYPE_CHECKING: - from sglang.kernels.ops.muse_spark.fi_trtllm_moe_finalize_with_rmsnorm_cutedsl import ( + from sglang.srt.layers.communication.muse_spark_finalize_workspace import ( FinalizePushWorkspace, ) from sglang.srt.layers.moe.token_dispatcher.flashinfer import ( diff --git a/python/sglang/srt/layers/n_gram_embedding.py b/python/sglang/srt/layers/n_gram_embedding.py index 8d6fe33ef4..d29bf27006 100644 --- a/python/sglang/srt/layers/n_gram_embedding.py +++ b/python/sglang/srt/layers/n_gram_embedding.py @@ -2,7 +2,7 @@ import torch from torch import nn from torch.nn import Parameter -from sglang.kernels.ops.speculative.ngram_embedding import compute_n_gram_ids +from sglang.kernels.ops.embeddings.ngram_embedding import compute_n_gram_ids from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding from sglang.srt.model_executor.forward_batch_info import ForwardBatch diff --git a/python/sglang/srt/layers/quantization/expert_pack.py b/python/sglang/srt/layers/quantization/expert_pack.py index 8e22c43362..59d4f74f2b 100644 --- a/python/sglang/srt/layers/quantization/expert_pack.py +++ b/python/sglang/srt/layers/quantization/expert_pack.py @@ -33,7 +33,9 @@ def _clamped_swiglu( ) -> torch.Tensor: if limit is not None: if gate.is_cuda: - from sglang.kernels.ops.moe.dsv4 import silu_and_mul_clamp + from sglang.kernels.ops.activation.silu_and_mul_clamp import ( + silu_and_mul_clamp, + ) gate_up = torch.cat((gate, up), dim=-1) output = torch.empty_like(gate) diff --git a/python/sglang/srt/layers/quantization/fp8_hip.py b/python/sglang/srt/layers/quantization/fp8_hip.py index 46ad696a57..c4c8a10a0e 100644 --- a/python/sglang/srt/layers/quantization/fp8_hip.py +++ b/python/sglang/srt/layers/quantization/fp8_hip.py @@ -14,19 +14,19 @@ from typing import Optional import torch -from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( - Fp8GridActivation, - Mxfp8Activation, - bf16_dequant_blockscaled_linear, - dequant_block_fp8_weight_to_bf16, - dequant_mxfp8_to_bf16, - mxfp8_operand, -) -from sglang.kernels.ops.quantization.mxfp8_native_amd_gfx95 import ( +from sglang.kernels.ops.gemm.mxfp8_amd_gfx95 import bf16_dequant_blockscaled_linear +from sglang.kernels.ops.gemm.mxfp8_native_amd_gfx95 import ( native_route_supports, prepare_mxfp8_native_weight, ue8m0_weight_scale, ) +from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( + Fp8GridActivation, + Mxfp8Activation, + dequant_block_fp8_weight_to_bf16, + dequant_mxfp8_to_bf16, + mxfp8_operand, +) from sglang.srt.environ import envs from sglang.srt.layers.utils import copy_or_rebind_param @@ -140,7 +140,7 @@ def apply_dense( def _flydsl_mxfp8_module(): """The mxfp8_flydsl_gfx95 module, or None when FlyDSL is missing.""" try: - from sglang.kernels.ops.quantization import mxfp8_flydsl_gfx95 + from sglang.kernels.ops.gemm import mxfp8_flydsl_gfx95 except ImportError as e: logger.warning("SGLANG_OPT_HIP_FLYDSL_MXFP8 ignored, FlyDSL unavailable: %s", e) return None diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index 39acc95de2..231b243ed9 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -54,7 +54,6 @@ from sglang.srt.utils import ( ) from sglang.srt.utils.common import torch_release from sglang.srt.utils.custom_op import register_custom_op -from sglang.utils import LazyImport if TYPE_CHECKING: from sglang.srt.layers.linear import LinearBase @@ -370,38 +369,6 @@ if _is_cuda: N = mat_b.shape[-1] return mat_a.new_empty((M, N), dtype=out_dtype) - # Resolved on first call: CUDA builds without FlashInfer must still import - # this module. - _raw_bmm_fp8_batched = LazyImport("flashinfer", "bmm_fp8") - - @register_custom_op(op_name="flashinfer_bmm_fp8_batched", mutates_args=["out"]) - def _bmm_fp8_batched_op( - A: torch.Tensor, - B: torch.Tensor, - out: torch.Tensor, - A_scale: torch.Tensor, - B_scale: torch.Tensor, - ) -> None: - _raw_bmm_fp8_batched(A, B, A_scale, B_scale, out.dtype, out) - - def bmm_fp8( - A: torch.Tensor, - B: torch.Tensor, - A_scale: torch.Tensor, - B_scale: torch.Tensor, - dtype: torch.dtype, - out: Optional[torch.Tensor] = None, - ) -> torch.Tensor: - """Batched (3D) per-tensor-scale FP8 matmul, via flashinfer's cuBLAS backend.""" - if out is None: - out = torch.empty( - (A.shape[0], A.shape[1], B.shape[2]), - device=A.device, - dtype=dtype, - ) - _bmm_fp8_batched_op(A, B, out, A_scale, B_scale) - return out - use_triton_w8a8_fp8_kernel = get_bool_env_var("USE_TRITON_W8A8_FP8_KERNEL") @@ -932,19 +899,19 @@ def dispatch_block_fp8_mxfp8_linear(backend: Mxfp8DenseGemmBackend) -> Callable: flashinfer_mxfp8_blockscaled_linear, backend="cute-dsl", pin_tactic=True ) if backend.is_gfx95_dot_scaled(): - from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( + from sglang.kernels.ops.gemm.mxfp8_amd_gfx95 import ( dot_scaled_mxfp8_blockscaled_linear, ) return dot_scaled_mxfp8_blockscaled_linear if backend.is_gfx95_mxfp8_native(): - from sglang.kernels.ops.quantization.mxfp8_native_amd_gfx95 import ( + from sglang.kernels.ops.gemm.mxfp8_native_amd_gfx95 import ( mxfp8_native_blockscaled_linear, ) return mxfp8_native_blockscaled_linear if backend.is_gfx95_mxfp8_aiter(): - from sglang.kernels.ops.quantization.mxfp8_aiter_gfx95 import ( + from sglang.kernels.ops.gemm.mxfp8_aiter_gfx95 import ( aiter_mxfp8_blockscaled_linear, ) @@ -993,7 +960,7 @@ def dispatch_w8a8_mxfp8_linear() -> Callable: elif backend.is_unsupported(): return _unsupported_mxfp8_linear - from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( + from sglang.kernels.ops.gemm.mxfp8_amd_gfx95 import ( dot_scaled_mxfp8_blockscaled_linear, ) diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index 5d90372611..0365d52aff 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -2007,7 +2007,7 @@ class ModelOptFp4LinearMethod(LinearMethodBase): ) if getattr(layer, "_interleave_for_swiglu_fusion", False): - from sglang.kernels.ops.quantization.nvfp4_gemm_swiglu_nvfp4_quant import ( + from sglang.kernels.ops.gemm.nvfp4_gemm_swiglu_nvfp4_quant import ( interleave_linear_and_gate, swizzle_blockscale_2d, ) diff --git a/python/sglang/srt/layers/rotary_embedding/base.py b/python/sglang/srt/layers/rotary_embedding/base.py index 03addb6acf..67bd3fcc56 100644 --- a/python/sglang/srt/layers/rotary_embedding/base.py +++ b/python/sglang/srt/layers/rotary_embedding/base.py @@ -67,7 +67,7 @@ if _is_npu: ) if _is_hip: - from sglang.kernels.ops.attention.utils import ( + from sglang.kernels.ops.attention.fused_qk_rope_reshape_and_cache import ( fused_qk_rope_reshape_and_cache, ) diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index e8a32b4231..9f6f404018 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -15,16 +15,10 @@ from typing import ( import torch -from sglang.kernels.ops.attention.dsa import index_buf_accessor from sglang.kernels.ops.attention.dsv4 import ( - clear_unaccepted_c128_draft_states, fused_k_norm_rope_flashmla, fused_store_cache, ) -from sglang.kernels.ops.attention.dsv4 import ( - index_buf_accessor as dsv4_index_buf_accessor, -) -from sglang.kernels.ops.attention.dsv4.index_buf_accessor import NopeFp8RopeBf16Pack from sglang.kernels.ops.attention.dsv4.kv_layout import ( KVLayout, is_valid_kv_layout_pair, @@ -33,6 +27,11 @@ from sglang.kernels.ops.attention.dsv4.kv_norm_rope_store import ( fused_k_norm_rope_uniform_fp8, ) from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import layout +from sglang.kernels.ops.kvcache import dsa_index_k_cache, dsv4_main_kv_store +from sglang.kernels.ops.kvcache.dsv4_c128_cleanup import ( + clear_unaccepted_c128_draft_states, +) +from sglang.kernels.ops.kvcache.dsv4_main_kv_store import NopeFp8RopeBf16Pack from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE from sglang.srt.environ import envs from sglang.srt.layers.attention.hip_flash_mla import resolve_hip_flashmla_backend @@ -157,7 +156,7 @@ def get_swa_ring_size(sliding_window: int, is_speculative: bool = False) -> int: def resolve_unified_kv_fp8(unified_fp8: Optional[bool] = None) -> bool: """Per-pool fp8 layout; None follows SGLANG_DSV4_UNIFIED_KV_FP8. False keeps this pool bf16 while target fused-Q still keys off the global switch.""" - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_fp8, ) @@ -339,7 +338,7 @@ class DeepSeekV4SingleKVPool(KVCache): "the (fp8 nope, bf16 rope, 7 scales) pack is the V4 layout; " f"a {self.kv_layout.value} pool is written through set_key_buffer_fused" ) - dsv4_index_buf_accessor.SetKAndS.execute( + dsv4_main_kv_store.SetKAndS.execute( pool=self, buf=self.kv_buffer[layer_id], loc=loc, @@ -675,7 +674,7 @@ class DeepSeekV4IndexerPool(KVCache): max_seq_len: int, ) -> Tuple[torch.Tensor, torch.Tensor]: buf = self.index_k_with_scale_buffer[layer_id] - return index_buf_accessor.GetKAndS.execute( + return dsa_index_k_cache.GetKAndS.execute( self, buf, page_indices=page_indices, @@ -692,7 +691,7 @@ class DeepSeekV4IndexerPool(KVCache): index_k_scale: torch.Tensor, ) -> None: buf = self.index_k_with_scale_buffer[layer_id - self.start_layer] - index_buf_accessor.SetKAndS.execute( + dsa_index_k_cache.SetKAndS.execute( pool=self, buf=buf, loc=loc, index_k=index_k, index_k_scale=index_k_scale ) @@ -719,7 +718,7 @@ class DeepSeekV4IndexerPool(KVCache): if self.uses_aiter_fp4_layout: # the FlyDSL kernels read the fp4 payload and the packed ue8m0 scales # from two buffers - from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( + from sglang.kernels.ops.kvcache.dsv4_fp4_index_k_cache_hip import ( store_fp4_index_k_cache_split, ) @@ -732,7 +731,7 @@ class DeepSeekV4IndexerPool(KVCache): rne=self.index_k_rne, page8=self.index_k_page8, ) - from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( + from sglang.kernels.ops.kvcache.dsv4_fp4_index_k_cache import ( store_fp4_index_k_cache, ) @@ -752,7 +751,7 @@ class DeepSeekV4IndexerPool(KVCache): assert self.use_fp4_indexer, "packed readback only applies to the fp4 layout" buf = self.index_k_with_scale_buffer[layer_id - self.start_layer] if buf.is_cuda: - from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( + from sglang.kernels.ops.kvcache.dsv4_fp4_index_k_cache import ( gather_fp4_index_k, ) @@ -1008,7 +1007,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): ) self.c4_logical_size = c4_logical_size self.c128_size = c128_size - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_fp8, is_unified_kv_triton, ) @@ -1987,7 +1986,7 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): f"a {self.kv_layout.value} pool is written through the fused setters" ) if self.request_window is not None: - dsv4_index_buf_accessor.SetKAndS.execute( + dsv4_main_kv_store.SetKAndS.execute( pool=self.request_window.state, buf=self.get_swa_raw_buffer(layer_id), loc=loc, @@ -2242,10 +2241,12 @@ class DeepSeekV4TokenToKVPool(BaseSWAKVPool): """unified_kv counterpart of set_swa_key_buffer_radix_fused_norm_rope; rows with swa_loc < 0 (uncommitted verify tokens) are skipped by the scatter.""" from sglang.kernels.ops.attention.dsv4 import fused_norm_rope_inplace - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime + from sglang.kernels.ops.kvcache.dsv4_unified_kv_store import ( + scatter_bf16_into_unified, + ) fused_norm_rope_inplace(kv, kv_weight, eps, freqs_cis, positions) - runtime.scatter_bf16_into_unified( + scatter_bf16_into_unified( kv=kv, loc=swa_loc, unified_kv=self.get_unified_kv(layer_id), diff --git a/python/sglang/srt/mem_cache/dsa_cache_layer_split.py b/python/sglang/srt/mem_cache/dsa_cache_layer_split.py index 9b7760df0a..9e6eb30321 100644 --- a/python/sglang/srt/mem_cache/dsa_cache_layer_split.py +++ b/python/sglang/srt/mem_cache/dsa_cache_layer_split.py @@ -35,7 +35,7 @@ from typing import TYPE_CHECKING, Optional import torch -from sglang.kernels.ops.attention.dsa import index_buf_accessor +from sglang.kernels.ops.kvcache import dsa_index_k_cache from sglang.srt.layers.cp.utils import get_layer_owner, get_layer_shard_range from sglang.srt.mem_cache.index_key_cache import IndexKeyCache from sglang.srt.mem_cache.memory_pool import ( @@ -106,7 +106,7 @@ class LayerSplitIndexKeyCache(IndexKeyCache): ): buf = self.get_buffer(layer_id) self.pool.prefetch_kv_buffer(layer_id) - return index_buf_accessor.GetKAndS.execute( + return dsa_index_k_cache.GetKAndS.execute( self.pool, buf, page_indices=page_indices, diff --git a/python/sglang/srt/mem_cache/dsv41_request_window.py b/python/sglang/srt/mem_cache/dsv41_request_window.py index 8c203ff1da..f71fc88dbc 100644 --- a/python/sglang/srt/mem_cache/dsv41_request_window.py +++ b/python/sglang/srt/mem_cache/dsv41_request_window.py @@ -4,7 +4,7 @@ import msgspec import torch from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout -from sglang.kernels.ops.attention.dsv4.request_window_copy import ( +from sglang.kernels.ops.kvcache.dsv4_request_window_copy import ( commit_packed_rows, copy_packed_rows, ) diff --git a/python/sglang/srt/mem_cache/index_key_cache.py b/python/sglang/srt/mem_cache/index_key_cache.py index 158ecf1d34..082d139944 100644 --- a/python/sglang/srt/mem_cache/index_key_cache.py +++ b/python/sglang/srt/mem_cache/index_key_cache.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING import torch -from sglang.kernels.ops.attention.dsa import index_buf_accessor +from sglang.kernels.ops.kvcache import dsa_index_k_cache from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE from sglang.srt.layers.attention.dsa.utils import ( INDEXER_K_CACHE_PRESHUFFLE_TILE, @@ -113,7 +113,7 @@ class IndexKeyCache: max_seq_len: int, ): buf = self.get_buffer(layer_id) - return index_buf_accessor.GetKAndS.execute( + return dsa_index_k_cache.GetKAndS.execute( self.pool, buf, page_indices=page_indices, @@ -130,7 +130,7 @@ class IndexKeyCache: index_k_scale: torch.Tensor, ) -> None: buf = self._kernel_view(self.buffer[layer_id - self.pool.start_layer]) - index_buf_accessor.SetKAndS.execute( + dsa_index_k_cache.SetKAndS.execute( pool=self.pool, buf=buf, loc=loc, diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 67fe450dc7..b8801be1cd 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -179,7 +179,7 @@ def _pd_prefill_skips_mamba_spec_state() -> bool: def unified_fp8_for_dsv4_pool(*, is_draft_worker: bool, spec_algorithm) -> bool: """Per-pool fp8 layout. DSpark draft writers scatter bf16, so that pool stays a bf16 ring; MTP/EAGLE NextN follows the env.""" - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_fp8, ) diff --git a/python/sglang/srt/mem_cache/kv_index_translator.py b/python/sglang/srt/mem_cache/kv_index_translator.py index 8d58e9d98c..821c8509cd 100644 --- a/python/sglang/srt/mem_cache/kv_index_translator.py +++ b/python/sglang/srt/mem_cache/kv_index_translator.py @@ -67,7 +67,7 @@ from typing import Dict, Optional import msgspec import torch -from sglang.kernels.ops.kvcache.kv_indices import ( +from sglang.kernels.ops.attention.kv_indices import ( create_flashinfer_kv_indices_triton, ) from sglang.kernels.ops.kvcache.kv_read_table import ( diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 5f519dd6ba..23eeaacea8 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -39,10 +39,6 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.attention.dsa.quant_k_cache import ( - quantize_k_cache, - quantize_k_cache_separate, -) from sglang.kernels.ops.kvcache.cache_move import ( copy_all_layer_kv_cache_func, set_kv_buffer_prefix_valid_tiled, @@ -50,6 +46,10 @@ from sglang.kernels.ops.kvcache.cache_move import ( ) from sglang.kernels.ops.kvcache.cache_ops import launch_reshape_and_cache_flash from sglang.kernels.ops.kvcache.kvcache import can_use_store_cache, store_cache +from sglang.kernels.ops.quantization.dsa_quant_k_cache import ( + quantize_k_cache, + quantize_k_cache_separate, +) from sglang.kernels.ops.quantization.fp8_kernel import ( fp8_dtype, is_fp8_fnuz, @@ -3004,7 +3004,7 @@ class MHATokenToKVPool(KVCache): # and viewed as ``store_dtype`` by ``set_kv_buffer``. if self.kv_cache_layout == "vectorized_5d": # Late-import to keep the NHD path import-clean. - from sglang.kernels.ops.attention.utils import ( + from sglang.kernels.ops.kvcache.shuffle_5d_cache import ( launch_reshape_and_cache_shuffle_5d, ) @@ -4020,7 +4020,9 @@ class MHATokenToKVPoolMXFP8(MHATokenToKVPool): # payload and the interleaved UE8M0 scales. if not self.mxfp8_sf_interleaved or cache_k.dtype == self.store_dtype: raise ValueError("MXFP8 KV cache requires K and V scale tensors.") - from sglang.kernels.ops.quantization.mxfp8_quant import quant_store_kv_mxfp8 + from sglang.kernels.ops.kvcache.mxfp8_quant_store_kv import ( + quant_store_kv_mxfp8, + ) quant_store_kv_mxfp8( cache_k, @@ -4061,7 +4063,7 @@ class MHATokenToKVPoolMXFP8(MHATokenToKVPool): """Write per-token UE8M0 K/V scales — interleaved into the FA4 BlockScaledBasicChunk layout for page_size==128, flat otherwise.""" if self.mxfp8_sf_interleaved: - from sglang.kernels.ops.quantization.mxfp8_interleave_sf import ( + from sglang.kernels.ops.kvcache.mxfp8_interleave_sf import ( store_sf_interleaved, ) @@ -4103,7 +4105,7 @@ class MHATokenToKVPoolMXFP8(MHATokenToKVPool): # scale rows must travel with their fp8 payload or dequant reads # mismatched exponents. if self.mxfp8_sf_interleaved: - from sglang.kernels.ops.quantization.mxfp8_interleave_sf import ( + from sglang.kernels.ops.kvcache.mxfp8_interleave_sf import ( store_sf_interleaved, ) diff --git a/python/sglang/srt/mem_cache/page_interleave_pool.py b/python/sglang/srt/mem_cache/page_interleave_pool.py index 2a932b34a8..2288e2f5a2 100644 --- a/python/sglang/srt/mem_cache/page_interleave_pool.py +++ b/python/sglang/srt/mem_cache/page_interleave_pool.py @@ -56,10 +56,10 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple import torch -from sglang.kernels.ops.attention.dsa import index_buf_accessor from sglang.kernels.ops.attention.dsa.dequant_k_cache import ( - dequantize_k_cache_paged, + dequantize_k_cache_paged_dsa, ) +from sglang.kernels.ops.kvcache import dsa_index_k_cache from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator from sglang.srt.mem_cache.index_key_cache import IndexKeyCache from sglang.srt.mem_cache.memory_pool import ( @@ -794,7 +794,7 @@ class _PageInterleaveIndexKeyCache(IndexKeyCache): "cannot write indexer keys for a skip-topk layer" ) if pool._shard_extend_active: - index_buf_accessor.SetKAndS.execute( + dsa_index_k_cache.SetKAndS.execute( pool=pool, buf=pool._slots[layer_id % 2].tensors["index_k"], loc=pool.translate_loc_to_scratch(loc), @@ -927,7 +927,7 @@ class PageInterleaveDSATokenToKVPool(PageInterleaveMLATokenToKVPool, DSATokenToK slot = self._acquire_slot_for_read(layer.layer_id) rows = self.translate_loc_to_scratch(loc) - kv = dequantize_k_cache_paged(self._scratch_kv(slot), rows) + kv = dequantize_k_cache_paged_dsa(self._scratch_kv(slot), rows) if dst_dtype is not None: kv = kv.to(dst_dtype) return ( diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 597e536918..afcaa90b31 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -3691,7 +3691,7 @@ class UnifiedRadixCache(BasePrefixCache): writes late-layer window KV only for an extend's tail rows, so a matched prefix may lack it; a streaming turn borrowing its own record has it. """ - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_triton, ) diff --git a/python/sglang/srt/model_executor/forward_batch_deepseek_mha_mixin.py b/python/sglang/srt/model_executor/forward_batch_deepseek_mha_mixin.py index b9c23152dd..1c08a363f3 100644 --- a/python/sglang/srt/model_executor/forward_batch_deepseek_mha_mixin.py +++ b/python/sglang/srt/model_executor/forward_batch_deepseek_mha_mixin.py @@ -5,7 +5,7 @@ from typing import List, Optional import torch -from sglang.kernels.ops.kvcache.kv_indices import ( +from sglang.kernels.ops.attention.kv_indices import ( create_chunked_prefix_cache_kv_indices, create_flashinfer_kv_indices_triton, ) diff --git a/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py b/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py index fc4a574a53..bb90840a36 100644 --- a/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py +++ b/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Optional import torch -from sglang.kernels.ops.speculative.ngram_embedding import update_token_table +from sglang.kernels.ops.embeddings.ngram_embedding import update_token_table from sglang.srt.configs.model_config import ModelConfig from sglang.srt.managers.schedule_batch import ForwardMode from sglang.srt.mem_cache.memory_pool import ReqToTokenPool diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index c993fef11b..63782a3dd1 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -1056,7 +1056,7 @@ class DSV4PoolConfigurator(MemoryPoolConfigurator): self.qk_rope_head_dim = cfg.qk_rope_head_dim self.indexer_head_dim = cfg.index_head_dim self.attn_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_fp8, is_unified_kv_triton, ) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 5cab65e43f..cf291612aa 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -49,7 +49,7 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Union import torch import tqdm -from sglang.kernels.ops.kvcache.kv_indices import ( +from sglang.kernels.ops.attention.kv_indices import ( create_chunked_prefix_cache_kv_indices, ) from sglang.srt.configs.model_config import is_qwen4_exp diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index 411ce33dd0..dfc4231688 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -10,7 +10,7 @@ import torch.nn.functional as F from torch import nn from transformers import PretrainedConfig -from sglang.kernels.ops.moe.router import ( +from sglang.kernels.ops.gemm.router_gate_matvec import ( ROUTER_GATE_MATVEC_MAX_M, router_gate_matvec, ) diff --git a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_gfx95_dense.py b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_gfx95_dense.py index 20680e5830..1a7d6953b5 100644 --- a/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_gfx95_dense.py +++ b/python/sglang/srt/models/deepseek_common/amd/deepseek_v4_gfx95_dense.py @@ -11,13 +11,13 @@ import torch from torch import nn from sglang.kernels.ops.layernorm.mhc_boundary_hip import rmsnorm_with_sinkhorn +from sglang.kernels.ops.layernorm.rmsnorm_fake_quant_amd_gfx95 import ( + rmsnorm_fake_quant_fp8, +) from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( Fp8GridActivation, Mxfp8Activation, ) -from sglang.kernels.ops.quantization.rmsnorm_fake_quant_amd_gfx95 import ( - rmsnorm_fake_quant_fp8, -) from sglang.srt.environ import envs from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py index 0ede89212c..a471c47178 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py @@ -4,8 +4,10 @@ from typing import TYPE_CHECKING, Optional import torch -from sglang.kernels.ops.attention.dsa.dequant_k_cache import dequantize_k_cache_paged -from sglang.kernels.ops.attention.utils import concat_and_cast_mha_k_triton +from sglang.kernels.ops.attention.dsa.dequant_k_cache import ( + dequantize_k_cache_paged_dsa, +) +from sglang.kernels.ops.attention.mla_qk_prep import concat_and_cast_mha_k_triton from sglang.srt.environ import envs from sglang.srt.layers.attention.index_topk_share import write_mtp_seed from sglang.srt.layers.attention.tbo_backend import TboAttnBackend @@ -664,7 +666,7 @@ class DeepseekMHAForwardMixin: if _use_aiter_gfx95: # ROCm (gfx950) stores the FP8 MLA KV in the raw # (kv_lora_rank + qk_rope_head_dim) layout, not the scaled 656-byte - # layout that dequantize_k_cache_paged expects (it asserts dim==656). + # layout that dequantize_k_cache_paged_dsa expects (it asserts dim==656). # Dequantize the raw layout via the pool's HIP-aware path instead — # the same routine _get_mla_kv_buffer uses for the BF16 MHA path. # Without this, a chunked-prefill split (extend_prefix_lens != 0) that @@ -687,7 +689,7 @@ class DeepseekMHAForwardMixin: kv_cache_fp8 = get_token_to_kv_pool().get_key_buffer(self.attn_mha.layer_id) - kv_latent_bf16 = dequantize_k_cache_paged(kv_cache_fp8, kv_indices) + kv_latent_bf16 = dequantize_k_cache_paged_dsa(kv_cache_fp8, kv_indices) kv_a = kv_latent_bf16[:, :, : self.kv_lora_rank].squeeze(1).contiguous() k_pe = kv_latent_bf16[:, :, self.kv_lora_rank :] diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha_rocm.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha_rocm.py index d631421747..c85774b91b 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha_rocm.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha_rocm.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING import torch -from sglang.kernels.ops.attention.utils import concat_and_cast_mha_k_triton +from sglang.kernels.ops.attention.mla_qk_prep import concat_and_cast_mha_k_triton from sglang.srt.layers.dcp import all_gather_kv_cache_for_mha_extend from sglang.srt.layers.layer_boundary import get_attn_tp_context from sglang.srt.layers.quantization.fp8_utils import ( diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py index 1f95f47379..0fae375ec4 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Optional import torch -from sglang.kernels.ops.kvcache.cache_ops import absorbed_bmm_concat_cast_q_fp8 +from sglang.kernels.ops.attention.mla_qk_prep import absorbed_bmm_concat_cast_q_fp8 from sglang.kernels.ops.quantization.fp8_kernel import ( per_tensor_quant_mla_fp8, per_token_group_quant_mla_deep_gemm_masked_fp8, diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 47feb0773e..f5b1052e3c 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -40,7 +40,7 @@ import torch.nn.functional as F from torch import nn from transformers import PretrainedConfig -from sglang.kernels.ops.moe.dsv4 import ( +from sglang.kernels.ops.activation.silu_and_mul_clamp import ( silu_and_mul_clamp, silu_and_mul_contig_post_quant, ) @@ -351,7 +351,7 @@ class DeepseekV2MLP(nn.Module): and self.swiglu_limit is None and not isinstance(x, tuple) ): - from sglang.kernels.ops.quantization.nvfp4_gemm_swiglu_nvfp4_quant import ( + from sglang.kernels.ops.gemm.nvfp4_gemm_swiglu_nvfp4_quant import ( nvfp4_gemm_swiglu_nvfp4_quant, ) from sglang.srt.layers.quantization.fp4_utils import fp4_quantize diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index bc08449dde..362ea60bf0 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -29,7 +29,6 @@ from sglang.kernels.ops.attention.dsv4 import ( fused_norm_rope_inplace, fused_q_norm_rope, fused_rope_inplace, - sglang_per_token_group_quant_fp8_dsv4_wo_a, ) from sglang.kernels.ops.attention.dsv4.wo_a import MAX_M as _FUSED_WO_A_MAX_TOKENS from sglang.kernels.ops.attention.dsv4.wo_a import fused_rope_wo_a_bf16 @@ -39,6 +38,9 @@ from sglang.kernels.ops.gemm.dsv4_wo_a import ( wo_a_bf16_small_batch, wo_a_bf16_small_batch_mxfp8, ) +from sglang.kernels.ops.quantization.fp8_group_major_quant_ue8m0 import ( + sglang_per_token_group_quant_fp8_dsv4_wo_a, +) from sglang.kernels.ops.quantization.fp8_kernel import ( sglang_per_token_group_quant_fp8, ) @@ -1034,7 +1036,7 @@ class MqaAttentionBase(nn.Module): parallel_group="attn_tp", ) - from sglang.kernels.ops.attention.deepseek_v4_rope import precompute_freqs_cis + from sglang.kernels.ops.attention.dsv4.rope import precompute_freqs_cis rope_theta, rope_scaling = get_rope_config(config) self.rope_scaling = dict(rope_scaling) if rope_scaling else None @@ -1344,7 +1346,7 @@ class MQALayer(MqaAttentionBase): ) self.mega_cos_sin_cache = None if self.use_mega_attention: - from sglang.kernels.ops.attention.deepseek_v4_rope import ( + from sglang.kernels.ops.attention.dsv4.rope import ( precompute_cos_sin_table, ) @@ -1554,7 +1556,7 @@ class MQALayer(MqaAttentionBase): is_batch_invariant_mode_enabled() or get_exec().deterministic.enable_deterministic_inference ): - from sglang.kernels.ops.layernorm.mxfp8_epilogue import rmsnorm_mxfp8 + from sglang.kernels.ops.layernorm.rmsnorm_mxfp8 import rmsnorm_mxfp8 y, quant, scale = rmsnorm_mxfp8( q, self.q_norm.weight, self.q_norm.variance_epsilon @@ -2044,7 +2046,7 @@ class MQALayer(MqaAttentionBase): use_cp = is_cp_active(forward_batch) kv: Optional[torch.Tensor] - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_fp8, is_unified_kv_triton, ) @@ -2456,7 +2458,7 @@ class MQALayer(MqaAttentionBase): ) ) ) - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_fp8, is_unified_kv_triton, ) @@ -3295,7 +3297,7 @@ class DeepseekV4DecoderLayerBase(nn.Module): self.hc_sinkhorn_iters, self.hc_eps, ) - from sglang.kernels.ops.layernorm.mhc import hc_combine + from sglang.kernels.ops.layernorm.mhc import mhc_combine # y is the post-norm activation fed into the MoE. Allocate it in the # symmetric memory pool so the downstream all-reduce uses the low-latency @@ -3306,7 +3308,7 @@ class DeepseekV4DecoderLayerBase(nn.Module): with use_symmetric_memory( get_parallel().tp_group, disabled=not is_allocation_symmetric() ): - y = hc_combine(x_flat, pre.squeeze(1), self.hc_mult, dtype) + y = mhc_combine(x_flat, pre.squeeze(1), self.hc_mult, dtype) return y, post.squeeze(1), comb.squeeze(1), False def hc_post( @@ -4364,9 +4366,9 @@ class DeepseekV4Model(nn.Module): pre_hc_head = hidden_states.flatten(1) if self.hc_pre_from_prev_sublayer: - from sglang.kernels.ops.layernorm.mhc import hc_combine + from sglang.kernels.ops.layernorm.mhc import mhc_combine - hidden_states = hc_combine( + hidden_states = mhc_combine( pre_hc_head.float(), last_pre, self.hc_mult, hidden_states.dtype ) else: @@ -4425,7 +4427,7 @@ class DeepseekV4ForCausalLM(nn.Module): # batched/contiguous-load rope kernels (faster on gfx95; . # Module-level toggles default OFF; flipped True here for DSV4 if _is_hip: - from sglang.kernels.ops.attention.deepseek_v4_rope import set_batched_rope + from sglang.kernels.ops.attention.dsv4.rope import set_batched_rope from sglang.srt.layers.quantization.fp8_utils import ( set_dense_mxscale, set_force_ck_w8a8, diff --git a/python/sglang/srt/models/deepseek_v4_dspark.py b/python/sglang/srt/models/deepseek_v4_dspark.py index 33da73e761..23a6a30264 100644 --- a/python/sglang/srt/models/deepseek_v4_dspark.py +++ b/python/sglang/srt/models/deepseek_v4_dspark.py @@ -11,14 +11,14 @@ import torch.nn.functional as F from torch import nn from sglang.kernels.ops.attention.dsv4 import fused_q_norm_rope, fused_rope_inplace -from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( - is_unified_kv_triton, -) from sglang.kernels.ops.speculative.dspark.dspark_draft_model import CommitKvProj from sglang.srt.configs.deepseek_v4 import DeepSeekV4Config from sglang.srt.distributed.device_communicators.vocab_gather import make_vocab_gather from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( + is_unified_kv_triton, +) from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layer_boundary import layer_stack from sglang.srt.layers.layer_boundary.residual import batch as residual_batch @@ -931,9 +931,9 @@ class DeepseekV4ForCausalLMDSpark(nn.Module): if self.hc_pre_from_prev_sublayer: pre, x = x.pre, x.residual if self.hc_pre_from_prev_sublayer: - from sglang.kernels.ops.layernorm.mhc import hc_combine + from sglang.kernels.ops.layernorm.mhc import mhc_combine - x = hc_combine(x.flatten(1).float(), pre, self.hc_mult, x.dtype) + x = mhc_combine(x.flatten(1).float(), pre, self.hc_mult, x.dtype) return LogitsProcessorOutput(next_token_logits=None, hidden_states=x) def collapse_hc_head(self, x: torch.Tensor) -> torch.Tensor: diff --git a/python/sglang/srt/models/deepseek_v4_mhc.py b/python/sglang/srt/models/deepseek_v4_mhc.py index 8cb607926b..a4da0b1e44 100644 --- a/python/sglang/srt/models/deepseek_v4_mhc.py +++ b/python/sglang/srt/models/deepseek_v4_mhc.py @@ -400,7 +400,7 @@ def _combine( quantized: Optional[list], ) -> torch.Tensor: """The combine computed from the residual's rows.""" - from sglang.kernels.ops.layernorm.mhc import hc_combine + from sglang.kernels.ops.layernorm.mhc import mhc_combine cfg, norm = hc.cfg, hc.norm x, apply_pre = state.residual, state.pre @@ -444,7 +444,7 @@ def _combine( from sglang.kernels.ops.layernorm.hc_combine_norm import hc_combine_norm return hc_combine_norm(x_flat, apply_pre, norm.weight, norm.variance_epsilon) - return norm(hc_combine(x_flat, apply_pre, cfg.mult, x.dtype)) + return norm(mhc_combine(x_flat, apply_pre, cfg.mult, x.dtype)) # --------------------------------------------------------------------------- diff --git a/python/sglang/srt/models/deepseek_v4_replay_tail.py b/python/sglang/srt/models/deepseek_v4_replay_tail.py index 77819d45fb..cf4ed28c87 100644 --- a/python/sglang/srt/models/deepseek_v4_replay_tail.py +++ b/python/sglang/srt/models/deepseek_v4_replay_tail.py @@ -174,7 +174,7 @@ class DecoderReplayTail: ) -> Tuple[bool, Optional[List[int]]]: """Whether this step uses a tail graph and, under DP attention, every rank's tail rows; also feeds the warm gate.""" - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_triton, ) @@ -261,7 +261,7 @@ class DecoderReplayTail: def breaks_out_of_prefill_graph( self, tail: LateLayerTail, *, num_tokens: int ) -> bool: - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import ( + from sglang.srt.layers.attention.dsv4.unified_kv_gate import ( is_unified_kv_triton, ) diff --git a/python/sglang/srt/models/grok.py b/python/sglang/srt/models/grok.py index 9976a3c2e0..cbfc80efb6 100644 --- a/python/sglang/srt/models/grok.py +++ b/python/sglang/srt/models/grok.py @@ -22,10 +22,10 @@ import torch.nn.functional as F from torch import nn from transformers import PretrainedConfig -from sglang.kernels.ops.elementwise.elementwise import ( +from sglang.kernels.ops.activation.gelu_and_mul_triton import gelu_and_mul_triton +from sglang.kernels.ops.layernorm.fused_rmsnorm import ( fused_dual_residual_rmsnorm, fused_rmsnorm, - gelu_and_mul_triton, ) from sglang.kernels.ops.moe.router import fused_moe_router_shim from sglang.srt.distributed import ( diff --git a/python/sglang/srt/models/hunyuan_v4.py b/python/sglang/srt/models/hunyuan_v4.py index fef1c23237..9c10e3de85 100644 --- a/python/sglang/srt/models/hunyuan_v4.py +++ b/python/sglang/srt/models/hunyuan_v4.py @@ -393,8 +393,8 @@ class HYV4Attention(DeepseekV2AttentionMLA): @staticmethod def _resolve_gate_fallback_backend() -> str: try: - from sglang.kernels.ops.moe.triton_sigmoid_gate_mul import ( # noqa: F401 - sigmoid_gate_mul, + from sglang.kernels.ops.elementwise.elementwise import ( # noqa: F401 + fused_sigmoid_mul, ) except ImportError: return "eager" @@ -498,11 +498,11 @@ class HYV4Attention(DeepseekV2AttentionMLA): f"output shape: {tuple(gate.shape)} != {tuple(attn_out.shape)}" ) if backend == "triton" and attn_out.is_cuda: - from sglang.kernels.ops.moe.triton_sigmoid_gate_mul import ( - sigmoid_gate_mul, + from sglang.kernels.ops.elementwise.elementwise import ( + fused_sigmoid_mul, ) - return sigmoid_gate_mul(attn_out, gate) + return fused_sigmoid_mul(attn_out, gate) return attn_out * torch.sigmoid(gate) def prepare_attention_output_gate(self, hidden_states): diff --git a/python/sglang/srt/models/inkling_common/attn.py b/python/sglang/srt/models/inkling_common/attn.py index af80f5f37c..d2b977055d 100644 --- a/python/sglang/srt/models/inkling_common/attn.py +++ b/python/sglang/srt/models/inkling_common/attn.py @@ -9,12 +9,12 @@ from torch import nn from sglang.kernels.ops.attention.flash_attn.cute.batch_invariance import ( is_batch_invariant, ) -from sglang.kernels.ops.attention.log_scaling_tau import ( - apply_log_scaling_tau as _apply_log_scaling_tau, -) from sglang.kernels.ops.attention.score_mod import ( relative_bias_score_mod as triton_relative_bias_score_mod, ) +from sglang.kernels.ops.elementwise.row_scale import ( + apply_log_scaling_tau as _apply_log_scaling_tau, +) from sglang.kernels.ops.gemm.inkling_rel_proj import rel_proj_small_t from sglang.kernels.ops.memory.row_compact import row_compact_bf16 from sglang.srt.distributed.utils import get_group_rank_size diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 690f7064ac..33e27f6c81 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -2287,7 +2287,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA): self._gate_hidden_states = None if gate_input is not None and not isinstance(x, tuple): gate = self._compute_output_gate(gate_input) - from sglang.kernels.ops.attention import mla_output_gate + from sglang.kernels.ops.elementwise import mla_output_gate if mla_output_gate.covered(x, gate): # One kernel for x * sigmoid(gate); double rounding diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index c388a31135..4fd904e9ac 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -24,7 +24,7 @@ import torch from torch import nn from transformers import PretrainedConfig -from sglang.kernels.ops.quantization.mxfp8_swizzled_triton import ( +from sglang.kernels.ops.activation.swiglu_mxfp8_swizzled import ( can_use_silu_mul_mxfp8, swiglu_oai_mxfp8, swiglu_oai_mxfp8_deepgemm, diff --git a/python/sglang/srt/models/muse_spark.py b/python/sglang/srt/models/muse_spark.py index 7899b710c7..8442ae4251 100644 --- a/python/sglang/srt/models/muse_spark.py +++ b/python/sglang/srt/models/muse_spark.py @@ -19,25 +19,23 @@ import torch import torch.nn.functional as F from torch import nn -from sglang.kernels.ops.muse_spark.all_gather_norm_residual import ( +from sglang.kernels.ops.attention.muse_spark_qk_norm_rope_store import FusedQKNorm +from sglang.kernels.ops.communication.muse_spark_all_gather import ( + gemm_ag_supported, +) +from sglang.kernels.ops.communication.muse_spark_all_gather_norm_residual import ( + all_gather_norm_residual, all_gather_norm_residual_supported, ) -from sglang.kernels.ops.muse_spark.fused_muse_spark_routing import ( - FusedMuseSparkRouting, +from sglang.kernels.ops.gemm.muse_spark_router_gemm import ( + MAX_TOKENS as ROUTER_GEMM_MAX_TOKENS, ) -from sglang.kernels.ops.muse_spark.fused_norm_gate import FusedRMSNormGate -from sglang.kernels.ops.muse_spark.fused_norm_residual import ( +from sglang.kernels.ops.layernorm.muse_spark_norm_gate import FusedRMSNormGate +from sglang.kernels.ops.layernorm.muse_spark_norm_residual import ( FusedPostNormResidualPreNorm, ) -from sglang.kernels.ops.muse_spark.fused_qk_norm import FusedQKNorm -from sglang.kernels.ops.muse_spark.gemm_ag import ( - gemm_ag, - gemm_ag_supported, - gemm_local, - reserve_projection_workspace, -) -from sglang.kernels.ops.muse_spark.router_gemm import ( - MAX_TOKENS as ROUTER_GEMM_MAX_TOKENS, +from sglang.kernels.ops.moe.muse_spark_routing import ( + FusedMuseSparkRouting, ) from sglang.srt.configs.muse_spark import ( MuseSparkTextConfig, @@ -48,6 +46,11 @@ from sglang.srt.environ import envs from sglang.srt.layers import layernorm_sp from sglang.srt.layers.attention.trtllm_mha_backend import TRTLLMHAAttnBackend from sglang.srt.layers.aux_hidden_states import pack_aux_hidden_states +from sglang.srt.layers.communication.muse_spark_gemm_ag import ( + gemm_ag, + gemm_local, + reserve_projection_workspace, +) from sglang.srt.layers.cp.utils import is_cp_active from sglang.srt.layers.dp_attention import ( dp_gather_replicate, @@ -176,6 +179,22 @@ def _use_split_bf16_router(num_tokens: int) -> bool: return True +def _fused_norm_boundary( + post_norm: nn.Module, residual_add: nn.Module, pre_norm: nn.Module +) -> FusedPostNormResidualPreNorm: + return FusedPostNormResidualPreNorm( + post_norm, + residual_add, + pre_norm, + post_norm_is_input_scale=isinstance( + post_norm, MuseSpark1xRMSNormWithInputScale + ), + use_residual_gate=isinstance( + residual_add, MuseSpark1xNormPreservingResidualConnection + ), + ) + + class MuseSpark1xRouter(nn.Module): """Router stored in HF linear layout: ``[experts, hidden]``.""" @@ -942,7 +961,7 @@ class MuseSpark1xDecoderLayer(nn.Module): # Each boundary -- post-norm, residual add, next pre-norm -- is one fused # kernel. The FFN boundary's pre-norm belongs to the next layer, so the # model binds it in bind_next_pre_attn_norm once every layer exists. - self._fused_attn_boundary = FusedPostNormResidualPreNorm( + self._fused_attn_boundary = _fused_norm_boundary( self.post_attn_norm, self.post_attention_residual_gate, self.pre_feedforward_layernorm, @@ -954,7 +973,7 @@ class MuseSpark1xDecoderLayer(nn.Module): return config.mlp_layer_types[layer_idx] == "sparse" def bind_next_pre_attn_norm(self, next_pre_attn_norm: MuseSpark1xRMSNorm) -> None: - self._fused_ffn_boundary = FusedPostNormResidualPreNorm( + self._fused_ffn_boundary = _fused_norm_boundary( self.post_feedforward_layernorm, self.post_feedforward_residual_gate, next_pre_attn_norm, @@ -990,8 +1009,11 @@ class MuseSpark1xDecoderLayer(nn.Module): ffn_output = self.mlp(ffn_input) # A TP-sliced branch means the unpooler gather was deferred to the FFN boundary. if ffn_output.shape[-1] != stream.shape[-1]: - next_attn_input, stream = self._fused_ffn_boundary.forward_with_ag( - stream, ffn_output, get_parallel().tp_group.ca_comm + next_attn_input, stream = all_gather_norm_residual( + get_parallel().tp_group.ca_comm, + ffn_output, + stream, + **self._fused_ffn_boundary.jit_kernel_kwargs(), ) return stream, next_attn_input if self._fused_ffn_boundary is None: @@ -1064,12 +1086,12 @@ class MuseSpark1xModel(nn.Module): self._sp_moe_dispatcher = None self._finalize_ar_workspace = None if is_cuda() and envs.SGLANG_ENABLE_MUSE_SPARK_FINALIZE_AR.get(): - from sglang.kernels.ops.muse_spark.fi_trtllm_moe_finalize_with_rmsnorm_cutedsl import ( - FinalizePushWorkspace, - ) from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( CustomAllReduceV2, ) + from sglang.srt.layers.communication.muse_spark_finalize_workspace import ( + FinalizePushWorkspace, + ) tp_group = get_parallel().tp_group if ( diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 9ae9d18c21..e5305134fe 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -527,7 +527,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module): shared_output, ) elif _is_hip: - from sglang.kernels.ops.moe.triton_sigmoid_gate_mul import ( + from sglang.kernels.ops.elementwise.elementwise import ( sigmoid_gate_mul_broadcast, ) diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index cf7444960a..57c7a397bd 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -806,11 +806,11 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module): if self.attn_output_gate: if _is_hip: - from sglang.kernels.ops.moe.triton_sigmoid_gate_mul import ( - sigmoid_gate_mul, + from sglang.kernels.ops.elementwise.elementwise import ( + fused_sigmoid_mul, ) - attn_output = sigmoid_gate_mul(attn_output, gate) + attn_output = fused_sigmoid_mul(attn_output, gate) else: gate = torch.sigmoid(gate) attn_output = attn_output * gate diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index 2477ce2837..a921a70c2a 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -13,7 +13,7 @@ import torch.nn.functional as F from torch import nn from transformers import PretrainedConfig -from sglang.kernels.ops.attention.utils import concat_and_cast_mha_k_triton +from sglang.kernels.ops.attention.mla_qk_prep import concat_and_cast_mha_k_triton from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.activation import SiluAndMul diff --git a/python/sglang/srt/speculative/spec_utils.py b/python/sglang/srt/speculative/spec_utils.py index 59b7c50cbb..bb3eed5486 100644 --- a/python/sglang/srt/speculative/spec_utils.py +++ b/python/sglang/srt/speculative/spec_utils.py @@ -993,7 +993,7 @@ def commit_mamba_states_after_verify( ): if batch.forward_mode.is_idle() or accept_index.numel() == 0: return - from sglang.kernels.ops.attention.fla.gdn_replayssm_spec_fold import ( + from sglang.kernels.ops.attention.linear.gdn_replayssm_spec_fold import ( commit_gdn_replayssm_fold_after_verify, ) @@ -1029,7 +1029,7 @@ def commit_mamba_states_after_verify( ): if batch.forward_mode.is_idle() or accept_index.numel() == 0: return - from sglang.kernels.ops.attention.fla.gdn_replayssm_spec_decode import ( + from sglang.kernels.ops.attention.linear.gdn_replayssm_spec_decode import ( commit_gdn_replayssm_circular, commit_gdn_replayssm_spec, ) @@ -1114,7 +1114,7 @@ def commit_mamba_states_after_verify( ): if batch.forward_mode.is_idle() or accept_index.numel() == 0: return - from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( + from sglang.kernels.ops.attention.linear.kda_replayssm_spec_decode import ( commit_kda_replayssm_after_verify, ) diff --git a/python/sglang/test/kernels/deepseek_v4/common.py b/python/sglang/test/kernels/deepseek_v4/common.py index a890cf226f..e3376aecda 100644 --- a/python/sglang/test/kernels/deepseek_v4/common.py +++ b/python/sglang/test/kernels/deepseek_v4/common.py @@ -255,3 +255,63 @@ def fp4_e2m1_code_ref(x: torch.Tensor) -> torch.Tensor: idx += (ax > threshold).to(torch.uint8) sign = ((x < 0) & (idx != 0)).to(torch.uint8) * 8 return idx | sign + + +# DSV4 FP4 index-K row: 128 values as 64 packed e2m1 bytes plus four ue8m0 +# exponents (one per 32 values), stored page-major as 64 payload bytes per slot +# followed by 4 scale bytes per slot. +_INDEX_HEAD_DIM = 128 +_INDEX_FP4_DIM = _INDEX_HEAD_DIM // 2 +_INDEX_GROUP_SIZE = 32 +_INDEX_SCALE_GROUPS = _INDEX_HEAD_DIM // _INDEX_GROUP_SIZE +_INDEX_SCALE_BYTES = 4 +_INDEX_PAGE_SIZE = 64 + + +def quantize_fp4_indexer_ref(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """Torch ``quantize_fp4_indexer_tensor`` (default rounding): packed e2m1 codes + uint8 ``[n, 64]`` and the four ue8m0 exponents packed into int32 ``[n]``.""" + x = x.contiguous().view(-1, _INDEX_HEAD_DIM).float() + groups = x.view(-1, _INDEX_SCALE_GROUPS, _INDEX_GROUP_SIZE) + scale_raw = (groups.abs().amax(dim=-1) / 6.0).clamp_min(1.0e-4) + scale_exp = ceil_ue8m0_exp_ref(scale_raw) + scale = (scale_exp << 23).contiguous().view(torch.float32) + + scaled = (groups / scale.unsqueeze(-1)).view(-1, _INDEX_HEAD_DIM) + code = fp4_e2m1_code_ref(scaled) + packed = (code[:, 0::2].to(torch.int16) | (code[:, 1::2].to(torch.int16) << 4)).to( + torch.uint8 + ) + + packed_sf = scale_exp[:, 0].clone() + for group_id in range(1, _INDEX_SCALE_GROUPS): + packed_sf |= scale_exp[:, group_id] << (8 * group_id) + return packed, packed_sf + + +def store_fp4_index_cache_ref( + x_fp4: torch.Tensor, + x_sf: torch.Tensor, + loc: torch.Tensor, + num_pages: int, +) -> torch.Tensor: + """The 64-slot paged index-K cache after writing row ``i`` to slot ``loc[i]``.""" + expected = torch.zeros( + num_pages, + _INDEX_PAGE_SIZE * (_INDEX_FP4_DIM + _INDEX_SCALE_BYTES), + device=x_fp4.device, + dtype=torch.uint8, + ) + sf_shifts = torch.arange(0, 32, 8, device=x_fp4.device, dtype=torch.int32) + for token_id in range(x_fp4.shape[0]): + cache_loc = int(loc[token_id].item()) + page = cache_loc // _INDEX_PAGE_SIZE + offset = cache_loc % _INDEX_PAGE_SIZE + expected[page, offset * _INDEX_FP4_DIM : (offset + 1) * _INDEX_FP4_DIM] = x_fp4[ + token_id + ] + sf_start = _INDEX_PAGE_SIZE * _INDEX_FP4_DIM + offset * _INDEX_SCALE_BYTES + expected[page, sf_start : sf_start + _INDEX_SCALE_BYTES] = ( + (x_sf[token_id] >> sf_shifts) & 0xFF + ).to(torch.uint8) + return expected diff --git a/python/sglang/test/kernels/deepseek_v4/dsv41_kv_quant_reference.py b/python/sglang/test/kernels/deepseek_v4/dsv41_kv_quant_reference.py index 92dc643cad..4727cc2fbb 100644 --- a/python/sglang/test/kernels/deepseek_v4/dsv41_kv_quant_reference.py +++ b/python/sglang/test/kernels/deepseek_v4/dsv41_kv_quant_reference.py @@ -4,7 +4,7 @@ from typing import Optional import torch -from sglang.kernels.ops.attention.dsv4.torch_quant import ceil_pow2 +from sglang.kernels.ops.quantization.fp4_fake_quant import ceil_pow2 KV_DIM = 512 FP4_GROUP = 16 # values per e4m3 scale diff --git a/python/sglang/test/kernels/deepseek_v4/fp4_index_k_reference.py b/python/sglang/test/kernels/deepseek_v4/fp4_index_k_reference.py new file mode 100644 index 0000000000..761e977c52 --- /dev/null +++ b/python/sglang/test/kernels/deepseek_v4/fp4_index_k_reference.py @@ -0,0 +1,40 @@ +"""Pure-torch readers for the DeepSeek-V4 FP4 index-K cache layouts.""" + +from typing import Tuple + +import torch + +from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( + _KV_BLOCK_SIZE, + index_k_page8_views, +) + + +def read_fp4_index_k_split( + payload: torch.Tensor, + scale: torch.Tensor, + slots: torch.Tensor, + *, + page_size: int, + page8: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Inverse of store_fp4_index_k_cache_split: (payload int8 [n, 64], scales + int32 [n] with chunk c's e8m0 byte at bits 8c..8c+7), the layout of + quantize_fp4_indexer_tensor.""" + assert page_size == _KV_BLOCK_SIZE, "the 16 x 4 slot transpose spans a 64-slot page" + slots = slots.to(torch.int64) + if page8: + kv8, ks8 = index_k_page8_views(payload, scale) + page, t = slots // 8, slots % 8 + kv8 = kv8.view(-1, 4, 4, 2, 16) # [page8, t % 4, chunk, t // 4, 16] + rows = kv8[page, t % 4, :, t // 4, :].reshape(-1, 64).view(torch.int8) + sf = ks8.view(-1, 4, 8)[page, :, t].to(torch.int32) # [n, 4] + packed = sf[:, 0] | (sf[:, 1] << 8) | (sf[:, 2] << 16) | (sf[:, 3] << 24) + return rows, packed + page, off = slots // page_size, slots % page_size + rows = payload.view(torch.uint8)[page, 0, :, off, :] # [n, 4, 16] + rows = rows.reshape(-1, 64).view(torch.int8) + shuffled = (off % 16) * 4 + off // 16 + sf = scale[page, 0, :, shuffled].to(torch.int32) # [n, 4] + packed = sf[:, 0] | (sf[:, 1] << 8) | (sf[:, 2] << 16) | (sf[:, 3] << 24) + return rows, packed diff --git a/python/sglang/kernels/ops/quantization/fp8_cvt.py b/python/sglang/test/kernels/deepseek_v4/fp8_cvt.py similarity index 100% rename from python/sglang/kernels/ops/quantization/fp8_cvt.py rename to python/sglang/test/kernels/deepseek_v4/fp8_cvt.py diff --git a/python/sglang/test/kernels/deepseek_v4/k_cache_reference.py b/python/sglang/test/kernels/deepseek_v4/k_cache_reference.py index 7a54a1e417..fe0a07c851 100644 --- a/python/sglang/test/kernels/deepseek_v4/k_cache_reference.py +++ b/python/sglang/test/kernels/deepseek_v4/k_cache_reference.py @@ -12,7 +12,7 @@ from sglang.kernels.ops.attention.dsv4.dequant_k_cache import ( PADDED_SCALE_PER_TOKEN, TILE_SIZE, ) -from sglang.kernels.ops.attention.dsv4.index_buf_accessor import NopeFp8RopeBf16Pack +from sglang.kernels.ops.kvcache.dsv4_main_kv_store import NopeFp8RopeBf16Pack from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz fp8_dtype = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn @@ -209,7 +209,7 @@ def dequantize_k_cache_paged_ref( page_table_1_flattened: torch.Tensor, page_size: int, ) -> torch.Tensor: - """Pure-torch reference for ``dsv4.dequant_k_cache.dequantize_k_cache_paged``. + """Pure-torch reference for ``dsv4.dequant_k_cache.dequantize_k_cache_paged_dsv4``. Decodes the same v4 paged layout with vectorized torch indexing instead of a Triton kernel. diff --git a/python/sglang/test/kernels/dsa_k_cache_reference.py b/python/sglang/test/kernels/dsa_k_cache_reference.py new file mode 100644 index 0000000000..7cb14ef870 --- /dev/null +++ b/python/sglang/test/kernels/dsa_k_cache_reference.py @@ -0,0 +1,57 @@ +"""Pure-torch reference for the DSA FP8 MLA K-cache quantizer.""" + +import torch + + +# Copied from original +def _quantize_k_cache_ref( + input_k_cache: torch.Tensor, # (num_blocks, block_size, h_k, d) + dv: int = 512, + tile_size: int = 128, +) -> torch.Tensor: + """ + Quantize the k-cache + Return a tensor with shape (num_blocks, block_size, h_k, dv + 4(dv/tile_size) + t(d-dv)) of dtype uint8_t, where t = input_k_cache.element_size() + For more detail about the layout of K/V, please refer to comments in flash_mla_interface.py or README.md + """ + assert dv % tile_size == 0 + num_tiles = dv // tile_size + num_blocks, block_size, h_k, d = input_k_cache.shape + assert h_k == 1 + input_k_cache = input_k_cache.squeeze(2) # [num_blocks, block_size, d] + input_elem_size = input_k_cache.element_size() + + result = torch.empty( + (num_blocks, block_size, dv + num_tiles * 4 + input_elem_size * (d - dv)), + dtype=torch.float8_e4m3fn, + device=input_k_cache.device, + ) + result_k_nope_part = result[..., :dv] + result_k_scale_factor = result[..., dv : dv + num_tiles * 4].view(torch.float32) + result_k_rope_part = result[..., dv + num_tiles * 4 :].view(input_k_cache.dtype) + result_k_rope_part[:] = input_k_cache[..., dv:] + + for tile_idx in range(0, num_tiles): + cur_scale_factors_inv = ( + torch.abs( + input_k_cache[..., tile_idx * tile_size : (tile_idx + 1) * tile_size] + ) + .max(dim=-1) + .values + / 448.0 + ) # [num_blocks, block_size] + result_k_scale_factor[:, :, tile_idx] = cur_scale_factors_inv + + cur_scale_factors_inv.unsqueeze_(-1) # [num_blocks, block_size, 1] + cur_quantized_nope = ( + input_k_cache[ + ..., tile_idx * tile_size : (tile_idx + 1) * tile_size + ].float() + / cur_scale_factors_inv.float() + ).to(torch.float8_e4m3fn) + result_k_nope_part[..., tile_idx * tile_size : (tile_idx + 1) * tile_size] = ( + cur_quantized_nope + ) + + result = result.view(num_blocks, block_size, 1, -1) + return result diff --git a/python/sglang/test/kernels/kv_canary/scatter_req_token_ids_reference.py b/python/sglang/test/kernels/kv_canary/scatter_req_token_ids_reference.py new file mode 100644 index 0000000000..6ab86cbfe8 --- /dev/null +++ b/python/sglang/test/kernels/kv_canary/scatter_req_token_ids_reference.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import torch + + +def scatter_req_token_ids_torch_reference( + *, + flat_in: torch.Tensor, + offsets: torch.Tensor, + req_pool_indices: torch.Tensor, + pool_out: torch.Tensor, +) -> None: + """Plain-PyTorch reference for :func:`launch_scatter_req_token_ids_kernel`.""" + bs = int(req_pool_indices.shape[0]) + offsets_host = offsets.detach().cpu().tolist() + req_pool_indices_host = req_pool_indices.detach().cpu().tolist() + flat_host = flat_in.detach().cpu() + pool_max_context_len = int(pool_out.shape[1]) + + for r in range(bs): + start = int(offsets_host[r]) + end = int(offsets_host[r + 1]) + if end <= start: + continue + rp = int(req_pool_indices_host[r]) + seg = flat_host[start:end].to(torch.int32) + write_len = min(int(seg.shape[0]), pool_max_context_len) + if write_len <= 0: + continue + pool_out[rp, :write_len] = seg[:write_len].to(pool_out.device) diff --git a/python/sglang/kernels/ops/muse_spark/triton_sum.py b/python/sglang/test/kernels/triton_sum.py similarity index 96% rename from python/sglang/kernels/ops/muse_spark/triton_sum.py rename to python/sglang/test/kernels/triton_sum.py index 41522d2c1b..a0c7e4c777 100644 --- a/python/sglang/kernels/ops/muse_spark/triton_sum.py +++ b/python/sglang/test/kernels/triton_sum.py @@ -7,7 +7,7 @@ uses tl.sum, whose reduction order is fixed by the Triton compiler and does not change across CUDA runtime versions. Usage: - from sglang.kernels.ops.muse_spark.triton_sum import triton_sum + from sglang.test.kernels.triton_sum import triton_sum score_sum = triton_sum(topk_scores, dim=-1, keepdim=True) """ diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py index 67bbc566d8..8063e9d644 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dsa_attention.py @@ -1394,7 +1394,7 @@ def run_dsa_sparse_fp8_prefill_case( ) -> None: """FP8-KV-cache prefill. With `flashmla_sparse` + EXTEND + non-empty prefix, `get_topk_transform_method` returns `RAGGED` (the only path - that exercises `dequantize_k_cache_paged` + the + that exercises `dequantize_k_cache_paged_dsa` + the `topk_indices_offset` shift). With `flashmla_kv` or `flashmla_auto` it stays on `PAGED` topk; the auto resolver picks `flashmla_kv` for FP8 KV cache (`set_dsa_prefill_impl`), so `flashmla_auto` and diff --git a/test/manual/attention/test_trtllm_mla_backend.py b/test/manual/attention/test_trtllm_mla_backend.py index 966122a0f6..8faf51f2db 100755 --- a/test/manual/attention/test_trtllm_mla_backend.py +++ b/test/manual/attention/test_trtllm_mla_backend.py @@ -10,7 +10,7 @@ from sglang.srt.runtime_context import get_parallel, get_server_args _parallel_override = get_parallel().override(attn_tp_size=1) _parallel_override.__enter__() -from sglang.kernels.ops.attention.utils import get_num_page_per_block_flashmla +from sglang.kernels.ops.attention.kv_indices import get_num_page_per_block_flashmla from sglang.srt.configs.model_config import AttentionArch from sglang.srt.layers.attention.flashinfer_mla_backend import FlashInferMLAAttnBackend from sglang.srt.layers.attention.trtllm_mla_backend import ( diff --git a/test/manual/kernels/attention/check_dsa_dequant_k_cache.py b/test/manual/kernels/attention/check_dsa_dequant_k_cache.py index 9c69e1562e..fbd16d291d 100644 --- a/test/manual/kernels/attention/check_dsa_dequant_k_cache.py +++ b/test/manual/kernels/attention/check_dsa_dequant_k_cache.py @@ -1,2 +1,2 @@ if __name__ == "__main__": - raise Exception("UT is in quant_k_cache.py") + raise Exception("UT is in ../quantization/check_dsa_quant_k_cache.py") diff --git a/test/manual/kernels/attention/check_dsa_transform_index.py b/test/manual/kernels/attention/check_dsa_transform_index.py index 63471cab69..2ba670fe8e 100644 --- a/test/manual/kernels/attention/check_dsa_transform_index.py +++ b/test/manual/kernels/attention/check_dsa_transform_index.py @@ -1,7 +1,7 @@ import torch from sglang.kernels.ops.attention.dsa.transform_index import ( - transform_index_page_table_decode_fast, + transform_index_page_table_decode, ) from sglang.test.kernels.dsa_transform_index import ( transform_index_page_table_decode_ref, @@ -13,6 +13,6 @@ if __name__ == "__main__": topk_indices = torch.full((bs, topk), -1, device="cuda") topk_indices[:, :1600] = torch.arange(1600).unsqueeze(0).repeat(bs, 1) ref_result = transform_index_page_table_decode_ref(page_table, topk_indices) - result = transform_index_page_table_decode_fast(page_table, topk_indices) + result = transform_index_page_table_decode(page_table, topk_indices) assert torch.all(result == ref_result) print("Passed") diff --git a/test/manual/kernels/attention/check_dsv4_dequant_k_cache.py b/test/manual/kernels/attention/check_dsv4_dequant_k_cache.py index 021b052c79..13fe94fe72 100644 --- a/test/manual/kernels/attention/check_dsv4_dequant_k_cache.py +++ b/test/manual/kernels/attention/check_dsv4_dequant_k_cache.py @@ -3,7 +3,7 @@ import torch from sglang.kernels.ops.attention.dsv4.dequant_k_cache import ( NOPE_ROPE_BYTES, PADDED_SCALE_PER_TOKEN, - dequantize_k_cache_paged, + dequantize_k_cache_paged_dsv4, ) from sglang.test.kernels.deepseek_v4.k_cache_reference import ( dequantize_k_cache_paged_ref, @@ -29,7 +29,7 @@ if __name__ == "__main__": 0, num_pages * page_size, (num_tokens,), dtype=torch.int32, device=device ) - out_kernel = dequantize_k_cache_paged(quant_k_cache, page_table, page_size) + out_kernel = dequantize_k_cache_paged_dsv4(quant_k_cache, page_table, page_size) out_ref = dequantize_k_cache_paged_ref(quant_k_cache, page_table, page_size) torch.testing.assert_close(out_kernel, out_ref, atol=0, rtol=0, equal_nan=True) diff --git a/test/manual/kernels/attention/test_flash_fwd_mla_sm100.py b/test/manual/kernels/attention/test_flash_fwd_mla_sm100.py new file mode 100644 index 0000000000..c2ca978229 --- /dev/null +++ b/test/manual/kernels/attention/test_flash_fwd_mla_sm100.py @@ -0,0 +1,800 @@ +# Copyright (c) 2026, Colfax International. +"""Correctness check and benchmark for FlashAttentionMLAForwardSm100 (SM100).""" + +import math +import time + +import cuda.bindings.driver as cuda +import cutlass.cute as cute +import torch +import torch.utils.benchmark as benchmark +from cutlass.cute.runtime import from_dlpack + +from sglang.kernels.ops.attention.flash_attn.cute.cute_dsl_utils import ( + dump_kernel_attributes, +) +from sglang.kernels.ops.attention.flash_attn.cute.flash_fwd_mla_sm100 import ( + FlashAttentionMLAForwardSm100, +) +from sglang.kernels.ops.attention.flash_attn.cute.testing import attention_ref + + +def test_mla_kernel( + seqlen_q=2048, + seqlen_k=2048, + topk_length=2048, + nheads=1, + batch=1, + iter=0, + compile_cache=dict(), + validate=True, + seed=0, + gather_kv=True, + pack_gqa=False, + is_causal=False, + varlen_q=False, + varlen_k=False, + disable_bitmask=False, + has_qk=True, + store_P=False, +): + torch.manual_seed(seed) + hdim = 64 + hdimv = 512 + softmax_scale = 1.0 / math.sqrt(hdim + hdimv) if has_qk else 1.0 / math.sqrt(hdimv) + + nheads_kv = 1 + qhead_per_kvhead = nheads + seqlen_k_rounded = (seqlen_k + 128 - 1) // 128 * 128 + P_k_length = seqlen_k_rounded if not gather_kv else topk_length + + torch_stream = torch.cuda.current_stream() + stream = cuda.CUstream(torch_stream.cuda_stream) + + compile_key = ( + is_causal, + gather_kv, + topk_length if gather_kv else None, + pack_gqa, + qhead_per_kvhead, + nheads_kv, + varlen_q, + varlen_k, + disable_bitmask, + has_qk, + ) + if compile_key not in compile_cache: + total_q_dummy = batch * seqlen_q + total_k_dummy = batch * seqlen_k + + if varlen_q: + Q = torch.randn( + total_q_dummy, nheads, hdim, dtype=torch.bfloat16, device="cuda" + ) + Qv = torch.randn( + total_q_dummy, nheads, hdimv, dtype=torch.bfloat16, device="cuda" + ) + O = torch.empty( + total_q_dummy, nheads, hdimv, dtype=torch.bfloat16, device="cuda" + ) + P = torch.empty( + total_q_dummy, nheads, P_k_length, dtype=torch.bfloat16, device="cuda" + ) + lse = torch.empty(total_q_dummy, nheads, dtype=torch.float32, device="cuda") + row_max = torch.empty( + total_q_dummy, + nheads, + P_k_length // 128, + dtype=torch.float32, + device="cuda", + ) + index_topk = ( + torch.rand(total_q_dummy, topk_length, device="cuda") + .argsort(dim=-1) + .to(torch.int32) + ) + cu_seqlens_q_dummy = torch.arange( + 0, (batch + 1) * seqlen_q, seqlen_q, dtype=torch.int32, device="cuda" + ) + else: + Q = torch.randn( + batch, seqlen_q, nheads, hdim, dtype=torch.bfloat16, device="cuda" + ) + Qv = torch.randn( + batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda" + ) + O = torch.empty( + batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda" + ) + P = torch.empty( + batch, seqlen_q, nheads, P_k_length, dtype=torch.bfloat16, device="cuda" + ) + lse = torch.empty( + batch, seqlen_q, nheads, dtype=torch.float32, device="cuda" + ) + row_max = torch.empty( + batch, + seqlen_q, + nheads, + P_k_length // 128, + dtype=torch.float32, + device="cuda", + ) + index_topk = ( + torch.rand(batch, seqlen_q, topk_length, device="cuda") + .argsort(dim=-1) + .to(torch.int32) + ) + + if varlen_k: + K = torch.randn( + total_k_dummy, nheads_kv, hdim, dtype=torch.bfloat16, device="cuda" + ) + V = torch.randn( + total_k_dummy, nheads_kv, hdimv, dtype=torch.bfloat16, device="cuda" + ) + cu_seqlens_k_dummy = torch.arange( + 0, (batch + 1) * seqlen_k, seqlen_k, dtype=torch.int32, device="cuda" + ) + else: + K = torch.randn( + batch, seqlen_k, nheads_kv, hdim, dtype=torch.bfloat16, device="cuda" + ) + V = torch.randn( + batch, seqlen_k, nheads_kv, hdimv, dtype=torch.bfloat16, device="cuda" + ) + + mQ = from_dlpack(Q, assumed_align=16).mark_layout_dynamic( + leading_dim=Q.ndim - 1 + ) + mQv = from_dlpack(Qv, assumed_align=16).mark_layout_dynamic( + leading_dim=Qv.ndim - 1 + ) + mK = from_dlpack(K, assumed_align=16).mark_layout_dynamic( + leading_dim=K.ndim - 1 + ) + mV = from_dlpack(V, assumed_align=16).mark_layout_dynamic( + leading_dim=V.ndim - 1 + ) + mO = from_dlpack(O, assumed_align=16).mark_layout_dynamic( + leading_dim=O.ndim - 1 + ) + mP = from_dlpack(P, assumed_align=16).mark_layout_dynamic( + leading_dim=P.ndim - 1 + ) + mLSE = from_dlpack(lse, assumed_align=4).mark_layout_dynamic( + leading_dim=lse.ndim - 1 + ) + mRowMax = from_dlpack(row_max, assumed_align=4).mark_layout_dynamic( + leading_dim=row_max.ndim - 1 + ) + if gather_kv: + mIndexTopk = from_dlpack(index_topk, assumed_align=16).mark_layout_dynamic( + leading_dim=index_topk.ndim - 1 + ) + else: + mIndexTopk = None + + compile_kwargs = dict(mIndexTopk=mIndexTopk) + if varlen_q: + compile_kwargs["mCuSeqlensQ"] = from_dlpack( + cu_seqlens_q_dummy, assumed_align=4 + ) + if varlen_k: + compile_kwargs["mCuSeqlensK"] = from_dlpack( + cu_seqlens_k_dummy, assumed_align=4 + ) + + if not has_qk: + mQ = mK = None + + if store_P is False: + mP = mRowMax = None + + kernel = cute.compile( + FlashAttentionMLAForwardSm100( + is_causal=is_causal, + use_cpasync_load_KV=gather_kv, + topk_length=topk_length if gather_kv else 2048, + is_topk_gather=gather_kv, + pack_gqa=pack_gqa, + qhead_per_kvhead=qhead_per_kvhead, + nheads_kv=nheads_kv, + is_varlen_q=varlen_q, + disable_bitmask=disable_bitmask, + has_qk=has_qk, + ), + mQ, + mQv, + mK, + mV, + mO, + mLSE, + softmax_scale, + mP, + mRowMax, + **compile_kwargs, + stream=stream, + options="--keep-ptx --keep-cubin --generate-line-info", + ) + dump_kernel_attributes(kernel) + compile_cache[compile_key] = kernel + + # ================================================================ + # ---- Generate variable seqlens for this run ---- + if varlen_q: + torch.manual_seed(seed + 1000) + # When causal without varlen_k, every per-batch seqlen_q must not exceed seqlen_k. + max_seqlen_q = seqlen_k if (is_causal and not varlen_k) else seqlen_q + seqlens_q = torch.randint(1, max_seqlen_q + 1, (batch,), dtype=torch.int32) + cu_seqlens_q = torch.zeros(batch + 1, dtype=torch.int32, device="cuda") + cu_seqlens_q[1:] = seqlens_q.cumsum(0).to(torch.int32).cuda() + total_q = cu_seqlens_q[-1].item() + else: + seqlens_q = torch.full((batch,), seqlen_q, dtype=torch.int32) + total_q = None # unused + + if varlen_k: + torch.manual_seed(seed + 2000) + # Each batch item must have at least topk_length keys so topk gather is valid. + min_seqlen_k = topk_length if gather_kv else 1 + seqlens_k = torch.randint( + min_seqlen_k, seqlen_k + 1, (batch,), dtype=torch.int32 + ) + # When causal, every batch item needs seqlens_k[b] >= seqlens_q[b]. + if is_causal: + seqlens_k = torch.maximum(seqlens_k, seqlens_q) + cu_seqlens_k = torch.zeros(batch + 1, dtype=torch.int32, device="cuda") + cu_seqlens_k[1:] = seqlens_k.cumsum(0).to(torch.int32).cuda() + total_k = cu_seqlens_k[-1].item() + else: + seqlens_k = torch.full((batch,), seqlen_k, dtype=torch.int32) + total_k = None # unused + + torch.manual_seed(seed) # restore main seed before drawing actual tensors + + # ---- Allocate Q / Qv / O / lse ---- + if varlen_q: + Q = torch.randn(total_q, nheads, hdim, dtype=torch.bfloat16, device="cuda") + Qv = torch.randn(total_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda") + O = torch.empty(total_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda") + P = torch.empty( + total_q, nheads, P_k_length, dtype=torch.bfloat16, device="cuda" + ) + lse = torch.empty(total_q, nheads, dtype=torch.float32, device="cuda") + row_max = torch.empty( + total_q_dummy, P_k_length // 128, nheads, dtype=torch.float32, device="cuda" + ) + else: + Q = torch.randn( + batch, seqlen_q, nheads, hdim, dtype=torch.bfloat16, device="cuda" + ) + Qv = torch.randn( + batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda" + ) + O = torch.empty( + batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda" + ) + P = torch.empty( + batch, seqlen_q, nheads, P_k_length, dtype=torch.bfloat16, device="cuda" + ) + lse = torch.empty(batch, seqlen_q, nheads, dtype=torch.float32, device="cuda") + row_max = torch.empty( + batch, + seqlen_q, + P_k_length // 128, + nheads, + dtype=torch.float32, + device="cuda", + ) + + # ---- Allocate K / V ---- + if varlen_k: + K = torch.randn(total_k, nheads_kv, hdim, dtype=torch.bfloat16, device="cuda") + V = torch.randn(total_k, nheads_kv, hdimv, dtype=torch.bfloat16, device="cuda") + else: + K = torch.randn( + batch, seqlen_k, nheads_kv, hdim, dtype=torch.bfloat16, device="cuda" + ) + V = torch.randn( + batch, seqlen_k, nheads_kv, hdimv, dtype=torch.bfloat16, device="cuda" + ) + + # ---- Generate index_topk with per-batch valid ranges when varlen_k ---- + # index_topk shape: (total_q, topk_length) if varlen_q else (batch, seqlen_q, topk_length) + if gather_kv: + topk_parts = [] + for b in range(batch): + sl_q_b = seqlens_q[b].item() + sl_k_b = seqlens_k[b].item() + # Draw topk_length unique indices from [0, sl_k_b) for each query in this batch item. + topk_b = ( + torch.rand(sl_q_b, sl_k_b, device="cuda") + .argsort(dim=-1)[..., :topk_length] + .to(torch.int32) + ) # (sl_q_b, topk_length), all < sl_k_b + topk_parts.append(topk_b) + + if varlen_q: + index_topk = torch.cat(topk_parts, dim=0) # (total_q, topk_length) + else: + index_topk = torch.stack( + topk_parts, dim=0 + ) # (batch, seqlen_q, topk_length) + else: + index_topk = None + + # ---- Reference computation (per-batch loop covers all four varlen combos) ---- + O_ref_list, O_pt_list, lse_ref_list, lse_pt_list = [], [], [], [] + for b in range(batch): + qs = cu_seqlens_q[b].item() if varlen_q else b * seqlen_q + qe = cu_seqlens_q[b + 1].item() if varlen_q else (b + 1) * seqlen_q + ks = cu_seqlens_k[b].item() if varlen_k else b * seqlen_k + ke = cu_seqlens_k[b + 1].item() if varlen_k else (b + 1) * seqlen_k + + Q_b = ( + Q[qs:qe].unsqueeze(0) if varlen_q else Q[b : b + 1] + ) # (1, sl_q, nheads, hdim) + Qv_b = ( + Qv[qs:qe].unsqueeze(0) if varlen_q else Qv[b : b + 1] + ) # (1, sl_q, nheads, hdimv) + K_b = ( + K[ks:ke].unsqueeze(0) if varlen_k else K[b : b + 1] + ) # (1, sl_k, nheads_kv, hdim) + V_b = ( + V[ks:ke].unsqueeze(0) if varlen_k else V[b : b + 1] + ) # (1, sl_k, nheads_kv, hdimv) + if gather_kv: + topk_b = ( + index_topk[qs:qe].unsqueeze(0) if varlen_q else index_topk[b : b + 1] + ) + else: + topk_b = None + + O_b, _, lse_b = attention_ref( + Q_b if has_qk else None, + K_b if has_qk else None, + V_b, + qv=Qv_b, + causal=is_causal, + return_lse=True, + gather_kv_indices=topk_b, + ) + O_pt_b, _, lse_pt_b = attention_ref( + Q_b if has_qk else None, + K_b if has_qk else None, + V_b, + qv=Qv_b, + causal=is_causal, + upcast=False, + reorder_ops=True, + return_lse=True, + gather_kv_indices=topk_b, + ) + O_ref_list.append(O_b.squeeze(0)) + O_pt_list.append(O_pt_b.squeeze(0)) + lse_ref_list.append(lse_b.squeeze(0)) + lse_pt_list.append(lse_pt_b.squeeze(0)) + + cat_dim_o = 0 if (varlen_q) else 0 # always 0: leading token/batch dim + cat_dim_lse = -1 if (varlen_q) else -1 # always last: token dim + + if varlen_q: + O_ref = torch.cat(O_ref_list, dim=0) # (total_q, nheads, hdimv) + O_pt = torch.cat(O_pt_list, dim=0) + lse_ref = torch.cat(lse_ref_list, dim=-1) # (nheads, total_q) + lse_pt = torch.cat(lse_pt_list, dim=-1) + else: + O_ref = torch.stack(O_ref_list, dim=0) # (batch, seqlen_q, nheads, hdimv) + O_pt = torch.stack(O_pt_list, dim=0) + lse_ref = torch.stack(lse_ref_list, dim=0) # (batch, nheads, seqlen_q) + lse_pt = torch.stack(lse_pt_list, dim=0) + + rtol = 2 + atol = 2 * (O_ref + 0.3 - 0.3 - O_ref).abs().max().item() + + # ---- CuTe tensor wrappers ---- + mQ = from_dlpack(Q, assumed_align=16).mark_layout_dynamic(leading_dim=Q.ndim - 1) + mQv = from_dlpack(Qv, assumed_align=16).mark_layout_dynamic(leading_dim=Qv.ndim - 1) + mK = from_dlpack(K, assumed_align=16).mark_layout_dynamic(leading_dim=K.ndim - 1) + mV = from_dlpack(V, assumed_align=16).mark_layout_dynamic(leading_dim=V.ndim - 1) + mO = from_dlpack(O, assumed_align=16).mark_layout_dynamic(leading_dim=O.ndim - 1) + mP = from_dlpack(P, assumed_align=16).mark_layout_dynamic(leading_dim=P.ndim - 1) + mLSE = from_dlpack(lse, assumed_align=4).mark_layout_dynamic( + leading_dim=lse.ndim - 1 + ) + mRowMax = from_dlpack(row_max, assumed_align=4).mark_layout_dynamic( + leading_dim=row_max.ndim - 1 + ) + if index_topk is not None: + mIndexTopk = from_dlpack(index_topk, assumed_align=16).mark_layout_dynamic( + leading_dim=index_topk.ndim - 1 + ) + else: + mIndexTopk = None + + run_kwargs = dict(mIndexTopk=mIndexTopk) + if varlen_q: + run_kwargs["mCuSeqlensQ"] = from_dlpack(cu_seqlens_q, assumed_align=4) + if varlen_k: + run_kwargs["mCuSeqlensK"] = from_dlpack(cu_seqlens_k, assumed_align=4) + + if not has_qk: + mQ = mK = None + + if store_P is False: + mP = mRowMax = None + + # ---- Run kernel ---- + compile_cache[compile_key]( + mQ, + mQv, + mK, + mV, + mO, + mLSE, + softmax_scale, + mP, + mRowMax, + **run_kwargs, + stream=stream, + ) + + O_ref_max = O_ref.abs().max().item() + O_max = O.abs().max().item() + print(f"Pytorch O max = {O_ref_max} and our O max = {O_max}") + print(f"Pytorch max O diff: {(O_pt - O_ref).abs().max().item()}") + print(f"Pytorch mean O diff: {(O_pt - O_ref).abs().mean().item()}") + print(f"Max abs diff O, O_ref: {(O - O_ref).abs().max().item()}") + print(f"Mean abs diff O, O_ref: {(O - O_ref).abs().mean().item()}") + + lse = lse.transpose(-1, -2) + lse_ref_max = lse_ref.abs().max().item() + lse_max = lse.abs().max().item() + print(f"Pytorch LSE max = {lse_ref_max} and our LSE max = {lse_max}") + print(f"Pytorch LSE max diff: {(lse_pt - lse_ref).abs().max().item()}") + print(f"Pytorch LSE mean diff: {(lse_pt - lse_ref).abs().mean().item()}") + print(f"Max abs diff LSE: {(lse - lse_ref).abs().max().item()}") + print(f"Mean abs diff LSE: {(lse - lse_ref).abs().mean().item()}") + + if validate: + assert (O - O_ref).abs().max().item() <= rtol * ( + O_pt - O_ref + ).abs().max().item() + atol + varlen_tag = "" + if varlen_q: + varlen_tag += f", total_q:{total_q}" + if varlen_k: + varlen_tag += f", total_k:{total_k}" + print( + f"batch:{batch:3d}, nheads:{nheads:3d}, seqlen_q:{seqlen_q:5d}, seqlen_k:{seqlen_k:5d}" + f"{varlen_tag}, iter:{iter:2d} PASSED" + ) + else: + print(mO) + print( + f"batch:{batch:3d}, nheads:{nheads:3d}, seqlen_q:{seqlen_q:5d}, seqlen_k:{seqlen_k:5d}" + f", iter:{iter:2d} RUN (NOT TESTING CORRECTNESS)" + ) + + return None + + +def timeit(fn, *args, **kwargs): + # Synchronize before timing + torch.cuda.synchronize() + + # Warmup + for _ in range(10): + fn(*args, **kwargs) + + # Benchmark using PyTorch's Timer + t = benchmark.Timer( + stmt="fn(*args, **kwargs)", globals={"fn": fn, "args": args, "kwargs": kwargs} + ) + + # Time it multiple runs + measurement = t.timeit(20) # 20 repeats + avg_time = measurement.mean # Average time in seconds + + time.sleep(1) + + return avg_time + + +def benchmark_mla_kernel( + batch=1, + seqlen_q=2048, + seqlen_k=2048, + topk_length=2048, + nheads=128, + hdim=64, + hdimv=512, + compile_cache=dict(), + gather_kv=True, + is_causal=False, + disable_bitmask=False, + store_P=False, +): + assert hdim == 64, "hdim must be 64" + assert hdimv == 512, "hdimv must be 512" + + qhead_per_kvhead = nheads + nheads_kv = 1 + pack_gqa = True + softmax_scale = 1.0 / math.sqrt(hdim + hdimv) + seqlen_k_rounded = (seqlen_k + 128 - 1) // 128 * 128 + P_k_length = seqlen_k_rounded if not gather_kv else topk_length + + torch_stream = torch.cuda.current_stream() + stream = cuda.CUstream(torch_stream.cuda_stream) + + compile_key = ( + is_causal, + gather_kv, + topk_length if gather_kv else None, + pack_gqa, + qhead_per_kvhead, + nheads_kv, + disable_bitmask, + ) + if compile_key not in compile_cache: + Q = torch.randn( + batch, seqlen_q, nheads, hdim, dtype=torch.bfloat16, device="cuda" + ) + Qv = torch.randn( + batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda" + ) + K = torch.randn( + batch, seqlen_k, nheads_kv, hdim, dtype=torch.bfloat16, device="cuda" + ) + V = torch.randn( + batch, seqlen_k, nheads_kv, hdimv, dtype=torch.bfloat16, device="cuda" + ) + O = torch.empty( + batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda" + ) + P = torch.empty( + batch, seqlen_q, nheads, P_k_length, dtype=torch.bfloat16, device="cuda" + ) + index_topk = ( + torch.rand(batch, seqlen_q, topk_length, device="cuda") + .argsort(dim=-1) + .to(torch.int32) + ) + + mQ = from_dlpack(Q, assumed_align=16).mark_layout_dynamic( + leading_dim=Q.ndim - 1 + ) + mQv = from_dlpack(Qv, assumed_align=16).mark_layout_dynamic( + leading_dim=Qv.ndim - 1 + ) + mK = from_dlpack(K, assumed_align=16).mark_layout_dynamic( + leading_dim=K.ndim - 1 + ) + mV = from_dlpack(V, assumed_align=16).mark_layout_dynamic( + leading_dim=V.ndim - 1 + ) + mO = from_dlpack(O, assumed_align=16).mark_layout_dynamic( + leading_dim=O.ndim - 1 + ) + mP = from_dlpack(P, assumed_align=16).mark_layout_dynamic( + leading_dim=P.ndim - 1 + ) + if gather_kv: + mIndexTopk = from_dlpack(index_topk, assumed_align=16).mark_layout_dynamic( + leading_dim=index_topk.ndim - 1 + ) + else: + mIndexTopk = None + + mLSE = None + + if store_P is False: + mP = None + + kernel = cute.compile( + FlashAttentionMLAForwardSm100( + is_causal=is_causal, + use_cpasync_load_KV=gather_kv, + topk_length=topk_length if gather_kv else 2048, + is_topk_gather=gather_kv, + pack_gqa=pack_gqa, + qhead_per_kvhead=qhead_per_kvhead, + nheads_kv=nheads_kv, + disable_bitmask=disable_bitmask, + ), + mQ, + mQv, + mK, + mV, + mO, + mLSE, + softmax_scale, + mP=mP, + mIndexTopk=mIndexTopk, + stream=stream, + ) + compile_cache[compile_key] = kernel + + Q = torch.randn(batch, seqlen_q, nheads, hdim, dtype=torch.bfloat16, device="cuda") + Qv = torch.randn( + batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda" + ) + K = torch.randn( + batch, seqlen_k, nheads_kv, hdim, dtype=torch.bfloat16, device="cuda" + ) + V = torch.randn( + batch, seqlen_k, nheads_kv, hdimv, dtype=torch.bfloat16, device="cuda" + ) + O = torch.empty(batch, seqlen_q, nheads, hdimv, dtype=torch.bfloat16, device="cuda") + P = torch.empty( + batch, seqlen_q, nheads, P_k_length, dtype=torch.bfloat16, device="cuda" + ) + + index_topk = ( + torch.rand(batch, seqlen_q, topk_length, device="cuda") + .argsort(dim=-1) + .to(torch.int32) + ) + + mQ = from_dlpack(Q, assumed_align=16).mark_layout_dynamic(leading_dim=Q.ndim - 1) + mQv = from_dlpack(Qv, assumed_align=16).mark_layout_dynamic(leading_dim=Qv.ndim - 1) + mK = from_dlpack(K, assumed_align=16).mark_layout_dynamic(leading_dim=K.ndim - 1) + mV = from_dlpack(V, assumed_align=16).mark_layout_dynamic(leading_dim=V.ndim - 1) + mO = from_dlpack(O, assumed_align=16).mark_layout_dynamic(leading_dim=O.ndim - 1) + mP = from_dlpack(P, assumed_align=16).mark_layout_dynamic(leading_dim=P.ndim - 1) + if gather_kv: + mIndexTopk = from_dlpack(index_topk, assumed_align=16).mark_layout_dynamic( + leading_dim=index_topk.ndim - 1 + ) + else: + mIndexTopk = None + mLSE = None + + if store_P is False: + mP = None + + exec_time_in_s = timeit( + compile_cache[compile_key], + mQ, + mQv, + mK, + mV, + mO, + mLSE, + softmax_scale, + mP=mP, + mIndexTopk=mIndexTopk, + stream=stream, + ) + + seqlen_k_eff = topk_length if gather_kv else seqlen_k + + FLOPs = 2 * batch * nheads * seqlen_q * seqlen_k_eff * (hdim + 2 * hdimv) + if is_causal and not gather_kv: + FLOPs /= 2 + + TFLOPS = FLOPs / exec_time_in_s / 1e12 + + q_bytes = 2 * batch * nheads * seqlen_q * hdim + qv_bytes = 2 * batch * nheads * seqlen_q * hdimv + k_bytes = 2 * batch * nheads_kv * seqlen_k_eff * hdim + v_bytes = 2 * batch * nheads_kv * seqlen_k_eff * hdimv + o_bytes = 2 * batch * nheads * seqlen_q * hdimv + total_bytes = q_bytes + qv_bytes + k_bytes + v_bytes + o_bytes + TBs = total_bytes / exec_time_in_s / 1e12 + + print( + f"batch: {batch}, seqlen_q: {seqlen_q}, seqlen_k: {seqlen_k}, nheads: {nheads}, -> {exec_time_in_s * 1e3:.2f} ms, {TFLOPS:.2f} TFLOPS, {TBs:.2f} TBs" + ) + + +if __name__ == "__main__": + run_test = True + run_benchmark = True + gather_kv = True + is_causal = False + pack_gqa = True + topk_length = 2048 + varlen_q = False + varlen_k = False + disable_bitmask = False + validate = True + has_qk = True + + if run_test: + if not gather_kv: + seqlen_q_test_values = range(1, 4002, 400) + seqlen_k_test_values = range(1, 4002, 400) + else: + seqlen_q_test_values = range(1, 1001, 200) + seqlen_k_test_values = range(topk_length, 9001, 2000) + seqlen_q_test_values = [4096] + seqlen_k_test_values = [4096] + nheads_test_values = [128] + batch_test_values = [1] + test_configs = [ + ( + batch, + nheads, + seqlen_q, + seqlen_k, + ) + for batch in batch_test_values + for nheads in nheads_test_values + for seqlen_q in seqlen_q_test_values + for seqlen_k in seqlen_k_test_values + ] + iters_per_config = 1 + compile_cache = dict() + print("=" * 40) + print("Testing MLA Kernel") + print("=" * 40) + for config in test_configs: + batch, nheads, seqlen_q, seqlen_k = config + # if is_causal and seqlen_k < seqlen_q: + # continue + for iter in range(iters_per_config): + test_mla_kernel( + seqlen_q=seqlen_q, + seqlen_k=seqlen_k, + topk_length=topk_length, + nheads=nheads, + batch=batch, + iter=iter, + compile_cache=compile_cache, + validate=validate, + seed=0, + gather_kv=gather_kv, + pack_gqa=pack_gqa, + is_causal=is_causal, + varlen_q=varlen_q, + varlen_k=varlen_k, + disable_bitmask=disable_bitmask, + has_qk=has_qk, + ) + if run_benchmark: + if gather_kv: + seqlen_q_benchmark_values = [1] + seqlen_k_benchmark_values = [8192 * 2] + nheads_benchmark_values = [128] + batch_benchmark_values = [128] + else: + seqlen_q_benchmark_values = [1] + seqlen_k_benchmark_values = [8192] + nheads_benchmark_values = [128] + batch_benchmark_values = [128] + # seqlen_q_benchmark_values = [4096] + # seqlen_k_benchmark_values = [4096] + # nheads_benchmark_values = [16] + # batch_benchmark_values = [8] + benchmark_configs = [ + ( + batch, + nheads, + seqlen_q, + seqlen_k, + ) + for batch in batch_benchmark_values + for nheads in nheads_benchmark_values + for seqlen_q in seqlen_q_benchmark_values + for seqlen_k in seqlen_k_benchmark_values + ] + compile_cache = dict() + print("=" * 40) + print("Benchmarking MLA Kernel") + print("=" * 40) + for config in benchmark_configs: + batch, nheads, seqlen_q, seqlen_k = config + benchmark_mla_kernel( + batch=batch, + seqlen_q=seqlen_q, + seqlen_k=seqlen_k, + topk_length=topk_length, + nheads=nheads, + gather_kv=gather_kv, + is_causal=is_causal, + disable_bitmask=disable_bitmask, + compile_cache=compile_cache, + ) diff --git a/python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_msa_fp8_parity.py b/test/manual/kernels/attention/test_minimax_msa_fp8_parity.py similarity index 97% rename from python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_msa_fp8_parity.py rename to test/manual/kernels/attention/test_minimax_msa_fp8_parity.py index 5939606636..409cc99d3f 100644 --- a/python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_msa_fp8_parity.py +++ b/test/manual/kernels/attention/test_minimax_msa_fp8_parity.py @@ -10,15 +10,18 @@ but their QK/accumulation orders differ, so MSA-fp8 vs Triton-fp8 tolerances cover two independent P-quantization errors (~1e-1 worst-case elementwise). Requires SM100 + fmha_sm100 (first run JIT-compiles the fp8 variants). -Run: pytest python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_msa_fp8_parity.py -v +Run: pytest test/manual/kernels/attention/test_minimax_msa_fp8_parity.py -v """ import pytest import torch -from sglang.srt.layers.attention.minimax_sparse_ops.decode.topk_sparse import ( +from sglang.kernels.ops.attention.minimax_sparse.decode.topk_sparse import ( flash_decode_with_gqa_share_sparse, ) +from sglang.kernels.ops.attention.minimax_sparse.prefill.topk_sparse import ( + flash_prefill_with_gqa_share_sparse, +) from sglang.srt.layers.attention.minimax_sparse_ops.msa import ( build_msa_decode_cg_plan, msa_available, @@ -26,9 +29,6 @@ from sglang.srt.layers.attention.minimax_sparse_ops.msa import ( msa_sparse_prefill_main, update_msa_decode_cg_meta, ) -from sglang.srt.layers.attention.minimax_sparse_ops.prefill.topk_sparse import ( - flash_prefill_with_gqa_share_sparse, -) DEVICE = "cuda" FP8 = torch.float8_e4m3fn diff --git a/test/manual/kernels/benchmark/attention/bench_gdn_replayssm_fold.py b/test/manual/kernels/benchmark/attention/bench_gdn_replayssm_fold.py index 785d7e92c1..80b416bf49 100644 --- a/test/manual/kernels/benchmark/attention/bench_gdn_replayssm_fold.py +++ b/test/manual/kernels/benchmark/attention/bench_gdn_replayssm_fold.py @@ -13,7 +13,7 @@ import argparse import torch import triton -from sglang.kernels.ops.attention.fla.gdn_replayssm_spec_fold import ( +from sglang.kernels.ops.attention.linear.gdn_replayssm_spec_fold import ( commit_gdn_replayssm_fold_all_layers, ) diff --git a/test/manual/layers/attention/dsa/test_get_k_scale_triton_kernel.py b/test/manual/kernels/kvcache/test_dsa_get_k_scale_triton_kernel.py similarity index 98% rename from test/manual/layers/attention/dsa/test_get_k_scale_triton_kernel.py rename to test/manual/kernels/kvcache/test_dsa_get_k_scale_triton_kernel.py index 78f35b085a..955e71c9bc 100644 --- a/test/manual/layers/attention/dsa/test_get_k_scale_triton_kernel.py +++ b/test/manual/kernels/kvcache/test_dsa_get_k_scale_triton_kernel.py @@ -1,6 +1,6 @@ import torch -from sglang.kernels.ops.attention.dsa.index_buf_accessor import ( +from sglang.kernels.ops.kvcache.dsa_index_k_cache import ( _get_k_and_s_triton_kernel, ) diff --git a/test/manual/layers/attention/dsa/test_index_buf_accessor.py b/test/manual/kernels/kvcache/test_dsa_index_k_cache.py similarity index 99% rename from test/manual/layers/attention/dsa/test_index_buf_accessor.py rename to test/manual/kernels/kvcache/test_dsa_index_k_cache.py index 461cb56bb2..4d195fb621 100644 --- a/test/manual/layers/attention/dsa/test_index_buf_accessor.py +++ b/test/manual/kernels/kvcache/test_dsa_index_k_cache.py @@ -8,7 +8,7 @@ torch gather of the K and S regions of the paged buffer. import pytest import torch -from sglang.kernels.ops.attention.dsa.index_buf_accessor import GetKAndS +from sglang.kernels.ops.kvcache.dsa_index_k_cache import GetKAndS class MockDSATokenToKVPool: diff --git a/test/manual/kernels/attention/check_dsa_quant_k_cache.py b/test/manual/kernels/quantization/check_dsa_quant_k_cache.py similarity index 94% rename from test/manual/kernels/attention/check_dsa_quant_k_cache.py rename to test/manual/kernels/quantization/check_dsa_quant_k_cache.py index 83f4570844..0d859b4164 100644 --- a/test/manual/kernels/attention/check_dsa_quant_k_cache.py +++ b/test/manual/kernels/quantization/check_dsa_quant_k_cache.py @@ -1,12 +1,12 @@ import torch import triton -from sglang.kernels.ops.attention.dsa.quant_k_cache import ( +from sglang.kernels.ops.quantization.dsa_quant_k_cache import ( _quantize_k_cache_fast_wrapped, - _quantize_k_cache_ref, quantize_k_cache, quantize_k_cache_separate, ) +from sglang.test.kernels.dsa_k_cache_reference import _quantize_k_cache_ref if __name__ == "__main__": import dequant_k_cache @@ -49,7 +49,7 @@ if __name__ == "__main__": page_table_1 = torch.arange( num_blocks * block_size, dtype=torch.int32, device="cuda" ) - actual_dequant_paged = dequant_k_cache.dequantize_k_cache_paged( + actual_dequant_paged = dequant_k_cache.dequantize_k_cache_paged_dsa( actual_quant, page_table_1 ).reshape(actual_actual_dequant.shape) print(f"{torch.mean(actual_actual_dequant - actual_dequant_paged)=}") @@ -127,7 +127,9 @@ if __name__ == "__main__": ) def run_ans(): - return dequant_k_cache.dequantize_k_cache_paged(actual_quant, page_table_1) + return dequant_k_cache.dequantize_k_cache_paged_dsa( + actual_quant, page_table_1 + ) ans_time: float = triton.testing.do_bench(run_ans, warmup=10, rep=20) / 1000 # type: ignore print(f"seq_kv: {num_blocks * block_size}, time: {ans_time * 1e6: 4.0f} us") diff --git a/test/manual/layers/attention/dsa/test_act_quant_triton.py b/test/manual/kernels/quantization/test_act_quant_triton.py similarity index 97% rename from test/manual/layers/attention/dsa/test_act_quant_triton.py rename to test/manual/kernels/quantization/test_act_quant_triton.py index e9a4d4f622..f1a1bede17 100644 --- a/test/manual/layers/attention/dsa/test_act_quant_triton.py +++ b/test/manual/kernels/quantization/test_act_quant_triton.py @@ -9,8 +9,10 @@ from typing import Tuple import pytest import torch -from sglang.kernels.ops.attention.dsa.tilelang_kernel import act_quant -from sglang.kernels.ops.attention.dsa.triton_kernel import act_quant as act_quant_triton +from sglang.kernels.ops.quantization.act_quant_tilelang import act_quant +from sglang.kernels.ops.quantization.act_quant_triton import ( + act_quant as act_quant_triton, +) def benchmark_kernel( diff --git a/test/manual/test_kda_target_verify.py b/test/manual/test_kda_target_verify.py index 03389e42f6..a578f9637d 100644 --- a/test/manual/test_kda_target_verify.py +++ b/test/manual/test_kda_target_verify.py @@ -2,7 +2,7 @@ import torch def test_kda_target_verify_equivalence(): - from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( + from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, ) @@ -118,7 +118,7 @@ def test_kda_target_verify_equivalence(): def test_kda_target_verify_bf16(): - from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( + from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, ) diff --git a/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py b/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py index 9715559a11..dd2a137627 100644 --- a/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py +++ b/test/registered/amd/test_deepseek_v4_pro_fp4_dspark.py @@ -17,7 +17,7 @@ import requests import torch from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import MAX_FUSED_ROWS -from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime +from sglang.kernels.ops.kvcache import dsv4_unified_kv_store from sglang.kernels.ops.speculative.dspark import dspark_verify_window from sglang.srt.layers.attention.deepseek_v4_backend_hip_radix import ( DeepseekV4HipRadixBackend, @@ -163,11 +163,15 @@ class TestDSparkUnifiedKVKernelsAMD(CustomTestCase): keep = loc >= 0 expected[loc[keep].long()] = kv[keep] - runtime.scatter_bf16_into_unified(kv=kv, loc=loc, unified_kv=unified) + dsv4_unified_kv_store.scatter_bf16_into_unified( + kv=kv, loc=loc, unified_kv=unified + ) self.assertTrue(torch.equal(unified, expected)) with self.assertRaises(AssertionError): - runtime.scatter_bf16_into_unified(kv=kv, loc=loc, unified_kv=unified.t()) + dsv4_unified_kv_store.scatter_bf16_into_unified( + kv=kv, loc=loc, unified_kv=unified.t() + ) class TestDeepseekV4DSparkUnifiedKVGSM8K(CustomTestCase): diff --git a/test/registered/amd/test_dsv4_hip_bcg_metadata.py b/test/registered/amd/test_dsv4_hip_bcg_metadata.py index fddd29a5d1..81d2349634 100644 --- a/test/registered/amd/test_dsv4_hip_bcg_metadata.py +++ b/test/registered/amd/test_dsv4_hip_bcg_metadata.py @@ -172,8 +172,7 @@ class TestDSV4HipBreakableCudaGraphMetadata(unittest.TestCase): with ( mock.patch( - "sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate." - "is_unified_kv_triton", + "sglang.srt.layers.attention.dsv4.unified_kv_gate.is_unified_kv_triton", return_value=True, ), mock.patch( @@ -214,7 +213,7 @@ class TestDSV4HipBreakableCudaGraphMetadata(unittest.TestCase): with ( mock.patch( - "sglang.kernels.ops.attention.dsv4_attn_metadata_kernels." + "sglang.kernels.ops.attention.dsv4.attn_metadata_kernels." "ExpandPrefillCausally.execute", return_value=SimpleNamespace( seq_lens_casual=core.seq_lens_casual, @@ -279,7 +278,7 @@ class TestDSV4HipBreakableCudaGraphMetadata(unittest.TestCase): with ( mock.patch( - "sglang.kernels.ops.attention.dsv4_attn_metadata_kernels." + "sglang.kernels.ops.attention.dsv4.attn_metadata_kernels." "ExpandPrefillCausally.execute", return_value=SimpleNamespace( seq_lens_casual=torch.ones(4, dtype=torch.int32), @@ -346,6 +345,7 @@ class TestDSV4HipBreakableCudaGraphMetadata(unittest.TestCase): def test_cp_unified_uses_inert_query_padding_and_logical_ring_rows(self): from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime + from sglang.kernels.ops.kvcache import dsv4_unified_kv_store backend = object.__new__(DeepseekV4HipRadixBackend) backend.token_to_kv_pool = SimpleNamespace( @@ -394,7 +394,7 @@ class TestDSV4HipBreakableCudaGraphMetadata(unittest.TestCase): ), ) as build_indices, mock.patch.object(runtime, "prefill", return_value=output), - mock.patch.object(runtime, "store_swa_into_unified") as store, + mock.patch.object(dsv4_unified_kv_store, "store_swa_into_unified") as store, ): result = backend._forward_unified_kv( q=torch.zeros(2, 1, 4), diff --git a/test/registered/attention/test_create_kvindices.py b/test/registered/attention/test_create_kvindices.py index 5124229d8f..85adb35a06 100644 --- a/test/registered/attention/test_create_kvindices.py +++ b/test/registered/attention/test_create_kvindices.py @@ -3,7 +3,7 @@ import unittest import numpy as np import torch -from sglang.kernels.ops.attention.utils import create_flashinfer_kv_indices_triton +from sglang.kernels.ops.attention.kv_indices import create_flashinfer_kv_indices_triton from sglang.srt.utils import get_device from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/registered/attention/test_fused_fp8_kv_write.py b/test/registered/attention/test_fused_fp8_kv_write.py index b8fca329f8..7675b03a5e 100644 --- a/test/registered/attention/test_fused_fp8_kv_write.py +++ b/test/registered/attention/test_fused_fp8_kv_write.py @@ -31,9 +31,7 @@ def _fp32_divide_write(k, v, k_cache, v_cache, loc, k_scale, v_scale, fp8_dtype) @unittest.skipUnless(_HAS_CUDA, "Triton kernels require a GPU") class TestFusedFp8KvWrite(unittest.TestCase): def _run(self, num_tokens, num_heads, head_dim, total_slots=None, seed=0xC0FFEE): - from sglang.kernels.ops.attention.utils import ( - launch_reshape_and_cache_flash, - ) + from sglang.kernels.ops.kvcache.cache_ops import launch_reshape_and_cache_flash from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype torch.manual_seed(seed) diff --git a/test/registered/attention/test_gdn_noncontiguous_stride.py b/test/registered/attention/test_gdn_noncontiguous_stride.py index 65b801d549..3c231e5185 100644 --- a/test/registered/attention/test_gdn_noncontiguous_stride.py +++ b/test/registered/attention/test_gdn_noncontiguous_stride.py @@ -8,8 +8,8 @@ import unittest import torch -from sglang.kernels.ops.attention.fla.fused_gdn_gating import fused_gdn_gating -from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( +from sglang.kernels.ops.attention.linear.fused_gdn_gating import fused_gdn_gating +from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, ) from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci diff --git a/test/registered/attention/test_gdn_prefill_cutedsl.py b/test/registered/attention/test_gdn_prefill_cutedsl.py index f9eed7a211..e6abdab4d8 100644 --- a/test/registered/attention/test_gdn_prefill_cutedsl.py +++ b/test/registered/attention/test_gdn_prefill_cutedsl.py @@ -17,7 +17,7 @@ from sglang.test.ci.ci_register import register_cuda_ci # suite is the right slot (matches existing jit_kernel test_*.py pattern). register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b200") -if not (torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 10): +if not (torch.cuda.is_available() and torch.cuda.get_device_capability()[0] == 10): pytest.skip( "GDN CuteDSL prefill requires CUDA SM10x (Blackwell).", allow_module_level=True, diff --git a/test/registered/attention/test_kda_kernels.py b/test/registered/attention/test_kda_kernels.py index e685e3990d..a62f72f8b0 100644 --- a/test/registered/attention/test_kda_kernels.py +++ b/test/registered/attention/test_kda_kernels.py @@ -8,15 +8,15 @@ from sglang.kernels.ops.attention.fla.cumsum import chunk_local_cumsum from sglang.kernels.ops.attention.fla.fused_recurrent import ( fused_recurrent_kda_packed_decode, ) -from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( - fused_sigmoid_gating_delta_rule_update, -) from sglang.kernels.ops.attention.fla.index import prepare_chunk_indices from sglang.kernels.ops.attention.fla.kda import ( chunk_kda, fused_recurrent_kda, kda_gate_chunk_cumsum, ) +from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( + fused_sigmoid_gating_delta_rule_update, +) from sglang.srt.utils.common import get_device from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/registered/attention/test_kda_prefill_cutedsl.py b/test/registered/attention/test_kda_prefill_cutedsl.py index 5ca95629b9..b12c964f31 100644 --- a/test/registered/attention/test_kda_prefill_cutedsl.py +++ b/test/registered/attention/test_kda_prefill_cutedsl.py @@ -19,7 +19,7 @@ from sglang.test.ci.ci_register import register_cuda_ci # same slot as the GDN prefill test. register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b200") -if not (torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 10): +if not (torch.cuda.is_available() and torch.cuda.get_device_capability()[0] == 10): pytest.skip( "KDA CuteDSL prefill requires CUDA SM10x (Blackwell).", allow_module_level=True, diff --git a/test/registered/attention/test_trtllm_mha_graph_metadata.py b/test/registered/attention/test_trtllm_mha_graph_metadata.py index 579b999166..d562873111 100644 --- a/test/registered/attention/test_trtllm_mha_graph_metadata.py +++ b/test/registered/attention/test_trtllm_mha_graph_metadata.py @@ -16,7 +16,7 @@ from torch.utils import _pytree as pytree from torch.utils._python_dispatch import TorchDispatchMode import sglang.srt.layers.attention.trtllm_mha_backend as trtllm_mha_backend -from sglang.kernels.ops.kvcache.trtllm_mha_graph_metadata import ( +from sglang.kernels.ops.attention.trtllm_mha_graph_metadata import ( Q_MODE_CUMSUM, Q_MODE_NONE, Q_MODE_STRIDED, diff --git a/test/registered/attention/test_trtllm_mha_page_table.py b/test/registered/attention/test_trtllm_mha_page_table.py index c70e852229..fc65ba34ee 100644 --- a/test/registered/attention/test_trtllm_mha_page_table.py +++ b/test/registered/attention/test_trtllm_mha_page_table.py @@ -18,7 +18,7 @@ from typing import Optional import torch -from sglang.kernels.ops.kvcache.trtllm_mha_page_table import ( +from sglang.kernels.ops.attention.trtllm_mha_page_table import ( build_trtllm_mha_page_table, ) from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci diff --git a/test/registered/attention/unittests/dsa/README.md b/test/registered/attention/unittests/dsa/README.md index d629a5578f..9cbc560e63 100644 --- a/test/registered/attention/unittests/dsa/README.md +++ b/test/registered/attention/unittests/dsa/README.md @@ -60,7 +60,7 @@ hardware/SDK. The variant tests live in `test_dsa.py` as - **Page size other than 1 (HIP legacy) or 64 (CUDA)** — the DSA indexer hard-asserts the page size: HIP legacy at `dsa/dsa_indexer.py:547-548, 724-725` (`assert page_size == 1`); CUDA at `dsa/dsa_indexer.py:550, 727, - 946, 1095` and `dsa/index_buf_accessor.py:436` (`assert page_size == 64`). + 946, 1095` and `kvcache/dsa_index_k_cache.py:436` (`assert page_size == 64`). The `dsa/transform_index.py:53, 79, 100, 121` helpers also assert `page_size == 1`. - **`Unsupported {forward_batch.forward_mode=}`** — `forward_extend` diff --git a/test/registered/attention/unittests/dsa/test_dsa.py b/test/registered/attention/unittests/dsa/test_dsa.py index 2d88f2d388..6c58176b42 100644 --- a/test/registered/attention/unittests/dsa/test_dsa.py +++ b/test/registered/attention/unittests/dsa/test_dsa.py @@ -257,7 +257,7 @@ class TestDSAAttentionBackendCorrectness(CustomTestCase): # FP8 + `flashmla_sparse` prefill + EXTEND + non-empty prefix is the # only combo that hits `TopkTransformMethod.RAGGED` # (`get_topk_transform_method`), which exercises - # `dequantize_k_cache_paged` and the `topk_indices_offset` shift — + # `dequantize_k_cache_paged_dsa` and the `topk_indices_offset` shift — # paths that the BF16 default suite never reaches. FP8_PREFILL_RAGGED_CASE = DSAAttentionCase( name="dsa_sparse_fp8_prefill_ragged_topk", diff --git a/test/registered/attention/unittests/dsv4/test_dsv41_fused_compress.py b/test/registered/attention/unittests/dsv4/test_dsv41_fused_compress.py index a930ed6729..3114c463f4 100644 --- a/test/registered/attention/unittests/dsv4/test_dsv41_fused_compress.py +++ b/test/registered/attention/unittests/dsv4/test_dsv41_fused_compress.py @@ -190,7 +190,7 @@ def _reference(t, ratio: int): from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( index_k_rope_pack, ) - from sglang.kernels.ops.attention.dsv4.torch_quant import fake_quant_compressed_kv + from sglang.kernels.ops.quantization.fp4_fake_quant import fake_quant_compressed_kv from sglang.srt.layers.attention.dsv4.dsv41_sparse import rope_tail if ratio == 1: @@ -254,7 +254,7 @@ def _reference(t, ratio: int): ) index_cache = tuple(torch.zeros_like(buf) for buf in t.index_cache) if t.index_k_split: - from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( + from sglang.kernels.ops.kvcache.dsv4_fp4_index_k_cache_hip import ( store_fp4_index_k_cache_split, ) from sglang.srt.layers.attention.dsv4.dsv41_sparse import _rope_fq4 diff --git a/test/registered/attention/unittests/dsv4/test_hip_flash_mla_aiter_sparse.py b/test/registered/attention/unittests/dsv4/test_hip_flash_mla_aiter_sparse.py index c1cd9d2858..87af7ce085 100644 --- a/test/registered/attention/unittests/dsv4/test_hip_flash_mla_aiter_sparse.py +++ b/test/registered/attention/unittests/dsv4/test_hip_flash_mla_aiter_sparse.py @@ -382,8 +382,8 @@ def _model_inverse_rope(x, freqs_real, positions): """The model's standalone inverse RoPE of the attention output: fused_rope_inplace(..., inverse=True) with the batched flat kernel (set_batched_rope(True)).""" - from sglang.kernels.ops.attention.deepseek_v4_rope import set_batched_rope - from sglang.kernels.ops.attention.dsv4.elementwise import fused_rope_inplace + from sglang.kernels.ops.attention.dsv4.norm_rope import fused_rope_inplace + from sglang.kernels.ops.attention.dsv4.rope import set_batched_rope set_batched_rope(True) freqs_cis = torch.view_as_complex(freqs_real.view(freqs_real.shape[0], -1, 2)) diff --git a/test/registered/attention/unittests/gdn/test_gdn_cutedsl_ring_verify.py b/test/registered/attention/unittests/gdn/test_gdn_cutedsl_ring_verify.py index d0039b3166..0d27a23f2d 100644 --- a/test/registered/attention/unittests/gdn/test_gdn_cutedsl_ring_verify.py +++ b/test/registered/attention/unittests/gdn/test_gdn_cutedsl_ring_verify.py @@ -21,8 +21,8 @@ from sglang.kernels.ops.attention.cutedsl_gdn_mtp_ring import ( _select_wide_vec_phase_b_unroll, gated_delta_rule_mtp, ) -from sglang.kernels.ops.attention.fla.fused_gdn_gating import fused_gdn_gating -from sglang.kernels.ops.attention.fla.gdn_replayssm_spec_fold import ( +from sglang.kernels.ops.attention.linear.fused_gdn_gating import fused_gdn_gating +from sglang.kernels.ops.attention.linear.gdn_replayssm_spec_fold import ( commit_gdn_replayssm_fold_all_layers, ) from sglang.test.ci.ci_register import register_cuda_ci diff --git a/test/registered/attention/unittests/gdn/test_gdn_replayssm_spec_fold.py b/test/registered/attention/unittests/gdn/test_gdn_replayssm_spec_fold.py index f5e531e228..5421a66bfd 100644 --- a/test/registered/attention/unittests/gdn/test_gdn_replayssm_spec_fold.py +++ b/test/registered/attention/unittests/gdn/test_gdn_replayssm_spec_fold.py @@ -11,10 +11,10 @@ import unittest import torch -from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( +from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, ) -from sglang.kernels.ops.attention.fla.gdn_replayssm_spec_fold import ( +from sglang.kernels.ops.attention.linear.gdn_replayssm_spec_fold import ( commit_gdn_replayssm_fold_all_layers, ) from sglang.test.ci.ci_register import register_cuda_ci diff --git a/test/registered/attention/unittests/gdn/test_linear_replayssm_decode.py b/test/registered/attention/unittests/gdn/test_linear_replayssm_decode.py index 0c3ef19e73..a5fd8846da 100644 --- a/test/registered/attention/unittests/gdn/test_linear_replayssm_decode.py +++ b/test/registered/attention/unittests/gdn/test_linear_replayssm_decode.py @@ -1,6 +1,6 @@ """Correctness test for the buffered output-only linear-attention decode kernel (ReplaySSM Part A), implemented in -``python/sglang/srt/layers/attention/fla/fused_recurrent_linear_replayssm.py``. +``python/sglang/kernels/ops/attention/linear/fused_recurrent_linear_replayssm.py``. One kernel, two gate granularities (selected by ``is_kda``): * GDN (``is_kda=False``): per-head SCALAR gate. Ground truth = @@ -98,7 +98,7 @@ class TestLinearReplaySSMDecode(CustomTestCase): fused_recurrent_gated_delta_rule_packed_decode, fused_recurrent_kda_packed_decode, ) - from sglang.kernels.ops.attention.fla.fused_recurrent_linear_replayssm import ( + from sglang.kernels.ops.attention.linear.fused_recurrent_linear_replayssm import ( fused_recurrent_linear_replayssm_decode, ) diff --git a/test/registered/e2e/dsv4/test_dsv4_unified_fp8_compress_store.py b/test/registered/e2e/dsv4/test_dsv4_unified_fp8_compress_store.py index 51c38390ce..ecb4ee45ee 100644 --- a/test/registered/e2e/dsv4/test_dsv4_unified_fp8_compress_store.py +++ b/test/registered/e2e/dsv4/test_dsv4_unified_fp8_compress_store.py @@ -16,12 +16,12 @@ import unittest import torch -from sglang.kernels.ops.attention.deepseek_v4_rope import precompute_freqs_cis from sglang.kernels.ops.attention.dsv4 import ( CompressorDecodePlan, CompressorPrefillPlan, compress_norm_rope_store, ) +from sglang.kernels.ops.attention.dsv4.rope import precompute_freqs_cis from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( DSV4_FP8_NOPE_ROW_BYTES, DSV4_FP8_QUANT_TILE, diff --git a/test/registered/e2e/dsv4/test_dsv4_unified_fp8_scatter.py b/test/registered/e2e/dsv4/test_dsv4_unified_fp8_scatter.py index 82255abe23..fbf2b6ab00 100644 --- a/test/registered/e2e/dsv4/test_dsv4_unified_fp8_scatter.py +++ b/test/registered/e2e/dsv4/test_dsv4_unified_fp8_scatter.py @@ -11,7 +11,7 @@ import unittest import torch -from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import runtime +from sglang.kernels.ops.kvcache import dsv4_unified_kv_store from sglang.srt.mem_cache.deepseek_v4_memory_pool import DSV4_FP8_NOPE_ROW_BYTES from sglang.test.ci.ci_register import register_amd_ci from sglang.test.test_utils import CustomTestCase @@ -73,7 +73,7 @@ def _bf16_rope(n_rows): def _store(kv, pool, state_slot, positions, final_pos=None, **kw): - runtime.store_swa_into_unified( + dsv4_unified_kv_store.store_swa_into_unified( kv=kv, state_slot=state_slot, positions=positions, @@ -258,7 +258,7 @@ class TestUnifiedFp8SwaScatter(CustomTestCase): expected = pool.clone() keep = loc >= 0 expected[loc[keep].long()] = kv[keep] - runtime.scatter_bf16_into_unified(kv=kv, loc=loc, unified_kv=pool) + dsv4_unified_kv_store.scatter_bf16_into_unified(kv=kv, loc=loc, unified_kv=pool) self.assertTrue(torch.equal(pool, expected)) def test_empty_batch_is_a_noop(self): diff --git a/test/registered/expert_pack/test_expert_pack_mxfp4.py b/test/registered/expert_pack/test_expert_pack_mxfp4.py index ea6311ee48..ac79828f90 100644 --- a/test/registered/expert_pack/test_expert_pack_mxfp4.py +++ b/test/registered/expert_pack/test_expert_pack_mxfp4.py @@ -5,7 +5,6 @@ import unittest import torch from sglang.kernels.ops.moe.expert_pack_mxfp4 import ( - mxfp4_marlin_repack, mxfp4_matvec, mxfp4_matvec_dual, ) @@ -32,7 +31,6 @@ _FP4_VALUES = ( -4.0, -6.0, ) -_PACK_INDEX = (0, 2, 4, 6, 1, 3, 5, 7) def _matvec_reference( @@ -95,96 +93,6 @@ def _fill_mxfp4_cache(slots, role_bytes, input_size, device): return cache_cpu.to(device=device) -def _load_raw_word(raw_cpu, slot, role_offset, row, blocks_per_row, packed_word): - block = packed_word // 4 - word_in_block = packed_word & 3 - offset = ( - slot * raw_cpu.stride(0) - + role_offset - + row * blocks_per_row * 17 - + block * 17 - + 1 - + word_in_block * 4 - ) - word = 0 - for byte in range(4): - word |= int(raw_cpu.view(-1)[offset + byte]) << (8 * byte) - return word - - -def _marlin_nibble(word, value_index): - return (word >> ((value_index & 7) * 4)) & 0xF - - -def _marlin_scale_perm(index): - local_perm = (0, 2, 1, 3) - interleaved = (index // 4) * 4 + local_perm[index & 3] - return ((interleaved & 7) * 8) + (interleaved >> 3) - - -def _repack_reference( - raw_cpu, source_slot, role_bytes, input_size, output_size, gate_up -): - blocks = input_size // 32 - total_words = (input_size // 16) * output_size * 2 - output = torch.empty(total_words, dtype=torch.int32) - tile_span = (output_size // 64) * 128 - rows_per_role = output_size // 2 if gate_up else output_size - for index in range(total_words): - tile_k, tile_rem = divmod(index, tile_span) - tile_n, local = divmod(tile_rem, 128) - warp, thread = local & 3, local >> 2 - cur_n = warp * 16 + thread // 4 - tc_row = (thread & 3) * 2 - values = [] - for high in (False, True): - source_row = tile_n * 64 + cur_n + (8 if high else 0) - role = ( - 1 if gate_up and source_row >= rows_per_role else (0 if gate_up else 2) - ) - row = source_row % rows_per_role if gate_up else source_row - for offset in (0, 1, 8, 9): - value_index = tc_row + offset - word = _load_raw_word( - raw_cpu, - source_slot, - role * role_bytes, - row, - blocks, - tile_k * 2 + value_index // 8, - ) - values.append(_marlin_nibble(word, value_index)) - packed = 0 - for output_index, value_index in enumerate(_PACK_INDEX): - packed |= values[value_index] << (output_index * 4) - # The CUDA kernel stores the packed uint32 bit pattern in int32. - output[index] = packed if packed < (1 << 31) else packed - (1 << 32) - return output - - -def _scale_reference( - raw_cpu, source_slot, role_bytes, input_size, output_size, gate_up -): - blocks = input_size // 32 - output = torch.empty(blocks * output_size, dtype=torch.uint8) - rows_per_role = output_size // 2 if gate_up else output_size - for index in range(output.numel()): - group, column = divmod(index, output_size) - source_column = (column // 64) * 64 + _marlin_scale_perm(column & 63) - role = ( - 1 if gate_up and source_column >= rows_per_role else (0 if gate_up else 2) - ) - row = source_column % rows_per_role if gate_up else source_column - offset = ( - source_slot * raw_cpu.stride(0) - + role * role_bytes - + row * blocks * 17 - + group * 17 - ) - output[index] = raw_cpu.view(-1)[offset] - return output - - @unittest.skipUnless(torch.cuda.is_available(), "MXFP4 kernel tests require CUDA") class TestExpertPackMxfp4(unittest.TestCase): def test_matvec_fp16_and_bf16(self): @@ -264,81 +172,6 @@ class TestExpertPackMxfp4(unittest.TestCase): torch.testing.assert_close(output_a, expected_a) torch.testing.assert_close(output_b, expected_b) - def test_marlin_repack_weights_and_scales(self): - hidden_size, intermediate_size = 64, 32 - w13_n, w2_n = 2 * intermediate_size, hidden_size - role_bytes = intermediate_size * (hidden_size // 32) * 17 - raw_cpu = torch.zeros((4, 3 * role_bytes), dtype=torch.uint8) - for slot in range(4): - for role, rows, size in ( - (0, intermediate_size, hidden_size), - (1, intermediate_size, hidden_size), - (2, hidden_size, intermediate_size), - ): - groups = size // 32 - for row in range(rows): - for group in range(groups): - offset = ( - slot * raw_cpu.stride(0) - + role * role_bytes - + row * groups * 17 - + group * 17 - ) - raw_cpu.view(-1)[offset] = ( - 100 + slot * 7 + role * 11 + row + group - ) % 256 - for byte in range(16): - raw_cpu.view(-1)[offset + byte + 1] = ( - (byte + row + role) % 16 - ) | (((15 - byte + slot + group) % 16) << 4) - raw = raw_cpu.cuda() - source_slots = torch.tensor([1, 0], dtype=torch.int32, device="cuda") - target_slots = torch.tensor([2, 3], dtype=torch.int32, device="cuda") - w13_words = (hidden_size // 16) * w13_n * 2 - w2_words = (intermediate_size // 16) * w2_n * 2 - w13_scales = (hidden_size // 32) * w13_n - w2_scales = (intermediate_size // 32) * w2_n - w13 = torch.full((4, w13_words), -1, dtype=torch.int32, device="cuda") - w2 = torch.full((4, w2_words), -1, dtype=torch.int32, device="cuda") - w13_scale = torch.full((4, w13_scales), 255, dtype=torch.uint8, device="cuda") - w2_scale = torch.full((4, w2_scales), 255, dtype=torch.uint8, device="cuda") - mxfp4_marlin_repack( - raw, - source_slots, - target_slots, - role_bytes=role_bytes, - hidden_size=hidden_size, - intermediate_size=intermediate_size, - w13=w13, - w2=w2, - w13_scale=w13_scale, - w2_scale=w2_scale, - ) - torch.cuda.synchronize() - for source_slot, target_slot in zip( - source_slots.cpu().tolist(), target_slots.cpu().tolist() - ): - expected_w13 = _repack_reference( - raw_cpu, source_slot, role_bytes, hidden_size, w13_n, True - ) - expected_w2 = _repack_reference( - raw_cpu, source_slot, role_bytes, intermediate_size, w2_n, False - ) - expected_w13_scale = _scale_reference( - raw_cpu, source_slot, role_bytes, hidden_size, w13_n, True - ) - expected_w2_scale = _scale_reference( - raw_cpu, source_slot, role_bytes, intermediate_size, w2_n, False - ) - torch.testing.assert_close(w13[target_slot].cpu(), expected_w13) - torch.testing.assert_close(w2[target_slot].cpu(), expected_w2) - torch.testing.assert_close(w13_scale[target_slot].cpu(), expected_w13_scale) - torch.testing.assert_close(w2_scale[target_slot].cpu(), expected_w2_scale) - self.assertTrue(torch.all(w13[0] == -1)) - self.assertTrue(torch.all(w2[0] == -1)) - self.assertTrue(torch.all(w13_scale[0] == 255)) - self.assertTrue(torch.all(w2_scale[0] == 255)) - def test_invalid_dimensions_are_rejected(self): input_tensor = torch.zeros((1, 31), dtype=torch.float16, device="cuda") cache = torch.zeros((1, 17), dtype=torch.uint8, device="cuda") diff --git a/test/registered/kernels/benchmark/elementwise/bench_bias_gelu.py b/test/registered/kernels/benchmark/activation/bench_bias_gelu.py similarity index 93% rename from test/registered/kernels/benchmark/elementwise/bench_bias_gelu.py rename to test/registered/kernels/benchmark/activation/bench_bias_gelu.py index 5c622b11d6..c210003f8d 100644 --- a/test/registered/kernels/benchmark/elementwise/bench_bias_gelu.py +++ b/test/registered/kernels/benchmark/activation/bench_bias_gelu.py @@ -2,7 +2,7 @@ import torch import torch.nn.functional as F from sglang.kernels.jit.benchmark import marker -from sglang.kernels.ops.elementwise.bias_gelu import bias_gelu_tanh +from sglang.kernels.ops.activation.bias_gelu import bias_gelu_tanh from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci( diff --git a/test/registered/kernels/benchmark/attention/bench_fp4_index_logits.py b/test/registered/kernels/benchmark/attention/bench_fp4_index_logits.py index a461457bff..ab777679af 100644 --- a/test/registered/kernels/benchmark/attention/bench_fp4_index_logits.py +++ b/test/registered/kernels/benchmark/attention/bench_fp4_index_logits.py @@ -4,12 +4,12 @@ from sglang.kernels.jit.benchmark import marker from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( fp4_index_logits_decode, fp4_index_logits_paged, - store_fp4_index_k_cache, ) from sglang.kernels.ops.attention.dsv4.topk import ( plan_topk_v2, topk_transform_paged_v2, ) +from sglang.kernels.ops.kvcache.dsv4_fp4_index_k_cache import store_fp4_index_k_cache from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci( diff --git a/test/registered/kernels/benchmark/attention/bench_online_c128_mtp.py b/test/registered/kernels/benchmark/attention/bench_online_c128_mtp.py index 80376b8dd5..6e4f09d022 100644 --- a/test/registered/kernels/benchmark/attention/bench_online_c128_mtp.py +++ b/test/registered/kernels/benchmark/attention/bench_online_c128_mtp.py @@ -16,7 +16,7 @@ from sglang.kernels.jit.benchmark.utils import ( run_benchmark_no_cudagraph, ) from sglang.kernels.ops.attention.dsv4.online_c128_mtp import ( - _jit_online_c128_mtp_module, + online_c128_mtp_write_prefix_states, ) from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci @@ -118,8 +118,8 @@ def make_case(batch_size: int, num_verify_tokens: int) -> BenchmarkCase: ) -def call_write_prefix(module, case: BenchmarkCase) -> None: - module.write_prefix_states( +def call_write_prefix(case: BenchmarkCase) -> None: + online_c128_mtp_write_prefix_states( case.kv_score_input, case.seq_lens, case.req_pool_indices, @@ -129,6 +129,7 @@ def call_write_prefix(module, case: BenchmarkCase) -> None: case.layer_bs, case.num_verify_tokens, case.state_slot_stride, + head_dim=HEAD_DIM, ) @@ -149,10 +150,7 @@ def benchmark( batch_size: int, num_verify_tokens: int, launch_mode: str ) -> tuple[float, float, float]: case = make_case(batch_size, num_verify_tokens) - module = _jit_online_c128_mtp_module( - HEAD_DIM, case.seq_lens.dtype, case.req_pool_indices.dtype, case.state.dtype - ) - fn = lambda: call_write_prefix(module, case) + fn = lambda: call_write_prefix(case) if launch_mode == "cuda_graph": return run_benchmark(fn) diff --git a/test/registered/kernels/benchmark/communication/bench_custom_all_reduce.py b/test/registered/kernels/benchmark/communication/bench_custom_all_reduce.py index 0bdd388f25..66ff7f9b14 100644 --- a/test/registered/kernels/benchmark/communication/bench_custom_all_reduce.py +++ b/test/registered/kernels/benchmark/communication/bench_custom_all_reduce.py @@ -13,7 +13,7 @@ import sglang.srt.distributed.parallel_state as ps from sglang.kernels.jit.benchmark import marker from sglang.kernels.jit.benchmark.utils import get_benchmark_range, multigpu_bench_main from sglang.kernels.jit.utils import cache_once, is_arch_support_pdl -from sglang.kernels.ops.communication.mp import register_comm_cleanup +from sglang.srt.distributed.comm_cleanup import register_comm_cleanup from sglang.srt.runtime_context import get_parallel from sglang.test.ci.ci_register import register_cuda_ci diff --git a/test/registered/kernels/benchmark/communication/bench_tp_qknorm.py b/test/registered/kernels/benchmark/communication/bench_tp_qknorm.py index 9ce8728718..45177086dd 100644 --- a/test/registered/kernels/benchmark/communication/bench_tp_qknorm.py +++ b/test/registered/kernels/benchmark/communication/bench_tp_qknorm.py @@ -32,7 +32,7 @@ from sglang.kernels.ops.communication.all_reduce import ( get_fused_parallel_qknorm_max_occupancy, get_fused_parallel_qknorm_module, ) -from sglang.kernels.ops.communication.mp import register_comm_cleanup +from sglang.srt.distributed.comm_cleanup import register_comm_cleanup from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( CustomAllReduceV2, ) diff --git a/test/registered/kernels/benchmark/speculative/bench_ngram_compute_decode.py b/test/registered/kernels/benchmark/embeddings/bench_ngram_compute_decode.py similarity index 98% rename from test/registered/kernels/benchmark/speculative/bench_ngram_compute_decode.py rename to test/registered/kernels/benchmark/embeddings/bench_ngram_compute_decode.py index 7f60bb4c73..2e10356fe9 100644 --- a/test/registered/kernels/benchmark/speculative/bench_ngram_compute_decode.py +++ b/test/registered/kernels/benchmark/embeddings/bench_ngram_compute_decode.py @@ -7,7 +7,7 @@ from sglang.kernels.jit.benchmark.utils import ( get_benchmark_range, run_benchmark_no_cudagraph, ) -from sglang.kernels.ops.speculative.ngram_embedding import ( +from sglang.kernels.ops.embeddings.ngram_embedding import ( compute_n_gram_ids, compute_n_gram_ids_decode, ) diff --git a/test/registered/kernels/benchmark/speculative/bench_ngram_update_token_table.py b/test/registered/kernels/benchmark/embeddings/bench_ngram_update_token_table.py similarity index 97% rename from test/registered/kernels/benchmark/speculative/bench_ngram_update_token_table.py rename to test/registered/kernels/benchmark/embeddings/bench_ngram_update_token_table.py index bb179fbac0..7c61579684 100644 --- a/test/registered/kernels/benchmark/speculative/bench_ngram_update_token_table.py +++ b/test/registered/kernels/benchmark/embeddings/bench_ngram_update_token_table.py @@ -7,7 +7,7 @@ from sglang.kernels.jit.benchmark.utils import ( get_benchmark_range, run_benchmark_no_cudagraph, ) -from sglang.kernels.ops.speculative.ngram_embedding import ( +from sglang.kernels.ops.embeddings.ngram_embedding import ( update_token_table, update_token_table_decode, ) diff --git a/test/registered/kernels/ops/elementwise/test_bias_gelu.py b/test/registered/kernels/ops/activation/test_bias_gelu.py similarity index 95% rename from test/registered/kernels/ops/elementwise/test_bias_gelu.py rename to test/registered/kernels/ops/activation/test_bias_gelu.py index 6ff8159586..2f48cca96e 100644 --- a/test/registered/kernels/ops/elementwise/test_bias_gelu.py +++ b/test/registered/kernels/ops/activation/test_bias_gelu.py @@ -4,7 +4,7 @@ import pytest import torch import torch.nn.functional as F -from sglang.kernels.ops.elementwise.bias_gelu import bias_gelu_tanh +from sglang.kernels.ops.activation.bias_gelu import bias_gelu_tanh from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large") diff --git a/test/registered/kernels/ops/activation/test_swiglu_mxfp8_swizzled.py b/test/registered/kernels/ops/activation/test_swiglu_mxfp8_swizzled.py new file mode 100644 index 0000000000..68604ac022 --- /dev/null +++ b/test/registered/kernels/ops/activation/test_swiglu_mxfp8_swizzled.py @@ -0,0 +1,120 @@ +"""SwiGLU MXFP8 producers against ``flashinfer.mxfp8_quantize`` (or the DeepGEMM +quantizer) of the bf16 tensor the unfused activation stores: payload and the +scale buffer, padding included, byte for byte.""" + +import sys + +import pytest +import torch + +from sglang.kernels.ops.activation import swiglu_mxfp8_swizzled +from sglang.kernels.ops.activation.swiglu_mxfp8_swizzled import ( + can_use_silu_mul_mxfp8, + silu_mul_mxfp8, + swiglu_oai_mxfp8, + swiglu_oai_mxfp8_deepgemm, +) +from sglang.kernels.ops.quantization.fp8_kernel import sglang_per_token_group_quant_fp8 +from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import ( + swiglu_no_interleaved_with_alpha_and_limit, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +ROWS, HIDDEN = 333, 5376 # rows and scale columns both need padding + + +def _assert_matches_flashinfer( + got: tuple[torch.Tensor, torch.Tensor], bf16: torch.Tensor +) -> None: + import flashinfer + + q, s = flashinfer.mxfp8_quantize(bf16.contiguous(), True) + assert torch.equal(got[0].view(torch.uint8), q.view(torch.uint8)) + assert torch.equal(got[1].view(torch.uint8), s.view(torch.uint8)) + + +def test_silu_mul_mxfp8_is_byte_exact(monkeypatch: pytest.MonkeyPatch) -> None: + g = torch.Generator(device="cuda").manual_seed(2) + x = (torch.randn((ROWS, 2 * HIDDEN), device="cuda", generator=g) * 2).to( + torch.bfloat16 + ) + assert can_use_silu_mul_mxfp8(x) + ref = torch.nn.functional.silu(x[:, :HIDDEN]) * x[:, HIDDEN:] + _poison_scale_alloc(monkeypatch) + _assert_matches_flashinfer(silu_mul_mxfp8(x), ref) + + +def _poison_scale_alloc(monkeypatch: pytest.MonkeyPatch) -> None: + alloc = swiglu_mxfp8_swizzled._alloc + + def poisoned(*args, **kwargs): + q, s = alloc(*args, **kwargs) + return q, s.fill_(0xFF) + + monkeypatch.setattr(swiglu_mxfp8_swizzled, "_alloc", poisoned) + + +@pytest.mark.parametrize("rows", [1, 7, 128, 333]) +@pytest.mark.parametrize("n", [768, 1056]) +@pytest.mark.parametrize("scale", [1e-3, 1.0, 30.0]) +def test_swiglu_oai_mxfp8_matches_eager_chain( + rows: int, n: int, scale: float, monkeypatch: pytest.MonkeyPatch +) -> None: + g = torch.Generator(device="cuda").manual_seed(rows * 31 + n) + x = (torch.randn((rows, 2 * n + 64), device="cuda", generator=g) * scale).to( + torch.bfloat16 + )[:, : 2 * n] + ref = swiglu_no_interleaved_with_alpha_and_limit(x, 1.702, 7.0) + _poison_scale_alloc(monkeypatch) + _assert_matches_flashinfer(swiglu_oai_mxfp8(x, 1.702, 7.0), ref) + + +def test_swiglu_oai_mxfp8_cuda_graph_replay() -> None: + rows, n = 40, 768 + x = torch.randn((rows, 2 * n), device="cuda").to(torch.bfloat16) + swiglu_oai_mxfp8(x, 1.702, 7.0) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + got = swiglu_oai_mxfp8(x, 1.702, 7.0) + for scale in [0.0, 1e-4, 1.0, 100.0]: + x.copy_(torch.randn_like(x, dtype=torch.float32) * scale) + snapshot = x.clone() + graph.replay() + assert torch.equal(x, snapshot) + ref = swiglu_no_interleaved_with_alpha_and_limit(x, 1.702, 7.0) + _assert_matches_flashinfer(got, ref) + + +@pytest.mark.parametrize("rows", [1, 7, 128, 333, 2050]) +@pytest.mark.parametrize("n", [768, 1536]) +@pytest.mark.parametrize("scale", [0.0, 1e-3, 1.0, 30.0]) +def test_swiglu_oai_mxfp8_deepgemm_matches_eager_chain( + rows: int, n: int, scale: float +) -> None: + g = torch.Generator(device="cuda").manual_seed(rows * 31 + n) + x = (torch.randn((rows, 2 * n + 64), device="cuda", generator=g) * scale).to( + torch.bfloat16 + )[:, : 2 * n] + ref_q, ref_s = sglang_per_token_group_quant_fp8( + swiglu_no_interleaved_with_alpha_and_limit(x, 1.702, 7.0).contiguous(), + 32, + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + ) + q, s = swiglu_oai_mxfp8_deepgemm(x, 1.702, 7.0) + assert s.shape == ref_s.shape and s.stride() == ref_s.stride() + assert torch.equal(q.view(torch.uint8), ref_q.view(torch.uint8)) + assert torch.equal(s, ref_s) + + +def test_predicates_reject_unsupported_input() -> None: + assert not can_use_silu_mul_mxfp8( + torch.randn(4, 96, device="cuda", dtype=torch.bfloat16) + ) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py b/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py index 14b095ca31..960d1a7984 100644 --- a/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py +++ b/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py @@ -4,7 +4,7 @@ import unittest import torch -from sglang.kernels.ops.attention.dsv4.elementwise import fused_rope_inplace +from sglang.kernels.ops.attention.dsv4.norm_rope import fused_rope_inplace from sglang.srt.utils import is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/registered/kernels/ops/attention/dsv4/test_v41_kv_store.py b/test/registered/kernels/ops/attention/dsv4/test_v41_kv_store.py index 9ffba6ecd9..e8f9e48b0f 100644 --- a/test/registered/kernels/ops/attention/dsv4/test_v41_kv_store.py +++ b/test/registered/kernels/ops/attention/dsv4/test_v41_kv_store.py @@ -12,19 +12,21 @@ from typing import Tuple import torch -from sglang.kernels.ops.attention.deepseek_v4_rope import set_batched_rope from sglang.kernels.ops.attention.dsv4.attn import fused_store_cache from sglang.kernels.ops.attention.dsv4.compress import ( CompressorDecodePlan, compress_norm_rope_store, ) -from sglang.kernels.ops.attention.dsv4.dequant_k_cache import dequantize_k_cache_paged -from sglang.kernels.ops.attention.dsv4.elementwise import ( +from sglang.kernels.ops.attention.dsv4.dequant_k_cache import ( + dequantize_k_cache_paged_dsv4, +) +from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout +from sglang.kernels.ops.attention.dsv4.norm_rope import ( fused_k_norm_rope_flashmla, fused_rope_inplace, ) -from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout -from sglang.kernels.ops.attention.dsv4.torch_quant import fake_quant_compressed_kv +from sglang.kernels.ops.attention.dsv4.rope import set_batched_rope +from sglang.kernels.ops.quantization.fp4_fake_quant import fake_quant_compressed_kv from sglang.srt.utils import is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kernels.deepseek_v4 import dsv41_kv_quant_reference as tq @@ -523,7 +525,9 @@ class TestV41KVDequant(CustomTestCase): device="cuda", dtype=torch.int32, ) - got = dequantize_k_cache_paged(pages, ids, page_size, layout=layout) + got = dequantize_k_cache_paged_dsv4( + pages, ids, page_size, layout=layout + ) self.assertTrue( torch.equal( bits(got), @@ -576,7 +580,7 @@ class TestV41KVDequant(CustomTestCase): workspace = torch.zeros( 305, 1, 512, dtype=torch.bfloat16, device="cuda" ) - out = dequantize_k_cache_paged( + out = dequantize_k_cache_paged_dsv4( pages, ids, page_size, out=workspace[5:], layout=layout ) # NaN payloads (fp8 0x7F / e4m3 NaN scales) compare through their bits. diff --git a/test/registered/kernels/ops/attention/test_combine_topk_swa_indices.py b/test/registered/kernels/ops/attention/test_combine_topk_swa_indices.py index 87a743f45e..e359a03b69 100644 --- a/test/registered/kernels/ops/attention/test_combine_topk_swa_indices.py +++ b/test/registered/kernels/ops/attention/test_combine_topk_swa_indices.py @@ -18,7 +18,7 @@ Checks the Triton kernel against a per-row torch reference: import pytest import torch -from sglang.srt.layers.attention.dsv4.sparse_prefill_utils import ( +from sglang.kernels.ops.attention.dsv4.sparse_prefill_kernels import ( combine_topk_swa_indices, ) from sglang.test.ci.ci_register import register_cuda_ci diff --git a/test/registered/kernels/ops/attention/test_cutedsl_gdn.py b/test/registered/kernels/ops/attention/test_cutedsl_gdn.py index 6328c59572..d53af23c5b 100644 --- a/test/registered/kernels/ops/attention/test_cutedsl_gdn.py +++ b/test/registered/kernels/ops/attention/test_cutedsl_gdn.py @@ -18,7 +18,7 @@ except ImportError: cutedsl_gdn = None try: - from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( + from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, ) diff --git a/test/registered/kernels/ops/attention/test_dense_prefill_indexer.py b/test/registered/kernels/ops/attention/test_dense_prefill_indexer.py index 77df9c4670..3f74f4e9e1 100644 --- a/test/registered/kernels/ops/attention/test_dense_prefill_indexer.py +++ b/test/registered/kernels/ops/attention/test_dense_prefill_indexer.py @@ -3,7 +3,9 @@ from unittest.mock import patch import torch -from sglang.kernels.ops.attention.dsv4.fp4_indexer import quantize_fp4_indexer_tensor +from sglang.kernels.ops.quantization.fp4_indexer_quant import ( + quantize_fp4_indexer_tensor, +) from sglang.srt.layers.attention.dsv4.v41_indexer import dense_blocks, scoring from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/registered/kernels/ops/attention/test_dsa_indexer.py b/test/registered/kernels/ops/attention/test_dsa_indexer.py index 317fe9ffa2..d4fdaa604e 100644 --- a/test/registered/kernels/ops/attention/test_dsa_indexer.py +++ b/test/registered/kernels/ops/attention/test_dsa_indexer.py @@ -953,7 +953,7 @@ class TestDSAIndexer(CustomTestCase): @patch( "sglang.srt.layers.attention.dsa.dsa_indexer.resolve_paged_mqa_logits_metadata_fn" ) - @patch("sglang.kernels.ops.attention.dsa.triton_kernel.act_quant") + @patch("sglang.kernels.ops.quantization.act_quant_triton.act_quant") def test_forward_extend_mode( self, mock_act_quant, @@ -1048,7 +1048,7 @@ class TestDSAIndexer(CustomTestCase): @patch( "sglang.srt.layers.attention.dsa.dsa_indexer.resolve_paged_mqa_logits_metadata_fn" ) - @patch("sglang.kernels.ops.attention.dsa.triton_kernel.act_quant") + @patch("sglang.kernels.ops.quantization.act_quant_triton.act_quant") def test_forward_decode_mode( self, mock_act_quant, diff --git a/test/registered/kernels/ops/attention/test_dsa_kpool_metadata_fusion.py b/test/registered/kernels/ops/attention/test_dsa_kpool_metadata_fusion.py index 3e1f427025..c92b667ae0 100644 --- a/test/registered/kernels/ops/attention/test_dsa_kpool_metadata_fusion.py +++ b/test/registered/kernels/ops/attention/test_dsa_kpool_metadata_fusion.py @@ -4,8 +4,8 @@ import unittest import torch -from sglang.kernels.ops.attention.dsa_kpool_metadata.verify import ( - fused_dsa_target_verify_metadata, +from sglang.kernels.ops.attention.dsa.kpool_metadata.verify import ( + fused_dsa_target_verify_metadata_kpool, ) from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase @@ -37,7 +37,7 @@ class TestKPoolMetadataFusion(CustomTestCase): addresses = {key: value.data_ptr() for key, value in buffers.items()} def refresh(): - fused_dsa_target_verify_metadata( + fused_dsa_target_verify_metadata_kpool( seq_lens=seq, req_pool_indices=req, req_to_token=table, diff --git a/test/registered/kernels/ops/attention/test_dsa_metadata.py b/test/registered/kernels/ops/attention/test_dsa_metadata.py index db31b57309..4b011a3659 100644 --- a/test/registered/kernels/ops/attention/test_dsa_metadata.py +++ b/test/registered/kernels/ops/attention/test_dsa_metadata.py @@ -2,7 +2,7 @@ import unittest import torch -from sglang.kernels.ops.attention.dsa_metadata import ( +from sglang.kernels.ops.attention.dsa.metadata import ( fused_dsa_decode_metadata, fused_dsa_draft_extend_metadata, fused_dsa_target_verify_metadata, diff --git a/test/registered/kernels/ops/attention/test_dsa_transform_index.py b/test/registered/kernels/ops/attention/test_dsa_transform_index.py index 7791e7cdd6..a24e876704 100644 --- a/test/registered/kernels/ops/attention/test_dsa_transform_index.py +++ b/test/registered/kernels/ops/attention/test_dsa_transform_index.py @@ -5,8 +5,8 @@ import torch import sglang.kernels.ops.attention.dsa.transform_index as transform_index_module from sglang.kernels.ops.attention.dsa.transform_index import ( - transform_index_page_table_decode_fast, - transform_index_page_table_prefill_fast, + transform_index_page_table_decode, + transform_index_page_table_prefill, ) from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kernels.dsa_transform_index import ( @@ -121,7 +121,7 @@ class TestDSATransformIndex(CustomTestCase): expected[topk_indices < 0] = -1 result = torch.empty_like(expected) if provide_result else None - actual = transform_index_page_table_decode_fast( + actual = transform_index_page_table_decode( page_table=page_table, topk_indices=topk_indices, result=result, @@ -158,7 +158,7 @@ class TestDSATransformIndex(CustomTestCase): topk_width, ) - actual = transform_index_page_table_prefill_fast( + actual = transform_index_page_table_prefill( page_table=page_table, topk_indices=topk_indices, extend_lens_cpu=extend_lens_cpu, @@ -176,10 +176,10 @@ class TestDSATransformIndex(CustomTestCase): with patch.object( transform_index_module, - "transform_index_page_table_decode_fast", + "transform_index_page_table_decode", side_effect=AssertionError("prefill must not launch decode per request"), ): - transform_index_page_table_prefill_fast( + transform_index_page_table_prefill( page_table=page_table, topk_indices=topk_indices, extend_lens_cpu=extend_lens_cpu, @@ -197,7 +197,7 @@ class TestDSATransformIndex(CustomTestCase): "tensor", side_effect=AssertionError("must reuse device-side metadata"), ): - transform_index_page_table_prefill_fast( + transform_index_page_table_prefill( page_table=page_table, topk_indices=topk_indices, extend_lens_cpu=extend_lens_cpu, @@ -282,7 +282,7 @@ class TestDSATransformIndex(CustomTestCase): page_table_is_expanded=page_table_is_expanded, ) expected = transform_index_page_table_prefill_ref(**kwargs) - actual = transform_index_page_table_prefill_fast(**kwargs) + actual = transform_index_page_table_prefill(**kwargs) torch.cuda.synchronize() torch.testing.assert_close(actual, expected, rtol=0, atol=0) diff --git a/test/registered/kernels/ops/attention/test_dsv32_indexer_fusion.py b/test/registered/kernels/ops/attention/test_dsv32_indexer_fusion.py index c42cc8c350..190ae60f8f 100644 --- a/test/registered/kernels/ops/attention/test_dsv32_indexer_fusion.py +++ b/test/registered/kernels/ops/attention/test_dsv32_indexer_fusion.py @@ -21,8 +21,8 @@ from sglang.kernels.ops.attention.dsa.indexer_k import ( fused_k_indexer_norm_rope, fused_k_indexer_norm_rope_store, ) -from sglang.kernels.ops.attention.dsv4 import fused_q_indexer_rope_first_quant -from sglang.kernels.ops.attention.fused_store_index_cache import ( +from sglang.kernels.ops.attention.dsa.indexer_q import fused_q_indexer_rope_first_quant +from sglang.kernels.ops.kvcache.dsa_fused_store_index_cache import ( can_use_dsa_fused_store, fused_store_index_k_cache, ) diff --git a/test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py b/test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py index 1b61ae3b03..ce117e621d 100644 --- a/test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py +++ b/test/registered/kernels/ops/attention/test_dsv41_prefill_sparse_indexer.py @@ -10,7 +10,9 @@ import torch from sglang.kernels.ops.attention.dsv4.candidate_blocks import ( select_candidate_block_ids, ) -from sglang.kernels.ops.attention.dsv4.fp4_indexer import quantize_fp4_indexer_tensor +from sglang.kernels.ops.quantization.fp4_indexer_quant import ( + quantize_fp4_indexer_tensor, +) from sglang.srt.layers.attention.dsv4.v41_indexer.scoring import ( DeepGEMMPrefillData, ) diff --git a/test/registered/kernels/ops/attention/test_dsv4_indexer_quant.py b/test/registered/kernels/ops/attention/test_dsv4_indexer_quant.py index fe2d562b9a..6751cd30a0 100644 --- a/test/registered/kernels/ops/attention/test_dsv4_indexer_quant.py +++ b/test/registered/kernels/ops/attention/test_dsv4_indexer_quant.py @@ -20,10 +20,8 @@ import pytest import sgl_kernel # noqa: F401 the ROCm path dispatches to torch.ops.sgl_kernel import torch -from sglang.kernels.ops.attention.dsv4 import ( - fused_q_indexer_rope_first_quant, - fused_q_indexer_rope_hadamard_quant, -) +from sglang.kernels.ops.attention.dsa.indexer_q import fused_q_indexer_rope_first_quant +from sglang.kernels.ops.attention.dsv4 import fused_q_indexer_rope_hadamard_quant from sglang.srt.utils import is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci diff --git a/test/registered/kernels/ops/attention/test_dsv4_q_rope_store.py b/test/registered/kernels/ops/attention/test_dsv4_q_rope_store.py index 1b418bca40..347fb65198 100644 --- a/test/registered/kernels/ops/attention/test_dsv4_q_rope_store.py +++ b/test/registered/kernels/ops/attention/test_dsv4_q_rope_store.py @@ -2,8 +2,8 @@ import unittest import torch -from sglang.kernels.ops.attention import deepseek_v4_rope -from sglang.kernels.ops.attention.dsv4.elementwise import ( +from sglang.kernels.ops.attention.dsv4 import rope as deepseek_v4_rope +from sglang.kernels.ops.attention.dsv4.norm_rope import ( fused_q_norm_rope, fused_rope_inplace, ) diff --git a/test/registered/kernels/ops/attention/test_dsv4_trtllm_q_metadata.py b/test/registered/kernels/ops/attention/test_dsv4_trtllm_q_metadata.py index 1cf9bcd715..fd7ce51afc 100644 --- a/test/registered/kernels/ops/attention/test_dsv4_trtllm_q_metadata.py +++ b/test/registered/kernels/ops/attention/test_dsv4_trtllm_q_metadata.py @@ -3,7 +3,7 @@ import sys import pytest import torch -from sglang.kernels.ops.attention.dsv4.elementwise import fused_q_norm_rope +from sglang.kernels.ops.attention.dsv4.norm_rope import fused_q_norm_rope from sglang.kernels.ops.attention.dsv4.trtllm_metadata import pack_sparse_tail from sglang.test.ci.ci_register import register_cuda_ci diff --git a/test/registered/kernels/ops/attention/test_fp4_indexer.py b/test/registered/kernels/ops/attention/test_fp4_indexer.py index 83aced3ca5..735ae4f683 100644 --- a/test/registered/kernels/ops/attention/test_fp4_indexer.py +++ b/test/registered/kernels/ops/attention/test_fp4_indexer.py @@ -7,10 +7,6 @@ from unittest.mock import patch import pytest import torch -from sglang.kernels.ops.attention.deepseek_v4_rope import ( - apply_rotary_emb_triton, - precompute_freqs_cis, -) from sglang.kernels.ops.attention.dsv4 import ( CompressorDecodePlan, compress_norm_rope_store, @@ -20,17 +16,20 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( finish_paged_indexer_topk, fp4_index_logits_decode, fp4_index_logits_paged, - quantize_fp4_indexer_tensor, - store_fp4_index_k_cache, ) +from sglang.kernels.ops.attention.dsv4.rope import ( + apply_rotary_emb_triton, + precompute_freqs_cis, +) +from sglang.kernels.ops.kvcache.dsv4_fp4_index_k_cache import store_fp4_index_k_cache from sglang.srt.utils import get_device, is_xpu from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kernels.deepseek_v4.common import ( - ceil_ue8m0_exp_ref, - fp4_e2m1_code_ref, + quantize_fp4_indexer_ref, + store_fp4_index_cache_ref, ) -register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_cuda_ci(est_time=50, stage="base-b-kernel-unit", runner_config="1-gpu-large") _is_xpu = is_xpu() if _is_xpu: @@ -40,177 +39,8 @@ else: HEAD_DIM = 128 FP4_DIM = HEAD_DIM // 2 -GROUP_SIZE = 32 -SCALE_GROUPS = HEAD_DIM // GROUP_SIZE SCALE_BYTES = 4 PAGE_SIZE = 64 -E2M1_MAX = 6.0 - - -def _ref_quantize_fp4_indexer(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: - x = x.contiguous().view(-1, HEAD_DIM).float() - groups = x.view(-1, SCALE_GROUPS, GROUP_SIZE) - scale_raw = (groups.abs().amax(dim=-1) / E2M1_MAX).clamp_min(1.0e-4) - scale_exp = ceil_ue8m0_exp_ref(scale_raw) - scale = (scale_exp << 23).contiguous().view(torch.float32) - - scaled = (groups / scale.unsqueeze(-1)).view(-1, HEAD_DIM) - code = fp4_e2m1_code_ref(scaled) - packed = (code[:, 0::2].to(torch.int16) | (code[:, 1::2].to(torch.int16) << 4)).to( - torch.uint8 - ) - - packed_sf = scale_exp[:, 0].clone() - for group_id in range(1, SCALE_GROUPS): - packed_sf |= scale_exp[:, group_id] << (8 * group_id) - return packed, packed_sf - - -def _ref_store_fp4_index_cache( - x_fp4: torch.Tensor, - x_sf: torch.Tensor, - loc: torch.Tensor, - num_pages: int, -) -> torch.Tensor: - expected = torch.zeros( - num_pages, - PAGE_SIZE * (FP4_DIM + SCALE_BYTES), - device=x_fp4.device, - dtype=torch.uint8, - ) - sf_shifts = torch.arange(0, 32, 8, device=x_fp4.device, dtype=torch.int32) - for token_id in range(x_fp4.shape[0]): - cache_loc = int(loc[token_id].item()) - page = cache_loc // PAGE_SIZE - offset = cache_loc % PAGE_SIZE - expected[page, offset * FP4_DIM : (offset + 1) * FP4_DIM] = x_fp4[token_id] - sf_start = PAGE_SIZE * FP4_DIM + offset * SCALE_BYTES - expected[page, sf_start : sf_start + SCALE_BYTES] = ( - (x_sf[token_id] >> sf_shifts) & 0xFF - ).to(torch.uint8) - return expected - - -@pytest.mark.parametrize("num_tokens", [1, 7, 96]) -def test_quantize_fp4_indexer_tensor(num_tokens: int) -> None: - torch.manual_seed(num_tokens) - x = torch.randn(num_tokens, HEAD_DIM, device=get_device(), dtype=torch.bfloat16) - x[0, :8] = torch.tensor( - [-8.0, -6.0, -3.0, -1.5, 0.0, 0.5, 2.0, 8.0], - device=get_device(), - dtype=torch.bfloat16, - ) - - x_fp4, x_sf = quantize_fp4_indexer_tensor(x) - ref_fp4, ref_sf = _ref_quantize_fp4_indexer(x) - - torch.testing.assert_close(x_fp4.view(torch.uint8), ref_fp4) - torch.testing.assert_close(x_sf, ref_sf) - - -@pytest.mark.skipif( - not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10, - reason="Vectorized prefill dispatch targets Blackwell", -) -@pytest.mark.parametrize("rows", [4096, 4097, 16384, 524288]) -@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) -@pytest.mark.parametrize("rne", [False, True]) -@pytest.mark.parametrize("strided", [False, True]) -def test_prefill_quantization_matches_single_row_and_replay(rows, dtype, rne, strided): - from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( - _quantize_fp4_indexer_kernel, - ) - - x = torch.randn(rows, 256 if strided else 128, device="cuda", dtype=dtype) - if strided: - x = x[:, ::2] - - def reference(): - q = torch.empty(rows, 64, device="cuda", dtype=torch.int8) - sf = torch.empty(rows, device="cuda", dtype=torch.int32) - _quantize_fp4_indexer_kernel[(rows,)]( - x.contiguous(), - q, - sf, - BLOCK_N=128, - GROUP_N=32, - RNE=rne, - ) - return q, sf - - for _ in range(3): - quantize_fp4_indexer_tensor(x, rne) - reference() - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - actual = quantize_fp4_indexer_tensor(x, rne) - boundaries = torch.tensor( - [0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0, 6.0], device="cuda", dtype=dtype - ) - for scale in (0.0, 1e-6, 1.0, 1e3): - x.normal_().mul_(scale) - x[:2] = boundaries.repeat(16) - x[1].neg_() - graph.replay() - for a, b in zip(actual, reference()): - assert torch.equal(a, b) - - -@pytest.mark.skipif( - not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10, - reason="Vectorized prefill dispatch targets Blackwell", -) -@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) -@pytest.mark.parametrize("rne", [False, True]) -def test_prefill_quantization_nonfinite_group_replay(dtype, rne): - from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( - _quantize_fp4_indexer_kernel, - ) - - rows = 4097 # Also exercise the final partial CTA. - x = torch.randn(rows, HEAD_DIM, device="cuda", dtype=dtype) - expected = ( - torch.empty(rows, FP4_DIM, device="cuda", dtype=torch.int8), - torch.empty(rows, device="cuda", dtype=torch.int32), - ) - for _ in range(3): - quantize_fp4_indexer_tensor(x, rne) - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - actual = quantize_fp4_indexer_tensor(x, rne) - for value in (float("inf"), float("-inf"), float("nan"), 0.0): - x.normal_() - x[:, 0] = value - x[:, 32:64] = value - x[-1, 96:] = value - graph.replay() - _quantize_fp4_indexer_kernel[(rows,)]( - x, *expected, BLOCK_N=HEAD_DIM, GROUP_N=GROUP_SIZE, RNE=rne - ) - for a, b in zip(actual, expected): - assert torch.equal(a, b) - - -@pytest.mark.parametrize("num_tokens", [1, 16, 96]) -def test_fp4_index_cache_store_layout(num_tokens: int) -> None: - torch.manual_seed(num_tokens) - num_pages = max(1, (num_tokens + PAGE_SIZE - 1) // PAGE_SIZE) - x = torch.randn(num_tokens, HEAD_DIM, device=get_device(), dtype=torch.bfloat16) - loc = torch.randperm(num_pages * PAGE_SIZE, device=get_device())[:num_tokens].to( - torch.int64 - ) - cache = torch.zeros( - num_pages, - PAGE_SIZE * (FP4_DIM + SCALE_BYTES), - device=get_device(), - dtype=torch.uint8, - ) - - store_fp4_index_k_cache(x, cache, loc, page_size=PAGE_SIZE) - - ref_fp4, ref_sf = _ref_quantize_fp4_indexer(x) - expected = _ref_store_fp4_index_cache(ref_fp4, ref_sf, loc, num_pages) - torch.testing.assert_close(cache, expected) @pytest.mark.parametrize("num_tokens", [1, 16, 96]) @@ -264,9 +94,9 @@ def test_fp4_fused_norm_rope_store_layout(num_tokens: int) -> None: rope_out[..., 1] = rope[..., 0] * freqs[..., 1] + rope[..., 1] * freqs[..., 0] ref[:, 64:] = rope_out.reshape(num_tokens, 64) ref = hadamard_transform(ref.contiguous(), scale=HEAD_DIM**-0.5) - ref_fp4, ref_sf = _ref_quantize_fp4_indexer(ref) + ref_fp4, ref_sf = quantize_fp4_indexer_ref(ref) - expected = _ref_store_fp4_index_cache( + expected = store_fp4_index_cache_ref( ref_fp4, ref_sf, loc.to(torch.int64), @@ -303,7 +133,7 @@ def test_fp4_fused_q_indexer_rope_hadamard_quant(batch_size: int) -> None: ref = q.clone() apply_rotary_emb_triton(ref[..., -rope_dim:], freqs_cis, positions=positions) ref = hadamard_transform(ref.contiguous(), scale=HEAD_DIM**-0.5) - ref_fp4, ref_sf = _ref_quantize_fp4_indexer(ref.view(-1, HEAD_DIM)) + ref_fp4, ref_sf = quantize_fp4_indexer_ref(ref.view(-1, HEAD_DIM)) ref_fp4 = ref_fp4.view(batch_size, num_heads, FP4_DIM) ref_sf = ref_sf.view(batch_size, num_heads) diff --git a/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py b/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py index 3192aea4da..856d14de2d 100644 --- a/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py +++ b/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py @@ -22,7 +22,6 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.attention.deepseek_v4_rope import precompute_freqs_cis from sglang.kernels.ops.attention.dsv4 import ( CompressorDecodePlan, compress_norm_rope_store, @@ -36,10 +35,6 @@ from sglang.kernels.ops.attention.dsv4.candidate_blocks_hip import ( with_candidate_page8_table, ) from sglang.kernels.ops.attention.dsv4.compress import CompressorPrefillPlan -from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( - _fp4_e2m1_code_rne, - quantize_fp4_indexer_tensor, -) from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( LOW_RATIO_PAGE_TABLE_BUCKET, FP4KWriteMetadata, @@ -54,18 +49,15 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer_hip import ( index_k_page8_views, index_q_rope_pack_flydsl, index_q_rope_pack_weights_flydsl, - indexer_head_weights, pack_fp4_query_flydsl, prepare_fp4_decode_workspace, prepare_fp4_k_write_metadata, prepare_fp4_prefill_workspace, prepare_fp4_rowgroup_decode_workspace, prepare_fp4_rowgroup_prefill_workspace, - read_fp4_index_k_split, rowgroup_paged_mqa_logits, rowgroup_prefill_query_start_loc, rowgroup_table_columns, - store_fp4_index_k_cache_split, ) from sglang.kernels.ops.attention.dsv4.fp4_indexer_rope_hip import ( index_k_norm_rope_pack_store_split, @@ -76,17 +68,29 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer_schedule_hip import ( from sglang.kernels.ops.attention.dsv4.fp4_rope_fake_quant import ( rope_tail_fake_quant_fp4, ) +from sglang.kernels.ops.attention.dsv4.rope import precompute_freqs_cis from sglang.kernels.ops.attention.dsv4.topk import ( plan_topk_v2, topk_transform_paged_v2, ) +from sglang.kernels.ops.gemm.dsv4_indexer_head_weights_hip import indexer_head_weights from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k +from sglang.kernels.ops.kvcache.dsv4_fp4_index_k_cache_hip import ( + store_fp4_index_k_cache_split, +) +from sglang.kernels.ops.quantization.fp4_indexer_quant import ( + _fp4_e2m1_code_rne, + quantize_fp4_indexer_tensor, +) from sglang.srt.utils import get_device, is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci from sglang.test.kernels.deepseek_v4.common import ( ceil_ue8m0_exp_ref, fp4_e2m1_code_ref, ) +from sglang.test.kernels.deepseek_v4.fp4_index_k_reference import ( + read_fp4_index_k_split, +) register_amd_ci(est_time=60, suite="stage-b-test-1-gpu-small-amd-mi35x") diff --git a/test/registered/kernels/ops/attention/test_fused_kda_conv_recurrent_verify.py b/test/registered/kernels/ops/attention/test_fused_kda_conv_recurrent_verify.py index 1e470d228f..e7426046df 100644 --- a/test/registered/kernels/ops/attention/test_fused_kda_conv_recurrent_verify.py +++ b/test/registered/kernels/ops/attention/test_fused_kda_conv_recurrent_verify.py @@ -3,10 +3,10 @@ import sys import pytest import torch -from sglang.kernels.ops.attention.fla.fused_kda_conv_recurrent_verify import ( +from sglang.kernels.ops.attention.linear.fused_kda_conv_recurrent_verify import ( fused_kda_conv_gating_verify, ) -from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( +from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, ) from sglang.kernels.ops.mamba.causal_conv1d_triton import ( diff --git a/test/registered/kernels/ops/attention/test_fused_metadata_copy.py b/test/registered/kernels/ops/attention/test_fused_metadata_copy.py index a4f093c247..4724c6ac5f 100644 --- a/test/registered/kernels/ops/attention/test_fused_metadata_copy.py +++ b/test/registered/kernels/ops/attention/test_fused_metadata_copy.py @@ -152,7 +152,7 @@ def test_fused_metadata_copy_dtype_validation(): if not torch.cuda.is_available(): pytest.skip("CUDA not available") - from sglang.kernels.ops.attention.fused_metadata_copy import ( + from sglang.kernels.ops.attention.dsa.fused_metadata_copy import ( fused_metadata_copy_cuda, ) @@ -271,7 +271,7 @@ def test_fused_metadata_copy(bs, has_real_page_table, has_flashmla): if not torch.cuda.is_available(): pytest.skip("CUDA not available") - from sglang.kernels.ops.attention.fused_metadata_copy import ( + from sglang.kernels.ops.attention.dsa.fused_metadata_copy import ( fused_metadata_copy_cuda, ) @@ -472,7 +472,7 @@ def test_fused_metadata_copy_multi_dtype_validation(): if not torch.cuda.is_available(): pytest.skip("CUDA not available") - from sglang.kernels.ops.attention.fused_metadata_copy import ( + from sglang.kernels.ops.attention.dsa.fused_metadata_copy import ( fused_metadata_copy_multi_cuda, ) @@ -580,7 +580,7 @@ def test_fused_metadata_copy_multi(bs, has_real_page_table, has_flashmla): if not torch.cuda.is_available(): pytest.skip("CUDA not available") - from sglang.kernels.ops.attention.fused_metadata_copy import ( + from sglang.kernels.ops.attention.dsa.fused_metadata_copy import ( fused_metadata_copy_multi_cuda, ) diff --git a/test/registered/kernels/ops/attention/test_fused_verify_triton_gdn.py b/test/registered/kernels/ops/attention/test_fused_verify_triton_gdn.py index 55f434fdf5..2055fb8054 100644 --- a/test/registered/kernels/ops/attention/test_fused_verify_triton_gdn.py +++ b/test/registered/kernels/ops/attention/test_fused_verify_triton_gdn.py @@ -15,14 +15,14 @@ from sglang.srt.utils import is_gfx95_supported, is_sm90_supported from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci try: - from sglang.kernels.ops.attention.fla import ( - fused_sigmoid_gating_recurrent as recurrent_module, - ) - from sglang.kernels.ops.attention.fla.fused_gdn_gating import fused_gdn_gating from sglang.kernels.ops.attention.fla.fused_recurrent import ( fused_recurrent_gated_delta_rule_update, ) - from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( + from sglang.kernels.ops.attention.linear import ( + fused_sigmoid_gating_recurrent as recurrent_module, + ) + from sglang.kernels.ops.attention.linear.fused_gdn_gating import fused_gdn_gating + from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( _select_recurrent_launch_config, fused_sigmoid_gating_delta_rule_update, ) diff --git a/test/registered/kernels/ops/attention/test_inkling_attn_prologue_tau.py b/test/registered/kernels/ops/attention/test_inkling_attn_prologue_tau.py index a5e4a78047..a1852ff301 100644 --- a/test/registered/kernels/ops/attention/test_inkling_attn_prologue_tau.py +++ b/test/registered/kernels/ops/attention/test_inkling_attn_prologue_tau.py @@ -111,7 +111,7 @@ def test_prologue_decode_tau_fold(t): def test_rel_logits_proj_prescale_tau(t): """RelLogitsProj's operand-side tau fold (r*tau before the einsum) must match the legacy output-side scale within bf16 rounding.""" - from sglang.kernels.ops.attention.log_scaling_tau import apply_log_scaling_tau + from sglang.kernels.ops.elementwise.row_scale import apply_log_scaling_tau from sglang.srt.models.inkling_common.attn import RelLogitsProj torch.manual_seed(t) @@ -164,7 +164,7 @@ def test_rel_logits_proj_strided_dispatch(t): assert torch.equal(out, ref) # The tau path on the same strided layout (prescale compacts first). - from sglang.kernels.ops.attention.log_scaling_tau import apply_log_scaling_tau + from sglang.kernels.ops.elementwise.row_scale import apply_log_scaling_tau tau = 1.0 + 0.1 * torch.rand(t, device="cuda", dtype=torch.float32) ref_tau = apply_log_scaling_tau(ref, tau.view(-1, 1, 1)) diff --git a/test/registered/kernels/ops/attention/test_kda_fused_verify_backend.py b/test/registered/kernels/ops/attention/test_kda_fused_verify_backend.py index 0a30844fd4..b26d0473e9 100644 --- a/test/registered/kernels/ops/attention/test_kda_fused_verify_backend.py +++ b/test/registered/kernels/ops/attention/test_kda_fused_verify_backend.py @@ -6,7 +6,7 @@ from unittest.mock import Mock import torch -from sglang.kernels.ops.attention.fla.fused_kda_conv_recurrent_verify import ( +from sglang.kernels.ops.attention.linear.fused_kda_conv_recurrent_verify import ( fused_kda_conv_gating_verify, ) from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( diff --git a/test/registered/kernels/ops/attention/test_kda_helion.py b/test/registered/kernels/ops/attention/test_kda_helion.py index 3325f2adca..5012ceda9f 100644 --- a/test/registered/kernels/ops/attention/test_kda_helion.py +++ b/test/registered/kernels/ops/attention/test_kda_helion.py @@ -9,10 +9,10 @@ import torch from sglang.kernels.ops.attention.fla.fused_recurrent import ( fused_recurrent_kda_packed_decode, ) -from sglang.kernels.ops.attention.fla.fused_recurrent_linear_replayssm import ( +from sglang.kernels.ops.attention.fla.kda import chunk_kda as triton_chunk_kda +from sglang.kernels.ops.attention.linear.fused_recurrent_linear_replayssm import ( fused_recurrent_linear_replayssm_decode, ) -from sglang.kernels.ops.attention.fla.kda import chunk_kda as triton_chunk_kda from sglang.test.ci.ci_register import register_cuda_ci try: diff --git a/test/registered/kernels/ops/attention/test_kda_mtp_cutedsl_replayssm_ring.py b/test/registered/kernels/ops/attention/test_kda_mtp_cutedsl_replayssm_ring.py index 86bb5afb38..05b9d27e94 100644 --- a/test/registered/kernels/ops/attention/test_kda_mtp_cutedsl_replayssm_ring.py +++ b/test/registered/kernels/ops/attention/test_kda_mtp_cutedsl_replayssm_ring.py @@ -32,12 +32,12 @@ if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10: allow_module_level=True, ) -from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( # noqa: E402 - commit_kda_replayssm_spec, -) from sglang.kernels.ops.attention.kda_decode_mtp import ( # noqa: E402 fused_kda_decode_mtp_dspark, ) +from sglang.kernels.ops.attention.linear.kda_replayssm_spec_decode import ( # noqa: E402 + commit_kda_replayssm_spec, +) DEV = "cuda" K = 128 diff --git a/test/registered/kernels/ops/attention/test_kda_replayssm_fold.py b/test/registered/kernels/ops/attention/test_kda_replayssm_fold.py index 249b400766..cae24498e6 100644 --- a/test/registered/kernels/ops/attention/test_kda_replayssm_fold.py +++ b/test/registered/kernels/ops/attention/test_kda_replayssm_fold.py @@ -35,10 +35,10 @@ if not torch.cuda.is_available(): allow_module_level=True, ) -from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( # noqa: E402 +from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( # noqa: E402 fused_sigmoid_gating_delta_rule_update, ) -from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( # noqa: E402 +from sglang.kernels.ops.attention.linear.kda_replayssm_spec_decode import ( # noqa: E402 commit_kda_replayssm_spec, ) diff --git a/test/registered/kernels/ops/attention/test_kda_replayssm_fold_batched.py b/test/registered/kernels/ops/attention/test_kda_replayssm_fold_batched.py index 938c7b1d4b..dfe2341b25 100644 --- a/test/registered/kernels/ops/attention/test_kda_replayssm_fold_batched.py +++ b/test/registered/kernels/ops/attention/test_kda_replayssm_fold_batched.py @@ -25,7 +25,7 @@ if not torch.cuda.is_available(): allow_module_level=True, ) -from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( # noqa: E402 +from sglang.kernels.ops.attention.linear.kda_replayssm_spec_decode import ( # noqa: E402 commit_kda_replayssm_spec, commit_kda_replayssm_spec_all_layers, ) diff --git a/test/registered/kernels/ops/attention/test_kda_replayssm_ring_fused.py b/test/registered/kernels/ops/attention/test_kda_replayssm_ring_fused.py index afbba8dbb4..c51b09cf4a 100644 --- a/test/registered/kernels/ops/attention/test_kda_replayssm_ring_fused.py +++ b/test/registered/kernels/ops/attention/test_kda_replayssm_ring_fused.py @@ -28,10 +28,10 @@ if not torch.cuda.is_available(): allow_module_level=True, ) -from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( # noqa: E402 +from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( # noqa: E402 fused_sigmoid_gating_delta_rule_update, ) -from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( # noqa: E402 +from sglang.kernels.ops.attention.linear.kda_replayssm_spec_decode import ( # noqa: E402 commit_kda_replayssm_spec, ) diff --git a/test/registered/kernels/ops/attention/test_kda_replayssm_ring_ragged.py b/test/registered/kernels/ops/attention/test_kda_replayssm_ring_ragged.py index daef77a704..4ecf7774ac 100644 --- a/test/registered/kernels/ops/attention/test_kda_replayssm_ring_ragged.py +++ b/test/registered/kernels/ops/attention/test_kda_replayssm_ring_ragged.py @@ -19,10 +19,10 @@ if not torch.cuda.is_available(): allow_module_level=True, ) -from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( # noqa: E402 +from sglang.kernels.ops.attention.linear.fused_sigmoid_gating_recurrent import ( # noqa: E402 fused_sigmoid_gating_delta_rule_update, ) -from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( # noqa: E402 +from sglang.kernels.ops.attention.linear.kda_replayssm_spec_decode import ( # noqa: E402 commit_kda_replayssm_spec, ) diff --git a/python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_flash_with_topk_idx.py b/test/registered/kernels/ops/attention/test_minimax_sparse_flash_with_topk_idx.py similarity index 98% rename from python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_flash_with_topk_idx.py rename to test/registered/kernels/ops/attention/test_minimax_sparse_flash_with_topk_idx.py index 81991455c1..8033adf174 100644 --- a/python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_flash_with_topk_idx.py +++ b/test/registered/kernels/ops/attention/test_minimax_sparse_flash_with_topk_idx.py @@ -8,6 +8,11 @@ from sglang.kernels.ops.attention.minimax_sparse.decode.flash_with_topk_idx impo ) from sglang.srt.environ import envs from sglang.srt.utils import get_device, is_xpu +from sglang.test.ci.ci_register import register_cuda_ci + +# ~8 min of Triton autotuning on a cold cache: nightly rather than per-commit. +register_cuda_ci(est_time=600, stage="nightly", runner_config="1-gpu-large") +register_cuda_ci(est_time=500, stage="nightly", runner_config="4-gpu-b200") DEVICE = get_device() RTOL_VS_REF = 5e-3 diff --git a/python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_fp8_attn_gemm.py b/test/registered/kernels/ops/attention/test_minimax_sparse_fp8_attn_gemm.py similarity index 96% rename from python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_fp8_attn_gemm.py rename to test/registered/kernels/ops/attention/test_minimax_sparse_fp8_attn_gemm.py index ad4c1b6425..6a7194ec24 100644 --- a/python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_fp8_attn_gemm.py +++ b/test/registered/kernels/ops/attention/test_minimax_sparse_fp8_attn_gemm.py @@ -14,18 +14,22 @@ indexer, non-unit k_scale/v_scale semantics, and bf16-path regression. import pytest import torch -from sglang.srt.layers.attention.minimax_sparse_ops.decode.flash_with_topk_idx import ( +from sglang.kernels.ops.attention.minimax_sparse.decode.flash_with_topk_idx import ( flash_decode_with_topk_idx, ) -from sglang.srt.layers.attention.minimax_sparse_ops.decode.topk_sparse import ( +from sglang.kernels.ops.attention.minimax_sparse.decode.topk_sparse import ( flash_decode_with_gqa_share_sparse, ) -from sglang.srt.layers.attention.minimax_sparse_ops.prefill.flash_with_topk_idx import ( +from sglang.kernels.ops.attention.minimax_sparse.prefill.flash_with_topk_idx import ( flash_prefill_with_topk_index, ) -from sglang.srt.layers.attention.minimax_sparse_ops.prefill.topk_sparse import ( +from sglang.kernels.ops.attention.minimax_sparse.prefill.topk_sparse import ( flash_prefill_with_gqa_share_sparse, ) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=150, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_cuda_ci(est_time=110, stage="base-b-kernel-unit", runner_config="4-gpu-b200") DEVICE = "cuda" FP8 = torch.float8_e4m3fn diff --git a/python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_sparse_gqa.py b/test/registered/kernels/ops/attention/test_minimax_sparse_gqa.py similarity index 97% rename from python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_sparse_gqa.py rename to test/registered/kernels/ops/attention/test_minimax_sparse_gqa.py index a0c9b3db52..86677a91ce 100644 --- a/python/sglang/srt/layers/attention/minimax_sparse_ops/tests/test_sparse_gqa.py +++ b/test/registered/kernels/ops/attention/test_minimax_sparse_gqa.py @@ -14,6 +14,10 @@ from sglang.kernels.ops.attention.minimax_sparse.decode.topk_sparse import ( flash_decode_with_gqa_share_sparse, ) from sglang.srt.utils import get_device +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_cuda_ci(est_time=40, stage="base-b-kernel-unit", runner_config="4-gpu-b200") DEVICE = get_device() RTOL = 5e-3 diff --git a/test/registered/kernels/ops/attention/test_muse_spark_qk_norm_rope_store.py b/test/registered/kernels/ops/attention/test_muse_spark_qk_norm_rope_store.py new file mode 100644 index 0000000000..652ae03cdc --- /dev/null +++ b/test/registered/kernels/ops/attention/test_muse_spark_qk_norm_rope_store.py @@ -0,0 +1,71 @@ +"""Muse Spark QK norm (+ interleaved RoPE, + KV store): Triton against the torch reference.""" + +import pytest +import torch + +from sglang.kernels.ops.attention.muse_spark_qk_norm_rope_store import FusedQKNorm +from sglang.kernels.spec import KernelBackend +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +_HEAD_DIM, _NUM_Q, _NUM_KV, _MAX_POS, _SLOTS = 64, 8, 2, 4096, 1024 + + +def _cos_sin_cache(): + inv_freq = 1.0 / ( + 10000.0 ** (torch.arange(0, _HEAD_DIM, 2, device="cuda") / _HEAD_DIM) + ) + freqs = torch.outer(torch.arange(_MAX_POS, device="cuda").float(), inv_freq) + return torch.cat([freqs.cos(), freqs.sin()], dim=-1) + + +@pytest.mark.parametrize("num_tokens", [1, 17, 700]) +@pytest.mark.parametrize("rope", [False, True]) +@pytest.mark.parametrize("cache_dtype", [None, torch.bfloat16, torch.float8_e4m3fn]) +def test_qk_norm_matches_native(num_tokens, rope, cache_dtype): + torch.manual_seed(num_tokens) + op = FusedQKNorm( + head_dim=_HEAD_DIM, num_q_heads=_NUM_Q, num_kv_heads=_NUM_KV, eps=1e-6 + ) + width = (_NUM_Q + 2 * _NUM_KV) * _HEAD_DIM + # A row-strided view, as the fused QKV-O projection output is. + qkv = torch.randn(num_tokens, width + 64, device="cuda", dtype=torch.bfloat16)[ + :, :width + ] + positions = torch.randint(0, _MAX_POS, (num_tokens,), device="cuda") + args = (positions, _cos_sin_cache()) if rope else (None, None) + + def kv_store(): + if cache_dtype is None: + return None + torch.manual_seed(0) + shape = (_SLOTS, _NUM_KV * _HEAD_DIM) + k_cache = torch.randn(shape, device="cuda").to(cache_dtype) + v_cache = torch.randn(shape, device="cuda").to(cache_dtype) + # Slot 0 is the CUDA-graph padding slot and is never written. + cache_loc = torch.randperm(_SLOTS, device="cuda")[:num_tokens] + return k_cache, v_cache, cache_loc + + ref_store, store = kv_store(), kv_store() + ref_q, ref_k = op.forward_native(qkv, *args, kv_store=ref_store) + q, k = op(qkv, *args, kv_store=store, backend=KernelBackend.TRITON) + + assert q.dtype == ref_q.dtype and k.dtype == ref_k.dtype + _assert_close(q, ref_q) + _assert_close(k, ref_k) + if cache_dtype is not None: + for out, ref in zip(store[:2], ref_store[:2]): + _assert_close(out, ref) + + +def _assert_close(out, ref): + # One e4m3 step is 2^-3 relative; Triton and torch may round a tie apart. + tol = 0.125 if out.dtype == torch.float8_e4m3fn else 2e-2 + torch.testing.assert_close(out.float(), ref.float(), rtol=tol, atol=tol) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/attention/test_q8kv8_dsa_nope_bf16_prefill.py b/test/registered/kernels/ops/attention/test_q8kv8_dsa_nope_bf16_prefill.py index c0b6e1c90b..daad06ebe2 100644 --- a/test/registered/kernels/ops/attention/test_q8kv8_dsa_nope_bf16_prefill.py +++ b/test/registered/kernels/ops/attention/test_q8kv8_dsa_nope_bf16_prefill.py @@ -16,7 +16,7 @@ from sglang.kernels.ops.attention.dsa.dequant_k_cache import ( concat_cast_kv_fp8_pad, gather_cast_kv_fp8_pad_paged, ) -from sglang.kernels.ops.kvcache.cache_ops import concat_and_cast_q_fp8_pad +from sglang.kernels.ops.attention.mla_qk_prep import concat_and_cast_q_fp8_pad from sglang.srt.utils import is_sm90_supported from sglang.test.ci.ci_register import register_cuda_ci diff --git a/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py b/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py index b3be0ef2d3..77d2fde16a 100644 --- a/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py +++ b/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py @@ -18,6 +18,8 @@ import torch from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout from sglang.kernels.ops.attention.sparse_mla_q8kv8_prefill_sm90 import ( + cast_q_fp8_for_q8kv8_prefill, + q8kv8_padded_num_heads, sparse_mla_q8kv8_prefill_fwd, ) from sglang.srt.layers.attention.deepseek_v4_backend import DeepseekV4AttnBackend @@ -443,6 +445,8 @@ def _patched_sparse_kernels( q8_module_name = "sglang.kernels.ops.attention.sparse_mla_q8kv8_prefill_sm90" q8_mod = types.ModuleType(q8_module_name) q8_mod.sparse_mla_q8kv8_prefill_fwd = fake_sparse_mla_q8kv8_prefill_fwd + q8_mod.q8kv8_padded_num_heads = q8kv8_padded_num_heads + q8_mod.cast_q_fp8_for_q8kv8_prefill = cast_q_fp8_for_q8kv8_prefill old_modules = { name: sys.modules.get(name) diff --git a/test/registered/kernels/ops/attention/test_qprep_bf16_fp8_sm90.py b/test/registered/kernels/ops/attention/test_qprep_bf16_fp8_sm90.py index 5a7858be74..ea71f79adc 100644 --- a/test/registered/kernels/ops/attention/test_qprep_bf16_fp8_sm90.py +++ b/test/registered/kernels/ops/attention/test_qprep_bf16_fp8_sm90.py @@ -50,10 +50,8 @@ def _make_inputs(T: int, H: int, K: int, seed: int = 1234, magnitude: float = 1. ) @pytest.mark.parametrize("pad_heads_extra", [0, 2]) def test_qprep_vs_triton_two_dot(T, h_k, pad_heads_extra): + from sglang.kernels.ops.attention.mla_qk_prep import absorbed_bmm_concat_cast_q_fp8 from sglang.kernels.ops.attention.qprep_bf16_fp8_sm90 import q8kv8_qprep_fwd - from sglang.kernels.ops.kvcache.cache_ops import ( - absorbed_bmm_concat_cast_q_fp8, - ) H, K = h_k q_nope, q_rope, w_kc = _make_inputs(T, H, K) @@ -88,10 +86,8 @@ def test_qprep_vs_triton_two_dot(T, h_k, pad_heads_extra): "h_k", [(64, 192), (128, 128)], ids=["glm_h64_k192", "ds_h128_k128"] ) def test_qprep_fp64_reference_parity(h_k): + from sglang.kernels.ops.attention.mla_qk_prep import absorbed_bmm_concat_cast_q_fp8 from sglang.kernels.ops.attention.qprep_bf16_fp8_sm90 import q8kv8_qprep_fwd - from sglang.kernels.ops.kvcache.cache_ops import ( - absorbed_bmm_concat_cast_q_fp8, - ) H, K = h_k T = 437 diff --git a/test/registered/kernels/ops/attention/test_sparse_mla_q8kv8_prefill_sm90.py b/test/registered/kernels/ops/attention/test_sparse_mla_q8kv8_prefill_sm90.py index fd7d5f64e9..abd46f33d9 100644 --- a/test/registered/kernels/ops/attention/test_sparse_mla_q8kv8_prefill_sm90.py +++ b/test/registered/kernels/ops/attention/test_sparse_mla_q8kv8_prefill_sm90.py @@ -691,9 +691,7 @@ def test_sparse_mla_q8kv8_prefill_large_skv(): ) @pytest.mark.parametrize("s_q,topk", [(437, 2048), (7, 128), (65, 256), (4096, 2048)]) def test_q8kv8_topk_length_backscan(s_q: int, topk: int): - from sglang.kernels.ops.kvcache.cache_ops import ( - q8kv8_topk_length_from_indices, - ) + from sglang.kernels.ops.attention.metadata import q8kv8_topk_length_from_indices generator = torch.Generator(device="cuda") generator.manual_seed(4000 + s_q + topk) diff --git a/test/registered/kernels/ops/attention/test_tilelang_dsa_zero_rope.py b/test/registered/kernels/ops/attention/test_tilelang_dsa_zero_rope.py index 5ccfe60101..f405dd8684 100644 --- a/test/registered/kernels/ops/attention/test_tilelang_dsa_zero_rope.py +++ b/test/registered/kernels/ops/attention/test_tilelang_dsa_zero_rope.py @@ -8,9 +8,9 @@ import torch from sglang.kernels.ops.attention.dsa.dequant_k_cache import ( _infer_dsa_dims, dequantize_k_cache, - dequantize_k_cache_paged, + dequantize_k_cache_paged_dsa, ) -from sglang.kernels.ops.attention.dsa.quant_k_cache import quantize_k_cache +from sglang.kernels.ops.quantization.dsa_quant_k_cache import quantize_k_cache from sglang.srt.utils import is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase @@ -48,7 +48,7 @@ class TestScaledCacheLayouts(CustomTestCase): torch.testing.assert_close(restored, source, atol=0.08, rtol=0.08) pages = torch.tensor([7, 1, 1, 4], device="cuda", dtype=torch.int32) - gathered = dequantize_k_cache_paged(packed, pages) + gathered = dequantize_k_cache_paged_dsa(packed, pages) expected = restored.view(8, 1, -1)[pages] torch.testing.assert_close(gathered, expected, atol=0, rtol=0) diff --git a/test/registered/kernels/ops/attention/test_trtllm_kv_norm_rope_store.py b/test/registered/kernels/ops/attention/test_trtllm_kv_norm_rope_store.py index 6204405fda..f9bfa7c476 100644 --- a/test/registered/kernels/ops/attention/test_trtllm_kv_norm_rope_store.py +++ b/test/registered/kernels/ops/attention/test_trtllm_kv_norm_rope_store.py @@ -5,10 +5,10 @@ import sys import pytest import torch -from sglang.kernels.ops.attention.deepseek_v4_rope import fused_norm_rope_inplace_triton from sglang.kernels.ops.attention.dsv4.kv_norm_rope_store import ( fused_k_norm_rope_uniform_fp8, ) +from sglang.kernels.ops.attention.dsv4.rope import fused_norm_rope_inplace_triton from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=40, stage="base-b-kernel-unit", runner_config="1-gpu-large") diff --git a/test/registered/kernels/ops/communication/test_ar_fusion.py b/test/registered/kernels/ops/communication/test_ar_fusion.py index cd357a4274..b61ed58295 100644 --- a/test/registered/kernels/ops/communication/test_ar_fusion.py +++ b/test/registered/kernels/ops/communication/test_ar_fusion.py @@ -26,7 +26,7 @@ import torch.distributed as dist import sglang.srt.distributed.parallel_state as ps from sglang.kernels.jit.utils import cache_once, get_ci_test_range from sglang.kernels.ops.communication import all_reduce_residual as all_reduce -from sglang.kernels.ops.communication.mp import register_comm_cleanup +from sglang.srt.distributed.comm_cleanup import register_comm_cleanup from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( CustomAllReduceV2, ) diff --git a/test/registered/kernels/ops/communication/test_collectives.py b/test/registered/kernels/ops/communication/test_collectives.py index d326bbe7d6..b5475b2041 100644 --- a/test/registered/kernels/ops/communication/test_collectives.py +++ b/test/registered/kernels/ops/communication/test_collectives.py @@ -12,10 +12,11 @@ from sglang.kernels.jit.utils import cache_once from sglang.kernels.ops.attention import attn_res from sglang.kernels.ops.communication import all_reduce_residual as all_reduce from sglang.kernels.ops.communication import ( + attn_res_collective, gemm_ag, sp_collective, ) -from sglang.kernels.ops.communication.mp import register_comm_cleanup +from sglang.srt.distributed.comm_cleanup import register_comm_cleanup from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( CustomAllReduceV2, ) @@ -73,7 +74,7 @@ def _init_comm(): raise RuntimeError("Kimi K3 collectives require multicast symmetric memory") all_reduce.register_comm(comm.obj) sp_collective.register_comm(comm.obj) - attn_res.register_comm(comm.obj) + attn_res_collective.register_comm(comm.obj) register_comm_cleanup(comm) return comm @@ -285,7 +286,7 @@ def test_attention_residual_direct_all_gather(): ) output, handle, multicast_ptr = _symmetric_tensor(tuple(full_reference.shape)) - attn_res.attn_res_fused_direct_ag( + attn_res_collective.attn_res_fused_direct_ag( comm.world_size, prefix, bank, diff --git a/test/registered/kernels/ops/communication/test_custom_all_reduce.py b/test/registered/kernels/ops/communication/test_custom_all_reduce.py index f05788fa33..55af9f1cff 100644 --- a/test/registered/kernels/ops/communication/test_custom_all_reduce.py +++ b/test/registered/kernels/ops/communication/test_custom_all_reduce.py @@ -33,7 +33,7 @@ from sglang.kernels.ops.communication.all_reduce import ( AllReduceAlgo, get_all_reduce_module, ) -from sglang.kernels.ops.communication.mp import register_comm_cleanup +from sglang.srt.distributed.comm_cleanup import register_comm_cleanup from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( CustomAllReduceV2, ) diff --git a/test/registered/kernels/ops/communication/test_muse_spark_moe_finalize_all_reduce.py b/test/registered/kernels/ops/communication/test_muse_spark_moe_finalize_all_reduce.py new file mode 100644 index 0000000000..4a279ef13e --- /dev/null +++ b/test/registered/kernels/ops/communication/test_muse_spark_moe_finalize_all_reduce.py @@ -0,0 +1,58 @@ +"""Muse Spark TRT-LLM MoE finalize (unpermute + post-expert norm + weighted sum). + +Single-GPU: the CuTeDSL kernel without the fused all-reduce against the torch +reference. +""" + +import pytest +import torch + +from sglang.kernels.ops.communication import muse_spark_moe_finalize_all_reduce_cutedsl +from sglang.kernels.ops.communication.muse_spark_moe_finalize_all_reduce import ( + FusedPostExpertNormWeightedSumTRTLLM, +) +from sglang.kernels.spec import KernelBackend +from sglang.test.ci.ci_register import register_cuda_ci + +# The SM100 pool has no single-GPU runner_config. +register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +pytestmark = pytest.mark.skipif( + not muse_spark_moe_finalize_all_reduce_cutedsl.HAS_CUTEDSL, + reason="needs the CuTeDSL runtime", +) + + +@pytest.mark.parametrize("num_tokens", [1, 20, 100, 300]) +@pytest.mark.parametrize("weight_first", [True, False]) +@pytest.mark.parametrize("weights_dtype", [torch.bfloat16, torch.float32]) +def test_finalize_matches_native(num_tokens, weight_first, weights_dtype): + torch.manual_seed(num_tokens) + topk, hidden, padded = 8, 4096, 4096 + 64 + op = FusedPostExpertNormWeightedSumTRTLLM( + topk=topk, hidden_dim=hidden, eps=1e-6, weight_first=weight_first + ) + num_slots = num_tokens * topk + gemm2_output = torch.randn(num_slots, padded, device="cuda", dtype=torch.bfloat16) + expert_weights = torch.rand(num_tokens, topk, device="cuda").to(weights_dtype) + permuted = torch.randperm(num_slots, device="cuda", dtype=torch.int32) + # -1 marks an expert slot owned by another EP rank. + permuted[torch.rand(num_slots, device="cuda") < 0.2] = -1 + norm_weight = 1.0 + 0.1 * torch.randn(hidden, device="cuda", dtype=torch.bfloat16) + + def run(backend): + out = torch.empty(num_tokens, hidden, device="cuda", dtype=torch.bfloat16) + op(gemm2_output, expert_weights, permuted, norm_weight, out, backend=backend) + return out + + # The reference adds each expert's bf16-rounded contribution into the bf16 + # output; the kernel accumulates in fp32 and rounds once. + torch.testing.assert_close( + run(KernelBackend.CUTE_DSL), run(KernelBackend.TORCH), rtol=2e-2, atol=1e-1 + ) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/communication/test_tp_qknorm.py b/test/registered/kernels/ops/communication/test_tp_qknorm.py index deed03f49a..6ef732c769 100644 --- a/test/registered/kernels/ops/communication/test_tp_qknorm.py +++ b/test/registered/kernels/ops/communication/test_tp_qknorm.py @@ -20,7 +20,7 @@ from sglang.kernels.ops.communication.all_reduce import ( get_all_reduce_module, get_fused_parallel_qknorm_module, ) -from sglang.kernels.ops.communication.mp import register_comm_cleanup +from sglang.srt.distributed.comm_cleanup import register_comm_cleanup from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( CustomAllReduceV2, ) diff --git a/test/registered/kernels/ops/diffusion/test_indexed_scale_shift_mxfp8.py b/test/registered/kernels/ops/diffusion/test_indexed_scale_shift_mxfp8.py new file mode 100644 index 0000000000..e77338d969 --- /dev/null +++ b/test/registered/kernels/ops/diffusion/test_indexed_scale_shift_mxfp8.py @@ -0,0 +1,47 @@ +"""The indexed scale-shift MXFP8 producer against ``flashinfer.mxfp8_quantize`` of +the bf16 tensor the unfused kernel stores: payload and the swizzled E8M0 scale +buffer, padding included, byte for byte.""" + +import sys + +import pytest +import torch + +from sglang.kernels.ops.diffusion import indexed_scale_shift_bf16_ +from sglang.kernels.ops.diffusion.modulate.indexed_scale_shift_mxfp8_triton import ( + indexed_scale_shift_mxfp8_, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=2, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +ROWS, HIDDEN = 333, 5376 # rows and scale columns both need padding + + +def _assert_matches_flashinfer( + got: tuple[torch.Tensor, torch.Tensor], bf16: torch.Tensor +) -> None: + import flashinfer + + q, s = flashinfer.mxfp8_quantize(bf16.contiguous(), True) + assert torch.equal(got[0].view(torch.uint8), q.view(torch.uint8)) + assert torch.equal(got[1].view(torch.uint8), s.view(torch.uint8)) + + +@pytest.mark.parametrize("keep_bf16", [True, False]) +def test_indexed_scale_shift_mxfp8_is_byte_exact(keep_bf16: bool) -> None: + g = torch.Generator(device="cuda").manual_seed(3) + x = torch.randn((ROWS, HIDDEN), device="cuda", generator=g).to(torch.bfloat16) + shift = torch.randn((3, HIDDEN), device="cuda", generator=g).to(torch.bfloat16) + scale = torch.randn((3, HIDDEN), device="cuda", generator=g).to(torch.bfloat16) + indices = torch.randint(0, 3, (ROWS,), device="cuda", generator=g) + ref = indexed_scale_shift_bf16_(x.clone(), shift, scale, indices) + kept, q, s = indexed_scale_shift_mxfp8_( + x, shift, scale, indices, keep_bf16=keep_bf16 + ) + _assert_matches_flashinfer((q, s), ref) + assert (kept is x and torch.equal(x, ref)) if keep_bf16 else kept is None + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/attention/test_mla_output_gate.py b/test/registered/kernels/ops/elementwise/test_mla_output_gate.py similarity index 95% rename from test/registered/kernels/ops/attention/test_mla_output_gate.py rename to test/registered/kernels/ops/elementwise/test_mla_output_gate.py index a463fc5367..2cb2988b98 100644 --- a/test/registered/kernels/ops/attention/test_mla_output_gate.py +++ b/test/registered/kernels/ops/elementwise/test_mla_output_gate.py @@ -2,7 +2,7 @@ import unittest import torch -from sglang.kernels.ops.attention.mla_output_gate import ( +from sglang.kernels.ops.elementwise.mla_output_gate import ( covered, kimi_k3_mla_output_gate, ) diff --git a/test/registered/kernels/ops/elementwise/test_row_scale.py b/test/registered/kernels/ops/elementwise/test_row_scale.py index 41bd1aebb8..5b0489f15b 100644 --- a/test/registered/kernels/ops/elementwise/test_row_scale.py +++ b/test/registered/kernels/ops/elementwise/test_row_scale.py @@ -5,10 +5,10 @@ including on the row-strided qkvr-slice layouts.""" import pytest import torch -from sglang.kernels.ops.attention.log_scaling_tau import ( +from sglang.kernels.ops.elementwise.row_scale import ( _apply_log_scaling_tau_kernel, + row_scale_bf16, ) -from sglang.kernels.ops.elementwise.row_scale import row_scale_bf16 from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large") @@ -45,7 +45,7 @@ def test_row_scale_bitexact(rows, inner, strided): def test_dispatch_through_apply_log_scaling_tau(): - from sglang.kernels.ops.attention.log_scaling_tau import apply_log_scaling_tau + from sglang.kernels.ops.elementwise.row_scale import apply_log_scaling_tau torch.manual_seed(0) for shape, view in (((7, 16, 16), (-1, 1, 1)), ((7, 2048), (-1, 1))): diff --git a/test/registered/kernels/ops/elementwise/test_row_sum.py b/test/registered/kernels/ops/elementwise/test_row_sum.py new file mode 100644 index 0000000000..f15ba6d875 --- /dev/null +++ b/test/registered/kernels/ops/elementwise/test_row_sum.py @@ -0,0 +1,31 @@ +"""row_sum must stay bit-identical to the tl.sum reference it replaces. + +Column counts that are multiples of 16 are excluded: Triton specializes such a +K and splits each row across lanes, which reorders its sum. The router top-k +that row_sum serves is 8. +""" + +import pytest +import torch + +from sglang.kernels.ops.elementwise.row_sum import row_sum +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kernels.triton_sum import triton_sum + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +@pytest.mark.parametrize("num_rows", [1, 7, 129, 4096]) +@pytest.mark.parametrize("num_cols", [1, 6, 8, 13, 31]) +def test_row_sum_matches_triton_sum_bitwise(num_rows, num_cols): + torch.manual_seed(num_rows * 64 + num_cols) + x = torch.rand(num_rows, num_cols, device="cuda", dtype=torch.float32) + ref = triton_sum(x, dim=-1) + assert torch.equal(row_sum(x, dim=-1), ref) + assert torch.equal(row_sum(x, dim=1, keepdim=True), ref.unsqueeze(-1)) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/moe/test_sigmoid_gate_mul.py b/test/registered/kernels/ops/elementwise/test_sigmoid_gate_mul.py similarity index 81% rename from test/registered/kernels/ops/moe/test_sigmoid_gate_mul.py rename to test/registered/kernels/ops/elementwise/test_sigmoid_gate_mul.py index 4a0cad31f5..a873393456 100644 --- a/test/registered/kernels/ops/moe/test_sigmoid_gate_mul.py +++ b/test/registered/kernels/ops/elementwise/test_sigmoid_gate_mul.py @@ -28,14 +28,14 @@ def reference_sigmoid_gate_mul(x, gate): ], ) def test_sigmoid_gate_mul_correctness(shape, dtype): - from sglang.kernels.ops.moe.triton_sigmoid_gate_mul import sigmoid_gate_mul + from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul torch.manual_seed(42) x = torch.randn(shape, dtype=dtype, device=DEVICE) gate = torch.randn(shape, dtype=dtype, device=DEVICE) ref = reference_sigmoid_gate_mul(x, gate) - out = sigmoid_gate_mul(x, gate) + out = fused_sigmoid_mul(x, gate) rtol = 1e-2 if dtype == torch.bfloat16 else 1e-3 atol = 2e-2 if dtype == torch.bfloat16 else 1e-3 @@ -44,7 +44,7 @@ def test_sigmoid_gate_mul_correctness(shape, dtype): @pytest.mark.parametrize("shape", [(4, 4096), (1, 128)]) def test_sigmoid_gate_mul_does_not_modify_inputs(shape): - from sglang.kernels.ops.moe.triton_sigmoid_gate_mul import sigmoid_gate_mul + from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul torch.manual_seed(42) x = torch.randn(shape, dtype=torch.bfloat16, device=DEVICE) @@ -52,28 +52,28 @@ def test_sigmoid_gate_mul_does_not_modify_inputs(shape): x_orig = x.clone() gate_orig = gate.clone() - sigmoid_gate_mul(x, gate) + fused_sigmoid_mul(x, gate) torch.testing.assert_close(x, x_orig, rtol=0, atol=0) torch.testing.assert_close(gate, gate_orig, rtol=0, atol=0) def test_sigmoid_gate_mul_output_dtype(): - from sglang.kernels.ops.moe.triton_sigmoid_gate_mul import sigmoid_gate_mul + from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul for dtype in [torch.bfloat16, torch.float16, torch.float32]: x = torch.randn(4, 4096, dtype=dtype, device=DEVICE) gate = torch.randn(4, 4096, dtype=dtype, device=DEVICE) - out = sigmoid_gate_mul(x, gate) + out = fused_sigmoid_mul(x, gate) assert out.dtype == dtype, f"Expected {dtype}, got {out.dtype}" def test_sigmoid_gate_mul_contiguous_output(): - from sglang.kernels.ops.moe.triton_sigmoid_gate_mul import sigmoid_gate_mul + from sglang.kernels.ops.elementwise.elementwise import fused_sigmoid_mul x = torch.randn(4, 4096, dtype=torch.bfloat16, device=DEVICE) gate = torch.randn(4, 4096, dtype=torch.bfloat16, device=DEVICE) - out = sigmoid_gate_mul(x, gate) + out = fused_sigmoid_mul(x, gate) assert out.is_contiguous() diff --git a/test/registered/kernels/ops/moe/test_sigmoid_gate_mul_broadcast.py b/test/registered/kernels/ops/elementwise/test_sigmoid_gate_mul_broadcast.py similarity index 89% rename from test/registered/kernels/ops/moe/test_sigmoid_gate_mul_broadcast.py rename to test/registered/kernels/ops/elementwise/test_sigmoid_gate_mul_broadcast.py index 8565aa721c..4d1d273b99 100644 --- a/test/registered/kernels/ops/moe/test_sigmoid_gate_mul_broadcast.py +++ b/test/registered/kernels/ops/elementwise/test_sigmoid_gate_mul_broadcast.py @@ -23,7 +23,7 @@ DEVICE = "cuda" ], ) def test_sigmoid_gate_mul_broadcast_correctness(shape, dtype): - from sglang.kernels.ops.moe.triton_sigmoid_gate_mul import ( + from sglang.kernels.ops.elementwise.elementwise import ( sigmoid_gate_mul_broadcast, ) @@ -42,7 +42,7 @@ def test_sigmoid_gate_mul_broadcast_correctness(shape, dtype): @pytest.mark.parametrize("shape", [(4, 4096), (1, 128)]) def test_sigmoid_gate_mul_broadcast_does_not_modify_inputs(shape): - from sglang.kernels.ops.moe.triton_sigmoid_gate_mul import ( + from sglang.kernels.ops.elementwise.elementwise import ( sigmoid_gate_mul_broadcast, ) @@ -60,7 +60,7 @@ def test_sigmoid_gate_mul_broadcast_does_not_modify_inputs(shape): def test_sigmoid_gate_mul_broadcast_output_dtype(): - from sglang.kernels.ops.moe.triton_sigmoid_gate_mul import ( + from sglang.kernels.ops.elementwise.elementwise import ( sigmoid_gate_mul_broadcast, ) @@ -72,7 +72,7 @@ def test_sigmoid_gate_mul_broadcast_output_dtype(): def test_sigmoid_gate_mul_broadcast_contiguous_output(): - from sglang.kernels.ops.moe.triton_sigmoid_gate_mul import ( + from sglang.kernels.ops.elementwise.elementwise import ( sigmoid_gate_mul_broadcast, ) diff --git a/test/registered/kernels/ops/speculative/test_ngram_embedding.py b/test/registered/kernels/ops/embeddings/test_ngram_embedding.py similarity index 98% rename from test/registered/kernels/ops/speculative/test_ngram_embedding.py rename to test/registered/kernels/ops/embeddings/test_ngram_embedding.py index 152c1fa1c1..2658c341c7 100644 --- a/test/registered/kernels/ops/speculative/test_ngram_embedding.py +++ b/test/registered/kernels/ops/embeddings/test_ngram_embedding.py @@ -3,7 +3,7 @@ import sys import pytest import torch -from sglang.kernels.ops.speculative.ngram_embedding import ( +from sglang.kernels.ops.embeddings.ngram_embedding import ( compute_n_gram_ids, compute_n_gram_ids_decode, update_token_table, diff --git a/test/registered/kernels/ops/quantization/test_mxfp8_amd_gfx95.py b/test/registered/kernels/ops/gemm/test_mxfp8_amd_gfx95.py similarity index 98% rename from test/registered/kernels/ops/quantization/test_mxfp8_amd_gfx95.py rename to test/registered/kernels/ops/gemm/test_mxfp8_amd_gfx95.py index f49f322789..d358a0bcf7 100644 --- a/test/registered/kernels/ops/quantization/test_mxfp8_amd_gfx95.py +++ b/test/registered/kernels/ops/gemm/test_mxfp8_amd_gfx95.py @@ -5,19 +5,19 @@ import unittest import torch -from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( - Fp8GridActivation, - Mxfp8Activation, - bf16_dequant_blockscaled_linear, - dequant_block_fp8_weight_to_bf16, - fake_quant_fp8_activation, - fp8_grid_quantize, -) -from sglang.kernels.ops.quantization.mxfp8_native_amd_gfx95 import ( +from sglang.kernels.ops.gemm.mxfp8_amd_gfx95 import bf16_dequant_blockscaled_linear +from sglang.kernels.ops.gemm.mxfp8_native_amd_gfx95 import ( mxfp8_gemv, mxfp8_native_blockscaled_linear, prepare_mxfp8_native_weight, ) +from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( + Fp8GridActivation, + Mxfp8Activation, + dequant_block_fp8_weight_to_bf16, + fake_quant_fp8_activation, + fp8_grid_quantize, +) from sglang.srt.layers.quantization import fp8_hip from sglang.srt.layers.quantization.fp8_utils import ( Mxfp8DenseGemmBackend, diff --git a/test/registered/kernels/ops/kv_canary/test_scatter_req_token_ids.py b/test/registered/kernels/ops/kv_canary/test_scatter_req_token_ids.py index 5c6fd23683..8fcc33dda1 100644 --- a/test/registered/kernels/ops/kv_canary/test_scatter_req_token_ids.py +++ b/test/registered/kernels/ops/kv_canary/test_scatter_req_token_ids.py @@ -8,9 +8,11 @@ import torch from sglang.kernels.ops.kv_canary.scatter_req_token_ids import ( _SCATTER_BATCH_BLOCK, launch_scatter_req_token_ids_kernel, - scatter_req_token_ids_torch_reference, ) from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.kernels.kv_canary.scatter_req_token_ids_reference import ( + scatter_req_token_ids_torch_reference, +) from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") diff --git a/test/registered/kernels/ops/attention/test_dsv41_request_window_copy.py b/test/registered/kernels/ops/kvcache/test_dsv41_request_window_copy.py similarity index 100% rename from test/registered/kernels/ops/attention/test_dsv41_request_window_copy.py rename to test/registered/kernels/ops/kvcache/test_dsv41_request_window_copy.py diff --git a/test/registered/kernels/ops/kvcache/test_dsv4_fp4_index_k_cache.py b/test/registered/kernels/ops/kvcache/test_dsv4_fp4_index_k_cache.py new file mode 100644 index 0000000000..d010c28cac --- /dev/null +++ b/test/registered/kernels/ops/kvcache/test_dsv4_fp4_index_k_cache.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import sys + +import pytest +import torch + +from sglang.kernels.ops.kvcache.dsv4_fp4_index_k_cache import store_fp4_index_k_cache +from sglang.srt.utils import get_device +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kernels.deepseek_v4.common import ( + quantize_fp4_indexer_ref, + store_fp4_index_cache_ref, +) + +register_cuda_ci(est_time=5, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +HEAD_DIM = 128 +FP4_DIM = HEAD_DIM // 2 +SCALE_BYTES = 4 +PAGE_SIZE = 64 + + +@pytest.mark.parametrize("num_tokens", [1, 16, 96]) +def test_fp4_index_cache_store_layout(num_tokens: int) -> None: + torch.manual_seed(num_tokens) + num_pages = max(1, (num_tokens + PAGE_SIZE - 1) // PAGE_SIZE) + x = torch.randn(num_tokens, HEAD_DIM, device=get_device(), dtype=torch.bfloat16) + loc = torch.randperm(num_pages * PAGE_SIZE, device=get_device())[:num_tokens].to( + torch.int64 + ) + cache = torch.zeros( + num_pages, + PAGE_SIZE * (FP4_DIM + SCALE_BYTES), + device=get_device(), + dtype=torch.uint8, + ) + + store_fp4_index_k_cache(x, cache, loc, page_size=PAGE_SIZE) + + ref_fp4, ref_sf = quantize_fp4_indexer_ref(x) + expected = store_fp4_index_cache_ref(ref_fp4, ref_sf, loc, num_pages) + torch.testing.assert_close(cache, expected) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/attention/test_fused_store_index_cache.py b/test/registered/kernels/ops/kvcache/test_fused_store_index_cache.py similarity index 99% rename from test/registered/kernels/ops/attention/test_fused_store_index_cache.py rename to test/registered/kernels/ops/kvcache/test_fused_store_index_cache.py index a842a7d9b0..a07e0c379d 100644 --- a/test/registered/kernels/ops/attention/test_fused_store_index_cache.py +++ b/test/registered/kernels/ops/kvcache/test_fused_store_index_cache.py @@ -25,7 +25,7 @@ import torch from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci try: - from sglang.kernels.ops.attention.fused_store_index_cache import ( + from sglang.kernels.ops.kvcache.dsa_fused_store_index_cache import ( can_use_dsa_fused_store, fused_store_index_k_cache, ) @@ -187,7 +187,7 @@ def _reference_quantize_and_store( def _import_act_quant(): try: - from sglang.kernels.ops.attention.dsa.triton_kernel import act_quant + from sglang.kernels.ops.quantization.act_quant_triton import act_quant return act_quant except Exception: diff --git a/test/registered/kernels/ops/layernorm/test_hc_boundary_hip.py b/test/registered/kernels/ops/layernorm/test_hc_boundary_hip.py index ba62c2b8f6..8dfd127656 100644 --- a/test/registered/kernels/ops/layernorm/test_hc_boundary_hip.py +++ b/test/registered/kernels/ops/layernorm/test_hc_boundary_hip.py @@ -4,7 +4,7 @@ import unittest import torch -from sglang.kernels.ops.layernorm.mhc import hc_combine, hc_mix_stats_sinkhorn +from sglang.kernels.ops.layernorm.mhc import hc_mix_stats_sinkhorn, mhc_combine from sglang.srt.utils import is_hip from sglang.test.ci.ci_register import register_amd_ci from sglang.test.kernels.mhc import hc_split_sinkhorn_torch @@ -111,8 +111,8 @@ class TestHcBoundaryFused(CustomTestCase): bool(((ulp <= 1) | near_zero).all()), f"m={m}: {int(((ulp > 1) & ~near_zero).sum())} elements beyond one bf16 ulp", ) - # The collapse of the stored residual, as hc_combine computes it. - y_ref = hc_combine(res_out.flatten(1), pre_prev, HC, torch.bfloat16) + # The collapse of the stored residual, as mhc_combine computes it. + y_ref = mhc_combine(res_out.flatten(1), pre_prev, HC, torch.bfloat16) self.assertTrue(torch.equal(y, y_ref)) # Coefficients of the stored residual against fp64. ref = _ref_coefficients(res_out, self.hc_fn, self.hc_scale, self.hc_base) @@ -124,7 +124,7 @@ class TestHcBoundaryFused(CustomTestCase): _, residual, _, _, pre_prev = self._inputs(33, 5) res_out, y, pre, post, comb = self._run(None, residual, None, None, pre_prev) self.assertIsNone(res_out) - y_ref = hc_combine(residual.flatten(1), pre_prev, HC, torch.bfloat16) + y_ref = mhc_combine(residual.flatten(1), pre_prev, HC, torch.bfloat16) self.assertTrue(torch.equal(y, y_ref)) ref = _ref_coefficients(residual, self.hc_fn, self.hc_scale, self.hc_base) for got, want in zip((pre, post, comb), ref): @@ -164,7 +164,7 @@ class TestRmsnormWithSinkhorn(CustomTestCase): hc_boundary_fused_deferred, rmsnorm_with_sinkhorn, ) - from sglang.kernels.ops.quantization.rmsnorm_fake_quant_amd_gfx95 import ( + from sglang.kernels.ops.layernorm.rmsnorm_fake_quant_amd_gfx95 import ( rmsnorm_fake_quant_fp8, ) diff --git a/test/registered/kernels/ops/layernorm/test_hc_combine.py b/test/registered/kernels/ops/layernorm/test_mhc_combine.py similarity index 82% rename from test/registered/kernels/ops/layernorm/test_hc_combine.py rename to test/registered/kernels/ops/layernorm/test_mhc_combine.py index 08173765a9..a8f620d00d 100644 --- a/test/registered/kernels/ops/layernorm/test_hc_combine.py +++ b/test/registered/kernels/ops/layernorm/test_mhc_combine.py @@ -3,14 +3,14 @@ import sys import pytest import torch -from sglang.kernels.ops.layernorm.mhc import hc_combine +from sglang.kernels.ops.layernorm.mhc import mhc_combine from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") pytestmark = pytest.mark.skipif( not torch.cuda.is_available() or torch.version.cuda is None, - reason="hc_combine requires CUDA", + reason="mhc_combine requires CUDA", ) @@ -24,17 +24,17 @@ def _reference( @pytest.mark.parametrize("m", [1, 7, 64, 1024, 2048]) @pytest.mark.parametrize("hc,h", [(4, 128), (4, 512), (8, 256)]) @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) -def test_hc_combine_matches_reference(m: int, hc: int, h: int, dtype: torch.dtype): +def test_mhc_combine_matches_reference(m: int, hc: int, h: int, dtype: torch.dtype): torch.manual_seed(0) x_flat = torch.randn(m, hc * h, device="cuda", dtype=dtype) pre = torch.randn(m, hc, device="cuda", dtype=dtype).contiguous() - got = hc_combine(x_flat, pre, hc, dtype) + got = mhc_combine(x_flat, pre, hc, dtype) ref = _reference(x_flat, pre, hc, dtype) assert got.shape == (m, h) assert got.dtype == dtype - # hc_combine accumulates in fp32 while the reference accumulates in `dtype`, + # mhc_combine accumulates in fp32 while the reference accumulates in `dtype`, # so compare against the fp32 result the kernel is approximating. ref_fp32 = _reference(x_flat.float(), pre.float(), hc, torch.float32) scale = ref_fp32.abs().max().clamp(min=1e-6) @@ -42,14 +42,14 @@ def test_hc_combine_matches_reference(m: int, hc: int, h: int, dtype: torch.dtyp assert (ref.float() - got.float()).abs().max() / scale < 5e-2 -def test_hc_combine_strided_pre(): +def test_mhc_combine_strided_pre(): torch.manual_seed(0) m, hc, h = 32, 4, 128 x_flat = torch.randn(m, hc * h, device="cuda", dtype=torch.bfloat16) # A non-contiguous view of pre, to check the kernel honours pre's strides. pre = torch.randn(m, hc, 2, device="cuda", dtype=torch.bfloat16)[..., 0] assert not pre.is_contiguous() - got = hc_combine(x_flat, pre, hc, torch.bfloat16) + got = mhc_combine(x_flat, pre, hc, torch.bfloat16) ref = _reference(x_flat, pre, hc, torch.bfloat16) scale = ref.float().abs().max().clamp(min=1e-6) assert (got.float() - ref.float()).abs().max() / scale < 5e-2 diff --git a/test/registered/kernels/ops/layernorm/test_mhc_kernels.py b/test/registered/kernels/ops/layernorm/test_mhc_kernels.py index d54456a4c5..37ad8717a0 100644 --- a/test/registered/kernels/ops/layernorm/test_mhc_kernels.py +++ b/test/registered/kernels/ops/layernorm/test_mhc_kernels.py @@ -298,7 +298,7 @@ def test_hopper_combine_norm(num_tokens): norm, ) actual = model_mhc.combine(hc, model_mhc.HcState(x, pre)) - expected = norm(mhc.hc_combine(x.flatten(1), pre, 4, x.dtype)) + expected = norm(mhc.mhc_combine(x.flatten(1), pre, 4, x.dtype)) torch.testing.assert_close(actual, expected, rtol=1 / 128, atol=1e-5) diff --git a/test/registered/kernels/ops/layernorm/test_muse_spark_norm_ops.py b/test/registered/kernels/ops/layernorm/test_muse_spark_norm_ops.py new file mode 100644 index 0000000000..a7c99a4c25 --- /dev/null +++ b/test/registered/kernels/ops/layernorm/test_muse_spark_norm_ops.py @@ -0,0 +1,118 @@ +"""Muse Spark fused norm ops: Triton/JIT backends against their torch reference.""" + +import pytest +import torch + +from sglang.kernels.ops.layernorm.muse_spark_norm_gate import FusedRMSNormGate +from sglang.kernels.ops.layernorm.muse_spark_norm_residual import ( + FusedPostNormResidualPreNorm, +) +from sglang.kernels.spec import KernelBackend +from sglang.srt.layers.muse_spark.muse_spark_layers import ( + MuseSpark1xNormPreservingResidualConnection, + MuseSpark1xRMSNorm, + MuseSpark1xRMSNormWithInputScale, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +_BACKENDS = [KernelBackend.TRITON, KernelBackend.JIT] + + +def _bf16_module(module): + with torch.no_grad(): + for p in module.parameters(): + p.copy_(0.1 * torch.randn_like(p)) + return module.to(device="cuda", dtype=torch.bfloat16) + + +@pytest.mark.parametrize("backend", _BACKENDS) +@pytest.mark.parametrize("num_tokens", [1, 9, 300]) +@pytest.mark.parametrize( + "has_norm, has_weight", [(True, False), (True, True), (False, False)] +) +@pytest.mark.parametrize("gate_groups", [None, 1, 4]) +def test_rms_norm_gate_matches_native( + backend, num_tokens, has_norm, has_weight, gate_groups +): + torch.manual_seed(num_tokens) + head_dim, num_heads = 64, 16 + o_norm = ( + _bf16_module( + MuseSpark1xRMSNorm( + head_dim, eps=1e-6, has_weight=has_weight, zero_centered_gamma=True + ) + ) + if has_norm + else None + ) + op = FusedRMSNormGate(o_norm, use_o_gate=gate_groups is not None, head_dim=head_dim) + width = num_heads * head_dim + x = torch.randn(num_tokens, width, device="cuda", dtype=torch.bfloat16) + gate = None + if gate_groups is not None: + # A strided grouped view, as the fused QKV-O projection hands it over. + packed = torch.randn( + num_tokens, + gate_groups, + width // gate_groups + 32, + device="cuda", + dtype=torch.bfloat16, + ) + gate = packed[..., : width // gate_groups] + if gate_groups == 1: + gate = gate.reshape(num_tokens, width) + ref = op.forward_native(x, gate) + out = op(x, gate, backend=backend) + torch.testing.assert_close(out, ref, rtol=2e-2, atol=2e-2) + + +def _boundary(post_input_scale, hidden): + post_cls = ( + MuseSpark1xRMSNormWithInputScale if post_input_scale else MuseSpark1xRMSNorm + ) + post_norm = _bf16_module( + post_cls( + hidden, eps=1e-5, has_weight=post_input_scale, zero_centered_gamma=True + ) + ) + pre_norm = _bf16_module( + MuseSpark1xRMSNorm(hidden, eps=1e-6, has_weight=True, zero_centered_gamma=True) + ) + residual = _bf16_module(MuseSpark1xNormPreservingResidualConnection(hidden)) + return FusedPostNormResidualPreNorm( + post_norm, + residual, + pre_norm, + post_norm_is_input_scale=post_input_scale, + use_residual_gate=True, + ) + + +@pytest.mark.parametrize("backend", _BACKENDS) +@pytest.mark.parametrize("num_tokens", [1, 7, 300]) +@pytest.mark.parametrize("post_input_scale", [False, True]) +@pytest.mark.parametrize("router_dtype", [None, torch.float32]) +def test_norm_residual_matches_native( + backend, num_tokens, post_input_scale, router_dtype +): + torch.manual_seed(num_tokens) + hidden = 4096 + op = _boundary(post_input_scale, hidden) + stream = torch.randn(num_tokens, hidden, device="cuda", dtype=torch.float32) + branch = torch.randn(num_tokens, hidden, device="cuda", dtype=torch.bfloat16) + ref = op.forward_native(stream, branch, router_dtype=router_dtype) + out = op(stream, branch, router_dtype=router_dtype, backend=backend) + torch.testing.assert_close(out[0], ref[0], rtol=2e-2, atol=2e-2) + torch.testing.assert_close(out[1], ref[1], rtol=1e-2, atol=1e-2) + if router_dtype is None: + assert out[2] is None and ref[2] is None + else: + torch.testing.assert_close(out[2], ref[2], rtol=2e-2, atol=2e-2) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/layernorm/test_muse_spark_rmsnorm_nvfp4_quant.py b/test/registered/kernels/ops/layernorm/test_muse_spark_rmsnorm_nvfp4_quant.py new file mode 100644 index 0000000000..b4082a01a3 --- /dev/null +++ b/test/registered/kernels/ops/layernorm/test_muse_spark_rmsnorm_nvfp4_quant.py @@ -0,0 +1,71 @@ +"""Muse Spark RMSNorm + per-token NVFP4 quant: JIT kernel against the torch reference.""" + +import pytest +import torch + +from sglang.kernels.ops.layernorm.muse_spark_rmsnorm_nvfp4_quant import ( + _FUSED_NORM_FP4_QUANT, +) +from sglang.kernels.spec import KernelBackend +from sglang.test.ci.ci_register import register_cuda_ci + +# The SM100 pool has no single-GPU runner_config. +register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] < 10, + reason="cvt.e2m1x2 needs SM100+", +) + + +def _run(backend, x, weight, quantize, want_normed): + return _FUSED_NORM_FP4_QUANT( + x, + weight, + 1e-6, + 1.0 / 2688.0, + quantize=quantize, + want_normed=want_normed, + backend=backend, + ) + + +@pytest.mark.parametrize("num_tokens", [1, 33, 512]) +@pytest.mark.parametrize("hidden_size", [4096, 8192]) +@pytest.mark.parametrize("has_weight", [True, False]) +@pytest.mark.parametrize( + "quantize, want_normed", [(True, False), (True, True), (False, True)] +) +def test_rmsnorm_nvfp4_quant_matches_native( + num_tokens, hidden_size, has_weight, quantize, want_normed +): + torch.manual_seed(num_tokens + hidden_size) + x = torch.randn(num_tokens, hidden_size, device="cuda", dtype=torch.bfloat16) + x[0, :16] = 0 # an all-zero scale block + weight = ( + 1.0 + 0.1 * torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) + if has_weight + else None + ) + out = _run(KernelBackend.JIT, x, weight, quantize, want_normed) + ref = _run(KernelBackend.TORCH, x, weight, quantize, want_normed) + + fp4, sf, scale, normed = out + ref_fp4, ref_sf, ref_scale, ref_normed = ref + if want_normed: + torch.testing.assert_close(normed, ref_normed, rtol=1e-2, atol=1e-2) + else: + assert normed is None and ref_normed is None + if not quantize: + assert fp4 is None and ref_fp4 is None + return + torch.testing.assert_close(scale, ref_scale, rtol=1e-2, atol=0) + # IEEE vs rcp.approx reciprocals: only values on a rounding boundary differ. + assert (sf != ref_sf).float().mean().item() < 1e-3 + assert (fp4 != ref_fp4).float().mean().item() < 1e-3 + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/layernorm/test_mxfp8_epilogue.py b/test/registered/kernels/ops/layernorm/test_mxfp8_epilogue.py index 0021dc0a25..8a738d961c 100644 --- a/test/registered/kernels/ops/layernorm/test_mxfp8_epilogue.py +++ b/test/registered/kernels/ops/layernorm/test_mxfp8_epilogue.py @@ -6,7 +6,7 @@ import flashinfer import pytest import torch -from sglang.kernels.ops.layernorm.mxfp8_epilogue import rmsnorm_mxfp8 +from sglang.kernels.ops.layernorm.rmsnorm_mxfp8 import rmsnorm_mxfp8 from sglang.srt.runtime_context import get_platform from sglang.test.ci.ci_register import register_cuda_ci diff --git a/test/registered/kernels/ops/moe/test_minimax_m3_mxfp8.py b/test/registered/kernels/ops/moe/test_minimax_m3_mxfp8.py index 1b3a4f5671..19349b33bd 100644 --- a/test/registered/kernels/ops/moe/test_minimax_m3_mxfp8.py +++ b/test/registered/kernels/ops/moe/test_minimax_m3_mxfp8.py @@ -27,8 +27,10 @@ if not is_hip(): if not torch.cuda.is_available(): pytest.skip("Requires a GPU.", allow_module_level=True) -from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( # noqa: E402 +from sglang.kernels.ops.gemm.mxfp8_amd_gfx95 import ( # noqa: E402 _mxfp8_dot_scaled_linear, +) +from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( # noqa: E402 _mxfp8_e4m3_quantize_torch, _mxfp8_e4m3_quantize_triton, dequant_mxfp8_to_bf16, @@ -110,7 +112,7 @@ def test_minimax_swiglu_mxfp8_quant_matches_unfused_fp32(m, inter): # the reference is the unfused fp32 swiglu followed by MXFP8 quant. Not # bit-identical because the reference quant runs in torch vs the fused triton # path, but numerically equivalent (tight relerr, scales agree within 1 ulp). - from sglang.kernels.ops.moe.minimax_m3_swiglu import ( + from sglang.kernels.ops.activation.minimax_m3_swiglu import ( swiglu_oai_mxfp8_quant, swiglu_oai_split, ) diff --git a/test/registered/kernels/ops/moe/test_minimax_quant_scatter.py b/test/registered/kernels/ops/moe/test_minimax_quant_scatter.py index 55515f5c76..2e2a7fcc61 100644 --- a/test/registered/kernels/ops/moe/test_minimax_quant_scatter.py +++ b/test/registered/kernels/ops/moe/test_minimax_quant_scatter.py @@ -11,12 +11,12 @@ from sglang.kernels.ops.moe.ep_moe_kernels import ( fill_gateup_input_triton_kernel, moe_ep_deepgemm_preprocess, ) +from sglang.kernels.ops.moe.minimax_quant_ue8m0 import ( + per_token_quant_fp8_ue8m0_scatter, +) from sglang.kernels.ops.quantization.fp8_kernel import ( sglang_per_token_group_quant_fp8, ) -from sglang.kernels.ops.quantization.minimax_quant_ue8m0 import ( - per_token_quant_fp8_ue8m0_scatter, -) from sglang.srt.environ import envs from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig from sglang.srt.layers.moe.moe_runner.deep_gemm import ( diff --git a/test/registered/kernels/ops/moe/test_muse_spark_routing.py b/test/registered/kernels/ops/moe/test_muse_spark_routing.py new file mode 100644 index 0000000000..29bb7f5a89 --- /dev/null +++ b/test/registered/kernels/ops/moe/test_muse_spark_routing.py @@ -0,0 +1,112 @@ +"""Muse Spark MoE routing: fused Triton/JIT backends against the torch reference.""" + +import pytest +import torch + +from sglang.kernels.ops.moe.muse_spark_post_expert_finalize_triton import ( + fused_post_expert_norm_weighted_sum, +) +from sglang.kernels.ops.moe.muse_spark_routing import FusedMuseSparkRouting +from sglang.kernels.spec import KernelBackend +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") + + +def _routing_op(num_experts, eps, multiplier, ep_map, num_valid): + op = FusedMuseSparkRouting( + expert_bias=0.1 * torch.randn(num_experts, device="cuda"), + route_scale=2.5, + router_output_multiplier=multiplier, + router_score_division_eps=eps, + ) + if ep_map: + op.set_local_expert_mapping( + torch.randperm(num_experts, device="cuda", dtype=torch.int32) + ) + if num_valid is not None: + op.set_num_valid_tokens( + torch.tensor([num_valid], device="cuda", dtype=torch.int32) + ) + return op + + +@pytest.mark.parametrize("backend", [KernelBackend.TRITON, KernelBackend.JIT]) +@pytest.mark.parametrize("num_tokens", [1, 33, 600]) +@pytest.mark.parametrize("num_experts, topk", [(256, 8), (128, 4)]) +@pytest.mark.parametrize( + "eps, multiplier, ep_map, padded", + [(None, None, False, False), (1e-20, 0.5, True, True)], +) +def test_routing_matches_native( + backend, num_tokens, num_experts, topk, eps, multiplier, ep_map, padded +): + torch.manual_seed(num_tokens * num_experts + topk) + num_valid = max(num_tokens - 3, 0) if padded else None + op = _routing_op(num_experts, eps, multiplier, ep_map, num_valid) + logits = torch.randn(num_tokens, num_experts, device="cuda") + hidden = torch.empty(num_tokens, 1, device="cuda") + + ref_w, ref_ids, ref_packed = op.forward_native( + hidden, logits, topk, True, want_packed=True + ) + w, ids, packed = op(hidden, logits, topk, True, want_packed=True, backend=backend) + + assert torch.equal(ids, ref_ids) + assert torch.equal(packed >> 16, ref_packed >> 16) + valid = slice(None) if num_valid is None else slice(0, num_valid) + torch.testing.assert_close(w[valid], ref_w[valid], rtol=1e-5, atol=1e-6) + + +@pytest.mark.parametrize("num_tokens", [1, 33, 4096]) +def test_jit_routing_is_bitwise_triton(num_tokens): + torch.manual_seed(num_tokens) + op = _routing_op(256, 1e-20, 0.5, False, None) + logits = torch.randn(num_tokens, 256, device="cuda") + hidden = torch.empty(num_tokens, 1, device="cuda") + triton_out = op( + hidden, logits, 8, True, want_packed=True, backend=KernelBackend.TRITON + ) + jit_out = op(hidden, logits, 8, True, want_packed=True, backend=KernelBackend.JIT) + for a, b in zip(jit_out, triton_out): + assert torch.equal(a, b) + + +def _post_expert_reference(x, scores, weight, eps, weight_first, ids): + f = x.float() + if ids is not None: + f = torch.where((ids >= 0).unsqueeze(-1), f, torch.zeros_like(f)) + w = weight.float() + if weight_first: + f = f * w + y = f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps) + else: + y = f * torch.rsqrt(f.pow(2).mean(-1, keepdim=True) + eps) * w + return (y * scores.float().unsqueeze(-1)).sum(dim=1) + + +@pytest.mark.parametrize("num_tokens", [5, 64]) +@pytest.mark.parametrize("weight_first", [True, False]) +@pytest.mark.parametrize("with_ids", [False, True]) +def test_post_expert_finalize_triton(num_tokens, weight_first, with_ids): + torch.manual_seed(num_tokens) + topk, hidden = 8, 1024 + x = torch.randn(num_tokens, topk, hidden, device="cuda", dtype=torch.bfloat16) + scores = torch.rand(num_tokens, topk, device="cuda") + weight = 1.0 + 0.1 * torch.randn(hidden, device="cuda", dtype=torch.bfloat16) + ids = ( + torch.randint(-1, 4, (num_tokens, topk), device="cuda", dtype=torch.int32) + if with_ids + else None + ) + out = fused_post_expert_norm_weighted_sum( + x, scores, weight, eps=1e-6, weight_first=weight_first, slot_expert_ids=ids + ) + ref = _post_expert_reference(x, scores, weight, 1e-6, weight_first, ids) + torch.testing.assert_close(out.float(), ref, rtol=2e-2, atol=2e-2) + + +if __name__ == "__main__": + import sys + + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/quantization/test_dsv4_fp8_cast.py b/test/registered/kernels/ops/quantization/test_dsv4_fp8_cast.py index abca9928ca..60753478d2 100644 --- a/test/registered/kernels/ops/quantization/test_dsv4_fp8_cast.py +++ b/test/registered/kernels/ops/quantization/test_dsv4_fp8_cast.py @@ -18,10 +18,10 @@ import unittest import torch -from sglang.kernels.ops.quantization.fp8_cvt import cvt_fp8_e4m3 from sglang.kernels.ops.quantization.fp8_kernel import fp8_dtype, fp8_max from sglang.srt.utils import is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.kernels.deepseek_v4.fp8_cvt import cvt_fp8_e4m3 from sglang.test.test_utils import CustomTestCase # the default amd runner is mi300, where pack_fp8 still takes the software path this diff --git a/test/registered/kernels/ops/quantization/test_fp4_indexer_quant.py b/test/registered/kernels/ops/quantization/test_fp4_indexer_quant.py new file mode 100644 index 0000000000..98f32f12b2 --- /dev/null +++ b/test/registered/kernels/ops/quantization/test_fp4_indexer_quant.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import sys + +import pytest +import torch + +from sglang.kernels.ops.quantization.fp4_indexer_quant import ( + quantize_fp4_indexer_tensor, +) +from sglang.srt.utils import get_device +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kernels.deepseek_v4.common import quantize_fp4_indexer_ref + +register_cuda_ci(est_time=5, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +HEAD_DIM = 128 +FP4_DIM = HEAD_DIM // 2 +GROUP_SIZE = 32 + + +@pytest.mark.parametrize("num_tokens", [1, 7, 96]) +def test_quantize_fp4_indexer_tensor(num_tokens: int) -> None: + torch.manual_seed(num_tokens) + x = torch.randn(num_tokens, HEAD_DIM, device=get_device(), dtype=torch.bfloat16) + x[0, :8] = torch.tensor( + [-8.0, -6.0, -3.0, -1.5, 0.0, 0.5, 2.0, 8.0], + device=get_device(), + dtype=torch.bfloat16, + ) + + x_fp4, x_sf = quantize_fp4_indexer_tensor(x) + ref_fp4, ref_sf = quantize_fp4_indexer_ref(x) + + torch.testing.assert_close(x_fp4.view(torch.uint8), ref_fp4) + torch.testing.assert_close(x_sf, ref_sf) + + +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10, + reason="Vectorized prefill dispatch targets Blackwell", +) +@pytest.mark.parametrize("rows", [4096, 4097, 16384, 524288]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) +@pytest.mark.parametrize("rne", [False, True]) +@pytest.mark.parametrize("strided", [False, True]) +def test_prefill_quantization_matches_single_row_and_replay(rows, dtype, rne, strided): + from sglang.kernels.ops.quantization.fp4_indexer_quant import ( + _quantize_fp4_indexer_kernel, + ) + + x = torch.randn(rows, 256 if strided else 128, device="cuda", dtype=dtype) + if strided: + x = x[:, ::2] + + def reference(): + q = torch.empty(rows, 64, device="cuda", dtype=torch.int8) + sf = torch.empty(rows, device="cuda", dtype=torch.int32) + _quantize_fp4_indexer_kernel[(rows,)]( + x.contiguous(), + q, + sf, + BLOCK_N=128, + GROUP_N=32, + RNE=rne, + ) + return q, sf + + for _ in range(3): + quantize_fp4_indexer_tensor(x, rne) + reference() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual = quantize_fp4_indexer_tensor(x, rne) + boundaries = torch.tensor( + [0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0, 6.0], device="cuda", dtype=dtype + ) + for scale in (0.0, 1e-6, 1.0, 1e3): + x.normal_().mul_(scale) + x[:2] = boundaries.repeat(16) + x[1].neg_() + graph.replay() + for a, b in zip(actual, reference()): + assert torch.equal(a, b) + + +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10, + reason="Vectorized prefill dispatch targets Blackwell", +) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32]) +@pytest.mark.parametrize("rne", [False, True]) +def test_prefill_quantization_nonfinite_group_replay(dtype, rne): + from sglang.kernels.ops.quantization.fp4_indexer_quant import ( + _quantize_fp4_indexer_kernel, + ) + + rows = 4097 # Also exercise the final partial CTA. + x = torch.randn(rows, HEAD_DIM, device="cuda", dtype=dtype) + expected = ( + torch.empty(rows, FP4_DIM, device="cuda", dtype=torch.int8), + torch.empty(rows, device="cuda", dtype=torch.int32), + ) + for _ in range(3): + quantize_fp4_indexer_tensor(x, rne) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual = quantize_fp4_indexer_tensor(x, rne) + for value in (float("inf"), float("-inf"), float("nan"), 0.0): + x.normal_() + x[:, 0] = value + x[:, 32:64] = value + x[-1, 96:] = value + graph.replay() + _quantize_fp4_indexer_kernel[(rows,)]( + x, *expected, BLOCK_N=HEAD_DIM, GROUP_N=GROUP_SIZE, RNE=rne + ) + for a, b in zip(actual, expected): + assert torch.equal(a, b) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/quantization/test_fp8_grid_producers_gfx95.py b/test/registered/kernels/ops/quantization/test_fp8_grid_producers_gfx95.py index 7c0e606a42..cca4b0c233 100644 --- a/test/registered/kernels/ops/quantization/test_fp8_grid_producers_gfx95.py +++ b/test/registered/kernels/ops/quantization/test_fp8_grid_producers_gfx95.py @@ -12,14 +12,14 @@ from sglang.kernels.ops.activation.silu_and_mul_clamp_hip import ( from sglang.kernels.ops.gemm.gfx95_batched_gemm_bf16_fp8_grid import ( batched_gemm_bf16_fp8_grid, ) +from sglang.kernels.ops.layernorm.rmsnorm_fake_quant_amd_gfx95 import ( + rmsnorm_fake_quant_fp8, +) from sglang.kernels.ops.quantization.mxfp8_amd_gfx95 import ( _mxfp8_e4m3_quantize_torch, dequant_mxfp8_to_bf16, fake_quant_fp8_activation, ) -from sglang.kernels.ops.quantization.rmsnorm_fake_quant_amd_gfx95 import ( - rmsnorm_fake_quant_fp8, -) from sglang.srt.utils import is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/registered/kernels/ops/attention/test_fp8_wo_a.py b/test/registered/kernels/ops/quantization/test_fp8_wo_a.py similarity index 97% rename from test/registered/kernels/ops/attention/test_fp8_wo_a.py rename to test/registered/kernels/ops/quantization/test_fp8_wo_a.py index 456f888773..7222d2fd4f 100644 --- a/test/registered/kernels/ops/attention/test_fp8_wo_a.py +++ b/test/registered/kernels/ops/quantization/test_fp8_wo_a.py @@ -9,8 +9,10 @@ import unittest import torch -import sglang.kernels.ops.attention.dsv4.fp8_wo_a as fp8_wo_a_module -from sglang.kernels.ops.attention.dsv4 import sglang_per_token_group_quant_fp8_dsv4_wo_a +import sglang.kernels.ops.quantization.fp8_group_major_quant_ue8m0 as fp8_wo_a_module +from sglang.kernels.ops.quantization.fp8_group_major_quant_ue8m0 import ( + sglang_per_token_group_quant_fp8_dsv4_wo_a, +) from sglang.kernels.ops.quantization.fp8_kernel import ( fp8_dtype, sglang_per_token_group_quant_fp8, diff --git a/test/registered/kernels/ops/quantization/test_mxfp8_swizzled.py b/test/registered/kernels/ops/quantization/test_mxfp8_swizzled.py index 8fb94747d5..30ca2827d9 100644 --- a/test/registered/kernels/ops/quantization/test_mxfp8_swizzled.py +++ b/test/registered/kernels/ops/quantization/test_mxfp8_swizzled.py @@ -1,30 +1,18 @@ -"""MXFP8 producers against ``flashinfer.mxfp8_quantize`` of the bf16 tensor the -unfused kernel stores: payload and the swizzled E8M0 scale buffer, padding -included, byte for byte.""" +"""The swizzled MXFP8 quantizer against ``flashinfer.mxfp8_quantize``: payload and +the swizzled E8M0 scale buffer, padding included, byte for byte.""" import sys import pytest import torch -from sglang.kernels.ops.diffusion import indexed_scale_shift_bf16_ -from sglang.kernels.ops.quantization import mxfp8_swizzled_triton -from sglang.kernels.ops.quantization.fp8_kernel import sglang_per_token_group_quant_fp8 from sglang.kernels.ops.quantization.mxfp8_swizzled_triton import ( can_use_mxfp8_swizzled, - can_use_silu_mul_mxfp8, - indexed_scale_shift_mxfp8_, mxfp8_quantize_swizzled, - silu_mul_mxfp8, - swiglu_oai_mxfp8, - swiglu_oai_mxfp8_deepgemm, -) -from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import ( - swiglu_no_interleaved_with_alpha_and_limit, ) from sglang.test.ci.ci_register import register_cuda_ci -register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="4-gpu-b200") +register_cuda_ci(est_time=3, stage="base-b-kernel-unit", runner_config="4-gpu-b200") ROWS, HIDDEN = 333, 5376 # rows and scale columns both need padding @@ -47,103 +35,10 @@ def test_quantize_swizzled_is_byte_exact() -> None: _assert_matches_flashinfer(mxfp8_quantize_swizzled(x), x) -def test_silu_mul_mxfp8_is_byte_exact(monkeypatch: pytest.MonkeyPatch) -> None: - g = torch.Generator(device="cuda").manual_seed(2) - x = (torch.randn((ROWS, 2 * HIDDEN), device="cuda", generator=g) * 2).to( - torch.bfloat16 - ) - assert can_use_silu_mul_mxfp8(x) - ref = torch.nn.functional.silu(x[:, :HIDDEN]) * x[:, HIDDEN:] - _poison_scale_alloc(monkeypatch) - _assert_matches_flashinfer(silu_mul_mxfp8(x), ref) - - -def _poison_scale_alloc(monkeypatch: pytest.MonkeyPatch) -> None: - alloc = mxfp8_swizzled_triton._alloc - - def poisoned(*args, **kwargs): - q, s = alloc(*args, **kwargs) - return q, s.fill_(0xFF) - - monkeypatch.setattr(mxfp8_swizzled_triton, "_alloc", poisoned) - - -@pytest.mark.parametrize("rows", [1, 7, 128, 333]) -@pytest.mark.parametrize("n", [768, 1056]) -@pytest.mark.parametrize("scale", [1e-3, 1.0, 30.0]) -def test_swiglu_oai_mxfp8_matches_eager_chain( - rows: int, n: int, scale: float, monkeypatch: pytest.MonkeyPatch -) -> None: - g = torch.Generator(device="cuda").manual_seed(rows * 31 + n) - x = (torch.randn((rows, 2 * n + 64), device="cuda", generator=g) * scale).to( - torch.bfloat16 - )[:, : 2 * n] - ref = swiglu_no_interleaved_with_alpha_and_limit(x, 1.702, 7.0) - _poison_scale_alloc(monkeypatch) - _assert_matches_flashinfer(swiglu_oai_mxfp8(x, 1.702, 7.0), ref) - - -def test_swiglu_oai_mxfp8_cuda_graph_replay() -> None: - rows, n = 40, 768 - x = torch.randn((rows, 2 * n), device="cuda").to(torch.bfloat16) - swiglu_oai_mxfp8(x, 1.702, 7.0) - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - got = swiglu_oai_mxfp8(x, 1.702, 7.0) - for scale in [0.0, 1e-4, 1.0, 100.0]: - x.copy_(torch.randn_like(x, dtype=torch.float32) * scale) - snapshot = x.clone() - graph.replay() - assert torch.equal(x, snapshot) - ref = swiglu_no_interleaved_with_alpha_and_limit(x, 1.702, 7.0) - _assert_matches_flashinfer(got, ref) - - -@pytest.mark.parametrize("rows", [1, 7, 128, 333, 2050]) -@pytest.mark.parametrize("n", [768, 1536]) -@pytest.mark.parametrize("scale", [0.0, 1e-3, 1.0, 30.0]) -def test_swiglu_oai_mxfp8_deepgemm_matches_eager_chain( - rows: int, n: int, scale: float -) -> None: - g = torch.Generator(device="cuda").manual_seed(rows * 31 + n) - x = (torch.randn((rows, 2 * n + 64), device="cuda", generator=g) * scale).to( - torch.bfloat16 - )[:, : 2 * n] - ref_q, ref_s = sglang_per_token_group_quant_fp8( - swiglu_no_interleaved_with_alpha_and_limit(x, 1.702, 7.0).contiguous(), - 32, - column_major_scales=True, - scale_tma_aligned=True, - scale_ue8m0=True, - ) - q, s = swiglu_oai_mxfp8_deepgemm(x, 1.702, 7.0) - assert s.shape == ref_s.shape and s.stride() == ref_s.stride() - assert torch.equal(q.view(torch.uint8), ref_q.view(torch.uint8)) - assert torch.equal(s, ref_s) - - -@pytest.mark.parametrize("keep_bf16", [True, False]) -def test_indexed_scale_shift_mxfp8_is_byte_exact(keep_bf16: bool) -> None: - g = torch.Generator(device="cuda").manual_seed(3) - x = torch.randn((ROWS, HIDDEN), device="cuda", generator=g).to(torch.bfloat16) - shift = torch.randn((3, HIDDEN), device="cuda", generator=g).to(torch.bfloat16) - scale = torch.randn((3, HIDDEN), device="cuda", generator=g).to(torch.bfloat16) - indices = torch.randint(0, 3, (ROWS,), device="cuda", generator=g) - ref = indexed_scale_shift_bf16_(x.clone(), shift, scale, indices) - kept, q, s = indexed_scale_shift_mxfp8_( - x, shift, scale, indices, keep_bf16=keep_bf16 - ) - _assert_matches_flashinfer((q, s), ref) - assert (kept is x and torch.equal(x, ref)) if keep_bf16 else kept is None - - def test_predicates_reject_unsupported_input() -> None: assert not can_use_mxfp8_swizzled( torch.randn(4, 64, device="cuda", dtype=torch.float16) ) - assert not can_use_silu_mul_mxfp8( - torch.randn(4, 96, device="cuda", dtype=torch.bfloat16) - ) if __name__ == "__main__": diff --git a/test/registered/kernels/ops/quantization/test_per_token_group_quant.py b/test/registered/kernels/ops/quantization/test_per_token_group_quant.py index 5e2e1127e8..60837e9e46 100644 --- a/test/registered/kernels/ops/quantization/test_per_token_group_quant.py +++ b/test/registered/kernels/ops/quantization/test_per_token_group_quant.py @@ -322,10 +322,10 @@ def test_fp32_silu_post_quant(group_size, hidden, swiglu_limit): round. Reuse the independent UE8M0 oracle, and compare both layouts only on active rows; an empty expert and a partial slab exercise masked counts. """ - from sglang.kernels.ops.moe.dsv4 import ( + from sglang.kernels.ops.activation.silu_and_mul_clamp import ( silu_and_mul_contig_post_quant, - silu_and_mul_masked_post_quant, ) + from sglang.kernels.ops.moe.dsv4 import silu_and_mul_masked_post_quant torch.manual_seed(123 + hidden) experts, capacity = 3, 32 diff --git a/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py b/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py index d93ad83dee..5c2f5ee736 100644 --- a/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py +++ b/test/registered/kernels/ops/speculative/test_spec_kv_indices_grid.py @@ -8,7 +8,7 @@ import unittest import torch import triton -from sglang.kernels.ops.attention.utils import ( +from sglang.kernels.ops.attention.kv_indices import ( create_flashinfer_kv_indices_triton, kv_indices_num_token_blocks, ) diff --git a/test/registered/spec/dspark/test_dspark_kernel_parity.py b/test/registered/spec/dspark/test_dspark_kernel_parity.py index 9422859059..41ccecd0e6 100644 --- a/test/registered/spec/dspark/test_dspark_kernel_parity.py +++ b/test/registered/spec/dspark/test_dspark_kernel_parity.py @@ -12,9 +12,7 @@ import unittest import torch -from sglang.kernels.ops.attention import ( - dsv4_attn_metadata_kernels as attn_metadata_kernels, -) +from sglang.kernels.ops.attention.dsv4 import attn_metadata_kernels from sglang.kernels.ops.speculative import ragged_verify_kernels from sglang.kernels.ops.speculative.dspark import ( dspark_accept, diff --git a/test/registered/unit/dsa/test_dsa_gfx950_fused_workspace.py b/test/registered/unit/dsa/test_dsa_gfx950_fused_workspace.py index d0d179a61c..cbef339ff7 100644 --- a/test/registered/unit/dsa/test_dsa_gfx950_fused_workspace.py +++ b/test/registered/unit/dsa/test_dsa_gfx950_fused_workspace.py @@ -5,7 +5,7 @@ double stands in for _Workspace, so only the bookkeeping is under test.""" import unittest from unittest import mock -from sglang.kernels.ops.attention.dsa.hip_gfx950 import fused_decode as fd +from sglang.srt.layers.attention.dsa import gfx950_fused_indexer as fd from sglang.test.ci.ci_register import register_cpu_ci # GLM-5.2: 78 target layers + 1 draft layer, each with its own Indexer. diff --git a/test/registered/unit/kernels/test_kernels_namespace.py b/test/registered/unit/kernels/test_kernels_namespace.py index c12429fa18..1eb7755c90 100644 --- a/test/registered/unit/kernels/test_kernels_namespace.py +++ b/test/registered/unit/kernels/test_kernels_namespace.py @@ -1,6 +1,7 @@ """GPU-free import / registry / selector tests for ``sglang.kernels`` (RFC #29630).""" import ast +import re import subprocess import sys from pathlib import Path @@ -173,7 +174,7 @@ def test_import_stays_metadata_only(): ("moe.situ_and_mul_masked_post_quant", KernelBackend.JIT, "cuda"), ("attention.attn_res_fused_tma", KernelBackend.JIT, "cuda"), ("attention.attn_res_hip", KernelBackend.TRITON, "hip"), - ("attention.kimi_k3_mla_output_gate", KernelBackend.JIT, "cuda"), + ("elementwise.kimi_k3_mla_output_gate", KernelBackend.JIT, "cuda"), ("attention.fused_kda_decode_mtp_dspark", KernelBackend.CUTE_DSL, "cuda"), ("attention.flydsl_kimi_k3_kda_decode_with_f_b", KernelBackend.FLYDSL, "hip"), ("communication.all_reduce_push_res", KernelBackend.JIT, "cuda"), @@ -189,6 +190,27 @@ def test_kimi_k3_kernels_are_inventoried_by_operator(op, backend, device): assert not K.capabilities_satisfied(spec.capabilities, _CPU) +def _readme() -> str: + return (Path(K.__file__).resolve().parent / "README.md").read_text() + + +def test_readme_operator_groups_match_registry(): + """The README group list is the documented contract for ``ops._GROUPS``.""" + paragraph = _readme().split("Operator groups", 1)[1].split("\n\n", 1)[0] + documented = re.findall(r"`([a-z_0-9]+)`", paragraph) + assert len(documented) == len(set(documented)), documented + assert set(documented) == set(K.ops.__all__) == set(K.ops._GROUPS) + + +def test_readme_layout_lists_every_top_level_directory(): + """A new directory directly under ``kernels/`` must be documented first.""" + layout = _readme().split("## Layout", 1)[1].split("```", 2)[1] + documented = set(re.findall(r"^ ([a-z_0-9]+)/", layout, re.MULTILINE)) + root = Path(K.__file__).resolve().parent + actual = {p.name for p in root.iterdir() if p.is_dir() and p.name != "__pycache__"} + assert documented == actual + + def test_operator_and_test_groups_agree(): """A new root-level model bundle must not bypass logical op grouping.""" root = Path(K.__file__).resolve().parents[3] @@ -227,12 +249,20 @@ def test_reclassified_public_entry_points_are_inventoried(): "attention.dsa.kpool_topk_transform", "attention.dsa.indexer_k", "attention.dsv4.wo_a", + "attention.helion.kda_replayssm", + "attention.helion.kda_verify", + "attention.minimax_sparse.common.index", + "attention.minimax_sparse.decode.indexer_cp", + "attention.minimax_sparse.decode.topk_sparse", + "attention.minimax_sparse.prefill.gluon_prefill", + "attention.minimax_sparse.prefill.topk_sparse", "embeddings.qwen4_ngram", "elementwise.qwen4_gate", "elementwise.row_scale", "elementwise.hc_combine_decode", "mamba.qwen4_short_conv", "mamba.lfm_short_conv", + "mamba.kda_prefill_track", "gemm.dsv4_wo_a", "gemm.inkling_rel_proj", "gemm.hopper_bf16_gemv", @@ -243,10 +273,15 @@ def test_reclassified_public_entry_points_are_inventoried(): "layernorm.rmsnorm_fanout", "layernorm.grouped_gemma_rmsnorm", "moe.dsv4", + "activation.silu_and_mul_clamp", "moe.gemma4_routing", "moe.shared_expert_gate", "memory.adler32", "memory.row_compact", + "kv_canary.plan.api", + "kv_canary.write", + "kv_canary.verify", + "kv_canary.scatter_req_token_ids", "mm.process.image", "speculative.row_argmax", ) @@ -265,6 +300,65 @@ def test_reclassified_public_entry_points_are_inventoried(): assert f"sglang.kernels.ops.{module}:{node.name}" in targets +def _module_defines(module: str, attr: str) -> bool: + """AST check that ``module`` defines ``attr`` (``name`` or ``name.method``). + + For ``instance.method`` the instance's class must define ``method``. + """ + root = Path(K.__file__).resolve().parents[2] + tree = ast.parse(root.joinpath(*module.split(".")).with_suffix(".py").read_text()) + top = {} + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.ClassDef)): + top[node.name] = node + elif isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name): + top[target.id] = node.value + name, _, method = attr.partition(".") + node = top.get(name) + if node is None or not method: + return node is not None + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + node = top.get(node.func.id) + return isinstance(node, ast.ClassDef) and any( + isinstance(item, ast.FunctionDef) and item.name == method for item in node.body + ) + + +@pytest.mark.parametrize( + "op, backends", + [ + ("attention.muse_spark_qk_norm_rope_store", {"torch", "triton", "helion"}), + ("communication.muse_spark_all_gather", {"jit"}), + ("communication.muse_spark_all_gather_norm_residual", {"jit"}), + ( + "communication.muse_spark_moe_finalize_all_reduce", + {"torch", "cute_dsl", "helion"}, + ), + ("communication.all_gather_matmul", {"cute_dsl"}), + ("communication.matmul_reduce_scatter", {"cute_dsl"}), + ("elementwise.row_sum", {"jit"}), + ("gemm.muse_spark_router_gemm", {"jit"}), + ("layernorm.muse_spark_norm_gate", {"torch", "jit", "helion", "triton"}), + ("layernorm.muse_spark_norm_residual", {"torch", "jit", "triton"}), + ("layernorm.muse_spark_rmsnorm_nvfp4_quant", {"torch", "jit", "helion"}), + ("moe.muse_spark_routing", {"torch", "jit", "triton"}), + ("moe.muse_spark_post_expert_finalize", {"triton"}), + ], +) +def test_muse_spark_kernels_are_inventoried_by_operator(op, backends): + specs = [s for s in K.registry.all_specs() if s.op == op] + assert {s.backend.value for s in specs} == backends + group = op.split(".")[0] + for spec in specs: + module, attr = spec.target.split(":", 1) + assert module.startswith(f"sglang.kernels.ops.{group}."), spec + assert _module_defines(module, attr), spec + if spec.backend is not KernelBackend.TORCH: + assert not K.capabilities_satisfied(spec.capabilities, _CPU), spec + + @pytest.mark.parametrize( "op, sm, expected", [ @@ -287,6 +381,13 @@ def test_reclassified_public_entry_points_are_inventoried(): ("attention.deep_select_topk", (10, 1), False), ("attention.deep_select_topk", (10, 3), True), ("attention.deep_select_topk", (12, 0), False), + ("attention.chunk_kda_fwd", (9, 0), False), + ("attention.chunk_kda_fwd", (10, 0), True), + ("attention.chunk_kda_fwd", (10, 1), False), + ("attention.chunk_kda_fwd", (10, 3), True), + ("attention.chunk_kda_fwd", (12, 0), False), + ("attention.absorbed_bmm_concat_cast_q_fp8", (9, 0), True), + ("attention.absorbed_bmm_concat_cast_q_fp8", (10, 0), False), ("gemm.convrot_int8_fused_linear", (8, 9), False), ("gemm.convrot_int8_fused_linear", (9, 0), True), ("gemm.convrot_int8_fused_linear", (10, 0), True), diff --git a/test/registered/unit/layers/attention/test_kpool_preshuffled_cache_layout.py b/test/registered/unit/layers/attention/test_kpool_preshuffled_cache_layout.py index 454c861c7a..23e2e2885f 100644 --- a/test/registered/unit/layers/attention/test_kpool_preshuffled_cache_layout.py +++ b/test/registered/unit/layers/attention/test_kpool_preshuffled_cache_layout.py @@ -5,7 +5,7 @@ from types import SimpleNamespace import torch -from sglang.kernels.ops.attention.dsa.index_buf_accessor import SetKAndS, _is_fp8_fnuz +from sglang.kernels.ops.kvcache.dsa_index_k_cache import SetKAndS, _is_fp8_fnuz from sglang.srt.layers.attention.dsa.kpool_fp8_index import ( INDEX_HEAD_DIM, gather_index_k_scale_prefix_into, diff --git a/test/registered/unit/layers/moe/test_aiter_runner.py b/test/registered/unit/layers/moe/test_aiter_runner.py index aed8b86aaa..0d3962fdd2 100644 --- a/test/registered/unit/layers/moe/test_aiter_runner.py +++ b/test/registered/unit/layers/moe/test_aiter_runner.py @@ -158,7 +158,7 @@ def test_mori_pre_permute_consumes_dispatcher_cap( MoriEPNormalDispatchOutput, ) - fake_utils = ModuleType("sglang.kernels.ops.moe.rocm_moe_utils") + fake_utils = ModuleType("sglang.kernels.ops.quantization.rocm_upscale") fake_utils.upscale = None fake_utils.upscale_mxfp4 = None monkeypatch.setitem(sys.modules, fake_utils.__name__, fake_utils) diff --git a/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py index 5ebe1273bb..b227a612f7 100644 --- a/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py +++ b/test/registered/unit/layers/test_dsv41_late_layer_tail_readers.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock, patch import torch -from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import ( +from sglang.kernels.ops.attention.dsv4.attn_metadata_kernels import ( late_layer_tail_layout, ) from sglang.srt.layers.attention import deepseek_v4_backend diff --git a/test/registered/unit/layers/test_kpool_stream_scheduling.py b/test/registered/unit/layers/test_kpool_stream_scheduling.py index c0f5b0779a..68e02c10b0 100644 --- a/test/registered/unit/layers/test_kpool_stream_scheduling.py +++ b/test/registered/unit/layers/test_kpool_stream_scheduling.py @@ -10,7 +10,7 @@ from unittest.mock import Mock, patch import torch import torch.nn.functional as F -from sglang.kernels.ops.attention.dsa import triton_kernel +from sglang.kernels.ops.quantization import act_quant_triton from sglang.srt.layers.attention.dsa import dsa_indexer_kpool as indexer_module from sglang.srt.layers.attention.dsa.dsa_indexer_kpool import IndexerKPool from sglang.srt.layers.attention.dsa.dsa_topk_backend import TopkTransformMethod @@ -285,7 +285,7 @@ class TestKPoolStreamScheduling(unittest.TestCase): torch.cuda, "current_stream", side_effect=lambda: streams.current ), patch.object(torch.cuda, "stream", side_effect=streams.use), - patch.object(triton_kernel, "act_quant", side_effect=quant), + patch.object(act_quant_triton, "act_quant", side_effect=quant), ): actual = IndexerKPool._forward_cuda_impl( indexer, x, x, torch.arange(num_tokens), batch, 0, return_indices diff --git a/test/registered/unit/mem_cache/test_dsv4_unified_fp8_pool.py b/test/registered/unit/mem_cache/test_dsv4_unified_fp8_pool.py index 158347fb84..c261a43952 100644 --- a/test/registered/unit/mem_cache/test_dsv4_unified_fp8_pool.py +++ b/test/registered/unit/mem_cache/test_dsv4_unified_fp8_pool.py @@ -243,14 +243,14 @@ class TestDSV4UnifiedRegionBuffers(CustomTestCase): class TestResolveUnifiedKvFp8(CustomTestCase): def test_override_false_wins_over_env(self): - env_mod = "sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate" + env_mod = "sglang.srt.layers.attention.dsv4.unified_kv_gate" with patch(f"{env_mod}.is_unified_kv_fp8", return_value=True): self.assertFalse(resolve_unified_kv_fp8(False)) self.assertTrue(resolve_unified_kv_fp8(True)) self.assertTrue(resolve_unified_kv_fp8(None)) def test_none_follows_env_off(self): - env_mod = "sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate" + env_mod = "sglang.srt.layers.attention.dsv4.unified_kv_gate" with patch(f"{env_mod}.is_unified_kv_fp8", return_value=False): self.assertFalse(resolve_unified_kv_fp8(None)) self.assertFalse(resolve_unified_kv_fp8(False)) @@ -258,7 +258,7 @@ class TestResolveUnifiedKvFp8(CustomTestCase): class TestDsv4PoolFp8Gate(CustomTestCase): - _ENV = "sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate" + _ENV = "sglang.srt.layers.attention.dsv4.unified_kv_gate" def _layout(self, *, is_draft_worker, algo, env_on=True): with patch(f"{self._ENV}.is_unified_kv_fp8", return_value=env_on): @@ -302,7 +302,7 @@ class TestUnifiedKvPoolFollowsCtorFp8(CustomTestCase): ) def test_dspark_draft_layout_has_no_rope_pool(self): - env = "sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate" + env = "sglang.srt.layers.attention.dsv4.unified_kv_gate" with patch(f"{env}.is_unified_kv_fp8", return_value=True): fp8 = unified_fp8_for_dsv4_pool( is_draft_worker=True, spec_algorithm=SpeculativeAlgorithm.DSPARK @@ -315,7 +315,7 @@ class TestUnifiedKvPoolFollowsCtorFp8(CustomTestCase): self.assertIsNone(pool.kv_buffer_rope[0]) def test_eagle_draft_layout_has_rope_pool(self): - env = "sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate" + env = "sglang.srt.layers.attention.dsv4.unified_kv_gate" with patch(f"{env}.is_unified_kv_fp8", return_value=True): fp8 = unified_fp8_for_dsv4_pool( is_draft_worker=True, spec_algorithm=SpeculativeAlgorithm.EAGLE diff --git a/test/registered/unit/mem_cache/test_unified_cache_linker.py b/test/registered/unit/mem_cache/test_unified_cache_linker.py index 88c55faf17..bf75811fe4 100644 --- a/test/registered/unit/mem_cache/test_unified_cache_linker.py +++ b/test/registered/unit/mem_cache/test_unified_cache_linker.py @@ -1298,10 +1298,10 @@ def test_check_hicache_events_commits_common_rank_results(): def test_swa_reuse_policy_tracks_layout_without_a_tier_condition( monkeypatch, swa_req_ring, enable_hicache ): - from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import env_gate + from sglang.srt.layers.attention.dsv4 import unified_kv_gate unified_kv = swa_req_ring is True - monkeypatch.setattr(env_gate, "is_unified_kv_triton", lambda: unified_kv) + monkeypatch.setattr(unified_kv_gate, "is_unified_kv_triton", lambda: unified_kv) component = SWAComponent.__new__(SWAComponent) component.sliding_window_size = 128 component.gates_prefix_match = not unified_kv diff --git a/test/registered/unit/mem_cache/test_unified_mla_block_table.py b/test/registered/unit/mem_cache/test_unified_mla_block_table.py index f200784fb0..d8ed36e6c3 100644 --- a/test/registered/unit/mem_cache/test_unified_mla_block_table.py +++ b/test/registered/unit/mem_cache/test_unified_mla_block_table.py @@ -81,7 +81,7 @@ def _fill_block_table(req_to_token, req_pool_indices, seq_lens, page_size, *, v2 def _fill_block_table_static(req_to_token, req_pool_indices, seq_lens, page_size): """The static-pool route: the stripped flashmla kernel, token//ps verbatim.""" - from sglang.kernels.ops.kvcache.kv_indices import ( + from sglang.kernels.ops.attention.kv_indices import ( create_flashmla_kv_indices_triton, get_num_kv_index_blocks_flashmla, ) diff --git a/test/registered/unit/model_executor/test_encoder_swa_replay.py b/test/registered/unit/model_executor/test_encoder_swa_replay.py index 42366a67d1..ee6b10393a 100644 --- a/test/registered/unit/model_executor/test_encoder_swa_replay.py +++ b/test/registered/unit/model_executor/test_encoder_swa_replay.py @@ -8,7 +8,7 @@ from unittest.mock import MagicMock, patch import torch -from sglang.kernels.ops.attention.dsv4_attn_metadata_kernels import ( +from sglang.kernels.ops.attention.dsv4.attn_metadata_kernels import ( late_layer_tail_layout, ) from sglang.srt.arg_groups import deepseek_v4_hook @@ -52,7 +52,7 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=10, suite="base-a-test-cpu") -_ENV_GATE = "sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate" +_ENV_GATE = "sglang.srt.layers.attention.dsv4.unified_kv_gate" MAX_CTX = 1024 diff --git a/test/registered/unit/models/test_deepseek_v4_cp_kv_store.py b/test/registered/unit/models/test_deepseek_v4_cp_kv_store.py index 3ace1f90f9..1cbfefec9e 100644 --- a/test/registered/unit/models/test_deepseek_v4_cp_kv_store.py +++ b/test/registered/unit/models/test_deepseek_v4_cp_kv_store.py @@ -11,7 +11,7 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=4, suite="base-a-test-cpu") -_ENV_GATE = "sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate" +_ENV_GATE = "sglang.srt.layers.attention.dsv4.unified_kv_gate" _PREFILL = SimpleNamespace( forward_mode=SimpleNamespace( is_decode_or_idle=lambda: False, diff --git a/test/registered/unit/models/test_deepseek_v4_rope_policy.py b/test/registered/unit/models/test_deepseek_v4_rope_policy.py index bae5e90ec2..3c2b9abaaa 100644 --- a/test/registered/unit/models/test_deepseek_v4_rope_policy.py +++ b/test/registered/unit/models/test_deepseek_v4_rope_policy.py @@ -8,7 +8,7 @@ import torch from torch import nn import sglang.srt.models.deepseek_v4 as deepseek_v4 -from sglang.kernels.ops.attention.deepseek_v4_rope import precompute_freqs_cis +from sglang.kernels.ops.attention.dsv4.rope import precompute_freqs_cis from sglang.srt.environ import envs from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase diff --git a/test/registered/unit/models/test_deepseek_v4_unified_fp8_q_pair.py b/test/registered/unit/models/test_deepseek_v4_unified_fp8_q_pair.py index ae17456fa4..d9f9869f46 100644 --- a/test/registered/unit/models/test_deepseek_v4_unified_fp8_q_pair.py +++ b/test/registered/unit/models/test_deepseek_v4_unified_fp8_q_pair.py @@ -14,8 +14,8 @@ from unittest.mock import patch import torch import sglang.srt.models.deepseek_v4 as deepseek_v4 -from sglang.kernels.ops.attention.dsv4.unified_kv_kernels import env_gate from sglang.srt.environ import envs +from sglang.srt.layers.attention.dsv4 import unified_kv_gate from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.ci.ci_register import register_cpu_ci @@ -138,8 +138,8 @@ def _run(fp8, mode=ForwardMode.DECODE, cp=False, fused_verify=True): with ( envs.SGLANG_OPT_USE_MULTI_STREAM_OVERLAP.override(False), envs.SGLANG_OPT_FUSED_QK_NORM_ROPE_VERIFY.override(fused_verify), - patch.object(env_gate, "is_unified_kv_triton", return_value=True), - patch.object(env_gate, "is_unified_kv_fp8", return_value=fp8), + patch.object(unified_kv_gate, "is_unified_kv_triton", return_value=True), + patch.object(unified_kv_gate, "is_unified_kv_fp8", return_value=fp8), patch.object(deepseek_v4, "get_token_to_kv_pool", return_value=_Pool(fp8)), patch.object( deepseek_v4, diff --git a/test/registered/unit/server_args/test_platform_prefill_cp_deprecation.py b/test/registered/unit/server_args/test_platform_prefill_cp_deprecation.py index 77455216fe..ed78c4f1b7 100644 --- a/test/registered/unit/server_args/test_platform_prefill_cp_deprecation.py +++ b/test/registered/unit/server_args/test_platform_prefill_cp_deprecation.py @@ -17,7 +17,7 @@ from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=2, suite="base-a-test-cpu") -_ENV_GATE = "sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate" +_ENV_GATE = "sglang.srt.layers.attention.dsv4.unified_kv_gate" def _cp_args(arch="DeepseekV4ForCausalLM", model_type="deepseek_v4", **overrides): diff --git a/test/registered/unit/spec/test_ngram_mamba_verify_update.py b/test/registered/unit/spec/test_ngram_mamba_verify_update.py index 23a4aa2cb0..ca95e9ffda 100644 --- a/test/registered/unit/spec/test_ngram_mamba_verify_update.py +++ b/test/registered/unit/spec/test_ngram_mamba_verify_update.py @@ -160,22 +160,22 @@ class TestPPReplaySSMVerifySourceRows(CustomTestCase): return spec_state def test_fold_helpers_read_pp_request_rows(self): - from sglang.kernels.ops.attention.fla.gdn_replayssm_spec_fold import ( + from sglang.kernels.ops.attention.linear.gdn_replayssm_spec_fold import ( commit_gdn_replayssm_fold_after_verify, ) - from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( + from sglang.kernels.ops.attention.linear.kda_replayssm_spec_decode import ( commit_kda_replayssm_after_verify, ) cases = ( ( commit_gdn_replayssm_fold_after_verify, - "sglang.kernels.ops.attention.fla.gdn_replayssm_spec_fold", + "sglang.kernels.ops.attention.linear.gdn_replayssm_spec_fold", "commit_gdn_replayssm_fold_all_layers", ), ( commit_kda_replayssm_after_verify, - "sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode", + "sglang.kernels.ops.attention.linear.kda_replayssm_spec_decode", "commit_kda_replayssm_spec_all_layers", ), ) @@ -232,11 +232,11 @@ class TestPPReplaySSMVerifySourceRows(CustomTestCase): return_value=True, ), patch( - "sglang.kernels.ops.attention.fla.gdn_replayssm_spec_decode." + "sglang.kernels.ops.attention.linear.gdn_replayssm_spec_decode." "commit_gdn_replayssm_spec" ), patch( - "sglang.kernels.ops.attention.fla.gdn_replayssm_spec_decode." + "sglang.kernels.ops.attention.linear.gdn_replayssm_spec_decode." "commit_gdn_replayssm_circular" ), patch( diff --git a/test/registered/xpu/test_dsa_indexer_xpu.py b/test/registered/xpu/test_dsa_indexer_xpu.py index c1420ec4e0..bfd5982f59 100644 --- a/test/registered/xpu/test_dsa_indexer_xpu.py +++ b/test/registered/xpu/test_dsa_indexer_xpu.py @@ -388,7 +388,7 @@ class TestDSAIndexerXPU(CustomTestCase): # Test: indexer forward — extend mode # ------------------------------------------------------------------ - @patch("sglang.kernels.ops.attention.dsa.triton_kernel.act_quant") + @patch("sglang.kernels.ops.quantization.act_quant_triton.act_quant") def test_forward_extend_mode(self, mock_act_quant): """Indexer forward in EXTEND mode calls sgl_kernel.fp8_mqa_logits on XPU.""" @@ -444,7 +444,7 @@ class TestDSAIndexerXPU(CustomTestCase): # Test: indexer forward — decode mode # ------------------------------------------------------------------ - @patch("sglang.kernels.ops.attention.dsa.triton_kernel.act_quant") + @patch("sglang.kernels.ops.quantization.act_quant_triton.act_quant") def test_forward_decode_mode(self, mock_act_quant): """Indexer forward in DECODE mode calls sgl_kernel.fp8_paged_mqa_logits on XPU."""