From 329072d99a7e150d72ba0731066525269b61b843 Mon Sep 17 00:00:00 2001 From: Xiaoyu Zhang <1182563586@qq.com> Date: Mon, 19 Oct 2026 08:55:47 +0800 Subject: [PATCH] [Refactor] kernels/diffusion/tests: Prune dead code and fix tests that never ran (#44690) Co-authored-by: Claude Opus 5.5 --- .github/workflows/pr-test-multimodal-gen.yml | 29 + .../Moonshotai/Kimi-K2.7-Code.mdx | 1 - .../LingBot-World/LingBot-World-2.0.mdx | 2 +- .../diffusion/LingBot-World/LingBot-World.mdx | 2 +- docs/cookbook/diffusion/SANA-WM/SANA-WM.mdx | 24 +- .../docs/references/environment_variables.mdx | 12 +- .../environment_variables.mdx | 25 - .../kimi-k27-code-deployment.jsx | 1 - python/sglang/kernels/README.md | 17 +- python/sglang/kernels/__init__.py | 4 +- .../aot/benchmark/bench_dsv4_norm_rope.py | 75 - .../benchmark/bench_moe_ep_post_reorder.py | 85 - .../bench_per_token_group_quant_8bit.py | 10 +- .../kernels/aot/csrc/common_extension.cc | 11 +- .../kernels/aot/csrc/common_extension_rocm.cc | 9 - .../aot/csrc/elementwise/dsv4_norm_rope.cu | 391 +--- .../kernels/aot/csrc/elementwise/topk.hip | 9 +- .../kernels/aot/include/sgl_kernel_ops.h | 38 +- .../kernels/aot/python/sgl_kernel/__init__.py | 4 - .../aot/python/sgl_kernel/elementwise.py | 53 - .../kernels/aot/python/sgl_kernel/utils.py | 12 - .../kernels/aot/tests/test_dsv4_norm_rope.py | 86 +- .../aot/tests/test_fused_qk_norm_rope.py | 14 +- .../kernels/aot/tests/test_merge_state_v2.py | 92 +- .../kernels/aot/tests/test_moe_align.py | 142 +- .../tests/test_per_token_group_quant_8bit.py | 104 +- python/sglang/kernels/aot/tests/test_topk.py | 5 +- python/sglang/kernels/fused_op.py | 46 +- python/sglang/kernels/jit/benchmark/utils.py | 2 +- .../kernels/jit/csrc/deepseek_v4/c128.cuh | 522 ----- .../jit/csrc/deepseek_v4/c128_online.cuh | 722 ------- .../jit/csrc/deepseek_v4/c128_online_v2.cuh | 4 +- .../kernels/jit/csrc/deepseek_v4/c4.cuh | 549 ----- .../kernels/jit/csrc/deepseek_v4/common.cuh | 208 -- .../deepseek_v4/mega_moe_pre_dispatch.cuh | 2 +- .../jit/csrc/distributed/nvlink_comm.cuh | 32 - .../jit/csrc/elementwise/adler32_checksum.cuh | 78 +- .../csrc/elementwise/qknorm_across_heads.cuh | 179 -- .../jit/csrc/inkling/inkling_all_reduce.cuh | 44 +- .../csrc/minimax/per_token_quant_ue8m0.cuh | 115 +- .../jit/csrc/moe/inkling_gate_topk_renorm.cuh | 273 --- .../kernels/jit/csrc/moe/moe_fused_gate.cuh | 376 ---- .../sgl_kernel/deepseek_v4/kv_layout.cuh | 2 +- .../sglang/kernels/ops/activation/__init__.py | 317 +-- .../sglang/kernels/ops/activation/softcap.py | 64 - .../sglang/kernels/ops/attention/__init__.py | 45 +- .../kernels/ops/attention/clamp_position.py | 13 +- .../kernels/ops/attention/cute_utils/cvt.py | 101 +- .../ops/attention/cutedsl_gdn_mtp_ring.py | 972 +-------- .../kernels/ops/attention/dcp_kernels.py | 43 - .../kernels/ops/attention/deepseek_v4_rope.py | 86 - .../ops/attention/dsa/index_buf_accessor.py | 302 +-- .../ops/attention/dsa/quant_k_cache.py | 51 - .../ops/attention/dsa/tilelang_kernel.py | 83 - .../ops/attention/dsa/transform_index.py | 53 - .../attention/dsa/triton_sparse_mla_decode.py | 442 +--- .../kernels/ops/attention/dsv4/__init__.py | 2 +- .../kernels/ops/attention/dsv4/compress.py | 2 +- .../ops/attention/dsv4/compress_old.py | 308 --- .../ops/attention/dsv4/dequant_k_cache.py | 83 - .../kernels/ops/attention/dsv4/elementwise.py | 37 + .../ops/attention/dsv4/fp4_indexer_hip.py | 6 +- .../attention/dsv4/fused_compress_triton.py | 1116 ---------- .../ops/attention/dsv4/index_buf_accessor.py | 78 - .../ops/attention/dsv4/quant_k_cache.py | 120 -- .../kernels/ops/attention/dsv4/torch_quant.py | 56 - .../dsv4/unified_kv_kernels/paged_decode.py | 4 +- .../kernels/ops/attention/extend_attention.py | 38 - .../fla/gdn_replayssm_spec_decode.py | 202 -- .../ops/attention/flash_attention_v4.py | 27 +- .../ops/attention/flash_attention_v4_sm120.py | 27 +- .../ops/attention/kda_flydsl/__init__.py | 2 - .../kda_flydsl/kernels/kimi_k3_kda_decode.py | 450 ---- .../kda_flydsl/kimi_k3_kda_decode.py | 109 +- .../linear/kda_nvidia_prefill/chunk_fwd.py | 104 +- .../fuse_k4_only_persistent.py | 53 +- .../kernels/ops/attention/linear/seg_la.py | 231 -- .../ops/attention/triton_gdn_fused_proj.py | 5 +- .../attention/vattn_asm_gfx950/__init__.py | 13 +- .../kernels/ops/attention/vision_rope.py | 10 +- .../kernels/ops/communication/__init__.py | 10 +- .../ops/communication/inkling_all_reduce.py | 27 - python/sglang/kernels/ops/communication/mp.py | 188 +- .../kernels/ops/communication/nvlink_comm.py | 66 +- .../ops/communication/sp_collective.py | 14 - .../sglang/kernels/ops/diffusion/__init__.py | 198 +- .../attention/sana_wm_gdn_chunkwise_triton.py | 29 +- .../diffusion/attention/sana_wm_gdn_triton.py | 71 +- .../ops/diffusion/common/fast_launch.py | 12 +- .../kernels/ops/diffusion/common/platform.py | 7 - .../ops/diffusion/norm/h3_vae_fused_norm.py | 114 - .../ops/diffusion/sites/bitexact_gate.py | 77 +- .../sites/flux2_nvfp4_swiglu_quant_site.py | 21 +- .../sites/fused_gate_rmsnorm_site.py | 56 +- .../diffusion/sites/fused_linear_gelu_site.py | 64 +- .../diffusion/sites/fused_ln_modulate_site.py | 33 +- ...esidual_site.py => gated_residual_site.py} | 43 +- .../sites/helios_gated_residual_site.py | 72 - .../diffusion/sites/hunyuan_qknorm_site.py | 31 +- .../sites/lingbot_video_rmsnorm_site.py | 32 +- .../sites/ltx2_qknorm_split_rope_site.py | 20 +- .../sites/ltx2_rmsnorm_modulate_site.py | 33 +- .../sites/minimax_h3_norm_modulate_site.py | 25 +- .../diffusion/sites/nvfp4_bias_gelu_site.py | 54 +- .../ops/diffusion/sites/quality_gate.py | 44 +- .../sites/qwen_image_added_qkv_site.py | 32 +- .../sites/sana_video_linear_attention_site.py | 34 +- .../kernels/ops/elementwise/__init__.py | 14 +- .../kernels/ops/elementwise/elementwise.py | 187 +- .../sglang/kernels/ops/elementwise/hc_mix.py | 11 - .../kernels/ops/elementwise/qwen4_gate.py | 30 +- .../sglang/kernels/ops/embeddings/__init__.py | 4 +- python/sglang/kernels/ops/gemm/__init__.py | 187 +- .../kernels/ops/gemm/cutedsl_bf16_gemm.py | 8 - .../dense_bf16_gemm_sm100_splitk_epilogue.py | 34 - python/sglang/kernels/ops/gemm/fp8_kernel.py | 18 - .../ops/gemm/smallm_fp8_gfx950/__init__.py | 7 +- python/sglang/kernels/ops/kvcache/__init__.py | 7 +- .../sglang/kernels/ops/layernorm/__init__.py | 529 +---- python/sglang/kernels/ops/layernorm/mhc.py | 45 +- .../ops/layernorm/minimax_m3_rmsnorm.py | 1 + python/sglang/kernels/ops/layernorm/norm.py | 35 - .../ops/layernorm/rms_normalize_hip.py | 52 - .../kernels/ops/lplb/cublasdx_solver.py | 14 - python/sglang/kernels/ops/lplb/cuda_solver.py | 72 +- .../sglang/kernels/ops/lplb/shmem_budget.py | 21 - .../sglang/kernels/ops/lplb/torch_solver.py | 56 - python/sglang/kernels/ops/mamba/__init__.py | 33 +- .../kernels/ops/mamba/qwen4_short_conv.py | 4 +- python/sglang/kernels/ops/memory/__init__.py | 22 +- python/sglang/kernels/ops/memory/adler32.py | 56 +- python/sglang/kernels/ops/mm/__init__.py | 4 +- python/sglang/kernels/ops/moe/__init__.py | 42 +- .../sglang/kernels/ops/moe/ep_moe_kernels.py | 42 - .../ops/moe/fused_moe_triton_kernels.py | 4 +- .../ops/moe/inkling_gate_topk_renorm.py | 88 +- .../sglang/kernels/ops/moe/moe_fused_gate.py | 80 +- .../kernels/ops/moe/moe_route_quant_fused.py | 9 +- .../sglang/kernels/ops/moe/rocm_moe_utils.py | 112 - .../ops/moe/smallm_moe_gfx950/__init__.py | 25 +- .../ops/moe/smallm_router_gfx950/__init__.py | 4 +- .../kernels/ops/quantization/__init__.py | 83 +- .../kernels/ops/quantization/awq_triton.py | 180 -- .../kernels/ops/quantization/fp8_kernel.py | 348 +--- .../kernels/ops/quantization/int8_kernel.py | 2 + .../ops/quantization/minimax_quant_ue8m0.py | 43 +- .../ops/quantization/mxfp8_amd_gfx95.py | 42 +- .../kernels/ops/quantization/mxfp8_quant.py | 164 -- .../nvfp4_gemm_swiglu_nvfp4_quant.py | 30 - .../per_token_group_quant_8bit_v2.py | 11 +- .../sglang/kernels/ops/sampling/__init__.py | 57 +- .../ops/sampling/top_p_renorm_triton.py | 127 -- .../kernels/ops/speculative/__init__.py | 13 +- .../dspark/dspark_verify_window.py | 78 - python/sglang/kernels/selector.py | 5 - python/sglang/kernels/spec.py | 47 +- .../existing-fast-paths.md | 4 +- .../sglang/multimodal_gen/.claude/CLAUDE.md | 1 - .../multimodal_gen/apps/realtime_webui/app.js | 91 +- .../apps/realtime_webui/decoder_worker.js | 58 - .../apps/realtime_webui/index.html | 2 +- .../configs/pipeline_configs/__init__.py | 2 - .../configs/pipeline_configs/base.py | 18 - .../configs/sample/hunyuan_image3.py | 17 - python/sglang/multimodal_gen/envs.py | 93 +- python/sglang/multimodal_gen/registry.py | 91 +- .../runtime/breakable_cuda_graph/runner.py | 11 - .../entrypoints/diffusion_generator.py | 49 - .../runtime/layers/flux3_fused_ops.py | 18 +- .../runtime/layers/layernorm.py | 9 +- .../runtime/layers/quantization/__init__.py | 33 - .../layers/quantization/bitsandbytes.py | 77 - .../layers/quantization/modelopt_quant.py | 11 - .../layers/quantization/weight_only_fp8.py | 21 - .../runtime/loader/readonly_safetensors.py | 5 - .../multimodal_gen/runtime/loader/utils.py | 20 - .../runtime/managers/gpu_worker.py | 99 +- .../memory_managers/auto_residency.py | 355 +--- .../memory_managers/component_manager.py | 176 +- .../memory_managers/layerwise_offload.py | 96 - .../decoders/ltx_2_5_diffusion_decoder.py | 38 +- .../runtime/models/dits/anima.py | 24 +- .../runtime/models/dits/causal_wanvideo.py | 3 +- .../runtime/models/dits/common.py | 41 +- .../runtime/models/dits/cosmos3video.py | 2 + .../runtime/models/dits/ernie_image.py | 94 +- .../runtime/models/dits/flux.py | 34 +- .../runtime/models/dits/flux3.py | 75 +- .../runtime/models/dits/flux3_image.py | 15 - .../runtime/models/dits/flux_2.py | 49 +- .../runtime/models/dits/glm_image.py | 93 +- .../runtime/models/dits/helios.py | 14 +- .../runtime/models/dits/hunyuanvideo.py | 16 +- .../runtime/models/dits/ideogram.py | 69 +- .../runtime/models/dits/joy_image.py | 63 +- .../runtime/models/dits/krea2.py | 12 +- .../runtime/models/dits/lingbot_video_moe.py | 17 +- .../runtime/models/dits/longcat_image.py | 79 +- .../runtime/models/dits/ltx_2.py | 25 +- .../runtime/models/dits/ming_image.py | 36 +- .../runtime/models/dits/minimax_h3.py | 11 +- .../runtime/models/dits/moss_image.py | 85 +- .../runtime/models/dits/qwen_image.py | 30 +- .../runtime/models/dits/sana.py | 12 - .../runtime/models/dits/sana_video.py | 52 +- .../runtime/models/dits/sana_wm.py | 35 - .../runtime/models/dits/sana_wm_parity.py | 82 - .../runtime/models/dits/wanvideo.py | 51 +- .../runtime/models/encoders/gemma_3.py | 11 - .../multimodal_gen/runtime/models/registry.py | 110 +- .../scheduling_minimax_h3_euler_ancestral.py | 213 -- .../neo_unify/modeling_fm_modules.py | 517 +---- .../neo_unify/modeling_neo_chat.py | 1849 +---------------- .../sensenova_u1/neo_unify/modeling_qwen3.py | 36 +- .../neo_unify/transformers_compat.py | 5 - .../models/sensenova_u1/neo_unify/utils.py | 144 -- .../runtime/models/vaes/autoencoder_dc.py | 41 +- .../models/vaes/autoencoder_kl_qwenimage.py | 8 - .../models/vaes/minimax_h3_vae_cuda_opt.py | 4 +- .../vaes/minimax_h3_video_vae/attention.py | 64 +- .../vaes/minimax_h3_video_vae/base_module.py | 69 +- .../models/vaes/minimax_h3_video_vae/klvae.py | 45 +- .../models/vaes/minimax_h3_video_vae/norm.py | 176 -- .../vaes/minimax_h3_video_vae/vae_cnn.py | 25 +- .../vaes/minimax_h3_video_vae/vae_vit.py | 5 +- .../vaes/minimax_h3_video_vae/vit_utils.py | 91 +- .../runtime/pipelines/flux3_image_weights.py | 12 - .../runtime/pipelines/minimax_h3_pipeline.py | 4 +- .../runtime/pipelines_core/schedule_batch.py | 29 - .../runtime/pipelines_core/stages/__init__.py | 2 - .../runtime/pipelines_core/stages/base.py | 43 - .../pipelines_core/stages/causal_denoising.py | 9 - .../pipelines_core/stages/denoising.py | 186 +- .../pipelines_core/stages/denoising_dmd.py | 6 - .../runtime/pipelines_core/stages/encoding.py | 125 -- .../stages/model_specific_stages/cosmos3.py | 5 - .../model_specific_stages/helios_denoising.py | 35 +- .../lingbot_world_causal_denoising.py | 1 - .../stages/model_specific_stages/longlive2.py | 3 - .../ltx_2/denoising_av.py | 6 - .../minimax_h3/stages/denoising.py | 5 - .../stages/model_specific_stages/mova.py | 5 - .../model_specific_stages/sana_video2.py | 3 - .../sana_wm/realtime_chain.py | 6 - .../model_specific_stages/sana_wm/refiner.py | 2 - .../sana_wm/streaming.py | 106 +- .../sana_wm/streaming_refiner.py | 34 - .../pipelines_core/stages/validators.py | 37 - .../multimodal_gen/runtime/platforms/cpu.py | 4 - .../multimodal_gen/runtime/platforms/cuda.py | 11 - .../runtime/platforms/interface.py | 19 - .../multimodal_gen/runtime/platforms/mps.py | 11 - .../multimodal_gen/runtime/platforms/musa.py | 18 - .../multimodal_gen/runtime/platforms/npu.py | 11 - .../multimodal_gen/runtime/platforms/rocm.py | 10 - .../multimodal_gen/runtime/platforms/xpu.py | 11 - .../multimodal_gen/runtime/realtime/video.py | 76 - .../runtime/server_args/disagg.py | 13 - .../runtime/server_args/server_args.py | 16 - .../runtime/utils/model_overlay.py | 34 - .../runtime/utils/perf_logger.py | 24 +- .../multimodal_gen/runtime/utils/profiler.py | 3 +- .../multimodal_gen/test/server/gpu_cases.py | 21 +- .../test/server/realtime_consistency.py | 35 +- .../test_diffusion_bcg_tp2_zimage_turbo.py | 18 +- .../test/unit/modelopt_fp8_stub.py | 21 + .../test_realtime_consistency_harness.py | 131 +- .../test_realtime_output_transport.py | 19 - .../test/unit/sana_wm/test_pipeline_config.py | 71 - .../unit/sana_wm/test_streaming_cached.py | 128 +- .../unit/test_attention_backend_selector.py | 1 + .../test/unit/test_auto_residency.py | 379 ---- .../test/unit/test_comfyui_h3.py | 5 - .../test/unit/test_comfyui_model_patcher.py | 28 - .../test/unit/test_comfyui_qwen_image21.py | 47 - .../test/unit/test_component_residency.py | 84 +- .../test_conditioning_model_boundaries.py | 29 - .../test/unit/test_convrot_int8_backends.py | 36 - .../multimodal_gen/test/unit/test_cosmos3.py | 18 - .../test/unit/test_diffusion_bcg_padding.py | 98 +- .../unit/test_flux2_fp8_norm_quant_gate.py | 36 +- .../test/unit/test_flux3_image_served.py | 1 - .../test/unit/test_glm_image_attention.py | 54 + .../unit/test_hunyuan_image3_tp_broadcast.py | 33 - .../multimodal_gen/test/unit/test_ipc_cuda.py | 6 - .../unit/test_kandinsky6_pipeline_contract.py | 62 - .../test/unit/test_kandinsky6_rope.py | 14 - .../test_kandinsky6_sr_reference_parity.py | 763 ------- .../test_latent_upsampler_group_norm_silu.py | 52 - .../test/unit/test_ltx2_5_config.py | 37 - .../test/unit/test_minimax_h3_denoise_loop.py | 36 +- .../test_minimax_h3_vae_parallel_modes.py | 9 - .../test/unit/test_performance_metrics.py | 117 +- .../test/unit/test_qwen_image21.py | 83 +- .../unit/test_qwen_image21_distributed.py | 24 +- .../unit/test_qwen_image_fp8_norm_quant.py | 26 +- .../test/unit/test_readonly_safetensors.py | 2 - .../test/unit/test_sensenova_u1.py | 58 - .../test/unit/test_sensenova_u1_backbone.py | 19 + .../test/unit/test_server_args.py | 301 +-- .../multimodal_gen/test/unit/test_storage.py | 78 +- .../test/unit/test_usp_ring_replicated.py | 8 + .../test/unit/test_vae_loader.py | 27 + .../test/unit/test_wan_animate_2_config.py | 28 - .../test/unit/test_wan_gelu_mlp.py | 28 +- .../test/unit/test_warmup_step_floor.py | 2 - .../test/unit/test_weight_utils.py | 13 - .../multimodal_gen/tools/convert_hf_to_fp8.py | 320 --- python/sglang/srt/environ.py | 12 +- .../srt/layers/attention/dsv4/compress_hip.py | 451 ---- .../srt/layers/attention/dsv4/compressor.py | 299 +-- .../srt/layers/attention/dsv4/indexer.py | 2 +- .../attention/linear/lightning_backend.py | 1 - .../layers/deep_gemm_wrapper/compile_utils.py | 2 +- python/sglang/srt/layers/engram.py | 3 +- .../moe/fused_moe_triton/fused_marlin_moe.py | 1 + .../sglang/srt/layers/moe/moe_runner/aiter.py | 5 +- python/sglang/srt/model_loader/loader.py | 2 +- .../sglang/srt/models/bailing_moe_linear.py | 3 +- .../triton_qk_rmsnorm.py | 1 + .../deepseek_common/deepseek_weight_loader.py | 4 +- .../srt/models/dots3_common/modeling.py | 4 +- python/sglang/srt/models/longcat_flash.py | 3 +- .../sglang/srt/models/longcat_flash_nextn.py | 3 +- python/sglang/srt/models/minimax_m3.py | 4 +- python/sglang/test/kernels/dcp.py | 46 + .../sglang/test/kernels/deepseek_v4/common.py | 58 + .../deepseek_v4/dsv41_kv_quant_reference.py | 58 +- .../kernels/deepseek_v4/k_cache_reference.py | 263 +++ .../test/kernels/dsa_transform_index.py | 60 + python/sglang/test/kernels/dspark.py | 92 + .../sglang/test/kernels/extend_attention.py | 43 + python/sglang/test/kernels/fp8.py | 47 +- python/sglang/test/kernels/lplb.py | 125 ++ python/sglang/test/kernels/mhc.py | 44 + python/sglang/test/kernels/moe_align.py | 135 ++ python/sglang/test/kernels/multigpu.py | 189 ++ .../test/kernels/per_token_group_quant.py | 146 ++ python/sglang/test/kernels/qknorm.py | 19 + python/sglang/test/kernels/utils.py | 23 +- .../attention_methods/dsv4_attention.py | 8 +- .../attention_methods/lightning_attention.py | 2 +- scripts/ci/cuda/warmup_deep_gemm.py | 2 +- scripts/ci/partition_rerun_tests.py | 1 - .../diffusion/compute_diffusion_partitions.py | 6 +- .../utils/diffusion/diffusion_case_parser.py | 30 +- .../diffusion/verify_diffusion_coverage.py | 6 +- .../attention/check_dsa_transform_index.py | 2 + .../attention/check_dsv4_dequant_k_cache.py | 2 + .../attention/dsa/test_index_buf_accessor.py | 317 +-- test/manual/quant/test_block_fp8.py | 16 +- .../test_triton_attention_kernels.py | 2 +- .../attention/test_wave_attention_kernels.py | 2 +- .../unittests/lightning/test_triton.py | 1 - .../test_mamba_extra_buffer_spec_grammar.py | 5 +- test/registered/eplb/test_lplb_distributed.py | 8 +- .../communication/bench_custom_all_reduce.py | 3 +- .../bench_symm_mem_all_gather.py | 3 +- .../communication/bench_tp_qknorm.py | 3 +- .../diffusion/bench_residual_gate_add.py | 20 +- .../diffusion/bench_scale_shift_4d.py | 20 +- .../benchmark/layernorm/bench_qknorm.py | 17 +- .../layernorm/bench_qknorm_across_heads.py | 125 -- .../benchmark/moe/bench_moe_fused_gate.py | 8 +- .../moe/bench_post_reorder_deepgemm.py | 22 +- .../bench_per_token_group_quant.py | 4 +- .../bench_per_token_group_quant_masked.py | 4 +- .../activation/test_fused_op_gpu_parity.py | 8 +- .../attention/dsv4/test_flydsl_v4nm_decode.py | 33 +- .../ops/attention/dsv4/test_mega_attention.py | 2 +- .../attention/dsv4/test_swapab_attention.py | 2 +- .../ops/attention/dsv4/test_v41_kv_store.py | 14 +- .../kda_flydsl/test_kimi_k3_kda_decode.py | 69 - .../kernels/ops/attention/test_c128_v2.py | 55 +- .../kernels/ops/attention/test_c4_v2.py | 57 +- .../kernels/ops/attention/test_cutedsl_gdn.py | 202 -- .../ops/attention/test_dcp_lse_combine.py | 16 +- .../ops/attention/test_dsa_transform_index.py | 4 +- .../ops/attention/test_flash_attention_4.py | 288 +-- .../kernels/ops/attention/test_fp4_indexer.py | 25 +- .../ops/attention/test_fp4_indexer_hip.py | 25 +- .../ops/attention/test_hy4_hpc_gated_mla.py | 10 +- .../ops/attention/test_kda_decode_mtp.py | 90 - .../test_q8kv8_sparse_prefill_backend.py | 10 +- .../ops/attention/test_replayssm_ring_fold.py | 142 -- .../kernels/ops/diffusion/test_fast_launch.py | 5 +- .../ops/diffusion/test_h3_vae_fused_norm.py | 34 +- .../test_longcat_image_norm_modulate.py | 14 +- .../ops/diffusion/test_model_fast_paths.py | 63 +- .../kernels/ops/diffusion/test_sites.py | 161 +- .../kernels/ops/gemm/test_fp8_kernel.py | 46 - .../ops/gemm/test_linear_bf16_fp32_hpc.py | 7 +- .../kernels/ops/gemm/test_tiny_gemm.py | 2 + .../ops/gemm/test_tiny_gemm_variants.py | 35 - .../kvcache/test_fused_fp8_qkv_kv_cache.py | 6 +- .../ops/layernorm/test_fused_op_gpu_parity.py | 107 - .../ops/layernorm/test_hc_boundary_hip.py | 9 +- .../kernels/ops/layernorm/test_hy4_hpc_ihc.py | 7 +- .../kernels/ops/layernorm/test_qknorm.py | 16 - .../ops/layernorm/test_qknorm_across_heads.py | 84 - .../ops/memory/test_adler32_checksum.py | 24 +- .../ops/moe/test_minimax_quant_scatter.py | 14 +- .../ops/moe/test_moe_align_block_size.py | 133 +- .../kernels/ops/moe/test_moe_fused_gate.py | 16 +- .../ops/moe/test_post_reorder_deepgemm.py | 12 +- .../moe/test_sigmoid_gate_mul_broadcast.py | 37 - .../ops/quantization/test_awq_dequant.py | 61 - .../quantization/test_per_token_quant_fp8.py | 1 - .../kernels/ops/sampling/test_top_p_renorm.py | 38 - .../spec/dspark/test_dspark_kernel_parity.py | 5 +- .../mlx/test_attention_patching.py | 2 +- test/registered/unit/kernels/test_fused_op.py | 72 +- .../unit/kernels/test_kernels_namespace.py | 27 +- .../test_minimax_h3_step_contract.py | 5 - 414 files changed, 3522 insertions(+), 26178 deletions(-) delete mode 100644 python/sglang/kernels/aot/benchmark/bench_dsv4_norm_rope.py delete mode 100644 python/sglang/kernels/aot/benchmark/bench_moe_ep_post_reorder.py delete mode 100644 python/sglang/kernels/jit/csrc/deepseek_v4/c128.cuh delete mode 100644 python/sglang/kernels/jit/csrc/deepseek_v4/c128_online.cuh delete mode 100644 python/sglang/kernels/jit/csrc/deepseek_v4/c4.cuh delete mode 100644 python/sglang/kernels/jit/csrc/deepseek_v4/common.cuh delete mode 100644 python/sglang/kernels/jit/csrc/elementwise/qknorm_across_heads.cuh delete mode 100644 python/sglang/kernels/jit/csrc/moe/moe_fused_gate.cuh delete mode 100644 python/sglang/kernels/ops/attention/dsv4/compress_old.py delete mode 100644 python/sglang/kernels/ops/attention/dsv4/fused_compress_triton.py delete mode 100644 python/sglang/kernels/ops/attention/dsv4/quant_k_cache.py delete mode 100644 python/sglang/kernels/ops/attention/kda_flydsl/kernels/kimi_k3_kda_decode.py rename python/sglang/kernels/ops/diffusion/sites/{lingbot_video_gated_residual_site.py => gated_residual_site.py} (59%) delete mode 100644 python/sglang/kernels/ops/diffusion/sites/helios_gated_residual_site.py delete mode 100644 python/sglang/kernels/ops/layernorm/rms_normalize_hip.py delete mode 100644 python/sglang/kernels/ops/lplb/cublasdx_solver.py delete mode 100644 python/sglang/kernels/ops/sampling/top_p_renorm_triton.py delete mode 100644 python/sglang/multimodal_gen/runtime/models/dits/sana_wm_parity.py delete mode 100644 python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_minimax_h3_euler_ancestral.py delete mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/encoding.py create mode 100644 python/sglang/multimodal_gen/test/unit/modelopt_fp8_stub.py delete mode 100644 python/sglang/multimodal_gen/test/unit/test_comfyui_model_patcher.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_glm_image_attention.py delete mode 100644 python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_reference_parity.py delete mode 100644 python/sglang/multimodal_gen/tools/convert_hf_to_fp8.py delete mode 100644 python/sglang/srt/layers/attention/dsv4/compress_hip.py create mode 100644 python/sglang/test/kernels/dcp.py create mode 100644 python/sglang/test/kernels/deepseek_v4/k_cache_reference.py create mode 100644 python/sglang/test/kernels/dsa_transform_index.py create mode 100644 python/sglang/test/kernels/dspark.py create mode 100644 python/sglang/test/kernels/extend_attention.py create mode 100644 python/sglang/test/kernels/lplb.py create mode 100644 python/sglang/test/kernels/mhc.py create mode 100644 python/sglang/test/kernels/moe_align.py create mode 100644 python/sglang/test/kernels/multigpu.py create mode 100644 python/sglang/test/kernels/per_token_group_quant.py create mode 100644 python/sglang/test/kernels/qknorm.py delete mode 100644 test/registered/kernels/benchmark/layernorm/bench_qknorm_across_heads.py delete mode 100644 test/registered/kernels/ops/attention/test_replayssm_ring_fold.py delete mode 100644 test/registered/kernels/ops/gemm/test_fp8_kernel.py delete mode 100644 test/registered/kernels/ops/gemm/test_tiny_gemm_variants.py delete mode 100644 test/registered/kernels/ops/layernorm/test_fused_op_gpu_parity.py delete mode 100644 test/registered/kernels/ops/layernorm/test_qknorm_across_heads.py delete mode 100644 test/registered/kernels/ops/sampling/test_top_p_renorm.py diff --git a/.github/workflows/pr-test-multimodal-gen.yml b/.github/workflows/pr-test-multimodal-gen.yml index ada973468a..27c3522673 100644 --- a/.github/workflows/pr-test-multimodal-gen.yml +++ b/.github/workflows/pr-test-multimodal-gen.yml @@ -175,6 +175,12 @@ jobs: run: | CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion + - name: Run SM120-only diffusion unit tests + timeout-minutes: 10 + working-directory: python + run: | + python3 -m pytest -v sglang/multimodal_gen/test/unit/test_fp8_fa_sm120_attn.py + - name: Run 5090 diffusion canary tests timeout-minutes: 120 env: @@ -427,6 +433,22 @@ jobs: run: | python3 -m pytest -v python/sglang/multimodal_gen/test/single_test_file/test_wan_vae_parallel_groups_4_gpu.py + - name: Run Blackwell-only diffusion unit tests + timeout-minutes: 10 + working-directory: python + run: | + python3 -m pytest -v \ + sglang/multimodal_gen/test/unit/test_mxfp8_online_gemm.py \ + sglang/multimodal_gen/test/unit/test_wan_gelu_mlp.py + + - name: Run 4-GPU diffusion unit tests + timeout-minutes: 15 + working-directory: python + run: | + python3 -m pytest -v \ + sglang/multimodal_gen/test/unit/test_kandinsky6_sequence_parallel.py::test_joint_video_audio_tensor_sequence_parallel \ + sglang/multimodal_gen/test/unit/test_kandinsky6_sr_sequence_parallel.py::test_sr_four_gpus + - name: Run diffusion server tests timeout-minutes: 240 env: @@ -490,6 +512,13 @@ jobs: run: | CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion + - name: Install Quant-VideoGen + timeout-minutes: 5 + run: | + # quant-videogen pins an incompatible torch, so install it without deps + # plus the import-time deps of the diffusion-qvg extra. + uv pip install --no-deps quant-videogen==0.1.0 "loguru>=0.7" "termcolor>=2.3" + - name: Run diffusion unit tests timeout-minutes: 60 run: | diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx index 4c1841b619..6296745e95 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K2.7-Code.mdx @@ -510,7 +510,6 @@ Deploy the MXFP4 checkpoint on AMD MI350X/MI355X (reasoning and tool-call parser SGLANG_USE_AITER=1 \ HIP_FORCE_DEV_KERNARG=1 \ SGLANG_EXPERT_PARALLEL_SIZE=1 \ -SGLANG_USE_DYNAMIC_MXFP4_LINEAR=0 \ TORCH_BLAS_PREFER_HIPBLASLT=1 \ TENSILE_STREAMK_DYNAMIC_GRID=6 \ AITER_QUICK_REDUCE_QUANTIZATION=INT4 \ diff --git a/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx b/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx index 51155544bd..2796d7ca1d 100644 --- a/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx +++ b/docs/cookbook/diffusion/LingBot-World/LingBot-World-2.0.mdx @@ -108,7 +108,7 @@ Send this MessagePack map immediately after the WebSocket opens. | `max_chunks` | integer | No | Stop after this many chunks. Omit for a continuous session. | | `realtime_causal_sink_size` | integer | No | Number of sink frames/tokens retained in the causal attention window. | | `realtime_causal_kv_cache_num_frames` | integer | No | Number of recent frames retained in the causal KV cache window. | -| `realtime_output_format` | `"webp"`, `"jpeg"`, `"raw"` | No | Preview/output transport. `webp` and `jpeg` send encoded preview frames; `raw` sends raw RGB; omit for lossless delta-gzip RGB. | +| `realtime_output_format` | `"webp"`, `"jpeg"`, `"raw"` | No | Preview/output transport. `webp` and `jpeg` send encoded preview frames; `raw` (the default when omitted) sends lossless raw RGB. | | `output_compression` | integer | No | Preview quality for `webp` or `jpeg`, from `1` to `100`. | | `enable_upscaling` | boolean | No | Enable server-side super resolution after frame decode. | | `upscaling_scale` | integer | No | Super-resolution scale. Current default is `4` when upscaling is enabled. | diff --git a/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx b/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx index 8d43bebaab..e4670ff571 100644 --- a/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx +++ b/docs/cookbook/diffusion/LingBot-World/LingBot-World.mdx @@ -103,7 +103,7 @@ Send this MessagePack map immediately after the WebSocket opens. | `max_chunks` | integer | No | Stop after this many chunks. Omit for a continuous session. | | `realtime_causal_sink_size` | integer | No | Number of sink frames/tokens retained in the causal attention window. | | `realtime_causal_kv_cache_num_frames` | integer | No | Number of recent frames retained in the causal KV cache window. | -| `realtime_output_format` | `"webp"`, `"jpeg"`, `"raw"` | No | Preview/output transport. `webp` and `jpeg` send encoded preview frames; `raw` sends raw RGB; omit for lossless delta-gzip RGB. | +| `realtime_output_format` | `"webp"`, `"jpeg"`, `"raw"` | No | Preview/output transport. `webp` and `jpeg` send encoded preview frames; `raw` (the default when omitted) sends lossless raw RGB. | | `output_compression` | integer | No | Preview quality for `webp` or `jpeg`, from `1` to `100`. | | `enable_upscaling` | boolean | No | Enable server-side super resolution after frame decode. | | `upscaling_scale` | integer | No | Super-resolution scale. Current default is `4` when upscaling is enabled. | diff --git a/docs/cookbook/diffusion/SANA-WM/SANA-WM.mdx b/docs/cookbook/diffusion/SANA-WM/SANA-WM.mdx index ed7d60add5..bae8b26bd4 100644 --- a/docs/cookbook/diffusion/SANA-WM/SANA-WM.mdx +++ b/docs/cookbook/diffusion/SANA-WM/SANA-WM.mdx @@ -331,14 +331,13 @@ Header fields: | `type` | `"frame_batch"` (always) | | `request_id` | Generation id | | `chunk_index` | Chunk index | -| `content_type` | `application/x-raw-rgb`, `application/x-raw-rgb-delta-gzip`, `image/webp`, or `image/jpeg` | +| `content_type` | `application/x-raw-rgb`, `image/webp`, or `image/jpeg` | | `num_frames` | Frames in this batch | -| `total_size` | Payload size in bytes (`len(payload)` — the compressed size for delta-gzip) | +| `total_size` | Payload size in bytes (`len(payload)`) | | `width`, `height`, `channels` | Frame geometry (`channels: 3`) | -| `bytes_per_frame` | Bytes per uncompressed frame (`width*height*3`) | +| `bytes_per_frame` | Bytes per raw frame (`width*height*3`) | | `format` | `rgb24` for raw | -| `encoding` | `raw`, `delta-gzip`, `webp`, or `jpeg` | -| `delta_reference` | `previous-frame` (present for delta-gzip) | +| `encoding` | `raw`, `webp`, or `jpeg` | | `event_id` | Echoes the steering event id; **omitted** from the header for INIT-only chunks | | `frame_batch_index`, `num_frame_batches` | Sequence multiple batches within a chunk | | `is_final_frame_batch` | `true` ends the chunk | @@ -348,29 +347,24 @@ Header fields: "type": "frame_batch", "request_id": "uuid-string", "chunk_index": 0, - "content_type": "application/x-raw-rgb-delta-gzip", + "content_type": "application/x-raw-rgb", "num_frames": 3, - "total_size": 1048576, + "total_size": 8110080, "width": 1280, "height": 704, "channels": 3, "bytes_per_frame": 2703360, "format": "rgb24", - "encoding": "delta-gzip", - "delta_reference": "previous-frame", + "encoding": "raw", "event_id": 1, "frame_batch_index": 0, "num_frame_batches": 1, "is_final_frame_batch": true, - "payload": "" + "payload": "" } ``` -**Encodings.** `application/x-raw-rgb` is uncompressed RGB24 (3 × uint8, `bytes_per_frame = width*height*3`). `application/x-raw-rgb-delta-gzip` is the zlib-compressed **per-frame XOR delta** against the preceding frame (each frame in the batch is XOR'd against the previous one; sent by default). `realtime_output_format: "raw"` forces uncompressed RGB; `"webp"` / `"jpeg"` send preview-encoded frames. - - -delta-gzip must be restored **frame-by-frame**: decompress the payload, then for each frame XOR it against the already-restored previous frame (the first frame of a batch references the last frame of the previous batch). See `restore_delta_gzip_raw_rgb_payload` in `runtime/realtime/video.py`. The `"raw"` format below avoids this. - +**Encodings.** `application/x-raw-rgb` is uncompressed RGB24 (3 × uint8, `bytes_per_frame = width*height*3`), sent by default or with `realtime_output_format: "raw"`; `"webp"` / `"jpeg"` send preview-encoded frames. ### Minimal client example diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx index 33c57fd0b2..c6ef4258b5 100644 --- a/docs/docs/references/environment_variables.mdx +++ b/docs/docs/references/environment_variables.mdx @@ -292,11 +292,6 @@ SGLang supports various environment variables that can be used to configure its Enable precompilation of DeepGEMM kernels `"true"` - - `SGLANG_JIT_DEEPGEMM_COMPILE_WORKERS` - Number of workers for parallel DeepGEMM kernel compilation - `4` - `SGLANG_IN_DEEPGEMM_PRECOMPILE_STAGE` Indicator flag used during the DeepGEMM precompile script @@ -309,7 +304,7 @@ SGLang supports various environment variables that can be used to configure its SGLANG_DG_USE_NVRTC - Use NVRTC (instead of Triton) for JIT compilation (Experimental) + Use NVRTC instead of NVCC for DeepGEMM JIT compilation (Experimental) "false" @@ -929,11 +924,6 @@ SGLang supports various environment variables that can be used to configure its Enable CPU-side quantization. false - - SGLANG_USE_DYNAMIC_MXFP4_LINEAR - Use dynamic MXFP4 quantization for linear layers. - false - USE_TRITON_W8A8_FP8_KERNEL Use the Triton W8A8 FP8 kernel. diff --git a/docs/docs/sglang-diffusion/environment_variables.mdx b/docs/docs/sglang-diffusion/environment_variables.mdx index 337ced9459..80b8ce404c 100644 --- a/docs/docs/sglang-diffusion/environment_variables.mdx +++ b/docs/docs/sglang-diffusion/environment_variables.mdx @@ -33,16 +33,6 @@ description: "Configure SGLang diffusion behavior with environment variables." not set Comma-separated allowlist for installed sglang.multimodal_gen.plugins hooks. When unset, all discovered diffusion hooks load. - - SGLANG_DIFFUSION_TARGET_DEVICE - cuda - Target device for inference (cuda, rocm, xpu, npu, musa, mps, cpu) - - - SGLANG_DIFFUSION_ATTENTION_BACKEND - not set - Override attention backend via env var (e.g. fa, torch_sdpa, sage_attn) - SGLANG_DIFFUSION_ATTENTION_CONFIG not set @@ -53,11 +43,6 @@ description: "Configure SGLang diffusion behavior with environment variables." false Enable per-stage timing logs - - SGLANG_DIFFUSION_SERVER_DEV_MODE - false - Enable dev-only HTTP endpoints for debugging - SGLANG_DIFFUSION_TORCH_PROFILER_DIR not set @@ -68,16 +53,6 @@ description: "Configure SGLang diffusion behavior with environment variables." ~/.cache/sgl_diffusion Root directory for cache files - - SGLANG_DIFFUSION_CONFIG_ROOT - ~/.config/sgl_diffusion - Root directory for configuration files - - - SGLANG_DIFFUSION_LOGGING_LEVEL - INFO - Default logging level - SGLANG_DIFFUSION_IPC_A2A true diff --git a/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx index 4d3cff8386..ec7b5b88aa 100644 --- a/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx +++ b/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx @@ -137,7 +137,6 @@ export const KimiK27CodeDeployment = () => { 'SGLANG_USE_AITER=1', 'HIP_FORCE_DEV_KERNARG=1', 'SGLANG_EXPERT_PARALLEL_SIZE=1', - 'SGLANG_USE_DYNAMIC_MXFP4_LINEAR=0', 'TORCH_BLAS_PREFER_HIPBLASLT=1', 'TENSILE_STREAMK_DYNAMIC_GRID=6', 'AITER_QUICK_REDUCE_QUANTIZATION=INT4', diff --git a/python/sglang/kernels/README.md b/python/sglang/kernels/README.md index a73e6e81a9..4be13dafb7 100644 --- a/python/sglang/kernels/README.md +++ b/python/sglang/kernels/README.md @@ -4,8 +4,8 @@ This package is the public in-tree import surface for callable kernels, per [RFC #29630](https://github.com/sgl-project/sglang/issues/29630). ```python -from sglang.kernels.ops.layernorm import rmsnorm from sglang.kernels.ops.activation import silu_and_mul +from sglang.kernels.ops.gemm import fp8_scaled_mm from sglang.kernels.ops.kvcache import reshape_and_cache_flash ``` @@ -13,8 +13,8 @@ from sglang.kernels.ops.kvcache import reshape_and_cache_flash ``` sglang/kernels/ - spec.py # KernelSpec, KernelBackend, FormatSignature, - # CapabilityRequirement, PlatformInfo + spec.py # KernelSpec, KernelBackend, CapabilityRequirement, + # PlatformInfo registry.py # process-wide KernelRegistry + register_kernel() selector.py # heuristic select_kernel() and cached get_kernel() fused_op.py # BaseFusedOp: per-operator multi-backend contract @@ -94,7 +94,7 @@ explicitly, e.g.: ```python from sglang.kernels import select_kernel, KernelBackend -jit_rmsnorm = select_kernel("layernorm.rmsnorm", backend=KernelBackend.JIT).load() +jit_moe_align = select_kernel("moe.moe_align_block_size", backend=KernelBackend.JIT).load() ``` ## `BaseFusedOp` — the unified operator contract @@ -128,8 +128,11 @@ Dispatch priority, highest first: explicit `forward(..., backend=...)` → global forced backend (`SGLANG_FORCE_FUSED_OP_BACKEND`) → OOT platform override → declared optimized kernel backends by `priority` (filtered by `backend_eligible()`, a `CapabilityRequirement`-vs-`PlatformInfo` check -extensible with per-call shape/dtype gates) → platform-specific forward → -`forward_native`. The static part of the decision is resolved once and cached +extensible with per-call applicability gates) → platform-specific forward → +`forward_native`. A gate only answers whether the kernel covers these operands, +so that falling through is a correct way to compute the same result; operands +that violate the op's contract (mismatched shapes or devices) are a caller bug +that the backend's launcher must raise, not a reason to fall back. The static part of the decision is resolved once and cached on the instance, so the hot path stays a single indirect call. `BaseFusedOp` also owns the torch.compile mode protocol @@ -148,7 +151,7 @@ What this buys (see the - **Unified correctness testing** — a generic harness enumerates `available_backends()` and asserts each one matches `forward_native` - (`test/registered/kernels/ops/layernorm/test_fused_op_gpu_parity.py`); new + (`test/registered/kernels/ops/activation/test_fused_op_gpu_parity.py`); new backends are picked up automatically. - **One-switch debugging** — `SGLANG_FORCE_FUSED_OP_BACKEND=torch` (or `set_fused_op_backend(KernelBackend.TORCH)`) flips *every* fused op to its diff --git a/python/sglang/kernels/__init__.py b/python/sglang/kernels/__init__.py index 40ef0629a5..420ba0fef6 100644 --- a/python/sglang/kernels/__init__.py +++ b/python/sglang/kernels/__init__.py @@ -3,8 +3,8 @@ SGLang runtime code and tests should import callable kernels from ``sglang.kernels.ops.``, e.g.:: - from sglang.kernels.ops.layernorm import rmsnorm from sglang.kernels.ops.activation import silu_and_mul + from sglang.kernels.ops.gemm import fp8_scaled_mm from sglang.kernels.ops.kvcache import reshape_and_cache_flash Implementations still live in ``sglang.kernels.jit`` (JIT CUDA), the @@ -39,7 +39,6 @@ from sglang.kernels.selector import get_kernel, select_kernel from sglang.kernels.spec import ( CapabilityRequirement, DeviceType, - FormatSignature, KernelBackend, KernelSpec, PlatformInfo, @@ -57,7 +56,6 @@ __all__ = [ "CapabilityRequirement", "DeviceType", "capabilities_satisfied", - "FormatSignature", "FusedOpTraceRecord", "KernelBackend", "KernelRegistry", diff --git a/python/sglang/kernels/aot/benchmark/bench_dsv4_norm_rope.py b/python/sglang/kernels/aot/benchmark/bench_dsv4_norm_rope.py deleted file mode 100644 index 3e52c415df..0000000000 --- a/python/sglang/kernels/aot/benchmark/bench_dsv4_norm_rope.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Benchmark for DeepSeek-V4 fused norm + RoPE kernels.""" - -import itertools - -import sgl_kernel -import torch -import triton -import triton.testing - -try: - from sglang.utils import is_in_ci - - IS_CI = is_in_ci() -except ImportError: - IS_CI = False - -batch_sizes = [1] if IS_CI else [1, 4, 16, 64, 256] -num_heads_list = [8] if IS_CI else [8, 16, 64] -head_dims = [192] if IS_CI else [128, 192] - -configs = list(itertools.product(batch_sizes, num_heads_list, head_dims)) - - -def torch_rmsnorm_rope( - q: torch.Tensor, freqs_cis: torch.Tensor, positions: torch.Tensor, eps: float -) -> torch.Tensor: - """Naive PyTorch reference: RMSNorm + RoPE.""" - rms = torch.sqrt(q.float().pow(2).mean(dim=-1, keepdim=True) + eps) - q_normed = (q.float() / rms).to(q.dtype) - return q_normed - - -@triton.testing.perf_report( - triton.testing.Benchmark( - x_names=["batch_size", "num_heads", "head_dim"], - x_vals=configs, - line_arg="provider", - line_vals=["sglang", "torch"], - line_names=["SGL Kernel", "PyTorch"], - styles=[("green", "-"), ("red", "--")], - ylabel="µs (median)", - plot_name="dsv4-q-norm-rope-performance", - args={}, - ) -) -def benchmark_q_norm_rope(batch_size, num_heads, head_dim, provider): - torch.manual_seed(42) - eps = 1e-6 - max_pos = 8192 - rope_dim = 64 - - q_input = torch.randn( - batch_size, num_heads, head_dim, dtype=torch.bfloat16, device="cuda" - ) - q_output = torch.empty_like(q_input) - freqs_cis = torch.randn(max_pos, rope_dim, dtype=torch.float32, device="cuda") - positions = torch.randint( - 0, max_pos, (batch_size,), dtype=torch.int32, device="cuda" - ) - - if provider == "sglang": - fn = lambda: sgl_kernel.dsv4_fused_q_norm_rope( - q_input, freqs_cis, positions, eps, q_output - ) - else: - fn = lambda: torch_rmsnorm_rope(q_input, freqs_cis, positions, eps) - - ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( - fn, quantiles=[0.5, 0.2, 0.8] - ) - return 1000 * ms, 1000 * max_ms, 1000 * min_ms - - -if __name__ == "__main__": - benchmark_q_norm_rope.run(print_data=True) diff --git a/python/sglang/kernels/aot/benchmark/bench_moe_ep_post_reorder.py b/python/sglang/kernels/aot/benchmark/bench_moe_ep_post_reorder.py deleted file mode 100644 index 1aa5177aaa..0000000000 --- a/python/sglang/kernels/aot/benchmark/bench_moe_ep_post_reorder.py +++ /dev/null @@ -1,85 +0,0 @@ -import torch -import triton - -from sglang.kernels.ops.moe.ep_moe_kernels import post_reorder_triton_kernel -from sglang.utils import is_in_ci - -IS_CI = is_in_ci() - -# CI environment uses simplified parameters -if IS_CI: - batch_sizes = [64, 128] # Only test 2 values in CI -else: - batch_sizes = [64, 128, 256, 512, 640, 768, 1024, 2048, 4096] - -configs = [(bs,) for bs in batch_sizes] - - -@triton.testing.perf_report( - triton.testing.Benchmark( - x_names=["batch_size"], - x_vals=[list(_) for _ in configs], - line_arg="provider", - line_vals=["triton"], - line_names=["Triton Kernel"], - styles=[("orange", "-")], - ylabel="us", - plot_name="ep-moe-post-reorder-performance", - args={}, - ) -) -def benchmark(batch_size, provider): - dtype = torch.bfloat16 - device = torch.device("cuda") - hidden_size, topk, start_expert_id, end_expert_id, block_size = 4096, 8, 0, 255, 512 - - def alloc_tensors(): - down_output = torch.randn( - batch_size * topk, hidden_size, dtype=dtype, device=device - ) - output = torch.zeros(batch_size, hidden_size, dtype=dtype, device=device) - src2dst = torch.randint( - 0, batch_size * topk, (batch_size, topk), dtype=torch.int32, device=device - ) - topk_ids = torch.randint( - start_expert_id, - end_expert_id + 1, - (batch_size, topk), - dtype=torch.int32, - device=device, - ) - topk_weights = torch.rand(batch_size, topk, dtype=dtype, device=device) - return down_output, output, src2dst, topk_ids, topk_weights - - quantiles = [0.5, 0.2, 0.8] - - if provider == "triton": - d_out, out, s2d, tk_ids, tk_weights = alloc_tensors() - - def run_triton(): - post_reorder_triton_kernel[(batch_size,)]( - d_out.view(-1), - out.view(-1), - s2d.view(-1), - tk_ids.view(-1), - tk_weights.view(-1), - start_expert_id, - end_expert_id, - topk, - hidden_size, - 0, - block_size, - ) - - ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( - run_triton, quantiles=quantiles - ) - - else: - raise ValueError(f"Unknown provider: {provider}") - - return 1000 * ms, 1000 * max_ms, 1000 * min_ms - - -if __name__ == "__main__": - benchmark.run(print_data=True) diff --git a/python/sglang/kernels/aot/benchmark/bench_per_token_group_quant_8bit.py b/python/sglang/kernels/aot/benchmark/bench_per_token_group_quant_8bit.py index 9023718a4b..1e22a544e9 100644 --- a/python/sglang/kernels/aot/benchmark/bench_per_token_group_quant_8bit.py +++ b/python/sglang/kernels/aot/benchmark/bench_per_token_group_quant_8bit.py @@ -5,14 +5,12 @@ import torch import triton from sgl_kernel.test_utils import create_per_token_group_quant_test_data -from sglang.kernels.ops.quantization.fp8_kernel import ( - per_token_group_quant_8bit as triton_per_token_group_quant_8bit, -) -from sglang.kernels.ops.quantization.fp8_kernel import ( - sglang_per_token_group_quant_8bit, -) from sglang.srt.utils import is_hip from sglang.srt.utils.bench_utils import bench_kineto +from sglang.test.kernels.per_token_group_quant import ( + sglang_per_token_group_quant_8bit, + triton_per_token_group_quant_8bit, +) from sglang.utils import is_in_ci IS_CI = is_in_ci() diff --git a/python/sglang/kernels/aot/csrc/common_extension.cc b/python/sglang/kernels/aot/csrc/common_extension.cc index 6f146de442..1b8d68cc17 100644 --- a/python/sglang/kernels/aot/csrc/common_extension.cc +++ b/python/sglang/kernels/aot/csrc/common_extension.cc @@ -178,16 +178,7 @@ TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { m.def("apply_shuffle_mul_sum(Tensor input, Tensor output, Tensor permutation, Tensor? factors) -> ()"); m.impl("apply_shuffle_mul_sum", torch::kCUDA, &apply_shuffle_mul_sum); - // DeepSeek-V4 fused norm + rope - m.def( - "dsv4_fused_q_norm_rope(Tensor q_input, Tensor! q_output, Tensor freqs_cis, Tensor positions, float eps) -> ()"); - m.impl("dsv4_fused_q_norm_rope", torch::kCUDA, &dsv4_fused_q_norm_rope); - - m.def( - "dsv4_fused_k_norm_rope_flashmla(Tensor kv, Tensor kv_weight, Tensor freqs_cis, Tensor positions, " - "Tensor out_loc, Tensor! kvcache, float eps, int page_size) -> ()"); - m.impl("dsv4_fused_k_norm_rope_flashmla", torch::kCUDA, &dsv4_fused_k_norm_rope_flashmla); - + // DeepSeek-V4 fused indexer rope + hadamard + quant m.def( "dsv4_fused_q_indexer_rope_hadamard_quant(Tensor q_input, Tensor! q_fp8, Tensor weight, " "Tensor! weights_out, float weight_scale, Tensor freqs_cis, Tensor positions) -> ()"); diff --git a/python/sglang/kernels/aot/csrc/common_extension_rocm.cc b/python/sglang/kernels/aot/csrc/common_extension_rocm.cc index fa2d5a7cfb..e3cd24f201 100644 --- a/python/sglang/kernels/aot/csrc/common_extension_rocm.cc +++ b/python/sglang/kernels/aot/csrc/common_extension_rocm.cc @@ -52,15 +52,6 @@ TORCH_LIBRARY_EXPAND(sgl_kernel, m) { "page_indices, int page_size, Tensor!? raw_indices, bool sort_output=False) -> ()"); m.impl("deepseek_v4_topk_transform_512", torch::kCUDA, &deepseek_v4_topk_transform_512); - m.def( - "dsv4_fused_q_norm_rope(Tensor q_input, Tensor! q_output, Tensor freqs_cis, Tensor positions, float eps) -> ()"); - m.impl("dsv4_fused_q_norm_rope", torch::kCUDA, &dsv4_fused_q_norm_rope); - - m.def( - "dsv4_fused_k_norm_rope_flashmla(Tensor kv, Tensor kv_weight, Tensor freqs_cis, Tensor positions, " - "Tensor out_loc, Tensor! kvcache, float eps, int page_size) -> ()"); - m.impl("dsv4_fused_k_norm_rope_flashmla", torch::kCUDA, &dsv4_fused_k_norm_rope_flashmla); - m.def( "dsv4_fused_q_indexer_rope_hadamard_quant(Tensor q_input, Tensor! q_fp8, Tensor weight, " "Tensor! weights_out, float weight_scale, Tensor freqs_cis, Tensor positions) -> ()"); diff --git a/python/sglang/kernels/aot/csrc/elementwise/dsv4_norm_rope.cu b/python/sglang/kernels/aot/csrc/elementwise/dsv4_norm_rope.cu index a69802c4c8..af4ecab9a8 100644 --- a/python/sglang/kernels/aot/csrc/elementwise/dsv4_norm_rope.cu +++ b/python/sglang/kernels/aot/csrc/elementwise/dsv4_norm_rope.cu @@ -13,9 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// DeepSeek-V4 fused norm + RoPE kernels, ported from JIT kernel -// python/sglang/kernels/jit/csrc/deepseek_v4/main_norm_rope.cuh -// to sgl-kernel AOT compilation with CUDA + HIP (ROCm) support. +// DeepSeek-V4 indexer Q RoPE + Hadamard + FP8 quant kernel, ported from the JIT kernel in +// python/sglang/kernels/jit/csrc/deepseek_v4/main_norm_rope.cuh for the ROCm build. #ifndef USE_ROCM #include @@ -59,14 +58,6 @@ using fp8x2_e4m3_t = uint16_t; static constexpr uint32_t kWarpSize = 32; -template -__device__ __forceinline__ float warp_reduce_sum(float val) { -#pragma unroll - for (uint32_t mask = kNumThreads / 2; mask > 0; mask >>= 1) - val += SGLANG_SHFL_XOR_SYNC(FULL_MASK, val, mask); - return val; -} - __device__ __forceinline__ float warp_reduce_max(float val) { #pragma unroll for (uint32_t mask = kWarpSize / 2; mask > 0; mask >>= 1) @@ -96,31 +87,10 @@ __device__ __forceinline__ float bf16_to_float(bf16_t v) { return __bfloat162float(v); } -__device__ __forceinline__ bf16_t float_to_bf16(float v) { -#ifndef USE_ROCM - return __float2bfloat16_rn(v); -#else - return __float2bfloat16(v); -#endif -} - // ============================================================================ // FP8 E4M3 helpers (portable CUDA + HIP) // ============================================================================ -// UE8M0 scale: round a positive float to the nearest power-of-two -// representable in UE8M0 (unsigned 8-bit exponent, no mantissa). -__device__ __forceinline__ int32_t cast_to_ue8m0(float x) { - uint32_t u = __float_as_uint(x); - int32_t exp = static_cast((u >> 23) & 0xFFu); - uint32_t mant = u & 0x7FFFFFu; - return exp + (mant != 0); -} - -__device__ __forceinline__ float inv_scale_ue8m0(int32_t exp) { - return __uint_as_float(static_cast((127 + 127 - exp) << 23)); -} - static constexpr float kFP8Max = 448.0f; #ifndef USE_ROCM @@ -188,236 +158,13 @@ __device__ __forceinline__ fp8x2_e4m3_t pack_fp8(float x, float y) { } #endif -// ============================================================================ -// Kernel 1: Fused Q Norm + RoPE -// warp-per-(token, head), rmsnorm-self (no weight) + RoPE + write to q_out. -// ============================================================================ - namespace { constexpr uint32_t kFusedQBlockSize = 128; constexpr uint32_t kFusedQNumWarps = kFusedQBlockSize / kWarpSize; -constexpr uint32_t kFusedKBlockSize = 256; -constexpr uint32_t kFusedKNumWarps = kFusedKBlockSize / kWarpSize; - -struct FusedQNormRopeParams { - const void* __restrict__ q_input; - void* __restrict__ q_output; - const float* __restrict__ freqs_cis; - const int32_t* __restrict__ positions; - int64_t q_input_stride_batch; - int64_t q_output_stride_batch; - uint32_t batch_size; - uint32_t num_q_heads; - float eps; -}; - -// Compute the largest power-of-2 vec size that divides both kHeadDim and -// fits in 16 bytes, while also dividing kRopeDim. -template -struct QKernelTraits { - static constexpr int64_t kMaxVecSize = 16 / sizeof(bf16_t); // 8 - // Use kRopeDim/kWarpSize (=2 for kRopeDim=64) as the vec size. - // This guarantees kRopeDim % kVecSize == 0 and works for all head dims - // that are multiples of kWarpSize*kVecSize. - static constexpr int64_t kVecSize = kRopeDim / kWarpSize; // 2 - static constexpr int64_t kLocalSize = kHeadDim / (kWarpSize * kVecSize); - static constexpr uint32_t kRopeSize = kRopeDim / kVecSize; - static_assert(kHeadDim % (kWarpSize * kVecSize) == 0); - static_assert(kRopeDim % kVecSize == 0); - static_assert(kRopeDim == kWarpSize * 2, "1 (real, imag) pair per lane"); -}; - -template -__global__ __launch_bounds__(kFusedQBlockSize, 16) void fused_q_norm_rope_kernel( - const __grid_constant__ FusedQNormRopeParams params) { - using Traits = QKernelTraits; - constexpr int64_t kVecSize = Traits::kVecSize; - constexpr int64_t kLocalSize = Traits::kLocalSize; - constexpr uint32_t kRopeSize = Traits::kRopeSize; - - using Storage = AlignedVec; - using Float2 = AlignedVec; - - const auto warp_id = threadIdx.x / kWarpSize; - const auto lane_id = threadIdx.x % kWarpSize; - const auto work_id = blockIdx.x * kFusedQNumWarps + warp_id; - - const uint32_t total_works = params.batch_size * params.num_q_heads; - if (work_id >= total_works) return; - - const uint32_t batch_id = work_id / params.num_q_heads; - const uint32_t head_id = work_id % params.num_q_heads; - const auto input_ptr = - static_cast(params.q_input) + batch_id * params.q_input_stride_batch + head_id * kHeadDim; - const auto output_ptr = - static_cast(params.q_output) + batch_id * params.q_output_stride_batch + head_id * kHeadDim; - const auto position = params.positions[batch_id]; - - __shared__ Storage s_rope[kFusedQNumWarps][kRopeSize]; - - // Prefetch freq pair. - Float2 freq; - freq.load(params.freqs_cis + position * kRopeDim, lane_id); - - // Part 1: rmsnorm-self (no weight). - Storage input_vec[kLocalSize]; -#pragma unroll - for (int i = 0; i < kLocalSize; ++i) { - input_vec[i].load(input_ptr, lane_id + i * kWarpSize); - } - - float sum_of_squares = 0.0f; -#pragma unroll - for (int i = 0; i < kLocalSize; ++i) { -#pragma unroll - for (int j = 0; j < kVecSize; ++j) { - float x = bf16_to_float(input_vec[i][j]); - sum_of_squares += x * x; - } - } - sum_of_squares = warp_reduce_sum(sum_of_squares); - const float norm_factor = rsqrtf(sum_of_squares / static_cast(kHeadDim) + params.eps); - -#pragma unroll - for (int i = 0; i < kLocalSize; ++i) { -#pragma unroll - for (int j = 0; j < kVecSize; ++j) { - float x = bf16_to_float(input_vec[i][j]); - input_vec[i][j] = float_to_bf16(x * norm_factor); - } - } - - // Stash rope tail into shared memory; write nope tiles to gmem. - const bool is_rope_lane = lane_id >= kWarpSize - kRopeSize; -#pragma unroll - for (int i = 0; i < kLocalSize; ++i) { - if (i == kLocalSize - 1 && is_rope_lane) { - const auto rope_id = lane_id - (kWarpSize - kRopeSize); - s_rope[warp_id][rope_id] = input_vec[i]; - } else { - input_vec[i].store(output_ptr, lane_id + i * kWarpSize); - } - } - __syncwarp(); - - // Part 2: RoPE on all 32 lanes -- one (real, imag) bf16x2 pair per lane. - auto elem_ptr = reinterpret_cast(&s_rope[warp_id][0]); - bf16x2_t elem = elem_ptr[lane_id]; -#ifndef USE_ROCM - float2 elem_f = __bfloat1622float2(elem); - float x_real = elem_f.x, x_imag = elem_f.y; -#else - float x_real = __bfloat162float(elem.x), x_imag = __bfloat162float(elem.y); -#endif - float freq_real = freq[0], freq_imag = freq[1]; - float rot_real = x_real * freq_real - x_imag * freq_imag; - float rot_imag = x_real * freq_imag + x_imag * freq_real; - bf16x2_t rotated = __float22bfloat162_rn(make_float2(rot_real, rot_imag)); - auto out_elem = reinterpret_cast(output_ptr + (kHeadDim - kRopeDim)); - out_elem[lane_id] = rotated; -} - // ============================================================================ -// Kernel 2: Fused K Norm + RoPE + FlashMLA Store -// block-per-token, rmsnorm (with kv_weight) + RoPE + FP8 quantized store. -// ============================================================================ - -struct FusedKNormRopeFlashMLAParams { - const void* __restrict__ kv; - const void* __restrict__ kv_weight; - const float* __restrict__ freqs_cis; - const int32_t* __restrict__ positions; - const int32_t* __restrict__ out_loc; - uint8_t* __restrict__ kvcache; - int64_t kv_stride_batch; - uint32_t batch_size; - float eps; -}; - -template -__global__ __launch_bounds__(kFusedKBlockSize, 8) void fused_k_norm_rope_flashmla_kernel( - const __grid_constant__ FusedKNormRopeFlashMLAParams params) { - constexpr int64_t kVecSize = 2; - constexpr uint32_t kRopeWarp = kFusedKNumWarps - 1; - constexpr int64_t kPageBytes = ((584ll << kPageBits) + 575) / 576 * 576; - static_assert(kHeadDim == kFusedKBlockSize * kVecSize); - static_assert(kRopeDim == kWarpSize * kVecSize); - - using Storage = AlignedVec; - - const auto tx = threadIdx.x; - const auto warp_id = tx / kWarpSize; - const auto lane_id = tx % kWarpSize; - const auto work_id = blockIdx.x; - if (work_id >= params.batch_size) return; - - const auto input_ptr = static_cast(params.kv) + work_id * params.kv_stride_batch; - const auto position = params.positions[work_id]; - const auto out_loc = params.out_loc[work_id]; - const auto freqs_cis = params.freqs_cis + position * kRopeDim; - - AlignedVec data, freq; - - // Part 1: norm with block-wide reduction. - { - __shared__ float partial_sums[kFusedKNumWarps]; - - Storage input_vec, weight_vec; - input_vec.load(input_ptr, tx); - weight_vec.load(params.kv_weight, tx); - if (warp_id == kRopeWarp) freq.load(freqs_cis, lane_id); - - float sum_of_squares = 0.0f; -#pragma unroll - for (int i = 0; i < kVecSize; ++i) { - float x = bf16_to_float(input_vec[i]); - sum_of_squares += x * x; - } - const float warp_sum = warp_reduce_sum(sum_of_squares); - if (lane_id == 0) partial_sums[warp_id] = warp_sum; - __syncthreads(); - sum_of_squares = warp_reduce_sum(partial_sums[lane_id % kFusedKNumWarps]); - const float norm_factor = rsqrtf(sum_of_squares / static_cast(kHeadDim) + params.eps); - -#pragma unroll - for (int i = 0; i < kVecSize; ++i) { - float x = bf16_to_float(input_vec[i]); - float w = bf16_to_float(weight_vec[i]); - data[i] = x * norm_factor * w; - } - } - - const int32_t page = out_loc >> kPageBits; - const int32_t offset = out_loc & ((1 << kPageBits) - 1); - const auto page_ptr = params.kvcache + page * kPageBytes; - const auto value_ptr = page_ptr + offset * 576; - - // Part 2: rope on last warp (BF16 store), per-warp UE8M0 quant + store on others. - if (warp_id == kRopeWarp) { - float x_real = data[0], x_imag = data[1]; - float freq_real = freq[0], freq_imag = freq[1]; - float rot_real = x_real * freq_real - x_imag * freq_imag; - float rot_imag = x_real * freq_imag + x_imag * freq_real; - bf16x2_t result = __float22bfloat162_rn(make_float2(rot_real, rot_imag)); - auto rope_ptr = value_ptr + 448; - reinterpret_cast(rope_ptr)[lane_id] = result; - } else { - float x = data[0], y = data[1]; - float abs_max = warp_reduce_max(fmaxf(fabsf(x), fabsf(y))); - float scale_raw = fmaxf(1e-4f, abs_max) / kFP8Max; - int32_t scale_ue8m0 = cast_to_ue8m0(scale_raw); - float inv_scale = inv_scale_ue8m0(scale_ue8m0); - fp8x2_e4m3_t result = pack_fp8(x * inv_scale, y * inv_scale); - auto scale_ptr = page_ptr + (576ll << kPageBits) + offset * 8; - reinterpret_cast(value_ptr)[tx] = result; - if (lane_id == 0) static_cast(scale_ptr)[warp_id] = static_cast(scale_ue8m0); - } -} - -// ============================================================================ -// Kernel 3: Fused Q Indexer RoPE + Hadamard + FP8 Quantization +// Fused Q Indexer RoPE + Hadamard + FP8 Quantization // warp-per-(token, head), no norm, RoPE + Hadamard + fp8 act-quant. // ============================================================================ @@ -537,138 +284,6 @@ __global__ __launch_bounds__(kFusedQBlockSize, 16) void fused_q_indexer_rope_had // Host-side launchers (PyTorch C++ extension API) // ============================================================================ -void dsv4_fused_q_norm_rope( - const at::Tensor& q_input, - at::Tensor& q_output, - const at::Tensor& freqs_cis, - const at::Tensor& positions, - double eps) { - TORCH_CHECK(q_input.is_cuda(), "q_input must be a CUDA tensor"); - TORCH_CHECK(q_output.is_cuda(), "q_output must be a CUDA tensor"); - TORCH_CHECK(q_input.scalar_type() == at::ScalarType::BFloat16, "q_input must be bfloat16"); - TORCH_CHECK(q_output.scalar_type() == at::ScalarType::BFloat16, "q_output must be bfloat16"); - TORCH_CHECK(q_input.dim() == 3, "q_input must be 3D: (B, H, D)"); - TORCH_CHECK(q_output.dim() == 3, "q_output must be 3D: (B, H, D)"); - TORCH_CHECK(positions.scalar_type() == at::ScalarType::Int, "positions must be int32"); - - const int64_t B = q_input.size(0); - const int64_t H = q_input.size(1); - const int64_t D = q_input.size(2); - TORCH_CHECK( - q_output.size(0) == B && q_output.size(1) == H && q_output.size(2) == D, "q_output shape must match q_input"); - TORCH_CHECK(q_input.stride(2) == 1 && q_output.stride(2) == 1, "last dim must be contiguous"); - TORCH_CHECK(q_input.stride(1) == D && q_output.stride(1) == D, "head dim must be contiguous"); - - if (B == 0) return; - - const auto stream = at::cuda::getCurrentCUDAStream(q_input.get_device()); - const auto params = FusedQNormRopeParams{ - .q_input = q_input.data_ptr(), - .q_output = q_output.data_ptr(), - .freqs_cis = freqs_cis.data_ptr(), - .positions = positions.data_ptr(), - .q_input_stride_batch = q_input.stride(0), - .q_output_stride_batch = q_output.stride(0), - .batch_size = static_cast(B), - .num_q_heads = static_cast(H), - .eps = static_cast(eps), - }; - const uint32_t total_works = static_cast(B * H); - const uint32_t num_blocks = CEILDIV(total_works, kFusedQNumWarps); - - // Dispatch on head_dim. DeepSeek V4 uses D=192 with kRopeDim=64. - constexpr int64_t kRopeDim = 64; - switch (D) { - case 128: - fused_q_norm_rope_kernel<128, kRopeDim><<>>(params); - break; - case 192: - fused_q_norm_rope_kernel<192, kRopeDim><<>>(params); - break; - default: - TORCH_CHECK(false, "Unsupported head_dim for dsv4_fused_q_norm_rope: ", D); - } -} - -void dsv4_fused_k_norm_rope_flashmla( - const at::Tensor& kv, - const at::Tensor& kv_weight, - const at::Tensor& freqs_cis, - const at::Tensor& positions, - const at::Tensor& out_loc, - at::Tensor& kvcache, - double eps, - int64_t page_size) { - TORCH_CHECK(kv.is_cuda(), "kv must be a CUDA tensor"); - TORCH_CHECK(kv.scalar_type() == at::ScalarType::BFloat16, "kv must be bfloat16"); - TORCH_CHECK(kv.dim() == 2, "kv must be 2D: (B, D)"); - TORCH_CHECK(positions.scalar_type() == at::ScalarType::Int, "positions must be int32"); - TORCH_CHECK(out_loc.scalar_type() == at::ScalarType::Int, "out_loc must be int32"); - - const int64_t B = kv.size(0); - const int64_t D = kv.size(1); - TORCH_CHECK(D == 512, "kv head_dim must be 512 for FlashMLA"); - TORCH_CHECK(kv_weight.size(0) == D, "kv_weight size must match head_dim"); - - if (B == 0) return; - - const auto stream = at::cuda::getCurrentCUDAStream(kv.get_device()); - const auto params = FusedKNormRopeFlashMLAParams{ - .kv = kv.data_ptr(), - .kv_weight = kv_weight.data_ptr(), - .freqs_cis = freqs_cis.data_ptr(), - .positions = positions.data_ptr(), - .out_loc = out_loc.data_ptr(), - .kvcache = static_cast(kvcache.data_ptr()), - .kv_stride_batch = kv.stride(0), - .batch_size = static_cast(B), - .eps = static_cast(eps), - }; - - constexpr int64_t kHeadDim = 512; - constexpr int64_t kRopeDim = 64; - - // Dispatch on page_size (must be power of 2). - TORCH_CHECK(page_size > 0 && (page_size & (page_size - 1)) == 0, "page_size must be a power of 2"); - -#define LAUNCH_K_KERNEL(PAGE_BITS) \ - fused_k_norm_rope_flashmla_kernel \ - <<(B), kFusedKBlockSize, 0, stream>>>(params) - - switch (page_size) { - case 1: - LAUNCH_K_KERNEL(0); - break; - case 2: - LAUNCH_K_KERNEL(1); - break; - case 4: - LAUNCH_K_KERNEL(2); - break; - case 8: - LAUNCH_K_KERNEL(3); - break; - case 16: - LAUNCH_K_KERNEL(4); - break; - case 32: - LAUNCH_K_KERNEL(5); - break; - case 64: - LAUNCH_K_KERNEL(6); - break; - case 128: - LAUNCH_K_KERNEL(7); - break; - case 256: - LAUNCH_K_KERNEL(8); - break; - default: - TORCH_CHECK(false, "Unsupported page_size: ", page_size); - } -#undef LAUNCH_K_KERNEL -} - void dsv4_fused_q_indexer_rope_hadamard_quant( const at::Tensor& q_input, at::Tensor& q_fp8, diff --git a/python/sglang/kernels/aot/csrc/elementwise/topk.hip b/python/sglang/kernels/aot/csrc/elementwise/topk.hip index 1b08e3e663..81cc991593 100644 --- a/python/sglang/kernels/aot/csrc/elementwise/topk.hip +++ b/python/sglang/kernels/aot/csrc/elementwise/topk.hip @@ -75,8 +75,13 @@ using Ws = CoopMbWorkspace; // which is the 32768/b1 line above -- 18% on a case the kernel already wins by 2x. int row_split(int batch, int64_t row_len_hint) { // Escape hatch for benchmarking the two paths against each other; 0 forces the - // one-block kernel. An empty value means unset, not zero. - if (const char* e = std::getenv("SGL_DSA_TOPK_ROW_SPLIT"); e != nullptr && *e != '\0') { + // one-block kernel. An empty value means unset, not zero. The legacy SGL_ name is + // read here too: without sglang's environ import nothing rewrites it to SGLANG_. + const char* e = std::getenv("SGLANG_DSA_TOPK_ROW_SPLIT"); + if (e == nullptr || *e == '\0') { + e = std::getenv("SGL_DSA_TOPK_ROW_SPLIT"); + } + if (e != nullptr && *e != '\0') { const int v = std::atoi(e); if (v >= 0) { return v; diff --git a/python/sglang/kernels/aot/include/sgl_kernel_ops.h b/python/sglang/kernels/aot/include/sgl_kernel_ops.h index 65c274b86c..0a2ce7a3b0 100644 --- a/python/sglang/kernels/aot/include/sgl_kernel_ops.h +++ b/python/sglang/kernels/aot/include/sgl_kernel_ops.h @@ -175,25 +175,8 @@ void deepseek_v4_topk_transform_512( #endif /* - * From csrc/elementwise (DeepSeek-V4 norm + rope) + * From csrc/elementwise (DeepSeek-V4 indexer rope + hadamard quant) */ -void dsv4_fused_q_norm_rope( - const at::Tensor& q_input, - at::Tensor& q_output, - const at::Tensor& freqs_cis, - const at::Tensor& positions, - double eps); - -void dsv4_fused_k_norm_rope_flashmla( - const at::Tensor& kv, - const at::Tensor& kv_weight, - const at::Tensor& freqs_cis, - const at::Tensor& positions, - const at::Tensor& out_loc, - at::Tensor& kvcache, - double eps, - int64_t page_size); - void dsv4_fused_q_indexer_rope_hadamard_quant( const at::Tensor& q_input, at::Tensor& q_fp8, @@ -316,25 +299,8 @@ void apply_shuffle_mul_sum( const std::optional& factors); /* - * From csrc/elementwise (DeepSeek-V4 norm + rope) + * From csrc/elementwise (DeepSeek-V4 indexer rope + hadamard quant) */ -void dsv4_fused_q_norm_rope( - const at::Tensor& q_input, - at::Tensor& q_output, - const at::Tensor& freqs_cis, - const at::Tensor& positions, - double eps); - -void dsv4_fused_k_norm_rope_flashmla( - const at::Tensor& kv, - const at::Tensor& kv_weight, - const at::Tensor& freqs_cis, - const at::Tensor& positions, - const at::Tensor& out_loc, - at::Tensor& kvcache, - double eps, - int64_t page_size); - void dsv4_fused_q_indexer_rope_hadamard_quant( const at::Tensor& q_input, at::Tensor& q_fp8, diff --git a/python/sglang/kernels/aot/python/sgl_kernel/__init__.py b/python/sglang/kernels/aot/python/sgl_kernel/__init__.py index 2820b64997..17c7f1bad2 100644 --- a/python/sglang/kernels/aot/python/sgl_kernel/__init__.py +++ b/python/sglang/kernels/aot/python/sgl_kernel/__init__.py @@ -32,9 +32,7 @@ else: concat_mla_absorb_q, concat_mla_k, copy_to_gpu_no_ce, - dsv4_fused_k_norm_rope_flashmla, dsv4_fused_q_indexer_rope_hadamard_quant, - dsv4_fused_q_norm_rope, fused_add_rmsnorm, gelu_and_mul, gelu_tanh_and_mul, @@ -155,9 +153,7 @@ else: "concat_mla_absorb_q", "concat_mla_k", "copy_to_gpu_no_ce", - "dsv4_fused_k_norm_rope_flashmla", "dsv4_fused_q_indexer_rope_hadamard_quant", - "dsv4_fused_q_norm_rope", "es_fp8_blockwise_scaled_grouped_mm", "es_sm100_mxfp8_blockscaled_grouped_mm", "es_sm100_mxfp8_blockscaled_grouped_quant", diff --git a/python/sglang/kernels/aot/python/sgl_kernel/elementwise.py b/python/sglang/kernels/aot/python/sgl_kernel/elementwise.py index d2315761c7..4e6668db1b 100644 --- a/python/sglang/kernels/aot/python/sgl_kernel/elementwise.py +++ b/python/sglang/kernels/aot/python/sgl_kernel/elementwise.py @@ -324,59 +324,6 @@ if torch.version.hip is not None: return out -def dsv4_fused_q_norm_rope( - q_input: torch.Tensor, - freqs_cis: torch.Tensor, - positions: torch.Tensor, - eps: float = 1e-6, - q_output: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """DeepSeek-V4 fused Q RMSNorm (no weight) + RoPE. - - Parameters - ---------- - q_input : (B, num_q_heads, head_dim) bfloat16 - freqs_cis: (max_pos, rope_dim) float32, re/im interleaved - positions: (B,) int32 - eps : RMSNorm epsilon - q_output : optional pre-allocated output tensor - """ - if q_output is None: - q_output = torch.empty_like(q_input) - torch.ops.sgl_kernel.dsv4_fused_q_norm_rope.default( - q_input, q_output, freqs_cis, positions, eps - ) - return q_output - - -def dsv4_fused_k_norm_rope_flashmla( - kv: torch.Tensor, - kv_weight: torch.Tensor, - freqs_cis: torch.Tensor, - positions: torch.Tensor, - out_loc: torch.Tensor, - kvcache: torch.Tensor, - eps: float = 1e-6, - page_size: int = 1, -) -> None: - """DeepSeek-V4 fused K RMSNorm + RoPE + FlashMLA FP8 store. - - Parameters - ---------- - kv : (B, 512) bfloat16 - kv_weight: (512,) bfloat16 - freqs_cis: (max_pos, 64) float32 - positions: (B,) int32 - out_loc : (B,) int32 cache slot ids - kvcache : (npages, page_bytes) uint8 - eps : RMSNorm epsilon - page_size: page size (power of 2) - """ - torch.ops.sgl_kernel.dsv4_fused_k_norm_rope_flashmla.default( - kv, kv_weight, freqs_cis, positions, out_loc, kvcache, eps, page_size - ) - - def dsv4_fused_q_indexer_rope_hadamard_quant( q_input: torch.Tensor, q_fp8: torch.Tensor, diff --git a/python/sglang/kernels/aot/python/sgl_kernel/utils.py b/python/sglang/kernels/aot/python/sgl_kernel/utils.py index 71c5679239..b1ea28fb36 100644 --- a/python/sglang/kernels/aot/python/sgl_kernel/utils.py +++ b/python/sglang/kernels/aot/python/sgl_kernel/utils.py @@ -14,21 +14,9 @@ # ============================================================================== import functools -from typing import Dict, Tuple import torch -_cache_buf: Dict[Tuple[str, torch.device], torch.Tensor] = {} - - -def _get_cache_buf(name: str, bytes: int, device: torch.device) -> torch.Tensor: - key = (name, device) - buf = _cache_buf.get(key) - if buf is None: - buf = torch.empty(bytes, dtype=torch.uint8, device=device) - _cache_buf[key] = buf - return buf - def _to_tensor_scalar_tuple(x): if isinstance(x, torch.Tensor): diff --git a/python/sglang/kernels/aot/tests/test_dsv4_norm_rope.py b/python/sglang/kernels/aot/tests/test_dsv4_norm_rope.py index 6ea444bdf7..d36680f78c 100644 --- a/python/sglang/kernels/aot/tests/test_dsv4_norm_rope.py +++ b/python/sglang/kernels/aot/tests/test_dsv4_norm_rope.py @@ -1,94 +1,10 @@ -"""Tests for DeepSeek-V4 fused norm + RoPE kernels.""" +"""Tests for the DeepSeek-V4 fused indexer Q RoPE + Hadamard + FP8 quant kernel.""" import pytest import sgl_kernel import torch -def _ref_rmsnorm_self(x: torch.Tensor, eps: float) -> torch.Tensor: - """Reference: RMSNorm without weight (identity weight).""" - rms = torch.sqrt(x.float().pow(2).mean(dim=-1, keepdim=True) + eps) - return (x.float() / rms).to(x.dtype) - - -def _ref_rope_interleaved( - x: torch.Tensor, freqs_cis: torch.Tensor, positions: torch.Tensor, rope_dim: int -) -> torch.Tensor: - """Reference: apply RoPE to the last `rope_dim` elements (interleaved re/im).""" - out = x.clone() - B = x.size(0) - head_dim = x.size(-1) - nope_dim = head_dim - rope_dim - - for b in range(B): - pos = positions[b].item() - freq = freqs_cis[pos] # (rope_dim,) interleaved [re0, im0, re1, im1, ...] - rope_part = out[b, ..., nope_dim:].float() - # Reshape to pairs - pairs = rope_part.reshape(*rope_part.shape[:-1], rope_dim // 2, 2) - x_real = pairs[..., 0] - x_imag = pairs[..., 1] - freq_pairs = freq.reshape(rope_dim // 2, 2) - f_real = freq_pairs[:, 0] - f_imag = freq_pairs[:, 1] - rot_real = x_real * f_real - x_imag * f_imag - rot_imag = x_real * f_imag + x_imag * f_real - result = torch.stack([rot_real, rot_imag], dim=-1).reshape(rope_part.shape) - out[b, ..., nope_dim:] = result.to(x.dtype) - return out - - -@pytest.mark.parametrize("batch_size", [1, 4, 16]) -@pytest.mark.parametrize("num_heads", [1, 8]) -@pytest.mark.parametrize("head_dim", [128, 192]) -def test_fused_q_norm_rope_correctness(batch_size, num_heads, head_dim): - """Test Q norm + rope against reference.""" - torch.manual_seed(42) - rope_dim = 64 - max_pos = 512 - eps = 1e-6 - - q_input = torch.randn( - batch_size, num_heads, head_dim, dtype=torch.bfloat16, device="cuda" - ) - freqs_cis = torch.randn(max_pos, rope_dim, dtype=torch.float32, device="cuda") - positions = torch.randint( - 0, max_pos, (batch_size,), dtype=torch.int32, device="cuda" - ) - - q_output = sgl_kernel.dsv4_fused_q_norm_rope(q_input, freqs_cis, positions, eps) - - # Reference - normed = _ref_rmsnorm_self(q_input, eps) - expected = _ref_rope_interleaved(normed, freqs_cis, positions, rope_dim) - - torch.testing.assert_close(q_output.float(), expected.float(), rtol=1e-2, atol=1e-2) - - -def test_fused_q_norm_rope_zero_batch(): - """Empty batch should not crash.""" - q_input = torch.empty(0, 8, 192, dtype=torch.bfloat16, device="cuda") - freqs_cis = torch.randn(512, 64, dtype=torch.float32, device="cuda") - positions = torch.empty(0, dtype=torch.int32, device="cuda") - q_output = sgl_kernel.dsv4_fused_q_norm_rope(q_input, freqs_cis, positions) - assert q_output.shape == q_input.shape - - -def test_fused_q_norm_rope_preallocated_output(): - """Test with pre-allocated output tensor.""" - torch.manual_seed(42) - B, H, D = 4, 8, 192 - q_input = torch.randn(B, H, D, dtype=torch.bfloat16, device="cuda") - freqs_cis = torch.randn(512, 64, dtype=torch.float32, device="cuda") - positions = torch.randint(0, 512, (B,), dtype=torch.int32, device="cuda") - q_output = torch.empty_like(q_input) - - result = sgl_kernel.dsv4_fused_q_norm_rope( - q_input, freqs_cis, positions, q_output=q_output - ) - assert result is q_output - - @pytest.mark.parametrize("batch_size", [1, 8]) def test_fused_q_indexer_rope_hadamard_quant_runs(batch_size): """Basic launch coverage with finite output checks.""" diff --git a/python/sglang/kernels/aot/tests/test_fused_qk_norm_rope.py b/python/sglang/kernels/aot/tests/test_fused_qk_norm_rope.py index 0011887436..6e46554f4a 100644 --- a/python/sglang/kernels/aot/tests/test_fused_qk_norm_rope.py +++ b/python/sglang/kernels/aot/tests/test_fused_qk_norm_rope.py @@ -8,21 +8,9 @@ from sglang.srt.server_args import ( ServerArgs, set_global_server_args_for_scheduler, ) -from sglang.srt.utils import ( - cpu_has_amx_support, - is_cpu, - is_cuda, - is_hip, - is_npu, - is_xpu, -) +from sglang.srt.utils import is_cuda _is_cuda = is_cuda() -_is_hip = is_hip() -_is_cpu = is_cpu() -_is_cpu_amx_available = cpu_has_amx_support() -_is_npu = is_npu() -_is_xpu = is_xpu() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") diff --git a/python/sglang/kernels/aot/tests/test_merge_state_v2.py b/python/sglang/kernels/aot/tests/test_merge_state_v2.py index 7b285d10ff..c7fd0bd0ca 100644 --- a/python/sglang/kernels/aot/tests/test_merge_state_v2.py +++ b/python/sglang/kernels/aot/tests/test_merge_state_v2.py @@ -3,99 +3,9 @@ from typing import Optional import pytest import torch -import triton -import triton.language as tl from sgl_kernel import merge_state_v2 - -@triton.jit -def merge_state_kernel( - output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_merged - output_lse, # [NUM_TOKENS, NUM_HEADS] s_merged - prefix_output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_a - prefix_lse, # [NUM_TOKENS, NUM_HEADS] s_a - suffix_output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_b - suffix_lse, # [NUM_TOKENS, NUM_HEADS] s_b - HEAD_SIZE: tl.constexpr, - PADDED_HEAD_SIZE: tl.constexpr, - OUTPUT_LSE: tl.constexpr, -): - token_idx = tl.program_id(0) - num_tokens = tl.num_programs(0) - head_idx = tl.program_id(1) - num_heads = tl.num_programs(1) - - p_lse = tl.load(prefix_lse + token_idx * num_heads + head_idx) - s_lse = tl.load(suffix_lse + token_idx * num_heads + head_idx) - p_lse = float("-inf") if p_lse == float("inf") else p_lse - s_lse = float("-inf") if s_lse == float("inf") else s_lse - - max_lse = tl.maximum(p_lse, s_lse) - p_lse = p_lse - max_lse - s_lse = s_lse - max_lse - out_se = tl.exp(p_lse) + tl.exp(s_lse) - - if OUTPUT_LSE: - out_lse = tl.log(out_se) + max_lse - tl.store(output_lse + token_idx * num_heads + head_idx, out_lse) - - head_arange = tl.arange(0, PADDED_HEAD_SIZE) - head_mask = head_arange < HEAD_SIZE - p_out = tl.load( - prefix_output - + token_idx * num_heads * HEAD_SIZE - + head_idx * HEAD_SIZE - + head_arange, - mask=head_mask, - ) - s_out = tl.load( - suffix_output - + token_idx * num_heads * HEAD_SIZE - + head_idx * HEAD_SIZE - + head_arange, - mask=head_mask, - ) - - p_scale = tl.exp(p_lse) / out_se - s_scale = tl.exp(s_lse) / out_se - out = p_out * p_scale + s_out * s_scale - tl.store( - output + token_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE + head_arange, - out, - mask=head_mask, - ) - - -def merge_state_triton( - prefix_output: torch.Tensor, - prefix_lse: torch.Tensor, - suffix_output: torch.Tensor, - suffix_lse: torch.Tensor, - output: Optional[torch.Tensor] = None, - output_lse: Optional[torch.Tensor] = None, -) -> None: - num_tokens = output.shape[0] - num_query_heads = output.shape[1] - head_size = output.shape[2] - padded_head_size = triton.next_power_of_2(head_size) - # Avoid creating new tensors if they are already provided - if output is None: - output = torch.empty_like(prefix_output) - if output_lse is None: - output_lse = torch.empty_like(prefix_lse) - - merge_state_kernel[(num_tokens, num_query_heads)]( - output, - output_lse, - prefix_output, - prefix_lse, - suffix_output, - suffix_lse, - head_size, - padded_head_size, - output_lse is not None, - ) - return output, output_lse +from sglang.kernels.ops.attention.merge_state import merge_state_triton # Naive PyTorch Implements of Merge Attn States diff --git a/python/sglang/kernels/aot/tests/test_moe_align.py b/python/sglang/kernels/aot/tests/test_moe_align.py index f0cfe5e0a4..06ee94a22c 100644 --- a/python/sglang/kernels/aot/tests/test_moe_align.py +++ b/python/sglang/kernels/aot/tests/test_moe_align.py @@ -3,10 +3,10 @@ import sys import pytest import torch -import triton -import triton.language as tl from sgl_kernel import moe_align_block_size, moe_sum +from sglang.test.kernels.moe_align import moe_align_block_size_triton + def is_hip() -> bool: return torch.version.hip is not None @@ -15,144 +15,14 @@ def is_hip() -> bool: _is_hip = is_hip() -def ceil_div(a, b): - return (a + b - 1) // b - - -@triton.jit -def moe_align_block_size_stage1( - topk_ids_ptr, - tokens_cnts_ptr, - num_experts: tl.constexpr, - numel: tl.constexpr, - tokens_per_thread: tl.constexpr, -): - pid = tl.program_id(0) - start_idx = pid * tokens_per_thread - off_c = (pid + 1) * num_experts - - for i in range(tokens_per_thread): - if start_idx + i < numel: - idx = tl.load(topk_ids_ptr + start_idx + i) - token_cnt = tl.load(tokens_cnts_ptr + off_c + idx) - tl.store(tokens_cnts_ptr + off_c + idx, token_cnt + 1) - - -@triton.jit -def moe_align_block_size_stage2( - tokens_cnts_ptr, - num_experts: tl.constexpr, -): - pid = tl.program_id(0) - last_cnt = 0 - for i in range(1, num_experts + 1): - token_cnt = tl.load(tokens_cnts_ptr + i * num_experts + pid) - last_cnt = last_cnt + token_cnt - tl.store(tokens_cnts_ptr + i * num_experts + pid, last_cnt) - - -@triton.jit -def moe_align_block_size_stage3( - total_tokens_post_pad_ptr, - tokens_cnts_ptr, - cumsum_ptr, - num_experts: tl.constexpr, - block_size: tl.constexpr, -): - last_cumsum = 0 - off_cnt = num_experts * num_experts - for i in range(1, num_experts + 1): - token_cnt = tl.load(tokens_cnts_ptr + off_cnt + i - 1) - last_cumsum = last_cumsum + tl.cdiv(token_cnt, block_size) * block_size - tl.store(cumsum_ptr + i, last_cumsum) - tl.store(total_tokens_post_pad_ptr, last_cumsum) - - -@triton.jit -def moe_align_block_size_stage4( - topk_ids_ptr, - sorted_token_ids_ptr, - expert_ids_ptr, - tokens_cnts_ptr, - cumsum_ptr, - num_experts: tl.constexpr, - block_size: tl.constexpr, - numel: tl.constexpr, - tokens_per_thread: tl.constexpr, -): - pid = tl.program_id(0) - start_idx = tl.load(cumsum_ptr + pid) - end_idx = tl.load(cumsum_ptr + pid + 1) - - for i in range(start_idx, end_idx, block_size): - tl.store(expert_ids_ptr + i // block_size, pid) - - start_idx = pid * tokens_per_thread - off_t = pid * num_experts - - for i in range(start_idx, tl.minimum(start_idx + tokens_per_thread, numel)): - expert_id = tl.load(topk_ids_ptr + i) - token_cnt = tl.load(tokens_cnts_ptr + off_t + expert_id) - rank_post_pad = token_cnt + tl.load(cumsum_ptr + expert_id) - tl.store(sorted_token_ids_ptr + rank_post_pad, i) - tl.store(tokens_cnts_ptr + off_t + expert_id, token_cnt + 1) - - -def moe_align_block_size_triton( - topk_ids: torch.Tensor, - num_experts: int, - block_size: int, - sorted_token_ids: torch.Tensor, - expert_ids: torch.Tensor, - num_tokens_post_pad: torch.Tensor, -) -> None: - numel = topk_ids.numel() - grid = (num_experts,) - tokens_cnts = torch.zeros( - (num_experts + 1, num_experts), dtype=torch.int32, device=topk_ids.device - ) - cumsum = torch.zeros((num_experts + 1,), dtype=torch.int32, device=topk_ids.device) - tokens_per_thread = ceil_div(numel, num_experts) - - moe_align_block_size_stage1[grid]( - topk_ids, - tokens_cnts, - num_experts, - numel, - tokens_per_thread, - ) - moe_align_block_size_stage2[grid]( - tokens_cnts, - num_experts, - ) - moe_align_block_size_stage3[(1,)]( - num_tokens_post_pad, - tokens_cnts, - cumsum, - num_experts, - block_size, - ) - moe_align_block_size_stage4[grid]( - topk_ids, - sorted_token_ids, - expert_ids, - tokens_cnts, - cumsum, - num_experts, - block_size, - numel, - tokens_per_thread, - ) - - @pytest.mark.parametrize( "block_size,num_tokens,topk,num_experts,pad_sorted_token_ids", list( itertools.product( - [32, 64, 128, 256], # block_size - [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096], # num_tokens - [1, 2, 4, 8, 16, 32, 64], # topk - [64, 160, 256, 257, 260, 264], # num_experts + [32, 256], # block_size + [1, 16, 4096], # num_tokens + [1, 8, 64], # topk + [64, 160, 257, 264], # num_experts [True, False], # pad_sorted_token_ids ) ), diff --git a/python/sglang/kernels/aot/tests/test_per_token_group_quant_8bit.py b/python/sglang/kernels/aot/tests/test_per_token_group_quant_8bit.py index 0432f4ecea..53c9737afd 100644 --- a/python/sglang/kernels/aot/tests/test_per_token_group_quant_8bit.py +++ b/python/sglang/kernels/aot/tests/test_per_token_group_quant_8bit.py @@ -8,58 +8,68 @@ from sgl_kernel.test_utils import ( create_per_token_group_quant_test_data, ) -from sglang.kernels.ops.quantization.fp8_kernel import ( - per_token_group_quant_8bit as triton_per_token_group_quant_8bit, -) -from sglang.kernels.ops.quantization.fp8_kernel import ( - sglang_per_token_group_quant_8bit, -) from sglang.srt.utils import is_hip +from sglang.test.kernels.per_token_group_quant import ( + sglang_per_token_group_quant_8bit, + triton_per_token_group_quant_8bit, +) _is_hip = is_hip() fp8_type_ = torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn -configs = list( - itertools.product( - [1, 4, 16, 64, 127, 128, 512, 1024, 4096, 8192], # num_tokens - [128, 256, 384, 512, 1024, 1536, 1664, 2048, 4096, 7168, 16384], # hidden_dim +_ROW_MAJOR = dict( + column_major_scales=False, + scale_tma_aligned=False, + scale_ue8m0=False, + fuse_silu_and_mul=False, + masked_layout_mode=None, +) +_COLUMN_MAJOR_FLAGS = [ + dict( + column_major_scales=True, + scale_tma_aligned=False, + scale_ue8m0=False, + fuse_silu_and_mul=False, + masked_layout_mode=None, + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=False, + fuse_silu_and_mul=False, + masked_layout_mode=None, + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + fuse_silu_and_mul=False, + masked_layout_mode=None, + ), +] + + +def _supported(group_size, dst_dtype, flags): + # int8 has no column-major scale layout; UE8M0 scales are only built for group 128. + if dst_dtype == torch.int8 and flags["column_major_scales"]: + return False + return not flags["scale_ue8m0"] or group_size == 128 + + +configs = [ + config + for config in itertools.product( + [1, 4, 127, 4096], # num_tokens + [128, 384, 1664, 7168], # hidden_dim [16, 32, 64, 128], # group_size [None], # num_ranks [fp8_type_, torch.int8], # dtype - [ - dict( - column_major_scales=False, - scale_tma_aligned=False, - scale_ue8m0=False, - fuse_silu_and_mul=False, - masked_layout_mode=None, - ), - dict( - column_major_scales=True, - scale_tma_aligned=False, - scale_ue8m0=False, - fuse_silu_and_mul=False, - masked_layout_mode=None, - ), - dict( - column_major_scales=True, - scale_tma_aligned=True, - scale_ue8m0=False, - fuse_silu_and_mul=False, - masked_layout_mode=None, - ), - dict( - column_major_scales=True, - scale_tma_aligned=True, - scale_ue8m0=True, - fuse_silu_and_mul=False, - masked_layout_mode=None, - ), - ], + [_ROW_MAJOR, *_COLUMN_MAJOR_FLAGS], ) -) + list( + if _supported(config[2], config[4], config[5]) +] + list( itertools.product( - [1, 4, 1 * 8, 4 * 8, 64 * 8, 256 * 8, 768 * 8], + [1, 4 * 8, 768 * 8], # TODO support more [2048], [128], @@ -117,21 +127,11 @@ def test_per_token_group_quant_with_column_major( arch_major, _ = torch.cuda.get_device_capability(torch.cuda.current_device()) if flags["scale_ue8m0"] and (arch_major <= 9): pytest.skip("Only Blackwell need ue8m0 fusion") - return - - if (flags["scale_ue8m0"] and (group_size != 128)) or ( - (dst_dtype == torch.int8) and flags["column_major_scales"] - ): - pytest.skip() - return x, masked_m = create_per_token_group_quant_test_data( num_tokens=num_tokens, hidden_dim=hidden_dim, num_ranks=num_ranks, flags=flags ) - # print("hack data!!!") - # x = torch.full_like(x, fill_value=100) - execute_kwargs = dict( x=x, masked_m=masked_m, diff --git a/python/sglang/kernels/aot/tests/test_topk.py b/python/sglang/kernels/aot/tests/test_topk.py index 73d53614f1..6d6a93d14c 100644 --- a/python/sglang/kernels/aot/tests/test_topk.py +++ b/python/sglang/kernels/aot/tests/test_topk.py @@ -539,10 +539,11 @@ def test_topk_variants_are_exact_on_variable_windows( kind: str, force_one_block: bool, monkeypatch: pytest.MonkeyPatch ) -> None: """Cover both dispatches and all three APIs on adversarial score distributions.""" + monkeypatch.delenv("SGL_DSA_TOPK_ROW_SPLIT", raising=False) if force_one_block: - monkeypatch.setenv("SGL_DSA_TOPK_ROW_SPLIT", "0") + monkeypatch.setenv("SGLANG_DSA_TOPK_ROW_SPLIT", "0") else: - monkeypatch.delenv("SGL_DSA_TOPK_ROW_SPLIT", raising=False) + monkeypatch.delenv("SGLANG_DSA_TOPK_ROW_SPLIT", raising=False) bs, k = 4, 2048 score = _make_scores(kind, bs, MAX_SEQ_LEN, seed=1200 + force_one_block) diff --git a/python/sglang/kernels/fused_op.py b/python/sglang/kernels/fused_op.py index 3617a6fad3..5ed99d74fb 100644 --- a/python/sglang/kernels/fused_op.py +++ b/python/sglang/kernels/fused_op.py @@ -35,7 +35,7 @@ Dispatch priority (highest first), resolved by :meth:`BaseFusedOp.forward`: whose method is overridden, that is *declared* in :attr:`capabilities`, and whose :class:`~sglang.kernels.spec.CapabilityRequirement` set matches the detected platform. Ops may extend :meth:`backend_eligible` with - per-call shape/dtype gates; overriding it switches this step from a + per-call applicability gates; overriding it switches this step from a statically cached choice to per-call selection. 5. **Platform-specific forward** — ``forward_cuda`` on CUDA, ``forward_hip`` (falling back to ``forward_cuda``) on ROCm, ``forward_musa`` on MUSA, @@ -86,12 +86,11 @@ from torch import nn from sglang.kernels.kernel_api_logging import debug_kernel_api from sglang.kernels.registry import register_kernel +from sglang.kernels.selector import _platform from sglang.kernels.spec import ( CapabilityRequirement, - FormatSignature, KernelBackend, KernelSpec, - PlatformInfo, capabilities_satisfied, ) @@ -159,11 +158,6 @@ _PLATFORM_METHODS: Dict[str, Tuple[str, ...]] = { } -@functools.lru_cache(maxsize=1) -def _platform() -> PlatformInfo: - return PlatformInfo.detect() - - @functools.lru_cache(maxsize=1) def _platform_key() -> str: """In-tree platform dispatch key, or ``""`` for plain-native platforms. @@ -207,13 +201,6 @@ def _oot_dispatch_key() -> Optional[str]: return None -def clear_platform_caches() -> None: - """Drop the cached platform detection (used by tests that mock platforms).""" - _platform.cache_clear() - _platform_key.cache_clear() - _oot_dispatch_key.cache_clear() - - # --- global backend override ------------------------------------------------ # Sentinel distinguishing "not resolved yet" from "resolved to None (no force)". @@ -350,7 +337,7 @@ class BaseFusedOp(nn.Module, ABC): Class attributes ---------------- op: - Operator id, ``"."`` (e.g. ``"layernorm.rmsnorm"``). + Operator id, ``"."`` (e.g. ``"activation.silu_and_mul"``). Required for :func:`register_fused_op`; layer-style subclasses that are not registered in the kernel registry may leave it empty. priority: @@ -368,10 +355,6 @@ class BaseFusedOp(nn.Module, ABC): the ``CapabilityRequirement.CUDA`` / ``.HIP`` / ``.NPU`` shortcuts, e.g. ``{KernelBackend.AOT: {CapabilityRequirement.CUDA, CapabilityRequirement.HIP}}``. - format_signature: - Data-contract description shared by all backends of this op. - descriptions: - Optional per-backend one-liners for the registry inventory. """ op: ClassVar[str] = "" @@ -379,8 +362,6 @@ class BaseFusedOp(nn.Module, ABC): capabilities: ClassVar[ Mapping[KernelBackend, AbstractSet[CapabilityRequirement]] ] = {} - format_signature: ClassVar[FormatSignature] = FormatSignature() - descriptions: ClassVar[Mapping[KernelBackend, str]] = {} # OOT forward registry shared by all fused ops: platform dispatch key -> # {op class -> forward fn}. Populated by out-of-tree platform plugins. @@ -438,6 +419,9 @@ class BaseFusedOp(nn.Module, ABC): def forward_cute_dsl(self, *args, **kwargs): raise NotImplementedError(f"{self._op_label()}: no cute_dsl backend") + def forward_flydsl(self, *args, **kwargs): + raise NotImplementedError(f"{self._op_label()}: no flydsl backend") + def forward_helion(self, *args, **kwargs): raise NotImplementedError(f"{self._op_label()}: no helion backend") @@ -480,10 +464,12 @@ class BaseFusedOp(nn.Module, ABC): The base implementation checks the backend's :class:`CapabilityRequirement` set (OR semantics) against the detected - platform. Subclasses may extend it with per-call shape/dtype gates so - auto-selection bounces to the next backend instead of raising; - overriding this method switches backend auto-selection from a cached - static choice to per-call resolution. + platform. Subclasses may extend it with per-call gates for operands the + kernel does not cover, where the next backend computes the same result; + operands that violate the op's contract are a caller bug the backend's + launcher must raise on, not a reason to bounce. Overriding this method + switches backend auto-selection from a cached static choice to per-call + resolution. """ return capabilities_satisfied( self.capabilities.get(backend, frozenset()), _platform() @@ -586,10 +572,6 @@ class BaseFusedOp(nn.Module, ABC): _record_trace(self, _dispatch_label(method), args, kwargs) return result - def dispatch_forward(self) -> Callable: - """The static dispatch target for this op on the current platform.""" - return self._resolve_forward_method() - # --- torch.compile mode --- def _torch_compile_forward(self, num_tokens: int) -> Optional[Callable]: @@ -684,7 +666,7 @@ def register_fused_op(instance: BaseFusedOp, module: str, attr: str) -> BaseFuse ``KernelSpec.load()`` can lazily resolve e.g. ``":.forward_aot"`` to the bound backend method. Returns ``instance`` so group packages can write - ``_RMSNORM = register_fused_op(_RMSNormOp(), __name__, "_RMSNORM")``. + ``_SILU_AND_MUL = register_fused_op(SiluAndMulOp(), __name__, "_SILU_AND_MUL")``. """ for backend in instance.available_backends(): register_kernel( @@ -693,8 +675,6 @@ def register_fused_op(instance: BaseFusedOp, module: str, attr: str) -> BaseFuse backend=backend, target=f"{module}:{attr}.{BACKEND_METHODS[backend]}", capabilities=frozenset(instance.capabilities.get(backend, ())), - format_signature=instance.format_signature, - description=instance.descriptions.get(backend, ""), ) ) return instance diff --git a/python/sglang/kernels/jit/benchmark/utils.py b/python/sglang/kernels/jit/benchmark/utils.py index 9f85a19451..db892dec46 100644 --- a/python/sglang/kernels/jit/benchmark/utils.py +++ b/python/sglang/kernels/jit/benchmark/utils.py @@ -5,7 +5,7 @@ from typing import Callable, List, Optional, Sequence, Tuple import torch import triton.testing -from sglang.kernels.ops.communication.mp import multigpu_launch +from sglang.test.kernels.multigpu import multigpu_launch from sglang.utils import is_in_ci diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/c128.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/c128.cuh deleted file mode 100644 index 622ff45f0b..0000000000 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/c128.cuh +++ /dev/null @@ -1,522 +0,0 @@ -#include -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -#include - -namespace sglang { - -using Plan128 = device::compress::PrefillPlan; -using IndiceT = int32_t; - -/// \brief Each thread will handle this many elements (split along head_dim) -constexpr int32_t kTileElements = 2; -/// \brief Each warp will handle this many elements (split along 128) -constexpr int32_t kElementsPerWarp = 8; -constexpr uint32_t kNumWarps = 128 / kElementsPerWarp; -constexpr uint32_t kBlockSize = device::kWarpThreads * kNumWarps; - -/// \brief Need to reduce register usage to increase occupancy -#define C128_KERNEL __global__ __launch_bounds__(kBlockSize, 2) - -struct Compress128DecodeParams { - /** - * \brief Shape: `[num_indices, 128, head_dim * 2]` \n - * last dimension layout: - * | kv current | score current | - */ - void* __restrict__ kv_score_buffer; - /** \brief Shape: `[batch_size, head_dim * 2]` */ - const void* __restrict__ kv_score_input; - /** \brief Shape: `[batch_size, head_dim]` */ - void* __restrict__ kv_compressed_output; - /** \brief Shape: `[128, head_dim]` (called `ape`) */ - const void* __restrict__ score_bias; - /** \brief Shape: `[batch_size, ]`*/ - const IndiceT* __restrict__ indices; - /** \brief Shape: `[batch_size, ]` */ - const IndiceT* __restrict__ seq_lens; - /** \NOTE: `batch_size` <= `num_indices` */ - uint32_t batch_size; -}; - -struct Compress128PrefillParams { - /** - * \brief Shape: `[num_indices, 128, head_dim * 2]` \n - * last dimension layout: - * | kv current | score current | - */ - void* __restrict__ kv_score_buffer; - /** \brief Shape: `[batch_size, head_dim * 2]` */ - const void* __restrict__ kv_score_input; - /** \brief Shape: `[batch_size, head_dim]` */ - void* __restrict__ kv_compressed_output; - /** \brief Shape: `[128, head_dim]` (called `ape`) */ - const void* __restrict__ score_bias; - /** \brief Shape: `[batch_size, ]`*/ - const IndiceT* __restrict__ indices; - /** \brief Shape: `[batch_size, ]`*/ - const int32_t* __restrict__ load_indices; - /** \brief The following part is plan info. */ - const Plan128* __restrict__ compress_plan; - const Plan128* __restrict__ write_plan; - uint32_t num_compress; - uint32_t num_write; -}; - -struct Compress128SharedBuffer { - using Storage = device::AlignedVector; - Storage data[kNumWarps][device::kWarpThreads + 1]; // padding to avoid bank conflict - SGL_DEVICE Storage& operator()(uint32_t warp_id, uint32_t lane_id) { - return data[warp_id][lane_id]; - } - SGL_DEVICE float& operator()(uint32_t warp_id, uint32_t lane_id, uint32_t tile_id) { - return data[warp_id][lane_id][tile_id]; - } -}; - -template -SGL_DEVICE void c128_write( - T* kv_score_buf, // - const T* kv_score_src, - const int64_t head_dim, - const int32_t write_pos, - const uint32_t lane_id) { - using namespace device; - - using Storage = AlignedVector; - const auto element_size = head_dim * 2; - const auto gmem = tile::Memory{lane_id, kWarpThreads}; - kv_score_buf += write_pos * element_size; - - /// NOTE: Layout | [0] = kv | [1] = score | - Storage kv_score[2]; -#pragma unroll - for (int32_t i = 0; i < 2; ++i) { - kv_score[i] = gmem.load(kv_score_src + head_dim * i); - } -#pragma unroll - for (int32_t i = 0; i < 2; ++i) { - gmem.store(kv_score_buf + head_dim * i, kv_score[i]); - } -} - -template -SGL_DEVICE void c128_forward( - const InFloat* kv_score_buf, - const InFloat* kv_score_src, - OutFloat* kv_out, - const InFloat* score_bias, - const int64_t head_dim, - const int32_t window_len, - const uint32_t warp_id, - const uint32_t lane_id) { - using namespace device; - - const auto element_size = head_dim * 2; - const auto score_offset = head_dim; - - /// NOTE: part 1: load kv + score - using StorageIn = AlignedVector; - const auto gmem_in = tile::Memory{lane_id, kWarpThreads}; - StorageIn kv[kElementsPerWarp]; - StorageIn score[kElementsPerWarp]; - StorageIn bias[kElementsPerWarp]; - const int32_t warp_offset = warp_id * kElementsPerWarp; - -#pragma unroll - for (int32_t i = 0; i < 8; ++i) { - const int32_t j = i + warp_offset; - bias[i] = gmem_in.load(score_bias + j * head_dim); - } - -#pragma unroll - for (int32_t i = 0; i < kElementsPerWarp; ++i) { - const int32_t j = i + warp_offset; - const InFloat* src; - __builtin_assume(j < 128); - if (j < window_len) { - src = kv_score_buf + j * element_size; - } else { - /// NOTE: k in [-127, 0]. We'll load from the ragged `kv_score_src` - const int32_t k = j - 127; - src = kv_score_src + k * element_size; - } - kv[i] = gmem_in.load(src); - score[i] = gmem_in.load(src + score_offset); - } - - /// NOTE: part 2: safe online softmax + weighted sum - using TmpStorage = typename Compress128SharedBuffer::Storage; - __shared__ Compress128SharedBuffer s_local_val_max; - __shared__ Compress128SharedBuffer s_local_exp_sum; - __shared__ Compress128SharedBuffer s_local_product; - - TmpStorage tmp_val_max; - TmpStorage tmp_exp_sum; - TmpStorage tmp_product; - -#pragma unroll - for (int32_t i = 0; i < kTileElements; ++i) { - float score_fp32[kElementsPerWarp]; - -#pragma unroll - for (int32_t j = 0; j < kElementsPerWarp; ++j) { - score_fp32[j] = cast(score[j][i]) + cast(bias[j][i]); - } - - float max_value = score_fp32[0]; - float sum_exp_value = 0.0f; - -#pragma unroll - for (int32_t j = 1; j < kElementsPerWarp; ++j) { - const auto fp32_score = score_fp32[j]; - max_value = fmaxf(max_value, fp32_score); - } - - float sum_product = 0.0f; -#pragma unroll - for (int32_t j = 0; j < 8; ++j) { - const auto fp32_score = score_fp32[j]; - const auto exp_score = expf(fp32_score - max_value); - sum_product += cast(kv[j][i]) * exp_score; - sum_exp_value += exp_score; - } - - tmp_val_max[i] = max_value; - tmp_exp_sum[i] = sum_exp_value; - tmp_product[i] = sum_product; - } - - // naturally aligned, so no bank conflict - s_local_val_max(warp_id, lane_id) = tmp_val_max; - s_local_exp_sum(warp_id, lane_id) = tmp_exp_sum; - s_local_product(warp_id, lane_id) = tmp_product; - - __syncthreads(); - - /// NOTE: part 3: online softmax - /// NOTE: We have `kTileElements * kWarpThreads * kNumWarps` values to reduce - /// each reduce will consume `kNumWarps` threads (use partial warp reduction) - constexpr uint32_t kReductionCount = kTileElements * kWarpThreads * kNumWarps; - constexpr uint32_t kIteration = kReductionCount / kBlockSize; - -#pragma unroll - for (uint32_t i = 0; i < kIteration; ++i) { - /// NOTE: Range `[0, kTileElements * kWarpThreads * kNumWarps)` - const uint32_t j = i * kBlockSize + warp_id * kWarpThreads + lane_id; - /// NOTE: Range `[0, kNumWarps)` - const uint32_t local_warp_id = j % kNumWarps; - /// NOTE: Range `[0, kTileElements * kWarpThreads)` - const uint32_t local_elem_id = j / kNumWarps; - /// NOTE: Range `[0, kTileElements)` - const uint32_t local_tile_id = local_elem_id % kTileElements; - /// NOTE: Range `[0, kWarpThreads)` - const uint32_t local_lane_id = local_elem_id / kTileElements; - /// NOTE: each warp will access the whole tile (all `kTileElements`) - /// and for different lanes, the memory access only differ in `local_warp_id` - /// so there's no bank conflict in shared memory access. - static_assert(kTileElements * kNumWarps == kWarpThreads, "TODO: support other configs"); - const auto local_val_max = s_local_val_max(local_warp_id, local_lane_id, local_tile_id); - const auto local_exp_sum = s_local_exp_sum(local_warp_id, local_lane_id, local_tile_id); - const auto local_product = s_local_product(local_warp_id, local_lane_id, local_tile_id); - const auto global_val_max = warp::reduce_max(local_val_max); - const auto rescale = expf(local_val_max - global_val_max); - const auto global_exp_sum = warp::reduce_sum(local_exp_sum * rescale); - const auto final_scale = rescale / global_exp_sum; - const auto global_product = warp::reduce_sum(local_product * final_scale); - kv_out[local_elem_id] = cast(global_product); - } -} - -template -C128_KERNEL void flash_c128_decode(const __grid_constant__ Compress128DecodeParams params) { - using namespace device; - - constexpr int64_t kTileDim = kTileElements * kWarpThreads; // 64 - constexpr uint32_t kNumSplit = kHeadDim / kTileDim; - constexpr int64_t kElementSize = kHeadDim * 2; - static_assert(kHeadDim % kTileDim == 0, "Head dim must be multiple of tile dim"); - - const auto& [ - _kv_score_buffer, _kv_score_input, _kv_compressed_output, _score_bias, // kv score - indices, seq_lens, batch_size // decode info - ] = params; - const uint32_t warp_id = threadIdx.x / kWarpThreads; - const uint32_t lane_id = threadIdx.x % kWarpThreads; - - const uint32_t global_bid = blockIdx.x / kNumSplit; // batch id - const uint32_t global_sid = blockIdx.x % kNumSplit; // split id - if (global_bid >= batch_size) return; - - const int32_t index = indices[global_bid]; - const int32_t seq_len = seq_lens[global_bid]; - const int64_t split_offset = global_sid * kTileDim; - - // kv score - const auto kv_score_buffer = static_cast(_kv_score_buffer); - const auto kv_buf = kv_score_buffer + index * (kElementSize * 128) + split_offset; - - // kv input - const auto kv_score_input = static_cast(_kv_score_input); - const auto kv_src = kv_score_input + global_bid * kElementSize + split_offset; - - // kv output - const auto kv_compressed_output = static_cast(_kv_compressed_output); - const auto kv_out = kv_compressed_output + global_bid * kHeadDim + split_offset; - - // score bias (ape) - const auto score_bias = static_cast(_score_bias) + split_offset; - - PDLWaitPrimary(); - - /// NOTE: the write must be visible to the subsequent c128_forward, - /// so only the last warp can write to HBM - /// In addition, `position` = `seq_len - 1`. To avoid underflow, we use `seq_len + 127` - if (warp_id == kNumWarps - 1) { - c128_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/(seq_len + 127) % 128, lane_id); - } - if (seq_len % 128 == 0) { - c128_forward(kv_buf, kv_src, kv_out, score_bias, kHeadDim, /*window_len=*/128, warp_id, lane_id); - } - - PDLTriggerSecondary(); -} - -// compress kernel -template -C128_KERNEL void flash_c128_prefill(const __grid_constant__ Compress128PrefillParams params) { - using namespace device; - - constexpr int64_t kTileDim = kTileElements * kWarpThreads; // 64 - constexpr uint32_t kNumSplit = kHeadDim / kTileDim; - constexpr int64_t kElementSize = kHeadDim * 2; - static_assert(kHeadDim % kTileDim == 0, "Head dim must be multiple of tile dim"); - - const auto& [ - _kv_score_buffer, _kv_score_input, _kv_compressed_output, _score_bias, // kv score - indices, load_indices, compress_plan, write_plan, num_compress, num_write // prefill plan - ] = params; - const uint32_t warp_id = threadIdx.x / kWarpThreads; - const uint32_t lane_id = threadIdx.x % kWarpThreads; - - uint32_t global_id; - if constexpr (kWrite) { - // for write kernel, we use global warp_id to dispatch work - global_id = (blockIdx.x * blockDim.x + threadIdx.x) / kWarpThreads; - } else { - // for compress kernel, we use block id to dispatch work - global_id = blockIdx.x; // block id - } - const uint32_t global_pid = global_id / kNumSplit; // plan id - const uint32_t global_sid = global_id % kNumSplit; // split id - - /// NOTE: compiler can optimize this if-else at compile time - const auto num_plans = kWrite ? num_write : num_compress; - const auto plan_ptr = kWrite ? write_plan : compress_plan; - if (global_pid >= num_plans) return; - - const auto& [ragged_id, global_bid, position, window_len] = plan_ptr[global_pid]; - const auto indices_ptr = kWrite ? indices : load_indices; - - const int64_t split_offset = global_sid * kTileDim; - - // kv input - const auto kv_score_input = static_cast(_kv_score_input); - const auto kv_src = kv_score_input + ragged_id * kElementSize + split_offset; - - // kv output - const auto kv_compressed_output = static_cast(_kv_compressed_output); - const auto kv_out = kv_compressed_output + ragged_id * kHeadDim + split_offset; - - // score bias (ape) - const auto score_bias = static_cast(_score_bias) + split_offset; - - if (ragged_id == 0xFFFFFFFF) [[unlikely]] - return; - - const int32_t index = indices_ptr[global_bid]; - // kv score - const auto kv_score_buffer = static_cast(_kv_score_buffer); - const auto kv_buf = kv_score_buffer + index * (kElementSize * 128) + split_offset; - - PDLWaitPrimary(); - - // only responsible for the compress part - if constexpr (kWrite) { - c128_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/position % 128, lane_id); - } else { - c128_forward(kv_buf, kv_src, kv_out, score_bias, kHeadDim, window_len, warp_id, lane_id); - } - - PDLTriggerSecondary(); -} - -template -struct FlashCompress128Kernel { - static constexpr auto decode_kernel = flash_c128_decode; - template - static constexpr auto prefill_kernel = flash_c128_prefill; - static constexpr auto prefill_c_kernel = prefill_kernel; - static constexpr auto prefill_w_kernel = prefill_kernel; - static constexpr int64_t kTileDim = kTileElements * device::kWarpThreads; // 64 - static constexpr uint32_t kNumSplit = kHeadDim / kTileDim; - static constexpr uint32_t kWriteBlockSize = 128; - static constexpr uint32_t kWarpsPerWriteBlock = kWriteBlockSize / device::kWarpThreads; - - static void run_decode( - const tvm::ffi::TensorView kv_score_buffer, - const tvm::ffi::TensorView kv_score_input, - const tvm::ffi::TensorView kv_compressed_output, - const tvm::ffi::TensorView ape, - const tvm::ffi::TensorView indices, - const tvm::ffi::TensorView seq_lens, - const tvm::ffi::Optional /* UNUSED */) { - using namespace host; - - // this should not happen in practice - auto B = SymbolicSize{"batch_size"}; - auto device = SymbolicDevice{}; - device.set_options(); - - TensorMatcher({-1, 128, kHeadDim * 2}) // kv score - .with_dtype() - .with_device(device) - .verify(kv_score_buffer); - TensorMatcher({B, kHeadDim * 2}) // kv score input - .with_dtype() - .with_device(device) - .verify(kv_score_input); - TensorMatcher({B, kHeadDim}) // kv compressed output - .with_dtype() - .with_device(device) - .verify(kv_compressed_output); - TensorMatcher({128, kHeadDim}) // ape - .with_dtype() - .with_device(device) - .verify(ape); - TensorMatcher({B}) // indices - .with_dtype() - .with_device(device) - .verify(indices); - TensorMatcher({B}) // seq lens - .with_dtype() - .with_device(device) - .verify(seq_lens); - - const auto batch_size = static_cast(B.unwrap()); - const auto params = Compress128DecodeParams{ - .kv_score_buffer = kv_score_buffer.data_ptr(), - .kv_score_input = kv_score_input.data_ptr(), - .kv_compressed_output = kv_compressed_output.data_ptr(), - .score_bias = ape.data_ptr(), - .indices = static_cast(indices.data_ptr()), - .seq_lens = static_cast(seq_lens.data_ptr()), - .batch_size = batch_size, - }; - - const uint32_t num_blocks = batch_size * kNumSplit; - LaunchKernel(num_blocks, kBlockSize, device.unwrap()) // - .enable_pdl(kUsePDL)(decode_kernel, params); - } - - static void run_prefill( - const tvm::ffi::TensorView kv_score_buffer, - const tvm::ffi::TensorView kv_score_input, - const tvm::ffi::TensorView kv_compressed_output, - const tvm::ffi::TensorView ape, - const tvm::ffi::TensorView indices, - const tvm::ffi::TensorView compress_plan, - const tvm::ffi::TensorView write_plan, - const tvm::ffi::Optional extra) { - using namespace host; - - auto B = SymbolicSize{"batch_size"}; - auto N = SymbolicSize{"num_q_tokens"}; - auto X = SymbolicSize{"compress_tokens"}; - auto Y = SymbolicSize{"write_tokens"}; - auto device_ = SymbolicDevice{}; - device_.set_options(); - - TensorMatcher({-1, 128, kHeadDim * 2}) // kv score - .with_dtype() - .with_device(device_) - .verify(kv_score_buffer); - TensorMatcher({N, kHeadDim * 2}) // kv score input - .with_dtype() - .with_device(device_) - .verify(kv_score_input); - TensorMatcher({N, kHeadDim}) // kv compressed output - .with_dtype() - .with_device(device_) - .verify(kv_compressed_output); - TensorMatcher({128, kHeadDim}) // ape - .with_dtype() - .with_device(device_) - .verify(ape); - TensorMatcher({B}) // indices - .with_dtype() - .with_device(device_) - .verify(indices); - TensorMatcher({X, compress::kPrefillPlanDim}) // compress plan - .with_dtype() - .with_device(device_) - .verify(compress_plan); - TensorMatcher({Y, compress::kPrefillPlanDim}) // write plan - .with_dtype() - .with_device(device_) - .verify(write_plan); - - // might be needed for prefill write - const auto load_indices = extra.value_or(indices); - TensorMatcher({B}) // [read_positions] - .with_dtype() - .with_device(device_) - .verify(load_indices); - - const auto device = device_.unwrap(); - const auto batch_size = static_cast(B.unwrap()); - const auto num_q_tokens = static_cast(N.unwrap()); - const auto num_c = static_cast(X.unwrap()); - const auto num_w = static_cast(Y.unwrap()); - const auto params = Compress128PrefillParams{ - .kv_score_buffer = kv_score_buffer.data_ptr(), - .kv_score_input = kv_score_input.data_ptr(), - .kv_compressed_output = kv_compressed_output.data_ptr(), - .score_bias = ape.data_ptr(), - .indices = static_cast(indices.data_ptr()), - .load_indices = static_cast(load_indices.data_ptr()), - .compress_plan = static_cast(compress_plan.data_ptr()), - .write_plan = static_cast(write_plan.data_ptr()), - .num_compress = num_c, - .num_write = num_w, - }; - RuntimeCheck(num_q_tokens >= batch_size, "num_q_tokens must be >= batch_size"); - RuntimeCheck(num_q_tokens >= std::max(num_c, num_w), "invalid prefill plan"); - - constexpr auto kBlockSize_C = kBlockSize; - constexpr auto kBlockSize_W = kWriteBlockSize; - if (const auto num_c_blocks = num_c * kNumSplit) { - LaunchKernel(num_c_blocks, kBlockSize_C, device) // - .enable_pdl(kUsePDL)(prefill_c_kernel, params); - } - if (const auto num_w_blocks = div_ceil(num_w * kNumSplit, kWarpsPerWriteBlock)) { - LaunchKernel(num_w_blocks, kBlockSize_W, device) // - .enable_pdl(kUsePDL)(prefill_w_kernel, params); - } - } -}; - -} // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/c128_online.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/c128_online.cuh deleted file mode 100644 index 325035e4fe..0000000000 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/c128_online.cuh +++ /dev/null @@ -1,722 +0,0 @@ -#include -#include - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include - -#include -#include -#include - -namespace sglang { - -namespace device::compress { - -/// \brief Plan entry for online compress 128 prefill. -/// Each entry describes a contiguous segment of tokens that lies inside a -/// single 128-chunk. Multiple segments can map to the same batch id when the -/// extend tokens span chunk boundaries. -/// -/// **Layout compatibility:** the field order/types match `PrefillPlan` so that -/// downstream kernels (e.g. `fused_norm_rope` in `CompressExtend` mode) can -/// consume the compress_plan tensor as-if it were a `PrefillPlan` tensor -- -/// they only read `ragged_id` and `position`, both of which carry identical -/// semantics here (the LAST token of the segment in q-ragged and global -/// coordinates respectively). -/// -/// Note that `window_len` here means "number of real tokens in this segment" -/// (1..128), which differs from `PrefillPlan::window_len`. Downstream kernels -/// that share the tensor MUST NOT read it under that name. -struct alignas(16) OnlinePrefillPlan { - /// \brief Ragged-q position of the LAST token in this segment. - /// Equal to `segment_start_ragged + window_len - 1`. - uint32_t ragged_id; - /// \brief Index into the `indices` / `load_indices` arrays. - uint32_t batch_id; - /// \brief Global position of the LAST token in this segment. - /// For compress plans, `position % 128 == 127` (chunk-closing); for write - /// plans, `position % 128 < 127`. - uint32_t position; - /// \brief Number of real tokens in this segment (1..128). - /// The first segment token sits at `position - window_len + 1` (global) and - /// at `ragged_id - window_len + 1` (ragged). - uint32_t window_len; -}; - -static_assert(alignof(OnlinePrefillPlan) == alignof(PrefillPlan)); -static_assert(sizeof(OnlinePrefillPlan) == sizeof(PrefillPlan)); - -} // namespace device::compress - -namespace host::compress { - -using device::compress::OnlinePrefillPlan; -using OnlinePrefillPlanTensorDtype = uint8_t; -inline constexpr int64_t kOnlinePrefillPlanDim = 16; - -static_assert(alignof(OnlinePrefillPlan) == sizeof(OnlinePrefillPlan)); -static_assert(sizeof(OnlinePrefillPlan) == kOnlinePrefillPlanDim * sizeof(OnlinePrefillPlanTensorDtype)); - -} // namespace host::compress - -using OnlinePlan = device::compress::OnlinePrefillPlan; -using IndiceT = int32_t; - -/// \brief Need to reduce register usage to increase occupancy -struct Compress128OnlineDecodeParams { - /** \brief Shape: `[num_indices, 1, head_dim * 3 (max, sum, kv) ]` \n */ - void* __restrict__ kv_score_buffer; - /** \brief Shape: `[batch_size, head_dim * 2]` */ - const void* __restrict__ kv_score_input; - /** \brief Shape: `[batch_size, head_dim]` */ - void* __restrict__ kv_compressed_output; - /** \brief Shape: `[128, head_dim]` (called `ape`) */ - const void* __restrict__ score_bias; - /** \brief Shape: `[batch_size, ]`*/ - const IndiceT* __restrict__ indices; - /** \brief Shape: `[batch_size, ]` */ - const IndiceT* __restrict__ seq_lens; - /** \NOTE: `batch_size` <= `num_indices` */ - uint32_t batch_size; -}; - -/// \brief Need to reduce register usage to increase occupancy -struct Compress128OnlinePrefillParams { - /** \brief Shape: `[num_indices, 1, head_dim * 3 (max, sum, kv) ]` \n */ - void* __restrict__ kv_score_buffer; - /** \brief Shape: `[num_q_tokens, head_dim * 2]` */ - const void* __restrict__ kv_score_input; - /** \brief Shape: `[num_q_tokens, head_dim]` */ - void* __restrict__ kv_compressed_output; - /** \brief Shape: `[128, head_dim]` (called `ape`) */ - const void* __restrict__ score_bias; - /** \brief Shape: `[batch_size, ]`*/ - const IndiceT* __restrict__ indices; - /** \brief Shape: `[batch_size, ]`*/ - const IndiceT* __restrict__ load_indices; - /// \brief Plan for segments that close a chunk (write to `kv_compressed_output`). - /// Shape: `[num_compress, 16]` (uint8). - const OnlinePlan* __restrict__ compress_plan; - /// \brief Plan for the trailing partial segment of each batch (write back to - /// `kv_score_buffer`). Shape: `[num_write, 16]` (uint8). - const OnlinePlan* __restrict__ write_plan; - uint32_t num_compress; - uint32_t num_write; -}; - -// 4 elements per thread, kHeadDim / 4 threads per block -template -__global__ void flash_c128_online_decode(const __grid_constant__ Compress128OnlineDecodeParams params) { - using namespace device; - constexpr uint32_t kVecSize = 4; - constexpr uint32_t kBlockSize = kHeadDim / kVecSize; - using Vec = AlignedVector; - const auto gmem = tile::Memory::cta(kBlockSize); - const auto batch_id = blockIdx.x; - const auto index = params.indices[batch_id]; - const auto seq_len = params.seq_lens[batch_id]; - - const auto kv_score_buffer = static_cast(params.kv_score_buffer); - const auto kv_buf = kv_score_buffer + index * (kHeadDim * 3); - const auto kv_score_input = static_cast(params.kv_score_input); - const auto kv_src = kv_score_input + batch_id * (kHeadDim * 2); - - /// NOTE: kv_score_buffer layout is [max, sum, kv] (slot 0 / 1 / 2). Reads, - /// writes, and the prefill kernel must all agree on this order. - const auto max_score_vec = gmem.load(kv_buf, 0); - const auto sum_score_vec = gmem.load(kv_buf, 1); - const auto old_kv_vec = gmem.load(kv_buf, 2); - - /// NOTE: kv_score_input layout is | kv | score | (head_dim each), matching - /// the offline c128 kernel and the online prefill kernel. - const auto new_kv_vec = gmem.load(kv_src, 0); - const auto new_score_raw_vec = gmem.load(kv_src, 1); - - /// NOTE: the new token sits at global position `seq_len - 1`, so its - /// position inside the 128-chunk is `(seq_len - 1) % 128`. The previous - /// `seq_len % 128` was off by one (`bias[127]` vs `bias[0]`, etc.). - const auto pos_in_chunk = (seq_len - 1) % 128; - const auto bias_vec = gmem.load(params.score_bias, pos_in_chunk); - - Vec out_kv_vec; - Vec out_max_vec; - Vec out_sum_vec; - if (pos_in_chunk != 0) { - // Mid-chunk: combine prior partial state with the new token via online softmax. -#pragma unroll - for (uint32_t i = 0; i < 4; ++i) { - const auto old_max = max_score_vec[i]; - const auto old_kv = old_kv_vec[i]; - const auto new_score = new_score_raw_vec[i] + bias_vec[i]; - const auto new_kv = new_kv_vec[i]; - const auto new_max = fmax(old_max, new_score); - const auto old_sum = sum_score_vec[i] * expf(old_max - new_max); - const auto new_exp = expf(new_score - new_max); - const auto new_sum = old_sum + new_exp; - out_kv_vec[i] = (old_kv * old_sum + new_kv * new_exp) / new_sum; - out_max_vec[i] = new_max; - out_sum_vec[i] = new_sum; - } - } else { - // First token of a new 128-chunk: initialize state with this token alone. -#pragma unroll - for (uint32_t i = 0; i < 4; ++i) { - out_kv_vec[i] = new_kv_vec[i]; - out_max_vec[i] = new_score_raw_vec[i] + bias_vec[i]; - out_sum_vec[i] = 1.0f; // exp(score - max) with max == score - } - } - - if (pos_in_chunk == 127) { - // Chunk just closed: emit the compressed kv. No need to update the buffer - // -- the next chunk's first token will overwrite it. - const auto kv_out = static_cast(params.kv_compressed_output) + batch_id * kHeadDim; - gmem.store(kv_out, out_kv_vec); - } else { - // Otherwise persist the running [max, sum, kv] state for the next step. - gmem.store(kv_buf, out_max_vec, 0); - gmem.store(kv_buf, out_sum_vec, 1); - gmem.store(kv_buf, out_kv_vec, 2); - } -} - -constexpr int32_t kTileElements = 2; // split (along head-dim) -/// \brief Each warp will handle this many elements (split along softmax-128) -constexpr int32_t kElementsPerWarp = 8; -constexpr uint32_t kNumWarps = 128 / kElementsPerWarp; -constexpr uint32_t kPrefillBlockSize = device::kWarpThreads * kNumWarps; -using PrefillStorage = device::AlignedVector; - -struct Compress128SharedBuffer { - using Storage = device::AlignedVector; - Storage data[kNumWarps][device::kWarpThreads + 1]; // padding to avoid bank conflict - SGL_DEVICE Storage& operator()(uint32_t warp_id, uint32_t lane_id) { - return data[warp_id][lane_id]; - } - SGL_DEVICE float& operator()(uint32_t warp_id, uint32_t lane_id, uint32_t tile_id) { - return data[warp_id][lane_id][tile_id]; - } -}; - -template -SGL_DEVICE void c128_prefill_forward( - const PrefillStorage (&kv)[kElementsPerWarp], - const PrefillStorage (&score)[kElementsPerWarp], - float* kv_out, - float* max_out, - float* sum_out, - const uint32_t warp_id, - const uint32_t lane_id) { - using namespace device; - - /// NOTE: part 2: safe online softmax + weighted sum - using TmpStorage = typename Compress128SharedBuffer::Storage; - __shared__ Compress128SharedBuffer s_local_val_max; - __shared__ Compress128SharedBuffer s_local_exp_sum; - __shared__ Compress128SharedBuffer s_local_product; - - TmpStorage tmp_val_max; - TmpStorage tmp_exp_sum; - TmpStorage tmp_product; - -#pragma unroll - for (int32_t i = 0; i < kTileElements; ++i) { - float score_fp32[kElementsPerWarp]; - -#pragma unroll - for (int32_t j = 0; j < kElementsPerWarp; ++j) { - score_fp32[j] = score[j][i]; - } - - float max_value = score_fp32[0]; - float sum_exp_value = 0.0f; - -#pragma unroll - for (int32_t j = 1; j < kElementsPerWarp; ++j) { - const auto fp32_score = score_fp32[j]; - max_value = fmaxf(max_value, fp32_score); - } - - float sum_product = 0.0f; -#pragma unroll - for (int32_t j = 0; j < 8; ++j) { - const auto fp32_score = score_fp32[j]; - const auto exp_score = expf(fp32_score - max_value); - sum_product += cast(kv[j][i]) * exp_score; - sum_exp_value += exp_score; - } - - tmp_val_max[i] = max_value; - tmp_exp_sum[i] = sum_exp_value; - tmp_product[i] = sum_product; - } - - // naturally aligned, so no bank conflict - s_local_val_max(warp_id, lane_id) = tmp_val_max; - s_local_exp_sum(warp_id, lane_id) = tmp_exp_sum; - s_local_product(warp_id, lane_id) = tmp_product; - - __syncthreads(); - - /// NOTE: part 3: online softmax - /// NOTE: We have `kTileElements * kWarpThreads * kNumWarps` values to reduce - /// each reduce will consume `kNumWarps` threads (use partial warp reduction) - constexpr uint32_t kReductionCount = kTileElements * kWarpThreads * kNumWarps; - constexpr uint32_t kIteration = kReductionCount / kPrefillBlockSize; - -#pragma unroll - for (uint32_t i = 0; i < kIteration; ++i) { - /// NOTE: Range `[0, kTileElements * kWarpThreads * kNumWarps)` - const uint32_t j = i * kPrefillBlockSize + warp_id * kWarpThreads + lane_id; - /// NOTE: Range `[0, kNumWarps)` - const uint32_t local_warp_id = j % kNumWarps; - /// NOTE: Range `[0, kTileElements * kWarpThreads)` - const uint32_t local_elem_id = j / kNumWarps; - /// NOTE: Range `[0, kTileElements)` - const uint32_t local_tile_id = local_elem_id % kTileElements; - /// NOTE: Range `[0, kWarpThreads)` - const uint32_t local_lane_id = local_elem_id / kTileElements; - /// NOTE: each warp will access the whole tile (all `kTileElements`) - /// and for different lanes, the memory access only differ in `local_warp_id` - /// so there's no bank conflict in shared memory access. - static_assert(kTileElements * kNumWarps == kWarpThreads, "TODO: support other configs"); - const auto local_val_max = s_local_val_max(local_warp_id, local_lane_id, local_tile_id); - const auto local_exp_sum = s_local_exp_sum(local_warp_id, local_lane_id, local_tile_id); - const auto local_product = s_local_product(local_warp_id, local_lane_id, local_tile_id); - const auto global_val_max = warp::reduce_max(local_val_max); - const auto rescale = expf(local_val_max - global_val_max); - const auto global_exp_sum = warp::reduce_sum(local_exp_sum * rescale); - const auto final_scale = rescale / global_exp_sum; - const auto global_product = warp::reduce_sum(local_product * final_scale); - kv_out[local_elem_id] = global_product; - if constexpr (kNeedData) { - max_out[local_elem_id] = global_val_max; - sum_out[local_elem_id] = global_exp_sum; - } - } - if constexpr (kNeedData) __syncthreads(); -} - -/// \brief Sentinel score for padded positions in a 128-segment. -/// Must be finite so that `score - max` never produces NaN even when an -/// entire warp has only padded positions. -constexpr float kPadScore = -FLT_MAX; - -/// \brief Online compress 128 prefill. Two passes share this body: -/// - `kWrite=false` (compress pass): handles segments that close a chunk. -/// May load prior partial state from the buffer, but never writes to it, -/// so concurrent blocks can read the same slot without racing. -/// - `kWrite=true` (write pass): handles the trailing partial segment of each -/// batch. Each batch contributes at most one such plan, so concurrent blocks -/// touch disjoint buffer slots. -/// -/// The two passes MUST run as separate kernel launches (in stream order) so -/// that all reads in pass 1 finish before any writes in pass 2 start. -template -__global__ __launch_bounds__(kPrefillBlockSize, 2) // - void flash_c128_online_prefill(const __grid_constant__ Compress128OnlinePrefillParams params) { - using namespace device; - - constexpr int64_t kTileDim = kTileElements * kWarpThreads; // 64 - constexpr uint32_t kNumSplit = kHeadDim / kTileDim; - static_assert(kHeadDim % kTileDim == 0, "Head dim must be multiple of tile dim"); - - /// NOTE: the compiler folds the if-else at compile time. - const auto num_plans = kWrite ? params.num_write : params.num_compress; - const auto plan_ptr = kWrite ? params.write_plan : params.compress_plan; - const uint32_t global_id = blockIdx.x; - const uint32_t global_pid = global_id / kNumSplit; // plan id - const uint32_t global_sid = global_id % kNumSplit; // split id - if (global_pid >= num_plans) return; - const auto [ragged_id, batch_id, position, window_len] = plan_ptr[global_pid]; - if (ragged_id == 0xFFFFFFFFu) [[unlikely]] - return; - - const uint32_t warp_id = threadIdx.x / kWarpThreads; - const uint32_t lane_id = threadIdx.x % kWarpThreads; - const int32_t split_offset = global_sid * kTileDim; // int32 is enough - - const auto kv_score_buffer = static_cast(params.kv_score_buffer); - const auto kv_score_input = static_cast(params.kv_score_input); - const auto kv_compressed_output = static_cast(params.kv_compressed_output); - const auto score_bias_base = static_cast(params.score_bias); - - constexpr int64_t kElementSize = kHeadDim * 2; // | kv | score | - const uint32_t chunk_offset = (position % 128u) + 1u - window_len; - const uint32_t window_end = chunk_offset + window_len; // exclusive, in [1, 128] - const int32_t segment_start = ragged_id - (position % 128u); // can be negative, but safe - const int32_t load_index = chunk_offset != 0 ? params.load_indices[batch_id] : -1; - const int32_t store_index = kWrite ? params.indices[batch_id] : -1; - - PDLWaitPrimary(); - - // 2 * 8 = 16 register per elem. in theory we should consume 48 register here - PrefillStorage kv[kElementsPerWarp]; - PrefillStorage score[kElementsPerWarp]; - PrefillStorage bias[kElementsPerWarp]; - const auto warp_offset = warp_id * kElementsPerWarp; - -#pragma unroll - for (uint32_t i = 0; i < kElementsPerWarp; ++i) { - const uint32_t j = i + warp_offset; - if (j >= chunk_offset && j < window_end) { - const auto kv_src_ptr = kv_score_input + (segment_start + j) * kElementSize + split_offset; - const auto score_src_ptr = kv_src_ptr + kHeadDim; - const auto bias_src_ptr = score_bias_base + j * kHeadDim + split_offset; - kv[i].load(kv_src_ptr, lane_id); - score[i].load(score_src_ptr, lane_id); - bias[i].load(bias_src_ptr, lane_id); - } - } - -#pragma unroll - for (uint32_t i = 0; i < kElementsPerWarp; ++i) { - const uint32_t j = i + warp_offset; - const bool is_valid = (j >= chunk_offset && j < window_end); -#pragma unroll - for (uint32_t ii = 0; ii < kTileElements; ++ii) { - score[i][ii] = is_valid ? score[i][ii] + bias[i][ii] : kPadScore; - /// NOTE: must zero out kv on padded slots -- `c128_prefill_forward` - /// computes `kv * exp_score` where `exp_score = expf(-FLT_MAX - max) ??? 0`, - /// and IEEE-754 makes `NaN * 0 = NaN` / `+-inf * 0 = NaN`. An - /// uninitialized register can hold a NaN/inf bit pattern, so without - /// this reset a single padded warp can poison the whole softmax. - kv[i][ii] = is_valid ? kv[i][ii] : 0.0f; - } - } - - __shared__ alignas(16) float seg_kv[kTileDim]; - __shared__ alignas(16) float seg_max[kTileDim]; - __shared__ alignas(16) float seg_sum[kTileDim]; - - c128_prefill_forward(kv, score, seg_kv, seg_max, seg_sum, warp_id, lane_id); - - PDLTriggerSecondary(); - - if (warp_id == 0) { - PrefillStorage out_kv_vec, out_max_vec, out_sum_vec; - out_kv_vec.load(seg_kv, lane_id); - out_max_vec.load(seg_max, lane_id); - out_sum_vec.load(seg_sum, lane_id); - if (chunk_offset != 0) { - /// NOTE: load (max, sum, kv) of the in-progress chunk for this index. - /// `load_indices` may differ from `indices` when the prior partial state - /// lives on a different slot than the slot we ultimately write to. - const auto buf_load = kv_score_buffer + load_index * (kHeadDim * 3) + split_offset; - PrefillStorage buf_max_vec, buf_sum_vec, buf_kv_vec; - buf_max_vec.load(buf_load + 0 * kHeadDim, lane_id); - buf_sum_vec.load(buf_load + 1 * kHeadDim, lane_id); - buf_kv_vec.load(buf_load + 2 * kHeadDim, lane_id); -#pragma unroll - for (uint32_t ii = 0; ii < kTileElements; ++ii) { - const float m1 = buf_max_vec[ii]; - const float s1 = buf_sum_vec[ii]; - const float k1 = buf_kv_vec[ii]; - const float m2 = out_max_vec[ii]; - const float s2 = out_sum_vec[ii]; - const float k2 = out_kv_vec[ii]; - const float new_max = fmaxf(m1, m2); - const float new_s1 = s1 * expf(m1 - new_max); - const float new_s2 = s2 * expf(m2 - new_max); - const float new_sum = new_s1 + new_s2; - const float new_kv = (k1 * new_s1 + k2 * new_s2) / new_sum; - out_max_vec[ii] = new_max; - out_sum_vec[ii] = new_sum; - out_kv_vec[ii] = new_kv; - } - } - - if constexpr (kWrite) { - const auto buf_store = kv_score_buffer + store_index * (kHeadDim * 3) + split_offset; - reinterpret_cast(buf_store + 0 * kHeadDim)[lane_id] = out_max_vec; - reinterpret_cast(buf_store + 1 * kHeadDim)[lane_id] = out_sum_vec; - reinterpret_cast(buf_store + 2 * kHeadDim)[lane_id] = out_kv_vec; - } else { - const auto out_ptr = kv_compressed_output + ragged_id * kHeadDim + split_offset; - reinterpret_cast(out_ptr)[lane_id] = out_kv_vec; - } - } -} - -template -struct FlashCompress128OnlineKernel { - static constexpr auto decode_kernel = flash_c128_online_decode; - template - static constexpr auto prefill_kernel = flash_c128_online_prefill; - static constexpr auto prefill_c_kernel = prefill_kernel; - static constexpr auto prefill_w_kernel = prefill_kernel; - static constexpr int64_t kTileDim = kTileElements * device::kWarpThreads; // 64 - static constexpr uint32_t kNumSplit = kHeadDim / kTileDim; - static constexpr uint32_t kDecodeBlockSize = kHeadDim / 4; - - static void run_decode( - const tvm::ffi::TensorView kv_score_buffer, - const tvm::ffi::TensorView kv_score_input, - const tvm::ffi::TensorView kv_compressed_output, - const tvm::ffi::TensorView ape, - const tvm::ffi::TensorView indices, - const tvm::ffi::TensorView seq_lens, - const tvm::ffi::Optional /* UNUSED */) { - using namespace host; - - auto B = SymbolicSize{"batch_size"}; - auto device = SymbolicDevice{}; - device.set_options(); - - TensorMatcher({-1, 1, kHeadDim * 3}) // kv score buffer (max, sum, kv) - .with_dtype() - .with_device(device) - .verify(kv_score_buffer); - TensorMatcher({B, kHeadDim * 2}) // kv score input - .with_dtype() - .with_device(device) - .verify(kv_score_input); - TensorMatcher({B, kHeadDim}) // kv compressed output - .with_dtype() - .with_device(device) - .verify(kv_compressed_output); - TensorMatcher({128, kHeadDim}) // ape - .with_dtype() - .with_device(device) - .verify(ape); - TensorMatcher({B}).with_dtype().with_device(device).verify(indices); - TensorMatcher({B}).with_dtype().with_device(device).verify(seq_lens); - - const auto batch_size = static_cast(B.unwrap()); - const auto params = Compress128OnlineDecodeParams{ - .kv_score_buffer = kv_score_buffer.data_ptr(), - .kv_score_input = kv_score_input.data_ptr(), - .kv_compressed_output = kv_compressed_output.data_ptr(), - .score_bias = ape.data_ptr(), - .indices = static_cast(indices.data_ptr()), - .seq_lens = static_cast(seq_lens.data_ptr()), - .batch_size = batch_size, - }; - LaunchKernel(batch_size, kDecodeBlockSize, device.unwrap()) // - .enable_pdl(kUsePDL)(decode_kernel, params); - } - - static void run_prefill( - const tvm::ffi::TensorView kv_score_buffer, - const tvm::ffi::TensorView kv_score_input, - const tvm::ffi::TensorView kv_compressed_output, - const tvm::ffi::TensorView ape, - const tvm::ffi::TensorView indices, - const tvm::ffi::TensorView compress_plan, - const tvm::ffi::TensorView write_plan, - const tvm::ffi::Optional extra) { - using namespace host; - using host::compress::kOnlinePrefillPlanDim; - using host::compress::OnlinePrefillPlanTensorDtype; - - auto B = SymbolicSize{"batch_size"}; - auto N = SymbolicSize{"num_q_tokens"}; - auto X = SymbolicSize{"compress_tokens"}; - auto Y = SymbolicSize{"write_tokens"}; - auto device_ = SymbolicDevice{}; - device_.set_options(); - - TensorMatcher({-1, 1, kHeadDim * 3}) // kv score buffer (max, sum, kv) ??? 2D - .with_dtype() - .with_device(device_) - .verify(kv_score_buffer); - TensorMatcher({N, kHeadDim * 2}) // kv score input - .with_dtype() - .with_device(device_) - .verify(kv_score_input); - TensorMatcher({N, kHeadDim}) // kv compressed output - .with_dtype() - .with_device(device_) - .verify(kv_compressed_output); - TensorMatcher({128, kHeadDim}) // ape - .with_dtype() - .with_device(device_) - .verify(ape); - TensorMatcher({B}) // indices - .with_dtype() - .with_device(device_) - .verify(indices); - TensorMatcher({X, kOnlinePrefillPlanDim}) // compress plan - .with_dtype() - .with_device(device_) - .verify(compress_plan); - TensorMatcher({Y, kOnlinePrefillPlanDim}) // write plan - .with_dtype() - .with_device(device_) - .verify(write_plan); - - /// NOTE: `extra` is `load_indices`. When the previous partial state lives - /// on a slot different from the destination slot (e.g. paged buffers), the - /// caller must supply this; otherwise it defaults to `indices`. - const auto load_indices = extra.value_or(indices); - TensorMatcher({B}).with_dtype().with_device(device_).verify(load_indices); - - const auto device = device_.unwrap(); - const auto num_c = static_cast(X.unwrap()); - const auto num_w = static_cast(Y.unwrap()); - const auto params = Compress128OnlinePrefillParams{ - .kv_score_buffer = kv_score_buffer.data_ptr(), - .kv_score_input = kv_score_input.data_ptr(), - .kv_compressed_output = kv_compressed_output.data_ptr(), - .score_bias = ape.data_ptr(), - .indices = static_cast(indices.data_ptr()), - .load_indices = static_cast(load_indices.data_ptr()), - .compress_plan = static_cast(compress_plan.data_ptr()), - .write_plan = static_cast(write_plan.data_ptr()), - .num_compress = num_c, - .num_write = num_w, - }; - - /// NOTE: pass 1 reads the buffer (for the first segment of each batch - /// that started mid-chunk) and writes only to `kv_compressed_output`. - /// Pass 2 then writes the trailing partial state of each batch back to - /// the buffer. Stream serialization between the two launches enforces - /// read-before-write on shared buffer slots. - if (const auto num_c_blocks = num_c * kNumSplit) { - LaunchKernel(num_c_blocks, kPrefillBlockSize, device) // - .enable_pdl(kUsePDL)(prefill_c_kernel, params); - } - if (const auto num_w_blocks = num_w * kNumSplit) { - LaunchKernel(num_w_blocks, kPrefillBlockSize, device) // - .enable_pdl(kUsePDL)(prefill_w_kernel, params); - } - } -}; - -namespace host::compress { - -using OnlinePlanResult = tvm::ffi::Tuple; - -struct OnlinePrefillCompressParams { - OnlinePrefillPlan* __restrict__ compress_plan; - OnlinePrefillPlan* __restrict__ write_plan; - const int64_t* __restrict__ seq_lens; - const int64_t* __restrict__ extend_lens; - uint32_t batch_size; - uint32_t num_tokens; -}; - -/// \brief Build the compress + write plans for online compress 128 prefill. -/// -/// Each batch's `[prefix_len, prefix_len + extend_len)` range is split at -/// 128-aligned boundaries. Every resulting segment falls into one of: -/// - **compress**: closes a 128-chunk (`chunk_offset + window_len == 128`). -/// These plans only read the buffer (when starting mid-chunk) and write the -/// compressed kv to `kv_compressed_output`. -/// - **write**: trailing partial of the batch (`chunk_offset + window_len < 128`). -/// May read the buffer and always writes the new partial state back to it. -/// Each batch produces at most one such plan. -/// -/// The two plans MUST be dispatched as separate kernel launches in stream -/// order so that pass-1 reads of a buffer slot complete before any pass-2 -/// write of the same slot. -inline OnlinePlanResult plan_online_prefill_host(const OnlinePrefillCompressParams& params, const bool use_cuda_graph) { - const auto& [compress_plan, write_plan, seq_lens, extend_lens, batch_size, num_tokens] = params; - - uint32_t counter = 0; - uint32_t compress_count = 0; - uint32_t write_count = 0; - for (const auto i : irange(batch_size)) { - const uint32_t seq_len = static_cast(seq_lens[i]); - const uint32_t extend_len = static_cast(extend_lens[i]); - RuntimeCheck(0 < extend_len && extend_len <= seq_len); - const uint32_t prefix_len = seq_len - extend_len; - const uint32_t end_pos = prefix_len + extend_len; - /// NOTE: split the extend range into per-128-chunk segments. Each segment - /// stays inside one chunk, so the kernel can decide load/store from - /// `chunk_offset` and `window_len` alone. - uint32_t pos = prefix_len; - while (pos < end_pos) { - const uint32_t chunk_start = (pos / 128u) * 128u; - const uint32_t seg_end = std::min(end_pos, chunk_start + 128u); // exclusive - const uint32_t seg_len = seg_end - pos; - const uint32_t chunk_off = pos - chunk_start; - /// NOTE: store last-token coordinates so that downstream consumers - /// (e.g. `fused_norm_rope`) can read `ragged_id` and `position` with the - /// same semantics as `PrefillPlan`. The segment start is recoverable as - /// `ragged_id - window_len + 1` and `position - window_len + 1`. - const uint32_t last_pos = seg_end - 1; - const uint32_t last_ragged = counter + (last_pos - prefix_len); - const auto plan = OnlinePrefillPlan{ - .ragged_id = last_ragged, - .batch_id = i, - .position = last_pos, - .window_len = seg_len, - }; - if (chunk_off + seg_len == 128u) { - // full chunk, must be complete, maybe read the buffer, no write - RuntimeCheck(compress_count < num_tokens); - compress_plan[compress_count++] = plan; - } else { - // last chunk, must be incomplete, maybe read the buffer, must write - RuntimeCheck(write_count < num_tokens); - write_plan[write_count++] = plan; - } - pos = seg_end; - } - counter += extend_len; - } - RuntimeCheck(counter == num_tokens, "input size ", counter, " != num_q_tokens ", num_tokens); - if (!use_cuda_graph) return OnlinePlanResult{compress_count, write_count}; - /// NOTE: pad both plans with sentinel entries so cuda-graph runs always see - /// the same number of blocks. The kernel skips plans whose `ragged_id` is -1. - constexpr auto kInvalid = static_cast(-1); - constexpr auto kInvalidPlan = OnlinePrefillPlan{kInvalid, kInvalid, kInvalid, kInvalid}; - for (const auto i : irange(compress_count, num_tokens)) { - compress_plan[i] = kInvalidPlan; - } - for (const auto i : irange(write_count, num_tokens)) { - write_plan[i] = kInvalidPlan; - } - return OnlinePlanResult{num_tokens, num_tokens}; -} - -inline OnlinePlanResult plan_online_prefill( - const tvm::ffi::TensorView extend_lens, - const tvm::ffi::TensorView seq_lens, - const tvm::ffi::TensorView compress_plan, - const tvm::ffi::TensorView write_plan, - const bool use_cuda_graph) { - auto N = SymbolicSize{"batch_size"}; - auto M = SymbolicSize{"num_tokens"}; - auto device = SymbolicDevice{}; - /// NOTE: only host (CPU/cuda-host) planning is implemented for now. The - device.set_options(); - TensorMatcher({N}) // - .with_dtype() - .with_device(device) - .verify(extend_lens) - .verify(seq_lens); - TensorMatcher({M, kOnlinePrefillPlanDim}) // - .with_dtype() - .with_device(device) - .verify(compress_plan) - .verify(write_plan); - const auto params = OnlinePrefillCompressParams{ - .compress_plan = static_cast(compress_plan.data_ptr()), - .write_plan = static_cast(write_plan.data_ptr()), - .seq_lens = static_cast(seq_lens.data_ptr()), - .extend_lens = static_cast(extend_lens.data_ptr()), - .batch_size = static_cast(N.unwrap()), - .num_tokens = static_cast(M.unwrap()), - }; - return plan_online_prefill_host(params, use_cuda_graph); -} - -} // namespace host::compress - -[[maybe_unused]] -constexpr auto& plan_compress_online_prefill = host::compress::plan_online_prefill; - -} // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/c128_online_v2.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/c128_online_v2.cuh index 097acb141a..ad4f1f28ce 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/c128_online_v2.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/c128_online_v2.cuh @@ -233,8 +233,8 @@ constexpr float kPadScore = -FLT_MAX; __syncthreads(); - // Cross-warp reduction. Same recipe as c128_online.cuh: each block-thread - // pair reduces a (tile_id, lane_id) slot using a kNumWarps-wide warp shuffle. + // Cross-warp reduction: each block-thread pair reduces a (tile_id, lane_id) + // slot using a kNumWarps-wide warp shuffle. constexpr uint32_t kReductionCount = kTileElements * kWarpThreads * kNumWarps; constexpr uint32_t kIteration = kReductionCount / kPrefillBlockSize; static_assert(kTileElements * kNumWarps == kWarpThreads, "TODO: support other configs"); diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/c4.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/c4.cuh deleted file mode 100644 index b5bec148c9..0000000000 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/c4.cuh +++ /dev/null @@ -1,549 +0,0 @@ -#include -#include - -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -#include - -namespace sglang { - -using Plan4 = device::compress::PrefillPlan; -using IndiceT = int32_t; - -/// \brief Each thread will handle this many elements (split along head_dim) -constexpr int kTileElements = 4; - -/// \brief Need to improve register usage to reduce latency -#define C4_KERNEL __global__ __launch_bounds__(128, 4) - -enum class PageMode { - RingBuffer = 8, - Page4Align = 4, -}; - -struct alignas(16) C4IndexBundle { - int32_t load_first_page; - int32_t load_second_page; - int32_t write_first_page; - int32_t last_position; -}; - -struct Compress4DecodeParams { - /** - * \brief Shape: `[num_indices, 8, head_dim * 4]` \n - * last dimension layout: - * | kv overlap | kv | score overlap | score | - */ - void* __restrict__ kv_score_buffer; - /** \brief Shape: `[batch_size, head_dim * 4]` */ - const void* __restrict__ kv_score_input; - /** \brief Shape: `[batch_size, head_dim]` */ - void* __restrict__ kv_compressed_output; - /** \brief Shape: `[8, head_dim]` (called `ape`) */ - const void* __restrict__ score_bias; - /** \brief Shape: `[batch_size, ]`*/ - const IndiceT* __restrict__ indices; - /** \brief Shape: `[batch_size, ]` */ - const IndiceT* __restrict__ seq_lens; - /** \brief Shape: `[batch_size, 1]` */ - const int32_t* __restrict__ extra; - /** \NOTE: `batch_size` <= `num_indices` */ - uint32_t batch_size; -}; - -struct Compress4PrefillParams { - /** - * \brief Shape: `[num_indices, 8, head_dim * 4]` \n - * last dimension layout: - * | kv overlap | kv | score overlap | score | - */ - void* __restrict__ kv_score_buffer; - /** \brief Shape: `[num_q_tokens, head_dim * 4]` */ - const void* __restrict__ kv_score_input; - /** \brief Shape: `[num_q_tokens, head_dim]` */ - void* __restrict__ kv_compressed_output; - /** \brief Shape: `[8, head_dim]` (called `ape`) */ - const void* __restrict__ score_bias; - /** \brief Shape: `[batch_size, ]`*/ - const IndiceT* __restrict__ indices; - /** \brief Shape: `[batch_size, 4]` */ - const C4IndexBundle* __restrict__ extra; - /** \brief The following part is plan info. */ - - const Plan4* __restrict__ compress_plan; - const Plan4* __restrict__ write_plan; - uint32_t num_compress; - uint32_t num_write; -}; - -template -SGL_DEVICE void c4_write( - T* kv_score_buf, // - const T* kv_score_src, - const int64_t head_dim, - const int32_t write_pos) { - using namespace device; - - using Storage = AlignedVector; - const auto element_size = head_dim * 4; - const auto gmem = tile::Memory::warp(); - kv_score_buf += write_pos * element_size; - - /// NOTE: Layout | [0] = kv overlap | [1] = kv | [2] = score overlap | [3] = score | - Storage kv_score[4]; -#pragma unroll - for (int32_t i = 0; i < 4; ++i) { - kv_score[i] = gmem.load(kv_score_src + head_dim * i); - } -#pragma unroll - for (int32_t i = 0; i < 4; ++i) { - gmem.store(kv_score_buf + head_dim * i, kv_score[i]); - } -} - -template -SGL_DEVICE void c4_forward( - const InFloat* kv_score_buf, - const InFloat* kv_score_src, - OutFloat* kv_out, - const InFloat* score_bias, - const int64_t head_dim, - const int32_t seq_len, - const int32_t window_len, - [[maybe_unused]] const InFloat* kv_score_overlap_buf = nullptr) { - using namespace device; - - const auto element_size = head_dim * 4; - const auto score_offset = head_dim * 2; - const auto overlap_stride = head_dim; - - /// NOTE: part 1: load kv + score - using StorageIn = AlignedVector; - const auto gmem_in = tile::Memory::warp(); - StorageIn kv[8]; - StorageIn score[8]; - StorageIn bias[8]; - -#pragma unroll - for (int32_t i = 0; i < 8; ++i) { - bias[i] = gmem_in.load(score_bias + i * head_dim); - } - -#pragma unroll - for (int32_t i = 0; i < 8; ++i) { - const bool is_overlap = i < 4; - const InFloat* src; - if (i < window_len) { - /// NOTE: `seq_len` must be a multiple of 4 here - if constexpr (kPaged) { - const auto kv_score_ptr = is_overlap ? kv_score_overlap_buf : kv_score_buf; - const int32_t k = i % 4; - src = kv_score_ptr + k * element_size; - } else { - const int32_t k = (seq_len + i) % 8; - src = kv_score_buf + k * element_size; - } - } else { - /// NOTE: k in [-7, 0]. We'll load from the ragged `kv_score_src` - const int32_t k = i - 7; - src = kv_score_src + k * element_size; - } - src += (is_overlap ? 0 : overlap_stride); - kv[i] = gmem_in.load(src); - score[i] = gmem_in.load(src + score_offset); - } - - if (seq_len == 4) { - [[unlikely]]; - constexpr float kFloatNegInf = -1e9f; -#pragma unroll - for (int32_t i = 0; i < 4; ++i) { - kv[i].fill(cast(0.0f)); - score[i].fill(cast(kFloatNegInf)); - } - } - - /// NOTE: part 2: safe online softmax + weighted sum - using StorageOut = AlignedVector; - const auto gmem_out = tile::Memory::warp(); - StorageOut result; - -#pragma unroll - for (int32_t i = 0; i < kTileElements; ++i) { - float score_fp32[8]; - -#pragma unroll - for (int32_t j = 0; j < 8; ++j) { - score_fp32[j] = cast(score[j][i]) + cast(bias[j][i]); - } - - float max_value = score_fp32[0]; - float sum_exp_value = 0.0f; - -#pragma unroll - for (int32_t j = 1; j < 8; ++j) { - const auto fp32_score = score_fp32[j]; - max_value = fmaxf(max_value, fp32_score); - } - - float sum_product = 0.0f; -#pragma unroll - for (int32_t j = 0; j < 8; ++j) { - const auto fp32_score = score_fp32[j]; - const auto exp_score = expf(fp32_score - max_value); - sum_product += cast(kv[j][i]) * exp_score; - sum_exp_value += exp_score; - } - - result[i] = cast(sum_product / sum_exp_value); - } - - gmem_out.store(kv_out, result); -} - -template -C4_KERNEL void flash_c4_decode(const __grid_constant__ Compress4DecodeParams params) { - using namespace device; - - constexpr int64_t kTileDim = kTileElements * kWarpThreads; // 128 - constexpr uint32_t kNumSplit = kHeadDim / kTileDim; - constexpr int64_t kElementSize = kHeadDim * 4; // `* 4` due to overlap transform + score - static_assert(kHeadDim % kTileDim == 0, "Head dim must be multiple of tile dim"); - - const auto& [ - _kv_score_buffer, _kv_score_input, _kv_compressed_output, _score_bias, // kv score - indices, seq_lens, extra, batch_size // decode info - ] = params; - const uint32_t global_tid = blockIdx.x * blockDim.x + threadIdx.x; - const uint32_t global_wid = global_tid / kWarpThreads; // warp id - const uint32_t global_bid = global_wid / kNumSplit; // batch id - const uint32_t global_sid = global_wid % kNumSplit; // split id - - if (global_bid >= batch_size) return; - - const int32_t index = indices[global_bid]; - const int32_t seq_len = seq_lens[global_bid]; - const int64_t split_offset = global_sid * kTileDim; - - // kv score - const auto kv_score_buffer = static_cast(_kv_score_buffer); - - // kv input - const auto kv_score_input = static_cast(_kv_score_input); - const auto kv_src = kv_score_input + global_bid * kElementSize + split_offset; - - // kv output - const auto kv_compressed_output = static_cast(_kv_compressed_output); - const auto kv_out = kv_compressed_output + global_bid * kHeadDim + split_offset; - - // score bias (ape) - const auto score_bias = static_cast(_score_bias) + split_offset; - - PDLWaitPrimary(); - - /// NOTE: `position` = `seq_len - 1`. To avoid underflow, we use `seq_len + page_size - 1` - if constexpr (kMode == PageMode::Page4Align) { - const auto index_prev = extra[global_bid]; - const auto kv_buf = kv_score_buffer + index * (kElementSize * 4) + split_offset; - c4_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/(seq_len + 3) % 4); - if (seq_len % 4 == 0) { - const auto kv_overlap = kv_buf + (index_prev - index) * (kElementSize * 4); - c4_forward(kv_buf, kv_src, kv_out, score_bias, kHeadDim, seq_len, 8, kv_overlap); - } - } else { - static_assert(kMode == PageMode::RingBuffer, "Unsupported PageMode"); - const auto kv_buf = kv_score_buffer + index * (kElementSize * 8) + split_offset; - c4_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/(seq_len + 7) % 8); - if (seq_len % 4 == 0) { - c4_forward(kv_buf, kv_src, kv_out, score_bias, kHeadDim, seq_len, /*window_size=*/8); - } - } - - PDLTriggerSecondary(); -} - -template -C4_KERNEL void flash_c4_prefill(const __grid_constant__ Compress4PrefillParams params) { - using namespace device; - - constexpr int64_t kTileDim = kTileElements * kWarpThreads; // 128 - constexpr uint32_t kNumSplit = kHeadDim / kTileDim; - constexpr int64_t kElementSize = kHeadDim * 4; // `* 4` due to overlap transform + score - static_assert(kHeadDim % kTileDim == 0, "Head dim must be multiple of tile dim"); - - const auto& [ - _kv_score_buffer, _kv_score_input, _kv_compressed_output, _score_bias, // kv score - indices, extra, compress_plan, write_plan, num_compress, num_write // prefill plan - ] = params; - - const uint32_t global_tid = blockIdx.x * blockDim.x + threadIdx.x; - const uint32_t global_wid = global_tid / kWarpThreads; // warp id - const uint32_t global_pid = global_wid / kNumSplit; // plan id - const uint32_t global_sid = global_wid % kNumSplit; // split id - - /// NOTE: compiler can optimize this if-else at compile time - const auto num_plans = kWrite ? num_write : num_compress; - const auto plan_ptr = kWrite ? write_plan : compress_plan; - if (global_pid >= num_plans) return; - - const auto& [ragged_id, global_bid, position, window_len] = plan_ptr[global_pid]; - const int64_t split_offset = global_sid * kTileDim; - - // kv score - const auto kv_score_buffer = static_cast(_kv_score_buffer); - - // kv input - const auto kv_score_input = static_cast(_kv_score_input); - const auto kv_src = kv_score_input + ragged_id * kElementSize + split_offset; - - // kv output - const auto kv_compressed_output = static_cast(_kv_compressed_output); - const auto kv_out = kv_compressed_output + ragged_id * kHeadDim + split_offset; - - if (ragged_id == 0xFFFFFFFF) [[unlikely]] - return; - - // score bias (ape) - const auto score_bias = static_cast(_score_bias) + split_offset; - const auto seq_len = position + 1; - const int32_t index = indices[global_bid]; - - PDLWaitPrimary(); - - if constexpr (kMode == PageMode::Page4Align) { - const auto write_second_page = index; - const auto [load_first_page, load_second_page, write_first_page, last_pos] = extra[global_bid]; - if constexpr (kWrite) { - int32_t index; - if (position < static_cast(last_pos)) { - index = write_first_page; - } else { - index = write_second_page; - } - const auto kv_buf = kv_score_buffer + index * (kElementSize * 4) + split_offset; - c4_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/position % 4); - } else { - int32_t index_overlap, index_normal; - if (window_len <= 4) { - index_overlap = load_second_page; - index_normal = load_second_page; // not used - } else { - index_overlap = load_first_page; - index_normal = load_second_page; - } - const auto kv_buf = kv_score_buffer + index_normal * (kElementSize * 4) + split_offset; - const auto kv_overlap = kv_score_buffer + index_overlap * (kElementSize * 4) + split_offset; - c4_forward(kv_buf, kv_src, kv_out, score_bias, kHeadDim, seq_len, window_len, kv_overlap); - } - } else { - static_assert(kMode == PageMode::RingBuffer, "Unsupported PageMode"); - const auto kv_buf = kv_score_buffer + index * (kElementSize * 8) + split_offset; - if constexpr (kWrite) { - c4_write(kv_buf, kv_src, kHeadDim, /*write_pos=*/position % 8); - } else { - c4_forward(kv_buf, kv_src, kv_out, score_bias, kHeadDim, seq_len, window_len); - } - } - - PDLTriggerSecondary(); -} - -template -struct FlashCompress4Kernel { - template - static constexpr auto decode_kernel = flash_c4_decode; - template - static constexpr auto prefill_kernel = flash_c4_prefill; - template - static constexpr auto prefill_c_kernel = prefill_kernel; - template - static constexpr auto prefill_w_kernel = prefill_kernel; - static constexpr uint32_t kBlockSize = 128; - static constexpr uint32_t kTileDim = kTileElements * device::kWarpThreads; - static constexpr uint32_t kNumSplit = kHeadDim / kTileDim; - static constexpr uint32_t kWarpsPerBlock = kBlockSize / device::kWarpThreads; - - using Self = FlashCompress4Kernel; - - static void run_decode( - const tvm::ffi::TensorView kv_score_buffer, - const tvm::ffi::TensorView kv_score_input, - const tvm::ffi::TensorView kv_compressed_output, - const tvm::ffi::TensorView ape, - const tvm::ffi::TensorView indices, - const tvm::ffi::TensorView seq_lens, - const tvm::ffi::Optional extra) { - using namespace host; - - // this should not happen in practice - auto B = SymbolicSize{"batch_size"}; - auto device_ = SymbolicDevice{}; - device_.set_options(); - const auto extra_ptr = _get_extra_pointer(B, device_, extra); - const auto page_size = extra_ptr != nullptr ? 4 : 8; - - TensorMatcher({-1, page_size, kHeadDim * 4}) // kv score - .with_dtype() - .with_device(device_) - .verify(kv_score_buffer); - TensorMatcher({B, kHeadDim * 4}) // kv score input - .with_dtype() - .with_device(device_) - .verify(kv_score_input); - TensorMatcher({B, kHeadDim}) // kv compressed output - .with_dtype() - .with_device(device_) - .verify(kv_compressed_output); - TensorMatcher({8, kHeadDim}) // ape - .with_dtype() - .with_device(device_) - .verify(ape); - TensorMatcher({B}) // indices - .with_dtype() - .with_device(device_) - .verify(indices); - TensorMatcher({B}) // seq lens - .with_dtype() - .with_device(device_) - .verify(seq_lens); - - const auto device = device_.unwrap(); - const auto batch_size = static_cast(B.unwrap()); - const auto params = Compress4DecodeParams{ - .kv_score_buffer = kv_score_buffer.data_ptr(), - .kv_score_input = kv_score_input.data_ptr(), - .kv_compressed_output = kv_compressed_output.data_ptr(), - .score_bias = ape.data_ptr(), - .indices = static_cast(indices.data_ptr()), - .seq_lens = static_cast(seq_lens.data_ptr()), - .extra = static_cast(extra_ptr), - .batch_size = batch_size, - }; - const auto kernel = extra_ptr != nullptr ? decode_kernel // - : decode_kernel; - const uint32_t num_blocks = div_ceil(batch_size * kNumSplit, kWarpsPerBlock); - LaunchKernel(num_blocks, kBlockSize, device) // - .enable_pdl(kUsePDL)(kernel, params); - } - - static void run_prefill( - const tvm::ffi::TensorView kv_score_buffer, - const tvm::ffi::TensorView kv_score_input, - const tvm::ffi::TensorView kv_compressed_output, - const tvm::ffi::TensorView ape, - const tvm::ffi::TensorView indices, - const tvm::ffi::TensorView compress_plan, - const tvm::ffi::TensorView write_plan, - const tvm::ffi::Optional extra) { - using namespace host; - - auto B = SymbolicSize{"batch_size"}; - auto N = SymbolicSize{"num_q_tokens"}; - auto X = SymbolicSize{"compress_tokens"}; - auto Y = SymbolicSize{"write_tokens"}; - auto device_ = SymbolicDevice{}; - device_.set_options(); - const auto extra_ptr = _get_extra_pointer(B, device_, extra, /*is_prefill=*/true); - const auto page_size = extra_ptr != nullptr ? 4 : 8; - - TensorMatcher({-1, page_size, kHeadDim * 4}) // kv score - .with_dtype() - .with_device(device_) - .verify(kv_score_buffer); - TensorMatcher({N, kHeadDim * 4}) // kv score input - .with_dtype() - .with_device(device_) - .verify(kv_score_input); - TensorMatcher({N, kHeadDim}) // kv compressed output - .with_dtype() - .with_device(device_) - .verify(kv_compressed_output); - TensorMatcher({8, kHeadDim}) // ape - .with_dtype() - .with_device(device_) - .verify(ape); - TensorMatcher({B}) // indices - .with_dtype() - .with_device(device_) - .verify(indices); - TensorMatcher({X, compress::kPrefillPlanDim}) // compress plan - .with_dtype() - .with_device(device_) - .verify(compress_plan); - TensorMatcher({Y, compress::kPrefillPlanDim}) // write plan - .with_dtype() - .with_device(device_) - .verify(write_plan); - - const auto device = device_.unwrap(); - const auto batch_size = static_cast(B.unwrap()); - const auto num_q_tokens = static_cast(N.unwrap()); - const auto num_c = static_cast(X.unwrap()); - const auto num_w = static_cast(Y.unwrap()); - const auto params = Compress4PrefillParams{ - .kv_score_buffer = kv_score_buffer.data_ptr(), - .kv_score_input = kv_score_input.data_ptr(), - .kv_compressed_output = kv_compressed_output.data_ptr(), - .score_bias = ape.data_ptr(), - .indices = static_cast(indices.data_ptr()), - .extra = static_cast(extra_ptr), - .compress_plan = static_cast(compress_plan.data_ptr()), - .write_plan = static_cast(write_plan.data_ptr()), - .num_compress = num_c, - .num_write = num_w, - }; - RuntimeCheck(num_q_tokens >= batch_size, "num_q_tokens must be >= batch_size"); - RuntimeCheck(num_q_tokens >= std::max(num_c, num_w), "invalid prefill plan"); - if (const auto num_c_blocks = div_ceil(num_c * kNumSplit, kWarpsPerBlock)) { - const auto c_kernel = extra_ptr != nullptr ? prefill_c_kernel // - : prefill_c_kernel; - LaunchKernel(num_c_blocks, kBlockSize, device) // - .enable_pdl(kUsePDL)(c_kernel, params); - } - if (const auto num_w_blocks = div_ceil(num_w * kNumSplit, kWarpsPerBlock)) { - const auto w_kernel = extra_ptr != nullptr ? prefill_w_kernel // - : prefill_w_kernel; - LaunchKernel(num_w_blocks, kBlockSize, device) // - .enable_pdl(kUsePDL)(w_kernel, params); - } - } - - // some auxiliary functions - private: - static const void* _get_extra_pointer( - host::SymbolicSize& B, // batch_size - host::SymbolicDevice& device, - const tvm::ffi::Optional& extra, - bool is_prefill = false) { - // only have value when using page-aligned mode - if (!extra.has_value()) return nullptr; - const auto& extra_tensor = extra.value(); - /// NOTE: the metadata layout is different for prefill and decode: - /// for prefill, last 4 are: - /// load overlap | load normal | write overlap | last written page - /// for decode, last 1 is the write (also load) overlap - host::TensorMatcher({B, is_prefill ? 4 : 1}) // extra tensor - .with_dtype() - .with_device(device) - .verify(extra_tensor); - const auto data_ptr = extra_tensor.data_ptr(); - host::RuntimeCheck(data_ptr != nullptr, "extra tensor data ptr is null"); - if (is_prefill) { - static_assert(alignof(C4IndexBundle) == 16); - host::RuntimeCheck(std::bit_cast(data_ptr) % 16 == 0, "extra tensor is not properly aligned"); - } - return data_ptr; - } -}; - -} // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/common.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/common.cuh deleted file mode 100644 index 16c8c70bee..0000000000 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/common.cuh +++ /dev/null @@ -1,208 +0,0 @@ -#include -#include - -#include - -#include - -namespace sglang { - -namespace host::compress { - -using PlanResult = tvm::ffi::Tuple; - -struct CompressParams { - PrefillPlan* __restrict__ compress_plan; - PrefillPlan* __restrict__ write_plan; - const int64_t* __restrict__ seq_lens; - const int64_t* __restrict__ extend_lens; - uint32_t batch_size; - uint32_t num_tokens; - uint32_t compress_ratio; - bool is_overlap; -}; - -inline constexpr uint32_t kBlockSize = 1024; - -#define PLAN_KERNEL __global__ __launch_bounds__(kBlockSize, 1) inline - -PLAN_KERNEL void plan_prefill_cuda(const __grid_constant__ CompressParams params) { - const auto &[ - compress_plan, write_plan, seq_lens, extend_lens, // pointers - batch_size, num_tokens, compress_ratio, is_overlap // values - ] = params; - - __shared__ uint32_t compress_counter; - __shared__ uint32_t write_counter; - - uint32_t batch_id = 0; - uint32_t counter = 0; - uint32_t extend_len = extend_lens[0]; - - const auto tid = threadIdx.x; - if (tid == 0) { - compress_counter = 0; - write_counter = 0; - } - __syncthreads(); - - for (uint32_t i = tid; i < num_tokens; i += blockDim.x) { - const uint32_t ragged_id = i; - uint32_t j = ragged_id - counter; - while (j >= extend_len) { - j -= extend_len; - batch_id += 1; - if (batch_id >= batch_size) [[unlikely]] - break; - counter += extend_len; - extend_len = extend_lens[batch_id]; - } - if (batch_id >= batch_size) [[unlikely]] - break; - const uint32_t seq_len = seq_lens[batch_id]; - const uint32_t extend_len = extend_lens[batch_id]; - const uint32_t prefix_len = seq_len - extend_len; - const uint32_t ratio = compress_ratio * (1 + is_overlap); - const uint32_t window_len = j + 1 < ratio ? ratio - (j + 1) : 0; - const uint32_t position = prefix_len + j; - const auto plan = PrefillPlan{ - .ragged_id = ragged_id, - .batch_id = batch_id, - .position = position, - .window_len = window_len, - }; - const uint32_t start_write_pos = [seq_len, compress_ratio, is_overlap] { - const uint32_t pos = seq_len / compress_ratio * compress_ratio; - if (!is_overlap) return pos; - return pos >= compress_ratio ? pos - compress_ratio : 0; - }(); - if ((position + 1) % compress_ratio == 0) { - const auto write_pos = atomicAdd(&compress_counter, 1); - compress_plan[write_pos] = plan; - } - if (position >= start_write_pos) { - const auto write_pos = atomicAdd(&write_counter, 1); - write_plan[write_pos] = plan; - } - } - __syncthreads(); - constexpr auto kInvalid = static_cast(-1); - const auto kInvalidPlan = PrefillPlan{kInvalid, kInvalid, kInvalid, kInvalid}; - const auto compress_count = compress_counter; - const auto write_count = write_counter; - for (uint32_t i = compress_count + tid; i < num_tokens; i += blockDim.x) { - compress_plan[i] = kInvalidPlan; - } - for (uint32_t i = write_count + tid; i < num_tokens; i += blockDim.x) { - write_plan[i] = kInvalidPlan; - } -} - -inline PlanResult plan_prefill_host(const CompressParams& params, const bool use_cuda_graph) { - const auto &[ - compress_ptr, write_ptr, seq_lens_ptr, extend_lens_ptr, // pointers - batch_size, num_tokens, compress_ratio, is_overlap // values - ] = params; - - uint32_t counter = 0; - uint32_t compress_counter = 0; - uint32_t write_counter = 0; - const auto ratio = compress_ratio * (1 + is_overlap); - for (const auto i : irange(batch_size)) { - const uint32_t seq_len = seq_lens_ptr[i]; - const uint32_t extend_len = extend_lens_ptr[i]; - const uint32_t prefix_len = seq_len - extend_len; - RuntimeCheck(0 < extend_len && extend_len <= seq_len); - /// NOTE: `start_write_pos` must be a multiple of `compress_ratio` - const uint32_t start_write_pos = [seq_len, compress_ratio, is_overlap] { - const uint32_t pos = seq_len / compress_ratio * compress_ratio; - if (!is_overlap) return pos; - /// NOTE: to avoid unsigned integer underflow, don't use `pos - compress_ratio` - return pos >= compress_ratio ? pos - compress_ratio : 0; - }(); - /// NOTE: `position` is within [prefix_len, seq_len) - for (const auto j : irange(extend_len)) { - const uint32_t position = prefix_len + j; - const auto plan = PrefillPlan{ - .ragged_id = counter + j, - .batch_id = i, - .position = position, - .window_len = ratio - std::min(j + 1, ratio), - }; - RuntimeCheck(plan.is_valid(compress_ratio, is_overlap), "Internal error!"); - if ((position + 1) % compress_ratio == 0) { - compress_ptr[compress_counter++] = plan; - } - if (position >= start_write_pos) { - write_ptr[write_counter++] = plan; - } - } - counter += extend_len; - } - RuntimeCheck(counter == num_tokens, "input size ", counter, " != num_q_tokens ", num_tokens); - if (!use_cuda_graph) return PlanResult{compress_counter, write_counter}; - constexpr auto kInvalid = static_cast(-1); - constexpr auto kInvalidPlan = PrefillPlan{kInvalid, kInvalid, kInvalid, kInvalid}; - for (const auto i : irange(compress_counter, num_tokens)) { - compress_ptr[i] = kInvalidPlan; - } - for (const auto i : irange(write_counter, num_tokens)) { - write_ptr[i] = kInvalidPlan; - } - return PlanResult{num_tokens, num_tokens}; -} - -inline PlanResult plan_prefill( - const tvm::ffi::TensorView extend_lens, - const tvm::ffi::TensorView seq_lens, - const tvm::ffi::TensorView compress_plan, - const tvm::ffi::TensorView write_plan, - const uint32_t compress_ratio, - const bool is_overlap, // for overlap transform, we have to keep 1 more extra window - const bool use_cuda_graph) { - auto N = SymbolicSize{"batch_size"}; - auto M = SymbolicSize{"num_tokens"}; - auto device = SymbolicDevice{}; - const bool is_cuda = [&] { - if (extend_lens.device().device_type == kDLCUDA) { - device.set_options(); - return true; - } else { - device.set_options(); - return false; - } - }(); - TensorMatcher({N}) // extend_lens and seq_lens - .with_dtype() - .with_device(device) - .verify(extend_lens) - .verify(seq_lens); - TensorMatcher({M, kPrefillPlanDim}) // compress_plan and write_plan - .with_dtype() - .with_device(device) - .verify(compress_plan) - .verify(write_plan); - - const auto params = CompressParams{ - .compress_plan = static_cast(compress_plan.data_ptr()), - .write_plan = static_cast(write_plan.data_ptr()), - .seq_lens = static_cast(seq_lens.data_ptr()), - .extend_lens = static_cast(extend_lens.data_ptr()), - .batch_size = static_cast(N.unwrap()), - .num_tokens = static_cast(M.unwrap()), - .compress_ratio = compress_ratio, - .is_overlap = is_overlap, - }; - - if (!is_cuda) return plan_prefill_host(params, use_cuda_graph); - /// NOTE: cuda kernel plan is naturally compatible with cuda graph - LaunchKernel(1, kBlockSize, device.unwrap())(plan_prefill_cuda, params); - return PlanResult{params.num_tokens, params.num_tokens}; -} - -} // namespace host::compress - -[[maybe_unused]] -constexpr auto& plan_compress_prefill = host::compress::plan_prefill; - -} // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/deepseek_v4/mega_moe_pre_dispatch.cuh b/python/sglang/kernels/jit/csrc/deepseek_v4/mega_moe_pre_dispatch.cuh index 9bd130ee47..203a96c5cf 100644 --- a/python/sglang/kernels/jit/csrc/deepseek_v4/mega_moe_pre_dispatch.cuh +++ b/python/sglang/kernels/jit/csrc/deepseek_v4/mega_moe_pre_dispatch.cuh @@ -43,7 +43,7 @@ struct MegaMoEPreDispatchParams { uint32_t top_k; }; -// kGroupSize must match sglang_per_token_group_quant_fp8_ue8m0(group_size=). +// kGroupSize must match the consuming MoE's activation scale group (quant_group_size). // kMultiChunk covers rows too wide for one chunk per thread. It is a template // parameter rather than a runtime check because this kernel is issue-bound: the // loop bookkeeping alone costs ~10% on the rows that fit a single chunk. diff --git a/python/sglang/kernels/jit/csrc/distributed/nvlink_comm.cuh b/python/sglang/kernels/jit/csrc/distributed/nvlink_comm.cuh index c846ecbedc..69f6af26cc 100644 --- a/python/sglang/kernels/jit/csrc/distributed/nvlink_comm.cuh +++ b/python/sglang/kernels/jit/csrc/distributed/nvlink_comm.cuh @@ -583,34 +583,12 @@ struct NVLinkComm { // specialized for each world size template static void - all_reduce_push(CommunicatorRef comm, TensorView in, TensorView out, tvm::ffi::Optional residual) { - return run_push(comm->get_push_obj(), in, out, residual); - } - template - static void all_gather_push(CommunicatorRef comm, TensorView in, TensorView out, tvm::ffi::Optional residual) { return run_push(comm->get_push_obj(), in, out, residual); } - template - static void - reduce_scatter_push(CommunicatorRef comm, TensorView in, TensorView out, tvm::ffi::Optional residual) { - return run_push(comm->get_push_obj(), in, out, residual); - } // only compile once for each world size template - static void all_reduce_pull( - CommunicatorRef comm, // only pull is needed - TensorView in, - TensorView out, - tvm::ffi::Optional residual, - int64_t in_mc_ptr, - int64_t out_mc_ptr, - uint32_t num_blocks_hint) { - return run_pull( - comm->get_pull_obj(), in, out, residual, in_mc_ptr, out_mc_ptr, num_blocks_hint); - } - template static void all_gather_pull( CommunicatorRef comm, // only pull is needed TensorView in, @@ -621,16 +599,6 @@ struct NVLinkComm { return run_pull( comm->get_pull_obj(), in, out, residual, 0, out_mc_ptr, num_blocks_hint); } - template - static void reduce_scatter_pull( - CommunicatorRef comm, // only pull is needed - TensorView in, - TensorView out, - tvm::ffi::Optional residual, - int64_t in_mc_ptr, - uint32_t num_blocks_hint) { - return run_pull(comm->get_pull_obj(), in, out, residual, in_mc_ptr, 0, num_blocks_hint); - } }; } // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/elementwise/adler32_checksum.cuh b/python/sglang/kernels/jit/csrc/elementwise/adler32_checksum.cuh index aa6cb8c859..677eeb8a93 100644 --- a/python/sglang/kernels/jit/csrc/elementwise/adler32_checksum.cuh +++ b/python/sglang/kernels/jit/csrc/elementwise/adler32_checksum.cuh @@ -1,9 +1,7 @@ // Adler-32 checksum CUDA kernels for tensor verification. // -// Two modes: -// 1. Whole-tensor: checksum all bytes of a contiguous tensor -// 2. Strided: checksum selected items across multiple tensors, -// given precomputed byte pointers and per-item lengths +// Checksums selected items across multiple tensors, given precomputed byte +// pointers and per-item lengths. // // Adler-32: // A = 1 + sum(bytes) (mod 65521) @@ -164,75 +162,7 @@ __device__ Adler32State adler32_block_reduce(Adler32State state, Adler32State* s return state; } -// --- Kernel 1: Whole tensor --- - -__global__ void adler32_whole_kernel( - const uint8_t* __restrict__ data, - uint32_t* __restrict__ out_a, - uint32_t* __restrict__ out_b, - uint64_t* __restrict__ out_len, - uint64_t num_bytes, - uint64_t chunk_size) { - extern __shared__ Adler32State smem[]; - - uint64_t global_tid = (uint64_t)blockIdx.x * blockDim.x + threadIdx.x; - uint64_t start = global_tid * chunk_size; - - Adler32State state; - if (start < num_bytes) { - uint64_t end = start + chunk_size; - if (end > num_bytes) end = num_bytes; - state = adler32_compute(data + start, end - start); - } else { - state = adler32_identity(); - } - - uint32_t threads_in_block = blockDim.x; - uint64_t block_start = (uint64_t)blockIdx.x * blockDim.x; - uint64_t total_threads = (num_bytes + chunk_size - 1) / chunk_size; - if (block_start + threads_in_block > total_threads) { - threads_in_block = (uint32_t)(total_threads - block_start); - } - - Adler32State result = adler32_block_reduce(state, smem, threads_in_block); - - if (threadIdx.x == 0) { - out_a[blockIdx.x] = result.a; - out_b[blockIdx.x] = result.b; - out_len[blockIdx.x] = result.len; - } -} - -struct Adler32WholeKernel { - static void run(tvm::ffi::TensorView data, tvm::ffi::TensorView states, int64_t num_bytes, int64_t chunk_size) { - using namespace host; - - const uint8_t* data_ptr = static_cast(data.data_ptr()); - - int64_t total_threads = (num_bytes + chunk_size - 1) / chunk_size; - int64_t num_blocks = (total_threads + kBlockSize - 1) / kBlockSize; - - uint32_t* out_a = static_cast(states.data_ptr()); - uint32_t* out_b = out_a + num_blocks; - uint64_t* out_len = reinterpret_cast(out_b + num_blocks); - - DLDevice device; - device.device_type = kDLCUDA; - device.device_id = data.device().device_id; - - size_t smem_bytes = (kBlockSize / 32) * sizeof(Adler32State); - LaunchKernel(num_blocks, kBlockSize, device, smem_bytes)( - adler32_whole_kernel, - data_ptr, - out_a, - out_b, - out_len, - static_cast(num_bytes), - static_cast(chunk_size)); - } -}; - -// --- Kernel 2: Strided items --- +// --- Kernel 1: Strided items --- __global__ void adler32_strided_kernel( const int64_t* __restrict__ ptrs, @@ -293,7 +223,7 @@ struct Adler32StridedKernel { } }; -// --- Kernel 3: Final reduction --- +// --- Kernel 2: Final reduction --- __global__ void adler32_reduce_kernel( const uint32_t* __restrict__ in_a, diff --git a/python/sglang/kernels/jit/csrc/elementwise/qknorm_across_heads.cuh b/python/sglang/kernels/jit/csrc/elementwise/qknorm_across_heads.cuh deleted file mode 100644 index 2e446cd7e8..0000000000 --- a/python/sglang/kernels/jit/csrc/elementwise/qknorm_across_heads.cuh +++ /dev/null @@ -1,179 +0,0 @@ -#include -#include - -#include -#include -#include -#include - -#include -#include - -#include -#include - -namespace sglang { - -template -struct VecTypeTrait; - -template <> -struct VecTypeTrait { - using packed_t = packed_t; - using vec_t = device::AlignedVector; -}; - -template <> -struct VecTypeTrait { - using packed_t = packed_t; - using vec_t = device::AlignedVector; -}; - -template <> -struct VecTypeTrait { - using packed_t = packed_t; - using vec_t = device::AlignedVector; -}; - -template <> -struct VecTypeTrait { - using packed_t = packed_t; - using vec_t = device::AlignedVector; -}; - -template -SGL_DEVICE packed_t rms(const packed_t& val, const packed_t& weight, float rsqrt_square_sum) { - float2 valf = device::cast(val); - float2 weightf = device::cast(weight); - return device::cast( - make_float2(valf.x * weightf.x * rsqrt_square_sum, valf.y * weightf.y * rsqrt_square_sum)); -} - -template -__global__ void qknorm_across_heads_reg_kernel( - T* __restrict__ q, - T* __restrict__ k, - const T* __restrict__ q_weight, - const T* __restrict__ k_weight, - int vec_hidden_size, - float eps) { - constexpr int inner_loop = VEC_SIZE_IN_BYTE == 16 ? 4 : 8; - - __shared__ float shared_memory[32]; - - using vec_t = typename VecTypeTrait::vec_t; - using packed_t = typename VecTypeTrait::packed_t; - vec_t v_data; - vec_t v_weight; - const int warp_id = threadIdx.x >> 5; - const int lane_id = threadIdx.x & 31; - const int warp_count = (blockDim.x + 31) >> 5; - const float inv_hidden_size = 1.0f / static_cast(vec_hidden_size * (VEC_SIZE_IN_BYTE / sizeof(T))); - const bool is_q = blockIdx.y == 0; - - const auto token_id = blockIdx.x; - float2 acc_square = make_float2(0.0f, 0.0f); - vec_t* data = reinterpret_cast(is_q ? q : k) + token_id * vec_hidden_size; - const vec_t* weight = reinterpret_cast(is_q ? q_weight : k_weight); - - if (threadIdx.x < vec_hidden_size) { - v_data = data[threadIdx.x]; - v_weight = weight[threadIdx.x]; - for (int i = 0; i < inner_loop; i++) { - float2 val = device::cast(v_data[i]); - acc_square.x += val.x * val.x; - acc_square.y += val.y * val.y; - } - } - - auto cg_warp = cooperative_groups::tiled_partition<32>(cooperative_groups::this_thread_block()); - float* buffer = shared_memory; - float warp_sum = cooperative_groups::reduce(cg_warp, acc_square.x + acc_square.y, cooperative_groups::plus()); - if (lane_id == 0) { - buffer[warp_id] = warp_sum; - } - - __syncthreads(); - if (threadIdx.x < 32) { - float cta_sum = cooperative_groups::reduce( - cg_warp, (threadIdx.x < warp_count) ? buffer[threadIdx.x] : 0.0f, cooperative_groups::plus()); - if (threadIdx.x == 0) { - buffer[0] = rsqrtf(eps + cta_sum * inv_hidden_size); - } - } - __syncthreads(); - - if (threadIdx.x < vec_hidden_size) { - float rsqrt_val = buffer[0]; - for (int i = 0; i < inner_loop; i++) { - v_data[i] = rms(v_data[i], v_weight[i], rsqrt_val); - } - data[threadIdx.x] = v_data; - } -} - -template -struct QKNormAcrossHeadsKernel { - static void - run(const tvm::ffi::TensorView q, - const tvm::ffi::TensorView k, - const tvm::ffi::TensorView q_weight, - const tvm::ffi::TensorView k_weight, - float eps) { - using namespace host; - auto N = SymbolicSize{"num_tokens"}; - auto D = SymbolicSize{"hidden_size"}; - auto device = SymbolicDevice{}; - device.set_options(); - - TensorMatcher({N, D}) // q - .with_strides({D, 1}) - .with_dtype() - .with_device(device) - .verify(q); - TensorMatcher({N, D}) // k - .with_strides({D, 1}) - .with_dtype() - .with_device(device) - .verify(k); - TensorMatcher({D}) // q_weight - .with_dtype() - .with_device(device) - .verify(q_weight); - TensorMatcher({D}) // k_weight - .with_dtype() - .with_device(device) - .verify(k_weight); - - int hidden_size = static_cast(D.unwrap()); - if (hidden_size <= (device::kMaxVecBytes == 32 ? 12288 : 8192)) { - int elements_in_vec = device::kMaxVecBytes / sizeof(DType); - int vec_hidden_size = hidden_size / elements_in_vec; - uint threads = (vec_hidden_size + 31) / 32 * 32; - - // Runtime check - host::RuntimeCheck( - hidden_size % elements_in_vec == 0, - "hidden_size", - hidden_size, - " can not align to elements_in_vec ", - elements_in_vec); - - auto kernel = qknorm_across_heads_reg_kernel; - - LaunchKernel(dim3(static_cast(N.unwrap()), 2), threads, device.unwrap()) - .enable_pdl(false)( - kernel, - reinterpret_cast(q.data_ptr()), - reinterpret_cast(k.data_ptr()), - reinterpret_cast(q_weight.data_ptr()), - reinterpret_cast(k_weight.data_ptr()), - vec_hidden_size, - eps); - } else { - host::RuntimeCheck(false, "Large hidden_sizes are not supported for now."); - } - } -}; - -} // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/inkling/inkling_all_reduce.cuh b/python/sglang/kernels/jit/csrc/inkling/inkling_all_reduce.cuh index f3749983b6..7653425c93 100644 --- a/python/sglang/kernels/jit/csrc/inkling/inkling_all_reduce.cuh +++ b/python/sglang/kernels/jit/csrc/inkling/inkling_all_reduce.cuh @@ -12,16 +12,13 @@ // slice S_r (in any buffer), so there is no write-write conflict, and each // per-element load completes before its store (data dependency). // -// Two variants: -// * ..._kernel (v1): no in-kernel sync; the caller fences with the symm-mem -// handle's barrier() on each side (3 launches total). -// * ..._fused_kernel (v2): an in-kernel per-block system barrier (entry: -// producers done + visible; exit: broadcasts done + visible), so the whole -// all-reduce is a single launch. The barrier uses a DEDICATED symmetric -// flags buffer (independent of torch's signal pad, so no interference with -// multimem) and a device-resident monotonic epoch counter per block, which -// keeps advancing across launches -- including CUDA-graph replays -- so -// flags never go stale (spin is `flag < epoch`, epoch strictly increasing). +// ..._fused_kernel (v2) uses an in-kernel per-block system barrier (entry: +// producers done + visible; exit: broadcasts done + visible), so the whole +// all-reduce is a single launch. The barrier uses a DEDICATED symmetric +// flags buffer (independent of torch's signal pad, so no interference with +// multimem) and a device-resident monotonic epoch counter per block, which +// keeps advancing across launches -- including CUDA-graph replays -- so +// flags never go stale (spin is `flag < epoch`, epoch strictly increasing). // // Fusion seam: the reduced `result` Storage below is where an epilogue (RMSNorm // / short-conv / bias) plugs in -- applied in registers before the broadcast @@ -162,15 +159,6 @@ __device__ __forceinline__ void two_shot_reduce_local(DType* (&input)[kNumGPU], } } -// v1: no in-kernel barrier (caller fences via hdl.barrier()). -template -__global__ __launch_bounds__(1024, 1) void inkling_two_shot_all_reduce_kernel( - void* const* __restrict__ peer_ptrs, const uint32_t rank, const uint32_t num_items) { - DType* input[kNumGPU]; - const uint32_t local_vecs = slice_setup(input, peer_ptrs, rank, num_items); - two_shot_reduce_local(input, local_vecs); -} - // v2: single-launch, fused entry + exit system barrier. `shared` (optional): // this rank's LOCAL shared-expert partials, folded into its own buffer before // the entry barrier (which then must publish -- see fold_shared_local). @@ -408,7 +396,7 @@ __global__ __launch_bounds__(1024, 1) void inkling_multimem_full_oneshot_kernel( // NO exit barrier: result is local & complete; input reuse is caller-managed. } -// Blocks needed to cover this rank's two-shot slice (v1/v2/v3 partition). +// Blocks needed to cover this rank's two-shot slice (v2/v3 partition). template uint32_t work_num_blocks(uint32_t n, uint32_t block_size) { using Trait = InklingAllReduceTrait; @@ -483,22 +471,6 @@ void validate(tvm::ffi::TensorView buf, int64_t peer_ptrs_dev, int64_t rank, int RuntimeCheck(rank >= 0 && rank < kNumGPU, "rank out of range"); } -template -void inkling_two_shot_all_reduce( - tvm::ffi::TensorView local_buffer, int64_t peer_ptrs_dev, int64_t rank, int64_t num_items) { - using namespace host; - uint32_t n; - validate(local_buffer, peer_ptrs_dev, rank, num_items, n); - const auto device = local_buffer.device(); - const uint32_t num_blocks = work_num_blocks(n, 1024u); // no in-kernel barrier -> uncapped - const auto stream = LaunchKernel::resolve_device(device); - LaunchKernel(num_blocks, 1024u, stream)( - inkling_two_shot_all_reduce_kernel, - reinterpret_cast(peer_ptrs_dev), - static_cast(rank), - n); -} - template void inkling_two_shot_all_reduce_fused( tvm::ffi::TensorView local_buffer, diff --git a/python/sglang/kernels/jit/csrc/minimax/per_token_quant_ue8m0.cuh b/python/sglang/kernels/jit/csrc/minimax/per_token_quant_ue8m0.cuh index 250688ee79..b648bc9332 100644 --- a/python/sglang/kernels/jit/csrc/minimax/per_token_quant_ue8m0.cuh +++ b/python/sglang/kernels/jit/csrc/minimax/per_token_quant_ue8m0.cuh @@ -19,81 +19,12 @@ namespace sglang { using deepseek_v4::fp8::cast_to_ue8m0; using deepseek_v4::fp8::pack_fp8; -// Per-token group quant to FP8-e4m3 with a fused UE8M0 scale. Each group of -// kGroupSize columns gets one UE8M0 exponent byte written contiguously in -// row-major order into ``x_sf`` (int32 [num_tokens, num_groups/4], 4 group -// bytes per int32). This is the deep_gemm "transform_sf" pack done inline in -// the quant, reusing the dsv4 ``cast_to_ue8m0``/``pack_fp8`` primitives -- it -// is byte-identical to ``per_token_group_quant_fp8(scale_ue8m0=True)`` followed -// by ``transform_sf_into_required_layout`` (both round via ceil(log2(absmax/ -// FP8_MAX))), but emits no separate transpose/pack kernel. -struct PerTokenQuantUe8m0Params { - const bf16_t* __restrict__ x; // [num_tokens, hidden] - fp8_e4m3_t* __restrict__ x_q; // [num_tokens, hidden] - int32_t* __restrict__ x_sf; // [num_tokens, num_groups/4]; written as bytes - uint32_t num_tokens; - uint32_t hidden; - uint32_t num_groups; // hidden / kGroupSize -}; - -template -__global__ __launch_bounds__(1024, 2) void // - per_token_quant_ue8m0_kernel(const PerTokenQuantUe8m0Params __grid_constant__ params) { - using namespace device; - constexpr uint32_t kVecElems = 8; // 8 bf16 = 16B load per thread - static_assert(kGroupSize % kVecElems == 0, "group_size must be a multiple of 8"); - constexpr uint32_t kThreadsPerGroup = kGroupSize / kVecElems; - using InputVec = AlignedVector; - using OutputVec = AlignedVector; - - const uint32_t token_id = blockIdx.x; - const uint32_t tid = threadIdx.x; - PDLWaitPrimary(); - - const auto token_in = params.x + static_cast(token_id) * params.hidden; - const auto token_out = params.x_q + static_cast(token_id) * params.hidden; - - InputVec in_vec; - in_vec.load(token_in, tid); - float local_max = 0.0f; - float vals[kVecElems]; -#pragma unroll - for (uint32_t i = 0; i < kVecElems / 2; ++i) { - const auto [v0, v1] = cast(in_vec[i]); - vals[2 * i + 0] = v0; - vals[2 * i + 1] = v1; - local_max = fmaxf(local_max, fmaxf(fabsf(v0), fabsf(v1))); - } - // Absmax across the kThreadsPerGroup threads that cover one group. - local_max = warp::reduce_max(local_max); - const float absmax = fmaxf(local_max, 1e-10f); - const float raw_scale = absmax / math::FP8_E4M3_MAX; - const uint32_t ue8m0_exp = cast_to_ue8m0(raw_scale); - const float inv_scale = __uint_as_float((127u + 127u - ue8m0_exp) << 23); - - OutputVec out_vec; -#pragma unroll - for (uint32_t i = 0; i < kVecElems / 2; ++i) { - out_vec[i] = pack_fp8(vals[2 * i + 0] * inv_scale, vals[2 * i + 1] * inv_scale); - } - out_vec.store(token_out, tid); - - const uint32_t group_id = tid / kThreadsPerGroup; - const uint32_t within_group_id = tid % kThreadsPerGroup; - if (within_group_id == 0 && group_id < params.num_groups) { - const uint32_t byte_off = token_id * params.num_groups + group_id; - reinterpret_cast(params.x_sf)[byte_off] = static_cast(ue8m0_exp); - } - PDLTriggerSecondary(); -} - -// Fused quant + scatter: like per_token_quant_ue8m0_kernel, but instead of -// writing the fp8/scale for the single source token, it scatters them straight -// into the permuted grouped-GEMM input -- replicating each token to its ``topk`` -// destination rows -- so the separate fill_gateup_input_triton_kernel launch (and -// the intermediate x_q/x_sf buffers) are eliminated. The fp8 value + UE8M0 scale -// are computed exactly once per token (identical to the non-fused kernel); only -// the stores differ. +// Fused per-token group quant to FP8-e4m3 with a UE8M0 scale + scatter: each +// group of kGroupSize columns gets one UE8M0 exponent (ceil(log2(absmax / +// FP8_MAX)), via the dsv4 ``cast_to_ue8m0``/``pack_fp8`` primitives), and the +// fp8 row + scale bytes are written straight into the permuted grouped-GEMM +// input -- replicating each token to its ``topk`` destination rows -- so no +// separate fill_gateup_input_triton_kernel launch or x_q/x_sf buffers are needed. struct PerTokenQuantUe8m0ScatterParams { const bf16_t* __restrict__ x; // [num_tokens, hidden] fp8_e4m3_t* __restrict__ gateup_input; // [E, m_max, hidden] @@ -242,38 +173,4 @@ void per_token_quant_ue8m0_scatter( .enable_pdl(kUsePDL)(kernel, params); } -template -void per_token_quant_ue8m0(tvm::ffi::TensorView x, tvm::ffi::TensorView x_q, tvm::ffi::TensorView x_sf) { - using namespace host; - auto device = SymbolicDevice{}; - auto M = SymbolicSize{"num_tokens"}; - auto H = SymbolicSize{"hidden"}; - auto G4 = SymbolicSize{"num_groups_div_4"}; - device.set_options(); - TensorMatcher({M, H}).with_dtype().with_device(device).verify(x); - TensorMatcher({M, H}).with_dtype().with_device(device).verify(x_q); - TensorMatcher({M, G4}).with_dtype().with_device(device).verify(x_sf); - - const uint32_t num_tokens = static_cast(M.unwrap()); - const uint32_t hidden = static_cast(H.unwrap()); - RuntimeCheck(hidden % kGroupSize == 0, "hidden ", hidden, " not divisible by group_size ", kGroupSize); - const uint32_t num_groups = hidden / static_cast(kGroupSize); - RuntimeCheck(static_cast(G4.unwrap()) * 4 == num_groups); - const uint32_t threads = hidden / 8; // kVecElems - RuntimeCheck(threads <= 1024, "hidden/8 must be <= 1024, got ", threads); - - const auto params = PerTokenQuantUe8m0Params{ - .x = static_cast(x.data_ptr()), - .x_q = static_cast(x_q.data_ptr()), - .x_sf = static_cast(x_sf.data_ptr()), - .num_tokens = num_tokens, - .hidden = hidden, - .num_groups = num_groups, - }; - if (num_tokens == 0) return; - constexpr auto kernel = per_token_quant_ue8m0_kernel; - LaunchKernel(num_tokens, threads, device.unwrap()) // - .enable_pdl(kUsePDL)(kernel, params); -} - } // namespace sglang diff --git a/python/sglang/kernels/jit/csrc/moe/inkling_gate_topk_renorm.cuh b/python/sglang/kernels/jit/csrc/moe/inkling_gate_topk_renorm.cuh index 7de7a5ed75..c7de0b8c17 100644 --- a/python/sglang/kernels/jit/csrc/moe/inkling_gate_topk_renorm.cuh +++ b/python/sglang/kernels/jit/csrc/moe/inkling_gate_topk_renorm.cuh @@ -18,7 +18,6 @@ static constexpr int kInklingRoutedExperts = 256; static constexpr int kInklingSharedExperts = 2; static constexpr int kInklingTotalExperts = kInklingRoutedExperts + kInklingSharedExperts; static constexpr int kInklingTopK = 6; -static constexpr int kInklingTopPow2 = 8; static constexpr int kInklingWarpSize = 32; static constexpr int kInklingValuesPerLane = kInklingRoutedExperts / kInklingWarpSize; @@ -26,10 +25,6 @@ __device__ __forceinline__ float inkling_sigmoid(float x) { return 1.0f / (1.0f + __expf(-x)); } -__device__ __forceinline__ bool inkling_score_better(float score, int idx, float best_score, int best_idx) { - return score > best_score || (score == best_score && idx < best_idx); -} - // Order-preserving uint32 key of an fp32 (same total order as the triton // kernel's fpval_to_key): flip sign bit for positives, all bits for negatives. __device__ __forceinline__ uint32_t inkling_fp_key(float f) { @@ -44,158 +39,6 @@ __device__ __forceinline__ int32_t inkling_pack_routed(int32_t id, float w) { return static_cast((static_cast(id) << 16) | wbits); } -template -__launch_bounds__(kInklingWarpSize* WarpsPerBlock) __global__ void inkling_gate_topk_renorm_kernel( - const float* __restrict__ logits, - const float* __restrict__ bias, - const float* __restrict__ global_scale, - float* __restrict__ routed_w, - float* __restrict__ shared_w, - int64_t* __restrict__ indices, - int32_t* __restrict__ packed, - int64_t M, - int64_t logits_stride_m, - float route_scale) { - const int lane = threadIdx.x; - const int warp_in_block = threadIdx.y; - const int64_t row = static_cast(blockIdx.x) * WarpsPerBlock + warp_in_block; - if (row >= M) { - return; - } - - const int64_t row_base = row * logits_stride_m; - float local_scores[kInklingValuesPerLane]; -#pragma unroll - for (int i = 0; i < kInklingValuesPerLane; ++i) { - const int expert = lane + i * kInklingWarpSize; - const float raw = logits[row_base + expert]; - local_scores[i] = inkling_sigmoid(raw) + bias[expert]; - } - - int selected_idx[kInklingTopK]; -#pragma unroll - for (int k = 0; k < kInklingTopK; ++k) { - float best_score = -FLT_MAX; - int best_idx = INT_MAX; -#pragma unroll - for (int i = 0; i < kInklingValuesPerLane; ++i) { - const int expert = lane + i * kInklingWarpSize; - const float score = local_scores[i]; - if (inkling_score_better(score, expert, best_score, best_idx)) { - best_score = score; - best_idx = expert; - } - } - -#pragma unroll - for (int offset = kInklingWarpSize / 2; offset > 0; offset >>= 1) { - const float other_score = __shfl_xor_sync(0xffffffff, best_score, offset); - const int other_idx = __shfl_xor_sync(0xffffffff, best_idx, offset); - if (inkling_score_better(other_score, other_idx, best_score, best_idx)) { - best_score = other_score; - best_idx = other_idx; - } - } - - selected_idx[k] = best_idx; - if (best_idx % kInklingWarpSize == lane) { - local_scores[best_idx / kInklingWarpSize] = -FLT_MAX; - } - __syncwarp(); - } - - if (lane != 0) { - return; - } - - float active[kInklingTopPow2]; -#pragma unroll - for (int i = 0; i < kInklingTopK; ++i) { - active[i] = inkling_sigmoid(logits[row_base + selected_idx[i]]); - } -#pragma unroll - for (int i = 0; i < kInklingSharedExperts; ++i) { - active[kInklingTopK + i] = inkling_sigmoid(logits[row_base + kInklingRoutedExperts + i]); - } - - float sum = 0.0f; -#pragma unroll - for (int i = 0; i < kInklingTopPow2; ++i) { - sum += active[i]; - } - const float scale = route_scale * global_scale[0] / sum; - -#pragma unroll - for (int i = 0; i < kInklingTopK; ++i) { - const float w = active[i] * scale; - if constexpr (ReturnPacked) { - packed[row * kInklingTopK + i] = inkling_pack_routed(selected_idx[i], w); - } else { - routed_w[row * kInklingTopK + i] = w; - indices[row * kInklingTopK + i] = static_cast(selected_idx[i]); - } - } -#pragma unroll - for (int i = 0; i < kInklingSharedExperts; ++i) { - shared_w[row * kInklingSharedExperts + i] = active[kInklingTopK + i] * scale; - } -} - -template -void launch_inkling_gate_topk_renorm( - const float* logits, - const float* bias, - const float* global_scale, - float* routed_w, - float* shared_w, - int64_t* indices, - int32_t* packed, - int64_t tokens, - int64_t logits_stride_m, - float route_scale, - DLDevice device) { - using namespace host; - const dim3 block(kInklingWarpSize, WarpsPerBlock); - const dim3 grid(static_cast(div_ceil(tokens, static_cast(WarpsPerBlock)))); - LaunchKernel(grid, block, device)( - inkling_gate_topk_renorm_kernel, - logits, - bias, - global_scale, - routed_w, - shared_w, - indices, - packed, - tokens, - logits_stride_m, - route_scale); -} - -template -void dispatch_inkling_gate_topk_renorm( - const float* logits, - const float* bias, - const float* global_scale, - float* routed_w, - float* shared_w, - int64_t* indices, - int32_t* packed, - int64_t tokens, - int64_t logits_stride_m, - float route_scale, - DLDevice device) { - if (tokens <= 64) { - launch_inkling_gate_topk_renorm<1, ReturnPacked>( - logits, bias, global_scale, routed_w, shared_w, indices, packed, tokens, logits_stride_m, route_scale, device); - } else if (tokens <= 1024) { - launch_inkling_gate_topk_renorm<4, ReturnPacked>( - logits, bias, global_scale, routed_w, shared_w, indices, packed, tokens, logits_stride_m, route_scale, device); - } else { - launch_inkling_gate_topk_renorm<8, ReturnPacked>( - logits, bias, global_scale, routed_w, shared_w, indices, packed, tokens, logits_stride_m, route_scale, device); - } -} - // Warp-per-row gate kernel with optional expert-per-block GEMV fusion. // Inkling gate GEMM shape: x [M, 6144] bf16 x W [264 (row-padded), 6144] bf16. @@ -736,122 +579,6 @@ void check_gate_row_alignment(const void* ptr, int64_t stride_m) { "logits rows must be device::kMaxVecBytes-aligned (production pitch is 264 floats)"); } -void inkling_gate_topk_renorm( - tvm::ffi::TensorView logits, - tvm::ffi::TensorView bias, - tvm::ffi::TensorView global_scale, - tvm::ffi::TensorView routed_w, - tvm::ffi::TensorView shared_w, - tvm::ffi::TensorView indices, - double route_scale) { - using namespace host; - - SymbolicSize M{"tokens"}; - SymbolicDevice device_; - device_.set_options(); - - TensorMatcher({M, kInklingTotalExperts}) - .with_strides({-1, 1}) - .with_dtype() - .with_device(device_) - .verify(logits); - TensorMatcher({kInklingRoutedExperts}).with_dtype().with_device(device_).verify(bias); - TensorMatcher({1}).with_dtype().with_device(device_).verify(global_scale); - TensorMatcher({M, kInklingTopK}).with_dtype().with_device(device_).verify(routed_w); - TensorMatcher({M, kInklingSharedExperts}).with_dtype().with_device(device_).verify(shared_w); - TensorMatcher({M, kInklingTopK}).with_dtype().with_device(device_).verify(indices); - - RuntimeCheck(logits.stride(1) == 1, "logits must be contiguous along the expert dimension"); - RuntimeCheck(bias.stride(0) == 1, "bias must be contiguous"); - RuntimeCheck(routed_w.stride(1) == 1, "routed_w must be contiguous along top-k dimension"); - RuntimeCheck(shared_w.stride(1) == 1, "shared_w must be contiguous along shared dimension"); - RuntimeCheck(indices.stride(1) == 1, "indices must be contiguous along top-k dimension"); - - const int64_t tokens = M.unwrap(); - if (tokens == 0) { - return; - } - - const auto* logits_ptr = static_cast(logits.data_ptr()); - const auto* bias_ptr = static_cast(bias.data_ptr()); - const auto* global_scale_ptr = static_cast(global_scale.data_ptr()); - auto* routed_w_ptr = static_cast(routed_w.data_ptr()); - auto* shared_w_ptr = static_cast(shared_w.data_ptr()); - auto* indices_ptr = static_cast(indices.data_ptr()); - const float route_scale_f = static_cast(route_scale); - const DLDevice device = device_.unwrap(); - - dispatch_inkling_gate_topk_renorm( - logits_ptr, - bias_ptr, - global_scale_ptr, - routed_w_ptr, - shared_w_ptr, - indices_ptr, - nullptr, - tokens, - logits.stride(0), - route_scale_f, - device); -} - -// Packed variant: emits packed[M, kInklingTopK] int32 ((id<<16)|bf16 weight) instead of -// the routed_w + indices pair; shared_w still written. -void inkling_gate_topk_renorm_packed( - tvm::ffi::TensorView logits, - tvm::ffi::TensorView bias, - tvm::ffi::TensorView global_scale, - tvm::ffi::TensorView packed, - tvm::ffi::TensorView shared_w, - double route_scale) { - using namespace host; - - SymbolicSize M{"tokens"}; - SymbolicDevice device_; - device_.set_options(); - - TensorMatcher({M, kInklingTotalExperts}) - .with_strides({-1, 1}) - .with_dtype() - .with_device(device_) - .verify(logits); - TensorMatcher({kInklingRoutedExperts}).with_dtype().with_device(device_).verify(bias); - TensorMatcher({1}).with_dtype().with_device(device_).verify(global_scale); - TensorMatcher({M, kInklingTopK}).with_dtype().with_device(device_).verify(packed); - TensorMatcher({M, kInklingSharedExperts}).with_dtype().with_device(device_).verify(shared_w); - - RuntimeCheck(logits.stride(1) == 1, "logits must be contiguous along the expert dimension"); - RuntimeCheck(bias.stride(0) == 1, "bias must be contiguous"); - RuntimeCheck(packed.stride(1) == 1, "packed must be contiguous along top-k dimension"); - RuntimeCheck(shared_w.stride(1) == 1, "shared_w must be contiguous along shared dimension"); - - const int64_t tokens = M.unwrap(); - if (tokens == 0) { - return; - } - - const auto* logits_ptr = static_cast(logits.data_ptr()); - const auto* bias_ptr = static_cast(bias.data_ptr()); - const auto* global_scale_ptr = static_cast(global_scale.data_ptr()); - auto* packed_ptr = static_cast(packed.data_ptr()); - auto* shared_w_ptr = static_cast(shared_w.data_ptr()); - const float route_scale_f = static_cast(route_scale); - const DLDevice device = device_.unwrap(); - - dispatch_inkling_gate_topk_renorm( - logits_ptr, - bias_ptr, - global_scale_ptr, - nullptr, - shared_w_ptr, - nullptr, - packed_ptr, - tokens, - logits.stride(0), - route_scale_f, - device); -} - // Uses int32 indices for MoeRunner, optional PDL, and tunable warps (0 = auto). // Logit rows must be device::kMaxVecBytes-aligned (the standard pitch is 264). void inkling_gate_topk_renorm_v2( diff --git a/python/sglang/kernels/jit/csrc/moe/moe_fused_gate.cuh b/python/sglang/kernels/jit/csrc/moe/moe_fused_gate.cuh deleted file mode 100644 index 7ea1a83e9b..0000000000 --- a/python/sglang/kernels/jit/csrc/moe/moe_fused_gate.cuh +++ /dev/null @@ -1,376 +0,0 @@ -#include -#include - -#include -#include -#include - -#include - -#include -#include - -// gfx1250 needs a 64-bit __shfl_*_sync mask (static_assert sizeof == 8); it's -// masked to wave32 internally, so 64-bit is fine everywhere. __gfx1250__ is -// device-pass only, so widen in the host pass too or the launch stub won't build. -#if defined(__gfx1250__) || (defined(__HIP_PLATFORM_AMD__) && !defined(__HIP_DEVICE_COMPILE__)) -#define SGL_WARP_SYNC_MASK 0xFFFFFFFFFFFFFFFFULL -#else -#define SGL_WARP_SYNC_MASK 0xFFFFFFFF -#endif - -namespace sglang { - -constexpr uint32_t kWarpSize = 32; -constexpr uint32_t kWarpsPerCTA = 6; -constexpr uint32_t kSmallTokenThreshold = 512; -constexpr uint32_t kMaxExperts = 512; -constexpr uint32_t kMaxTopK = 16; - -enum class ScoringFunc : uint32_t { - kSigmoid = 0, - kSqrtSoftplus = 1, -}; - -struct MoEFusedGateParams { - const float* __restrict__ input; - const float* __restrict__ bias; - float* __restrict__ output; - int32_t* __restrict__ indices; - uint32_t num_rows; - uint32_t num_experts; - uint32_t topk; - uint32_t num_fused_shared_experts; - bool renormalize; - float routed_scaling_factor; - bool apply_routed_scaling_factor_on_output; -}; - -template -__device__ __forceinline__ float compute_score(float x) { - if constexpr (kScoringFunc == ScoringFunc::kSigmoid) { - // sigmoid(x) = 1 / (1 + exp(-x)) - return 1.0f / (1.0f + expf(-x)); - } else { - // sqrt(softplus(x)); sign folded out because expf overflows above 88.7. - const float softplus = fmaxf(x, 0.0f) + log1pf(expf(-fabsf(x))); - return sqrtf(softplus); - } -} - -template -__global__ void moe_fused_gate_kernel_small_token(const MoEFusedGateParams __grid_constant__ params) { - const auto& [input, bias, output, indices, num_rows, num_experts, topk, num_fused_shared_experts, renormalize, routed_scaling_factor, apply_routed_scaling_factor_on_output] = - params; - - uint32_t row_idx = blockIdx.x; - if (row_idx >= num_rows) return; - - // number of routed experts to select (excluding fused shared experts) - const uint32_t topk_routed = topk - num_fused_shared_experts; - - uint32_t tid = threadIdx.x; - uint32_t warp_id = tid / kWarpSize; - uint32_t lane_id = tid % kWarpSize; - // Actual warps launched (<= kWarpsPerToken). num_experts that need fewer than - // kWarpsPerToken warps leave the upper warp_maxs/warp_experts slots unwritten, - // so the cross-warp reduction below must only read the launched warps. - const uint32_t num_warps = blockDim.x / kWarpSize; - - extern __shared__ float shared_mem[]; - float* shared_scores = shared_mem; - float* shared_original_scores = shared_mem + num_experts; - - // For warp-level reduction - __shared__ float warp_maxs[kWarpsPerToken]; - __shared__ int warp_experts[kWarpsPerToken]; - __shared__ int selected_experts[kMaxTopK]; - - for (uint32_t e = tid; e < num_experts; e += blockDim.x) { - float input_val = input[row_idx * num_experts + e]; - float bias_val = bias[e]; - float score_val = compute_score(input_val); - float biased_val = score_val + bias_val; - shared_scores[e] = biased_val; - shared_original_scores[e] = score_val; - } - - __syncthreads(); - - // only select topk_routed experts (excluding shared experts) - for (uint32_t k = 0; k < topk_routed; k++) { - float my_val = -FLT_MAX; - int my_expert = -1; - for (uint32_t e = tid; e < num_experts; e += blockDim.x) { - if (shared_scores[e] > my_val) { - my_val = shared_scores[e]; - my_expert = e; - } - } - - float warp_max_val = my_val; - int warp_max_expert = my_expert; - -#pragma unroll - for (int offset = 16; offset > 0; offset /= 2) { - float other_val = __shfl_down_sync(SGL_WARP_SYNC_MASK, warp_max_val, offset); - int other_expert = __shfl_down_sync(SGL_WARP_SYNC_MASK, warp_max_expert, offset); - if (other_val > warp_max_val) { - warp_max_val = other_val; - warp_max_expert = other_expert; - } - } - - if (lane_id == 0 && warp_id < kWarpsPerToken) { - warp_maxs[warp_id] = warp_max_val; - warp_experts[warp_id] = warp_max_expert; - } - - __syncthreads(); - - if (warp_id == 0) { - float final_max = (lane_id < num_warps) ? warp_maxs[lane_id] : -FLT_MAX; - int final_expert = (lane_id < num_warps) ? warp_experts[lane_id] : -1; - -#pragma unroll - for (int offset = 16; offset > 0; offset /= 2) { - float other_val = __shfl_down_sync(SGL_WARP_SYNC_MASK, final_max, offset); - int other_expert = __shfl_down_sync(SGL_WARP_SYNC_MASK, final_expert, offset); - if (other_val > final_max) { - final_max = other_val; - final_expert = other_expert; - } - } - - if (lane_id == 0) { - selected_experts[k] = final_expert; - } - } - - __syncthreads(); - - int selected = selected_experts[k]; - if (selected >= 0 && tid == 0) { - shared_scores[selected] = -FLT_MAX; - } - - __syncthreads(); - } - - static_assert(kMaxTopK <= device::kWarpThreads); - if (tid >= device::kWarpThreads) return; - - // only use the first warp to perform write to global operation - float routed_weight = 0.0f; - int32_t selected_expert = 0; - if (tid < topk_routed) { - int expert_id = selected_experts[tid]; - float score = shared_original_scores[expert_id]; - if (expert_id >= 0 && expert_id < static_cast(num_experts)) { - routed_weight = score; - selected_expert = expert_id; - } - } - const auto routed_sum = device::warp::reduce_sum(routed_weight); - if (tid < topk) { - const bool is_shared = tid >= topk_routed; - const auto output_offset = row_idx * topk + tid; - const auto weight = is_shared ? (routed_sum / routed_scaling_factor) : routed_weight; - const auto expert_id = is_shared ? (num_experts + tid - topk_routed) : selected_expert; - const auto scale = apply_routed_scaling_factor_on_output ? routed_scaling_factor : 1.0f; - const auto norm = renormalize && routed_sum > 0.0f ? routed_sum : 1.0f; - output[output_offset] = weight / norm * scale; - indices[output_offset] = expert_id; - } -} - -template -__global__ void moe_fused_gate_kernel(const MoEFusedGateParams __grid_constant__ params) { - const auto& [input, bias, output, indices, num_rows, num_experts, topk, num_fused_shared_experts, renormalize, routed_scaling_factor, apply_routed_scaling_factor_on_output] = - params; - - uint32_t row_idx = blockIdx.x * kWarpsPerCTA + threadIdx.y; - if (row_idx >= num_rows) return; - - // number of routed experts to select (excluding fused shared experts) - const uint32_t topk_routed = topk - num_fused_shared_experts; - - uint32_t lane_id = threadIdx.x; - uint32_t warp_id = threadIdx.y; - - extern __shared__ float shared_mem[]; - float* shared_scores = shared_mem + warp_id * num_experts * 2; - float* shared_original_scores = shared_scores + num_experts; - __shared__ int selected_experts[kWarpsPerCTA][kMaxTopK]; - int* warp_selected_experts = selected_experts[warp_id]; - - for (uint32_t e = lane_id; e < num_experts; e += kWarpSize) { - float input_val = input[row_idx * num_experts + e]; - float bias_val = bias[e]; - float score_val = compute_score(input_val); - float biased_val = score_val + bias_val; - shared_scores[e] = biased_val; - shared_original_scores[e] = score_val; - } - - __syncwarp(); - - // only select topk_routed experts - for (uint32_t k = 0; k < topk_routed; k++) { - float max_val = -FLT_MAX; - int max_expert = -1; - - for (uint32_t expert = lane_id; expert < num_experts; expert += kWarpSize) { - if (shared_scores[expert] > max_val) { - max_val = shared_scores[expert]; - max_expert = expert; - } - } - - for (int offset = kWarpSize / 2; offset > 0; offset /= 2) { - float other_val = __shfl_down_sync(SGL_WARP_SYNC_MASK, max_val, offset); - int other_expert = __shfl_down_sync(SGL_WARP_SYNC_MASK, max_expert, offset); - - if (other_val > max_val || (other_val == max_val && other_expert < max_expert)) { - max_val = other_val; - max_expert = other_expert; - } - } - - if (lane_id == 0) { - warp_selected_experts[k] = max_expert; - if (max_expert != -1) { - shared_scores[max_expert] = -FLT_MAX; - } - } - - __syncwarp(); - } - - static_assert(kMaxTopK <= device::kWarpThreads); - - float routed_weight = 0.0f; - int32_t selected_expert = 0; - if (lane_id < topk_routed) { - int expert_id = warp_selected_experts[lane_id]; - if (expert_id >= 0 && expert_id < static_cast(num_experts)) { - routed_weight = shared_original_scores[expert_id]; - selected_expert = expert_id; - } - } - const auto routed_sum = device::warp::reduce_sum(routed_weight); - if (lane_id < topk) { - const bool is_shared = lane_id >= topk_routed; - const auto output_idx = row_idx * topk + lane_id; - const auto weight = is_shared ? (routed_sum / routed_scaling_factor) : routed_weight; - const auto expert_id = is_shared ? (num_experts + lane_id - topk_routed) : selected_expert; - const auto scale = apply_routed_scaling_factor_on_output ? routed_scaling_factor : 1.0f; - const auto norm = renormalize && routed_sum > 0.0f ? routed_sum : 1.0f; - output[output_idx] = weight / norm * scale; - indices[output_idx] = expert_id; - } -} - -template -void dispatch_small_token_kernel( - uint32_t num_rows, - uint32_t threads_per_block, - uint32_t warps_per_token, - DLDevice device, - size_t smem_per_row, - const MoEFusedGateParams& params) { - using namespace host; - if (warps_per_token <= 8) { - LaunchKernel(num_rows, threads_per_block, device, smem_per_row)( - moe_fused_gate_kernel_small_token<8, kScoringFunc>, params); - } else if (warps_per_token <= 12) { - LaunchKernel(num_rows, threads_per_block, device, smem_per_row)( - moe_fused_gate_kernel_small_token<12, kScoringFunc>, params); - } else { - LaunchKernel(num_rows, threads_per_block, device, smem_per_row)( - moe_fused_gate_kernel_small_token<16, kScoringFunc>, params); - } -} - -struct MoEFusedGateKernel { - static void - run(const tvm::ffi::TensorView input, - const tvm::ffi::TensorView bias, - const tvm::ffi::TensorView output, - const tvm::ffi::TensorView indices, - uint32_t topk, - uint32_t scoring_func, // 0 = sigmoid, 1 = sqrtsoftplus - uint32_t num_fused_shared_experts, - bool renormalize, - float routed_scaling_factor, - bool apply_routed_scaling_factor_on_output) { - using namespace host; - - auto N = SymbolicSize{"num_rows"}; - auto E = SymbolicSize{"num_experts"}; - auto K = SymbolicSize{"topk"}; - auto device = SymbolicDevice{}; - K.set_value(topk); - device.set_options(); - - TensorMatcher({N, E}).with_dtype().with_device(device).verify(input); - TensorMatcher({E}).with_dtype().with_device(device).verify(bias); - TensorMatcher({N, K}).with_dtype().with_device(device).verify(output); - TensorMatcher({N, K}).with_dtype().with_device(device).verify(indices); - - const auto num_rows = static_cast(N.unwrap()); - const auto num_experts = static_cast(E.unwrap()); - - RuntimeCheck(num_experts <= kMaxExperts, "num_experts exceeds maximum supported value"); - RuntimeCheck(scoring_func <= 1, "scoring_func must be 0 (sigmoid) or 1 (sqrtsoftplus)"); - RuntimeCheck(topk > num_fused_shared_experts, "topk must be greater than num_fused_shared_experts"); - - const auto params = MoEFusedGateParams{ - .input = static_cast(input.data_ptr()), - .bias = static_cast(bias.data_ptr()), - .output = static_cast(output.data_ptr()), - .indices = static_cast(indices.data_ptr()), - .num_rows = num_rows, - .num_experts = num_experts, - .topk = topk, - .num_fused_shared_experts = num_fused_shared_experts, - .renormalize = renormalize, - .routed_scaling_factor = routed_scaling_factor, - .apply_routed_scaling_factor_on_output = apply_routed_scaling_factor_on_output, - }; - - const size_t smem_per_row = 2 * num_experts * sizeof(float); - - bool use_small_token_kernel = num_rows <= kSmallTokenThreshold; - - if (use_small_token_kernel) { - // 1 token per block - uint32_t warps_per_token = div_ceil(num_experts, kWarpSize); - warps_per_token = std::min(warps_per_token, 16u); - uint32_t threads_per_block = warps_per_token * kWarpSize; - - if (scoring_func == 0) { - dispatch_small_token_kernel( - num_rows, threads_per_block, warps_per_token, device.unwrap(), smem_per_row, params); - } else { - dispatch_small_token_kernel( - num_rows, threads_per_block, warps_per_token, device.unwrap(), smem_per_row, params); - } - } else { - // multiple tokens per block - uint32_t num_blocks = div_ceil(num_rows, kWarpsPerCTA); - dim3 block_dim(kWarpSize, kWarpsPerCTA); - size_t large_smem = smem_per_row * kWarpsPerCTA; - - if (scoring_func == 0) { - LaunchKernel(num_blocks, block_dim, device.unwrap(), large_smem)( - moe_fused_gate_kernel, params); - } else { - LaunchKernel(num_blocks, block_dim, device.unwrap(), large_smem)( - moe_fused_gate_kernel, params); - } - } - } -}; - -} // namespace sglang diff --git a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh index 028766b9e7..1dbddd91b2 100644 --- a/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh +++ b/python/sglang/kernels/jit/include/sgl_kernel/deepseek_v4/kv_layout.cuh @@ -29,7 +29,7 @@ // The reader requires the rows of a page to be contiguous and the page stride // to be a multiple of kPageAlign (its TMA row stride), which is what // kv_page_bytes pads to. The pure-torch reference of the V41 (fp8) quantizer is -// `sglang.kernels.ops.attention.dsv4.torch_quant`. +// `sglang.test.kernels.deepseek_v4.k_cache_reference`. namespace sglang { diff --git a/python/sglang/kernels/ops/activation/__init__.py b/python/sglang/kernels/ops/activation/__init__.py index 14359b322a..3128b3d2e6 100644 --- a/python/sglang/kernels/ops/activation/__init__.py +++ b/python/sglang/kernels/ops/activation/__init__.py @@ -1,10 +1,11 @@ -"""Fused gated-activation kernels (``act(x[:h]) * x[h:]``). +"""Activation kernels. -Each operator is a :class:`~sglang.kernels.fused_op.BaseFusedOp` with a -pure-``torch`` reference (``forward_native``) plus AOT (``sgl_kernel``) and -JIT CUDA backends behind one ``(input, out)`` signature. The JIT backend -additionally accepts ``expert_ids`` / ``expert_step`` — call -``forward_jit`` directly when those are needed. +``silu_and_mul`` (``silu(x[..., :d]) * x[..., d:]``) is a +:class:`~sglang.kernels.fused_op.BaseFusedOp` with a pure-``torch`` reference +(``forward_native``) plus AOT (``sgl_kernel``), JIT CUDA and aiter backends +behind one ``(input, out)`` signature. The JIT backend additionally accepts +``expert_ids`` / ``expert_step`` — call ``forward_jit`` directly when those are +needed. """ from __future__ import annotations @@ -14,8 +15,8 @@ from typing import TYPE_CHECKING, Optional from sglang.kernels.fused_op import BaseFusedOp, register_fused_op from sglang.kernels.registry import register_kernel from sglang.kernels.spec import ( + CUDA_ONLY, CapabilityRequirement, - FormatSignature, KernelBackend, KernelSpec, ) @@ -23,85 +24,11 @@ from sglang.kernels.spec import ( if TYPE_CHECKING: import torch -_ACT_DTYPES = ("float16", "bfloat16") -_CUDA = frozenset({CapabilityRequirement.CUDA}) -_HIP = frozenset({CapabilityRequirement.HIP}) -# sgl_kernel's gated-activation ops build for CUDA *and* ROCm (production -# imports them from sgl_kernel on both), so the AOT backend spans both devices -# — the canonical OR-semantics case that a device-baked backend name couldn't. -_CUDA_HIP = frozenset({CapabilityRequirement.CUDA, CapabilityRequirement.HIP}) -# JIT before AOT to match the production path (srt/layers/activation.py imports -# from sglang.kernels.ops.activation.activation on CUDA); auto-selection must not invert it. -_ACT_PRIORITY = ( - KernelBackend.JIT, - KernelBackend.AOT, - KernelBackend.TORCH, -) - -class _GatedActivationOp(BaseFusedOp): - """Shared structure for ``act(x[..., :d]) * x[..., d:]`` operators.""" - - # Set by subclasses: sgl_kernel / jit_kernel attr name (same for both). - kernel_attr: str - - priority = _ACT_PRIORITY - capabilities = { - KernelBackend.AOT: _CUDA_HIP, - KernelBackend.JIT: _CUDA, - } - format_signature = FormatSignature( - supported_dtypes=_ACT_DTYPES, - description="gated activation; returns tensor", - ) - - def _act(self, gate: torch.Tensor) -> torch.Tensor: - raise NotImplementedError - - def forward_native( - self, input: torch.Tensor, out: Optional[torch.Tensor] = None - ) -> torch.Tensor: - d = input.shape[-1] // 2 - result = self._act(input[..., :d]) * input[..., d:] - if out is None: - return result - out.copy_(result) - return out - - def forward_aot( - self, input: torch.Tensor, out: Optional[torch.Tensor] = None - ) -> torch.Tensor: - import sgl_kernel - - return getattr(sgl_kernel, self.kernel_attr)(input, out) - - def forward_jit( - self, - input: torch.Tensor, - out: Optional[torch.Tensor] = None, - expert_ids: Optional[torch.Tensor] = None, - expert_step: int = 1, - ) -> torch.Tensor: - import sglang.kernels.ops.activation.activation as jit_activation - - return getattr(jit_activation, self.kernel_attr)( - input, out, expert_ids, expert_step - ) - - -class SiluAndMulOp(_GatedActivationOp): - """``out = silu(input[..., :d]) * input[..., d:]`` with ``d = input.shape[-1] // 2``. - - Adds an ``AITER`` backend on ``device=HIP``: on ROCm this op has a native - ``aiter`` kernel (``srt/layers/activation.py`` uses it in production). Note - the sibling gelu ops below deliberately do *not* register AITER — ROCm/aiter - coverage is a per-``(op, backend)`` subset, which the decoupled backend/device - model expresses directly (a device-agnostic ``KernelBackend`` name plus a - per-backend ``CapabilityRequirement``). - """ +class SiluAndMulOp(BaseFusedOp): + """``out = silu(input[..., :d]) * input[..., d:]`` with ``d = input.shape[-1] // 2``.""" op = "activation.silu_and_mul" - kernel_attr = "silu_and_mul" # AOT spans CUDA+HIP; JIT is CUDA; AITER is an opt-in HIP path. By priority, # CUDA resolves to JIT and HIP resolves to AOT (matching production # defaults); AITER is registered and HIP-eligible but sits below AOT, so it @@ -113,21 +40,44 @@ class SiluAndMulOp(_GatedActivationOp): KernelBackend.TORCH, ) capabilities = { - KernelBackend.AOT: _CUDA_HIP, - KernelBackend.JIT: _CUDA, - KernelBackend.AITER: _HIP, - } - descriptions = { - KernelBackend.AOT: "silu_and_mul (sgl_kernel wheel).", - KernelBackend.JIT: "silu_and_mul (sglang.kernels.jit).", - KernelBackend.AITER: "silu_and_mul (aiter, ROCm).", - KernelBackend.TORCH: "silu_and_mul (pure-torch reference).", + KernelBackend.AOT: frozenset( + {CapabilityRequirement.CUDA, CapabilityRequirement.HIP} + ), + KernelBackend.JIT: CUDA_ONLY, + KernelBackend.AITER: frozenset({CapabilityRequirement.HIP}), } - def _act(self, gate: torch.Tensor) -> torch.Tensor: + def forward_native( + self, input: torch.Tensor, out: Optional[torch.Tensor] = None + ) -> torch.Tensor: import torch.nn.functional as F - return F.silu(gate) + d = input.shape[-1] // 2 + result = F.silu(input[..., :d]) * input[..., d:] + if out is None: + return result + out.copy_(result) + return out + + def forward_aot( + self, input: torch.Tensor, out: Optional[torch.Tensor] = None + ) -> torch.Tensor: + from sgl_kernel import silu_and_mul as aot_silu_and_mul + + return aot_silu_and_mul(input, out) + + def forward_jit( + self, + input: torch.Tensor, + out: Optional[torch.Tensor] = None, + expert_ids: Optional[torch.Tensor] = None, + expert_step: int = 1, + ) -> torch.Tensor: + from sglang.kernels.ops.activation.activation import ( + silu_and_mul as jit_silu_and_mul, + ) + + return jit_silu_and_mul(input, out, expert_ids, expert_step) def forward_aiter( self, input: torch.Tensor, out: Optional[torch.Tensor] = None @@ -146,132 +96,7 @@ class SiluAndMulOp(_GatedActivationOp): return out -class GeluAndMulOp(_GatedActivationOp): - """``out = gelu(input[..., :d]) * input[..., d:]`` (erf-based GELU).""" - - op = "activation.gelu_and_mul" - kernel_attr = "gelu_and_mul" - descriptions = { - KernelBackend.AOT: "gelu_and_mul (sgl_kernel wheel).", - KernelBackend.JIT: "gelu_and_mul (sglang.kernels.jit).", - KernelBackend.TORCH: "gelu_and_mul (pure-torch reference).", - } - - def _act(self, gate: torch.Tensor) -> torch.Tensor: - import torch.nn.functional as F - - return F.gelu(gate, approximate="none") - - -class GeluTanhAndMulOp(_GatedActivationOp): - """``out = gelu_tanh(input[..., :d]) * input[..., d:]`` (tanh-approximated GELU).""" - - op = "activation.gelu_tanh_and_mul" - kernel_attr = "gelu_tanh_and_mul" - descriptions = { - KernelBackend.AOT: "gelu_tanh_and_mul (sgl_kernel wheel).", - KernelBackend.JIT: "gelu_tanh_and_mul (sglang.kernels.jit).", - KernelBackend.TORCH: "gelu_tanh_and_mul (pure-torch reference).", - } - - def _act(self, gate: torch.Tensor) -> torch.Tensor: - import torch.nn.functional as F - - return F.gelu(gate, approximate="tanh") - - -class ReLU2Op(BaseFusedOp): - """``out = relu(input) ** 2`` (single-input, not gated). - - The real kernel is the CUDA JIT path (``sglang.kernels.ops.activation.activation.relu2``, - used in production on CUDA); elsewhere the torch reference runs. - """ - - op = "activation.relu2" - priority = (KernelBackend.JIT, KernelBackend.TORCH) - capabilities = {KernelBackend.JIT: _CUDA} - format_signature = FormatSignature( - supported_dtypes=_ACT_DTYPES, - description="relu(x) ** 2; returns tensor", - ) - descriptions = { - KernelBackend.JIT: "relu(x)**2 (sglang.kernels.jit).", - KernelBackend.TORCH: "relu(x)**2 (pure-torch reference).", - } - - def forward_native( - self, input: torch.Tensor, out: Optional[torch.Tensor] = None - ) -> torch.Tensor: - import torch.nn.functional as F - - x = F.relu(input) - result = x * x - if out is None: - return result - out.copy_(result) - return out - - def forward_jit( - self, input: torch.Tensor, out: Optional[torch.Tensor] = None - ) -> torch.Tensor: - from sglang.kernels.ops.activation.activation import relu2 - - result = relu2(input) - if out is None: - return result - out.copy_(result) - return out - - -class QuickGELUOp(BaseFusedOp): - """``out = input * sigmoid(1.702 * input)`` (single-input, not gated). - - Only ROCm has a native kernel (``sgl_kernel.gelu_quick``, AOT on HIP); CUDA - uses the torch reference in production, so no CUDA backend is registered. - """ - - op = "activation.gelu_quick" - priority = (KernelBackend.AOT, KernelBackend.TORCH) - capabilities = {KernelBackend.AOT: _HIP} - format_signature = FormatSignature( - supported_dtypes=_ACT_DTYPES, - description="x * sigmoid(1.702 * x); returns tensor", - ) - descriptions = { - KernelBackend.AOT: "quick GELU (sgl_kernel wheel, ROCm).", - KernelBackend.TORCH: "quick GELU (pure-torch reference).", - } - - def forward_native( - self, input: torch.Tensor, out: Optional[torch.Tensor] = None - ) -> torch.Tensor: - import torch - - result = input * torch.sigmoid(1.702 * input) - if out is None: - return result - out.copy_(result) - return out - - def forward_aot( - self, input: torch.Tensor, out: Optional[torch.Tensor] = None - ) -> torch.Tensor: - import torch - from sgl_kernel import gelu_quick - - if out is None: - out = torch.empty(input.shape, dtype=input.dtype, device=input.device) - gelu_quick(input, out) - return out - - _SILU_AND_MUL = register_fused_op(SiluAndMulOp(), __name__, "_SILU_AND_MUL") -_GELU_AND_MUL = register_fused_op(GeluAndMulOp(), __name__, "_GELU_AND_MUL") -_GELU_TANH_AND_MUL = register_fused_op( - GeluTanhAndMulOp(), __name__, "_GELU_TANH_AND_MUL" -) -_RELU2 = register_fused_op(ReLU2Op(), __name__, "_RELU2") -_GELU_QUICK = register_fused_op(QuickGELUOp(), __name__, "_GELU_QUICK") def silu_and_mul( @@ -281,30 +106,6 @@ def silu_and_mul( return _SILU_AND_MUL(input, out) -def gelu_and_mul( - input: torch.Tensor, out: Optional[torch.Tensor] = None -) -> torch.Tensor: - """``out = gelu(input[..., :d]) * input[..., d:]``.""" - return _GELU_AND_MUL(input, out) - - -def gelu_tanh_and_mul( - input: torch.Tensor, out: Optional[torch.Tensor] = None -) -> torch.Tensor: - """``out = gelu_tanh(input[..., :d]) * input[..., d:]``.""" - return _GELU_TANH_AND_MUL(input, out) - - -def relu2(input: torch.Tensor, out: Optional[torch.Tensor] = None) -> torch.Tensor: - """``out = relu(input) ** 2``.""" - return _RELU2(input, out) - - -def gelu_quick(input: torch.Tensor, out: Optional[torch.Tensor] = None) -> torch.Tensor: - """``out = input * sigmoid(1.702 * input)``.""" - return _GELU_QUICK(input, out) - - def situ_and_mul( input: torch.Tensor, out: Optional[torch.Tensor], @@ -319,29 +120,19 @@ def situ_and_mul( __all__ = [ "situ_and_mul", "SiluAndMulOp", - "GeluAndMulOp", - "GeluTanhAndMulOp", - "ReLU2Op", - "QuickGELUOp", "silu_and_mul", - "gelu_and_mul", - "gelu_tanh_and_mul", - "relu2", - "gelu_quick", ] # Triton kernel migrated into this group (from layers/triton_ops/softcap); # registered for inventory. Import it from its module. -for _fn in ("softcap_out", "softcap_inplace_logits"): - register_kernel( - KernelSpec( - op=f"activation.{_fn}", - backend=KernelBackend.TRITON, - target=f"sglang.kernels.ops.activation.softcap:{_fn}", - ) +register_kernel( + KernelSpec( + op="activation.softcap_inplace_logits", + backend=KernelBackend.TRITON, + target="sglang.kernels.ops.activation.softcap:softcap_inplace_logits", ) -del _fn +) register_kernel( @@ -349,11 +140,7 @@ register_kernel( op="activation.softcap_to_float32_logits", backend=KernelBackend.TRITON, target="sglang.kernels.ops.activation.softcap:softcap_to_float32_logits", - capabilities=_CUDA, - format_signature=FormatSignature( - supported_dtypes=("float16", "bfloat16", "float32"), - description="Softcap logits into a caller-provided FP32 output.", - ), + capabilities=CUDA_ONLY, ) ) @@ -364,6 +151,6 @@ register_kernel( op="activation.situ_and_mul", backend=KernelBackend.JIT, target="sglang.kernels.ops.activation._jit_situ_and_mul:situ_and_mul", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) diff --git a/python/sglang/kernels/ops/activation/softcap.py b/python/sglang/kernels/ops/activation/softcap.py index a03b9f22a9..1040e68aed 100644 --- a/python/sglang/kernels/ops/activation/softcap.py +++ b/python/sglang/kernels/ops/activation/softcap.py @@ -3,70 +3,6 @@ import triton import triton.language as tl from triton.language.extra import libdevice -softcap_out_autotune = triton.autotune( - configs=[ - triton.Config(kwargs={"BLOCK_SIZE": 128}, num_warps=4), - triton.Config(kwargs={"BLOCK_SIZE": 128}, num_warps=8), - triton.Config(kwargs={"BLOCK_SIZE": 128}, num_warps=16), - triton.Config(kwargs={"BLOCK_SIZE": 256}, num_warps=4), - triton.Config(kwargs={"BLOCK_SIZE": 256}, num_warps=8), - triton.Config(kwargs={"BLOCK_SIZE": 512}, num_warps=4), - triton.Config(kwargs={"BLOCK_SIZE": 512}, num_warps=8), - triton.Config(kwargs={"BLOCK_SIZE": 512}, num_warps=16), - triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=4), - triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=8), - triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=16), - triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=32), - triton.Config(kwargs={"BLOCK_SIZE": 2048}, num_warps=32), - triton.Config(kwargs={"BLOCK_SIZE": 4096}, num_warps=32), - triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=32), - triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=32), - triton.Config(kwargs={"BLOCK_SIZE": 32768}, num_warps=32), - ], - key=["n_ele"], -) - - -@triton.jit -def softcap_out_kernel( - output_ptr, - input_ptr, - n_ele, - softcap_const: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - pid = tl.program_id(axis=0) - block_start = pid * BLOCK_SIZE - offsets = block_start + tl.arange(0, BLOCK_SIZE) - mask = offsets < n_ele - x = tl.load(input_ptr + offsets, mask=mask) - fx = x.to(tl.float32) - fxs = fx / softcap_const - exped = tl.exp(2 * fxs) - top = exped - 1 - bottom = exped + 1 - output = top / bottom * softcap_const - tl.store(output_ptr + offsets, output, mask=mask) - - -softcap_out_kernel_autotuned = softcap_out_autotune(softcap_out_kernel) - - -def softcap_out(x, softcap_const, autotune=False): - output = torch.empty_like(x, dtype=torch.float32) - n_elements = output.numel() - if autotune: - - def grid(meta): - return (triton.cdiv(n_elements, meta["BLOCK_SIZE"]),) - - softcap_out_kernel_autotuned[grid](output, x, n_elements, softcap_const) - else: - softcap_out_kernel[(triton.cdiv(n_elements, 128),)]( - output, x, n_elements, softcap_const, BLOCK_SIZE=128, num_warps=8 - ) - return output - @triton.jit def softcap_inplace_logits_kernel( diff --git a/python/sglang/kernels/ops/attention/__init__.py b/python/sglang/kernels/ops/attention/__init__.py index 4d277dd438..f8fe9fc3de 100644 --- a/python/sglang/kernels/ops/attention/__init__.py +++ b/python/sglang/kernels/ops/attention/__init__.py @@ -13,8 +13,8 @@ from typing import TYPE_CHECKING from sglang.kernels.registry import register_kernel from sglang.kernels.selector import get_kernel from sglang.kernels.spec import ( + CUDA_ONLY, CapabilityRequirement, - FormatSignature, KernelBackend, KernelSpec, ) @@ -66,15 +66,6 @@ register_kernel( capabilities=frozenset( {CapabilityRequirement.cuda(min_sm=(12, 1), max_sm=(12, 1))} ), - format_signature=FormatSignature( - supported_dtypes=("bfloat16",), - description=( - "Qwen3.8 packed QSA decode: D=256, 12:1 GQA, 1 <= q_rows <= 128" - ), - ), - description=( - "SM121 Qwen3.8 QSA decode optimized by Codex/Kimi K3 through KDA-1.5." - ), ) ) @@ -265,12 +256,6 @@ for _mod, _fn, _backend, _device in [ KernelBackend.CUTE_DSL, CapabilityRequirement.CUDA, ), - ( - "kda_flydsl.kimi_k3_kda_decode", - "flydsl_kimi_k3_kda_decode", - KernelBackend.FLYDSL, - CapabilityRequirement.HIP, - ), ( "kda_flydsl.kimi_k3_kda_decode", "flydsl_kimi_k3_kda_decode_with_f_b", @@ -308,7 +293,7 @@ register_kernel( op="attention.get_block_table", backend=KernelBackend.JIT, target="sglang.kernels.ops.attention.minicpm_sala.get_block_table:get_block_table", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -316,7 +301,7 @@ register_kernel( op="attention.fast_kpool_topk_transform_fused", backend=KernelBackend.JIT, target="sglang.kernels.ops.attention.dsa.kpool_topk_transform:fast_kpool_topk_transform_fused", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -324,7 +309,7 @@ register_kernel( op="attention.fast_topk", backend=KernelBackend.JIT, target="sglang.kernels.ops.attention.fast_topk:fast_topk", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -344,7 +329,7 @@ register_kernel( op="attention.fused_k_indexer_norm_rope", backend=KernelBackend.JIT, target="sglang.kernels.ops.attention.dsa.indexer_k:fused_k_indexer_norm_rope", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -352,7 +337,7 @@ register_kernel( op="attention.fused_k_indexer_norm_rope_store", backend=KernelBackend.JIT, target="sglang.kernels.ops.attention.dsa.indexer_k:fused_k_indexer_norm_rope_store", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -370,7 +355,7 @@ register_kernel( op="attention.fp4_index_logits_decode", backend=KernelBackend.TRITON, target="sglang.kernels.ops.attention.dsv4.fp4_indexer:fp4_index_logits_decode", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) @@ -379,8 +364,7 @@ register_kernel( op="attention.pack_prefix_current", backend=KernelBackend.TRITON, target="sglang.kernels.ops.attention.dllm_kv_pack:pack_prefix_current", - capabilities=frozenset({CapabilityRequirement.CUDA}), - description="Pack CSR prefix and current K/V into caller-provided dense buffers.", + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -391,10 +375,6 @@ register_kernel( capabilities=frozenset( {CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(11, 9))} ), - format_signature=FormatSignature( - supported_dtypes=("float16", "bfloat16"), - description="SM100/SM110 D512 GQA with separate K/V and optional page tables.", - ), ) ) register_kernel( @@ -402,12 +382,7 @@ register_kernel( op="attention.gemma_qkv_norm_rope", backend=KernelBackend.TRITON, target="sglang.kernels.ops.attention.gemma_qkv_norm_rope:gemma_qkv_norm_rope", - capabilities=frozenset({CapabilityRequirement.CUDA}), - format_signature=FormatSignature( - supported_dtypes=("float16", "bfloat16"), - in_place=True, - description="Strided QKV normalization and partial NeoX RoPE.", - ), + capabilities=CUDA_ONLY, ) ) @@ -417,7 +392,7 @@ for _fn in ("fp4_index_logits_paged", "finish_paged_indexer_topk"): op=f"attention.{_fn}", backend=KernelBackend.TRITON, target=f"sglang.kernels.ops.attention.dsv4.fp4_indexer:{_fn}", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) del _fn diff --git a/python/sglang/kernels/ops/attention/clamp_position.py b/python/sglang/kernels/ops/attention/clamp_position.py index 8753bfcaa3..1931746f1d 100644 --- a/python/sglang/kernels/ops/attention/clamp_position.py +++ b/python/sglang/kernels/ops/attention/clamp_position.py @@ -6,11 +6,7 @@ import torch from sglang.kernels.fused_op import BaseFusedOp, register_fused_op from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args -from sglang.kernels.spec import ( - CapabilityRequirement, - FormatSignature, - KernelBackend, -) +from sglang.kernels.spec import CapabilityRequirement, KernelBackend if TYPE_CHECKING: from tvm_ffi.module import Module @@ -51,13 +47,6 @@ class ClampPositionOp(BaseFusedOp): {CapabilityRequirement.CUDA, CapabilityRequirement.HIP} ) } - format_signature = FormatSignature( - supported_dtypes=("int32", "int64"), - description="clamp(seq_lens - 1, min=0); returns int64 positions", - ) - descriptions = { - KernelBackend.JIT: "Fused clamp-position kernel (sglang.kernels.jit).", - } def forward_native(self, seq_lens: torch.Tensor) -> torch.Tensor: return torch.clamp(seq_lens - 1, min=0).to(torch.int64) diff --git a/python/sglang/kernels/ops/attention/cute_utils/cvt.py b/python/sglang/kernels/ops/attention/cute_utils/cvt.py index 1eccb09a11..847012323e 100644 --- a/python/sglang/kernels/ops/attention/cute_utils/cvt.py +++ b/python/sglang/kernels/ops/attention/cute_utils/cvt.py @@ -1,9 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # Adapted from https://github.com/vllm-project/vllm/blob/4868b542c9dfd166662eecc4bb8be3a36a3feaa2/vllm/cute_utils/cvt.py -from cutlass import Constexpr, Float32, Uint32, cute -from cutlass._mlir import ir -from cutlass._mlir.dialects import llvm, vector +from cutlass import Float32, Uint32, cute +from cutlass._mlir.dialects import llvm from cutlass.cutlass_dsl import T, dsl_user_op @@ -48,99 +47,3 @@ def bf16x2_to_fp32x2(data, *, loc=None, ip=None) -> tuple[Float32, Float32]: else: raise ValueError(f"Unsupported type {type(data)}") - - -@dsl_user_op -def fp8x4_to_bf16x4(x: Uint32, *, loc=None, ip=None) -> cute.TensorSSA: - # there is only fp8->fp16 conversion, hence we need to go - # round trip through fp16. - out = llvm.inline_asm( - llvm.StructType.get_literal([T.i32()] * 2), - [x.ir_value(loc=loc, ip=ip)], - "{\n\t" - ".reg .b16 x0, x1;\n\t" - ".reg .b16 t00, t01, t10, t11;\n\t" - "mov.b32 {x0, x1}, $2;\n\t" - "cvt.rn.f16x2.e4m3x2 $0, x0;\n\t" - "cvt.rn.f16x2.e4m3x2 $1, x1;\n\t" - "mov.b32 {t00, t01}, $0;\n\t" - "mov.b32 {t10, t11}, $1;\n\t" - "cvt.rn.bf16.f16 t00, t00;\n\t" - "cvt.rn.bf16.f16 t01, t01;\n\t" - "cvt.rn.bf16.f16 t10, t10;\n\t" - "cvt.rn.bf16.f16 t11, t11;\n\t" - "mov.b32 $0, {t00, t01};\n\t" - "mov.b32 $1, {t10, t11};\n\t" - "}\n", - "=r,=r,r", - has_side_effects=False, - is_align_stack=False, - ) - vec = vector.from_elements( - ir.VectorType.get([2], T.i32(), loc=loc), - [llvm.extractvalue(T.i32(), out, [i], loc=loc, ip=ip) for i in range(2)], - loc=loc, - ip=ip, - ) - return cute.TensorSSA(vec, 2, Uint32) - - -@dsl_user_op -def fp32x4_to_fp8x4( - a0: Float32, - a1: Float32, - a2: Float32, - a3: Float32, - *, - loc=None, - ip=None, -) -> Uint32: - # Pack four FP32 values into one b32 of four e4m3 bytes, byte order - # {a0, a1, a2, a3} from low to high address. - out = llvm.inline_asm( - T.i32(), - [ - a0.ir_value(loc=loc, ip=ip), - a1.ir_value(loc=loc, ip=ip), - a2.ir_value(loc=loc, ip=ip), - a3.ir_value(loc=loc, ip=ip), - ], - "{\n\t" - ".reg .b16 t0, t1;\n\t" - "cvt.rn.satfinite.e4m3x2.f32 t0, $2, $1;\n\t" - "cvt.rn.satfinite.e4m3x2.f32 t1, $4, $3;\n\t" - "mov.b32 $0, {t0, t1};\n\t" - "}\n", - "=r,f,f,f,f", - has_side_effects=False, - is_align_stack=False, - ) - return Uint32(out) - - -@dsl_user_op -def fp32x8_to_fp4x8( - vals: cute.Tensor, - offset: Constexpr[int], - *, - loc=None, - ip=None, -) -> Uint32: - # Pack eight scaled FP32 values into four E2M1x2 bytes, returned as one b32. - assert vals.element_type is Float32 - out = llvm.inline_asm( - T.i32(), - [vals[offset + i].ir_value(loc=loc, ip=ip) for i in range(8)], - "{\n\t" - ".reg .b8 x0, x1, x2, x3;\n\t" - "cvt.rn.satfinite.e2m1x2.f32 x0, $2, $1;\n\t" - "cvt.rn.satfinite.e2m1x2.f32 x1, $4, $3;\n\t" - "cvt.rn.satfinite.e2m1x2.f32 x2, $6, $5;\n\t" - "cvt.rn.satfinite.e2m1x2.f32 x3, $8, $7;\n\t" - "mov.b32 $0, {x0, x1, x2, x3};\n\t" - "}\n", - "=r,f,f,f,f,f,f,f,f", - has_side_effects=False, - is_align_stack=False, - ) - return Uint32(out) diff --git a/python/sglang/kernels/ops/attention/cutedsl_gdn_mtp_ring.py b/python/sglang/kernels/ops/attention/cutedsl_gdn_mtp_ring.py index 64c0817b0f..f3e91678ca 100644 --- a/python/sglang/kernels/ops/attention/cutedsl_gdn_mtp_ring.py +++ b/python/sglang/kernels/ops/attention/cutedsl_gdn_mtp_ring.py @@ -32,16 +32,14 @@ PR #2905) are supported natively by both kernels. ``K = V = 128`` is required. Public API: -- ``gated_delta_rule()``: T=1 single-token decode with BF16 state. - ``gated_delta_rule_mtp()``: multi-token prediction (T>=1) with BF16 state. -Both entries dispatch to one of: +It dispatches to one of: - ``gdn_wide_vec_kernel`` — the fast path (LDG.E.128 / STG.E.128). Covers - T=1 with ``B*HV >= 512`` and T>=2 with ``B*HV >= 128``, single-pool or - split-pool. + T>=2 with ``B*HV >= 128``, single-pool or split-pool. - ``gdn_decode_bf16state_mtp_ilp4_kernel`` (ILP=4) — higher-occupancy - fallback for the low-throughput tail (B=1 at HV=64; T=1 small batch - with ``tile_v < 64``). Also single-pool or split-pool. + fallback for the low-throughput tail (B=1 at HV=64) and T=1. Also + single-pool or split-pool. """ import math @@ -124,8 +122,7 @@ ILP_ROWS = 4 # 4 V-rows held in regs per thread per iter # Processes 4 V-rows per group iteration (vs the original ILP=8 design). # ILP=4 uses ~48 regs/thread → ~62% occupancy, which covers the T=2 inline # g/beta recompute stall and the small-batch latency tail. Dispatched when -# wide_vec gates out — i.e. work_units <= 128 (B=1 at HV=64) or T=1 small -# batch with tile_v < 64. +# wide_vec gates out — i.e. work_units <= 128 (B=1 at HV=64) or T=1. # # Supports split-pool writes via ``h0_out_indices``: when the dispatcher # passes a separate write-indices tensor (output_state_indices != @@ -1969,497 +1966,6 @@ def gdn_wide_vec_kernel( cute.autovec_copy(r_hb3, ht_w3) -# ============================================================================== -# T=1 LEGACY KERNEL -# ============================================================================== -# Dedicated T=1 wide_vec variant. The wide_vec kernel above (used for T>=2) -# accumulated several changes — disable_output gating, packed STG via -# r_o4_bf16, kq_partial removal, Q+K loop split — that collectively -# introduced an ~8-9% regression at T=1 BS>=16 which no individual revert -# resolved. This variant preserves the pre-regression T=1 code path. -# -# Maintenance note: this kernel is FROZEN; do not apply future kernel -# improvements here. It exists solely to restore T=1 perf. The T>=2 -# kernel is the active optimization target. -# ============================================================================== - - -@cute.kernel -def gdn_wide_vec_kernel_t1( - h0_source: cute.Tensor, - intermediate_states: cute.Tensor, - A_log: cute.Tensor, - a: cute.Tensor, - dt_bias: cute.Tensor, - q: cute.Tensor, - k: cute.Tensor, - v: cute.Tensor, - b_gate: cute.Tensor, - o: cute.Tensor, - h0_indices: cute.Tensor, - h0_out_indices: cute.Tensor, - softplus_beta: cutlass.Constexpr[float], - softplus_threshold: cutlass.Constexpr[float], - scale: cutlass.Constexpr[float], - HV: cutlass.Constexpr[int], - T: cutlass.Constexpr[int], - H: cutlass.Constexpr[int], - K: cutlass.Constexpr[int], - V: cutlass.Constexpr[int], - tile_v: cutlass.Constexpr[int], - num_v_tiles: cutlass.Constexpr[int], - use_qk_l2norm: cutlass.Constexpr[bool], - disable_state_update: cutlass.Constexpr[bool], - cache_intermediate_states: cutlass.Constexpr[bool], - use_packed_fma: cutlass.Constexpr[bool], - same_pool: cutlass.Constexpr[bool], -): - tidx, _, _ = cute.arch.thread_idx() - lane_in_warp = tidx % 32 - warp_idx = cute.arch.warp_idx() - warp_idx = cute.arch.make_warp_uniform(warp_idx) - - # 8 groups of 16 threads. Within a CTA: - # group_idx in [0..8) - # lane_in_group in [0..16) — owns K[lane_in_group*8 : +8] - group_idx = tidx // LANES_PER_ROW - lane_in_group = tidx % LANES_PER_ROW - k_start = lane_in_group * ELEMS_PER_LANE - - ROWS_PER_GROUP: cutlass.Constexpr[int] = tile_v // NUM_GROUPS - ITERS_PER_GROUP: cutlass.Constexpr[int] = ROWS_PER_GROUP // ILP_ROWS - - batch_idx, _, _ = cute.arch.block_idx() - # Grid: (num_v_tiles × HV × B) linearized. Decode (i_n, i_hv, i_v) so each - # CTA handles one V-tile of one (n, hv) pair. - i_v = batch_idx % num_v_tiles - tmp = batch_idx // num_v_tiles - i_hv = tmp % HV - i_n = tmp // HV - i_h = i_hv // (HV // H) - - # Int64 widen: same wrap threshold as the T>=2 wide_vec kernel above - # (stride[0] = V*K = 16384 BF16 elements). PR #3230. - cache_idx = h0_indices[i_n] - - r_A_log = cutlass.Float32(A_log[i_hv]) - r_dt_bias = cutlass.Float32(dt_bias[i_hv]) - - # ----- SMEM ----- - smem = cutlass.utils.SmemAllocator() - sQ = smem.allocate_tensor( - cutlass.Float32, cute.make_layout((T, K), stride=(K + 8, 1)), 16 - ) - sK = smem.allocate_tensor( - cutlass.Float32, cute.make_layout((T, K), stride=(K + 8, 1)), 16 - ) - sGB = smem.allocate_tensor( - cutlass.Float32, cute.make_layout((T, 3), stride=(3, 1)), 16 - ) - - # ----- registers ----- - vec: cutlass.Constexpr[int] = ELEMS_PER_LANE # 8 - r_h = cute.make_rmem_tensor( - cute.make_layout((ILP_ROWS, vec), stride=(vec, 1)), cutlass.Float32 - ) - r_q = cute.make_rmem_tensor(cute.make_layout((vec,), stride=(1,)), cutlass.Float32) - r_k = cute.make_rmem_tensor(cute.make_layout((vec,), stride=(1,)), cutlass.Float32) - r_q_bf16 = cute.make_rmem_tensor( - cute.make_layout((vec,), stride=(1,)), cutlass.BFloat16 - ) - r_k_bf16 = cute.make_rmem_tensor( - cute.make_layout((vec,), stride=(1,)), cutlass.BFloat16 - ) - r_hb0 = cute.make_rmem_tensor( - cute.make_layout((vec,), stride=(1,)), cutlass.BFloat16 - ) - r_hb1 = cute.make_rmem_tensor( - cute.make_layout((vec,), stride=(1,)), cutlass.BFloat16 - ) - r_hb2 = cute.make_rmem_tensor( - cute.make_layout((vec,), stride=(1,)), cutlass.BFloat16 - ) - r_hb3 = cute.make_rmem_tensor( - cute.make_layout((vec,), stride=(1,)), cutlass.BFloat16 - ) - - if cache_idx < 0: - cache_idx = cutlass.Int32(0) - - # Split-pool write index: distinct slot to write the updated H state. - # When same_pool=True (compile-time, set by the dispatcher whenever the - # caller's read and write indices alias), nvcc DCEs the LDG + - # negative-redirect compare. When False, the kernel reads the - # write-indices tensor and applies the same null-slot redirect as the - # read side. - if cutlass.const_expr(same_pool): - write_cache_idx = cache_idx - else: - write_cache_idx = h0_out_indices[i_n] - if write_cache_idx < 0: - write_cache_idx = cutlass.Int32(0) - - if cache_idx >= 0: - # 4D slot indexing: see gdn_decode_bf16state_mtp_ilp4_kernel for the - # rationale (vLLM-style padded pools have stride[0] > HV*V*K, which - # would silently clone if we reshaped to 3D). cute uses the - # caller-supplied strides; Int64 wrap on cache_idx prevents int32 - # overflow at large pool sizes. - h0_slot_r = h0_source[(cutlass.Int64(cache_idx), i_hv, None, None)] - if cutlass.const_expr(same_pool): - h0_slot_w = h0_slot_r - else: - h0_slot_w = h0_source[(cutlass.Int64(write_cache_idx), i_hv, None, None)] - - # ================================================================== - # Phase 0: precompute q/k/g/beta/kq into SMEM (all 4 warps) - # Each warp handles one token per pass; 4 warps -> ceil(T/4) passes. - # Within a warp, threads cooperate by K-lane (16 threads cover K=128). - # ================================================================== - # Use the same 16-thread-per-row layout for precompute. Only the first - # 16 lanes of each warp participate; the second 16 lanes sit idle for - # q/k loads but contribute identical work for reduction consistency. - # To keep it simple: all 32 threads of a warp co-load via 2 groups of 16. - # Here lane_in_warp < 16 loads q/k for the warp's assigned token; the - # upper 16 lanes redundantly load/compute (their writes land in the same - # SMEM slots, idempotent). - num_precompute_passes: cutlass.Constexpr[int] = (T + NUM_WARPS - 1) // NUM_WARPS - member_pre = lane_in_warp % LANES_PER_ROW - k_start_pre = member_pre * ELEMS_PER_LANE - for pass_idx in cutlass.range_constexpr(num_precompute_passes): - i_t_pre = pass_idx * NUM_WARPS + warp_idx - if i_t_pre < T: - q_tile_pre = cute.local_tile( - q, (1, 1, 1, vec), (i_n, i_t_pre, i_h, member_pre) - ) - k_tile_pre = cute.local_tile( - k, (1, 1, 1, vec), (i_n, i_t_pre, i_h, member_pre) - ) - cute.autovec_copy(q_tile_pre, r_q_bf16) - cute.autovec_copy(k_tile_pre, r_k_bf16) - for i in cutlass.range_constexpr(vec): - r_q[i] = cutlass.Float32(r_q_bf16[i]) - r_k[i] = cutlass.Float32(r_k_bf16[i]) - - if cutlass.const_expr(use_qk_l2norm): - sum_q = cutlass.Float32(0.0) - sum_k = cutlass.Float32(0.0) - for i in cutlass.range_constexpr(vec): - sum_q += r_q[i] * r_q[i] - sum_k += r_k[i] * r_k[i] - # 4-stage butterfly within 16-thread subgroup - for offset in [8, 4, 2, 1]: - sum_q += cute.arch.shuffle_sync_bfly( - sum_q, offset=offset, mask=-1, mask_and_clamp=31 - ) - sum_k += cute.arch.shuffle_sync_bfly( - sum_k, offset=offset, mask=-1, mask_and_clamp=31 - ) - inv_norm_q_scaled = cute.rsqrt(sum_q + 1e-6, fastmath=True) * scale - inv_norm_k = cute.rsqrt(sum_k + 1e-6, fastmath=True) - for i in cutlass.range_constexpr(vec): - r_q[i] = r_q[i] * inv_norm_q_scaled - r_k[i] = r_k[i] * inv_norm_k - else: - for i in cutlass.range_constexpr(vec): - r_q[i] = r_q[i] * scale - - # Write q, k to SMEM (both 16-thread subgroups write same data) - for i in cutlass.range_constexpr(vec): - sQ[(i_t_pre, k_start_pre + i)] = r_q[i] - sK[(i_t_pre, k_start_pre + i)] = r_k[i] - - # kq partial and reduce within 16-thread subgroup - kq_partial = cutlass.Float32(0.0) - for i in cutlass.range_constexpr(vec): - kq_partial += r_k[i] * r_q[i] - for offset in [8, 4, 2, 1]: - kq_partial += cute.arch.shuffle_sync_bfly( - kq_partial, offset=offset, mask=-1, mask_and_clamp=31 - ) - - # g, beta, kq to sGB (lane 0 of warp writes) - r_a_pre = cutlass.Float32(a[i_n, i_t_pre, i_hv]) - r_b_pre = cutlass.Float32(b_gate[i_n, i_t_pre, i_hv]) - x_pre = r_a_pre + r_dt_bias - beta_x_pre = softplus_beta * x_pre - exp_beta_x_pre = cute.exp(beta_x_pre, fastmath=True) - softplus_val_pre = (cutlass.Float32(1.0) / softplus_beta) * cute.log( - cutlass.Float32(1.0) + exp_beta_x_pre, fastmath=True - ) - use_softplus_pre = ( - cutlass.Float32(1.0) - if beta_x_pre <= softplus_threshold - else cutlass.Float32(0.0) - ) - softplus_x_pre = ( - use_softplus_pre * softplus_val_pre - + (cutlass.Float32(1.0) - use_softplus_pre) * x_pre - ) - r_g_pre = cute.exp( - -cute.exp(r_A_log, fastmath=True) * softplus_x_pre, fastmath=True - ) - r_beta_pre = cutlass.Float32(1.0) / ( - cutlass.Float32(1.0) + cute.exp(-r_b_pre, fastmath=True) - ) - - if lane_in_warp == 0: - sGB[(i_t_pre, 0)] = r_g_pre - sGB[(i_t_pre, 1)] = r_beta_pre - sGB[(i_t_pre, 2)] = kq_partial - - cute.arch.barrier() - - # ================================================================== - # Phase 1: main compute loop - # Each group of 16 threads owns ROWS_PER_GROUP=16 V-rows. - # With ILP_ROWS=4, iterate 4 times. Each iter holds 4 V-rows in r_h. - # ================================================================== - for iter_idx in cutlass.range_constexpr(ITERS_PER_GROUP): - v_base = i_v * tile_v + group_idx * ROWS_PER_GROUP + iter_idx * ILP_ROWS - v0 = v_base + 0 - v1 = v_base + 1 - v2 = v_base + 2 - v3 = v_base + 3 - - # Load 4 V-rows of h (LDG.128 each) into r_h from the read slot. - ht0 = cute.local_tile(h0_slot_r, (1, vec), (v0, lane_in_group)) - ht1 = cute.local_tile(h0_slot_r, (1, vec), (v1, lane_in_group)) - ht2 = cute.local_tile(h0_slot_r, (1, vec), (v2, lane_in_group)) - ht3 = cute.local_tile(h0_slot_r, (1, vec), (v3, lane_in_group)) - # Write-side tiles. In single-pool (same_pool=True), they alias - # the read tiles — nvcc DCEs the write-side base-pointer - # arithmetic (the source of the +5-7 % T=1 large-B regression). - # In split-pool (same_pool=False), separate STG.128 destinations - # at the split-pool write slot. - if cutlass.const_expr(same_pool): - ht_w0 = ht0 - ht_w1 = ht1 - ht_w2 = ht2 - ht_w3 = ht3 - else: - ht_w0 = cute.local_tile(h0_slot_w, (1, vec), (v0, lane_in_group)) - ht_w1 = cute.local_tile(h0_slot_w, (1, vec), (v1, lane_in_group)) - ht_w2 = cute.local_tile(h0_slot_w, (1, vec), (v2, lane_in_group)) - ht_w3 = cute.local_tile(h0_slot_w, (1, vec), (v3, lane_in_group)) - cute.autovec_copy(ht0, r_hb0) - cute.autovec_copy(ht1, r_hb1) - cute.autovec_copy(ht2, r_hb2) - cute.autovec_copy(ht3, r_hb3) - for i in cutlass.range_constexpr(vec): - r_h[0, i] = cutlass.Float32(r_hb0[i]) - r_h[1, i] = cutlass.Float32(r_hb1[i]) - r_h[2, i] = cutlass.Float32(r_hb2[i]) - r_h[3, i] = cutlass.Float32(r_hb3[i]) - - # Process each token sequentially (state is carried in registers). - # Non-fused form for numerical robustness: compute s = h_decayed @ k, - # then update h, then compute o = h_new @ q. Two reductions per token - # instead of one, but matches baseline's accumulation order. - for i_t in cutlass.range(T, unroll=1, unroll_full=(T <= 1)): - r_g = sGB[(i_t, 0)] - r_beta = sGB[(i_t, 1)] - - # Decay + h @ k (in one K-loop) - s0 = cutlass.Float32(0.0) - s1 = cutlass.Float32(0.0) - s2 = cutlass.Float32(0.0) - s3 = cutlass.Float32(0.0) - for i in cutlass.range_constexpr(0, vec, 2): - kv0 = sK[(i_t, k_start + i)] - kv1 = sK[(i_t, k_start + i + 1)] - if cutlass.const_expr(use_packed_fma): - r_h[0, i], r_h[0, i + 1] = cute.arch.fma_packed_f32x2( - src_a=(r_h[0, i], r_h[0, i + 1]), - src_b=(r_g, r_g), - src_c=(cutlass.Float32(0.0), cutlass.Float32(0.0)), - ) - r_h[1, i], r_h[1, i + 1] = cute.arch.fma_packed_f32x2( - src_a=(r_h[1, i], r_h[1, i + 1]), - src_b=(r_g, r_g), - src_c=(cutlass.Float32(0.0), cutlass.Float32(0.0)), - ) - r_h[2, i], r_h[2, i + 1] = cute.arch.fma_packed_f32x2( - src_a=(r_h[2, i], r_h[2, i + 1]), - src_b=(r_g, r_g), - src_c=(cutlass.Float32(0.0), cutlass.Float32(0.0)), - ) - r_h[3, i], r_h[3, i + 1] = cute.arch.fma_packed_f32x2( - src_a=(r_h[3, i], r_h[3, i + 1]), - src_b=(r_g, r_g), - src_c=(cutlass.Float32(0.0), cutlass.Float32(0.0)), - ) - else: - r_h[0, i], r_h[0, i + 1] = fma_pair_mul( - r_h[0, i], r_h[0, i + 1], r_g, r_g - ) - r_h[1, i], r_h[1, i + 1] = fma_pair_mul( - r_h[1, i], r_h[1, i + 1], r_g, r_g - ) - r_h[2, i], r_h[2, i + 1] = fma_pair_mul( - r_h[2, i], r_h[2, i + 1], r_g, r_g - ) - r_h[3, i], r_h[3, i + 1] = fma_pair_mul( - r_h[3, i], r_h[3, i + 1], r_g, r_g - ) - s0 = s0 + r_h[0, i] * kv0 + r_h[0, i + 1] * kv1 - s1 = s1 + r_h[1, i] * kv0 + r_h[1, i + 1] * kv1 - s2 = s2 + r_h[2, i] * kv0 + r_h[2, i + 1] * kv1 - s3 = s3 + r_h[3, i] * kv0 + r_h[3, i + 1] * kv1 - - # Butterfly reduce s across 16-thread subgroup (4 stages) - for offset in [8, 4, 2, 1]: - s0 += cute.arch.shuffle_sync_bfly( - s0, offset=offset, mask=-1, mask_and_clamp=31 - ) - s1 += cute.arch.shuffle_sync_bfly( - s1, offset=offset, mask=-1, mask_and_clamp=31 - ) - s2 += cute.arch.shuffle_sync_bfly( - s2, offset=offset, mask=-1, mask_and_clamp=31 - ) - s3 += cute.arch.shuffle_sync_bfly( - s3, offset=offset, mask=-1, mask_and_clamp=31 - ) - - # Delta rule: v_new = (v - s) * beta - v_val0 = cutlass.Float32(v[(i_n, i_t, i_hv, v0)]) - v_val1 = cutlass.Float32(v[(i_n, i_t, i_hv, v1)]) - v_val2 = cutlass.Float32(v[(i_n, i_t, i_hv, v2)]) - v_val3 = cutlass.Float32(v[(i_n, i_t, i_hv, v3)]) - vn0 = (v_val0 - s0) * r_beta - vn1 = (v_val1 - s1) * r_beta - vn2 = (v_val2 - s2) * r_beta - vn3 = (v_val3 - s3) * r_beta - - # Rank-1 update + h @ q (in one K-loop) - o0 = cutlass.Float32(0.0) - o1 = cutlass.Float32(0.0) - o2 = cutlass.Float32(0.0) - o3 = cutlass.Float32(0.0) - for i in cutlass.range_constexpr(0, vec, 2): - qv0 = sQ[(i_t, k_start + i)] - qv1 = sQ[(i_t, k_start + i + 1)] - kv0 = sK[(i_t, k_start + i)] - kv1 = sK[(i_t, k_start + i + 1)] - if cutlass.const_expr(use_packed_fma): - r_h[0, i], r_h[0, i + 1] = cute.arch.fma_packed_f32x2( - src_a=(kv0, kv1), - src_b=(vn0, vn0), - src_c=(r_h[0, i], r_h[0, i + 1]), - ) - r_h[1, i], r_h[1, i + 1] = cute.arch.fma_packed_f32x2( - src_a=(kv0, kv1), - src_b=(vn1, vn1), - src_c=(r_h[1, i], r_h[1, i + 1]), - ) - r_h[2, i], r_h[2, i + 1] = cute.arch.fma_packed_f32x2( - src_a=(kv0, kv1), - src_b=(vn2, vn2), - src_c=(r_h[2, i], r_h[2, i + 1]), - ) - r_h[3, i], r_h[3, i + 1] = cute.arch.fma_packed_f32x2( - src_a=(kv0, kv1), - src_b=(vn3, vn3), - src_c=(r_h[3, i], r_h[3, i + 1]), - ) - else: - r_h[0, i], r_h[0, i + 1] = fma_pair( - kv0, kv1, vn0, vn0, r_h[0, i], r_h[0, i + 1] - ) - r_h[1, i], r_h[1, i + 1] = fma_pair( - kv0, kv1, vn1, vn1, r_h[1, i], r_h[1, i + 1] - ) - r_h[2, i], r_h[2, i + 1] = fma_pair( - kv0, kv1, vn2, vn2, r_h[2, i], r_h[2, i + 1] - ) - r_h[3, i], r_h[3, i + 1] = fma_pair( - kv0, kv1, vn3, vn3, r_h[3, i], r_h[3, i + 1] - ) - # h_new @ q with updated r_h - o0 = o0 + r_h[0, i] * qv0 + r_h[0, i + 1] * qv1 - o1 = o1 + r_h[1, i] * qv0 + r_h[1, i + 1] * qv1 - o2 = o2 + r_h[2, i] * qv0 + r_h[2, i + 1] * qv1 - o3 = o3 + r_h[3, i] * qv0 + r_h[3, i + 1] * qv1 - - # Butterfly reduce o - for offset in [8, 4, 2, 1]: - o0 += cute.arch.shuffle_sync_bfly( - o0, offset=offset, mask=-1, mask_and_clamp=31 - ) - o1 += cute.arch.shuffle_sync_bfly( - o1, offset=offset, mask=-1, mask_and_clamp=31 - ) - o2 += cute.arch.shuffle_sync_bfly( - o2, offset=offset, mask=-1, mask_and_clamp=31 - ) - o3 += cute.arch.shuffle_sync_bfly( - o3, offset=offset, mask=-1, mask_and_clamp=31 - ) - - # Output scalar write (lane_in_group==0 only) - if lane_in_group == 0: - o[(i_n, i_t, i_hv, v0)] = cutlass.BFloat16(o0) - o[(i_n, i_t, i_hv, v1)] = cutlass.BFloat16(o1) - o[(i_n, i_t, i_hv, v2)] = cutlass.BFloat16(o2) - o[(i_n, i_t, i_hv, v3)] = cutlass.BFloat16(o3) - - # Intermediate write (for every token when caching) - if cutlass.const_expr(cache_intermediate_states): - for i in cutlass.range_constexpr(vec): - r_hb0[i] = cutlass.BFloat16(r_h[0, i]) - r_hb1[i] = cutlass.BFloat16(r_h[1, i]) - r_hb2[i] = cutlass.BFloat16(r_h[2, i]) - r_hb3[i] = cutlass.BFloat16(r_h[3, i]) - # The intermediate_states buffer is sized [B, T, HV, V, K] - # (batch-scoped, NOT pool-scoped), so this index uses i_n - # (the per-call batch index) and not cache_idx (the pool - # slot). Using cache_idx here writes OOB whenever - # initial_state_indices points at slots >= B (i.e. any - # realistic pool_size > B serving config). Fix mirrors - # upstream PR #3145. Int64 widening per PR #3230. - flat_idx = i_n * T * HV + i_t * HV + i_hv - it0 = cute.local_tile( - intermediate_states, - (1, 1, vec), - (flat_idx, v0, lane_in_group), - ) - it1 = cute.local_tile( - intermediate_states, - (1, 1, vec), - (flat_idx, v1, lane_in_group), - ) - it2 = cute.local_tile( - intermediate_states, - (1, 1, vec), - (flat_idx, v2, lane_in_group), - ) - it3 = cute.local_tile( - intermediate_states, - (1, 1, vec), - (flat_idx, v3, lane_in_group), - ) - cute.autovec_copy(r_hb0, it0) - cute.autovec_copy(r_hb1, it1) - cute.autovec_copy(r_hb2, it2) - cute.autovec_copy(r_hb3, it3) - - # Final state write-back to the split-pool WRITE slot. Skipped when - # caching is enabled (inter[T-1] already holds the final state). - if cutlass.const_expr( - not disable_state_update and not cache_intermediate_states - ): - for i in cutlass.range_constexpr(vec): - r_hb0[i] = cutlass.BFloat16(r_h[0, i]) - r_hb1[i] = cutlass.BFloat16(r_h[1, i]) - r_hb2[i] = cutlass.BFloat16(r_h[2, i]) - r_hb3[i] = cutlass.BFloat16(r_h[3, i]) - cute.autovec_copy(r_hb0, ht_w0) - cute.autovec_copy(r_hb1, ht_w1) - cute.autovec_copy(r_hb2, ht_w2) - cute.autovec_copy(r_hb3, ht_w3) - - # ============================================================================== # LAUNCH WRAPPER (MTP ILP=4 version) # ============================================================================== @@ -2686,83 +2192,6 @@ def _run_wide_vec( ) -# ============================================================================== -# LAUNCH WRAPPER (T=1 legacy) — corresponds to gdn_wide_vec_kernel_t1 -# ============================================================================== - - -@cute.jit -def _run_wide_vec_t1( - h0_source: cute.Tensor, - intermediate_states: cute.Tensor, - A_log: cute.Tensor, - a: cute.Tensor, - dt_bias: cute.Tensor, - q: cute.Tensor, - k: cute.Tensor, - v: cute.Tensor, - b_gate: cute.Tensor, - o: cute.Tensor, - h0_indices: cute.Tensor, - h0_out_indices: cute.Tensor, - softplus_beta: cutlass.Constexpr[float], - softplus_threshold: cutlass.Constexpr[float], - scale: cutlass.Constexpr[float], - HV: cutlass.Constexpr[int], - T: cutlass.Constexpr[int], - H: cutlass.Constexpr[int], - K: cutlass.Constexpr[int], - V: cutlass.Constexpr[int], - tile_v: cutlass.Constexpr[int], - use_qk_l2norm: cutlass.Constexpr[bool], - disable_state_update: cutlass.Constexpr[bool], - cache_intermediate_states: cutlass.Constexpr[bool], - use_packed_fma: cutlass.Constexpr[bool], - same_pool: cutlass.Constexpr[bool], - stream: cuda.CUstream, -): - num_v_tiles: cutlass.Constexpr[int] = V // tile_v - B = cute.size(q.shape[0]) - grid_size = B * HV * num_v_tiles - smem_bytes = ( - 4 * T * (K + 8) + 4 * T * (K + 8) + 4 * T * 3 + 256 # sQ FP32 # sK FP32 # sGB - ) - gdn_wide_vec_kernel_t1( - h0_source, - intermediate_states, - A_log, - a, - dt_bias, - q, - k, - v, - b_gate, - o, - h0_indices, - h0_out_indices, - softplus_beta, - softplus_threshold, - scale, - HV, - T, - H, - K, - V, - tile_v, - num_v_tiles, - use_qk_l2norm, - disable_state_update, - cache_intermediate_states, - use_packed_fma, - same_pool, - ).launch( - grid=(grid_size, 1, 1), - block=[NUM_THREADS, 1, 1], - smem=smem_bytes, - stream=stream, - ) - - # ============================================================================== # PUBLIC API # ============================================================================== @@ -2775,135 +2204,6 @@ _GPU_MAJOR, _ = torch.cuda.get_device_capability(0) _USE_PACKED_FMA = _GPU_MAJOR >= 10 -def gated_delta_rule( - A_log: torch.Tensor, - a: torch.Tensor, - dt_bias: torch.Tensor, - softplus_beta: float = 1.0, - softplus_threshold: float = 20.0, - q: Optional[torch.Tensor] = None, - k: Optional[torch.Tensor] = None, - v: Optional[torch.Tensor] = None, - b: Optional[torch.Tensor] = None, - initial_state_source: Optional[torch.Tensor] = None, - initial_state_indices: Optional[torch.Tensor] = None, - output_state_indices: Optional[torch.Tensor] = None, - output: Optional[torch.Tensor] = None, - use_qk_l2norm_in_kernel: bool = True, - scale: Optional[float] = None, -) -> torch.Tensor: - """ - GDN decode T=1 with BF16 state (pool mode, K=V=128 only). - - Dispatches to wide_vec when work_units is large enough - (`B * HV >= 512`, i.e. tile_v >= 64 at T=1); otherwise falls through - to the MTP T=1 path which picks tile_v via ``_get_bf16_mtp_config`` - and runs ``gdn_decode_bf16state_mtp_ilp4_kernel``. Both kernels - handle split-pool natively (``output_state_indices`` != - ``initial_state_indices``). - - Args: - A_log: [HV] float32 - a: [B, 1, HV] bf16 - dt_bias: [HV] float32 - q: [B, 1, H, K] bf16 - k: [B, 1, H, K] bf16 - v: [B, 1, HV, V] bf16 - b: [B, 1, HV] bf16 - initial_state_source: [pool_size, HV, V, K] bf16 — shared state pool - (modified in-place at the slots given by indices). - initial_state_indices: [B] int32 — pool slots to read. - Negative entries redirect to slot 0 (null buffer). REQUIRED. - output_state_indices: Optional [B] int32 — pool slots to write. - Defaults to initial_state_indices when None. Forwarded to the - kernel so split-pool is supported on either dispatch path. - output: Optional pre-allocated [B, 1, HV, V] bf16 output - scale: Optional, default 1/sqrt(K) - - Returns: - output: [B, 1, HV, V] bf16 - """ - assert q is not None and k is not None and v is not None - assert b is not None and initial_state_source is not None - - B, T, H, K = q.shape - assert T == 1, f"This kernel only supports T=1, got T={T}" - HV = v.shape[2] - V = v.shape[3] - assert K == 128 and V == 128, f"K and V must be 128, got K={K}, V={V}" - assert initial_state_source.dtype == torch.bfloat16 - assert initial_state_indices is not None, ( - "Pool mode is required: pass initial_state_indices. " - "Non-pool mode is no longer supported by the BF16 GDN kernels." - ) - - if scale is None: - scale = 1.0 / math.sqrt(K) - - if output_state_indices is not None and output_state_indices.dtype != torch.int32: - output_state_indices = output_state_indices.to(torch.int32) - - # Wide_vec T=1 fast path. Wide_vec uses LDG.E.128 / STG.E.128 on H, halving - # LSU instruction count vs the baseline ILP=4 kernel. SMEM-precompute phase - # runs ceil(T/NUM_WARPS)=1 pass at T=1, so wide_vec degenerates gracefully. - # - # Gate: tile_v >= 64. At T=1 the wide_vec Phase 0 precompute overhead is - # fixed per CTA while the main loop shrinks with tile_v; tile_v=32 gives - # only 1 ILP iter per subgroup, insufficient to amortize Phase 0. - # Measured at HV=64: tile_v=32 regresses at B=4 (0.91x); tile_v=64 wins - # at B=8 (1.05x). Split-pool writes are now natively supported by - # wide_vec so no longer gated on `output_state_indices is None`. - wv_tile_v = _select_wide_vec_tile_v(B, HV) - if wv_tile_v is not None and wv_tile_v < 64: - wv_tile_v = None # tile_v=32 at T=1 loses to MTP fallback - if wv_tile_v is not None: - # Route T=1 to the legacy wide_vec wrapper (gated_delta_rule_t1_wide_vec - # → gdn_wide_vec_kernel_t1). The v3-v8 kernel changes regressed T=1 - # by 8% at BS>=16; the legacy kernel restores that perf. T>=2 uses - # gated_delta_rule_mtp_wide_vec (the current v8 kernel) instead. - return gated_delta_rule_t1_wide_vec( - A_log=A_log, - a=a, - dt_bias=dt_bias, - softplus_beta=softplus_beta, - softplus_threshold=softplus_threshold, - q=q, - k=k, - v=v, - b=b, - initial_state_source=initial_state_source, - initial_state_indices=initial_state_indices, - output_state_indices=output_state_indices, - intermediate_states_buffer=None, # T=1 has no cache - disable_state_update=False, # T=1 default: write final state - use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, - scale=scale, - output=output, - tile_v=wv_tile_v, - ) - - # Wide_vec didn't fire (B*HV too small at T=1, i.e. tile_v < 64). - # Route through the MTP T=1 path which dispatches to mtp_ilp4_kernel - # via _get_bf16_mtp_config. - return gated_delta_rule_mtp( - A_log=A_log, - a=a, - dt_bias=dt_bias, - softplus_beta=softplus_beta, - softplus_threshold=softplus_threshold, - q=q, - k=k, - v=v, - b=b, - initial_state_source=initial_state_source, - initial_state_indices=initial_state_indices, - output_state_indices=output_state_indices, - output=output, - use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, - scale=scale, - ) - - # ============================================================================== # MTP PUBLIC API # ============================================================================== @@ -2939,9 +2239,9 @@ def _get_bf16_mtp_config( With ILP=4: ~48 regs/thread → ~62% occupancy. Wide_vec now covers every shape where ILP=8 was historically a win - (B*HV >= 128 at T>=2; B>=8 at T=1 with HV=64). The MTP fallback is - only reached at low work_units / T=1 small-batch redirect, where - ILP=4's higher occupancy beats ILP=8's larger per-CTA work amount. + (B*HV >= 128 at T>=2). The MTP fallback is only reached at low + work_units or T=1, where ILP=4's higher occupancy beats ILP=8's larger + per-CTA work amount. Returns ``(tile_v, ilp_rows)`` with ``ilp_rows == 4``. """ @@ -3018,7 +2318,7 @@ def _select_wide_vec_phase_b_unroll( # ============================================================================== -# PYTHON ENTRY (wide_vec) — called from gated_delta_rule and gated_delta_rule_mtp +# PYTHON ENTRY (wide_vec) — called from gated_delta_rule_mtp # ============================================================================== @@ -3054,7 +2354,7 @@ def gated_delta_rule_mtp_wide_vec( """Wide-vector BF16 GDN MTP decode. Prefer calling via the production entry point `gated_delta_rule_mtp` - (T>=2) or `gated_delta_rule` (T=1), which auto-dispatch to this + (T>=2), which auto-dispatches to this kernel when `B * HV >= 128` (tile_v=32) up through `>= 1024` (tile_v=128). Call this symbol directly only when you know your work size hits the fast path. @@ -3463,246 +2763,6 @@ def gated_delta_rule_mtp_wide_vec( return output -# ============================================================================== -# T=1 PUBLIC API (legacy) — wraps _run_wide_vec_t1, called from gated_delta_rule -# ============================================================================== - - -def gated_delta_rule_t1_wide_vec( - A_log: torch.Tensor, - a: torch.Tensor, - dt_bias: torch.Tensor, - softplus_beta: float = 1.0, - softplus_threshold: float = 20.0, - q: Optional[torch.Tensor] = None, - k: Optional[torch.Tensor] = None, - v: Optional[torch.Tensor] = None, - b: Optional[torch.Tensor] = None, - initial_state_source: Optional[torch.Tensor] = None, - initial_state_indices: Optional[torch.Tensor] = None, - output_state_indices: Optional[torch.Tensor] = None, - intermediate_states_buffer: Optional[torch.Tensor] = None, - disable_state_update: bool = False, - use_qk_l2norm_in_kernel: bool = True, - scale: Optional[float] = None, - output: Optional[torch.Tensor] = None, - tile_v: int = 128, -) -> torch.Tensor: - """Wide-vector BF16 GDN MTP decode. - - Prefer calling via the production entry point `gated_delta_rule_mtp` - (T>=2) or `gated_delta_rule` (T=1), which auto-dispatch to this - kernel when `B * HV >= 128` (tile_v=32) up through `>= 1024` - (tile_v=128). Call this symbol directly only when you know your - work size hits the fast path. - - When `intermediate_states_buffer is not None`, skips the final state - writeback; caller must read the final state from `buffer[:, T-1]`. - - `output_state_indices` enables split-pool semantics: when non-None and - different from `initial_state_indices`, the kernel reads from the read - slots and writes the updated H state to the write slots. When None - (or pointing at the same tensor as `initial_state_indices`), the - kernel reads and writes the same slot (single-pool). - """ - global _compiled_kernels_wide_vec - - assert q is not None and k is not None and v is not None - assert b is not None and initial_state_source is not None - - B_val, T_val, H_val, K_val = q.shape - HV_val = v.shape[2] - V_val = v.shape[3] - pool_size = initial_state_source.shape[0] - assert K_val == 128 and V_val == 128 - assert initial_state_source.dtype == torch.bfloat16 - assert tile_v in (32, 64, 128), f"tile_v must be 32/64/128, got {tile_v}" - assert V_val % tile_v == 0 and (tile_v // NUM_GROUPS) % ILP_ROWS == 0, ( - f"tile_v={tile_v} incompatible with 8 groups × ILP=4 layout" - ) - - if scale is None: - scale = 1.0 / math.sqrt(K_val) - - # Pass the 4D [pool_size, HV, V, K] tensor as-is. The kernel uses cute's - # 4D stride-aware indexing so non-contiguous per-slot strides (vLLM's - # padded pool layout) work without a silent .contiguous() clone. See - # tests/gdn/test_decode_pretranspose_bf16_padded_pool.py and PR #3268. - h0_source = initial_state_source - - cache_intermediate_states = intermediate_states_buffer is not None - if cache_intermediate_states: - # The cache buffer is BATCH-scoped: shape [B, T, HV, V, K]. The kernel - # indexes it by i_n (the per-call batch index), NOT by cache_idx (the - # pool slot), so a pool_size-sized buffer would be OOB-prone. Fix - # mirrors upstream PR #3145. - buffer_size = intermediate_states_buffer.shape[0] - cache_steps = intermediate_states_buffer.shape[1] - assert buffer_size == B_val, ( - f"intermediate_states_buffer dim 0 ({buffer_size}) must equal " - f"batch size B={B_val}; the buffer is batch-scoped, not pool-scoped" - ) - assert cache_steps >= T_val - assert intermediate_states_buffer.dtype == torch.bfloat16 - intermediate_states = intermediate_states_buffer.reshape( - B_val * cache_steps * HV_val, V_val, K_val - ) - if not intermediate_states.is_contiguous(): - intermediate_states = intermediate_states.contiguous() - # Skip the redundant final writeback when caching is on. - effective_disable_final = True - else: - intermediate_states = h0_source[:1, :1, :1] - effective_disable_final = disable_state_update - - stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) - use_packed_fma = _USE_PACKED_FMA - # Single-pool callers either pass output_state_indices=None (defaults to - # initial_state_indices below) or pass the same tensor for both. In both - # cases the kernel can elide write-side base-pointer arithmetic via the - # same_pool Constexpr; nvcc DCEs the dead branch in the compiled cubin. - same_pool = ( - output_state_indices is None or output_state_indices is initial_state_indices - ) - - # Contiguous pool -> sentinel keys + slot dim marked dynamic (pool-size - # agnostic); padded/strided pool keeps real pool_size/stride in the key. - contiguous_pool = initial_state_source.is_contiguous() - if contiguous_pool: - pool_size_key = -1 - pool_slot_stride: tuple[int, ...] = (-1,) - else: - pool_size_key = pool_size - pool_slot_stride = tuple(int(s) for s in initial_state_source.stride()) - cache_key = ( - "v3_mtp_bf16_tiled_dynB", - T_val, - H_val, - HV_val, - K_val, - V_val, - pool_size_key, - pool_slot_stride, - tile_v, - effective_disable_final, - cache_intermediate_states, - use_qk_l2norm_in_kernel, - scale, - softplus_beta, - softplus_threshold, - use_packed_fma, - same_pool, - ) - - if cache_key not in _compiled_kernels_wide_vec: - # Compile-time template tensors at B=1; cubin runs at caller's - # actual B via _mark_batch_dynamic. cache_key omits B by design - # (upstream batch-dynamic). - _placeholder_indices = torch.arange(1, dtype=torch.int32, device=q.device) - _placeholder_output = torch.empty( - 1, T_val, HV_val, V_val, device=q.device, dtype=q.dtype - ) - - if contiguous_pool: - h_ = _mark_slot_dynamic(h0_source) - else: - h_ = from_dlpack(h0_source, assumed_align=32, enable_tvm_ffi=True) - if cache_intermediate_states: - inter_ = _mark_slot_dynamic(intermediate_states) - else: - # Caching-off dummy ([1,1,1]) is never read; don't mark it. - inter_ = from_dlpack( - intermediate_states, assumed_align=32, enable_tvm_ffi=True - ) - q_ = _mark_batch_dynamic(q) - k_ = _mark_batch_dynamic(k) - v_ = _mark_batch_dynamic(v) - a_ = _mark_batch_dynamic(a) - b_ = _mark_batch_dynamic(b) - A_log_ = from_dlpack(A_log, assumed_align=32, enable_tvm_ffi=True) - dt_bias_ = from_dlpack(dt_bias, assumed_align=32, enable_tvm_ffi=True) - o_ = _mark_batch_dynamic(output if output is not None else _placeholder_output) - h0_idx_ = _mark_index_dynamic( - initial_state_indices - if initial_state_indices is not None - else _placeholder_indices - ) - h0_out_idx_ = h0_idx_ - - _compiled_kernels_wide_vec[cache_key] = { - "compiled": cute.compile( - _run_wide_vec_t1, - h_, - inter_, - A_log_, - a_, - dt_bias_, - q_, - k_, - v_, - b_, - o_, - h0_idx_, - h0_out_idx_, - softplus_beta, - softplus_threshold, - scale, - HV_val, - T_val, - H_val, - K_val, - V_val, - tile_v, - use_qk_l2norm_in_kernel, - effective_disable_final, - cache_intermediate_states, - use_packed_fma, - same_pool, - stream, - options="--enable-tvm-ffi --generate-line-info --opt-level 3", - ), - # Per-B default tensors (B-dependent shapes — see batch-dynamic - # correctness note in gated_delta_rule_mtp_wide_vec). - "defaults_by_B": {}, - } - - cache = _compiled_kernels_wide_vec[cache_key] - defaults_by_B = cache["defaults_by_B"] - if B_val not in defaults_by_B: - defaults_by_B[B_val] = { - "indices": torch.arange(B_val, dtype=torch.int32, device=q.device), - "output": torch.empty( - B_val, T_val, HV_val, V_val, device=q.device, dtype=q.dtype - ), - } - defs = defaults_by_B[B_val] - if initial_state_indices is None: - initial_state_indices = defs["indices"] - if output_state_indices is None: - # Single-pool: read==write. Reuse the same indices tensor — no extra - # allocation, kernel still produces the same address for both slots. - output_state_indices = initial_state_indices - if output is None: - output = defs["output"] - - cache["compiled"]( - h0_source, - intermediate_states, - A_log, - a, - dt_bias, - q, - k, - v, - b, - output, - initial_state_indices, - output_state_indices, - stream, - ) - return output - - def gated_delta_rule_mtp( A_log: torch.Tensor, a: torch.Tensor, @@ -3888,8 +2948,7 @@ def gated_delta_rule_mtp( # ∈ {32, 64, 128} so wide_vec covers ``B*HV >= 128`` at T>=2; below # that it returns None and we fall back to mtp_ilp4. Wide_vec # supports split-pool natively (PR #2905); ``output_state_indices`` - # is forwarded to the kernel. T=1 dispatches via gated_delta_rule - # (different gate, requires tile_v >= 64). + # is forwarded to the kernel. wv_tile_v = _select_wide_vec_tile_v(B, HV) if T >= 2 else None if recovery_steps > 0: assert wv_tile_v is not None, ( @@ -3943,8 +3002,8 @@ def gated_delta_rule_mtp( replayssm_beta=replayssm_beta, ) - # Wide_vec didn't fire (work_units < 128 at T>=2, or T=1 small batch - # redirected here). Falls to the ILP=4 MTP path + # Wide_vec didn't fire (work_units < 128 at T>=2, or T=1). Falls to + # the ILP=4 MTP path # (mtp_ilp4_kernel), which natively supports both single- and # split-pool, so the config picker is independent of pool mode. tile_v, ilp_rows = _get_bf16_mtp_config(B, T, HV, V) @@ -4158,8 +3217,3 @@ def gated_delta_rule_mtp( ) return output - - -# Backward-compatible aliases -gated_delta_rule_bf16state_cooprow = gated_delta_rule -gated_delta_rule_bf16state_cooprow_mtp = gated_delta_rule_mtp diff --git a/python/sglang/kernels/ops/attention/dcp_kernels.py b/python/sglang/kernels/ops/attention/dcp_kernels.py index 62f0a5b269..06869c43f8 100644 --- a/python/sglang/kernels/ops/attention/dcp_kernels.py +++ b/python/sglang/kernels/ops/attention/dcp_kernels.py @@ -671,46 +671,3 @@ def dcp_lse_combine_triton( RETURN_LSE=return_lse, ) return out, (out_lse if return_lse else None) - - -def _lse_weighted_combine_cpu( - partial_outputs: torch.Tensor, - partial_lses: torch.Tensor, - is_lse_base_on_e: bool = True, -) -> torch.Tensor: - """CPU reference: combine N partial attention outputs using LSE weights. - - Args: - partial_outputs: [N, B, H_local, D] - partial_lses: [N, B, H_local] - is_lse_base_on_e: base-e (True) or base-2 (False) - - Returns: - [B, H_local, D] combined output - """ - N, B, H_local, D = partial_outputs.shape - partial_outputs = partial_outputs.float() - partial_lses = partial_lses.float() - - # Sanitize - partial_lses = torch.where( - torch.isnan(partial_lses) | torch.isinf(partial_lses), - torch.full_like(partial_lses, float("-inf")), - partial_lses, - ) - - # max LSE for numerical stability - lse_max, _ = partial_lses.max(dim=0) - lse_max = torch.where(lse_max == float("-inf"), torch.zeros_like(lse_max), lse_max) - - centered = partial_lses - lse_max.unsqueeze(0) - if is_lse_base_on_e: - weights = torch.exp(centered) - else: - weights = torch.pow(2.0, centered) - - weight_sum = weights.sum(dim=0, keepdim=True) - weights = weights / weight_sum.clamp_min(1.0) - - combined = (partial_outputs * weights.unsqueeze(-1)).sum(dim=0) - return combined diff --git a/python/sglang/kernels/ops/attention/deepseek_v4_rope.py b/python/sglang/kernels/ops/attention/deepseek_v4_rope.py index da61ee69f2..93d9cb2623 100644 --- a/python/sglang/kernels/ops/attention/deepseek_v4_rope.py +++ b/python/sglang/kernels/ops/attention/deepseek_v4_rope.py @@ -519,92 +519,6 @@ def _fused_norm_rope_kernel( ) -@triton.jit -def _fused_softmax_pool_kernel( - kv_score_ptr, - out_ptr, - stride_bs: tl.constexpr, - stride_k: tl.constexpr, - K: tl.constexpr, - HEAD_DIM: tl.constexpr, - HEAD_BLOCK: tl.constexpr, -): - pid = tl.program_id(0) - base = pid * stride_bs - - offs = tl.arange(0, HEAD_BLOCK) - mask = offs < HEAD_DIM - - max_val = tl.full([HEAD_BLOCK], float("-inf"), dtype=tl.float32) - for k in range(K): - s = tl.load( - kv_score_ptr + base + k * stride_k + HEAD_DIM + offs, - mask=mask, - other=float("-inf"), - ).to(tl.float32) - max_val = tl.maximum(max_val, s) - - sum_exp = tl.zeros([HEAD_BLOCK], dtype=tl.float32) - weighted = tl.zeros([HEAD_BLOCK], dtype=tl.float32) - for k in range(K): - s = tl.load( - kv_score_ptr + base + k * stride_k + HEAD_DIM + offs, - mask=mask, - other=float("-inf"), - ).to(tl.float32) - v = tl.load( - kv_score_ptr + base + k * stride_k + offs, - mask=mask, - other=0.0, - ).to(tl.float32) - w = tl.exp(s - max_val) - sum_exp += w - weighted += v * w - - result = weighted / sum_exp - tl.store( - out_ptr + pid * HEAD_DIM + offs, result.to(out_ptr.dtype.element_ty), mask=mask - ) - - -def fused_softmax_pool_triton( - kv_score: torch.Tensor, - head_dim: int, -) -> torch.Tensor: - """Fused softmax-weighted-sum: out = (kv * softmax(score, dim=1)).sum(dim=1). - - Replaces the generic cunn_SpatialSoftMaxForward + elementwise multiply + sum - with a single Triton kernel. - - Args: - kv_score: [bs, K, 2 * head_dim] where first head_dim is kv, second is score. - head_dim: dimension of each of kv and score. - Returns: - output: [bs, head_dim] - """ - assert kv_score.dim() == 3 - bs, K, last = kv_score.shape - assert last == 2 * head_dim - assert kv_score.is_contiguous() - - out = torch.empty(bs, head_dim, dtype=kv_score.dtype, device=kv_score.device) - if bs == 0: - return out - - HEAD_BLOCK = triton.next_power_of_2(head_dim) - grid = (bs,) - _fused_softmax_pool_kernel[grid]( - kv_score, - out, - stride_bs=kv_score.stride(0), - stride_k=kv_score.stride(1), - K=K, - HEAD_DIM=head_dim, - HEAD_BLOCK=HEAD_BLOCK, - ) - return out - - def fused_norm_rope_inplace_triton( kv: torch.Tensor, weight: Optional[torch.Tensor], diff --git a/python/sglang/kernels/ops/attention/dsa/index_buf_accessor.py b/python/sglang/kernels/ops/attention/dsa/index_buf_accessor.py index ed40d0b85b..559119f8d3 100644 --- a/python/sglang/kernels/ops/attention/dsa/index_buf_accessor.py +++ b/python/sglang/kernels/ops/attention/dsa/index_buf_accessor.py @@ -9,12 +9,11 @@ from sglang.srt.layers.attention.dsa.utils import ( INDEXER_K_CACHE_PRESHUFFLE_TILE, aiter_can_use_preshuffle_paged_mqa, ) -from sglang.srt.utils import get_bool_env_var, is_hip, is_xpu +from sglang.srt.utils import is_hip, is_xpu _is_hip = is_hip() _is_xpu = is_xpu() _is_fp8_fnuz = is_fp8_fnuz() -_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip # aiter cp_gather kernel with preshuffle=True is only valid when the indexer # uses the page_size=64 preshuffle layout (i.e. when the matching MQA gluon path # is also enabled). @@ -32,149 +31,6 @@ s: scale, 1 item per token, fp32 """ -class GetK: - @classmethod - def execute(cls, *args, **kwargs): - return cls.triton(*args, **kwargs) - - @classmethod - def slow( - cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor - ): - page_size = pool.index_page_size - num_pages = (seq_len + page_size - 1) // page_size - seq_len_ = num_pages * page_size - index_k_fp8 = torch.empty( - (seq_len_, pool.index_head_dim), - dtype=torch.uint8, - device=pool.device, - ) - for i in range(num_pages): - page_index = page_indices[i] - index_k_fp8[i * page_size : (i + 1) * page_size] = buf[page_index][ - : page_size * pool.index_head_dim - ].view(-1, pool.index_head_dim) - - return index_k_fp8[:seq_len] - - @classmethod - def torch_fast( - cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor - ): - """ - :param page_indices: (num_pages,), int32 - :return: (seq_len, index_head_dim), uint8 - """ - - # can handle per 128B instead of per element - - # page_indices: (num_pages,), element := a page index - buf_numel_per_page = buf.shape[1] - - num_k_bytes_per_page = pool.index_page_size * pool.index_head_dim - num_k_bytes_per_token = pool.index_head_dim - - # buf: (num_pages, page_size 64 * head_dim 128 + page_size 64 * fp32_nbytes 4), uint8 - # flat_buf: (whatever,), uint8 - flat_buf = buf.flatten() - - # flat_indices: (num_pages, num_k_bytes_per_page), int32, element := an index into flat_buf that we want to access - flat_indices = (page_indices * buf_numel_per_page)[:, None] + torch.arange( - num_k_bytes_per_page, dtype=torch.int32, device="cuda" - )[None, :] - flat_indices = flat_indices.flatten()[: seq_len * num_k_bytes_per_token] - - out = flat_buf[flat_indices] - return out.view(-1, 128) - - @classmethod - def triton( - cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor - ): - """ - Triton implementation for gathering K data from paged buffer. - :param page_indices: (num_pages,), int32/int64 - :return: (seq_len, index_head_dim), uint8 - """ - return _get_k_triton( - buf=buf, - page_indices=page_indices, - seq_len=seq_len, - page_size=pool.index_page_size, - index_head_dim=pool.index_head_dim, - ) - - -class GetS: - @classmethod - def execute(cls, *args, **kwargs): - return cls.triton(*args, **kwargs) - - @classmethod - def slow( - cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor - ): - page_size = pool.index_page_size - num_pages = (seq_len + page_size - 1) // page_size - seq_len_ = num_pages * page_size - assert pool.index_head_dim // pool.quant_block_size == 1 - index_k_scale_fp8 = torch.empty( - (seq_len_, 4), - dtype=torch.uint8, - device=pool.device, - ) - for i in range(num_pages): - page_index = page_indices[i] - index_k_scale_fp8[i * page_size : (i + 1) * page_size] = buf[page_index][ - page_size * pool.index_head_dim : - ].view(-1, 4) - return index_k_scale_fp8[:seq_len] - - @classmethod - def torch_fast( - cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor - ): - """ - :param page_indices: (num_pages,), int32 - :return: (seq_len, index_head_dim // quant_block_size), uint8 - """ - buf_numel_per_page = buf.shape[1] - - s_offset_in_page = pool.index_page_size * pool.index_head_dim - num_s_bytes_per_page = buf.shape[1] - s_offset_in_page - num_s_bytes_per_token = pool.index_head_dim // pool.quant_block_size * 4 - - flat_buf = buf.flatten() - flat_indices = ( - (page_indices * buf_numel_per_page)[:, None] - + torch.arange(num_s_bytes_per_page, dtype=torch.int32, device="cuda")[ - None, : - ] - + s_offset_in_page - ) - flat_indices = flat_indices.flatten()[: seq_len * num_s_bytes_per_token] - - out = flat_buf[flat_indices] - return out.view(-1, 4) - - @classmethod - def triton( - cls, pool: "DSATokenToKVPool", buf, seq_len: int, page_indices: torch.Tensor - ): - """ - Triton implementation for gathering S (scale) data from paged buffer. - :param page_indices: (num_pages,), int32/int64 - :return: (seq_len, 4), uint8 - """ - return _get_s_triton( - buf=buf, - page_indices=page_indices, - seq_len=seq_len, - page_size=pool.index_page_size, - index_head_dim=pool.index_head_dim, - ) - - class GetKAndS: @classmethod def execute(cls, *args, **kwargs): @@ -415,161 +271,6 @@ def _set_k_and_s_triton_kernel( tl.store(buf_fp32_ptr + out_s_offset, k_scale) -def _get_k_triton( - buf: torch.Tensor, - page_indices: torch.Tensor, - seq_len: int, - page_size: int, - index_head_dim: int, -): - """ - Gather K (key) data from paged buffer using Triton. - - :param buf: (num_pages, page_size * 128 + page_size * 4), uint8 - :param page_indices: (num_pages,), int32/int64 - :param seq_len: int, number of tokens to gather - :param page_size: int, typically 64 - :param index_head_dim: int, typically 128 - :return: (seq_len, index_head_dim), uint8 - """ - num_pages, buf_numel_per_page = buf.shape - - # Allocate output - out = torch.empty((seq_len, index_head_dim), dtype=torch.uint8, device=buf.device) - - # Launch kernel with one thread per token - grid = (seq_len,) - _get_k_triton_kernel[grid]( - buf, - page_indices, - out, - seq_len, - page_size, - buf_numel_per_page, - index_head_dim, - BLOCK_SIZE=128, - ) - - return out - - -@triton.jit -def _get_k_triton_kernel( - buf_ptr, - page_indices_ptr, - out_ptr, - seq_len: tl.constexpr, - page_size: tl.constexpr, - buf_numel_per_page: tl.constexpr, - index_head_dim: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - """ - Each program handles one token (seq_len tokens total). - Loads 128 bytes from the appropriate page. - """ - token_id = tl.program_id(0) - - # Calculate which page and offset within page - page_idx = token_id // page_size - token_offset_in_page = token_id % page_size - - # Load the page index from page_indices - page_index = tl.load(page_indices_ptr + page_idx) - - # Calculate source offset in buf - # buf[page_index, token_offset_in_page * index_head_dim : ...] - src_base_offset = ( - page_index * buf_numel_per_page + token_offset_in_page * index_head_dim - ) - - # Load 128 bytes (index_head_dim elements) - offsets = tl.arange(0, BLOCK_SIZE) - mask = offsets < index_head_dim - data = tl.load(buf_ptr + src_base_offset + offsets, mask=mask) - - # Store to output - dst_offset = token_id * index_head_dim - tl.store(out_ptr + dst_offset + offsets, data, mask=mask) - - -def _get_s_triton( - buf: torch.Tensor, - page_indices: torch.Tensor, - seq_len: int, - page_size: int, - index_head_dim: int, -): - """ - Gather S (scale) data from paged buffer using Triton. - - :param buf: (num_pages, page_size * 128 + page_size * 4), uint8 - :param page_indices: (num_pages,), int32/int64 - :param seq_len: int, number of tokens to gather - :param page_size: int, typically 64 - :param index_head_dim: int, typically 128 - :return: (seq_len, 4), uint8 (representing fp32 scale) - """ - num_pages, buf_numel_per_page = buf.shape - s_offset_in_page = page_size * index_head_dim # Scales start after K data - - # Allocate output - out = torch.empty((seq_len, 4), dtype=torch.uint8, device=buf.device) - - # Launch kernel with one thread per token - grid = (seq_len,) - _get_s_triton_kernel[grid]( - buf, - page_indices, - out, - seq_len, - page_size, - buf_numel_per_page, - s_offset_in_page, - ) - - return out - - -@triton.jit -def _get_s_triton_kernel( - buf_ptr, - page_indices_ptr, - out_ptr, - seq_len: tl.constexpr, - page_size: tl.constexpr, - buf_numel_per_page: tl.constexpr, - s_offset_in_page: tl.constexpr, -): - """ - Each program handles one token (seq_len tokens total). - Loads 4 bytes (fp32 scale) from the appropriate page. - """ - token_id = tl.program_id(0) - - # Calculate which page and offset within page - page_idx = token_id // page_size - token_offset_in_page = token_id % page_size - - # Load the page index from page_indices - page_index = tl.load(page_indices_ptr + page_idx) - - # Calculate source offset in buf - # Scales are stored after K data: page_size * index_head_dim offset - # buf[page_index, s_offset_in_page + token_offset_in_page * 4 : ...] - src_base_offset = ( - page_index * buf_numel_per_page + s_offset_in_page + token_offset_in_page * 4 - ) - - # Load 4 bytes (fp32 scale) - offsets = tl.arange(0, 4) - data = tl.load(buf_ptr + src_base_offset + offsets) - - # Store to output - dst_offset = token_id * 4 - tl.store(out_ptr + dst_offset + offsets, data) - - def _get_k_and_s_triton( buf: torch.Tensor, page_indices: torch.Tensor, @@ -581,7 +282,6 @@ def _get_k_and_s_triton( ): """ Fused gather of both K (key) and S (scale) data from paged buffer using Triton. - This is more efficient than calling GetK and GetS separately. :param buf: (num_pages, page_size * 128 + page_size * 4), uint8 :param page_indices: (num_pages,), int32/int64 diff --git a/python/sglang/kernels/ops/attention/dsa/quant_k_cache.py b/python/sglang/kernels/ops/attention/dsa/quant_k_cache.py index 536aae4658..67c9a93bda 100644 --- a/python/sglang/kernels/ops/attention/dsa/quant_k_cache.py +++ b/python/sglang/kernels/ops/attention/dsa/quant_k_cache.py @@ -3,57 +3,6 @@ import triton import triton.language as tl -def gather_dsa_kv_scales( - scale_src, - scale_dst, - kv_indices, - kv_indptr, - kv_indptr_idx, -): - _gather_dsa_kv_scales[(32,)]( - scale_src, - scale_dst, - kv_indices, - kv_indptr, - scale_src.stride(0), - KV_INDPTR_IDX=kv_indptr_idx, - NUM_TILES=scale_src.shape[-1], - BLOCK=256, - ) - - -@triton.jit -def _gather_dsa_kv_scales( - scale_src, - scale_dst, - kv_indices, - kv_indptr, - scale_src_stride, - KV_INDPTR_IDX: tl.constexpr, - NUM_TILES: tl.constexpr, - BLOCK: tl.constexpr, -): - pid = tl.program_id(0) - num_programs = tl.num_programs(0) - active = tl.load(kv_indptr + KV_INDPTR_IDX) - block_start = pid * BLOCK - tiles = tl.arange(0, NUM_TILES) - while block_start < active: - offsets = block_start + tl.arange(0, BLOCK) - mask = offsets < active - rows = tl.load(kv_indices + offsets, mask=mask, other=0) - values = tl.load( - scale_src + rows[:, None] * scale_src_stride + tiles[None, :], - mask=mask[:, None], - ) - tl.store( - scale_dst + rows[:, None] * NUM_TILES + tiles[None, :], - values, - mask=mask[:, None], - ) - block_start += num_programs * BLOCK - - def quantize_k_cache(cache_k, dv: int | None = None): return _quantize_k_cache_fast_wrapped(cache_k, dv=dv) diff --git a/python/sglang/kernels/ops/attention/dsa/tilelang_kernel.py b/python/sglang/kernels/ops/attention/dsa/tilelang_kernel.py index b6a1d6a7e4..9c044d7c4c 100644 --- a/python/sglang/kernels/ops/attention/dsa/tilelang_kernel.py +++ b/python/sglang/kernels/ops/attention/dsa/tilelang_kernel.py @@ -169,89 +169,6 @@ def act_quant( return y, s -@tilelang.jit(out_idx=[4], pass_configs=pass_configs) -def fp8_index_kernel(h: int, d: int, clear_accum=True): - b = T.symbolic("b") - m = T.symbolic("m") - n = T.symbolic("n") - - blk_n1 = 512 - blk_n2 = 128 - - @T.prim_func - def fp8_index_kernel_( - q: T.Tensor[(b, m, h, d), FP8], - q_s: T.Tensor[(b, m, h), FP32], - k: T.Tensor[(b, n, d), FP8], - k_s: T.Tensor[(b, n), FP32], - o: T.Tensor[(b, m, n), FP32], - ) -> None: - with T.Kernel(b, m, T.ceildiv(n, blk_n1)) as (i_b, i_m, i1_n): - q_smem = T.alloc_shared((h, d), FP8) - T.copy(q[i_b, i_m, 0, 0], q_smem) - - q_s_frag = T.alloc_fragment(h, FP32) - T.copy(q_s[i_b, i_m, 0], q_s_frag) - - for i2_n in T.Pipelined(blk_n1 // blk_n2, num_stages=2): - k_smem = T.alloc_shared((blk_n2, d), FP8) - T.copy(k[i_b, i1_n * blk_n1 + i2_n * blk_n2, 0], k_smem) - - k_s_frag = T.alloc_fragment(blk_n2, FP32) - T.copy(k_s[i_b, i1_n * blk_n1 + i2_n * blk_n2], k_s_frag) - - logits = T.alloc_fragment((blk_n2, h), FP32) - if not clear_accum: - T.fill(logits, 0) - T.gemm( - k_smem, - q_smem, - logits, - transpose_A=False, - transpose_B=True, - clear_accum=clear_accum, - ) - - for i_h, i3_n in T.Parallel(h, blk_n2): - logits[i3_n, i_h] = T.max(logits[i3_n, i_h], 0) * q_s_frag[i_h] - - logits_sum = T.alloc_fragment(blk_n2, FP32) - T.reduce_sum(logits, logits_sum, dim=1) - - for i3_n in T.Parallel(blk_n2): - logits_sum[i3_n] *= k_s_frag[i3_n] - - T.copy(logits_sum, o[i_b, i_m, i1_n * blk_n1 + i2_n * blk_n2]) - - return fp8_index_kernel_ - - -def fp8_index( - q: torch.Tensor, - q_s: torch.Tensor, - k: torch.Tensor, - k_s: torch.Tensor, -) -> torch.Tensor: - """ - Perform index score using FP8 precision. - - Args: - q (torch.Tensor): The Q tensor, must be contiguous. - q_s (torch.Tensor): The scaling factor for Q (float), must be contiguous. - k (torch.Tensor): The K tensor, must be contiguous. - k_s (torch.Tensor): The scaling factor for K (e8m0 here), must be contiguous. - - fp8 q @ fp8 k -> fp32 logits - relu(fp32 logits) * q_s (weights) -> fp32 logits - fp32 logits -> fp32 logits_sum - fp32 logits_sum * k_s (e8m0) -> fp32 index_score - """ - if _is_hip: - return fp8_index_kernel(q.shape[2], q.shape[3], False)(q, q_s, k, k_s) - else: - return fp8_index_kernel(q.shape[2], q.shape[3])(q, q_s, k, k_s) - - @tilelang.jit( out_idx=[-1], pass_configs={ diff --git a/python/sglang/kernels/ops/attention/dsa/transform_index.py b/python/sglang/kernels/ops/attention/dsa/transform_index.py index 0f9746109c..7cfb4af3f6 100644 --- a/python/sglang/kernels/ops/attention/dsa/transform_index.py +++ b/python/sglang/kernels/ops/attention/dsa/transform_index.py @@ -371,56 +371,3 @@ def transform_index_page_table_prefill_fast( num_warps=4, ) return result - - -def transform_index_page_table_decode_ref( - page_table: torch.Tensor, - topk_indices: torch.Tensor, - result: Optional[torch.Tensor] = None, - page_size: int = 1, -) -> torch.Tensor: - assert page_size == 1 - assert page_table.shape[0] == topk_indices.shape[0] - if result is None: - result = torch.empty_like(topk_indices, dtype=torch.int32) - assert result.shape == topk_indices.shape - torch.gather( - page_table.to(result.dtype), - dim=1, - index=topk_indices.clamp(min=0), - out=result, - ) - result[topk_indices < 0] = -1 - return result - - -def transform_index_page_table_prefill_ref( - page_table: torch.Tensor, - topk_indices: torch.Tensor, - extend_lens_cpu: List[int], - page_size: int = 1, - output_num_tokens: Optional[int] = None, - page_table_is_expanded: bool = False, -) -> torch.Tensor: - assert page_size == 1 - real_num_tokens = sum(extend_lens_cpu) - result = _allocate_prefill_result(topk_indices, real_num_tokens, output_num_tokens) - - if page_table_is_expanded: - if real_num_tokens > 0: - transform_index_page_table_decode_ref( - page_table[:real_num_tokens], - topk_indices[:real_num_tokens], - result=result[:real_num_tokens], - ) - return result - - offset = 0 - for i, l in enumerate(extend_lens_cpu): - transform_index_page_table_decode_ref( - page_table[i].unsqueeze(0).expand(l, -1), - topk_indices[offset : offset + l], - result=result[offset : offset + l], - ) - offset += l - return result diff --git a/python/sglang/kernels/ops/attention/dsa/triton_sparse_mla_decode.py b/python/sglang/kernels/ops/attention/dsa/triton_sparse_mla_decode.py index c4b7a3d3d2..4062087281 100644 --- a/python/sglang/kernels/ops/attention/dsa/triton_sparse_mla_decode.py +++ b/python/sglang/kernels/ops/attention/dsa/triton_sparse_mla_decode.py @@ -1,21 +1,18 @@ -"""Triton sparse MLA decode kernel with FP8 and BF16 KV cache support. +"""Triton sparse MLA decode with FP8 and BF16 KV cache support. -Adapted from aiter's unified_attention_sparse_mla kernel for DSA shapes: +Decode-tuned launcher for the fused / split-K kernels of ``triton_sparse_mla`` +on DSA shapes: q: [bs, H, DIM] fp8/bf16 (DIM=576 = D_V+D_TAIL) kv: [num_pages, 1, DIM] fp8/bf16 indices: [bs, 1, topk] int32 output: [1, bs, H, D_V] bf16 - -Two variants: - 1. Base: single-pass per-token kernel (adapted from aiter) - 2. Split-K: adaptive split-K with fused fast path (adapted from DSv4) """ import torch -import triton -import triton.language as tl from sglang.kernels.ops.attention.dsa.triton_sparse_mla import ( + _FP8_MAX, + _LOG2E, _PREFERRED_BLOCK_K, _cu_count, _gfx950_sparse_mla_kv_splits, @@ -28,14 +25,11 @@ from sglang.kernels.ops.attention.dsa.triton_sparse_mla import ( _reduce_d_chunk, _row_strides, _sparse_mla_block_k, + _sparse_mla_fused_kernel, _sparse_mla_reduce_kernel, + _sparse_mla_split_k_kernel, _validate_input_dtypes, ) -from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz - -_IS_FNUZ = is_fp8_fnuz() -_FP8_MAX = 240.0 if _IS_FNUZ else 448.0 -_G = tl.constexpr(128) _SplitKWorkspace = list[tuple[torch.Tensor, torch.Tensor]] _splitk_bufs: dict[tuple[torch.device, int], _SplitKWorkspace] = {} @@ -82,420 +76,6 @@ def _get_splitk_bufs( return lse, acc -# --------------------------------------------------------------------------- -# Split-K kernel (adapted from DSv4 paged_decode.py) -# --------------------------------------------------------------------------- - -LOG2E = 1.4426950408889634 - - -@triton.jit -def _sparse_mla_decode_fused_kernel( - q_nope_ptr, # [N, H, D_V] - q_rope_ptr, # [N, H, D_TAIL] - kv_ptr, # [num_pages, 1, KV_DIM] - idx_ptr, # [N, topk] - out_ptr, # [N, H, D_V] - qk_scale, - fp8_max, - topk: tl.constexpr, - H: tl.constexpr, - KV_DIM: tl.constexpr, - D_V: tl.constexpr, - D_TAIL: tl.constexpr, - NUM_GROUPS: tl.constexpr, - STRIDE_QN_T: tl.constexpr, - STRIDE_QN_H: tl.constexpr, - STRIDE_QR_T: tl.constexpr, - STRIDE_QR_H: tl.constexpr, - USE_FP8_DOT: tl.constexpr, - BLOCK_H: tl.constexpr, - BLOCK_K: tl.constexpr, - USE_I64_PAGE: tl.constexpr = True, -): - t = tl.program_id(0) - pid_h = tl.program_id(1) - - h_offs = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) - h_mask = h_offs < H - dt = tl.arange(0, D_TAIL) - g = tl.arange(0, _G) - - input_type = kv_ptr.dtype.element_ty if USE_FP8_DOT else tl.bfloat16 - if USE_FP8_DOT: - p_dot_scale = 1.0 / fp8_max - else: - p_dot_scale = 1.0 - - qn_base = q_nope_ptr + t * STRIDE_QN_T - qn_row = qn_base + h_offs[:, None] * STRIDE_QN_H - q0 = tl.load( - qn_row + g[None, :], - mask=h_mask[:, None], - other=0.0, - ).to(input_type) - if NUM_GROUPS >= 2: - q1 = tl.load( - qn_row + (_G + g)[None, :], - mask=h_mask[:, None], - other=0.0, - ).to(input_type) - if NUM_GROUPS >= 3: - q2 = tl.load( - qn_row + (2 * _G + g)[None, :], - mask=h_mask[:, None], - other=0.0, - ).to(input_type) - if NUM_GROUPS >= 4: - q3 = tl.load( - qn_row + (3 * _G + g)[None, :], - mask=h_mask[:, None], - other=0.0, - ).to(input_type) - q_tail = tl.load( - q_rope_ptr + t * STRIDE_QR_T + h_offs[:, None] * STRIDE_QR_H + dt[None, :], - mask=h_mask[:, None], - other=0.0, - ).to(input_type) - - neg_large = -3.4028234663852886e38 - m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32) - l_i = tl.zeros((BLOCK_H,), dtype=tl.float32) - acc0 = tl.zeros((BLOCK_H, _G), dtype=tl.float32) - if NUM_GROUPS >= 2: - acc1 = tl.zeros((BLOCK_H, _G), dtype=tl.float32) - if NUM_GROUPS >= 3: - acc2 = tl.zeros((BLOCK_H, _G), dtype=tl.float32) - if NUM_GROUPS >= 4: - acc3 = tl.zeros((BLOCK_H, _G), dtype=tl.float32) - - k_offs = tl.arange(0, BLOCK_K) - num_tiles = tl.cdiv(topk, BLOCK_K) - - for j in tl.range(0, num_tiles, num_stages=3): - k_start = j * BLOCK_K - k_pos = k_start + k_offs - valid = k_pos < topk - - slot = tl.load(idx_ptr + t * topk + k_pos, mask=valid, other=0) - valid = valid & (slot >= 0) - page = tl.where(valid, slot, 0) - if USE_I64_PAGE: - page = page.to(tl.int64) - - kv_base = kv_ptr + page[:, None] * KV_DIM - kv0 = tl.load( - kv_base + g[None, :], - mask=valid[:, None], - other=0.0, - ).to(input_type) - if NUM_GROUPS >= 2: - kv1 = tl.load( - kv_base + (_G + g)[None, :], - mask=valid[:, None], - other=0.0, - ).to(input_type) - if NUM_GROUPS >= 3: - kv2 = tl.load( - kv_base + (2 * _G + g)[None, :], - mask=valid[:, None], - other=0.0, - ).to(input_type) - if NUM_GROUPS >= 4: - kv3 = tl.load( - kv_base + (3 * _G + g)[None, :], - mask=valid[:, None], - other=0.0, - ).to(input_type) - kv_tail = tl.load( - kv_base + (D_V + dt)[None, :], - mask=valid[:, None], - other=0.0, - ).to(input_type) - - scores = tl.dot(q0, tl.trans(kv0)) - if NUM_GROUPS >= 2: - scores += tl.dot(q1, tl.trans(kv1)) - if NUM_GROUPS >= 3: - scores += tl.dot(q2, tl.trans(kv2)) - if NUM_GROUPS >= 4: - scores += tl.dot(q3, tl.trans(kv3)) - scores += tl.dot(q_tail, tl.trans(kv_tail)) - scores = scores * qk_scale - scores = tl.where(valid[None, :], scores, neg_large) - - m_block = tl.max(scores, axis=1) - m_new = tl.maximum(m_i, m_block) - alpha = tl.exp2(m_i - m_new) - p = tl.exp2(scores - m_new[:, None]) - l_new = l_i * alpha + tl.sum(p, axis=1) - - if USE_FP8_DOT: - p_dot = (p * fp8_max).to(input_type) - else: - p_dot = p.to(input_type) - acc0 = acc0 * alpha[:, None] + tl.dot(p_dot, kv0).to(tl.float32) * p_dot_scale - if NUM_GROUPS >= 2: - acc1 = ( - acc1 * alpha[:, None] + tl.dot(p_dot, kv1).to(tl.float32) * p_dot_scale - ) - if NUM_GROUPS >= 3: - acc2 = ( - acc2 * alpha[:, None] + tl.dot(p_dot, kv2).to(tl.float32) * p_dot_scale - ) - if NUM_GROUPS >= 4: - acc3 = ( - acc3 * alpha[:, None] + tl.dot(p_dot, kv3).to(tl.float32) * p_dot_scale - ) - m_i = m_new - l_i = l_new - - denom = tl.maximum(l_i, 1.0e-30) - inv_denom = 1.0 / denom - acc0 = tl.where(l_i[:, None] > 0.0, acc0 * inv_denom[:, None], 0.0) - if NUM_GROUPS >= 2: - acc1 = tl.where(l_i[:, None] > 0.0, acc1 * inv_denom[:, None], 0.0) - if NUM_GROUPS >= 3: - acc2 = tl.where(l_i[:, None] > 0.0, acc2 * inv_denom[:, None], 0.0) - if NUM_GROUPS >= 4: - acc3 = tl.where(l_i[:, None] > 0.0, acc3 * inv_denom[:, None], 0.0) - - o_base = out_ptr + t * H * D_V - tl.store( - o_base + h_offs[:, None] * D_V + g[None, :], - acc0.to(tl.bfloat16), - mask=h_mask[:, None], - ) - if NUM_GROUPS >= 2: - tl.store( - o_base + h_offs[:, None] * D_V + (_G + g)[None, :], - acc1.to(tl.bfloat16), - mask=h_mask[:, None], - ) - if NUM_GROUPS >= 3: - tl.store( - o_base + h_offs[:, None] * D_V + (2 * _G + g)[None, :], - acc2.to(tl.bfloat16), - mask=h_mask[:, None], - ) - if NUM_GROUPS >= 4: - tl.store( - o_base + h_offs[:, None] * D_V + (3 * _G + g)[None, :], - acc3.to(tl.bfloat16), - mask=h_mask[:, None], - ) - - -@triton.jit -def _sparse_mla_decode_split_kernel( - q_nope_ptr, # [N, H, D_V] - q_rope_ptr, # [N, H, D_TAIL] - kv_ptr, # [num_pages, 1, KV_DIM] - idx_ptr, # [N, topk] - lse_partial_ptr, # [N, KV_SPLITS, H_padded] fp32 - acc_partial_ptr, # [N, KV_SPLITS, H_padded, D_V] bf16 - qk_scale, - fp8_max, - topk: tl.constexpr, - H: tl.constexpr, - KV_DIM: tl.constexpr, - D_V: tl.constexpr, - D_TAIL: tl.constexpr, - NUM_GROUPS: tl.constexpr, - STRIDE_QN_T: tl.constexpr, - STRIDE_QN_H: tl.constexpr, - STRIDE_QR_T: tl.constexpr, - STRIDE_QR_H: tl.constexpr, - USE_FP8_DOT: tl.constexpr, - KV_SPLITS: tl.constexpr, - BLOCK_H: tl.constexpr, - BLOCK_K: tl.constexpr, -): - t = tl.program_id(0) - pid_h = tl.program_id(1) - pid_k = tl.program_id(2) - - h_offs = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) - h_mask = h_offs < H - dt = tl.arange(0, D_TAIL) - g = tl.arange(0, _G) - - input_type = kv_ptr.dtype.element_ty if USE_FP8_DOT else tl.bfloat16 - if USE_FP8_DOT: - p_dot_scale = 1.0 / fp8_max - else: - p_dot_scale = 1.0 - - qn_base = q_nope_ptr + t * STRIDE_QN_T - qn_row = qn_base + h_offs[:, None] * STRIDE_QN_H - q0 = tl.load( - qn_row + g[None, :], - mask=h_mask[:, None], - other=0.0, - ).to(input_type) - if NUM_GROUPS >= 2: - q1 = tl.load( - qn_row + (_G + g)[None, :], - mask=h_mask[:, None], - other=0.0, - ).to(input_type) - if NUM_GROUPS >= 3: - q2 = tl.load( - qn_row + (2 * _G + g)[None, :], - mask=h_mask[:, None], - other=0.0, - ).to(input_type) - if NUM_GROUPS >= 4: - q3 = tl.load( - qn_row + (3 * _G + g)[None, :], - mask=h_mask[:, None], - other=0.0, - ).to(input_type) - q_tail = tl.load( - q_rope_ptr + t * STRIDE_QR_T + h_offs[:, None] * STRIDE_QR_H + dt[None, :], - mask=h_mask[:, None], - other=0.0, - ).to(input_type) - - tiles_per_segment = tl.cdiv(topk, KV_SPLITS * BLOCK_K) - if pid_k * tiles_per_segment * BLOCK_K >= topk: - return - num_tiles = tl.cdiv(topk, BLOCK_K) - tile_start = pid_k * tiles_per_segment - tile_end = tl.minimum((pid_k + 1) * tiles_per_segment, num_tiles) - - neg_large = -3.4028234663852886e38 - m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32) - l_i = tl.zeros((BLOCK_H,), dtype=tl.float32) - acc0 = tl.zeros((BLOCK_H, _G), dtype=tl.float32) - if NUM_GROUPS >= 2: - acc1 = tl.zeros((BLOCK_H, _G), dtype=tl.float32) - if NUM_GROUPS >= 3: - acc2 = tl.zeros((BLOCK_H, _G), dtype=tl.float32) - if NUM_GROUPS >= 4: - acc3 = tl.zeros((BLOCK_H, _G), dtype=tl.float32) - - k_offs = tl.arange(0, BLOCK_K) - for j in tl.range(tile_start, tile_end, num_stages=3): - k_start = j * BLOCK_K - k_pos = k_start + k_offs - valid = k_pos < topk - - slot = tl.load(idx_ptr + t * topk + k_pos, mask=valid, other=0) - valid = valid & (slot >= 0) - page = tl.where(valid, slot, 0).to(tl.int64) - - kv_base = kv_ptr + page[:, None] * KV_DIM - kv0 = tl.load( - kv_base + g[None, :], - mask=valid[:, None], - other=0.0, - ).to(input_type) - if NUM_GROUPS >= 2: - kv1 = tl.load( - kv_base + (_G + g)[None, :], - mask=valid[:, None], - other=0.0, - ).to(input_type) - if NUM_GROUPS >= 3: - kv2 = tl.load( - kv_base + (2 * _G + g)[None, :], - mask=valid[:, None], - other=0.0, - ).to(input_type) - if NUM_GROUPS >= 4: - kv3 = tl.load( - kv_base + (3 * _G + g)[None, :], - mask=valid[:, None], - other=0.0, - ).to(input_type) - kv_tail = tl.load( - kv_base + (D_V + dt)[None, :], - mask=valid[:, None], - other=0.0, - ).to(input_type) - - scores = tl.dot(q0, tl.trans(kv0)) - if NUM_GROUPS >= 2: - scores += tl.dot(q1, tl.trans(kv1)) - if NUM_GROUPS >= 3: - scores += tl.dot(q2, tl.trans(kv2)) - if NUM_GROUPS >= 4: - scores += tl.dot(q3, tl.trans(kv3)) - scores += tl.dot(q_tail, tl.trans(kv_tail)) - scores = scores * qk_scale - scores = tl.where(valid[None, :], scores, neg_large) - - m_block = tl.max(scores, axis=1) - m_new = tl.maximum(m_i, m_block) - alpha = tl.exp2(m_i - m_new) - p = tl.exp2(scores - m_new[:, None]) - l_new = l_i * alpha + tl.sum(p, axis=1) - - if USE_FP8_DOT: - p_dot = (p * fp8_max).to(input_type) - else: - p_dot = p.to(input_type) - acc0 = acc0 * alpha[:, None] + tl.dot(p_dot, kv0).to(tl.float32) * p_dot_scale - if NUM_GROUPS >= 2: - acc1 = ( - acc1 * alpha[:, None] + tl.dot(p_dot, kv1).to(tl.float32) * p_dot_scale - ) - if NUM_GROUPS >= 3: - acc2 = ( - acc2 * alpha[:, None] + tl.dot(p_dot, kv2).to(tl.float32) * p_dot_scale - ) - if NUM_GROUPS >= 4: - acc3 = ( - acc3 * alpha[:, None] + tl.dot(p_dot, kv3).to(tl.float32) * p_dot_scale - ) - m_i = m_new - l_i = l_new - - neg_large = -1073741824.0 - denom = tl.maximum(l_i, 1.0e-30) - inv_denom = 1.0 / denom - has_data = l_i > 0.0 - acc0 = tl.where(has_data[:, None], acc0 * inv_denom[:, None], 0.0) - if NUM_GROUPS >= 2: - acc1 = tl.where(has_data[:, None], acc1 * inv_denom[:, None], 0.0) - if NUM_GROUPS >= 3: - acc2 = tl.where(has_data[:, None], acc2 * inv_denom[:, None], 0.0) - if NUM_GROUPS >= 4: - acc3 = tl.where(has_data[:, None], acc3 * inv_denom[:, None], 0.0) - - lse = tl.where(has_data, tl.log2(l_i) + m_i, neg_large) - - H_padded = tl.cdiv(H, BLOCK_H) * BLOCK_H - lse_base = t * KV_SPLITS * H_padded + pid_k * H_padded - tl.store(lse_partial_ptr + lse_base + h_offs, lse, mask=h_mask) - - ap_base = t * KV_SPLITS * H_padded * D_V + pid_k * H_padded * D_V - tl.store( - acc_partial_ptr + ap_base + h_offs[:, None] * D_V + g[None, :], - acc0.to(tl.bfloat16), - mask=h_mask[:, None], - ) - if NUM_GROUPS >= 2: - tl.store( - acc_partial_ptr + ap_base + h_offs[:, None] * D_V + (_G + g)[None, :], - acc1.to(tl.bfloat16), - mask=h_mask[:, None], - ) - if NUM_GROUPS >= 3: - tl.store( - acc_partial_ptr + ap_base + h_offs[:, None] * D_V + (2 * _G + g)[None, :], - acc2.to(tl.bfloat16), - mask=h_mask[:, None], - ) - if NUM_GROUPS >= 4: - tl.store( - acc_partial_ptr + ap_base + h_offs[:, None] * D_V + (3 * _G + g)[None, :], - acc3.to(tl.bfloat16), - mask=h_mask[:, None], - ) - - def triton_sparse_mla_decode_splitk( q_nope: torch.Tensor, q_rope: torch.Tensor, @@ -581,7 +161,7 @@ def triton_sparse_mla_decode_splitk( else: kv_splits = min(kv_splits, max_kv_splits) - qk_scale = float(sm_scale) * LOG2E + qk_scale = float(sm_scale) * _LOG2E if kv_splits == 1: fused_num_warps = 4 @@ -589,11 +169,12 @@ def triton_sparse_mla_decode_splitk( fused_num_warps = _gfx950_sparse_mla_num_warps(base_ctas, 1, num_cu) out = torch.empty(bs, H, d_v, device=q_nope.device, dtype=torch.bfloat16) with _no_async_copy(): - _sparse_mla_decode_fused_kernel[(bs, n_head_blocks)]( + _sparse_mla_fused_kernel[(bs, n_head_blocks)]( q_nope, q_rope, kv, idx_flat, + idx_flat, out, qk_scale, _FP8_MAX, @@ -608,6 +189,7 @@ def triton_sparse_mla_decode_splitk( STRIDE_QR_T=stride_qr_t, STRIDE_QR_H=stride_qr_h, USE_FP8_DOT=use_fp8_dot, + USE_TOPK_LENGTH=False, BLOCK_H=BLOCK_H, BLOCK_K=BLOCK_K, USE_I64_PAGE=not ( @@ -637,7 +219,7 @@ def triton_sparse_mla_decode_splitk( grid_split = (bs, n_head_blocks, kv_splits) with _no_async_copy(): - _sparse_mla_decode_split_kernel[grid_split]( + _sparse_mla_split_k_kernel[grid_split]( q_nope, q_rope, kv, diff --git a/python/sglang/kernels/ops/attention/dsv4/__init__.py b/python/sglang/kernels/ops/attention/dsv4/__init__.py index a92f72bb1a..3b4e6647ab 100644 --- a/python/sglang/kernels/ops/attention/dsv4/__init__.py +++ b/python/sglang/kernels/ops/attention/dsv4/__init__.py @@ -13,9 +13,9 @@ from .compress import ( compress_forward, compress_norm_rope_store, ) -from .compress_old import fused_norm_rope_inplace from .elementwise import ( fused_k_norm_rope_flashmla, + fused_norm_rope_inplace, fused_q_indexer_rope_first_quant, fused_q_indexer_rope_hadamard_fp4_quant, fused_q_indexer_rope_hadamard_quant, diff --git a/python/sglang/kernels/ops/attention/dsv4/compress.py b/python/sglang/kernels/ops/attention/dsv4/compress.py index 4048d2fac4..bbe5e111f0 100644 --- a/python/sglang/kernels/ops/attention/dsv4/compress.py +++ b/python/sglang/kernels/ops/attention/dsv4/compress.py @@ -150,7 +150,7 @@ _PREFILL_PLAN_BYTES = 24 # ---------------------------------------------------------------------------- # Plan dataclasses. The element at index 1 is the consumer for -# `compress_fused_norm_rope_inplace` (which reads ragged_id / seq_len from a +# `compress_norm_rope_store` (which reads ragged_id / seq_len from a # 16-byte plan tensor --- both DecodePlan and CompressPlan satisfy that layout). # ---------------------------------------------------------------------------- diff --git a/python/sglang/kernels/ops/attention/dsv4/compress_old.py b/python/sglang/kernels/ops/attention/dsv4/compress_old.py deleted file mode 100644 index 7af950b520..0000000000 --- a/python/sglang/kernels/ops/attention/dsv4/compress_old.py +++ /dev/null @@ -1,308 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING, Literal, NamedTuple, Optional, Union - -import torch - -from sglang.kernels.jit.utils import ( - cache_once, - is_arch_support_pdl, - load_jit, - make_cpp_args, -) -from sglang.srt.environ import envs - -from .utils import make_name - -if TYPE_CHECKING: - from tvm_ffi.module import Module - - -@cache_once -def _jit_common_module() -> Module: - return load_jit( - make_name("common"), - cuda_files=["deepseek_v4/common.cuh"], - cuda_wrappers=[("plan_compress_prefill", "plan_compress_prefill")], - ) - - -@cache_once -def _jit_compress_128_online_plan_module() -> Module: - """Host-side plan generator for online compress 128 (no template args).""" - return load_jit( - make_name("compress_128_online_plan"), - cuda_files=["deepseek_v4/c128_online.cuh"], - cuda_wrappers=[ - ("plan_compress_online_prefill", "plan_compress_online_prefill"), - ], - ) - - -@cache_once -def _jit_compress_128_online_module(head_dim: int) -> Module: - """Online compress 128 kernel: ring_size=1, per-index (max, sum, kv) state.""" - args = make_cpp_args(head_dim, is_arch_support_pdl()) - kernel_class = f"FlashCompress128OnlineKernel<{args}>" - return load_jit( - make_name("compress_128_online"), - *args, - cuda_files=["deepseek_v4/c128_online.cuh"], - cuda_wrappers=[ - ("decode", f"{kernel_class}::run_decode"), - ("prefill", f"{kernel_class}::run_prefill"), - ], - extra_cuda_cflags=["-use_fast_math"], - ) - - -@cache_once -def norm_rope_module( - dtype: torch.dtype, - head_dim: int, - rope_dim: int, -) -> Module: - args = make_cpp_args(dtype, head_dim, rope_dim, is_arch_support_pdl()) - return load_jit( - make_name("fused_norm_rope"), - *args, - cuda_files=["deepseek_v4/fused_norm_rope.cuh"], - cuda_wrappers=[ - ("forward", f"FusedNormRopeKernel<{args}>::forward"), - ], - ) - - -@cache_once -def _jit_compress_module( - head_dim: int, - dtype_in: torch.dtype, - dtype_out: torch.dtype, - ratio: Literal[4, 128], -) -> Module: - args = make_cpp_args(head_dim, dtype_in, dtype_out, is_arch_support_pdl()) - kernel_class = f"FlashCompress{ratio}Kernel<{args}>" - return load_jit( - make_name(f"compress_{ratio}"), - *args, - cuda_files=[f"deepseek_v4/c{ratio}.cuh"], - cuda_wrappers=[ - ("decode", f"{kernel_class}::run_decode"), - ("prefill", f"{kernel_class}::run_prefill"), - ], - extra_cuda_cflags=["-use_fast_math"], - ) - - -class CompressorPrefillPlan(NamedTuple): - compress_ratio: int - compress_plan: torch.Tensor - write_plan: torch.Tensor - - def copy_(self, other: CompressorPrefillPlan) -> None: - assert self.compress_ratio == other.compress_ratio - self.compress_plan.copy_(other.compress_plan) - self.write_plan.copy_(other.write_plan) - - @staticmethod - def generate( - compress_ratio: Literal[4, 128], - num_q_tokens: int, - seq_lens: torch.Tensor, - extend_lens: torch.Tensor, - device: torch.device, - use_cuda_graph: bool = False, - ) -> CompressorPrefillPlan: - from sglang.srt.environ import envs - - # Online c128 keeps the same NamedTuple shape (compress_plan, write_plan) - # so call sites that splat `*plan[1:]` continue to work, but the C++ - # plan struct semantics differ (last-token coords + window_len). - if compress_ratio == 128 and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get(): - return CompressorPrefillPlan._generate_online( - num_q_tokens=num_q_tokens, - seq_lens=seq_lens, - extend_lens=extend_lens, - device=device, - use_cuda_graph=use_cuda_graph, - ) - assert seq_lens.device == extend_lens.device - seq_lens = seq_lens.to(torch.int64) - extend_lens = extend_lens.to(torch.int64) - plan_tensor = torch.empty( - (2, num_q_tokens, 16), - dtype=torch.uint8, - device=seq_lens.device, - pin_memory=seq_lens.is_cpu, - ) - module = _jit_common_module() - is_overlap = compress_ratio == 4 - plan_lens = module.plan_compress_prefill( - extend_lens, - seq_lens, - plan_tensor[0], - plan_tensor[1], - compress_ratio, - is_overlap, - use_cuda_graph, - ) - return CompressorPrefillPlan( - compress_ratio, - plan_tensor[0, : plan_lens[0]].to(device, non_blocking=True), - plan_tensor[1, : plan_lens[1]].to(device, non_blocking=True), - ) - - @staticmethod - def _generate_online( - num_q_tokens: int, - seq_lens: torch.Tensor, - extend_lens: torch.Tensor, - device: torch.device, - use_cuda_graph: bool, - ) -> CompressorPrefillPlan: - # Online plan host-side path: only CPU/cuda-host implemented today. - # Move inputs to CPU pinned memory then bounce the result to device. - seq_lens_cpu = seq_lens.detach().to(torch.int64).cpu() - extend_lens_cpu = extend_lens.detach().to(torch.int64).cpu() - plan_tensor = torch.empty( - (2, num_q_tokens, 16), - dtype=torch.uint8, - device="cpu", - pin_memory=True, - ) - module = _jit_compress_128_online_plan_module() - plan_lens = module.plan_compress_online_prefill( - extend_lens_cpu, - seq_lens_cpu, - plan_tensor[0], - plan_tensor[1], - use_cuda_graph, - ) - return CompressorPrefillPlan( - 128, - plan_tensor[0, : plan_lens[0]].to(device, non_blocking=True), - plan_tensor[1, : plan_lens[1]].to(device, non_blocking=True), - ) - - @property - def is_decode(self) -> bool: - return False - - -class CompressorDecodePlan(NamedTuple): - compress_ratio: int - seq_lens: torch.Tensor - - def copy_(self, other: CompressorDecodePlan) -> None: - assert self.compress_ratio == other.compress_ratio - self.seq_lens.copy_(other.seq_lens) - - @property - def is_decode(self) -> bool: - return True - - -def compress_plan( - compress_ratio: Literal[4, 128], - num_q_tokens: int, - seq_lens: torch.Tensor, - extend_lens: Optional[torch.Tensor], - device: torch.device, -) -> Union[CompressorDecodePlan, CompressorPrefillPlan]: - if extend_lens is not None: - return CompressorPrefillPlan.generate( - compress_ratio, - num_q_tokens, - seq_lens, - extend_lens, - device, - ) - else: - assert num_q_tokens == len(seq_lens) - seq_lens = seq_lens.to(device, non_blocking=True) - return CompressorDecodePlan(compress_ratio, seq_lens) - - -def compress_forward( - kv_score_buffer: torch.Tensor, - kv_score_input: torch.Tensor, - ape: torch.Tensor, - indices: torch.Tensor, - plan: Union[CompressorDecodePlan, CompressorPrefillPlan, None] = None, - extra_data: Optional[torch.Tensor] = None, - *, - head_dim: int, - compress_ratio: Literal[4, 128], - out: Optional[torch.Tensor] = None, - seq_lens: Optional[torch.Tensor] = None, - extend_lens: Optional[torch.Tensor] = None, -) -> torch.Tensor: - assert head_dim % 128 == 0 - num_q_tokens = kv_score_input.shape[0] - if out is None: - out = kv_score_input.new_empty((num_q_tokens, head_dim)) - if plan is None: - assert seq_lens is not None - plan = compress_plan( - compress_ratio, - num_q_tokens, - seq_lens, - extend_lens, - kv_score_input.device, - ) - assert plan.compress_ratio == compress_ratio, "Mismatched compress ratio in plan!" - # Online c128: separate JIT module, fp32 state, no compile-time dtypes. - if compress_ratio == 128 and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get(): - online_module = _jit_compress_128_online_module(head_dim=head_dim) - F = online_module.decode if plan.is_decode else online_module.prefill - F(kv_score_buffer, kv_score_input, out, ape, indices, *plan[1:], extra_data) - return out - module = _jit_compress_module( - head_dim, - kv_score_input.dtype, - out.dtype, - compress_ratio, - ) - F = module.decode if plan.is_decode else module.prefill - F(kv_score_buffer, kv_score_input, out, ape, indices, *plan[1:], extra_data) - return out - - -def compress_fused_norm_rope_inplace( - kv: torch.Tensor, - weight: torch.Tensor, - eps: float, - freq_cis: torch.Tensor, - plan: Union[CompressorDecodePlan, CompressorPrefillPlan], -) -> None: - freq_cis = torch.view_as_real(freq_cis).flatten(-2) - module = norm_rope_module(kv.dtype, kv.shape[-1], freq_cis.shape[-1]) - module.forward( - kv, - weight, - plan[1], - freq_cis, - int(plan.is_decode), - eps, - plan.compress_ratio, - ) - - -def fused_norm_rope_inplace( - kv: torch.Tensor, - weight: torch.Tensor, - eps: float, - freq_cis: torch.Tensor, - positions: torch.Tensor, -) -> None: - freq_cis = torch.view_as_real(freq_cis).flatten(-2) - module = norm_rope_module(kv.dtype, kv.shape[-1], freq_cis.shape[-1]) - module.forward( - kv, - weight, - positions, - freq_cis, - 2, - eps, - 0, - ) diff --git a/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py b/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py index 20ad157570..7cc9915a62 100644 --- a/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py +++ b/python/sglang/kernels/ops/attention/dsv4/dequant_k_cache.py @@ -509,86 +509,3 @@ def _gather_dequant_requant_fp8_paged_kernel( output_ptr + out_row_base + DIM_NOPE + rope_offs, rope_data.to(output_ptr.dtype.element_ty), ) - - -def dequantize_k_cache_paged_ref( - quant_k_cache: torch.Tensor, - page_table_1_flattened: torch.Tensor, - page_size: int, -) -> torch.Tensor: - """Pure-torch reference for :func:`dequantize_k_cache_paged`. - - Decodes the same v4 paged layout with vectorized torch indexing instead of - a Triton kernel. Used to validate the kernel (see the ``__main__`` block - below); not on any hot path. - """ - assert page_table_1_flattened.dtype in (torch.int32, torch.int64) - u8 = quant_k_cache.view(torch.uint8) - bytes_per_page = u8.shape[-1] - s_offset_bytes = page_size * NOPE_ROPE_BYTES - - flat_u8 = u8.reshape(-1) - flat_fp8 = u8.view(fp8_dtype).reshape(-1) - flat_bf16 = u8.view(torch.bfloat16).reshape(-1) - - loc = page_table_1_flattened.to(torch.int64) - page_idx = loc // page_size - in_page = loc % page_size - page_byte_base = page_idx * bytes_per_page - token_data_base = page_byte_base + in_page * NOPE_ROPE_BYTES - token_scale_base = ( - page_byte_base + s_offset_bytes + in_page * PADDED_SCALE_PER_TOKEN - ) - - device = quant_k_cache.device - nope_byte = ( - token_data_base[:, None] + torch.arange(DIM_NOPE, device=device)[None, :] - ) - nope_fp8 = flat_fp8[nope_byte].to(torch.float32) - scale_byte = ( - token_scale_base[:, None] - + torch.arange(NUM_SCALE_TILES, device=device)[None, :] - ) - scale_u8 = flat_u8[scale_byte].to(torch.int32) - scale_pow2 = torch.exp2((scale_u8 - 127).to(torch.float32)) - scale_pow2 = torch.where( - scale_pow2 < (2.0**-126), torch.zeros_like(scale_pow2), scale_pow2 - ) - scale_full = scale_pow2.repeat_interleave(TILE_SIZE, dim=1) - nope = nope_fp8 * scale_full - - rope_bf16_base = (token_data_base + DIM_NOPE) // 2 - rope_idx = rope_bf16_base[:, None] + torch.arange(DIM_ROPE, device=device)[None, :] - rope = flat_bf16[rope_idx] - - out = torch.empty( - (loc.shape[0], 1, DIM_NOPE + DIM_ROPE), - dtype=torch.bfloat16, - device=device, - ) - out[:, 0, :DIM_NOPE] = nope.to(torch.bfloat16) - out[:, 0, DIM_NOPE:] = rope - return out - - -def gather_dequant_requant_fp8_paged_ref( - quant_k_cache: torch.Tensor, - page_table_1_flattened: torch.Tensor, - page_size: int, - extra_rows: int = 0, -) -> torch.Tensor: - """Torch reference for :func:`gather_dequant_requant_fp8_paged`.""" - active = dequantize_k_cache_paged_ref( - quant_k_cache, - page_table_1_flattened, - page_size, - ).to(fp8_dtype) - if extra_rows == 0: - return active - out = torch.zeros( - (active.shape[0] + extra_rows, 1, DIM_NOPE + DIM_ROPE), - dtype=fp8_dtype, - device=active.device, - ) - out[: active.shape[0]] = active - return out diff --git a/python/sglang/kernels/ops/attention/dsv4/elementwise.py b/python/sglang/kernels/ops/attention/dsv4/elementwise.py index ceceb1860c..8dd7e526dc 100644 --- a/python/sglang/kernels/ops/attention/dsv4/elementwise.py +++ b/python/sglang/kernels/ops/attention/dsv4/elementwise.py @@ -32,6 +32,23 @@ def _jit_fused_rope_module(): ) +@cache_once +def _jit_fused_norm_rope_module( + dtype: torch.dtype, + head_dim: int, + rope_dim: int, +): + args = make_cpp_args(dtype, head_dim, rope_dim, is_arch_support_pdl()) + return load_jit( + make_name("fused_norm_rope"), + *args, + cuda_files=["deepseek_v4/fused_norm_rope.cuh"], + cuda_wrappers=[ + ("forward", f"FusedNormRopeKernel<{args}>::forward"), + ], + ) + + @cache_once def _jit_main_q_norm_rope_module( dtype: torch.dtype, @@ -168,6 +185,26 @@ def fused_rope_inplace( module.forward(q, k, freqs_real, positions, inverse) +def fused_norm_rope_inplace( + kv: torch.Tensor, + weight: torch.Tensor, + eps: float, + freq_cis: torch.Tensor, + positions: torch.Tensor, +) -> None: + freq_cis = torch.view_as_real(freq_cis).flatten(-2) + module = _jit_fused_norm_rope_module(kv.dtype, kv.shape[-1], freq_cis.shape[-1]) + module.forward( + kv, + weight, + positions, + freq_cis, + 2, + eps, + 0, + ) + + def fused_q_norm_rope( q_input: torch.Tensor, q_output: torch.Tensor, diff --git a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py index 8b957c65a5..ac834263bc 100644 --- a/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py +++ b/python/sglang/kernels/ops/attention/dsv4/fp4_indexer_hip.py @@ -4,7 +4,6 @@ from __future__ import annotations import functools import itertools -import os from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional, Tuple, Union import torch @@ -22,6 +21,7 @@ from sglang.kernels.ops.attention.dsv4.fp4_rope_fake_quant import ( rope_tail_fake_quant_fp4_row, ) from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k +from sglang.srt.environ import envs if TYPE_CHECKING: from sglang.kernels.ops.attention.dsv4.compress import ( @@ -62,9 +62,7 @@ C4_ROWGROUP_MIN_VERIFY_ROWS = 64 # (see `logits_rows_per_chunk`), so this caps the indexer's transient footprint # independently of context length and chunked-prefill size; smaller budgets only # buy more row chunks. 2 GiB covers 4096 rows over ~512K tokens of context. -_LOGITS_BUDGET_ELEMS = ( - int(os.environ.get("SGLANG_DSV4_FP4_LOGITS_BUDGET_MB", "2048")) * 2**20 // 4 -) +_LOGITS_BUDGET_ELEMS = envs.SGLANG_DSV4_FP4_LOGITS_BUDGET_MB.get() * 2**20 // 4 _LOGITS_POOL: dict = {} diff --git a/python/sglang/kernels/ops/attention/dsv4/fused_compress_triton.py b/python/sglang/kernels/ops/attention/dsv4/fused_compress_triton.py deleted file mode 100644 index d7de20eae2..0000000000 --- a/python/sglang/kernels/ops/attention/dsv4/fused_compress_triton.py +++ /dev/null @@ -1,1116 +0,0 @@ -"""HIP fused compressor kernels using the NV/main metadata contract. - -The public wrappers mirror ``compress_forward``: - - decode: indices, seq_lens, extra_data - prefill: indices, compress_plan, write_plan, extra_data - -Prefill plans are the upstream 16-byte ``PrefillPlan`` structs stored as -``uint8[:, 16]``. The wrappers reinterpret them as ``int32[:, 4]`` before -launching Triton kernels. -""" - -from __future__ import annotations - -from typing import Optional, Union - -import torch -import triton -import triton.language as tl - -from sglang.kernels.ops.attention.dsv4.compress_old import ( - CompressorDecodePlan, - CompressorPrefillPlan, -) - - -@triton.jit -def _fused_ape_pool_norm_rope_kernel( - kv_score_ptr, - kv_score_stride_b, - kv_score_stride_k, - ape_ptr, - ape_stride_r, - rms_weight_ptr, - rms_eps, - freqs_ptr, - freqs_stride_b, - out_ptr, - out_stride_b, - head_dim, - rope_head_dim, - half_dim, - RATIO: tl.constexpr, - K_POOL: tl.constexpr, - BLOCK_D: tl.constexpr, - HALF_ROPE: tl.constexpr, - OVERLAP: tl.constexpr, -): - bid = tl.program_id(0) - d = tl.arange(0, BLOCK_D) - d_mask = d < head_dim - - m_prev = tl.full([BLOCK_D], float("-inf"), tl.float32) - kv_acc = tl.zeros([BLOCK_D], tl.float32) - w_acc = tl.zeros([BLOCK_D], tl.float32) - - batch_base = bid * kv_score_stride_b - - for k in tl.range(0, K_POOL): - if OVERLAP: - is_b = k >= RATIO - col_off = tl.where(is_b, head_dim, 0) - else: - col_off = 0 - - row_off = batch_base + k * kv_score_stride_k - kv_val = tl.load( - kv_score_ptr + row_off + col_off + d, mask=d_mask, other=0.0 - ).to(tl.float32) - sc_val = tl.load( - kv_score_ptr + row_off + half_dim + col_off + d, mask=d_mask, other=0.0 - ).to(tl.float32) - - ape_val = tl.load( - ape_ptr + (k % RATIO) * ape_stride_r + col_off + d, mask=d_mask, other=0.0 - ).to(tl.float32) - score_k = sc_val + ape_val - - m_new = tl.maximum(m_prev, score_k) - exp_old = tl.where(m_prev == float("-inf"), 0.0, tl.exp(m_prev - m_new)) - exp_cur = tl.where(score_k == float("-inf"), 0.0, tl.exp(score_k - m_new)) - kv_acc = kv_acc * exp_old + exp_cur * kv_val - w_acc = w_acc * exp_old + exp_cur - m_prev = m_new - - compressed = kv_acc / w_acc - rms_w = tl.load(rms_weight_ptr + d, mask=d_mask, other=0.0) - c_sq = tl.where(d_mask, compressed * compressed, 0.0) - var = tl.sum(c_sq, axis=0) / head_dim - normed = compressed * tl.rsqrt(var + rms_eps) * rms_w - - out_base = out_ptr + bid * out_stride_b - tl.store(out_base + d, normed.to(out_ptr.dtype.element_ty), mask=d_mask) - - rope_start = head_dim - rope_head_dim - p = tl.arange(0, HALF_ROPE) - pmask = p < (rope_head_dim // 2) - xr = tl.load(out_base + rope_start + 2 * p, mask=pmask, other=0.0).to(tl.float32) - xi = tl.load(out_base + rope_start + 2 * p + 1, mask=pmask, other=0.0).to( - tl.float32 - ) - - freq_base = bid * freqs_stride_b - fr = tl.load(freqs_ptr + freq_base + 2 * p, mask=pmask, other=1.0).to(tl.float32) - fi = tl.load(freqs_ptr + freq_base + 2 * p + 1, mask=pmask, other=0.0).to( - tl.float32 - ) - - tl.store( - out_base + rope_start + 2 * p, - (xr * fr - xi * fi).to(out_ptr.dtype.element_ty), - mask=pmask, - ) - tl.store( - out_base + rope_start + 2 * p + 1, - (xr * fi + xi * fr).to(out_ptr.dtype.element_ty), - mask=pmask, - ) - - -def fused_ape_pool_norm_rope( - kv_score_gathered: torch.Tensor, - ape: torch.Tensor, - rms_weight: torch.Tensor, - rms_eps: float, - freqs_cis_real: torch.Tensor, - head_dim: int, - rope_head_dim: int, - ratio: int, - overlap: bool, -) -> torch.Tensor: - """Fused APE-add + overlap-transform + softmax-pool + RMSNorm + RoPE.""" - coff = 2 if overlap else 1 - bs = kv_score_gathered.shape[0] - k_in = kv_score_gathered.shape[1] - last_dim = kv_score_gathered.shape[2] - half_dim = last_dim // 2 - assert k_in == ratio * coff, f"k_in={k_in} != ratio*coff={ratio}*{coff}" - - out = torch.empty( - bs, head_dim, dtype=torch.float32, device=kv_score_gathered.device - ) - if bs == 0: - return out - - block_d = triton.next_power_of_2(head_dim) - half_rope = triton.next_power_of_2(rope_head_dim // 2) - num_warps = 4 if head_dim <= 256 else 8 - - _fused_ape_pool_norm_rope_kernel[(bs,)]( - kv_score_gathered, - kv_score_gathered.stride(0), - kv_score_gathered.stride(1), - ape, - ape.stride(0), - rms_weight, - rms_eps, - freqs_cis_real, - freqs_cis_real.stride(0), - out, - out.stride(0), - head_dim, - rope_head_dim, - half_dim, - RATIO=ratio, - K_POOL=k_in, - BLOCK_D=block_d, - HALF_ROPE=half_rope, - OVERLAP=int(overlap), - num_warps=num_warps, - ) - return out - - -@triton.jit -def _c4_decode_kernel( - kv_in_ptr, - out_ptr, - buffer_ptr, - ape_ptr, - indices_ptr, - seq_lens_ptr, - extra_ptr, - kv_in_row_stride, - out_row_stride, - buffer_page_stride, - buffer_slot_stride, - ape_row_stride, - HEAD_DIM: tl.constexpr, - BLOCK_D: tl.constexpr, -): - bid = tl.program_id(0) - pid_d = tl.program_id(1) - d_offs = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) - d_mask = d_offs < HEAD_DIM - - index = tl.load(indices_ptr + bid).to(tl.int64) - index_prev = tl.load(extra_ptr + bid).to(tl.int64) - seq_len = tl.load(seq_lens_ptr + bid).to(tl.int32) - write_slot = (seq_len + 3) % 4 - - in_base = bid.to(tl.int64) * kv_in_row_stride - page_base = ( - index * buffer_page_stride + write_slot.to(tl.int64) * buffer_slot_stride - ) - - valid_index = index >= 0 - for ch in tl.static_range(4): - ch_off = ch * HEAD_DIM - val = tl.load(kv_in_ptr + in_base + ch_off + d_offs, mask=d_mask, other=0.0).to( - tl.float32 - ) - tl.store( - buffer_ptr + page_base + ch_off + d_offs, - val, - mask=d_mask & valid_index, - ) - - NEG_BIG: tl.constexpr = -1.0e9 - running_max = tl.full((BLOCK_D,), NEG_BIG, tl.float32) - running_sum = tl.zeros((BLOCK_D,), tl.float32) - weighted = tl.zeros((BLOCK_D,), tl.float32) - - for slot in tl.static_range(8): - if slot < 4: - page = index_prev - kv_off = 0 - score_off = 2 * HEAD_DIM - else: - page = index - kv_off = HEAD_DIM - score_off = 3 * HEAD_DIM - - src_pos = seq_len - 8 + slot - is_input = slot == 7 - write_pos = ((seq_len - 1) // 4) * 4 - page = tl.where(src_pos < write_pos, index_prev, index) - slot_in_page = src_pos % 4 - slot_base = ( - page * buffer_page_stride + slot_in_page.to(tl.int64) * buffer_slot_stride - ) - valid = src_pos >= 0 - if slot == 7: - kv = tl.load( - kv_in_ptr + in_base + kv_off + d_offs, - mask=d_mask & valid, - other=0.0, - ).to(tl.float32) - score = tl.load( - kv_in_ptr + in_base + score_off + d_offs, - mask=d_mask & valid, - other=NEG_BIG, - ).to(tl.float32) - else: - kv = tl.load( - buffer_ptr + slot_base + kv_off + d_offs, - mask=d_mask & valid, - other=0.0, - ) - score = tl.load( - buffer_ptr + slot_base + score_off + d_offs, - mask=d_mask & valid, - other=NEG_BIG, - ) - bias = tl.load(ape_ptr + slot * ape_row_stride + d_offs, mask=d_mask, other=0.0) - s = score + bias - new_max = tl.maximum(running_max, s) - factor = tl.exp(running_max - new_max) - e = tl.where(valid, tl.exp(s - new_max), 0.0) - running_sum = running_sum * factor + e - weighted = weighted * factor + kv * e - running_max = new_max - - tl.store( - out_ptr + bid.to(tl.int64) * out_row_stride + d_offs, - (weighted / running_sum).to(out_ptr.dtype.element_ty), - mask=d_mask, - ) - - -@triton.jit -def _c4_prefill_compress_kernel( - kv_in_ptr, - out_ptr, - buffer_ptr, - ape_ptr, - indices_ptr, - extra_ptr, - plan_ptr, - kv_in_row_stride, - out_row_stride, - buffer_page_stride, - buffer_slot_stride, - ape_row_stride, - plan_row_stride, - HEAD_DIM: tl.constexpr, - BLOCK_D: tl.constexpr, -): - pid_p = tl.program_id(0) - pid_d = tl.program_id(1) - d_offs = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) - d_mask = d_offs < HEAD_DIM - - plan_base = plan_ptr + pid_p * plan_row_stride - ragged_id = tl.load(plan_base + 0).to(tl.int32) - batch_id = tl.load(plan_base + 1).to(tl.int32) - position = tl.load(plan_base + 2).to(tl.int32) - window_len = tl.load(plan_base + 3).to(tl.int32) - if ragged_id < 0: - return - - extra_base = extra_ptr + batch_id.to(tl.int64) * 4 - load_first_page = tl.load(extra_base + 0).to(tl.int64) - load_second_page = tl.load(extra_base + 1).to(tl.int64) - - NEG_BIG: tl.constexpr = -1.0e9 - running_max = tl.full((BLOCK_D,), NEG_BIG, tl.float32) - running_sum = tl.zeros((BLOCK_D,), tl.float32) - weighted = tl.zeros((BLOCK_D,), tl.float32) - - for slot in tl.static_range(8): - in_state = slot < window_len - if slot < 4: - page = tl.where(window_len <= 4, load_second_page, load_first_page) - kv_off = 0 - score_off = 2 * HEAD_DIM - slot_in_page = slot - else: - page = load_second_page - kv_off = HEAD_DIM - score_off = 3 * HEAD_DIM - slot_in_page = slot - 4 - - src_pos = position - 7 + slot - state_valid = in_state & (src_pos >= 0) - slot_base = page * buffer_page_stride + slot_in_page * buffer_slot_stride - in_row = ragged_id - (7 - slot) - in_row_safe = tl.where(in_state, 0, in_row) - in_base = in_row_safe.to(tl.int64) * kv_in_row_stride - - kv_state = tl.load( - buffer_ptr + slot_base + kv_off + d_offs, - mask=d_mask & state_valid, - other=0.0, - ) - score_state = tl.load( - buffer_ptr + slot_base + score_off + d_offs, - mask=d_mask & state_valid, - other=NEG_BIG, - ) - kv_input = tl.load( - kv_in_ptr + in_base + kv_off + d_offs, - mask=d_mask & (~in_state), - other=0.0, - ).to(tl.float32) - score_input = tl.load( - kv_in_ptr + in_base + score_off + d_offs, - mask=d_mask & (~in_state), - other=NEG_BIG, - ).to(tl.float32) - kv = tl.where(in_state, kv_state, kv_input) - score = tl.where(in_state, score_state, score_input) - bias = tl.load(ape_ptr + slot * ape_row_stride + d_offs, mask=d_mask, other=0.0) - - s = score + bias - new_max = tl.maximum(running_max, s) - factor = tl.exp(running_max - new_max) - e = tl.exp(s - new_max) - running_sum = running_sum * factor + e - weighted = weighted * factor + kv * e - running_max = new_max - - tl.store( - out_ptr + ragged_id.to(tl.int64) * out_row_stride + d_offs, - (weighted / running_sum).to(out_ptr.dtype.element_ty), - mask=d_mask, - ) - - -@triton.jit -def _c4_prefill_write_kernel( - kv_in_ptr, - buffer_ptr, - indices_ptr, - extra_ptr, - plan_ptr, - kv_in_row_stride, - buffer_page_stride, - buffer_slot_stride, - plan_row_stride, - HEAD_DIM: tl.constexpr, - BLOCK_D: tl.constexpr, -): - pid_p = tl.program_id(0) - pid_d = tl.program_id(1) - d_offs = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) - d_mask = d_offs < HEAD_DIM - - plan_base = plan_ptr + pid_p * plan_row_stride - ragged_id = tl.load(plan_base + 0).to(tl.int32) - batch_id = tl.load(plan_base + 1).to(tl.int32) - position = tl.load(plan_base + 2).to(tl.int32) - if ragged_id < 0: - return - - extra_base = extra_ptr + batch_id.to(tl.int64) * 4 - write_first_page = tl.load(extra_base + 2).to(tl.int64) - last_position = tl.load(extra_base + 3).to(tl.int32) - write_second_page = tl.load(indices_ptr + batch_id).to(tl.int64) - page = tl.where(position < last_position, write_first_page, write_second_page) - slot = position % 4 - - in_base = ragged_id.to(tl.int64) * kv_in_row_stride - dst_base = page * buffer_page_stride + slot.to(tl.int64) * buffer_slot_stride - for ch in tl.static_range(4): - ch_off = ch * HEAD_DIM - val = tl.load(kv_in_ptr + in_base + ch_off + d_offs, mask=d_mask, other=0.0).to( - tl.float32 - ) - tl.store(buffer_ptr + dst_base + ch_off + d_offs, val, mask=d_mask) - - -@triton.jit -def _c128_decode_kernel( - kv_in_ptr, - out_ptr, - buffer_ptr, - ape_ptr, - indices_ptr, - seq_lens_ptr, - extra_ptr, - kv_in_row_stride, - out_row_stride, - buffer_page_stride, - buffer_slot_stride, - ape_row_stride, - HEAD_DIM: tl.constexpr, - BLOCK_D: tl.constexpr, - BLOCK_S: tl.constexpr, -): - bid = tl.program_id(0) - pid_d = tl.program_id(1) - d_offs = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) - d_mask = d_offs < HEAD_DIM - - index = tl.load(indices_ptr + bid).to(tl.int64) - index_prev = tl.load(extra_ptr + bid).to(tl.int64) - seq_len = tl.load(seq_lens_ptr + bid).to(tl.int32) - write_slot = (seq_len + 127) % 128 - in_base = bid.to(tl.int64) * kv_in_row_stride - dst_base = index * buffer_page_stride + write_slot.to(tl.int64) * buffer_slot_stride - - for ch in tl.static_range(2): - ch_off = ch * HEAD_DIM - val = tl.load(kv_in_ptr + in_base + ch_off + d_offs, mask=d_mask, other=0.0).to( - tl.float32 - ) - tl.store(buffer_ptr + dst_base + ch_off + d_offs, val, mask=d_mask) - - NEG_BIG: tl.constexpr = -1.0e9 - running_max = tl.full((BLOCK_D,), NEG_BIG, tl.float32) - running_sum = tl.zeros((BLOCK_D,), tl.float32) - weighted = tl.zeros((BLOCK_D,), tl.float32) - - for chunk_start in tl.static_range(0, 128, BLOCK_S): - slot_offs = chunk_start + tl.arange(0, BLOCK_S) - src_pos = seq_len - 128 + slot_offs - valid = src_pos >= 0 - is_input = slot_offs == 127 - write_pos = ((seq_len - 1) // 128) * 128 - pages = tl.where(src_pos < write_pos, index_prev, index) - slot_in_page = src_pos % 128 - slot_bases = ( - pages * buffer_page_stride + slot_in_page.to(tl.int64) * buffer_slot_stride - ) - kv_tile = tl.load( - buffer_ptr + slot_bases[:, None] + d_offs[None, :], - mask=valid[:, None] & (~is_input)[:, None] & d_mask[None, :], - other=0.0, - ) - score_tile = tl.load( - buffer_ptr + slot_bases[:, None] + HEAD_DIM + d_offs[None, :], - mask=valid[:, None] & (~is_input)[:, None] & d_mask[None, :], - other=NEG_BIG, - ) - kv_input_tile = tl.load( - kv_in_ptr + in_base + d_offs[None, :], - mask=valid[:, None] & is_input[:, None] & d_mask[None, :], - other=0.0, - ).to(tl.float32) - score_input_tile = tl.load( - kv_in_ptr + in_base + HEAD_DIM + d_offs[None, :], - mask=valid[:, None] & is_input[:, None] & d_mask[None, :], - other=NEG_BIG, - ).to(tl.float32) - kv_tile = tl.where(is_input[:, None], kv_input_tile, kv_tile) - score_tile = tl.where(is_input[:, None], score_input_tile, score_tile) - bias_tile = tl.load( - ape_ptr + slot_offs[:, None] * ape_row_stride + d_offs[None, :], - mask=d_mask[None, :], - other=0.0, - ) - s = score_tile + bias_tile - local_max = tl.max(s, axis=0) - new_max = tl.maximum(running_max, local_max) - exp_s = tl.exp(s - new_max[None, :]) - exp_s = tl.where(valid[:, None], exp_s, 0.0) - factor = tl.exp(running_max - new_max) - running_sum = running_sum * factor + tl.sum(exp_s, axis=0) - weighted = weighted * factor + tl.sum(kv_tile * exp_s, axis=0) - running_max = new_max - - tl.store( - out_ptr + bid.to(tl.int64) * out_row_stride + d_offs, - (weighted / running_sum).to(out_ptr.dtype.element_ty), - mask=d_mask, - ) - - -@triton.jit -def _c128_prefill_compress_kernel( - kv_in_ptr, - out_ptr, - buffer_ptr, - ape_ptr, - indices_ptr, - plan_ptr, - kv_in_row_stride, - out_row_stride, - buffer_page_stride, - buffer_slot_stride, - ape_row_stride, - plan_row_stride, - HEAD_DIM: tl.constexpr, - BLOCK_D: tl.constexpr, - BLOCK_S: tl.constexpr, -): - pid_p = tl.program_id(0) - pid_d = tl.program_id(1) - d_offs = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) - d_mask = d_offs < HEAD_DIM - - plan_base = plan_ptr + pid_p * plan_row_stride - ragged_id = tl.load(plan_base + 0).to(tl.int32) - batch_id = tl.load(plan_base + 1).to(tl.int32) - position = tl.load(plan_base + 2).to(tl.int32) - window_len = tl.load(plan_base + 3).to(tl.int32) - if ragged_id < 0: - return - - index = tl.load(indices_ptr + batch_id).to(tl.int64) - NEG_BIG: tl.constexpr = -1.0e9 - running_max = tl.full((BLOCK_D,), NEG_BIG, tl.float32) - running_sum = tl.zeros((BLOCK_D,), tl.float32) - weighted = tl.zeros((BLOCK_D,), tl.float32) - - for chunk_start in tl.static_range(0, 128, BLOCK_S): - slot_offs = chunk_start + tl.arange(0, BLOCK_S) - is_state = slot_offs < window_len - src_pos = position - 127 + slot_offs - state_valid = is_state & (src_pos >= 0) - slot_bases = ( - index * buffer_page_stride + slot_offs.to(tl.int64) * buffer_slot_stride - ) - in_rows = ragged_id - (127 - slot_offs) - in_rows_safe = tl.where(is_state, tl.zeros_like(in_rows), in_rows) - in_bases = in_rows_safe.to(tl.int64) * kv_in_row_stride - - kv_state = tl.load( - buffer_ptr + slot_bases[:, None] + d_offs[None, :], - mask=state_valid[:, None] & d_mask[None, :], - other=0.0, - ) - score_state = tl.load( - buffer_ptr + slot_bases[:, None] + HEAD_DIM + d_offs[None, :], - mask=state_valid[:, None] & d_mask[None, :], - other=NEG_BIG, - ) - kv_input = tl.load( - kv_in_ptr + in_bases[:, None] + d_offs[None, :], - mask=(~is_state)[:, None] & d_mask[None, :], - other=0.0, - ).to(tl.float32) - score_input = tl.load( - kv_in_ptr + in_bases[:, None] + HEAD_DIM + d_offs[None, :], - mask=(~is_state)[:, None] & d_mask[None, :], - other=NEG_BIG, - ).to(tl.float32) - kv_tile = tl.where(is_state[:, None], kv_state, kv_input) - score_tile = tl.where(is_state[:, None], score_state, score_input) - bias_tile = tl.load( - ape_ptr + slot_offs[:, None] * ape_row_stride + d_offs[None, :], - mask=d_mask[None, :], - other=0.0, - ) - - s = score_tile + bias_tile - local_max = tl.max(s, axis=0) - new_max = tl.maximum(running_max, local_max) - exp_s = tl.exp(s - new_max[None, :]) - # Keep input-path entries valid; only state-path entries need src_pos guard. - valid = state_valid | (~is_state) - exp_s = tl.where(valid[:, None], exp_s, 0.0) - factor = tl.exp(running_max - new_max) - running_sum = running_sum * factor + tl.sum(exp_s, axis=0) - weighted = weighted * factor + tl.sum(kv_tile * exp_s, axis=0) - running_max = new_max - - tl.store( - out_ptr + ragged_id.to(tl.int64) * out_row_stride + d_offs, - (weighted / running_sum).to(out_ptr.dtype.element_ty), - mask=d_mask, - ) - - -@triton.jit -def _c128_prefill_write_kernel( - kv_in_ptr, - buffer_ptr, - indices_ptr, - plan_ptr, - kv_in_row_stride, - buffer_page_stride, - buffer_slot_stride, - plan_row_stride, - HEAD_DIM: tl.constexpr, - BLOCK_D: tl.constexpr, -): - pid_p = tl.program_id(0) - pid_d = tl.program_id(1) - d_offs = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) - d_mask = d_offs < HEAD_DIM - - plan_base = plan_ptr + pid_p * plan_row_stride - ragged_id = tl.load(plan_base + 0).to(tl.int32) - batch_id = tl.load(plan_base + 1).to(tl.int32) - position = tl.load(plan_base + 2).to(tl.int32) - if ragged_id < 0: - return - - index = tl.load(indices_ptr + batch_id).to(tl.int64) - slot = position % 128 - in_base = ragged_id.to(tl.int64) * kv_in_row_stride - dst_base = index * buffer_page_stride + slot.to(tl.int64) * buffer_slot_stride - - for ch in tl.static_range(2): - ch_off = ch * HEAD_DIM - val = tl.load(kv_in_ptr + in_base + ch_off + d_offs, mask=d_mask, other=0.0).to( - tl.float32 - ) - tl.store(buffer_ptr + dst_base + ch_off + d_offs, val, mask=d_mask) - - -@triton.jit -def _compress_norm_rope_kernel( - kv_ptr, - weight_ptr, - freqs_ptr, - handle_ptr, - eps, - kv_row_stride, - freqs_row_stride, - plan_row_stride, - HEAD_DIM: tl.constexpr, - ROPE_DIM: tl.constexpr, - HEAD_BLOCK: tl.constexpr, - ROPE_PAIR_BLOCK: tl.constexpr, - COMPRESS_RATIO: tl.constexpr, - IS_DECODE: tl.constexpr, -): - work_id = tl.program_id(0) - - if IS_DECODE: - row = work_id - seq_len = tl.load(handle_ptr + work_id).to(tl.int32) - position = ((seq_len - 1) // COMPRESS_RATIO) * COMPRESS_RATIO - else: - plan_base = handle_ptr + work_id * plan_row_stride - row = tl.load(plan_base + 0).to(tl.int32) - plan_position = tl.load(plan_base + 2).to(tl.int32) - if row < 0: - return - position = plan_position + 1 - COMPRESS_RATIO - - base = row.to(tl.int64) * kv_row_stride - offs = tl.arange(0, HEAD_BLOCK) - mask = offs < HEAD_DIM - x = tl.load(kv_ptr + base + offs, mask=mask, other=0.0).to(tl.float32) - w = tl.load(weight_ptr + offs, mask=mask, other=0.0).to(tl.float32) - rms_inv = tl.rsqrt(tl.sum(x * x, axis=0) / HEAD_DIM + eps) - x_normed = x * rms_inv * w - - rope_start: tl.constexpr = HEAD_DIM - ROPE_DIM - pair_offs = tl.arange(0, ROPE_PAIR_BLOCK) - pair_mask = pair_offs < (ROPE_DIM // 2) - x_real = tl.load( - kv_ptr + base + rope_start + 2 * pair_offs, - mask=pair_mask, - other=0.0, - ).to(tl.float32) - x_imag = tl.load( - kv_ptr + base + rope_start + 2 * pair_offs + 1, - mask=pair_mask, - other=0.0, - ).to(tl.float32) - w_real = tl.load( - weight_ptr + rope_start + 2 * pair_offs, - mask=pair_mask, - other=1.0, - ).to(tl.float32) - w_imag = tl.load( - weight_ptr + rope_start + 2 * pair_offs + 1, - mask=pair_mask, - other=1.0, - ).to(tl.float32) - x_real = x_real * rms_inv * w_real - x_imag = x_imag * rms_inv * w_imag - - freq_base = position.to(tl.int64) * freqs_row_stride - f_real = tl.load(freqs_ptr + freq_base + 2 * pair_offs, mask=pair_mask, other=0.0) - f_imag = tl.load( - freqs_ptr + freq_base + 2 * pair_offs + 1, - mask=pair_mask, - other=0.0, - ) - out_real = x_real * f_real - x_imag * f_imag - out_imag = x_real * f_imag + x_imag * f_real - - tl.store(kv_ptr + base + offs, x_normed, mask=mask & (offs < rope_start)) - tl.store(kv_ptr + base + rope_start + 2 * pair_offs, out_real, mask=pair_mask) - tl.store(kv_ptr + base + rope_start + 2 * pair_offs + 1, out_imag, mask=pair_mask) - - -@triton.jit -def _compress_norm_rope_hadamard_kernel( - kv_ptr, - weight_ptr, - freqs_ptr, - handle_ptr, - eps, - hadamard_scale, - kv_row_stride, - freqs_row_stride, - plan_row_stride, - HEAD_DIM: tl.constexpr, - ROPE_DIM: tl.constexpr, - HEAD_BLOCK: tl.constexpr, - ROPE_PAIR_BLOCK: tl.constexpr, - COMPRESS_RATIO: tl.constexpr, - IS_DECODE: tl.constexpr, - LOG2_HEAD_DIM: tl.constexpr, -): - work_id = tl.program_id(0) - - if IS_DECODE: - row = work_id - seq_len = tl.load(handle_ptr + work_id).to(tl.int32) - position = ((seq_len - 1) // COMPRESS_RATIO) * COMPRESS_RATIO - else: - plan_base = handle_ptr + work_id * plan_row_stride - row = tl.load(plan_base + 0).to(tl.int32) - plan_position = tl.load(plan_base + 2).to(tl.int32) - if row < 0: - return - position = plan_position + 1 - COMPRESS_RATIO - - base = row.to(tl.int64) * kv_row_stride - offs = tl.arange(0, HEAD_BLOCK) - mask = offs < HEAD_DIM - x = tl.load(kv_ptr + base + offs, mask=mask, other=0.0).to(tl.float32) - w = tl.load(weight_ptr + offs, mask=mask, other=0.0).to(tl.float32) - rms_inv = tl.rsqrt(tl.sum(x * x, axis=0) / HEAD_DIM + eps) - x_normed = x * rms_inv * w - - rope_start: tl.constexpr = HEAD_DIM - ROPE_DIM - pair_offs = tl.arange(0, ROPE_PAIR_BLOCK) - pair_mask = pair_offs < (ROPE_DIM // 2) - x_real = tl.load( - kv_ptr + base + rope_start + 2 * pair_offs, - mask=pair_mask, - other=0.0, - ).to(tl.float32) - x_imag = tl.load( - kv_ptr + base + rope_start + 2 * pair_offs + 1, - mask=pair_mask, - other=0.0, - ).to(tl.float32) - w_real = tl.load( - weight_ptr + rope_start + 2 * pair_offs, - mask=pair_mask, - other=1.0, - ).to(tl.float32) - w_imag = tl.load( - weight_ptr + rope_start + 2 * pair_offs + 1, - mask=pair_mask, - other=1.0, - ).to(tl.float32) - x_real = x_real * rms_inv * w_real - x_imag = x_imag * rms_inv * w_imag - - freq_base = position.to(tl.int64) * freqs_row_stride - f_real = tl.load(freqs_ptr + freq_base + 2 * pair_offs, mask=pair_mask, other=0.0) - f_imag = tl.load( - freqs_ptr + freq_base + 2 * pair_offs + 1, - mask=pair_mask, - other=0.0, - ) - out_real = x_real * f_real - x_imag * f_imag - out_imag = x_real * f_imag + x_imag * f_real - - # Store norm+rope result to kv_ptr (will be used for butterfly stages) - tl.store(kv_ptr + base + offs, x_normed, mask=mask & (offs < rope_start)) - tl.store(kv_ptr + base + rope_start + 2 * pair_offs, out_real, mask=pair_mask) - tl.store(kv_ptr + base + rope_start + 2 * pair_offs + 1, out_imag, mask=pair_mask) - - # Walsh-Hadamard butterfly transform via store-reload through L1 cache. - # Barriers are required because multiple warps share the same row in memory; - # without them a fast warp can overwrite a partner value before a slow warp reads it. - for stage in tl.static_range(LOG2_HEAD_DIM): - stride = 1 << stage - is_even = ((offs >> stage) & 1) == 0 - partner = tl.where(is_even, offs + stride, offs - stride) - tl.debug_barrier() - x_self = tl.load(kv_ptr + base + offs, mask=mask) - x_partner = tl.load(kv_ptr + base + partner, mask=mask) - result = tl.where(is_even, x_self + x_partner, x_partner - x_self) - if stage == LOG2_HEAD_DIM - 1: - result = result * hadamard_scale - tl.debug_barrier() - tl.store(kv_ptr + base + offs, result, mask=mask) - - -def _plan_as_i32(plan: torch.Tensor) -> torch.Tensor: - assert plan.dtype == torch.uint8 and plan.dim() == 2 and plan.shape[1] == 16 - return plan.view(torch.int32).view(-1, 4) - - -def _block_d(head_dim: int) -> int: - return min(32, triton.next_power_of_2(head_dim)) - - -def _check_common( - kv_score_buffer: torch.Tensor, - kv_score_input: torch.Tensor, - out: torch.Tensor, - ape: torch.Tensor, - indices: torch.Tensor, - head_dim: int, - compress_ratio: int, -) -> None: - coff = 2 if compress_ratio == 4 else 1 - assert kv_score_input.is_cuda and kv_score_buffer.is_cuda - assert kv_score_input.dim() == 2 and kv_score_input.dtype in ( - torch.float32, - torch.bfloat16, - ) - assert kv_score_input.shape[1] == 2 * coff * head_dim - assert kv_score_buffer.dim() == 3 and kv_score_buffer.dtype == torch.float32 - assert kv_score_buffer.shape[1:] == (compress_ratio, 2 * coff * head_dim) - assert out.shape == (kv_score_input.shape[0], head_dim) - assert out.is_cuda and out.dtype in (torch.float32, torch.bfloat16) - assert ape.shape == (compress_ratio * coff, head_dim) - assert ape.dtype == torch.float32 and ape.is_cuda - assert indices.dtype == torch.int32 and indices.is_cuda - - -def _is_decode_plan(plan: Union[CompressorDecodePlan, CompressorPrefillPlan]) -> bool: - return isinstance(plan, CompressorDecodePlan) - - -def hip_compress_forward( - *, - kv_score_buffer: torch.Tensor, - kv_score_input: torch.Tensor, - ape: torch.Tensor, - indices: torch.Tensor, - plan: Union[CompressorDecodePlan, CompressorPrefillPlan], - extra_data: Optional[torch.Tensor], - head_dim: int, - compress_ratio: int, - out: Optional[torch.Tensor] = None, -) -> torch.Tensor: - if compress_ratio not in (4, 128): - raise ValueError(f"unsupported {compress_ratio=}") - if out is None: - out = kv_score_input.new_empty((kv_score_input.shape[0], head_dim)) - is_decode = _is_decode_plan(plan) - if not is_decode: - out.fill_(10000.0) - - _check_common( - kv_score_buffer, - kv_score_input, - out, - ape, - indices, - head_dim, - compress_ratio, - ) - - BLOCK_D = _block_d(head_dim) - num_d_chunks = triton.cdiv(head_dim, BLOCK_D) - - if is_decode: - seq_lens = plan.seq_lens - assert seq_lens.dtype == torch.int32 and seq_lens.is_cuda - assert seq_lens.shape == indices.shape - grid = (seq_lens.numel(), num_d_chunks) - if compress_ratio == 4: - assert extra_data is not None - assert extra_data.shape == (seq_lens.numel(), 1) - _c4_decode_kernel[grid]( - kv_score_input, - out, - kv_score_buffer, - ape, - indices, - seq_lens, - extra_data, - kv_score_input.stride(0), - out.stride(0), - kv_score_buffer.stride(0), - kv_score_buffer.stride(1), - ape.stride(0), - HEAD_DIM=head_dim, - BLOCK_D=BLOCK_D, - ) - else: - assert extra_data is not None - assert extra_data.shape == seq_lens.shape - _c128_decode_kernel[grid]( - kv_score_input, - out, - kv_score_buffer, - ape, - indices, - seq_lens, - extra_data, - kv_score_input.stride(0), - out.stride(0), - kv_score_buffer.stride(0), - kv_score_buffer.stride(1), - ape.stride(0), - HEAD_DIM=head_dim, - BLOCK_D=BLOCK_D, - BLOCK_S=64, - ) - return out - - compress_plan = _plan_as_i32(plan.compress_plan) - write_plan = _plan_as_i32(plan.write_plan) - if compress_ratio == 4: - assert extra_data is not None - assert extra_data.dim() == 2 and extra_data.shape[1] == 4 - compress_grid = (compress_plan.shape[0], num_d_chunks) - write_grid = (write_plan.shape[0], num_d_chunks) - _c4_prefill_compress_kernel[compress_grid]( - kv_score_input, - out, - kv_score_buffer, - ape, - indices, - extra_data, - compress_plan, - kv_score_input.stride(0), - out.stride(0), - kv_score_buffer.stride(0), - kv_score_buffer.stride(1), - ape.stride(0), - compress_plan.stride(0), - HEAD_DIM=head_dim, - BLOCK_D=BLOCK_D, - ) - _c4_prefill_write_kernel[write_grid]( - kv_score_input, - kv_score_buffer, - indices, - extra_data, - write_plan, - kv_score_input.stride(0), - kv_score_buffer.stride(0), - kv_score_buffer.stride(1), - write_plan.stride(0), - HEAD_DIM=head_dim, - BLOCK_D=BLOCK_D, - ) - else: - load_indices = indices if extra_data is None else extra_data - assert load_indices.dim() == 1 and load_indices.dtype == torch.int32 - compress_grid = (compress_plan.shape[0], num_d_chunks) - write_grid = (write_plan.shape[0], num_d_chunks) - _c128_prefill_compress_kernel[compress_grid]( - kv_score_input, - out, - kv_score_buffer, - ape, - load_indices, - compress_plan, - kv_score_input.stride(0), - out.stride(0), - kv_score_buffer.stride(0), - kv_score_buffer.stride(1), - ape.stride(0), - compress_plan.stride(0), - HEAD_DIM=head_dim, - BLOCK_D=BLOCK_D, - BLOCK_S=64, - ) - _c128_prefill_write_kernel[write_grid]( - kv_score_input, - kv_score_buffer, - indices, - write_plan, - kv_score_input.stride(0), - kv_score_buffer.stride(0), - kv_score_buffer.stride(1), - write_plan.stride(0), - HEAD_DIM=head_dim, - BLOCK_D=BLOCK_D, - ) - return out - - -def hip_compress_fused_norm_rope_inplace( - kv: torch.Tensor, - weight: torch.Tensor, - eps: float, - freqs_cis: torch.Tensor, - plan: Union[CompressorDecodePlan, CompressorPrefillPlan], -) -> None: - assert kv.dim() == 2 and kv.stride(-1) == 1 - assert weight.shape == (kv.shape[1],) - freqs_real = torch.view_as_real(freqs_cis).flatten(-2) - head_dim = kv.shape[1] - rope_dim = freqs_real.shape[-1] - assert head_dim >= rope_dim and rope_dim % 2 == 0 - - is_decode = _is_decode_plan(plan) - if is_decode: - handle = plan.seq_lens - else: - handle = _plan_as_i32(plan.compress_plan) - - if handle.numel() == 0: - return - - HEAD_BLOCK = triton.next_power_of_2(head_dim) - ROPE_PAIR_BLOCK = max(triton.next_power_of_2(rope_dim // 2), 1) - _compress_norm_rope_kernel[(handle.shape[0],)]( - kv, - weight, - freqs_real, - handle, - eps, - kv.stride(0), - freqs_real.stride(0), - handle.stride(0) if not is_decode else 0, - HEAD_DIM=head_dim, - ROPE_DIM=rope_dim, - HEAD_BLOCK=HEAD_BLOCK, - ROPE_PAIR_BLOCK=ROPE_PAIR_BLOCK, - COMPRESS_RATIO=plan.compress_ratio, - IS_DECODE=is_decode, - ) - - -def hip_compress_fused_norm_rope_hadamard_inplace( - kv: torch.Tensor, - weight: torch.Tensor, - eps: float, - freqs_cis: torch.Tensor, - plan: Union[CompressorDecodePlan, CompressorPrefillPlan], - head_dim: int, -) -> None: - assert kv.dim() == 2 and kv.stride(-1) == 1 - assert weight.shape == (kv.shape[1],) - assert kv.shape[1] == head_dim - freqs_real = torch.view_as_real(freqs_cis).flatten(-2) - rope_dim = freqs_real.shape[-1] - assert head_dim >= rope_dim and rope_dim % 2 == 0 - assert (head_dim & (head_dim - 1)) == 0, "head_dim must be power of 2" - - is_decode = _is_decode_plan(plan) - if is_decode: - handle = plan.seq_lens - else: - handle = _plan_as_i32(plan.compress_plan) - - if handle.numel() == 0: - return - - import math - - log2_head_dim = int(math.log2(head_dim)) - hadamard_scale = head_dim**-0.5 - - HEAD_BLOCK = triton.next_power_of_2(head_dim) - ROPE_PAIR_BLOCK = max(triton.next_power_of_2(rope_dim // 2), 1) - _compress_norm_rope_hadamard_kernel[(handle.shape[0],)]( - kv, - weight, - freqs_real, - handle, - eps, - hadamard_scale, - kv.stride(0), - freqs_real.stride(0), - handle.stride(0) if not is_decode else 0, - HEAD_DIM=head_dim, - ROPE_DIM=rope_dim, - HEAD_BLOCK=HEAD_BLOCK, - ROPE_PAIR_BLOCK=ROPE_PAIR_BLOCK, - COMPRESS_RATIO=plan.compress_ratio, - IS_DECODE=is_decode, - LOG2_HEAD_DIM=log2_head_dim, - ) diff --git a/python/sglang/kernels/ops/attention/dsv4/index_buf_accessor.py b/python/sglang/kernels/ops/attention/dsv4/index_buf_accessor.py index 50738d8f8d..14c8b49c58 100644 --- a/python/sglang/kernels/ops/attention/dsv4/index_buf_accessor.py +++ b/python/sglang/kernels/ops/attention/dsv4/index_buf_accessor.py @@ -36,10 +36,6 @@ class SetKAndS: def execute(cls, pool, buf, loc, nope_fp8_rope_bf16_pack: NopeFp8RopeBf16Pack): cls.triton(pool, buf, loc, nope_fp8_rope_bf16_pack) - @classmethod - def torch(cls, pool, buf, loc, nope_fp8_rope_bf16_pack: NopeFp8RopeBf16Pack): - _set_k_and_s_torch(buf, loc, nope_fp8_rope_bf16_pack, pool.page_size) - @classmethod def triton(cls, pool, buf, loc, nope_fp8_rope_bf16_pack: NopeFp8RopeBf16Pack): _set_k_and_s_triton(buf, loc, nope_fp8_rope_bf16_pack, pool.page_size) @@ -183,77 +179,3 @@ def _set_k_and_s_triton_kernel( tl.store(buf_fp8_ptr + out_k_nope_offsets, k_nope, mask=nope_mask) tl.store(buf_bf16_ptr + out_k_rope_offsets, k_rope) tl.store(buf_uint8_ptr + out_s_offsets, k_scale, mask=scale_mask) - - -def _set_k_and_s_torch( - buf: torch.Tensor, - loc: torch.Tensor, - nope_fp8_rope_bf16_pack: NopeFp8RopeBf16Pack, - page_size: int, -): - num_pages, buf_numel_per_page = buf.shape - (num_tokens_to_write,) = loc.shape - - k_nope, k_rope, scale_k_nope = ( - nope_fp8_rope_bf16_pack.k_nope_fp8, - nope_fp8_rope_bf16_pack.k_rope_bf16, - nope_fp8_rope_bf16_pack.scale_k_nope_ue8m0, - ) - - num_tokens_to_write_nope, nope_dim = k_nope.shape - num_tokens_to_write_rope, rope_dim = k_rope.shape - num_tokens_to_write_scale, scale_dim = scale_k_nope.shape - - assert ( - num_tokens_to_write - == num_tokens_to_write_nope - == num_tokens_to_write_rope - == num_tokens_to_write_scale - ), ( - f"{num_tokens_to_write=} {num_tokens_to_write_nope=} {num_tokens_to_write_rope=} {num_tokens_to_write_scale=}" - ) - - assert buf.dtype == torch.uint8 - assert loc.dtype in [ - torch.int64, - torch.int32, - ], f"{loc.dtype=}" - - assert k_nope.dtype == fp8_dtype - assert k_rope.dtype == torch.bfloat16 - assert scale_k_nope.dtype == torch.uint8 - - assert buf.is_contiguous() - assert loc.is_contiguous() - assert k_nope.is_contiguous() - assert k_rope.is_contiguous() - assert scale_k_nope.is_contiguous() - - buf_fp8 = buf.view(fp8_dtype).flatten() - buf_bf16 = buf.view(torch.bfloat16).flatten() - buf_scale = buf.view(torch.uint8).flatten() - - loc_page_index = loc // page_size - loc_token_offset_in_page = loc % page_size - - s_offset_nbytes_in_page = page_size * (nope_dim + rope_dim * 2) - - nope_offset = loc_page_index * buf_numel_per_page + loc_token_offset_in_page * ( - nope_dim + rope_dim * 2 - ) - - rope_offset = ( - loc_page_index * buf_numel_per_page // 2 - + (loc_token_offset_in_page * (nope_dim + rope_dim * 2) + nope_dim) // 2 - ) - - s_offset = ( - loc_page_index * buf_numel_per_page - + s_offset_nbytes_in_page - + loc_token_offset_in_page * (scale_dim + 1) - ) - - for i in range(num_tokens_to_write): - buf_fp8[nope_offset[i] : nope_offset[i] + nope_dim] = k_nope[i] - buf_bf16[rope_offset[i] : rope_offset[i] + rope_dim] = k_rope[i] - buf_scale[s_offset[i] : s_offset[i] + scale_dim] = scale_k_nope[i] diff --git a/python/sglang/kernels/ops/attention/dsv4/quant_k_cache.py b/python/sglang/kernels/ops/attention/dsv4/quant_k_cache.py deleted file mode 100644 index 6cca6be047..0000000000 --- a/python/sglang/kernels/ops/attention/dsv4/quant_k_cache.py +++ /dev/null @@ -1,120 +0,0 @@ -import torch -import triton -import triton.language as tl - -from sglang.kernels.ops.attention.dsv4.index_buf_accessor import NopeFp8RopeBf16Pack -from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz - -fp8_dtype = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn - - -@triton.jit -def _quant_k_cache_fused_kernel( - k_bf16_ptr, - k_nope_fp8_ptr, - k_rope_bf16_ptr, - scale_k_nope_uint8_ptr, - k_bf16_stride_0, - k_nope_fp8_stride_0, - k_rope_bf16_stride_0, - scale_stride_0, - DIM_NOPE: tl.constexpr, - DIM_ROPE: tl.constexpr, - TILE_SIZE: tl.constexpr, - NUM_TILES: tl.constexpr, - FP8_MIN: tl.constexpr, - FP8_MAX: tl.constexpr, - EPS: tl.constexpr, -): - token_id = tl.program_id(0).to(tl.int64) - tile_id = tl.program_id(1) - - if tile_id == NUM_TILES: - rope_range = tl.arange(0, TILE_SIZE) - rope_mask = rope_range < DIM_ROPE - - in_rope_offsets = token_id * k_bf16_stride_0 + DIM_NOPE + rope_range - rope_data = tl.load(k_bf16_ptr + in_rope_offsets, mask=rope_mask, other=0.0) - - out_rope_offsets = token_id * k_rope_bf16_stride_0 + rope_range - tl.store(k_rope_bf16_ptr + out_rope_offsets, rope_data, mask=rope_mask) - else: - tile_range = tl.arange(0, TILE_SIZE) - - in_tile_offsets = token_id * k_bf16_stride_0 + tile_id * TILE_SIZE + tile_range - x_bf16 = tl.load(k_bf16_ptr + in_tile_offsets) - x_fp32 = x_bf16.to(tl.float32) - - abs_x = tl.abs(x_fp32) - max_abs = tl.max(abs_x) - max_abs_clamped = tl.maximum(max_abs, EPS) - scale = max_abs_clamped / FP8_MAX - - log2_scale = tl.log2(scale) - ceil_log2 = tl.math.ceil(log2_scale) - scale_pow2_fp32 = tl.exp2(ceil_log2) - scale_inv = 1.0 / scale_pow2_fp32 - x_scaled = x_fp32 * scale_inv - x_fp8 = tl.clamp(x_scaled, FP8_MIN, FP8_MAX).to(k_nope_fp8_ptr.dtype.element_ty) - - out_fp8_offsets = ( - token_id * k_nope_fp8_stride_0 + tile_id * TILE_SIZE + tile_range - ) - tl.store(k_nope_fp8_ptr + out_fp8_offsets, x_fp8) - - exponent = ceil_log2.to(tl.int32) - scale_uint8 = (exponent + 127).to(tl.uint8) - - out_scale_offset = token_id * scale_stride_0 + tile_id - tl.store(scale_k_nope_uint8_ptr + out_scale_offset, scale_uint8) - - -def quant_to_nope_fp8_rope_bf16_pack_triton( - k_bf16: torch.Tensor, -) -> NopeFp8RopeBf16Pack: - assert k_bf16.dtype == torch.bfloat16 - num_tokens, hidden_dim = k_bf16.shape - assert hidden_dim == 512 - dim_nope = 448 - dim_rope = 64 - tile_size = 64 - num_tiles = dim_nope // tile_size - - k_bf16 = k_bf16.contiguous() - - k_nope_fp8 = torch.empty( - (num_tokens, dim_nope), dtype=fp8_dtype, device=k_bf16.device - ) - k_rope_bf16 = torch.empty( - (num_tokens, dim_rope), dtype=torch.bfloat16, device=k_bf16.device - ) - scale_k_nope_ue8m0 = torch.empty( - (num_tokens, num_tiles), dtype=torch.uint8, device=k_bf16.device - ) - - fp8_dtype_info = torch.finfo(fp8_dtype) - - grid = (num_tokens, num_tiles + 1) - _quant_k_cache_fused_kernel[grid]( - k_bf16, - k_nope_fp8, - k_rope_bf16, - scale_k_nope_ue8m0, - k_bf16.stride(0), - k_nope_fp8.stride(0), - k_rope_bf16.stride(0), - scale_k_nope_ue8m0.stride(0), - DIM_NOPE=dim_nope, - DIM_ROPE=dim_rope, - TILE_SIZE=tile_size, - NUM_TILES=num_tiles, - FP8_MIN=fp8_dtype_info.min, - FP8_MAX=fp8_dtype_info.max, - EPS=1e-8, - ) - - return NopeFp8RopeBf16Pack( - k_nope_fp8=k_nope_fp8, - k_rope_bf16=k_rope_bf16, - scale_k_nope_ue8m0=scale_k_nope_ue8m0, - ) diff --git a/python/sglang/kernels/ops/attention/dsv4/torch_quant.py b/python/sglang/kernels/ops/attention/dsv4/torch_quant.py index 8683a24383..9db02a5878 100644 --- a/python/sglang/kernels/ops/attention/dsv4/torch_quant.py +++ b/python/sglang/kernels/ops/attention/dsv4/torch_quant.py @@ -4,8 +4,6 @@ Indexer values use per-32 UE8M0 scales; compressed KV uses per-16 E4M3 scales. Both paths round to the E2M1 grid with ties to even. """ -from typing import Optional - import torch FP8_MAX = 448.0 @@ -62,57 +60,3 @@ def fake_quant_compressed_kv(x: torch.Tensor) -> torch.Tensor: scaled = (blocks / scale).clamp(-FP4_MAX, FP4_MAX) deq = round_fp4(scaled) * scale return deq.flatten(-2).to(x.dtype) - - -# --------------------------------------------------------------------------- -# Pure-torch reference of the paged V4.1 fp8 KV cache format read by the sparse -# decode kernel (528 B/token, "V41"). -# --------------------------------------------------------------------------- - - -def ceil_pow2_scale(x: torch.Tensor) -> torch.Tensor: - """``2 ** ceil(log2(max(x, 1e-4)))`` as fp32, computed on the IEEE bits so - that it is exact at (and just above) powers of two.""" - x = x.float() - scale = ceil_pow2(torch.clamp_min(x, 1e-4)) - # ceil_pow2 works on the bits of a finite value; a NaN or inf amax passes - # through (both become the ue8m0 NaN byte, but only the NaN one turns the - # whole tile's payload into NaN). - return torch.where(torch.isfinite(x), scale, x) - - -def quantize_k_cache_v41( - k: torch.Tensor, page_bytes: Optional[int] = None -) -> torch.Tensor: - """``k`` ``[num_pages, page_size, 512]`` -> uint8 ``[num_pages, page_bytes]`` - pages of the V41 layout: 512 e4m3 per token, then 16 ue8m0 scales per token - (one per 32 values), ``scale = 2 ** ceil(log2(max(amax / 448, 1e-4)))``.""" - num_pages, page_size, d = k.shape - assert d == 512 - x = k.float().view(num_pages, page_size, 16, 32) - scale = ceil_pow2_scale(x.abs().amax(dim=-1) / 448.0) - data = (x / scale.unsqueeze(-1)).to(torch.float8_e4m3fn).view(torch.uint8) - scale_u8 = scale.to(torch.float8_e8m0fnu).view(torch.uint8) - raw = page_size * 528 - if page_bytes is None: - page_bytes = -(-raw // 512) * 512 - assert page_bytes >= raw - out = torch.zeros((num_pages, page_bytes), dtype=torch.uint8, device=k.device) - out[:, : page_size * 512] = data.reshape(num_pages, page_size * 512) - out[:, page_size * 512 : raw] = scale_u8.reshape(num_pages, page_size * 16) - return out - - -def dequantize_k_cache_v41(pages: torch.Tensor, page_size: int) -> torch.Tensor: - """Inverse of :func:`quantize_k_cache_v41`: ``[num_pages, page_size, 512]`` bf16.""" - num_pages = pages.shape[0] - pages = pages.view(torch.uint8) - data = pages[:, : page_size * 512].reshape(num_pages, page_size, 512) - scale = pages[:, page_size * 512 : page_size * 528].reshape( - num_pages, page_size, 16 - ) - values = data.view(torch.float8_e4m3fn).to(torch.bfloat16) - scale_bf16 = scale.view(torch.float8_e8m0fnu).to(torch.bfloat16) - return (values.view(num_pages, page_size, 16, 32) * scale_bf16.unsqueeze(-1)).view( - num_pages, page_size, 512 - ) diff --git a/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/paged_decode.py b/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/paged_decode.py index ae70c9eafd..6fa3bc8382 100644 --- a/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/paged_decode.py +++ b/python/sglang/kernels/ops/attention/dsv4/unified_kv_kernels/paged_decode.py @@ -52,13 +52,13 @@ the captured launch sequence. from __future__ import annotations import functools -import os import torch import triton import triton.language as tl from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz +from sglang.srt.environ import envs from sglang.srt.utils import is_hip from sglang.srt.utils.common import is_gfx1250_supported @@ -117,7 +117,7 @@ _FP8_DTYPE = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn # End to end on DSv4 / MI355X / c128, n-weighted at matched batch size against # the same config with the plain heuristic: decode step 121.76 -> 114.12 ms # (-7.50 ms, -6.2 %). At c256 the gain is smaller (-5.37 ms of 156.19). -_HCA_KV_SPLITS = int(os.environ.get("SGLANG_MLA_HCA_KV_SPLITS", "4")) +_HCA_KV_SPLITS = envs.SGLANG_MLA_HCA_KV_SPLITS.get() def _kv_splits_for_stream(compress_ratio: int) -> int | None: diff --git a/python/sglang/kernels/ops/attention/extend_attention.py b/python/sglang/kernels/ops/attention/extend_attention.py index 4ca5962001..025ec0d596 100644 --- a/python/sglang/kernels/ops/attention/extend_attention.py +++ b/python/sglang/kernels/ops/attention/extend_attention.py @@ -24,7 +24,6 @@ import triton import triton.language as tl from sglang.kernels.ops.attention.decode_attention import _extract_kv_strides -from sglang.kernels.ops.attention.prefill_attention import context_attention_fwd from sglang.kernels.ops.attention.score_mod import unpack_aux_tensors from sglang.srt.environ import envs from sglang.srt.utils import is_cuda, is_gfx95_supported, is_gfx1250_supported, is_hip @@ -1170,43 +1169,6 @@ def extend_attention_fwd( ) -def redundant_attention( - q_extend, - o_extend, - k_buffer, - v_buffer, - b_req_idx, - b_start_loc, - b_seq_len, - b_seq_len_prefix, - max_len_in_batch, -): - total_token_num = k_buffer.shape[0] - B, H_Q, D = b_req_idx.shape[0], q_extend.shape[-2], q_extend.shape[-1] - q_buffer = torch.empty( - (total_token_num, H_Q, D), dtype=q_extend.dtype, device=q_extend.device - ) - - pt = 0 - for i in range(B): - cur_seq_len_extend = b_seq_len[i] - b_seq_len_prefix[i] - pl, pr = b_start_loc[i] + b_seq_len_prefix[i], b_start_loc[i] + b_seq_len[i] - q_buffer[pl:pr] = q_extend[pt : pt + cur_seq_len_extend] - pt += cur_seq_len_extend - - o_buffer = torch.empty_like(q_buffer) - context_attention_fwd( - q_buffer, k_buffer, v_buffer, o_buffer, b_start_loc, b_seq_len, max_len_in_batch - ) - - pt = 0 - for i in range(B): - cur_seq_len_extend = b_seq_len[i] - b_seq_len_prefix[i] - pl, pr = b_start_loc[i] + b_seq_len_prefix[i], b_start_loc[i] + b_seq_len[i] - o_extend[pt : pt + cur_seq_len_extend] = o_buffer[pl:pr] - pt += cur_seq_len_extend - - @triton.jit def _fwd_kernel_unified( Q, diff --git a/python/sglang/kernels/ops/attention/fla/gdn_replayssm_spec_decode.py b/python/sglang/kernels/ops/attention/fla/gdn_replayssm_spec_decode.py index 389de92a3a..fa6150eafd 100644 --- a/python/sglang/kernels/ops/attention/fla/gdn_replayssm_spec_decode.py +++ b/python/sglang/kernels/ops/attention/fla/gdn_replayssm_spec_decode.py @@ -594,145 +594,6 @@ def gdn_replayssm_exact_fold_kernel( tl.store(p_h0, b_h.to(p_h0.dtype.element_ty), mask=mask_h) -@triton.jit -def gdn_replayssm_circular_commit_kernel( - h0, # [num_layers, num_slots, HV, V, K] - rawv_cache, # [num_layers, num_slots, HV, L, V] - rawk_cache, # [num_layers, num_slots, H, L, K] - g_cache, # [num_layers, num_slots, HV, L] - beta_cache, # [num_layers, num_slots, HV, L] - ssm_state_indices, # [B] - write_pos, # [num_slots], already advanced by the accepted length - cache_base, # [num_slots] - is_flush_flags, # [num_slots] - accept_lens, # [B] - mamba_track_indices, # [B] - mamba_steps_to_track, # [B] - stride_state_slot: tl.constexpr, - stride_rawv_slot: tl.constexpr, - stride_rawk_slot: tl.constexpr, - stride_g_slot: tl.constexpr, - stride_beta_slot: tl.constexpr, - stride_state_layer: tl.constexpr, - stride_rawv_layer: tl.constexpr, - stride_rawk_layer: tl.constexpr, - stride_g_layer: tl.constexpr, - stride_beta_layer: tl.constexpr, - stride_indices: tl.constexpr, - stride_accept: tl.constexpr, - stride_track: tl.constexpr, - stride_steps: tl.constexpr, - H: tl.constexpr, - HV: tl.constexpr, - K: tl.constexpr, - V: tl.constexpr, - BK: tl.constexpr, - BV: tl.constexpr, - MAX_CACHE_LEN: tl.constexpr, - USE_QK_L2NORM_IN_KERNEL: tl.constexpr, - NULL_BLOCK_ID: tl.constexpr, - HAS_TRACK: tl.constexpr, -): - """Materialize circular history only at capacity or radix-track boundaries.""" - i_v = tl.program_id(0) - i_n = tl.program_id(1) - i_hvl = tl.program_id(2) - i_layer = (i_hvl // HV).to(tl.int64) - i_hv = i_hvl % HV - i_h = i_hv // (HV // H) - - state_idx = tl.load(ssm_state_indices + i_n * stride_indices).to(tl.int64) - if state_idx <= NULL_BLOCK_ID: - return - n_history = tl.load(write_pos + state_idx).to(tl.int32) - fold_active = tl.load(is_flush_flags + state_idx) != 0 - - if HAS_TRACK: - track_idx = tl.load(mamba_track_indices + i_n * stride_track).to(tl.int64) - track_step = tl.load(mamba_steps_to_track + i_n * stride_steps).to(tl.int32) - has_track = (track_idx > NULL_BLOCK_ID) & (track_step >= 0) - n_accepted = tl.load(accept_lens + i_n * stride_accept).to(tl.int32) - previous_history = n_history - n_accepted - track_pos = previous_history + track_step + 1 - else: - track_idx = NULL_BLOCK_ID - has_track = False - track_pos = 0 - - if (not fold_active) and (not has_track): - return - replay_len = tl.maximum(tl.where(fold_active, n_history, 0), track_pos) - if replay_len <= 0: - return - - h0 = h0 + i_layer * stride_state_layer - rawv_cache = rawv_cache + i_layer * stride_rawv_layer - rawk_cache = rawk_cache + i_layer * stride_rawk_layer - g_cache = g_cache + i_layer * stride_g_layer - beta_cache = beta_cache + i_layer * stride_beta_layer - base = tl.load(cache_base + state_idx).to(tl.int32) - - o_k = tl.arange(0, BK) - o_v = i_v * BV + tl.arange(0, BV) - mask_k = o_k < K - mask_v = o_v < V - mask_h = mask_k[:, None] & mask_v[None, :] - p_h0 = ( - h0 - + state_idx * stride_state_slot - + i_hv * V * K - + o_v[None, :] * K - + o_k[:, None] - ) - b_h = tl.load(p_h0, mask=mask_h, other=0.0).to(tl.float32) - - for t in range(0, replay_len): - phys = ((base + t) & (MAX_CACHE_LEN - 1)).to(tl.int64) - b_k = tl.load( - rawk_cache - + state_idx * stride_rawk_slot - + (i_h * MAX_CACHE_LEN + phys) * K - + o_k, - mask=mask_k, - other=0.0, - ).to(tl.float32) - b_v = tl.load( - rawv_cache - + state_idx * stride_rawv_slot - + (i_hv * MAX_CACHE_LEN + phys) * V - + o_v, - mask=mask_v, - other=0.0, - ).to(tl.float32) - b_g = tl.load( - g_cache + state_idx * stride_g_slot + i_hv * MAX_CACHE_LEN + phys - ).to(tl.float32) - b_beta = tl.load( - beta_cache + state_idx * stride_beta_slot + i_hv * MAX_CACHE_LEN + phys - ).to(tl.float32) - if USE_QK_L2NORM_IN_KERNEL: - b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6) - b_h *= tl.exp(b_g) - b_v -= tl.sum(b_h * b_k[:, None], 0) - b_v *= b_beta - b_h += b_k[:, None] * b_v[None, :] - - if HAS_TRACK: - if has_track and (t + 1 == track_pos): - tl.store( - h0 - + track_idx * stride_state_slot - + i_hv * V * K - + o_v[None, :] * K - + o_k[:, None], - b_h.to(h0.dtype.element_ty), - mask=mask_h, - ) - - if fold_active: - tl.store(p_h0, b_h.to(p_h0.dtype.element_ty), mask=mask_h) - - @triton.jit def gdn_replayssm_compact_commit_kernel( h0, @@ -1011,39 +872,6 @@ def _advance_gdn_spec_cursors_kernel( tl.store(is_flush_ptr + blk, next_is_flush, mask=valid) -@triton.jit -def _reset_gdn_replayssm_spec_cursors_kernel( - write_pos_ptr, - cache_base_ptr, - is_flush_ptr, - do_reset_ptr, - state_batch_indices_ptr, - n_rows, - stride_sbi: tl.constexpr, - stride_reset: tl.constexpr, - INIT_FLUSH: tl.constexpr, - BLOCK: tl.constexpr, - NULL_BLOCK_ID: tl.constexpr, -): - offs = tl.arange(0, BLOCK) - row_mask = offs < n_rows - blk = tl.load( - state_batch_indices_ptr + offs * stride_sbi, mask=row_mask, other=NULL_BLOCK_ID - ).to(tl.int64) - do_reset = tl.load(do_reset_ptr + offs * stride_reset, mask=row_mask, other=0).to( - tl.int32 - ) - do = row_mask & (blk > NULL_BLOCK_ID) & (do_reset != 0) - - tl.store(write_pos_ptr + blk, tl.zeros_like(blk).to(tl.int32), mask=do) - tl.store(cache_base_ptr + blk, tl.zeros_like(blk).to(tl.int32), mask=do) - tl.store( - is_flush_ptr + blk, - tl.full([BLOCK], INIT_FLUSH, dtype=tl.int8), - mask=do, - ) - - # --------------------------------------------------------------------------- # Python wrappers. # --------------------------------------------------------------------------- @@ -1541,33 +1369,3 @@ def commit_gdn_replayssm_circular( BLOCK=block, NULL_BLOCK_ID=null_block_id, ) - - -def reset_gdn_replayssm_spec_cursors( - write_pos: torch.Tensor, - cache_base: torch.Tensor, - is_flush: torch.Tensor, - do_reset: torch.Tensor, # [n_rows] int/bool 1 for first-decode rows - state_batch_indices: torch.Tensor, # [n_rows] int - max_cache_len: int, - max_spec_len: int, - null_block_id: int = 0, -): - """Reset the cursors of first-decode rows (prefill->decode handoff).""" - n_rows = state_batch_indices.shape[0] - BLOCK = triton.next_power_of_2(max(1, n_rows)) - # Early-flush margin (2 * max_spec_len, strict '>'): mirror _advance_gdn_spec_cursors_kernel. - init_flush = 1 if 2 * max_spec_len > max_cache_len else 0 - _reset_gdn_replayssm_spec_cursors_kernel[(1,)]( - write_pos, - cache_base, - is_flush, - do_reset, - state_batch_indices, - n_rows, - stride_sbi=state_batch_indices.stride(0), - stride_reset=do_reset.stride(0), - INIT_FLUSH=init_flush, - BLOCK=BLOCK, - NULL_BLOCK_ID=null_block_id, - ) diff --git a/python/sglang/kernels/ops/attention/flash_attention_v4.py b/python/sglang/kernels/ops/attention/flash_attention_v4.py index d22a68aa76..31140a1226 100644 --- a/python/sglang/kernels/ops/attention/flash_attention_v4.py +++ b/python/sglang/kernels/ops/attention/flash_attention_v4.py @@ -1,7 +1,5 @@ from __future__ import annotations -import inspect -import os from functools import cache, lru_cache from typing import Callable, Optional, Tuple, Union @@ -11,14 +9,9 @@ import torch.nn.functional as F from sglang.kernels.kernel_api_logging import debug_kernel_api try: - if os.environ.get("SGLANG_INKLING_FA4_USE_PIP") == "1": - # A/B debug escape hatch: route through the pip flash-attn-4 package - # (dev's stack). rel_bias is vendored-only, so SHEARED must be 0. - from flash_attn.cute import flash_attn_varlen_func as _flash_attn_varlen_func - else: - from sglang.kernels.ops.attention.flash_attn.cute import ( - flash_attn_varlen_func as _flash_attn_varlen_func, - ) + from sglang.kernels.ops.attention.flash_attn.cute import ( + flash_attn_varlen_func as _flash_attn_varlen_func, + ) except Exception as _e: # pragma: no cover _flash_attn_varlen_func = None _flash_attn_import_error = _e @@ -26,14 +19,6 @@ else: _flash_attn_import_error = None -def _takes_out(kernel) -> bool: - return kernel is not None and "out" in inspect.signature(kernel).parameters - - -# Released flash-attn-4 builds may lack the vendored kernel's out=. -_kernel_takes_out = _takes_out(_flash_attn_varlen_func) - - def is_flash_attention_v4_available() -> bool: return _flash_attn_varlen_func is not None @@ -293,11 +278,7 @@ def flash_attn_varlen_func( rel_bias_kwargs["rel_bias_prep_cache"] = rel_bias_prep_cache # The kernel writes into out, except where it pads MLA heads: its output # then has more heads than out, and the result is a new tensor. - out_kwargs = ( - {"out": out} - if out is not None and mla_head_padding is None and _kernel_takes_out - else {} - ) + out_kwargs = {"out": out} if out is not None and mla_head_padding is None else {} result = _flash_attn_varlen_func( q=q, k=k, diff --git a/python/sglang/kernels/ops/attention/flash_attention_v4_sm120.py b/python/sglang/kernels/ops/attention/flash_attention_v4_sm120.py index d534618df2..b515ed71ea 100644 --- a/python/sglang/kernels/ops/attention/flash_attention_v4_sm120.py +++ b/python/sglang/kernels/ops/attention/flash_attention_v4_sm120.py @@ -3,13 +3,17 @@ from __future__ import annotations -import os from dataclasses import dataclass from typing import Callable, Optional, Tuple, Union import torch from sglang.kernels.kernel_api_logging import debug_kernel_api +from sglang.kernels.ops.attention.fa4_sm120.dispatch import ( + get_forward_arch, + resolve_runtime_policy, + try_cached_paged_decode, +) from sglang.kernels.ops.attention.flash_attention_v4 import ( _flash_attn_import_error, _flash_attn_varlen_func, @@ -18,18 +22,6 @@ from sglang.kernels.ops.attention.flash_attention_v4 import ( _unpad_mla_result, ) -if os.environ.get("SGLANG_INKLING_FA4_USE_PIP") == "1": - # The pip escape hatch deliberately bypasses SGLang-owned SM12x kernels. - get_forward_arch = None - resolve_runtime_policy = None - try_cached_paged_decode = None -else: - from sglang.kernels.ops.attention.fa4_sm120.dispatch import ( - get_forward_arch, - resolve_runtime_policy, - try_cached_paged_decode, - ) - @dataclass(frozen=True) class FlashAttentionV4SM120RuntimePolicy: @@ -44,13 +36,6 @@ def get_flash_attention_v4_sm120_runtime_policy( deterministic: bool, ) -> FlashAttentionV4SM120RuntimePolicy: """Resolve the SM12x FA4 launch policy exposed to SGLang.""" - if resolve_runtime_policy is None: - num_splits = 1 if deterministic or device_capability < (9, 0) else 0 - return FlashAttentionV4SM120RuntimePolicy( - num_splits=num_splits, - decode_num_splits=num_splits, - decode_uses_static_max_seqlen_k=False, - ) num_splits, decode_num_splits, decode_uses_static_max_seqlen_k = ( resolve_runtime_policy( device_capability=device_capability, @@ -251,7 +236,7 @@ def flash_attn_with_kvcache( (k_cache.shape[0],), cache_seqlens, dtype=torch.int32, device=k_cache.device ) - forward_arch = get_forward_arch(q.device) if get_forward_arch is not None else None + forward_arch = get_forward_arch(q.device) if ( forward_arch is not None and not return_softmax_lse diff --git a/python/sglang/kernels/ops/attention/kda_flydsl/__init__.py b/python/sglang/kernels/ops/attention/kda_flydsl/__init__.py index 93823f2856..00dacdad8b 100644 --- a/python/sglang/kernels/ops/attention/kda_flydsl/__init__.py +++ b/python/sglang/kernels/ops/attention/kda_flydsl/__init__.py @@ -5,13 +5,11 @@ import aiter as _aiter # noqa: F401 from .kimi_k3_kda_decode import ( - flydsl_kimi_k3_kda_decode, flydsl_kimi_k3_kda_decode_with_f_b, is_flydsl_kimi_k3_kda_decode_supported, ) __all__ = [ - "flydsl_kimi_k3_kda_decode", "flydsl_kimi_k3_kda_decode_with_f_b", "is_flydsl_kimi_k3_kda_decode_supported", ] diff --git a/python/sglang/kernels/ops/attention/kda_flydsl/kernels/kimi_k3_kda_decode.py b/python/sglang/kernels/ops/attention/kda_flydsl/kernels/kimi_k3_kda_decode.py deleted file mode 100644 index 34862900b6..0000000000 --- a/python/sglang/kernels/ops/attention/kda_flydsl/kernels/kimi_k3_kda_decode.py +++ /dev/null @@ -1,450 +0,0 @@ -# SPDX-License-Identifier: MIT -# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. - -"""FlyDSL kernel for the fused Kimi-K3 KDA decode path on gfx950.""" - -import functools -import math - -import flydsl.compiler as flyc -import flydsl.expr as fx -from aiter.ops.flydsl.kernels.tensor_shim import GTensor -from flydsl.expr import range_constexpr -from flydsl.expr.typing import T - -_HEADS = 12 -_DIM = 128 -_LOG2E = math.log2(math.e) -_SCALE = _DIM**-0.5 -_BLOCK_THREADS = 256 -_NUM_WARPS = 4 -_WARP_SIZE = 64 -_WARP_THREADS_K = 8 -_VALUES_PER_THREAD_K = 4 -_WARP_TILE_K = _WARP_THREADS_K * _VALUES_PER_THREAD_K -_K_ITERS = _DIM // _WARP_TILE_K -_WARP_THREADS_V = _WARP_SIZE // _WARP_THREADS_K -_V_GROUP_TILE = _NUM_WARPS * _WARP_THREADS_V -_V_ITERS = _DIM // _V_GROUP_TILE -_WAVES_PER_EU = 3 - - -@functools.cache -def create_kimi_k3_kda_decode_kernel(norm_eps: float, lower_bound: float): - """Build the fixed gfx950 BF16 Kimi-K3 decode specialization.""" - - @fx.struct - class SharedStorage: - q: fx.Array[fx.BFloat16, _DIM, 16] - k: fx.Array[fx.BFloat16, _DIM, 16] - v: fx.Array[fx.BFloat16, _DIM, 16] - recurrent_out: fx.Array[fx.BFloat16, _DIM, 16] - norm_partial: fx.Array[fx.Float32, 2, 16] - - @flyc.kernel( - name="kimi_k3_kda_decode_bf16_gfx950", - known_block_size=[_BLOCK_THREADS, 1, 1], - ) - def kernel( - x_mem: fx.Tensor, - weight_mem: fx.Tensor, - conv_state_mem: fx.Tensor, - raw_g_mem: fx.Tensor, - raw_beta_mem: fx.Tensor, - A_log_mem: fx.Tensor, - dt_bias_mem: fx.Tensor, - state_mem: fx.Tensor, - state_indices_mem: fx.Tensor, - output_gate_mem: fx.Tensor, - norm_weight_mem: fx.Tensor, - out_mem: fx.Tensor, - batch_size: fx.Int32, - stride_x_token: fx.Int32, - stride_weight_channel: fx.Int32, - stride_weight_width: fx.Int32, - stride_conv_slot: fx.Int32, - stride_conv_channel: fx.Int32, - stride_conv_width: fx.Int32, - stride_g_token: fx.Int32, - stride_beta_token: fx.Int32, - stride_state_slot: fx.Int32, - stride_gate_token: fx.Int32, - stride_gate_head: fx.Int32, - stride_out_token: fx.Int32, - stride_out_head: fx.Int32, - ): - del batch_size - - x = GTensor(x_mem, dtype=T.bf16, shape=(-1,)) - weight = GTensor(weight_mem, dtype=T.f32, shape=(-1,)) - conv_state = GTensor(conv_state_mem, dtype=T.bf16, shape=(-1,)) - raw_g = GTensor(raw_g_mem, dtype=T.bf16, shape=(-1,)) - raw_beta = GTensor(raw_beta_mem, dtype=T.bf16, shape=(-1,)) - A_log = GTensor(A_log_mem, dtype=T.f32, shape=(-1,)) - dt_bias = GTensor(dt_bias_mem, dtype=T.f32, shape=(-1,)) - state = GTensor(state_mem, dtype=T.f32, shape=(-1,)) - state_indices = GTensor(state_indices_mem, dtype=T.i32, shape=(-1,)) - output_gate = GTensor(output_gate_mem, dtype=T.bf16, shape=(-1,)) - norm_weight = GTensor(norm_weight_mem, dtype=T.bf16, shape=(-1,)) - out = GTensor(out_mem, dtype=T.bf16, shape=(-1,)) - - shared = fx.SharedAllocator().allocate(SharedStorage).peek() - q_lds = shared.q.ptr - k_lds = shared.k.ptr - v_lds = shared.v.ptr - out_lds = shared.recurrent_out.ptr - norm_lds = shared.norm_partial.ptr - - tid = fx.thread_idx.x - block = fx.block_idx.x - batch = block // fx.Int32(_HEADS) - head = block % fx.Int32(_HEADS) - lane = tid % fx.Int32(_WARP_SIZE) - warp = tid // fx.Int32(_WARP_SIZE) - lane_k = lane % fx.Int32(_WARP_THREADS_K) - - state_idx = fx.Int32(state_indices[batch]) - valid = state_idx > fx.Int32(0) - - # Branch bodies are closures: a bare `if` would make the AST rewriter - # carry the GTensors whose methods the body calls out as scf.if results. - def _valid_then(): - # A workgroup exclusively owns all three convolution channels for - # its (batch, head), so every cache entry is shifted exactly once. - def _conv_then(): - channel_local = tid - q_channel = head * fx.Int32(_DIM) + channel_local - k_channel = ( - fx.Int32(_HEADS * _DIM) + head * fx.Int32(_DIM) + channel_local - ) - v_channel = ( - fx.Int32(2 * _HEADS * _DIM) + head * fx.Int32(_DIM) + channel_local - ) - - def convolve_channel(channel): - cs_base = ( - state_idx * stride_conv_slot + channel * stride_conv_channel - ) - c0 = fx.Float32(conv_state[cs_base]) - c1 = fx.Float32(conv_state[cs_base + stride_conv_width]) - c2 = fx.Float32( - conv_state[cs_base + fx.Int32(2) * stride_conv_width] - ) - current = fx.BFloat16(x[batch * stride_x_token + channel]) - current_f32 = fx.Float32(current) - w_base = channel * stride_weight_channel - acc = c0 * fx.Float32(weight[w_base]) - acc = acc + c1 * fx.Float32(weight[w_base + stride_weight_width]) - acc = acc + c2 * fx.Float32( - weight[w_base + fx.Int32(2) * stride_weight_width] - ) - acc = acc + current_f32 * fx.Float32( - weight[w_base + fx.Int32(3) * stride_weight_width] - ) - silu = acc / ( - fx.Float32(1.0) + fx.math.exp2(-acc * fx.Float32(_LOG2E)) - ) - conv_state.store( - cs_base, - fx.BFloat16(c1), - ) - conv_state.store( - cs_base + stride_conv_width, - fx.BFloat16(c2), - ) - conv_state.store( - cs_base + fx.Int32(2) * stride_conv_width, - current, - ) - return silu.to(fx.BFloat16) - - q_conv = convolve_channel(q_channel) - k_conv = convolve_channel(k_channel) - v_conv = convolve_channel(v_channel) - fx.ptr_store(q_conv, q_lds + tid) - fx.ptr_store(k_conv, k_lds + tid) - fx.ptr_store(v_conv, v_lds + tid) - - if tid < fx.Int32(_DIM): - _conv_then() - - fx.gpu.barrier() - - # Four waves split V into 32-row groups. Eight-lane subgroups - # reduce K; each lane issues one aligned f32x4 state transaction. - k_vec_start = lane_k * fx.Int32(_VALUES_PER_THREAD_K) - global_v_start = warp * fx.Int32(_WARP_THREADS_V) + lane // fx.Int32( - _WARP_THREADS_K - ) - vec_f32 = T.vec(_VALUES_PER_THREAD_K, T.f32) - vec_bf16 = T.vec(_VALUES_PER_THREAD_K, T.bf16) - zero_vec = fx.full( - _VALUES_PER_THREAD_K, - 0.0, - fx.Float32, - ) - - q_vecs = [] - k_vecs = [] - decay_vecs = [] - sum_q_partial = fx.Float32(0.0) - sum_k_partial = fx.Float32(0.0) - a = fx.math.exp2(fx.Float32(A_log[head]) * fx.Float32(_LOG2E)) - - for ki in range_constexpr(_K_ITERS): - k_base = k_vec_start + fx.Int32(ki * _WARP_TILE_K) - q_bf16 = fx.ptr_load( - q_lds + k_base, - result_type=vec_bf16, - ) - k_bf16 = fx.ptr_load( - k_lds + k_base, - result_type=vec_bf16, - ) - q_f32 = q_bf16.extf(vec_f32) - k_f32 = k_bf16.extf(vec_f32) - q_vecs.append(q_f32) - k_vecs.append(k_f32) - sum_q_vec = q_f32 * q_f32 - sum_k_vec = k_f32 * k_f32 - sum_q_partial = sum_q_partial + fx.Vector(sum_q_vec).reduce("add") - sum_k_partial = sum_k_partial + fx.Vector(sum_k_vec).reduce("add") - - gate_bf16 = raw_g.vec_load( - (batch * stride_g_token + head * fx.Int32(_DIM) + k_base,), - _VALUES_PER_THREAD_K, - ) - gate_f32 = gate_bf16.extf(vec_f32) - dt = dt_bias.vec_load( - (head * fx.Int32(_DIM) + k_base,), - _VALUES_PER_THREAD_K, - ) - sigmoid_arg = (gate_f32 + dt) * a - gate = fx.Float32(lower_bound) / ( - fx.Float32(1.0) + fx.math.exp2(-sigmoid_arg * fx.Float32(_LOG2E)) - ) - decay_vecs.append(fx.math.exp2(gate * fx.Float32(_LOG2E))) - - width = fx.Int32(_WARP_SIZE) - for offset in (1, 2, 4): - sum_q_partial = sum_q_partial + fx.gpu.shuffle_xor( - sum_q_partial, fx.Int32(offset), width - ) - sum_k_partial = sum_k_partial + fx.gpu.shuffle_xor( - sum_k_partial, fx.Int32(offset), width - ) - - subgroup_leader = (lane // fx.Int32(_WARP_THREADS_K)) * fx.Int32( - _WARP_THREADS_K - ) - norm_q = fx.gpu.shuffle_idx(sum_q_partial, subgroup_leader, width) - norm_k = fx.gpu.shuffle_idx(sum_k_partial, subgroup_leader, width) - inv_q = fx.math.rsqrt(fx.Float32(norm_q) + fx.Float32(1e-6)) - inv_k = fx.math.rsqrt(fx.Float32(norm_k) + fx.Float32(1e-6)) - - for ki in range_constexpr(_K_ITERS): - q_vecs[ki] = q_vecs[ki] * fx.Float32(inv_q) * fx.Float32(_SCALE) - k_vecs[ki] = k_vecs[ki] * fx.Float32(inv_k) - - dot_kq_vec = zero_vec - for ki in range_constexpr(_K_ITERS): - dot_kq_vec = fx.fma(k_vecs[ki], q_vecs[ki], dot_kq_vec) - dot_kq = fx.Vector(dot_kq_vec).reduce("add") - for offset in (1, 2, 4): - dot_kq = dot_kq + fx.gpu.shuffle_xor(dot_kq, fx.Int32(offset), width) - - beta_value = fx.Float32(raw_beta[batch * stride_beta_token + head]) - beta = fx.Float32(1.0) / ( - fx.Float32(1.0) + fx.math.exp2(-beta_value * fx.Float32(_LOG2E)) - ) - state_head_base = state_idx * stride_state_slot + head * fx.Int32( - _DIM * _DIM - ) - - state_vecs = [] - for vi in range_constexpr(_V_ITERS): - global_v = global_v_start + fx.Int32(vi * _V_GROUP_TILE) - for ki in range_constexpr(_K_ITERS): - k_base = k_vec_start + fx.Int32(ki * _WARP_TILE_K) - state_off = state_head_base + global_v * fx.Int32(_DIM) + k_base - state_vecs.append( - state.vec_load( - (state_off,), - _VALUES_PER_THREAD_K, - ) - ) - - for vi in range_constexpr(_V_ITERS): - global_v = global_v_start + fx.Int32(vi * _V_GROUP_TILE) - sum_hk_vec = zero_vec - sum_hq_vec = zero_vec - for ki in range_constexpr(_K_ITERS): - state_pos = vi * _K_ITERS + ki - decayed = state_vecs[state_pos] * decay_vecs[ki] - state_vecs[state_pos] = decayed - sum_hk_vec = fx.fma(decayed, k_vecs[ki], sum_hk_vec) - sum_hq_vec = fx.fma(decayed, q_vecs[ki], sum_hq_vec) - - sum_hk = fx.Vector(sum_hk_vec).reduce("add") - sum_hq = fx.Vector(sum_hq_vec).reduce("add") - for offset in (1, 2, 4): - sum_hk = sum_hk + fx.gpu.shuffle_xor( - sum_hk, fx.Int32(offset), width - ) - sum_hq = sum_hq + fx.gpu.shuffle_xor( - sum_hq, fx.Int32(offset), width - ) - - conv_v = fx.Float32(fx.ptr_load(v_lds + global_v)) - v_new = (conv_v - fx.Float32(sum_hk)) * beta - v_new = fx.gpu.shuffle_idx(v_new, subgroup_leader, width) - recurrent_value = fx.Float32(sum_hq) + fx.Float32(v_new) * fx.Float32( - dot_kq - ) - v_new_vec = fx.full(_VALUES_PER_THREAD_K, v_new, fx.Float32) - - for ki in range_constexpr(_K_ITERS): - state_pos = vi * _K_ITERS + ki - updated = fx.fma(k_vecs[ki], v_new_vec, state_vecs[state_pos]) - k_base = k_vec_start + fx.Int32(ki * _WARP_TILE_K) - state_off = state_head_base + global_v * fx.Int32(_DIM) + k_base - state.vec_store( - (state_off,), - updated, - _VALUES_PER_THREAD_K, - ) - - if lane_k == fx.Int32(0): - fx.ptr_store( - fx.BFloat16(recurrent_value), - out_lds + global_v, - ) - - fx.gpu.barrier() - - # Preserve the model's BF16 boundary before RMSNorm and gating. - def _norm_then(): - recurrent_bf16 = fx.ptr_load(out_lds + tid) - recurrent_f32 = fx.Float32(recurrent_bf16) - square = recurrent_f32 * recurrent_f32 - for offset in (32, 16, 8, 4, 2, 1): - square = square + fx.gpu.shuffle_xor( - square, fx.Int32(offset), width - ) - if lane == fx.Int32(0): - fx.ptr_store(square, norm_lds + warp) - - if tid < fx.Int32(_DIM): - _norm_then() - - fx.gpu.barrier() - - def _output_then(): - norm_sum = fx.Float32(fx.ptr_load(norm_lds)) - norm_sum = norm_sum + fx.Float32(fx.ptr_load(norm_lds + fx.Int32(1))) - inv_rms = fx.math.rsqrt( - norm_sum * fx.Float32(1.0 / _DIM) + fx.Float32(norm_eps) - ) - recurrent_f32 = fx.Float32(fx.ptr_load(out_lds + tid)) - norm_w = fx.Float32(norm_weight[tid]) - gate_value = fx.Float32( - output_gate[ - batch * stride_gate_token + head * stride_gate_head + tid - ] - ) - output_sigmoid = fx.Float32(1.0) / ( - fx.Float32(1.0) + fx.math.exp2(-gate_value * fx.Float32(_LOG2E)) - ) - result = recurrent_f32 * inv_rms * norm_w * output_sigmoid - out.store( - batch * stride_out_token + head * stride_out_head + tid, - result.to(fx.BFloat16), - ) - - if tid < fx.Int32(_DIM): - _output_then() - - def _zero_then(): - out.store( - batch * stride_out_token + head * stride_out_head + tid, - fx.BFloat16(0.0), - ) - - if valid: - _valid_then() - elif tid < fx.Int32(_DIM): - _zero_then() - - @flyc.jit - def launch( - x_mem: fx.Tensor, - weight_mem: fx.Tensor, - conv_state_mem: fx.Tensor, - raw_g_mem: fx.Tensor, - raw_beta_mem: fx.Tensor, - A_log_mem: fx.Tensor, - dt_bias_mem: fx.Tensor, - state_mem: fx.Tensor, - state_indices_mem: fx.Tensor, - output_gate_mem: fx.Tensor, - norm_weight_mem: fx.Tensor, - out_mem: fx.Tensor, - batch_size: fx.Int32, - stride_x_token: fx.Int32, - stride_weight_channel: fx.Int32, - stride_weight_width: fx.Int32, - stride_conv_slot: fx.Int32, - stride_conv_channel: fx.Int32, - stride_conv_width: fx.Int32, - stride_g_token: fx.Int32, - stride_beta_token: fx.Int32, - stride_state_slot: fx.Int32, - stride_gate_token: fx.Int32, - stride_gate_head: fx.Int32, - stride_out_token: fx.Int32, - stride_out_head: fx.Int32, - stream: fx.Stream = fx.Stream(None), # noqa: B008 - ): - kernel( - x_mem, - weight_mem, - conv_state_mem, - raw_g_mem, - raw_beta_mem, - A_log_mem, - dt_bias_mem, - state_mem, - state_indices_mem, - output_gate_mem, - norm_weight_mem, - out_mem, - batch_size, - stride_x_token, - stride_weight_channel, - stride_weight_width, - stride_conv_slot, - stride_conv_channel, - stride_conv_width, - stride_g_token, - stride_beta_token, - stride_state_slot, - stride_gate_token, - stride_gate_head, - stride_out_token, - stride_out_head, - ).launch( - grid=(batch_size * fx.Int32(_HEADS), 1, 1), - block=(_BLOCK_THREADS, 1, 1), - stream=stream, - ) - - launch.compile_hints = { - "waves_per_eu": _WAVES_PER_EU, - "llvm_options": { - "amdgpu-expert-scheduling-mode": True, - }, - } - return launch - - -__all__ = ["create_kimi_k3_kda_decode_kernel"] diff --git a/python/sglang/kernels/ops/attention/kda_flydsl/kimi_k3_kda_decode.py b/python/sglang/kernels/ops/attention/kda_flydsl/kimi_k3_kda_decode.py index df0dd8c39e..cf34d62a73 100644 --- a/python/sglang/kernels/ops/attention/kda_flydsl/kimi_k3_kda_decode.py +++ b/python/sglang/kernels/ops/attention/kda_flydsl/kimi_k3_kda_decode.py @@ -11,9 +11,6 @@ from collections.abc import Iterable import torch from aiter.ops.flydsl.kernels.tensor_shim import _run_compiled -from .kernels.kimi_k3_kda_decode import ( - create_kimi_k3_kda_decode_kernel, -) from .kernels.kimi_k3_kda_decode_fb import ( create_kimi_k3_kda_decode_fb_kernel, ) @@ -127,7 +124,7 @@ def _validate_kda_inputs( norm_weight: torch.Tensor, out: torch.Tensor | None, ) -> torch.Tensor: - """Validate operands shared by both explicit KDA specializations.""" + """Validate the KDA decode operands.""" if not is_flydsl_kimi_k3_kda_decode_supported(device): raise RuntimeError(f"`{api_name}` requires a gfx950 GPU.") if batch <= 0: @@ -256,109 +253,6 @@ def _validate_kda_inputs( return out -def flydsl_kimi_k3_kda_decode( - x: torch.Tensor, - conv_weight: torch.Tensor, - conv_bias: torch.Tensor | None, - conv_state: torch.Tensor, - raw_g: torch.Tensor, - raw_beta: torch.Tensor, - A_log: torch.Tensor, - dt_bias: torch.Tensor, - lower_bound: float | None, - state: torch.Tensor, - state_indices: torch.Tensor, - output_gate: torch.Tensor, - norm_weight: torch.Tensor, - norm_eps: float, - out: torch.Tensor | None = None, -) -> torch.Tensor: - """Run fused Kimi-K3 KDA decode on MI350-series GPUs. - - This pure-decode specialization fuses the packed width-4 Q/K/V causal - convolution, the FP32 recurrent-state update, and the BF16 - RMSNorm/sigmoid output gate. Slot zero is reserved: non-positive - ``state_indices`` produce zero output without modifying either cache. - - The layout is fixed to Kimi-K3 TP8: 12 local heads and 128-dimensional - key/value state. Call - :func:`is_flydsl_kimi_k3_kda_decode_supported` before dispatching from a - model implementation. - """ - if x.ndim != 2: - raise ValueError(f"`x` must have rank 2, got rank {x.ndim}.") - if not x.is_cuda: - raise ValueError("`x` must be a CUDA tensor.") - device = x.device - batch = x.shape[0] - out = _validate_kda_inputs( - api_name="flydsl_kimi_k3_kda_decode", - batch_source="x", - device=device, - batch=batch, - x=x, - conv_weight=conv_weight, - conv_bias=conv_bias, - conv_state=conv_state, - raw_beta=raw_beta, - A_log=A_log, - dt_bias=dt_bias, - lower_bound=lower_bound, - state=state, - state_indices=state_indices, - output_gate=output_gate, - norm_weight=norm_weight, - out=out, - ) - _check_same_device((("raw_g", raw_g),), device) - _check_tensor( - "raw_g", - raw_g, - shape=(1, batch, _HEADS, _DIM), - dtype=torch.bfloat16, - device=device, - inner_strides=(_DIM, 1), - ) - - executable = create_kimi_k3_kda_decode_kernel( - float(norm_eps), - float(lower_bound), - ) - with torch.cuda.device(device): - stream = torch.cuda.current_stream(device) - _run_compiled( - executable, - x, - conv_weight, - conv_state, - raw_g, - raw_beta, - A_log, - dt_bias, - state, - state_indices, - output_gate, - norm_weight, - out, - batch, - x.stride(0), - conv_weight.stride(0), - conv_weight.stride(1), - conv_state.stride(0), - conv_state.stride(1), - conv_state.stride(2), - raw_g.stride(1), - raw_beta.stride(1), - state.stride(0), - output_gate.stride(0), - output_gate.stride(1), - out.stride(1), - out.stride(2), - stream, - ) - return out - - def flydsl_kimi_k3_kda_decode_with_f_b( f_a: torch.Tensor, f_b_weight: torch.Tensor, @@ -471,7 +365,6 @@ def flydsl_kimi_k3_kda_decode_with_f_b( __all__ = [ - "flydsl_kimi_k3_kda_decode", "flydsl_kimi_k3_kda_decode_with_f_b", "is_flydsl_kimi_k3_kda_decode_supported", ] diff --git a/python/sglang/kernels/ops/attention/linear/kda_nvidia_prefill/chunk_fwd.py b/python/sglang/kernels/ops/attention/linear/kda_nvidia_prefill/chunk_fwd.py index 187d088df3..2ab2e67199 100644 --- a/python/sglang/kernels/ops/attention/linear/kda_nvidia_prefill/chunk_fwd.py +++ b/python/sglang/kernels/ops/attention/linear/kda_nvidia_prefill/chunk_fwd.py @@ -37,20 +37,12 @@ from sglang.kernels.ops.attention.fla.index import ( prepare_chunk_indices, ) -try: - from .Akk_inverse_lower_triangle_bf16 import akk_inv_host as _akk_inv_host - from .fuse_k4_only_persistent import BYTES_PER_TENSORMAP as _K4P_BTM - from .fuse_k4_only_persistent import NUM_TENSORMAPS as _K4P_NTM - from .fuse_k4_only_persistent import make_host_fn as _k4p_make_host - from .fuse_kernel123_persistent import CHUNKS_PER_BLOCK as _K123_CHUNKS_PER_BLOCK - from .fuse_kernel123_persistent import make_host_function as _fused_make_host -except ImportError: - from Akk_inverse_lower_triangle_bf16 import akk_inv_host as _akk_inv_host - from fuse_k4_only_persistent import BYTES_PER_TENSORMAP as _K4P_BTM - from fuse_k4_only_persistent import NUM_TENSORMAPS as _K4P_NTM - from fuse_k4_only_persistent import make_host_fn as _k4p_make_host - from fuse_kernel123_persistent import CHUNKS_PER_BLOCK as _K123_CHUNKS_PER_BLOCK - from fuse_kernel123_persistent import make_host_function as _fused_make_host +from .Akk_inverse_lower_triangle_bf16 import akk_inv_host as _akk_inv_host +from .fuse_k4_only_persistent import BYTES_PER_TENSORMAP as _K4P_BTM +from .fuse_k4_only_persistent import NUM_TENSORMAPS as _K4P_NTM +from .fuse_k4_only_persistent import make_host_fn as _k4p_make_host +from .fuse_kernel123_persistent import CHUNKS_PER_BLOCK as _K123_CHUNKS_PER_BLOCK +from .fuse_kernel123_persistent import make_host_function as _fused_make_host def _ct(t, etype): @@ -219,62 +211,6 @@ _k4p_cache = {} _k4p_tm_ws = {} -# Beta absorption (K1 fusion reverted; K123 stays at 375us baseline). -# Two-stage strategy on side stream: -# 1) v*beta runs PARALLEL with K123 (no data dep on K123 outputs) -# 2) k_scaled*beta runs AFTER K123 on side stream, PARALLEL with akk_inv -@torch.compile(fullgraph=True) -def _v_absorb(v, beta): - return v * beta.unsqueeze(-1).to(v.dtype) - - -@torch.compile(fullgraph=True) -def _ks_absorb(k_scaled, beta): - return k_scaled * beta.unsqueeze(-1).to(k_scaled.dtype) - - -# Side stream cache for v_absorb || K123 overlap. -_side_streams = {} - - -def _get_side_stream(dev): - idx = dev.index or 0 - if idx not in _side_streams: - _side_streams[idx] = torch.cuda.Stream(device=dev) - return _side_streams[idx] - - -# ===== Per-section timing (K123 / K4) ===== -# Set _TIMING_ENABLED=True to record CUDA-event timings for each call. -# Times accumulate in _TIMING_STATS; use reset_timings() / get_timings() to manage. -_TIMING_ENABLED = False -_TIMING_STATS = {"k123_us": 0.0, "k4_us": 0.0, "count": 0} - - -def enable_timing(enabled=True): - """Enable/disable per-section CUDA event timing in chunk_kda_fwd. - Each call adds CUDA event records around K123 and K4 launches and a sync - at end of forward, so this slows down execution. Use only for benchmarking.""" - global _TIMING_ENABLED - _TIMING_ENABLED = enabled - - -def reset_timings(): - _TIMING_STATS["k123_us"] = 0.0 - _TIMING_STATS["k4_us"] = 0.0 - _TIMING_STATS["count"] = 0 - - -def get_timings(): - """Returns (k123_avg_us, k4_avg_us, n_calls). Avgs are over all calls since last reset.""" - n = max(_TIMING_STATS["count"], 1) - return ( - _TIMING_STATS["k123_us"] / n, - _TIMING_STATS["k4_us"] / n, - _TIMING_STATS["count"], - ) - - # Buffer cache: avoid re-allocating ~67us of intermediate tensors per call. # Also caches cute.Tensor wrappers (saves ~7us each call from from_dlpack). _buf_cache = {} @@ -406,11 +342,6 @@ def _get_buffers(dev, dtype_k, B, T, H, K_dim, V_dim, NT, N_seqs, BT): s_ct = from_dlpack(S_out, assumed_align=16) s_ct.element_type = fp32 - # Cache PyTorch streams once per buf_cache entry — torch.cuda.current_stream - # is a torch C call that costs ~2us each invocation. - main_stream_cached = torch.cuda.current_stream(dev) - side_stream_cached = _get_side_stream(dev) - cute_wrappers = dict( a_ct=a_ct, aqc_ct=aqc_ct, @@ -424,8 +355,6 @@ def _get_buffers(dev, dtype_k, B, T, H, K_dim, V_dim, NT, N_seqs, BT): akk_in_view=akk_in_view, akk_out_view=akk_out_view, s_ct=s_ct, - main_stream=main_stream_cached, - side_stream=side_stream_cached, # Filled lazily on first launch — saves cache_key tuple build + # outer dict lookup on subsequent calls. _k123_fns={}, @@ -936,18 +865,12 @@ def chunk_kda_fwd( # only pays off for huge multi-seq varlen states (~192MB at N=32); for # serving single-seq engagements the state is <1MB and the side stream # was the last remaining cross-stream surface in this path. - main_stream = torch.cuda.current_stream(device) needs_copy = S_in.data_ptr() != S_out.data_ptr() if needs_copy: S_out.copy_(S_in) # Beta is fused entirely in akk_inv kernel epilogue (post-inv column-scale). # No host v*beta and no K1 k_scaled*beta any more. - if _TIMING_ENABLED: - k123_s = torch.cuda.Event(enable_timing=True) - k123_e = torch.cuda.Event(enable_timing=True) - k123_s.record(stream=main_stream) - _launch_fused_k123_inv( q, k, @@ -974,9 +897,6 @@ def chunk_kda_fwd( varlen_pure_override=(True if needs_varlen_single_pad else varlen_pure), ) - if _TIMING_ENABLED: - k123_e.record(stream=main_stream) - # ===== K4: persistent kernel (eqlen + varlen via cu_seqlens) ===== if is_varlen: # GPU-side cumsum + cache by id(cu_seqlens). No host sync. @@ -987,11 +907,6 @@ def chunk_kda_fwd( # State copy now runs on the caller's stream; K4 is stream-ordered after it. - if _TIMING_ENABLED: - k4_s = torch.cuda.Event(enable_timing=True) - k4_e = torch.cuda.Event(enable_timing=True) - k4_s.record(stream=main_stream) - _launch_k4_persistent( cute_wrappers, v, @@ -1005,13 +920,6 @@ def chunk_kda_fwd( use_fast_sync=(not is_varlen), ) - if _TIMING_ENABLED: - k4_e.record(stream=cute_wrappers["main_stream"]) - torch.cuda.synchronize(device) - _TIMING_STATS["k123_us"] += k123_s.elapsed_time(k123_e) * 1000.0 - _TIMING_STATS["k4_us"] += k4_s.elapsed_time(k4_e) * 1000.0 - _TIMING_STATS["count"] += 1 - o = O_flat A_qk_out = A_qk A_kk_out = A_kk diff --git a/python/sglang/kernels/ops/attention/linear/kda_nvidia_prefill/fuse_k4_only_persistent.py b/python/sglang/kernels/ops/attention/linear/kda_nvidia_prefill/fuse_k4_only_persistent.py index 526cd61706..c084564bcb 100644 --- a/python/sglang/kernels/ops/attention/linear/kda_nvidia_prefill/fuse_k4_only_persistent.py +++ b/python/sglang/kernels/ops/attention/linear/kda_nvidia_prefill/fuse_k4_only_persistent.py @@ -36,8 +36,6 @@ State management: - single state_ready_mbar after both passes (TMEM no-overlap constraint) """ -import os -import sys import warnings import cutlass.cutlass_dsl as _dsl_mod @@ -62,54 +60,11 @@ from cutlass.cute.nvgpu import cpasync, tcgen05 from cutlass.cute.nvgpu.tcgen05 import Field, OperandMajorMode, OperandSource from cutlass.cute.runtime import from_dlpack from cutlass.cutlass_dsl import Int32, T - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -# GDN tile scheduler: import from the public FlashInfer release -# (`pip install flashinfer-python>=0.6.13`). Fall back to a local -# dynamic-kernel-generator checkout only if FlashInfer is unavailable. -try: - from flashinfer.gdn_kernels.blackwell.gated_delta_net_tile_scheduler import ( - GDNTileScheduler, - GDNTileSchedulerParams, - ) -except ImportError: - # Try multiple locations for GDNTileScheduler (workspace layout vs kda_optimized layout) - _GDN_PATHS = [ - # workspace/flash-linear-attention/prefill -> ../../../dynamic-kernel-generator - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "..", - "..", - "..", - "dynamic-kernel-generator", - "cutlass_ir", - "compiler", - "python", - "examples", - "blackwell", - "gated_delta_net", - ), - # scripts/kda_optimized -> ../../dynamic-kernel-generator - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "..", - "..", - "dynamic-kernel-generator", - "cutlass_ir", - "compiler", - "python", - "examples", - "blackwell", - "gated_delta_net", - ), - ] - for _p in _GDN_PATHS: - if os.path.isdir(_p): - sys.path.insert(0, _p) - break - from gated_delta_net_tile_scheduler import GDNTileSchedulerParams, GDNTileScheduler - from cutlass.utils import TensorMapManager, TensorMapUpdateMode +from flashinfer.gdn_kernels.blackwell.gated_delta_net_tile_scheduler import ( + GDNTileScheduler, + GDNTileSchedulerParams, +) def transform_partitioned_tensor_layout(tensor): diff --git a/python/sglang/kernels/ops/attention/linear/seg_la.py b/python/sglang/kernels/ops/attention/linear/seg_la.py index 9f5f6a44cc..fd7bf95886 100644 --- a/python/sglang/kernels/ops/attention/linear/seg_la.py +++ b/python/sglang/kernels/ops/attention/linear/seg_la.py @@ -29,178 +29,6 @@ class SegLaMeta: mask: Optional[torch.Tensor] = None # Currently not supported -# fused -@triton.jit -def seg_la_kernel( - Q, - K, - V, - S, - Out, - softmax_scale, - stride_q, - stride_k, - stride_v, - stride_s, - stride_o, - s_offsets, - q_offsets, - q_lengths, - s_scales, - decay_scales, - HEAD_DIM: tl.constexpr, - SPLIT_DIM: tl.constexpr, - BLOCK: tl.constexpr, - EVEN: tl.constexpr, - DECOUPLE: tl.constexpr, -): - bid = tl.program_id(0) - hid = tl.program_id(1) - sid = tl.program_id(2) - - # s_scale is 0 (prefill) or 1 (decode) - s_scale = tl.load(s_scales + bid) - q_length = tl.load(q_lengths + bid) - q_offset = tl.load(q_offsets + bid) - s_offset = tl.load(s_offsets + bid) - decay_scale = -tl.load(decay_scales + hid) - - offs_b = tl.arange(0, BLOCK) - offs_d = tl.arange(0, HEAD_DIM) - offs_s = tl.arange(0, SPLIT_DIM) - - if s_offset == -1: - return - - q_ptrs = ( - Q - + q_offset * stride_q - + hid * HEAD_DIM - + (offs_b[:, None] * stride_q + offs_d[None, :]) - ) - k_ptrs = ( - K - + q_offset * stride_k - + hid * HEAD_DIM - + (offs_b[:, None] * stride_k + offs_d[None, :]) - ) - v_ptrs = ( - V - + q_offset * stride_v - + hid * HEAD_DIM - + sid * SPLIT_DIM - + (offs_b[:, None] * stride_v + offs_s[None, :]) - ) - out_ptrs = ( - Out - + q_offset * stride_o - + hid * HEAD_DIM - + sid * SPLIT_DIM - + (offs_b[:, None] * stride_o + offs_s[None, :]) - ) - s_ptrs = ( - S - + s_offset * stride_s - + hid * HEAD_DIM * HEAD_DIM - + sid * SPLIT_DIM - + (offs_d[:, None] * HEAD_DIM + offs_s[None, :]) - ) - state = tl.load(s_ptrs, mask=s_scale > 0).to(tl.float32) - - if BLOCK > 1: - for n in range(0, q_length, BLOCK): - n = tl.multiple_of(n, BLOCK) - - if EVEN: - q = tl.load(q_ptrs + n * stride_q).to(tl.float32) - k = tl.trans(tl.load(k_ptrs + n * stride_k)).to(tl.float32) - v = tl.load(v_ptrs + n * stride_k).to(tl.float32) - else: - q = tl.load( - q_ptrs + n * stride_q, - mask=(n + offs_b)[:, None] < q_length, - other=0.0, - ).to(tl.float32) - k = tl.trans( - tl.load( - k_ptrs + n * stride_k, - mask=(n + offs_b)[:, None] < q_length, - other=0.0, - ) - ).to(tl.float32) - v = tl.load( - v_ptrs + n * stride_k, - mask=(n + offs_b)[:, None] < q_length, - other=0.0, - ).to(tl.float32) - - if DECOUPLE: - # only work with small scales - if EVEN: - b = BLOCK - else: - b = min(BLOCK, q_length - n) - b_offs = b - 1 - offs_b - - edb = tl.exp(decay_scale * b_offs) - decays = tl.where(b_offs >= 0, edb, 0) - inv_decays = tl.where(b_offs >= 0, 1 / edb, 0) - - q = q * inv_decays[:, None] - k = k * decays[None, :] - qk = tl.dot(q, k) * softmax_scale - qk = tl.where(offs_b[None, :] <= offs_b[:, None], qk, 0.0) - o = tl.dot(qk, v) - - block_decay = tl.exp(decay_scale * b) - block_decay_plus = block_decay * softmax_scale - o = tl.dot(q, state) * block_decay_plus + o - - state = state * block_decay + tl.dot(k, v) - else: - qk = tl.dot(q, k) * softmax_scale - decays = tl.exp(decay_scale * (offs_b[:, None] - offs_b[None, :])) - decays = tl.where(offs_b[None, :] <= offs_b[:, None], decays, 0.0) - qk *= decays - o = tl.dot(qk, v) - - decay_arr = tl.exp(decay_scale * (offs_b[:, None] + 1)) * softmax_scale - o = tl.dot(q * decay_arr, state, acc=o) - - if EVEN: - b = BLOCK - else: - b = min(BLOCK, q_length - n) - b_offs = b - 1 - offs_b - b_offs = tl.where(b_offs >= 0, b_offs, 10000) - decays = tl.exp(decay_scale * b_offs) - block_decay = tl.exp(decay_scale * b) - state = state * block_decay + tl.dot(k * decays[None, :], v) - - if EVEN: - tl.store(out_ptrs + n * stride_o, o.to(Out.dtype.element_ty)) - else: - tl.store( - out_ptrs + n * stride_o, - o.to(Out.dtype.element_ty), - mask=(n + offs_b)[:, None] < q_length, - ) - - tl.store(s_ptrs, state.to(S.dtype.element_ty)) - - else: - q = tl.trans(tl.load(q_ptrs)).to(tl.float32) * softmax_scale - k = tl.trans(tl.load(k_ptrs)).to(tl.float32) - v = tl.load(v_ptrs).to(tl.float32) - state = state * tl.exp(decay_scale) + k * v - - o = tl.sum(q * state, axis=0, keep_dims=True) - - tl.store(out_ptrs, o.to(Out.dtype.element_ty)) - - tl.store(s_ptrs, state.to(S.dtype.element_ty)) - - # used for prefilling @triton.jit def seg_la_p_kernel( @@ -691,7 +519,6 @@ def seg_la_fwd( track_lens=None, track_state_indices=None, softmax_scale=None, - decouple=False, ): length, qo_heads, HEAD_DIM = q.shape _, kv_heads, _ = k.shape @@ -706,7 +533,6 @@ def seg_la_fwd( if MAX_LENGTH > 1: # prefill with partitioning q/k/v - # BLOCK should <= 64 with decouple K_SPLIT_DIM = 32 V_SPLIT_DIM = 32 if bs <= 2 else 64 @@ -884,61 +710,4 @@ def seg_la_fwd( else: o = tmp[0] - # if fallback: - # # prefill/decode with partitioning v only - # o = torch.empty(q.shape, device=q.device, dtype=q.dtype) - # if MAX_LENGTH == 1: - # # decode - # BLOCK = 1 - # EVEN = False - # SPLIT_DIM = 32 - # num_warps = 8 - # num_stages = 2 - # num_dim_block = HEAD_DIM // SPLIT_DIM - # grid = (batch, kv_heads, num_dim_block) - # else: - # # prefill - # if decouple: - # BLOCK = 64 - # SPLIT_DIM = 16 - # else: - # BLOCK = HEAD_DIM - # SPLIT_DIM = 32 - # # EVEN = all([x % BLOCK == 0 for x in meta.qls]) - # EVEN = False - # num_warps = 8 - # num_stages = 2 - # # prop = torch.cuda.get_device_properties(q.device.index) - # # arch = prop.major * 10 + prop.minor - # # if arch not in (80, 90): - # # num_stages = 1 - - # num_dim_block = HEAD_DIM // SPLIT_DIM - # grid = (batch, kv_heads, num_dim_block) - - # seg_la_kernel[grid]( - # q, - # k, - # v, - # s, - # o, - # softmax_scale, - # q.stride(0), - # k.stride(0), - # v.stride(0), - # s.stride(0), - # o.stride(0), - # meta.s_offsets, - # meta.q_offsets, - # meta.q_lengths, - # meta.s_scales, - # decay_scales, - # HEAD_DIM=HEAD_DIM, - # SPLIT_DIM=SPLIT_DIM, - # BLOCK=BLOCK, - # EVEN=EVEN, - # DECOUPLE=decouple, - # num_warps=num_warps, - # num_stages=num_stages - # ) return o diff --git a/python/sglang/kernels/ops/attention/triton_gdn_fused_proj.py b/python/sglang/kernels/ops/attention/triton_gdn_fused_proj.py index dba707f25e..2ca0e1f7c1 100644 --- a/python/sglang/kernels/ops/attention/triton_gdn_fused_proj.py +++ b/python/sglang/kernels/ops/attention/triton_gdn_fused_proj.py @@ -4,10 +4,11 @@ import torch import triton import triton.language as tl -from sglang.srt.utils import get_bool_env_var, is_hip +from sglang.srt.environ import envs +from sglang.srt.utils import is_hip _is_hip = is_hip() -_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip +_use_aiter = envs.SGLANG_USE_AITER.get() and _is_hip # ============================================================================= # Fused kernel — reads INTERLEAVED input format diff --git a/python/sglang/kernels/ops/attention/vattn_asm_gfx950/__init__.py b/python/sglang/kernels/ops/attention/vattn_asm_gfx950/__init__.py index 29545fbf83..771d5b2603 100644 --- a/python/sglang/kernels/ops/attention/vattn_asm_gfx950/__init__.py +++ b/python/sglang/kernels/ops/attention/vattn_asm_gfx950/__init__.py @@ -15,6 +15,7 @@ HIP-graph-capture safe (no host sync). """ import ctypes +import logging import os import re import subprocess @@ -24,6 +25,8 @@ import torch from sglang.srt.distributed.device_communicators.cuda_wrapper import find_loaded_library +logger = logging.getLogger(__name__) + _DIR = os.path.dirname(os.path.abspath(__file__)) _LOG2E = 1.4426950408889634 @@ -205,10 +208,9 @@ def asm_kernel_available() -> bool: def _disable(reason): global _disabled_reason _disabled_reason = reason - print( + logger.warning( f"[vattn_asm] disabling the gfx950 asm attention kernel, falling back to " - f"the Triton path: {reason}", - flush=True, + f"the Triton path: {reason}" ) raise AsmKernelUnavailable(reason) @@ -264,10 +266,9 @@ def _get_kernel(hq): except RuntimeError as e: _disable(f"loading {co} failed: {e}") _kernels[hq] = kern - print( + logger.info( f"[vattn_asm] gfx950 asm verify-attention kernel active " - f"(in-tree loader, GQA ratio {hq}, built {co})", - flush=True, + f"(in-tree loader, GQA ratio {hq}, built {co})" ) return kern diff --git a/python/sglang/kernels/ops/attention/vision_rope.py b/python/sglang/kernels/ops/attention/vision_rope.py index 74bc0ba6d2..e68dfb5256 100644 --- a/python/sglang/kernels/ops/attention/vision_rope.py +++ b/python/sglang/kernels/ops/attention/vision_rope.py @@ -79,11 +79,13 @@ def can_use_fused_qk_complex_rope( device: torch.device, freqs_cis: torch.Tensor, ) -> bool: - """Select the fused path before projecting the vision Q/K tensors.""" + """Select the fused path before projecting the vision Q/K tensors. + + ``freqs_cis`` sharing the activations' device is the kernel's contract, + which :func:`apply_fused_qk_complex_rope` asserts, not a dispatch choice. + """ if not ( device.type == "cuda" - and freqs_cis.is_cuda - and device == freqs_cis.device and dtype in (torch.bfloat16, torch.float16) and freqs_cis.dtype == torch.complex64 ): @@ -115,7 +117,7 @@ def apply_fused_qk_complex_rope( f"freqs={freqs_cis.shape}/{freqs_cis.dtype}/{freqs_cis.device}" ) - assert q.device == k.device and q.dtype == k.dtype + assert q.device == k.device == freqs_cis.device and q.dtype == k.dtype assert q.shape == k.shape and q.ndim >= 3 assert q.shape[-1] % 2 == 0 assert freqs_cis.shape == q.shape[:-2] + (q.shape[-1] // 2,) diff --git a/python/sglang/kernels/ops/communication/__init__.py b/python/sglang/kernels/ops/communication/__init__.py index 93b52b67af..a2fb10cbda 100644 --- a/python/sglang/kernels/ops/communication/__init__.py +++ b/python/sglang/kernels/ops/communication/__init__.py @@ -6,7 +6,12 @@ records metadata; importing this group does not load the implementations. """ from sglang.kernels.registry import register_kernel -from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec +from sglang.kernels.spec import ( + CUDA_ONLY, + CapabilityRequirement, + KernelBackend, + KernelSpec, +) __all__ = [] @@ -16,7 +21,6 @@ register_kernel( backend=KernelBackend.JIT, target="sglang.kernels.ops.communication.all_reduce_fusion:moe_finalize_shared_gate_all_reduce", capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}), - description="BF16 routed finalize, FP32 shared gate, and clustered all-reduce.", ) ) @@ -39,7 +43,7 @@ for _mod, _fn in [ op=f"communication.{_fn}", backend=KernelBackend.JIT, target=f"sglang.kernels.ops.communication.{_mod}:{_fn}", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) del _mod, _fn diff --git a/python/sglang/kernels/ops/communication/inkling_all_reduce.py b/python/sglang/kernels/ops/communication/inkling_all_reduce.py index e7b7af2e73..39a4139dec 100644 --- a/python/sglang/kernels/ops/communication/inkling_all_reduce.py +++ b/python/sglang/kernels/ops/communication/inkling_all_reduce.py @@ -24,7 +24,6 @@ def _jit_inkling_all_reduce_module(dtype: torch.dtype, world_size: int) -> Modul *args, cuda_files=["inkling/inkling_all_reduce.cuh"], cuda_wrappers=[ - ("two_shot_all_reduce", f"inkling_two_shot_all_reduce<{args}>"), ("two_shot_all_reduce_fused", f"inkling_two_shot_all_reduce_fused<{args}>"), ("multimem_one_shot_fused", f"inkling_multimem_one_shot_fused<{args}>"), ("multimem_full_oneshot", f"inkling_multimem_full_oneshot<{args}>"), @@ -143,32 +142,6 @@ def compile_inkling_all_reduce(dtype: torch.dtype, world_size: int) -> None: _jit_inkling_all_reduce_module(dtype, world_size) -def inkling_two_shot_all_reduce( - buffer: torch.Tensor, - peer_ptrs_dev: int, - rank: int, - world_size: int, - num_items: int, -) -> None: - """Two-shot all-reduce in place over ``num_items`` elements of the symm buffer. - - Args: - buffer: this rank's symm buffer (1D, contiguous, bf16), sliced to - ``num_items``; used for device/dtype validation. The producer must - have already written this rank's shard into it. - peer_ptrs_dev: ``hdl.buffer_ptrs_dev`` -- device address of the array of - ``world_size`` peer buffer base pointers. - rank: this rank within the TP group. - world_size: TP world size (compile-time template arg). - num_items: number of elements to reduce (multiple of 8 for bf16). - - The caller is responsible for ``hdl.barrier()`` before (producers done) and - after (result visible) this call. - """ - module = _jit_inkling_all_reduce_module(buffer.dtype, world_size) - module.two_shot_all_reduce(buffer, peer_ptrs_dev, rank, num_items) - - def inkling_two_shot_all_reduce_fused( buffer: torch.Tensor, data_ptrs_dev: int, diff --git a/python/sglang/kernels/ops/communication/mp.py b/python/sglang/kernels/ops/communication/mp.py index 817d158f1e..b8435e33ad 100644 --- a/python/sglang/kernels/ops/communication/mp.py +++ b/python/sglang/kernels/ops/communication/mp.py @@ -1,30 +1,9 @@ -"""Multi-process / multi-GPU launching utilities (torchrun-based). - -Shared `multigpu_launch` helper that both `sglang.test.kernels.utils` and -`sglang.kernels.jit.benchmark.utils` build their domain-specific entry points on -top of (`multigpu_pytest_main`, `multigpu_bench_main`). - -When a script that calls one of those wrappers is run with plain `python`, the -launcher relaunches the same file under `torchrun` once for each `N` in -`num_gpus`. When the inner workers run (identified by an env_key being set), -the same launcher calls `inner()` on every rank, silences stdout on non-zero -ranks, and exits with its return code. -""" +"""Shutdown helper for custom all-reduce communicators.""" from __future__ import annotations import atexit -import logging -import os -import signal -import subprocess -import sys -from typing import Any, Callable, List, NoReturn, Optional, Sequence - -import psutil -import torch - -logger = logging.getLogger(__name__) +from typing import Any def register_comm_cleanup(comm: Any) -> None: @@ -52,166 +31,3 @@ def register_comm_cleanup(comm: Any) -> None: pass atexit.register(_safe_close) - - -def _kill_pgroup(pgid: int) -> None: - try: - os.killpg(pgid, signal.SIGKILL) - except ProcessLookupError: - pass - - -def _kill_descendants(pid: int) -> None: - """Snapshot every descendant of `pid` *now* and SIGKILL them all. - - Must be called BEFORE the direct child (torchrun) dies -- once it does, - its workers get reparented to init and we lose them via the process tree. - """ - try: - root = psutil.Process(pid) - except psutil.NoSuchProcess: - return - descendants = root.children(recursive=True) - for proc in descendants: - try: - proc.kill() - except psutil.Error: - # NoSuchProcess (already gone) or AccessDenied -- nothing to do. - pass - psutil.wait_procs(descendants, timeout=5) - - -def _extract_num_gpus_override( - argv: list[str], -) -> tuple[list[int] | None, list[str]]: - """Pop `--num-gpu(s)` flags out of `argv` and return them separately. - - Accepts `--num-gpu N`, `--num-gpu=N`, `--num-gpus ...`, and comma-separated - lists like `--num-gpu 2,4,8`. May be repeated. - """ - override: list[int] = [] - remaining: list[str] = [] - i = 0 - while i < len(argv): - a = argv[i] - if a in ("--num-gpu", "--num-gpus"): - if i + 1 >= len(argv): - raise ValueError(f"missing value for {a} (expected e.g. `{a} 2,4`)") - override.extend(int(x) for x in argv[i + 1].split(",")) - i += 2 - elif a.startswith("--num-gpu=") or a.startswith("--num-gpus="): - _, val = a.split("=", 1) - override.extend(int(x) for x in val.split(",")) - i += 1 - else: - remaining.append(a) - i += 1 - return (override if override else None), remaining - - -def multigpu_launch( - name: str, - file: str, - num_gpus: Sequence[int], - env_key: str, - inner: Callable[[], int], - kind: str, - pre_launch_fn: Optional[Callable[[List[int]], None]] = None, - timeout: Optional[int] = None, -) -> NoReturn | None: - """Shared torchrun-based launcher. - - See module docstring. `name` is the caller's `__name__`; `file` is its - `__file__`. `env_key` is a unique string per kind (test/benchmark) used to - detect the inside-torchrun state. `inner` returns an exit code. - - `pre_launch_fn`, if given, runs once in the outer process *before* any - torchrun child is spawned. It receives the list of world sizes that will - actually be launched (already filtered against the host's GPU count and - any ``--num-gpu`` override). Use it for parallel JIT precompilation so the - on-disk kernel cache is warm by the time the torchrun children import - their kernels. - - `timeout`, if given, bounds each per-world-size torchrun invocation (in - seconds). On expiry the child's whole process group is killed and the - launcher exits non-zero. `None` (the default) waits indefinitely. - """ - pid_key = env_key + "_PID" - if env_key in os.environ: - assert pid_key in os.environ - if name != "__main__": - return - rank = int(os.environ["LOCAL_RANK"]) - if rank != 0: - sys.stdout = open(os.devnull, "w") - torch.cuda.set_device(rank) - return sys.exit(inner()) - assert pid_key not in os.environ - if name != "__main__": - return logger.warning( - f"{file} can not directly run with `pytest`. " - "Use `python` to invoke it, which will internally relaunch it " - "under torchrun for each requested number of GPUs." - ) - num_devices = torch.cuda.device_count() - override, forwarded_args = _extract_num_gpus_override(sys.argv[1:]) - if override is not None: - logger.info(f"--num-gpu override: running only with {override}") - num_gpus = override - for N in num_gpus: - if N <= 1 or N > num_devices: - raise ValueError( - f"Invalid number of GPUs requested: {N} (available: {num_devices})" - ) - os.environ[env_key] = "1" - os.environ[pid_key] = str(os.getpid()) - os.environ.setdefault("OMP_NUM_THREADS", "1") - os.environ.setdefault("GLOO_SOCKET_IFNAME", "lo") # single-machine setup - # Unbuffered child stdout: when a worker is killed on timeout, pytest's - # block-buffered progress output is otherwise lost or flushed out of - # order into the CI log, making it impossible to tell which test hung. - os.environ.setdefault("PYTHONUNBUFFERED", "1") - signal.signal(signal.SIGINT, signal.default_int_handler) - runnable: List[int] = [] - for N in sorted(num_gpus): - assert N > 1 - if N > num_devices: - logger.warning(f"Skipping {kind} with {N} GPUs ({num_devices} available)") - continue - runnable.append(N) - if pre_launch_fn is not None and runnable: - logger.info(f"Running pre-launch hook for world sizes {runnable}") - pre_launch_fn(runnable) - for N in runnable: - logger.info(f"Running {kind} with {N} GPUs") - cmd = [ - "torchrun", - "--nproc_per_node", - str(N), - "--local-addr", - "127.0.0.1", - file, - ] - cmd += forwarded_args - proc = subprocess.Popen(cmd, start_new_session=True) - pgid = proc.pid - returncode = -1 - timed_out = False - try: - returncode = proc.wait(timeout=timeout) - except subprocess.TimeoutExpired: - timed_out = True - finally: - _kill_descendants(os.getpid()) - _kill_pgroup(pgid) - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - pass - if timed_out: - logger.error(f"{kind} (nproc={N}) timed out after {timeout} seconds") - sys.exit(1) - if returncode != 0: - logger.error(f"{kind} failed with {N} GPUs (exit {returncode})") - sys.exit(returncode) - logger.info(f"All {kind}s passed") diff --git a/python/sglang/kernels/ops/communication/nvlink_comm.py b/python/sglang/kernels/ops/communication/nvlink_comm.py index 6a7a0f4854..609de8b4e3 100644 --- a/python/sglang/kernels/ops/communication/nvlink_comm.py +++ b/python/sglang/kernels/ops/communication/nvlink_comm.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: from sglang.kernels.ops.communication.all_reduce import Communicator -_PRIMITIVES: Final = ["all_reduce", "all_gather", "reduce_scatter"] +_PRIMITIVES: Final = ["all_gather"] def get_multicast_ptr(tensor: torch.Tensor) -> int: @@ -61,17 +61,6 @@ def _jit_push_module(dtype: torch.dtype, world_size: int) -> Module: # `residual` on any of these is folded into the reduction; it may be shaped like # this rank's shard or like the whole tensor, of which this rank's slice is taken. -def all_reduce_push( - comm: Communicator, - input: torch.Tensor, - output: torch.Tensor, - residual: torch.Tensor | None = None, -) -> None: - _jit_push_module(input.dtype, comm.world_size).all_reduce( - comm, input, output, residual - ) - - def all_gather_push( comm: Communicator, input: torch.Tensor, @@ -83,39 +72,6 @@ def all_gather_push( ) -def reduce_scatter_push( - comm: Communicator, - input: torch.Tensor, - output: torch.Tensor, - residual: torch.Tensor | None = None, -) -> None: - _jit_push_module(input.dtype, comm.world_size).reduce_scatter( - comm, input, output, residual - ) - - -def all_reduce_pull( - comm: Communicator, - input: torch.Tensor, - output: torch.Tensor, - residual: torch.Tensor | None = None, - *, - in_mc_ptr: int = 0, - out_mc_ptr: int = 0, - num_unroll=4, - num_blocks_hint: int = 0, -) -> None: - _jit_pull_module(input.dtype, num_unroll).all_reduce( - comm, - input, - output, - residual, - in_mc_ptr or get_multicast_ptr(input), - out_mc_ptr or get_multicast_ptr(output), - num_blocks_hint, - ) - - def all_gather_pull( comm: Communicator, input: torch.Tensor, @@ -134,23 +90,3 @@ def all_gather_pull( out_mc_ptr or get_multicast_ptr(output), num_blocks_hint, ) - - -def reduce_scatter_pull( - comm: Communicator, - input: torch.Tensor, - output: torch.Tensor, - residual: torch.Tensor | None = None, - *, - in_mc_ptr: int = 0, - num_unroll=4, - num_blocks_hint: int = 0, -) -> None: - _jit_pull_module(input.dtype, num_unroll).reduce_scatter( - comm, - input, - output, - residual, - in_mc_ptr or get_multicast_ptr(input), - num_blocks_hint, - ) diff --git a/python/sglang/kernels/ops/communication/sp_collective.py b/python/sglang/kernels/ops/communication/sp_collective.py index 2e269605b3..cc397799b3 100644 --- a/python/sglang/kernels/ops/communication/sp_collective.py +++ b/python/sglang/kernels/ops/communication/sp_collective.py @@ -117,20 +117,6 @@ def get_dispatch( ) -def get_tuning( - kind: str, - world_size: int, - hidden_size: int, - num_tokens: int, - device: torch.device, -) -> Optional[Tuning]: - """Compatibility helper for callers that only support staging push.""" - dispatch = get_dispatch(kind, world_size, hidden_size, num_tokens, device) - if dispatch is None or dispatch.strategy != "push": - return None - return dispatch.tuning - - def get_fusion_dispatch( kind: str, world_size: int, diff --git a/python/sglang/kernels/ops/diffusion/__init__.py b/python/sglang/kernels/ops/diffusion/__init__.py index e4d5ee31ac..bad9abdef9 100644 --- a/python/sglang/kernels/ops/diffusion/__init__.py +++ b/python/sglang/kernels/ops/diffusion/__init__.py @@ -30,12 +30,7 @@ from __future__ import annotations from typing import Any from sglang.kernels.registry import register_kernel -from sglang.kernels.spec import ( - CapabilityRequirement, - FormatSignature, - KernelBackend, - KernelSpec, -) +from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec _CUDA = frozenset({CapabilityRequirement.CUDA}) _CUDA_SM90_PLUS = frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}) @@ -49,654 +44,556 @@ _CUDA_SM10X = frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, # ``scale_residual_norm_scale_shift`` in Triton, CuTe-DSL and FlyDSL) are # inventory: callers name the one they want via ``select_kernel``. # --------------------------------------------------------------------------- -_SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = ( +_SPECS: tuple[tuple[str, KernelBackend, str, frozenset], ...] = ( ( "diffusion.residual_gate_fp32", KernelBackend.TRITON, "modulate.residual_gate_fp32:residual_gate_fp32", _CUDA, - "Residual gating with separate FP32 product and sum rounding.", ), ( "diffusion.matrix_rope", KernelBackend.TRITON, "rope.matrix_rope:apply_matrix_rope", _CUDA, - "Interleaved FP32 matrix RoPE with separate product and sum rounding.", ), ( "diffusion.fp8_rowwise", KernelBackend.TRITON, "quantization.fp8_rowwise_triton:fp8_rowwise", frozenset({CapabilityRequirement.cuda(min_sm=(8, 9))}), - "FLUX.3 dynamic rowwise E4M3 quantization and zero row padding.", ), ( "diffusion.gelu_tanh_cat", KernelBackend.JIT, "activation.gelu_tanh_cat_jit:fused_gelu_tanh_cat", _CUDA, - "BF16 tanh GELU plus channel concatenation.", ), ( "diffusion.rmsnorm_cast_before_weight", KernelBackend.TRITON, "norm.rmsnorm_cast_before_weight_bitexact:rmsnorm_cast_before_weight_bitexact", _CUDA, - "MOSSImage RMSNorm with ATen-ordered reduction and BF16 rounding before gamma.", ), ( "diffusion.rmsnorm_cast_before_weight_modulate", KernelBackend.TRITON, "norm.rmsnorm_cast_before_weight_bitexact:rmsnorm_cast_before_weight_modulate_bitexact", _CUDA, - "MOSSImage ATen-ordered RMSNorm with exact BF16 modulation.", ), ( "diffusion.rmsnorm_cast_before_weight_rope", KernelBackend.TRITON, "norm.rmsnorm_cast_before_weight_bitexact:rmsnorm_cast_before_weight_rope_bitexact", _CUDA, - "MOSSImage ATen-ordered head RMSNorm with exact rotate-half RoPE.", ), ( "diffusion.apply_group_norm_silu", KernelBackend.TRITON, "norm.group_norm_silu:apply_group_norm_silu", _CUDA, - "Fused GroupNorm + SiLU.", ), ( "diffusion.group_norm_silu_4d", KernelBackend.TRITON, "norm.group_norm_silu_twopass_triton:group_norm_silu_4d", _CUDA, - "Channels-last two-pass GroupNorm(+SiLU), 4D.", ), ( "diffusion.group_norm_silu_rows", KernelBackend.TRITON, "norm.group_norm_silu_twopass_triton:group_norm_silu_rows", _CUDA, - "Channels-last two-pass GroupNorm(+SiLU) over (N, L, C) rows.", ), ( "diffusion.wan_rmsnorm_silu", KernelBackend.TRITON, "norm.wan_rmsnorm_silu_triton:wan_rmsnorm_silu", _CUDA, - "Wan VAE channels_last_3d RMSNorm + SiLU.", ), ( "diffusion.wan_norm_silu_post", KernelBackend.JIT, "norm.wan_norm_silu_post:wan_norm_silu_post", _CUDA, - "Wan VAE FP32 normalization post-ops with the native denominator.", ), ( "diffusion.rmsnorm_scale_shift", KernelBackend.TRITON, "norm.rmsnorm_scale_shift_bitexact:fused_rmsnorm_scale_shift_bitexact", _CUDA, - "Bit-exact RMSNorm + adaLN scale/shift.", ), ( "diffusion.scale_residual_norm_scale_shift", KernelBackend.KDA, "sglang.kernels.kda_kernels.norm_scale_shift_jit:kda_scale_residual_norm_scale_shift", _CUDA_SM100_PLUS, - "KDA B200 native CUDA residual + LayerNorm + scale/shift (#27392).", ), ( "diffusion.scale_residual_norm_scale_shift", KernelBackend.TRITON, "norm.rmsnorm_scale_shift_bitexact:fused_scale_residual_rmsnorm_scale_shift_bitexact", _CUDA, - "Bit-exact residual-gate add + RMSNorm + scale/shift.", ), ( "diffusion.scale_residual_norm_scale_shift", KernelBackend.CUTE_DSL, "norm.scale_residual_norm_cutedsl:fused_scale_residual_norm_scale_shift", _CUDA, - "CuTe-DSL residual + norm + scale/shift.", ), ( "diffusion.scale_residual_norm_scale_shift", KernelBackend.FLYDSL, "norm.fused_residual_norm_flydsl:flydsl_fused_residual_norm_scale_shift", _HIP, - "FlyDSL (ROCm gfx950) residual + norm + scale/shift.", ), ( "diffusion.scale_residual_norm_scale_shift_nvfp4", KernelBackend.JIT, "sglang.kernels.kda_kernels.norm_scale_shift_jit:try_fused_scale_residual_norm_scale_shift_nvfp4", _CUDA, - "Qwen residual LayerNorm/modulation + NVFP4 quantization.", ), ( "diffusion.norm_scale_shift", KernelBackend.KDA, "sglang.kernels.kda_kernels.norm_scale_shift_jit:kda_norm_scale_shift", _CUDA_SM100_PLUS, - "KDA B200 native CUDA LayerNorm + scale/shift (#27392).", ), ( "diffusion.norm_scale_shift", KernelBackend.CUTE_DSL, "norm.scale_residual_norm_cutedsl:fused_norm_scale_shift", _CUDA, - "CuTe-DSL norm + scale/shift.", ), ( "diffusion.norm_scale_shift", KernelBackend.FLYDSL, "norm.fused_residual_norm_flydsl:flydsl_norm_scale_shift", _HIP, - "FlyDSL (ROCm gfx950) norm + scale/shift.", ), ( "diffusion.layernorm_modulate", KernelBackend.TRITON, "sglang.kernels.kda_kernels.layernorm_modulate_triton:fused_layernorm_modulate", _CUDA, - "Bit-exact LayerNorm + adaLN modulate.", ), ( "diffusion.qk_head_layernorm", KernelBackend.TRITON, "sglang.kernels.kda_kernels.layernorm_modulate_triton:fused_qk_head_layernorm", _CUDA, - "Bit-exact per-head LayerNorm for q/k.", ), ( "diffusion.qk_rmsnorm_native", KernelBackend.TRITON, "norm.zimage_qk_rmsnorm_triton:zimage_qk_rmsnorm_native", _CUDA, - "Z-Image bf16-native per-head QK RMSNorm.", ), ( "diffusion.rmsnorm_scale", KernelBackend.TRITON, "norm.native_bf16_rmsnorm_triton:rmsnorm_scale", _CUDA, - "BF16-native RMSNorm * scale.", ), ( "diffusion.rmsnorm_tanh_residual", KernelBackend.TRITON, "norm.native_bf16_rmsnorm_triton:rmsnorm_tanh_residual", _CUDA, - "BF16-native x + tanh(gate) * RMSNorm(y).", ), ( "diffusion.modulate_scale_shift", KernelBackend.JIT, "modulate.modulate_scale_shift_jit:modulate_scale_shift", _CUDA, - "Bit-exact adaLN modulate x * (1 + scale) + shift.", ), ( "diffusion.residual_gate_add", KernelBackend.KDA, "sglang.kernels.kda_kernels.residual_gate_add_jit:residual_gate_add", _CUDA, - "KDA native CUDA residual + gate * update (#29361).", ), ( "diffusion.residual_gate_add2", KernelBackend.JIT, "modulate.residual_gate_add2_jit:residual_gate_add2", _CUDA, - "Fused residual + gate * (update_a + update_b) for two-branch blocks.", ), ( "diffusion.flux3_fp8_linear", KernelBackend.FLASHINFER, "gemm.flux3_fp8_prims_ts:flux3_clustered_fp8_linear", _CUDA_SM10X, - "FLUX.3 rowwise-FP8 PrimsTS GEMM.", ), ( "diffusion.flux3_fp8_linear_swiglu", KernelBackend.FLASHINFER, "gemm.flux3_fp8_prims_ts:flux3_fused_mlp_up", _CUDA_SM10X, - "FLUX.3 rowwise-FP8 PrimsTS GEMM with SwiGLU epilogue.", ), ( "diffusion.flux3_fp8_qkv_qknorm_rope", KernelBackend.FLASHINFER, "gemm.flux3_fp8_prims_ts:flux3_clustered_fp8_qkv_qknorm_rope", _CUDA_SM10X, - "FLUX.3 rowwise-FP8 PrimsTS QKV GEMM with QK-norm/RoPE epilogue.", ), ( "diffusion.flux3_nvfp4_linear", KernelBackend.FLASHINFER, "gemm.flux3_nvfp4_prims_ts:flux3_nvfp4_linear", _CUDA_SM10X, - "FLUX.3 NVFP4 PrimsTS GEMM.", ), ( "diffusion.flux3_nvfp4_linear_swiglu", KernelBackend.FLASHINFER, "gemm.flux3_nvfp4_prims_ts:flux3_nvfp4_linear_swiglu", _CUDA_SM10X, - "FLUX.3 NVFP4 PrimsTS GEMM with SwiGLU epilogue.", ), ( "diffusion.flux3_nvfp4_qkv_qknorm_rope", KernelBackend.FLASHINFER, "gemm.flux3_nvfp4_prims_ts:flux3_nvfp4_qkv_qknorm_rope", _CUDA_SM10X, - "FLUX.3 NVFP4 PrimsTS QKV GEMM with QK-norm/RoPE epilogue.", ), ( "diffusion.timestep_embedding", KernelBackend.JIT, "modulate.timestep_embedding_jit:timestep_embedding", _CUDA, - "Sinusoidal timestep embedding.", ), ( "diffusion.temb_table_slices", KernelBackend.TRITON, "modulate.wan_temb_table_slices_triton:fused_temb_table_slices", _CUDA, - "Contiguous adaLN slices for Wan2.2-TI2V.", ), ( "diffusion.ltx2_ada_values", KernelBackend.TRITON, "modulate.ltx2_ada_values_triton:ltx2_ada_values9", _CUDA, - "LTX-2 nine-way adaLN value split.", ), ( "diffusion.fused_inplace_qknorm_rope", KernelBackend.JIT, "rope.qknorm_rope_jit:fused_inplace_qknorm_rope", _CUDA, - "Fused in-place QK RMS-norm + RoPE.", ), ( "diffusion.vsa_block_sparse_sm100", KernelBackend.JIT, "attention.vsa_block_sparse_sm100_jit:vsa_block_sparse_sm100", frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 3))}), - "FastVideo's warp-specialized tcgen05 block-sparse VSA forward (64-token tiles).", ), ( "diffusion.h3_vae_rmsnorm", KernelBackend.TRITON, "norm.h3_vae_fused_norm:h3_vae_rmsnorm", _CUDA, - "FP32-accumulation RMSNorm for the MiniMax-H3 video VAE decoder.", ), ( "diffusion.h3_vae_scale_add_rmsnorm", KernelBackend.TRITON, "norm.h3_vae_fused_norm:h3_vae_scale_add_rmsnorm", _CUDA, - "Scaled residual add plus the following RMSNorm for the H3 VAE decoder.", - ), - ( - "diffusion.h3_vae_qk_rmsnorm_rope", - KernelBackend.TRITON, - "norm.h3_vae_fused_norm:h3_vae_qk_rmsnorm_rope", - _CUDA, - "Affine-free Q/K RMSNorm plus partial NeoX RoPE for the H3 VAE decoder.", ), ( "diffusion.flux2_layernorm_modulate_fp8_quant", KernelBackend.KDA, "sglang.kernels.kda_kernels.layernorm_modulate_triton:fused_layernorm_modulate_fp8_quant_raw", _CUDA, - "KDA-generated FLUX.2 LayerNorm + adaLN modulation + static FP8 quantization.", ), ( "diffusion.flux2_qkv_epilogue", KernelBackend.KDA, "sglang.kernels.kda_kernels.flux2_qkv_epilogue_jit:try_fused_flux2_qkv_epilogue", _CUDA, - "KDA-generated FLUX.2 QK RMS-norm + RoPE + joint QKV packing.", ), ( "diffusion.flux2_token_cat_fp8", KernelBackend.KDA, "sglang.kernels.kda_kernels.flux2_token_cat_fp8_triton:try_flux2_token_cat_fp8", _CUDA, - "KDA-generated FLUX.2 token concatenation + static FP8 quantization.", ), ( "diffusion.qwen_qkv_epilogue", KernelBackend.JIT, "rope.qwen_qkv_epilogue_jit:try_fused_qwen_qkv_epilogue", _CUDA_SM90_PLUS, - "Qwen-Image QK RMS-norm, RoPE, and joint QKV writes.", ), ( "diffusion.ltx2_qknorm_split_rope", KernelBackend.KDA, "sglang.kernels.kda_kernels.ltx2_qknorm_split_rope_jit:ltx2_qknorm_split_rope_cuda", _CUDA_SM100_PLUS, - "KDA native CUDA LTX-2 QK-norm + split RoPE (#29708).", ), ( "diffusion.ltx25_decoder_rope", KernelBackend.JIT, "rope.ltx25_decoder_rope_jit:fused_ltx25_decoder_rope", _CUDA, - "Paired LTX-2.5 decoder 3D RoPE.", ), ( "diffusion.rope_rotate_half_fp32", KernelBackend.TRITON, "rope.rope_rotate_half_fp32:fused_rope_rotate_half_fp32", _CUDA, - "Paired split-half RoPE with FP32 arithmetic and one output cast.", ), ( "diffusion.rope_rotate_half", KernelBackend.TRITON, "rope.rope_rotate_half_bitexact:fused_rope_rotate_half_bitexact", _CUDA, - "Bit-exact rotate-half RoPE.", ), ( "diffusion.interleaved_rope_fp64", KernelBackend.JIT, "rope.interleaved_rope_fp64_jit:fused_interleaved_rope_fp64", _CUDA, - "Paired interleaved RoPE with fp64 Diffusers semantics.", ), ( "diffusion.helios_qk_rope", KernelBackend.JIT, "rope.helios_qk_rope_jit:fused_inplace_helios_qk_rope", _CUDA, - "Paired in-place Helios transposed Q/K RoPE.", ), ( "diffusion.complex_rope", KernelBackend.TRITON, "rope.complex_rope_triton:fused_complex_rope", _CUDA, - "Paired RoPE preserving PyTorch complex64 multiplication rounding.", ), ( "diffusion.hunyuan_qkv_rope_pack", KernelBackend.TRITON, "rope.hunyuan_qkv_pack_triton:hunyuan_qkv_rope_pack", _CUDA, - "HunyuanVideo QKV pack + RoPE.", ), ( "diffusion.joint_qkv_cat", KernelBackend.TRITON, "layout.joint_qkv_cat_triton:joint_qkv_cat", _CUDA, - "Concatenate image/text QKV views into joint attention inputs.", ), ( "diffusion.rmsnorm_preserve_reduction", KernelBackend.TRITON, "norm.rmsnorm_preserve_reduction:rmsnorm_preserve_reduction", _CUDA, - "Cast-before-weight RMSNorm preserving the native FP32 mean reduction.", ), ( "diffusion.silu_mul", KernelBackend.TRITON, "activation.silu_mul_bitexact:fused_silu_mul_bitexact", _CUDA, - "Bit-exact silu(a) * b for split-projection SwiGLU.", ), ( "diffusion.situ_softcap_mul", KernelBackend.TRITON, "activation.situ_softcap_mul_bitexact:fused_situ_softcap_mul_bitexact", _CUDA, - "Bit-exact bounded SiTU for MOSSImage FFN projections.", ), ( "diffusion.bias_silu", KernelBackend.TRITON, "activation.sana_conv_post_triton:fused_bias_silu", _CUDA, - "Bit-exact conv bias + SiLU (Sana GLUMB).", ), ( "diffusion.bias_glu", KernelBackend.TRITON, "activation.sana_conv_post_triton:fused_bias_glu", _CUDA, - "Bit-exact conv bias + GLU (Sana GLUMB).", ), ( "diffusion.linear_gelu_tanh", KernelBackend.AOT, "sites.fused_linear_gelu_site:fused_linear_gelu_tanh", _CUDA, - "Linear + tanh-GELU via the cublasLt epilogue.", ), ( "diffusion.sparse_linear_attn_fwd", KernelBackend.TRITON, "attention.sparse_linear_attn_triton:_attn_fwd", _CUDA, - "Sparse linear attention forward.", ), ( "diffusion.bigdn", KernelBackend.TRITON, "attention.sana_wm_gdn_triton:fused_bigdn_func", _CUDA, - "Sana-WM bidirectional gated delta-net.", ), ( "diffusion.fused_qknorm_rope_out_of_place", KernelBackend.JIT, "rope.qknorm_rope_jit:fused_qknorm_rope_out_of_place", _CUDA, - "Out-of-place fused QK-norm + RoPE (raw q/k preserved).", ), ( "diffusion.qknorm_complex_rope_cuda", KernelBackend.JIT, "rope.qknorm_complex_rope_jit:qknorm_complex_rope_cuda", _CUDA, - "Bit-exact 128-wide RMSNorm + complex RoPE (Qwen-Image 2.1), CUDA.", ), ( "diffusion.qknorm_complex_rope_pack_", KernelBackend.JIT, "rope.qknorm_complex_rope_jit:qknorm_complex_rope_pack_", _CUDA, - "Q/K RMSNorm + complex RoPE with prefix K/V packing into [B, P+S, H, 128] buffers, CUDA.", ), ( "diffusion.channel_rmsnorm_finish_silu", KernelBackend.JIT, "norm.channel_rmsnorm_finish_silu_jit:channel_rmsnorm_finish_silu", _CUDA, - "Bit-exact channel-first RMSNorm finish + SiLU on aten's fp32 channel norm (Qwen-Image 2.1 VAE), CUDA.", ), ( "diffusion.channel_rmsnorm_silu_nhwc", KernelBackend.JIT, "norm.channel_rmsnorm_silu_nhwc_jit:channel_rmsnorm_silu_nhwc", _CUDA, - "Channels-last channel RMSNorm + SiLU with an in-warp reduction (Qwen-Image 2.1 VAE, quality-gated), CUDA.", ), ( "diffusion.bias_residual_add", KernelBackend.JIT, "norm.bias_residual_add_jit:bias_residual_add", _CUDA, - "Bit-exact conv bias + residual add in one pass for channels_last or NCHW bf16 tensors, CUDA.", ), ( "diffusion.vdn_delta_factors", KernelBackend.JIT, "attention.vdn_delta_factors_jit:vdn_delta_factors", _CUDA, - "VDN-H3 delta rule: fused (I + A)^-1 -> transition / injection (fp32, head_dim 128).", ), ( "diffusion.vdn_temporal_conv_act", KernelBackend.TRITON, "attention.vdn_linear_branch_triton:vdn_temporal_conv_act", _CUDA, - "VDN-H3 linear branch: 5-tap temporal conv + SiLU + L2 norm.", ), ( "diffusion.vdn_silu_l2norm", KernelBackend.TRITON, "attention.vdn_linear_branch_triton:vdn_silu_l2norm", _CUDA, - "VDN-H3 linear branch: SiLU + L2 norm over head_dim.", ), ( "diffusion.vdn_frame_stats_prep", KernelBackend.TRITON, "attention.vdn_linear_branch_triton:vdn_frame_stats_prep", _CUDA, - "VDN-H3 linear branch: frame-statistics GEMM operands in one pass.", ), ( "diffusion.vdn_gather_linear_state", KernelBackend.TRITON, "attention.vdn_linear_branch_triton:vdn_gather_linear_state", _CUDA, - "VDN-H3 linear branch: alpha-bridged boundary gather in one pass.", ), ( "diffusion.vdn_linear_epilogue", KernelBackend.TRITON, "attention.vdn_linear_branch_triton:vdn_linear_epilogue", _CUDA, - "VDN-H3 linear branch: RMSNorm * gate readout epilogue.", ), ( "diffusion.mxfp8_quantize_swizzled", KernelBackend.TRITON, "sglang.kernels.ops.quantization.mxfp8_swizzled_triton:mxfp8_quantize_swizzled", _CUDA, - "bf16 -> MXFP8 (e4m3, block-32 E8M0 scales in the cuBLASLt swizzled layout).", ), ( "diffusion.silu_mul_mxfp8", KernelBackend.TRITON, "sglang.kernels.ops.quantization.mxfp8_swizzled_triton:silu_mul_mxfp8", _CUDA, - "SwiGLU + MXFP8 quant for the online mxfp8 fc2 input.", ), ( "diffusion.indexed_scale_shift_mxfp8_", KernelBackend.TRITON, "sglang.kernels.ops.quantization.mxfp8_swizzled_triton:indexed_scale_shift_mxfp8_", _CUDA, - "Indexed adaLN modulation + MXFP8 quant for the online mxfp8 qkv/fc1 inputs.", ), ( "diffusion.group_limited_topk", KernelBackend.TRITON, "routing.group_limited_topk_triton:group_limited_topk", _CUDA, - "LingBot Video group-limited MoE top-k expert selection.", ), ( "diffusion.usp_merge_heads", KernelBackend.JIT, "layout.usp_relayout_jit:usp_merge_heads", _CUDA, - "USP all-to-all output head merge.", ), ( "diffusion.pack_qkv_destination_major", KernelBackend.TRITON, "layout.ulysses_qkv_triton:pack_qkv_destination_major", _CUDA, - "Ulysses destination-major QKV pack.", ), ( "diffusion.varlen_pack_qkv", KernelBackend.TRITON, "layout.varlen_pack_pad_triton:fused_pack_qkv", _CUDA, - "Varlen gather of Q/K/V at valid positions.", ), ( "diffusion.varlen_pack_segmented_qkv", KernelBackend.TRITON, "layout.varlen_pack_pad_triton:fused_pack_segmented_qkv", _CUDA, - "Varlen gather from a virtual prefix/main Q/K/V sequence.", ), ( "diffusion.varlen_scatter_to_padded", KernelBackend.TRITON, "layout.varlen_pack_pad_triton:fused_scatter_to_padded", _CUDA, - "Varlen scatter back to the dense layout.", ), ( "diffusion.causal_conv3d_cat_pad", KernelBackend.KDA, "sglang.kernels.kda_kernels.causal_conv3d_cat_pad_jit:fused_causal_conv3d_cat_pad_cuda", _CUDA, - "KDA native CUDA causal Conv3d cat + pad (#29281).", ), ( "diffusion.causal_conv3d_cat_pad", KernelBackend.TRITON, "layout.causal_conv3d_cat_pad_triton:fused_causal_conv3d_cat_pad", _CUDA, - "Causal Conv3d cat + pad (Triton).", ), ( "diffusion.cat_pad_channels_last_3d", KernelBackend.TRITON, "layout.wan_causal_cache_triton:cat_pad_channels_last_3d", _CUDA, - "Wan causal VAE cat + pad in channels_last_3d.", ), ( "diffusion.dup_up3d_add", KernelBackend.TRITON, "layout.wan_causal_cache_triton:dup_up3d_add", _CUDA, - "Wan causal VAE main + DupUp3D(src).", ), ( "diffusion.conv_bias_epilogue", KernelBackend.TRITON, "modulate.conv_bias_epilogue_triton:conv_bias_epilogue", _CUDA, - "Bit-exact conv bias (+ residual) epilogue on channels_last outputs.", ), ( "diffusion.nearest_upsample_nhwc", KernelBackend.TRITON, "layout.nearest_upsample_nhwc_triton:nearest_upsample_nhwc", _CUDA, - "Wan-family VAE channels_last integer-factor nearest upsample.", ), ( "diffusion.flux2_token_cat_nvfp4", KernelBackend.JIT, "layout.flux2_token_cat_nvfp4_jit:try_flux2_token_cat_nvfp4", _CUDA, - "FLUX.2 single-block token concatenation + NVFP4 quantization.", ), ) -for _op, _backend, _target, _caps, _description in _SPECS: +for _op, _backend, _target, _caps in _SPECS: register_kernel( KernelSpec( op=_op, @@ -707,8 +604,6 @@ for _op, _backend, _target, _caps, _description in _SPECS: else f"sglang.kernels.ops.diffusion.{_target}" ), capabilities=_caps, - format_signature=FormatSignature(description=_description), - description=_description, ) ) @@ -786,6 +681,12 @@ _EXPORTS: dict[str, str] = { "wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton", "can_use_qk_rmsnorm_native": "norm.zimage_qk_rmsnorm_triton", "zimage_qk_rmsnorm_native": "norm.zimage_qk_rmsnorm_triton", + "can_use_rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton", + "can_use_rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton", + "rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton", + "rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton", + "h3_vae_rmsnorm": "norm.h3_vae_fused_norm", + "h3_vae_scale_add_rmsnorm": "norm.h3_vae_fused_norm", # adaLN modulation, gating and timestep conditioning "indexed_gate_bf16": "modulate.indexed_modulation_triton", "indexed_gate_bf16_": "modulate.indexed_modulation_triton", @@ -906,80 +807,31 @@ _EXPORTS: dict[str, str] = { "BitExactFusionGate": "sites.bitexact_gate", "flashinfer_rmsnorm_diagnostic_hint": "sites.bitexact_gate", "tensors_equal": "sites.bitexact_gate", - "fused_gate_rmsnorm_active": "sites.fused_gate_rmsnorm_site", + "FLUX2_NVFP4_SWIGLU_QUANT": "sites.flux2_nvfp4_swiglu_quant_site", + "FUSED_GATE_RMSNORM": "sites.fused_gate_rmsnorm_site", "fused_rmsnorm_scale": "sites.fused_gate_rmsnorm_site", "fused_rmsnorm_tanh_residual": "sites.fused_gate_rmsnorm_site", - "mark_fused_gate_rmsnorm_site": "sites.fused_gate_rmsnorm_site", - "mount_fused_gate_rmsnorm": "sites.fused_gate_rmsnorm_site", - "unmount_fused_gate_rmsnorm": "sites.fused_gate_rmsnorm_site", + "FUSED_LINEAR_GELU": "sites.fused_linear_gelu_site", "can_use_linear_gelu": "sites.fused_linear_gelu_site", - "fused_gelu_active": "sites.fused_linear_gelu_site", "fused_linear_gelu_tanh": "sites.fused_linear_gelu_site", - "mark_fused_gelu_site": "sites.fused_linear_gelu_site", - "mount_fused_linear_gelu": "sites.fused_linear_gelu_site", - "unmount_fused_linear_gelu": "sites.fused_linear_gelu_site", - "flux2_nvfp4_swiglu_quant_active": "sites.flux2_nvfp4_swiglu_quant_site", - "mark_flux2_nvfp4_swiglu_quant_site": "sites.flux2_nvfp4_swiglu_quant_site", - "mount_flux2_nvfp4_swiglu_quant": "sites.flux2_nvfp4_swiglu_quant_site", - "unmount_flux2_nvfp4_swiglu_quant": "sites.flux2_nvfp4_swiglu_quant_site", - "mark_nvfp4_bias_gelu_site": "sites.nvfp4_bias_gelu_site", - "mount_nvfp4_bias_gelu": "sites.nvfp4_bias_gelu_site", - "nvfp4_bias_gelu_active": "sites.nvfp4_bias_gelu_site", - "unmount_nvfp4_bias_gelu": "sites.nvfp4_bias_gelu_site", - "mark_qwen_image_added_qkv_site": "sites.qwen_image_added_qkv_site", - "mount_qwen_image_added_qkv": "sites.qwen_image_added_qkv_site", - "qwen_image_added_qkv_active": "sites.qwen_image_added_qkv_site", - "unmount_qwen_image_added_qkv": "sites.qwen_image_added_qkv_site", + "FUSED_LN_MODULATE": "sites.fused_ln_modulate_site", "can_use_ln_modulate": "sites.fused_ln_modulate_site", "fused_ln_modulate": "sites.fused_ln_modulate_site", - "fused_ln_modulate_active": "sites.fused_ln_modulate_site", - "mark_fused_ln_modulate_site": "sites.fused_ln_modulate_site", - "mount_fused_ln_modulate": "sites.fused_ln_modulate_site", - "unmount_fused_ln_modulate": "sites.fused_ln_modulate_site", - "mark_hunyuan_qknorm_site": "sites.hunyuan_qknorm_site", - "mount_hunyuan_qknorm": "sites.hunyuan_qknorm_site", + "HELIOS_GATED_RESIDUAL": "sites.gated_residual_site", + "LINGBOT_VIDEO_GATED_RESIDUAL": "sites.gated_residual_site", + "try_gated_residual": "sites.gated_residual_site", + "HUNYUAN_QKNORM": "sites.hunyuan_qknorm_site", "try_hunyuan_qknorm": "sites.hunyuan_qknorm_site", - "unmount_hunyuan_qknorm": "sites.hunyuan_qknorm_site", - "fused_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site", - "ltx2_rms_norm_modulate_active": "sites.ltx2_rmsnorm_modulate_site", - "mark_ltx2_rms_norm_modulate_site": "sites.ltx2_rmsnorm_modulate_site", - "mount_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site", - "unmount_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site", - "ltx2_qknorm_split_rope_active": "sites.ltx2_qknorm_split_rope_site", - "mark_ltx2_qknorm_split_rope_site": "sites.ltx2_qknorm_split_rope_site", - "mount_ltx2_qknorm_split_rope": "sites.ltx2_qknorm_split_rope_site", - "unmount_ltx2_qknorm_split_rope": "sites.ltx2_qknorm_split_rope_site", - "lingbot_video_rmsnorm_active": "sites.lingbot_video_rmsnorm_site", - "mark_lingbot_video_rmsnorm_site": "sites.lingbot_video_rmsnorm_site", - "mount_lingbot_video_rmsnorm": "sites.lingbot_video_rmsnorm_site", + "LINGBOT_VIDEO_RMSNORM": "sites.lingbot_video_rmsnorm_site", "try_lingbot_video_rmsnorm": "sites.lingbot_video_rmsnorm_site", - "unmount_lingbot_video_rmsnorm": "sites.lingbot_video_rmsnorm_site", - "helios_gated_residual_active": "sites.helios_gated_residual_site", - "mark_helios_gated_residual_site": "sites.helios_gated_residual_site", - "mount_helios_gated_residual": "sites.helios_gated_residual_site", - "try_helios_gated_residual": "sites.helios_gated_residual_site", - "unmount_helios_gated_residual": "sites.helios_gated_residual_site", - "lingbot_video_gated_residual_active": "sites.lingbot_video_gated_residual_site", - "mark_lingbot_video_gated_residual_site": "sites.lingbot_video_gated_residual_site", - "mount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site", - "try_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site", - "unmount_lingbot_video_gated_residual": "sites.lingbot_video_gated_residual_site", - "mark_minimax_h3_norm_modulate_site": "sites.minimax_h3_norm_modulate_site", - "minimax_h3_norm_modulate_active": "sites.minimax_h3_norm_modulate_site", - "mount_minimax_h3_norm_modulate": "sites.minimax_h3_norm_modulate_site", - "unmount_minimax_h3_norm_modulate": "sites.minimax_h3_norm_modulate_site", - "can_use_rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton", - "can_use_rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton", - "rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton", - "rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton", - "h3_vae_rmsnorm": "norm.h3_vae_fused_norm", - "h3_vae_scale_add_rmsnorm": "norm.h3_vae_fused_norm", - "h3_vae_qk_rmsnorm_rope": "norm.h3_vae_fused_norm", - "mark_sana_video_linear_attention_site": "sites.sana_video_linear_attention_site", - "mount_sana_video_linear_attention": "sites.sana_video_linear_attention_site", - "sana_video_linear_attention_active": "sites.sana_video_linear_attention_site", + "LTX2_QKNORM_SPLIT_ROPE": "sites.ltx2_qknorm_split_rope_site", + "LTX2_RMS_NORM_MODULATE": "sites.ltx2_rmsnorm_modulate_site", + "fused_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site", + "MINIMAX_H3_NORM_MODULATE": "sites.minimax_h3_norm_modulate_site", + "NVFP4_BIAS_GELU": "sites.nvfp4_bias_gelu_site", + "QWEN_IMAGE_ADDED_QKV": "sites.qwen_image_added_qkv_site", + "SANA_VIDEO_LINEAR_ATTENTION": "sites.sana_video_linear_attention_site", "try_sana_video_linear_attention": "sites.sana_video_linear_attention_site", - "unmount_sana_video_linear_attention": "sites.sana_video_linear_attention_site", "QualityGatedFusion": "sites.quality_gate", # JIT C++/CUDA extensions (not kernels, not in the registry) "interpolate": "ext.hunyuan3d_rasterizer", diff --git a/python/sglang/kernels/ops/diffusion/attention/sana_wm_gdn_chunkwise_triton.py b/python/sglang/kernels/ops/diffusion/attention/sana_wm_gdn_chunkwise_triton.py index 1272fae4a9..0ada35897b 100644 --- a/python/sglang/kernels/ops/diffusion/attention/sana_wm_gdn_chunkwise_triton.py +++ b/python/sglang/kernels/ops/diffusion/attention/sana_wm_gdn_chunkwise_triton.py @@ -166,7 +166,7 @@ _CHUNKWISE_TUNING: dict[tuple[str, str], _ChunkwiseCfg] = { # nw=16 OOMs the 102 KB SRAM cap at TF32 (needs 131 KB); nw=8 fits and is # within noise. The D-tile path (auto-enabled on spark, see # `_pick_phase_b_d_splits`) is ~2.6× faster at TF32 / ~13% at IEEE — these - # baseline params only apply when PHASE_B_D_SPLITS=1 is forced. + # baseline params only apply where that picks d_splits=1. B=_PhaseCfg(nw=8, use_acc=False, ns=1), C=_PhaseCfg(nw=8, BS=16, ns=1), # binding constraint: M.fp32 64 KB + Q stage ), @@ -1206,20 +1206,7 @@ def _pick_phase_b_d_splits(BLOCK_D: int, dot_precision: int = 0): `d_splits>1` → `_phase_b_dtile_kernel` with nw/ns/acc overrides. Per-arch optima from a 96-config sweep (2026-04-29, T=11 B=1 IEEE) and a multi-arch bf16/TF32 sweep (2026-05-06, F=11 S=920); see inline cfg comments. - Env overrides: PHASE_B_D_SPLITS, PHASE_B_DTILE_NW/NS/ACC (1=True / 0=False). """ - import os - - env_d = os.environ.get("PHASE_B_D_SPLITS", None) - if env_d is not None: - d = int(env_d) - if d < 1 or BLOCK_D % d != 0: - return (1, None, None, None) - nw = int(os.environ.get("PHASE_B_DTILE_NW", "0")) or None - ns = int(os.environ.get("PHASE_B_DTILE_NS", "0")) or None - acc_env = os.environ.get("PHASE_B_DTILE_ACC", None) - acc = bool(int(acc_env)) if acc_env is not None else None - return (d, nw, ns, acc) try: import torch @@ -1617,18 +1604,6 @@ def fused_bigdn_bidi_chunkwise( return out -def _default_dot_prec() -> int: - try: - from sglang.kernels.ops.diffusion.attention.sana_wm_gdn_triton import ( - _resolve_launch_config, - ) - - _, dot_prec, _, _ = _resolve_launch_config() - return dot_prec - except Exception: - return 0 - - def _cam_identity_tables( *, B: int, @@ -1683,7 +1658,7 @@ def cam_scan_bidi_chunkwise( assert decay.shape == (B, H, F) if dot_precision is None: - dot_precision = _default_dot_prec() + dot_precision = 0 qkv = torch.empty(B, N, 3, H, D, device=q.device, dtype=q.dtype) qkv[:, :, 0].copy_(q.permute(0, 3, 1, 2)) diff --git a/python/sglang/kernels/ops/diffusion/attention/sana_wm_gdn_triton.py b/python/sglang/kernels/ops/diffusion/attention/sana_wm_gdn_triton.py index e7873d852f..1848981bb0 100644 --- a/python/sglang/kernels/ops/diffusion/attention/sana_wm_gdn_triton.py +++ b/python/sglang/kernels/ops/diffusion/attention/sana_wm_gdn_triton.py @@ -14,85 +14,16 @@ # # SPDX-License-Identifier: Apache-2.0 -"""Inference-side helpers for the bidirectional fused GDN path. - -Precision knob: env var ``FUSED_GDN_PRECISION`` or ``PRECISION_OVERRIDE``: - 0=IEEE fp32 dots, 1=TF32, 2=bf16 TC + fp32 state [default], 3=bf16 TC + bf16 state. -""" +"""Inference-side helpers for the bidirectional fused GDN path.""" # ruff: noqa: E501 from __future__ import annotations -import os - import torch import triton import triton.language as tl -# ===================================================================== -# GPU-adaptive kernel config -# ===================================================================== - - -def _get_kernel_config() -> dict: - """Return optimal kernel parameters for the current GPU. - - STATE_FP32 (fp32 state_prev) needs ~128KB SRAM (H100 228KB), vs ~96KB for - bf16 state_prev (fits GB10's 101KB). - """ - if not torch.cuda.is_available(): - return {"BLOCK_S": 64, "num_stages": 1, "num_warps": 4, "STATE_FP32": False} - smem = torch.cuda.get_device_properties(0).shared_memory_per_multiprocessor - state_fp32 = smem >= 150 * 1024 # H100 (228KB) yes, GB10 (101KB) no - return {"BLOCK_S": 64, "num_stages": 1, "num_warps": 8, "STATE_FP32": state_fp32} - - -_KCFG = None - - -def _kcfg(): - global _KCFG - if _KCFG is None: - _KCFG = _get_kernel_config() - return _KCFG - - -# precision=0 → IEEE fp32 dots + fp32 state (DOT_PRECISION=2, STATE_FP32=1) -# precision=1 → TF32 dots + fp32 state (DOT_PRECISION=1, STATE_FP32=1) -# precision=2 → bf16 dots + fp32 state (DOT_PRECISION=0, STATE_FP32=1) [default] -# precision=3 → bf16 dots + bf16 state (DOT_PRECISION=0, STATE_FP32=0) -def _precision_params(precision: int) -> tuple: - if precision == 0: - return 2, True - elif precision == 1: - return 1, True - elif precision == 3: - return 0, False - else: # default - return 0, True - - -_env_prec = os.environ.get("FUSED_GDN_PRECISION", None) -PRECISION_OVERRIDE: int | None = int(_env_prec) if _env_prec is not None else None - - -def _resolve_launch_config() -> tuple: - """Returns (prec, dot_prec, state_fp32, num_warps). - - Uses ``PRECISION_OVERRIDE`` when set, else ``_kcfg()`` (per-GPU SRAM). - num_warps clamped to 4 when dots run on fp32 operands (more registers). - """ - cfg = _kcfg() - prec = PRECISION_OVERRIDE if PRECISION_OVERRIDE is not None else 2 - dot_prec, state_fp32 = _precision_params(prec) - if PRECISION_OVERRIDE is None: - state_fp32 = cfg["STATE_FP32"] - nw = cfg["num_warps"] - if dot_prec >= 1: - nw = min(nw, 4) - return prec, dot_prec, state_fp32, nw - def prepare_rope_tables( rotary_emb, N: int, D: int, device diff --git a/python/sglang/kernels/ops/diffusion/common/fast_launch.py b/python/sglang/kernels/ops/diffusion/common/fast_launch.py index 2aa8590fc8..18642f8d81 100644 --- a/python/sglang/kernels/ops/diffusion/common/fast_launch.py +++ b/python/sglang/kernels/ops/diffusion/common/fast_launch.py @@ -33,11 +33,16 @@ from typing import Any, Callable import torch from torch.utils._python_dispatch import TorchDispatchMode from torch.utils._pytree import tree_leaves -from triton import knobs -from triton.knobs import HookChain from triton.runtime import driver from triton.runtime.jit import JITFunction +try: + from triton import knobs + from triton.knobs import HookChain +except ImportError: + # Vendor Triton builds may not expose the upstream launch-hook API. + knobs = None + logger = logging.getLogger(__name__) _ENABLED = os.environ.get( @@ -162,6 +167,9 @@ class _Plan: def _launch_hooked() -> bool: """A profiler such as proton hooks Triton launches; those go through the runner.""" + if knobs is None: + # Keep the regular runner when its hooks cannot be inspected safely. + return True return any( hook is not None and not (isinstance(hook, HookChain) and not hook.calls) for hook in (knobs.runtime.launch_enter_hook, knobs.runtime.launch_exit_hook) diff --git a/python/sglang/kernels/ops/diffusion/common/platform.py b/python/sglang/kernels/ops/diffusion/common/platform.py index be476c713a..4a32a430e9 100644 --- a/python/sglang/kernels/ops/diffusion/common/platform.py +++ b/python/sglang/kernels/ops/diffusion/common/platform.py @@ -27,8 +27,6 @@ from typing import Callable, TypeVar F = TypeVar("F", bound=Callable) -_CUDA_LIKE = frozenset({"cuda", "hip"}) - def platform_key() -> str: """Return the live device family: ``cuda``/``hip``/``npu``/``mps``/``musa``/``xpu``/``cpu``. @@ -72,11 +70,6 @@ def is_xpu() -> bool: return current_platform.is_xpu() -def has_triton() -> bool: - """True when the live device runs the Triton implementations.""" - return platform_key() in _CUDA_LIKE - - def lazy_fallback(kind: str, name: str) -> Callable: """Name a fallback without importing its module. diff --git a/python/sglang/kernels/ops/diffusion/norm/h3_vae_fused_norm.py b/python/sglang/kernels/ops/diffusion/norm/h3_vae_fused_norm.py index 46664de81b..7e77d2d0df 100644 --- a/python/sglang/kernels/ops/diffusion/norm/h3_vae_fused_norm.py +++ b/python/sglang/kernels/ops/diffusion/norm/h3_vae_fused_norm.py @@ -7,9 +7,6 @@ promotes the residual to FP32, so later layers carry an FP32 residual and an FP16 attention/FFN update. ``h3_vae_scale_add_rmsnorm`` does the add and the next RMSNorm in one launch: the sum stays in registers for the variance, and the stored residual matches the eager FP32 add. - -``h3_vae_qk_rmsnorm_rope`` is the affine-free head-64 path: RMSNorm, round to -the activation dtype, then partial NeoX RoPE on the first 48 channels. """ from __future__ import annotations @@ -79,59 +76,6 @@ def _scale_add_rmsnorm_kernel( tl.store(out_ptr + row * D + cols, acc * rstd * weight, mask=mask) -@triton.jit -def _qk_rmsnorm_rope_kernel( - q_ptr, - k_ptr, - qo_ptr, - ko_ptr, - cos_ptr, - sin_ptr, - stride_qb, - stride_qs, - stride_qh, - stride_kb, - stride_ks, - stride_kh, - seq_len, - n_heads, - eps, -): - row = tl.program_id(0).to(tl.int64) - heads = n_heads - token = row // heads - head = row % heads - seq = token % seq_len - batch = token // seq_len - cols = tl.arange(0, 64) - rope_mask = cols < 48 - q_base = q_ptr + batch * stride_qb + seq * stride_qs + head * stride_qh - k_base = k_ptr + batch * stride_kb + seq * stride_ks + head * stride_kh - q = tl.load(q_base + cols).to(tl.float32) - k = tl.load(k_base + cols).to(tl.float32) - q = (q * tl.rsqrt(tl.sum(q * q, axis=0) / 64 + eps)).to(tl.float16) - k = (k * tl.rsqrt(tl.sum(k * k, axis=0) / 64 + eps)).to(tl.float16) - cos = tl.load(cos_ptr + seq * 48 + cols, mask=rope_mask, other=0.0).to(tl.float16) - sin = tl.load(sin_ptr + seq * 48 + cols, mask=rope_mask, other=0.0).to(tl.float16) - # NeoX rotate-half: [-x2, x1] on the first 48 channels. - src = tl.where(cols < 24, cols + 24, cols - 24) - sign = tl.where(cols < 24, -1.0, 1.0) - q_rot = (tl.gather(q.to(tl.float32), src, 0) * sign).to(tl.float16) - k_rot = (tl.gather(k.to(tl.float32), src, 0) * sign).to(tl.float16) - yq = ( - (q.to(tl.float32) * cos.to(tl.float32)).to(tl.float16).to(tl.float32) - + (q_rot.to(tl.float32) * sin.to(tl.float32)).to(tl.float16).to(tl.float32) - ).to(tl.float16) - yk = ( - (k.to(tl.float32) * cos.to(tl.float32)).to(tl.float16).to(tl.float32) - + (k_rot.to(tl.float32) * sin.to(tl.float32)).to(tl.float16).to(tl.float32) - ).to(tl.float16) - yq = tl.where(rope_mask, yq, q) - yk = tl.where(rope_mask, yk, k) - tl.store(qo_ptr + row * 64 + cols, yq) - tl.store(ko_ptr + row * 64 + cols, yk) - - def _out_kind(dtype: torch.dtype) -> int: if dtype == torch.float32: return 0 @@ -239,61 +183,3 @@ def h3_vae_scale_add_rmsnorm( if inplace: return residual, normed.view(out_shape) return residual_out, normed.view(out_shape) - - -def h3_vae_qk_rmsnorm_rope( - query: torch.Tensor, - key: torch.Tensor, - eps: float, - cos: torch.Tensor, - sin: torch.Tensor, -) -> tuple[torch.Tensor, torch.Tensor] | None: - """Affine-free RMSNorm plus partial NeoX RoPE for head 64 / rotary 48. - - Returns None when the tensors are outside that contract so the caller can - keep the unfused norm and RoPE. - """ - if ( - not query.is_cuda - or query.shape != key.shape - or query.dtype != key.dtype - or query.dtype != torch.float16 - or query.dim() != 4 - or query.shape[-1] != 64 - or query.stride(-1) != 1 - or key.stride(-1) != 1 - or cos.shape != sin.shape - or cos.dim() != 2 - or cos.shape[-1] != 48 - or cos.shape[0] != query.shape[1] - or cos.device != query.device - or sin.device != query.device - or key.device != query.device - ): - return None - batch, seq_len, n_heads, _ = query.shape - rows = batch * seq_len * n_heads - if rows == 0: - return None - query_out = torch.empty(rows, 64, device=query.device, dtype=query.dtype) - key_out = torch.empty(rows, 64, device=key.device, dtype=key.dtype) - _qk_rmsnorm_rope_kernel[(rows,)]( - query, - key, - query_out, - key_out, - cos.contiguous(), - sin.contiguous(), - query.stride(0), - query.stride(1), - query.stride(2), - key.stride(0), - key.stride(1), - key.stride(2), - seq_len, - n_heads, - float(eps), - num_warps=1, - ) - view = (batch, seq_len, n_heads, 64) - return query_out.view(view), key_out.view(view) diff --git a/python/sglang/kernels/ops/diffusion/sites/bitexact_gate.py b/python/sglang/kernels/ops/diffusion/sites/bitexact_gate.py index 9eb8ae182d..e19bbe4d41 100644 --- a/python/sglang/kernels/ops/diffusion/sites/bitexact_gate.py +++ b/python/sglang/kernels/ops/diffusion/sites/bitexact_gate.py @@ -79,9 +79,9 @@ class BitExactFusionGate: The first-sight check runs the eager reference chain plus a host sync, so it must never happen inside ``torch.compile`` tracing or CUDA graph - capture. Once-for-all callers get both guards from - :meth:`can_attempt_once`; per-signature callers must keep their own - compile/capture checks next to the signature lookup (see FLUX / Sana). + capture. :meth:`run` applies both guards in either mode; hand-written + sites get them from :meth:`can_attempt_once` (once-for-all) or keep their + own compile/capture checks next to the signature lookup (FLUX / Sana). """ __slots__ = ("name", "disabled", "verified", "verified_sigs") @@ -114,14 +114,49 @@ class BitExactFusionGate: return False if self.verified: return True - # First-sight verify runs the eager reference chain and a host sync: - # attempt neither inside compile tracing nor CUDA graph capture (the - # sync would abort the capture; BCG then blocks the signature). Once - # verified, the fused kernel runs alone and is compile/capture-safe. - if torch.compiler.is_compiling(): - return False - return not ( - torch.cuda.is_available() and torch.cuda.is_current_stream_capturing() + return _can_verify_now() + + def run( + self, + fused: Callable[[], T], + reference: Callable[[], T], + *, + sig: Any = None, + equal: EqualFn | None = None, + logger: logging.Logger | None = None, + diagnostic_hint: DiagnosticHintFn | None = None, + reraise: tuple[type[Exception], ...] = (), + ) -> T: + """Return ``fused()`` once it has matched ``reference()`` bit-for-bit. + + The caller has already checked that the fast path applies. An + unverified ``sig`` is verified on this call unless it is inside compile + tracing or CUDA graph capture, where ``reference()`` runs instead. A + launch failure (other than ``reraise``) or a mismatch disables the + gate and returns ``reference()``. + """ + if self.disabled: + return reference() + sigs = self.verified_sigs + verified = self.verified if sigs is None else sig in sigs + if not verified and not _can_verify_now(): + return reference() + try: + out = fused() + except reraise: + raise + except Exception as exc: + self.on_exception(exc, logger=logger) + return reference() + if verified: + return out + return self.accept_or_fallback( + out, + reference(), + sig=sig, + equal=equal, + logger=logger, + diagnostic_hint=diagnostic_hint, ) def on_exception( @@ -145,7 +180,6 @@ class BitExactFusionGate: sig: Any = None, equal: EqualFn | None = None, logger: logging.Logger | None = None, - mismatch_msg: str | None = None, diagnostic_hint: DiagnosticHintFn | None = None, ) -> T: """Return ``out`` when bit-exact; otherwise disable and return ``ref``.""" @@ -156,10 +190,6 @@ class BitExactFusionGate: self.mark_verified(sig) return out if logger is not None: - message = mismatch_msg or ( - f"{self.name} fast path is not bit-exact against this " - "platform's reference dispatch; falling back to eager" - ) details = ( "Correctness is preserved because the eager reference output " "is used. A platform-specific reference kernel or reduction-order " @@ -172,11 +202,24 @@ class BitExactFusionGate: diagnostic_details = None if diagnostic_details: details = f"{details}. {diagnostic_details.rstrip('.')}" - logger.warning_once(f"{message.rstrip('.')}. {details}.") + logger.warning_once( + f"{self.name} fast path is not bit-exact against this platform's " + f"reference dispatch; falling back to eager. {details}." + ) self.disable() return ref +def _can_verify_now() -> bool: + # First-sight verify runs the eager reference chain and a host sync: + # attempt neither inside compile tracing nor CUDA graph capture (the + # sync would abort the capture; BCG then blocks the signature). Once + # verified, the fused kernel runs alone and is compile/capture-safe. + if torch.compiler.is_compiling(): + return False + return not (torch.cuda.is_available() and torch.cuda.is_current_stream_capturing()) + + def tensors_equal(a: Any, b: Any) -> bool: """``torch.equal`` for a tensor or a sequence of tensors.""" if isinstance(a, torch.Tensor): diff --git a/python/sglang/kernels/ops/diffusion/sites/flux2_nvfp4_swiglu_quant_site.py b/python/sglang/kernels/ops/diffusion/sites/flux2_nvfp4_swiglu_quant_site.py index 03adeb0bdb..dafb9b1fbe 100644 --- a/python/sglang/kernels/ops/diffusion/sites/flux2_nvfp4_swiglu_quant_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/flux2_nvfp4_swiglu_quant_site.py @@ -8,29 +8,10 @@ Keep it disabled below the approximate tier and mount it only for from __future__ import annotations -from torch import nn - from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion -_FUSION = QualityGatedFusion( +FLUX2_NVFP4_SWIGLU_QUANT = QualityGatedFusion( name="FLUX.2 NVFP4 FC1+SwiGLU+quant", marker_attr="_sgl_flux2_nvfp4_swiglu_quant_site", enabled_attr="_sgl_flux2_nvfp4_swiglu_quant_enabled", ) - - -def mark_flux2_nvfp4_swiglu_quant_site(module: nn.Module) -> None: - """Mark an eligible FLUX.2 feed-forward site, disabled by default.""" - _FUSION.mark(module) - - -def flux2_nvfp4_swiglu_quant_active(module: nn.Module) -> bool: - return _FUSION.is_enabled(module) - - -def mount_flux2_nvfp4_swiglu_quant(root: nn.Module) -> bool: - return _FUSION.mount(root) - - -def unmount_flux2_nvfp4_swiglu_quant(root: nn.Module) -> None: - _FUSION.unmount(root) diff --git a/python/sglang/kernels/ops/diffusion/sites/fused_gate_rmsnorm_site.py b/python/sglang/kernels/ops/diffusion/sites/fused_gate_rmsnorm_site.py index 6e5c73a896..08eab37f85 100644 --- a/python/sglang/kernels/ops/diffusion/sites/fused_gate_rmsnorm_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/fused_gate_rmsnorm_site.py @@ -12,9 +12,8 @@ native-bf16 reference RMSNorm. Ideogram's reference norm is ``F.rms_norm`` (fp32 internal statistics), so the fused path is numerically close (the norm statistics round through bf16) but **not bit-exact**. Following the fused-linear-GELU precedent, sites are therefore mounted only for -``quality="high"`` requests via :func:`mount_fused_gate_rmsnorm` / -:func:`unmount_fused_gate_rmsnorm` at batch boundaries; the default -``"exact"`` path keeps the unmodified reference chain bit-for-bit. +``quality="high"`` requests at batch boundaries; the default ``"exact"`` path +keeps the unmodified reference chain bit-for-bit. Mounting is all-or-nothing per transformer: if any marked site fails the static guards (non-bf16 norm weight, hidden size above the kernel limit, ...) @@ -23,7 +22,6 @@ every site on that transformer stays on the reference path. from __future__ import annotations -import logging from importlib import import_module import torch @@ -31,17 +29,6 @@ import torch.nn as nn from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion -logger = logging.getLogger(__name__) - -# Attributes of the site protocol (set by ``mark_fused_gate_rmsnorm_site``). -_SITE_NORM_ATTRS = "_sgl_fused_gate_rmsnorm_norm_attrs" -_SITE_ENABLED_ATTR = "_sgl_fused_gate_rmsnorm_enabled" -_FUSION = QualityGatedFusion( - name="fused gate RMSNorm", - marker_attr=_SITE_NORM_ATTRS, - enabled_attr=_SITE_ENABLED_ATTR, -) - # The Triton kernels mask a single block over the hidden dim. _MAX_HIDDEN_SIZE = 8192 @@ -78,7 +65,7 @@ def _static_reject_reason(site: nn.Module) -> str | None: import_module("triton") except ImportError: return "triton unavailable" - for attr in _FUSION.metadata(site, ()): + for attr in FUSED_GATE_RMSNORM.metadata(site, ()): norm = getattr(site, attr, None) weight = getattr(norm, "weight", None) if weight is None or weight.dim() != 1: @@ -90,33 +77,10 @@ def _static_reject_reason(site: nn.Module) -> str | None: return None -def mark_fused_gate_rmsnorm_site(module: nn.Module, norm_attrs: tuple[str, ...]): - """Declare ``module`` as a fused RMSNorm modulate/gate site. - - ``norm_attrs`` names the site's RMSNorm submodules (checked by the static - guards at mount time). The site starts unmounted - (``_sgl_fused_gate_rmsnorm_enabled = False``): the module's forward must - keep the reference path bit-exact until :func:`mount_fused_gate_rmsnorm` - enables it. - """ - _FUSION.mark(module, tuple(norm_attrs)) - - -def fused_gate_rmsnorm_active(module: nn.Module) -> bool: - """Whether the quality-gated fused path is mounted on ``module``.""" - return _FUSION.is_enabled(module) - - -def mount_fused_gate_rmsnorm(root: nn.Module) -> bool: - """Enable the fused kernels on every marked site under ``root``. - - All-or-nothing: if any marked site fails the static guards, every site is - left (or reset) on the reference path and False is returned. Returns False - as well when ``root`` has no marked sites. - """ - return _FUSION.mount(root, reject_reason=_static_reject_reason, logger=logger) - - -def unmount_fused_gate_rmsnorm(root: nn.Module) -> None: - """Reset every marked site under ``root`` to the bit-exact reference path.""" - _FUSION.unmount(root) +# A site's marker metadata is the tuple of its RMSNorm submodule names. +FUSED_GATE_RMSNORM = QualityGatedFusion( + name="fused gate RMSNorm", + marker_attr="_sgl_fused_gate_rmsnorm_norm_attrs", + enabled_attr="_sgl_fused_gate_rmsnorm_enabled", + reject_reason=_static_reject_reason, +) diff --git a/python/sglang/kernels/ops/diffusion/sites/fused_linear_gelu_site.py b/python/sglang/kernels/ops/diffusion/sites/fused_linear_gelu_site.py index 88828baf2f..fc77d78095 100644 --- a/python/sglang/kernels/ops/diffusion/sites/fused_linear_gelu_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/fused_linear_gelu_site.py @@ -11,10 +11,9 @@ reference only at bf16/fp16 rounding-order level -- close, but not bit-exact. Because it is not bit-exact, the fused path is **mounted only for ``quality="high"`` requests** (see ``SamplingParams.quality``): model code marks -its GELU up-projection sites with :func:`mark_fused_gelu_site` (default: off, -reference path, bit-exact), and the denoising stage calls -:func:`mount_fused_linear_gelu` / :func:`unmount_fused_linear_gelu` at batch -boundaries. Mounting is all-or-nothing per transformer: if any marked site +its GELU up-projection sites with ``FUSED_LINEAR_GELU.mark`` (default: off, +reference path, bit-exact), and the denoising stage mounts / unmounts +``FUSED_LINEAR_GELU`` at batch boundaries. Mounting is all-or-nothing per transformer: if any marked site fails the static guards (quantized weights, missing bias, non-half dtype, ...) no site on that transformer is fused. @@ -25,7 +24,6 @@ single opaque op under ``torch.compile`` -- no graph break. from __future__ import annotations -import logging import weakref from typing import Any @@ -36,22 +34,11 @@ from sglang.kernels.jit.utils import get_jit_cuda_arch from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion from sglang.srt.utils.custom_op import register_custom_op -logger = logging.getLogger(__name__) - # ``torch._addmm_activation`` is the (private but stable) entry point to the # cublasLt GEMM+bias+activation epilogue. Guard for builds where it is absent # so the reference path is always available. _HAS_ADDMM_ACTIVATION = hasattr(torch, "_addmm_activation") -# Attributes of the site protocol (set by ``mark_fused_gelu_site``). -_SITE_LINEAR_ATTR = "_sgl_fused_gelu_linear_attr" -_SITE_ENABLED_ATTR = "_sgl_fused_gelu_enabled" -_FUSION = QualityGatedFusion( - name="fused linear+GELU", - marker_attr=_SITE_LINEAR_ATTR, - enabled_attr=_SITE_ENABLED_ATTR, -) - def _fused_linear_gelu_tanh_fake( x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor @@ -157,39 +144,26 @@ def can_use_linear_gelu(linear: Any, x: torch.Tensor) -> bool: return verdict -def mark_fused_gelu_site(module: nn.Module, linear_attr: str) -> None: - """Declare ``module`` as a tanh-GELU up-projection fusion site. - - ``getattr(module, linear_attr)`` must be the up-projection linear whose - output feeds ``F.gelu(..., approximate="tanh")``. The site starts unmounted - (``_sgl_fused_gelu_enabled = False``): the module's forward must keep the - reference path bit-exact until :func:`mount_fused_linear_gelu` enables it. - """ - _FUSION.mark(module, linear_attr) - - -def fused_gelu_active(module: nn.Module) -> bool: - """Whether the quality-gated fused path is mounted on ``module``.""" - return _FUSION.is_enabled(module) - - def _site_reject_reason(site: nn.Module) -> str | None: - linear = getattr(site, _FUSION.metadata(site), None) + linear = getattr(site, FUSED_LINEAR_GELU.metadata(site), None) return "missing linear" if linear is None else _static_reject_reason(linear) -def mount_fused_linear_gelu(root: nn.Module) -> bool: - """Enable the fused epilogue on every marked site under ``root``. +class _LinearGeluFusion(QualityGatedFusion): + def mount(self, root: nn.Module) -> bool: + _VERDICTS.clear() + return super().mount(root) - All-or-nothing: if any marked site fails the static guards, every site is - left (or reset) on the reference path and False is returned. Returns False - as well when ``root`` has no marked sites. - """ - _VERDICTS.clear() - return _FUSION.mount(root, reject_reason=_site_reject_reason, logger=logger) + def unmount(self, root: nn.Module) -> None: + _VERDICTS.clear() + super().unmount(root) -def unmount_fused_linear_gelu(root: nn.Module) -> None: - """Reset every marked site under ``root`` to the bit-exact reference path.""" - _VERDICTS.clear() - _FUSION.unmount(root) +# A site's marker metadata names its up-projection linear, whose output feeds +# ``F.gelu(..., approximate="tanh")``. +FUSED_LINEAR_GELU = _LinearGeluFusion( + name="fused linear+GELU", + marker_attr="_sgl_fused_gelu_linear_attr", + enabled_attr="_sgl_fused_gelu_enabled", + reject_reason=_site_reject_reason, +) diff --git a/python/sglang/kernels/ops/diffusion/sites/fused_ln_modulate_site.py b/python/sglang/kernels/ops/diffusion/sites/fused_ln_modulate_site.py index a818309785..8c73de92c6 100644 --- a/python/sglang/kernels/ops/diffusion/sites/fused_ln_modulate_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/fused_ln_modulate_site.py @@ -8,48 +8,27 @@ the result is not bit-exact vs the reference (half-precision rounding-order differences only). Because it is not bit-exact the fold is opt-in per batch: model code marks -its LN+modulate sites with :func:`mark_fused_ln_modulate_site` (default off, -reference path), and the denoising stage calls -:func:`mount_fused_ln_modulate` / :func:`unmount_fused_ln_modulate` at batch -boundaries for ``quality="high"`` requests. +its LN+modulate sites with ``FUSED_LN_MODULATE.mark`` (default off, +reference path), and the denoising stage mounts / unmounts +``FUSED_LN_MODULATE`` at batch boundaries for ``quality="high"`` requests. """ from __future__ import annotations import torch import torch.nn.functional as F -from torch import nn from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion -_SITE_MARKER_ATTR = "_sgl_fused_ln_modulate_site" -_SITE_ENABLED_ATTR = "_sgl_fused_ln_modulate_enabled" -_FUSION = QualityGatedFusion( +FUSED_LN_MODULATE = QualityGatedFusion( name="fused LN+modulate", - marker_attr=_SITE_MARKER_ATTR, - enabled_attr=_SITE_ENABLED_ATTR, + marker_attr="_sgl_fused_ln_modulate_site", + enabled_attr="_sgl_fused_ln_modulate_enabled", ) _SUPPORTED_DTYPES = (torch.float16, torch.bfloat16) -def mark_fused_ln_modulate_site(module: nn.Module) -> None: - """Mark ``module`` as an LN+modulate fusion site (mounted off).""" - _FUSION.mark(module) - - -def fused_ln_modulate_active(module: nn.Module) -> bool: - return _FUSION.is_enabled(module) - - -def mount_fused_ln_modulate(root: nn.Module) -> bool: - return _FUSION.mount(root) - - -def unmount_fused_ln_modulate(root: nn.Module) -> None: - _FUSION.unmount(root) - - def can_use_ln_modulate( x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor ) -> bool: diff --git a/python/sglang/kernels/ops/diffusion/sites/lingbot_video_gated_residual_site.py b/python/sglang/kernels/ops/diffusion/sites/gated_residual_site.py similarity index 59% rename from python/sglang/kernels/ops/diffusion/sites/lingbot_video_gated_residual_site.py rename to python/sglang/kernels/ops/diffusion/sites/gated_residual_site.py index 56aadc90b6..34b5fb4b80 100644 --- a/python/sglang/kernels/ops/diffusion/sites/lingbot_video_gated_residual_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/gated_residual_site.py @@ -1,8 +1,8 @@ -"""LingBot Video per-token gated-residual fusion, gated by request quality. +"""Per-token gated-residual fusion for Helios and LingBot Video, gated by quality. -Each LingBot block applies ``residual + (gate * update).to(residual.dtype)`` at -the attention and FFN updates, where ``update`` (post-norm) and the per-token -``gate`` (``[B, S, 1]``) stay in FP32 while ``residual`` is BF16. The shared +Each block applies ``residual + (gate * update).to(residual.dtype)`` at its +attention and FFN updates, where ``update`` and the per-token ``gate`` +(``[B, S, 1]``) stay in FP32 while ``residual`` is BF16. The shared ``residual_gate_add`` kernel computes ``residual + update * gate`` in a single pass but requires one dtype, so the gate and update are first cast to BF16. That reordering of the FP32 multiply is numerically equivalent only at @@ -13,51 +13,36 @@ half-precision rounding level (not bit-exact), so the fusion is opt-in: from __future__ import annotations -import logging - import torch import torch.nn as nn from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion -logger = logging.getLogger(__name__) - -_FUSION = QualityGatedFusion( +HELIOS_GATED_RESIDUAL = QualityGatedFusion( + name="Helios per-token gated residual", + marker_attr="_sgl_helios_gated_residual_site", + enabled_attr="_sgl_helios_gated_residual_enabled", +) +LINGBOT_VIDEO_GATED_RESIDUAL = QualityGatedFusion( name="LingBot Video per-token gated residual", marker_attr="_sgl_lingbot_video_gated_residual_site", enabled_attr="_sgl_lingbot_video_gated_residual_enabled", ) -def mark_lingbot_video_gated_residual_site(module: nn.Module) -> None: - """Mark a LingBot block; it starts on the reference path.""" - _FUSION.mark(module) - - -def lingbot_video_gated_residual_active(module: nn.Module) -> bool: - return _FUSION.is_enabled(module) - - -def mount_lingbot_video_gated_residual(root: nn.Module) -> bool: - return _FUSION.mount(root, logger=logger) - - -def unmount_lingbot_video_gated_residual(root: nn.Module) -> None: - _FUSION.unmount(root) - - -def try_lingbot_video_gated_residual( +def try_gated_residual( + fusion: QualityGatedFusion, site: nn.Module, residual: torch.Tensor, update: torch.Tensor, gate: torch.Tensor, ) -> torch.Tensor | None: - """Return the fused ``residual + update * gate`` when the site is enabled. + """Return the fused ``residual + update * gate`` when ``site`` is enabled. Returns ``None`` (caller runs the reference path) when the site is off or the tensors are not eligible for the per-token fast path. """ - if not _FUSION.is_enabled(site): + if not fusion.is_enabled(site): return None from sglang.kernels.ops.diffusion import ( can_use_residual_gate_add_cuda, diff --git a/python/sglang/kernels/ops/diffusion/sites/helios_gated_residual_site.py b/python/sglang/kernels/ops/diffusion/sites/helios_gated_residual_site.py deleted file mode 100644 index 83a7cb8852..0000000000 --- a/python/sglang/kernels/ops/diffusion/sites/helios_gated_residual_site.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Helios per-token gated-residual fusion, gated by request quality. - -Each Helios block applies ``residual + (gate * update).to(residual.dtype)`` at -the self-attention and FFN updates, where ``update`` (post-attn / post-FFN) and -the per-token ``gate`` (``[B, S, 1]``) stay in FP32 while ``residual`` is BF16. -The shared ``residual_gate_add`` kernel computes ``residual + update * gate`` -in a single pass but requires one dtype, so the gate and update are first cast -to BF16. That reordering of the FP32 multiply is numerically equivalent only at -half-precision rounding level (not bit-exact), so the fusion is opt-in: -``quality="lossless"`` and ``quality="high"`` mount it, while the default -``quality="exact"`` keeps the reference FP32-multiply form bit-for-bit. -""" - -from __future__ import annotations - -import logging - -import torch -import torch.nn as nn - -from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion - -logger = logging.getLogger(__name__) - -_FUSION = QualityGatedFusion( - name="Helios per-token gated residual", - marker_attr="_sgl_helios_gated_residual_site", - enabled_attr="_sgl_helios_gated_residual_enabled", -) - - -def mark_helios_gated_residual_site(module: nn.Module) -> None: - """Mark a Helios block; it starts on the reference path.""" - _FUSION.mark(module) - - -def helios_gated_residual_active(module: nn.Module) -> bool: - return _FUSION.is_enabled(module) - - -def mount_helios_gated_residual(root: nn.Module) -> bool: - return _FUSION.mount(root, logger=logger) - - -def unmount_helios_gated_residual(root: nn.Module) -> None: - _FUSION.unmount(root) - - -def try_helios_gated_residual( - site: nn.Module, - residual: torch.Tensor, - update: torch.Tensor, - gate: torch.Tensor, -) -> torch.Tensor | None: - """Return the fused ``residual + update * gate`` when the site is enabled. - - Returns ``None`` (caller runs the reference path) when the site is off or - the tensors are not eligible for the per-token fast path. - """ - if not _FUSION.is_enabled(site): - return None - from sglang.kernels.ops.diffusion import ( - can_use_residual_gate_add_cuda, - residual_gate_add, - ) - - if residual.dtype != update.dtype or residual.dtype != gate.dtype: - update = update.to(residual.dtype) - gate = gate.to(residual.dtype) - if not can_use_residual_gate_add_cuda(residual, update, gate): - return None - return residual_gate_add(residual, update, gate) diff --git a/python/sglang/kernels/ops/diffusion/sites/hunyuan_qknorm_site.py b/python/sglang/kernels/ops/diffusion/sites/hunyuan_qknorm_site.py index 12b7e75f44..a4b15fd521 100644 --- a/python/sglang/kernels/ops/diffusion/sites/hunyuan_qknorm_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/hunyuan_qknorm_site.py @@ -2,7 +2,6 @@ from __future__ import annotations -import logging from functools import cache import torch @@ -10,14 +9,6 @@ import torch.nn as nn from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion -logger = logging.getLogger(__name__) - -_FUSION = QualityGatedFusion( - name="HunyuanVideo strided QK RMSNorm", - marker_attr="_sgl_hunyuan_qknorm_site", - enabled_attr="_sgl_hunyuan_qknorm_enabled", -) - @cache def _get_qk_rmsnorm_cute(): @@ -30,26 +21,18 @@ def _get_qk_rmsnorm_cute(): return qk_rmsnorm_cute -def mark_hunyuan_qknorm_site(module: nn.Module) -> None: - _FUSION.mark(module) - - def _site_reject_reason(_site: nn.Module) -> str | None: if _get_qk_rmsnorm_cute() is None: return "FlashInfer CuTe QK RMSNorm unavailable" return None -def mount_hunyuan_qknorm(root: nn.Module) -> bool: - return _FUSION.mount( - root, - reject_reason=_site_reject_reason, - logger=logger, - ) - - -def unmount_hunyuan_qknorm(root: nn.Module) -> None: - _FUSION.unmount(root) +HUNYUAN_QKNORM = QualityGatedFusion( + name="HunyuanVideo strided QK RMSNorm", + marker_attr="_sgl_hunyuan_qknorm_site", + enabled_attr="_sgl_hunyuan_qknorm_enabled", + reject_reason=_site_reject_reason, +) def try_hunyuan_qknorm( @@ -62,7 +45,7 @@ def try_hunyuan_qknorm( ) -> tuple[torch.Tensor, torch.Tensor] | None: """Normalize strided Hunyuan Q/K views without materializing inputs.""" if not ( - _FUSION.is_enabled(site) + HUNYUAN_QKNORM.is_enabled(site) and not torch.compiler.is_compiling() and q.is_cuda and q.dtype == torch.bfloat16 diff --git a/python/sglang/kernels/ops/diffusion/sites/lingbot_video_rmsnorm_site.py b/python/sglang/kernels/ops/diffusion/sites/lingbot_video_rmsnorm_site.py index 6c4a6f4863..32f442686a 100644 --- a/python/sglang/kernels/ops/diffusion/sites/lingbot_video_rmsnorm_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/lingbot_video_rmsnorm_site.py @@ -13,7 +13,6 @@ remains unchanged. from __future__ import annotations -import logging from importlib import import_module import torch @@ -21,23 +20,6 @@ import torch.nn as nn from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion -logger = logging.getLogger(__name__) - -_FUSION = QualityGatedFusion( - name="LingBot Video fused RMSNorm", - marker_attr="_sgl_lingbot_video_rmsnorm_site", - enabled_attr="_sgl_lingbot_video_rmsnorm_enabled", -) - - -def mark_lingbot_video_rmsnorm_site(module: nn.Module) -> None: - """Mark a LingBot RMSNorm module; it starts on the reference path.""" - _FUSION.mark(module) - - -def lingbot_video_rmsnorm_active(module: nn.Module) -> bool: - return _FUSION.is_enabled(module) - def _site_reject_reason(site: nn.Module) -> str | None: try: @@ -54,12 +36,12 @@ def _site_reject_reason(site: nn.Module) -> str | None: return None -def mount_lingbot_video_rmsnorm(root: nn.Module) -> bool: - return _FUSION.mount(root, reject_reason=_site_reject_reason, logger=logger) - - -def unmount_lingbot_video_rmsnorm(root: nn.Module) -> None: - _FUSION.unmount(root) +LINGBOT_VIDEO_RMSNORM = QualityGatedFusion( + name="LingBot Video fused RMSNorm", + marker_attr="_sgl_lingbot_video_rmsnorm_site", + enabled_attr="_sgl_lingbot_video_rmsnorm_enabled", + reject_reason=_site_reject_reason, +) def try_lingbot_video_rmsnorm( @@ -70,7 +52,7 @@ def try_lingbot_video_rmsnorm( ) -> torch.Tensor | None: """Return the quality-gated RMSNorm result, or ``None`` to fall back.""" if not ( - _FUSION.is_enabled(site) + LINGBOT_VIDEO_RMSNORM.is_enabled(site) and hidden_states.is_cuda and hidden_states.dtype in (torch.float16, torch.bfloat16) and hidden_states.is_contiguous() diff --git a/python/sglang/kernels/ops/diffusion/sites/ltx2_qknorm_split_rope_site.py b/python/sglang/kernels/ops/diffusion/sites/ltx2_qknorm_split_rope_site.py index 1f95edbbcb..65c2783278 100644 --- a/python/sglang/kernels/ops/diffusion/sites/ltx2_qknorm_split_rope_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/ltx2_qknorm_split_rope_site.py @@ -8,28 +8,10 @@ kernel fusions. from __future__ import annotations -from torch import nn - from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion -_FUSION = QualityGatedFusion( +LTX2_QKNORM_SPLIT_ROPE = QualityGatedFusion( name="LTX-2 Hopper QKNorm+split-RoPE", marker_attr="_sgl_ltx2_qknorm_split_rope_site", enabled_attr="_sgl_ltx2_qknorm_split_rope_enabled", ) - - -def mark_ltx2_qknorm_split_rope_site(module: nn.Module) -> None: - _FUSION.mark(module) - - -def ltx2_qknorm_split_rope_active(module: nn.Module) -> bool: - return _FUSION.is_enabled(module) - - -def mount_ltx2_qknorm_split_rope(root: nn.Module) -> bool: - return _FUSION.mount(root) - - -def unmount_ltx2_qknorm_split_rope(root: nn.Module) -> None: - _FUSION.unmount(root) diff --git a/python/sglang/kernels/ops/diffusion/sites/ltx2_rmsnorm_modulate_site.py b/python/sglang/kernels/ops/diffusion/sites/ltx2_rmsnorm_modulate_site.py index 980681d9f2..66e1b0a4e1 100644 --- a/python/sglang/kernels/ops/diffusion/sites/ltx2_rmsnorm_modulate_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/ltx2_rmsnorm_modulate_site.py @@ -10,51 +10,30 @@ The fused kernel reproduces the RMSNorm math via ``rsqrt.approx`` (the flashinfer CuTe form), which differs from aten's refined ``rsqrtf`` by at most one bf16 ULP on a small fraction of elements. It is therefore *not* bit-exact vs the eager reference, so the fold is opt-in per batch: model code -marks its adaLN sites with :func:`mark_ltx2_rms_norm_modulate_site` (default -off, reference path) and the denoising stage calls -:func:`mount_ltx2_rms_norm_modulate` / :func:`unmount_ltx2_rms_norm_modulate` -at batch boundaries for ``quality="high"`` requests. +marks its adaLN sites with ``LTX2_RMS_NORM_MODULATE.mark`` (default off, +reference path) and the denoising stage mounts / unmounts +``LTX2_RMS_NORM_MODULATE`` at batch boundaries for ``quality="high"`` requests. """ from __future__ import annotations import torch -from torch import nn from sglang.kernels.ops.diffusion.norm.rmsnorm_scale_shift_bitexact import ( fused_rmsnorm_scale_shift_bitexact, ) from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion -_SITE_MARKER_ATTR = "_sgl_ltx2_rms_norm_modulate_site" -_SITE_ENABLED_ATTR = "_sgl_ltx2_rms_norm_modulate_enabled" -_FUSION = QualityGatedFusion( +LTX2_RMS_NORM_MODULATE = QualityGatedFusion( name="LTX-2 RMSNorm+modulate", - marker_attr=_SITE_MARKER_ATTR, - enabled_attr=_SITE_ENABLED_ATTR, + marker_attr="_sgl_ltx2_rms_norm_modulate_site", + enabled_attr="_sgl_ltx2_rms_norm_modulate_enabled", ) # ``RMSNormNoWeight`` applies no scale, so a ones weight reproduces it exactly. _ONES_WEIGHT_CACHE: dict[tuple[torch.device, int], torch.Tensor] = {} -def mark_ltx2_rms_norm_modulate_site(module: nn.Module) -> None: - """Mark ``module`` as an LTX-2 RMSNorm+modulate fusion site (mounted off).""" - _FUSION.mark(module) - - -def ltx2_rms_norm_modulate_active(module: nn.Module) -> bool: - return _FUSION.is_enabled(module) - - -def mount_ltx2_rms_norm_modulate(root: nn.Module) -> bool: - return _FUSION.mount(root) - - -def unmount_ltx2_rms_norm_modulate(root: nn.Module) -> None: - _FUSION.unmount(root) - - def _ones_weight(x: torch.Tensor) -> torch.Tensor: key = (x.device, int(x.shape[-1])) w = _ONES_WEIGHT_CACHE.get(key) diff --git a/python/sglang/kernels/ops/diffusion/sites/minimax_h3_norm_modulate_site.py b/python/sglang/kernels/ops/diffusion/sites/minimax_h3_norm_modulate_site.py index 8d3e01f0a5..ffcbba0c12 100644 --- a/python/sglang/kernels/ops/diffusion/sites/minimax_h3_norm_modulate_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/minimax_h3_norm_modulate_site.py @@ -9,33 +9,10 @@ mount it, while ``"exact"`` keeps the reference chain bit-for-bit. from __future__ import annotations -import logging - -import torch.nn as nn - from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion -logger = logging.getLogger(__name__) - -_FUSION = QualityGatedFusion( +MINIMAX_H3_NORM_MODULATE = QualityGatedFusion( name="MiniMax-H3 fused RMSNorm + AdaLN", marker_attr="_sgl_minimax_h3_norm_modulate_site", enabled_attr="_sgl_minimax_h3_norm_modulate_enabled", ) - - -def mark_minimax_h3_norm_modulate_site(module: nn.Module) -> None: - """Mark an H3 DiT block; it starts on the reference path.""" - _FUSION.mark(module) - - -def minimax_h3_norm_modulate_active(module: nn.Module) -> bool: - return _FUSION.is_enabled(module) - - -def mount_minimax_h3_norm_modulate(root: nn.Module) -> bool: - return _FUSION.mount(root, logger=logger) - - -def unmount_minimax_h3_norm_modulate(root: nn.Module) -> None: - _FUSION.unmount(root) diff --git a/python/sglang/kernels/ops/diffusion/sites/nvfp4_bias_gelu_site.py b/python/sglang/kernels/ops/diffusion/sites/nvfp4_bias_gelu_site.py index 38866dab06..e9a7867ec2 100644 --- a/python/sglang/kernels/ops/diffusion/sites/nvfp4_bias_gelu_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/nvfp4_bias_gelu_site.py @@ -8,30 +8,10 @@ only for the existing ``quality="high"`` contract. from __future__ import annotations -import logging - import torch.nn as nn from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion -logger = logging.getLogger(__name__) - -_FUSION = QualityGatedFusion( - name="Wan NVFP4 bias+GELU", - marker_attr="_sgl_nvfp4_bias_gelu_site", - enabled_attr="_sgl_nvfp4_bias_gelu_enabled", -) - - -def mark_nvfp4_bias_gelu_site(module: nn.Module) -> None: - """Mark an MLP whose ``fc_in`` bias can be deferred for this fusion.""" - _FUSION.mark(module) - - -def nvfp4_bias_gelu_active(module: nn.Module) -> bool: - """Whether the quality-gated fusion is mounted on ``module``.""" - return _FUSION.is_enabled(module) - def _site_reject_reason(site: nn.Module) -> str | None: if not getattr(site, "fuse_bias_gelu_tanh", False): @@ -51,17 +31,27 @@ def _site_reject_reason(site: nn.Module) -> str | None: return None -def mount_nvfp4_bias_gelu(root: nn.Module) -> bool: - """Enable every eligible marked site under ``root``.""" - sites = list(_FUSION.iter_sites(root)) - mounted = _FUSION.mount(root, reject_reason=_site_reject_reason, logger=logger) - for site in sites: - site.fc_in.skip_bias_add = mounted - return mounted +class _NVFP4BiasGeluFusion(QualityGatedFusion): + """A mounted site defers its ``fc_in`` bias add to the fused GELU.""" + + __slots__ = () + + def mount(self, root: nn.Module) -> bool: + mounted = super().mount(root) + for site in self.iter_sites(root): + site.fc_in.skip_bias_add = mounted + return mounted + + def unmount(self, root: nn.Module) -> None: + super().unmount(root) + for site in self.iter_sites(root): + site.fc_in.skip_bias_add = False -def unmount_nvfp4_bias_gelu(root: nn.Module) -> None: - """Restore every marked site to its original linear+GELU path.""" - _FUSION.unmount(root) - for site in _FUSION.iter_sites(root): - site.fc_in.skip_bias_add = False +# Marks an MLP whose ``fc_in`` bias can be deferred for this fusion. +NVFP4_BIAS_GELU = _NVFP4BiasGeluFusion( + name="Wan NVFP4 bias+GELU", + marker_attr="_sgl_nvfp4_bias_gelu_site", + enabled_attr="_sgl_nvfp4_bias_gelu_enabled", + reject_reason=_site_reject_reason, +) diff --git a/python/sglang/kernels/ops/diffusion/sites/quality_gate.py b/python/sglang/kernels/ops/diffusion/sites/quality_gate.py index 1893c45d18..9b5e84ff6d 100644 --- a/python/sglang/kernels/ops/diffusion/sites/quality_gate.py +++ b/python/sglang/kernels/ops/diffusion/sites/quality_gate.py @@ -8,6 +8,8 @@ from typing import Any from torch import nn +logger = logging.getLogger(__name__) + RejectReason = Callable[[nn.Module], str | None] @@ -16,15 +18,24 @@ class QualityGatedFusion: The marker metadata describes the model attribute(s) owned by a site. The enabled flag remains a plain module attribute so compiled model forwards - can read it without depending on this helper object. + can read it without depending on this helper object. ``reject_reason`` + holds the static guards :meth:`mount` checks on every marked site. """ - __slots__ = ("enabled_attr", "marker_attr", "name") + __slots__ = ("enabled_attr", "marker_attr", "name", "reject_reason") - def __init__(self, *, name: str, marker_attr: str, enabled_attr: str) -> None: + def __init__( + self, + *, + name: str, + marker_attr: str, + enabled_attr: str, + reject_reason: RejectReason | None = None, + ) -> None: self.name = name self.marker_attr = marker_attr self.enabled_attr = enabled_attr + self.reject_reason = reject_reason def mark(self, module: nn.Module, metadata: Any = True) -> None: setattr(module, self.marker_attr, metadata) @@ -41,32 +52,25 @@ class QualityGatedFusion: if hasattr(module, self.marker_attr): yield module - def mount( - self, - root: nn.Module, - *, - reject_reason: RejectReason | None = None, - logger: logging.Logger | None = None, - ) -> bool: + def mount(self, root: nn.Module) -> bool: """Enable every eligible site, or leave the whole family disabled.""" sites = list(self.iter_sites(root)) if not sites: return False - if reject_reason is not None: + if self.reject_reason is not None: for site in sites: - reason = reject_reason(site) + reason = self.reject_reason(site) if reason is None: continue self._set_enabled(sites, False) - if logger is not None: - logger.info( - "%s: %s site failed static guards (%s); keeping the " - "whole model on the reference path", - self.name, - type(site).__name__, - reason, - ) + logger.info( + "%s: %s site failed static guards (%s); keeping the " + "whole model on the reference path", + self.name, + type(site).__name__, + reason, + ) return False self._set_enabled(sites, True) diff --git a/python/sglang/kernels/ops/diffusion/sites/qwen_image_added_qkv_site.py b/python/sglang/kernels/ops/diffusion/sites/qwen_image_added_qkv_site.py index 5156c8df2c..926bf034d2 100644 --- a/python/sglang/kernels/ops/diffusion/sites/qwen_image_added_qkv_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/qwen_image_added_qkv_site.py @@ -8,30 +8,10 @@ slices independently. ``quality="high"`` mounts the single-GEMM path. from __future__ import annotations -import logging - import torch.nn as nn from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion -logger = logging.getLogger(__name__) - -_FUSION = QualityGatedFusion( - name="Qwen-Image fused added-QKV", - marker_attr="_sgl_qwen_image_added_qkv_site", - enabled_attr="_sgl_qwen_image_added_qkv_enabled", -) - - -def mark_qwen_image_added_qkv_site(module: nn.Module) -> None: - """Mark an unquantized Qwen-Image attention site; it starts unmounted.""" - _FUSION.mark(module) - - -def qwen_image_added_qkv_active(module: nn.Module) -> bool: - """Whether the request-scoped packed added-QKV GEMM is mounted.""" - return _FUSION.is_enabled(module) - def _site_reject_reason(site: nn.Module) -> str | None: linear = getattr(site, "to_added_qkv", None) @@ -44,9 +24,9 @@ def _site_reject_reason(site: nn.Module) -> str | None: return None -def mount_qwen_image_added_qkv(root: nn.Module) -> bool: - return _FUSION.mount(root, reject_reason=_site_reject_reason, logger=logger) - - -def unmount_qwen_image_added_qkv(root: nn.Module) -> None: - _FUSION.unmount(root) +QWEN_IMAGE_ADDED_QKV = QualityGatedFusion( + name="Qwen-Image fused added-QKV", + marker_attr="_sgl_qwen_image_added_qkv_site", + enabled_attr="_sgl_qwen_image_added_qkv_enabled", + reject_reason=_site_reject_reason, +) diff --git a/python/sglang/kernels/ops/diffusion/sites/sana_video_linear_attention_site.py b/python/sglang/kernels/ops/diffusion/sites/sana_video_linear_attention_site.py index d782f42cd4..470636f6c9 100644 --- a/python/sglang/kernels/ops/diffusion/sites/sana_video_linear_attention_site.py +++ b/python/sglang/kernels/ops/diffusion/sites/sana_video_linear_attention_site.py @@ -13,31 +13,11 @@ eligible; unsupported dtypes and layouts fall back to the reference path. from __future__ import annotations -import logging - import torch import torch.nn as nn from sglang.kernels.ops.diffusion.sites.quality_gate import QualityGatedFusion -logger = logging.getLogger(__name__) - -_FUSION = QualityGatedFusion( - name="SANA-Video BF16-input linear attention", - marker_attr="_sgl_sana_video_linear_attention_site", - enabled_attr="_sgl_sana_video_linear_attention_enabled", -) - - -def mark_sana_video_linear_attention_site(module: nn.Module) -> None: - """Mark a SANA-Video linear-attention module; it starts unmounted.""" - _FUSION.mark(module) - - -def sana_video_linear_attention_active(module: nn.Module) -> bool: - """Whether the request-scoped BF16-input path is mounted on ``module``.""" - return _FUSION.is_enabled(module) - def _site_reject_reason(_site: nn.Module) -> str | None: if torch.version.cuda is None: @@ -45,12 +25,12 @@ def _site_reject_reason(_site: nn.Module) -> str | None: return None -def mount_sana_video_linear_attention(root: nn.Module) -> bool: - return _FUSION.mount(root, reject_reason=_site_reject_reason, logger=logger) - - -def unmount_sana_video_linear_attention(root: nn.Module) -> None: - _FUSION.unmount(root) +SANA_VIDEO_LINEAR_ATTENTION = QualityGatedFusion( + name="SANA-Video BF16-input linear attention", + marker_attr="_sgl_sana_video_linear_attention_site", + enabled_attr="_sgl_sana_video_linear_attention_enabled", + reject_reason=_site_reject_reason, +) def try_sana_video_linear_attention( @@ -62,7 +42,7 @@ def try_sana_video_linear_attention( ) -> torch.Tensor | None: """Return the quality-gated attention result, or ``None`` to fall back.""" if not ( - _FUSION.is_enabled(site) + SANA_VIDEO_LINEAR_ATTENTION.is_enabled(site) and query_rotate.is_cuda and query_rotate.dtype == torch.bfloat16 and key_rotate.dtype == query_rotate.dtype diff --git a/python/sglang/kernels/ops/elementwise/__init__.py b/python/sglang/kernels/ops/elementwise/__init__.py index a0c3aecab7..4de164ba29 100644 --- a/python/sglang/kernels/ops/elementwise/__init__.py +++ b/python/sglang/kernels/ops/elementwise/__init__.py @@ -9,7 +9,12 @@ they logically belong to. """ from sglang.kernels.registry import register_kernel -from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec +from sglang.kernels.spec import ( + CUDA_ONLY, + CapabilityRequirement, + KernelBackend, + KernelSpec, +) __all__ = [] @@ -20,7 +25,6 @@ for _fn in ("hc_combine_gate", "hc_combine_apply", "hc_combine_apply_norm"): backend=KernelBackend.JIT, target=f"sglang.kernels.ops.elementwise.hc_combine_decode:{_fn}", capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}), - description="BF16 HC decode with four branches of width 2560 and PDL.", ) ) del _fn @@ -32,7 +36,7 @@ register_kernel( op="elementwise.fused_qwen4_gate_value", backend=KernelBackend.TRITON, target="sglang.kernels.ops.elementwise.qwen4_gate:fused_qwen4_gate_value", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -40,7 +44,7 @@ register_kernel( op="elementwise.fused_qwen4_gate_reduce", backend=KernelBackend.TRITON, target="sglang.kernels.ops.elementwise.qwen4_gate:fused_qwen4_gate_reduce", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -48,6 +52,6 @@ register_kernel( op="elementwise.row_scale_bf16", backend=KernelBackend.JIT, target="sglang.kernels.ops.elementwise.row_scale:row_scale_bf16", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) diff --git a/python/sglang/kernels/ops/elementwise/elementwise.py b/python/sglang/kernels/ops/elementwise/elementwise.py index 8d096f53df..de793ccb06 100644 --- a/python/sglang/kernels/ops/elementwise/elementwise.py +++ b/python/sglang/kernels/ops/elementwise/elementwise.py @@ -8,48 +8,6 @@ from sglang.srt.utils import is_hip _is_hip = is_hip() -rmsnorm_autotune = triton.autotune( - configs=[ - triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=4, num_stages=1), - triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=8, num_stages=1), - triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=16, num_stages=1), - triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=4), - triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=8), - triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=16), - triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=4, num_stages=4), - triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=8, num_stages=4), - triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=16, num_stages=4), - triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=8, num_stages=8), - triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=16, num_stages=8), - triton.Config(kwargs={"BLOCK_SIZE": 2048}, num_warps=8), - triton.Config(kwargs={"BLOCK_SIZE": 2048}, num_warps=16), - triton.Config(kwargs={"BLOCK_SIZE": 2048}, num_warps=8, num_stages=4), - triton.Config(kwargs={"BLOCK_SIZE": 2048}, num_warps=16, num_stages=4), - triton.Config(kwargs={"BLOCK_SIZE": 4096}, num_warps=8), - triton.Config(kwargs={"BLOCK_SIZE": 4096}, num_warps=16), - triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=8), - triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=16), - triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=32), - triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=8, num_stages=1), - triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=16, num_stages=1), - triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=32, num_stages=1), - triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=8, num_stages=4), - triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=16, num_stages=4), - triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=32, num_stages=4), - triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=8), - triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=16), - triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=32), - triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=8, num_stages=1), - triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=16, num_stages=1), - triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=32, num_stages=1), - triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=8, num_stages=4), - triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=16, num_stages=4), - triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=32, num_stages=4), - ], - key=["hidden_dim"], -) - - @triton.jit def fused_dual_residual_rmsnorm_kernel( output_ptr, @@ -96,46 +54,37 @@ def fused_dual_residual_rmsnorm_kernel( ) -fused_dual_residual_rmsnorm_kernel_autotune = rmsnorm_autotune( - fused_dual_residual_rmsnorm_kernel -) - - -def fused_dual_residual_rmsnorm(x, residual, weight1, weight2, eps, autotune=False): +def fused_dual_residual_rmsnorm(x, residual, weight1, weight2, eps): assert len(x.shape) == 2 assert x.shape == residual.shape and x.dtype == residual.dtype, ( f"{x.shape=} {residual.shape=} {x.dtype=} {residual.dtype=}" ) output, mid = torch.empty_like(x), torch.empty_like(x) bs, hidden_dim = x.shape - if autotune: - fused_dual_residual_rmsnorm_kernel_autotune[(bs,)]( - output, mid, x, residual, weight1, weight2, eps=eps, hidden_dim=hidden_dim - ) - else: - max_warps = 16 if _is_hip else 32 - config = { - "BLOCK_SIZE": triton.next_power_of_2(hidden_dim), - "num_warps": max( - min(triton.next_power_of_2(triton.cdiv(hidden_dim, 256)), max_warps), 4 - ), - } + max_warps = 16 if _is_hip else 32 + config = { + "BLOCK_SIZE": triton.next_power_of_2(hidden_dim), + "num_warps": max( + min(triton.next_power_of_2(triton.cdiv(hidden_dim, 256)), max_warps), 4 + ), + } - fused_dual_residual_rmsnorm_kernel[(bs,)]( - output, - mid, - x, - residual, - weight1, - weight2, - eps=eps, - hidden_dim=hidden_dim, - **config, - ) + fused_dual_residual_rmsnorm_kernel[(bs,)]( + output, + mid, + x, + residual, + weight1, + weight2, + eps=eps, + hidden_dim=hidden_dim, + **config, + ) return output, mid +# Not layernorm.rmsnorm_fp32: dividing by sqrt(var + eps) rounds differently from rsqrt. @triton.jit def fused_rmsnorm_kernel( output_ptr, @@ -167,7 +116,7 @@ def fused_rmsnorm_kernel( ) -def fused_rmsnorm(x, weight, eps, autotune=False, inplace=False): +def fused_rmsnorm(x, weight, eps, inplace=False): assert len(x.shape) == 2 if inplace: output = x @@ -282,100 +231,6 @@ def gelu_and_mul_triton( return out_hidden_states, None -# silu on first half of vector -@triton.jit -def silu_and_mul_kernel( - out_hidden_states_ptr, # (bs, hidden_dim) - out_scales_ptr, # (bs,) - hidden_states_ptr, # (bs, hidden_dim * 2) - quant_max: tl.constexpr, - static_scale: tl.constexpr, - hidden_dim: tl.constexpr, # the output hidden_dim - BLOCK_SIZE: tl.constexpr, -): - pid = tl.program_id(axis=0) - - input_start = pid * hidden_dim * 2 - output_start = pid * hidden_dim - - input1_offs = tl.arange(0, BLOCK_SIZE) - mask = tl.arange(0, BLOCK_SIZE) < hidden_dim # shared for input1, input3, output - input3_offs = hidden_dim + tl.arange(0, BLOCK_SIZE) - output_offs = tl.arange(0, BLOCK_SIZE) - - x1 = tl.load( - hidden_states_ptr + input_start + input1_offs, mask=mask, other=0.0 - ).to(tl.float32) - x3 = tl.load( - hidden_states_ptr + input_start + input3_offs, mask=mask, other=0.0 - ).to(tl.float32) - - # silu - # cast down before mul to better match training? - silu_x1 = x1 * tl.sigmoid(x1) - out = x3 * silu_x1.to(hidden_states_ptr.dtype.element_ty) - - if quant_max is not None: - raise NotImplementedError() - - tl.store(out_hidden_states_ptr + output_start + output_offs, out, mask=mask) - - -def silu_and_mul_triton( - hidden_states, - scales=None, - quantize=None, # dtype to quantize to - out=None, -): - bs, in_hidden_dim = hidden_states.shape - hidden_dim = in_hidden_dim // 2 - - if out is None: - out_hidden_states = torch.empty( - (bs, hidden_dim), - dtype=quantize or hidden_states.dtype, - device=hidden_states.device, - ) - else: - assert out.shape == (bs, hidden_dim) - assert out.dtype == (quantize or hidden_states.dtype) - out_hidden_states = out - out_scales = None - static_scale = False - if quantize is not None: - if scales is None: - out_scales = torch.empty( - (bs,), dtype=torch.float32, device=hidden_states.device - ) - else: - out_scales = scales - static_scale = True - - max_warps = 16 if _is_hip else 32 - config = { - # 8 ele per thread (not tuned) - "num_warps": max( - min(triton.next_power_of_2(triton.cdiv(hidden_dim, 8 * 32)), max_warps), 4 - ), - } - - silu_and_mul_kernel[(bs,)]( - out_hidden_states, - out_scales, - hidden_states, - quant_max=torch.finfo(quantize).max if quantize is not None else None, - static_scale=static_scale, - hidden_dim=hidden_dim, - BLOCK_SIZE=triton.next_power_of_2(hidden_dim), - **config, - ) - - if quantize is not None: - return out_hidden_states, out_scales - else: - return out_hidden_states, None - - @triton.jit def _fused_sigmoid_mul_kernel( output_ptr, diff --git a/python/sglang/kernels/ops/elementwise/hc_mix.py b/python/sglang/kernels/ops/elementwise/hc_mix.py index 3919b33e00..cc6054f0d8 100644 --- a/python/sglang/kernels/ops/elementwise/hc_mix.py +++ b/python/sglang/kernels/ops/elementwise/hc_mix.py @@ -10,17 +10,6 @@ def pad_lowrank(lowrank: int) -> int: return (lowrank + _KPAD_ALIGN - 1) // _KPAD_ALIGN * _KPAD_ALIGN -def hc_mix_shape_ok(hc_count: int, hidden_size: int, lowrank: int) -> bool: - k = hc_count * hidden_size - return ( - hidden_size % 8 == 0 - and lowrank > 0 - and lowrank % 8 == 0 - and k % _KPAD_ALIGN == 0 - and k % 128 == 0 - ) - - def permute_pad_up_weight(w_up: torch.Tensor, hc_count: int) -> torch.Tensor: k, lowrank = w_up.shape hidden_size = k // hc_count diff --git a/python/sglang/kernels/ops/elementwise/qwen4_gate.py b/python/sglang/kernels/ops/elementwise/qwen4_gate.py index 1959fb23eb..362a1217f2 100644 --- a/python/sglang/kernels/ops/elementwise/qwen4_gate.py +++ b/python/sglang/kernels/ops/elementwise/qwen4_gate.py @@ -4,20 +4,12 @@ import torch import triton import triton.language as tl +from sglang.kernels.numerics import round_bf16_to_fp32 + _QWEN4_HC_COUNT = 4 _QWEN4_HIDDEN_SIZE = 2560 -@triton.jit -def _round_bf16_to_fp32(value): - """RNE-round fp32 to BF16 precision while retaining an fp32 register.""" - - bits = value.to(tl.int32, bitcast=True) - rounding_bias = 0x7FFF + ((bits >> 16) & 1) - rounded_bits = (bits + rounding_bias) & -65536 - return rounded_bits.to(tl.float32, bitcast=True) - - @triton.jit def _qwen4_gate_value_kernel( gate_ptr, @@ -37,10 +29,10 @@ def _qwen4_gate_value_kernel( # every remaining eager bf16 rounding boundary is reproduced below. gate = tl.load(gate_ptr + token_group).to(tl.float32) magnitude = tl.maximum(tl.abs(gate), 1.0e-6) - root = _round_bf16_to_fp32(tl.sqrt(magnitude)) + root = round_bf16_to_fp32(tl.sqrt(magnitude)) sign = tl.where(gate > 0.0, 1.0, tl.where(gate < 0.0, -1.0, 0.0)) - transformed = _round_bf16_to_fp32(root * sign) - activated = _round_bf16_to_fp32(tl.sigmoid(transformed)) + transformed = round_bf16_to_fp32(root * sign) + activated = round_bf16_to_fp32(tl.sigmoid(transformed)) value = tl.load(value_ptr + token * HIDDEN_SIZE + hidden, mask=mask, other=0.0).to( tl.float32 @@ -106,13 +98,13 @@ def _qwen4_gate_reduce_kernel( query = tl.load(query_ptr + group * HIDDEN_SIZE + hidden, mask, other=0).to( tl.float32 ) - product = _round_bf16_to_fp32(key * query) - gate = _round_bf16_to_fp32(tl.sum(product, 0)) - gate = _round_bf16_to_fp32(gate * (HIDDEN_SIZE**-0.5)) - magnitude = _round_bf16_to_fp32(tl.maximum(tl.abs(gate), 1.0e-6)) - root = _round_bf16_to_fp32(tl.sqrt(magnitude)) + product = round_bf16_to_fp32(key * query) + gate = round_bf16_to_fp32(tl.sum(product, 0)) + gate = round_bf16_to_fp32(gate * (HIDDEN_SIZE**-0.5)) + magnitude = round_bf16_to_fp32(tl.maximum(tl.abs(gate), 1.0e-6)) + root = round_bf16_to_fp32(tl.sqrt(magnitude)) sign = tl.where(gate > 0, 1.0, tl.where(gate < 0, -1.0, 0.0)) - activated = _round_bf16_to_fp32(tl.sigmoid(_round_bf16_to_fp32(root * sign))) + activated = round_bf16_to_fp32(tl.sigmoid(round_bf16_to_fp32(root * sign))) value = tl.load( value_ptr + (group // HC_COUNT) * HIDDEN_SIZE + hidden, mask, other=0 ).to(tl.float32) diff --git a/python/sglang/kernels/ops/embeddings/__init__.py b/python/sglang/kernels/ops/embeddings/__init__.py index 07d916b344..d2f339e864 100644 --- a/python/sglang/kernels/ops/embeddings/__init__.py +++ b/python/sglang/kernels/ops/embeddings/__init__.py @@ -1,7 +1,7 @@ """Embedding kernels.""" from sglang.kernels.registry import register_kernel -from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec +from sglang.kernels.spec import CUDA_ONLY, KernelBackend, KernelSpec register_kernel( KernelSpec( @@ -63,6 +63,6 @@ register_kernel( op="embeddings.fused_qwen4_ngram_hash", backend=KernelBackend.TRITON, target="sglang.kernels.ops.embeddings.qwen4_ngram:fused_qwen4_ngram_hash", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) diff --git a/python/sglang/kernels/ops/gemm/__init__.py b/python/sglang/kernels/ops/gemm/__init__.py index 47b4a0e4ad..515d4c943e 100644 --- a/python/sglang/kernels/ops/gemm/__init__.py +++ b/python/sglang/kernels/ops/gemm/__init__.py @@ -6,12 +6,13 @@ from typing import TYPE_CHECKING, Optional from sglang.kernels.fused_op import BaseFusedOp, register_fused_op from sglang.kernels.registry import register_kernel -from sglang.kernels.selector import get_kernel +from sglang.kernels.selector import _platform, get_kernel from sglang.kernels.spec import ( + CUDA_ONLY, CapabilityRequirement, - FormatSignature, KernelBackend, KernelSpec, + capabilities_satisfied, ) if TYPE_CHECKING: @@ -19,7 +20,6 @@ if TYPE_CHECKING: from sglang.kernels.ops.gemm.cutedsl_dual_gemm import DualGemmQuantMode -_CUDA = frozenset({CapabilityRequirement.CUDA}) _SM90 = frozenset({CapabilityRequirement.cuda(min_sm=(9, 0), max_sm=(9, 0))}) _SM120 = frozenset({CapabilityRequirement.cuda(min_sm=(12, 0), max_sm=(12, 0))}) _SM12X = frozenset({CapabilityRequirement.cuda(min_sm=(12, 0), max_sm=(12, 9))}) @@ -80,20 +80,7 @@ class Fp8ScaledMMOp(BaseFusedOp): op = "gemm.fp8_scaled_mm" priority = (KernelBackend.AOT, KernelBackend.TORCH) - capabilities = { - KernelBackend.AOT: _CUDA, - KernelBackend.TORCH: _SM90, - } - format_signature = FormatSignature( - supported_dtypes=("float8_e4m3fn",), - description=( - "C[M,N] = (A_fp8[M,K] @ B_fp8[K,N]) * scale_a[M,1] * scale_b[1,N] (+ bias)" - ), - ) - descriptions = { - KernelBackend.AOT: "Row/column-scaled FP8 matmul (sgl_kernel wheel).", - KernelBackend.TORCH: "Row/column-scaled FP8 matmul (Torch NVJet on SM90).", - } + capabilities = {KernelBackend.AOT: CUDA_ONLY} def backend_eligible( self, @@ -105,17 +92,11 @@ class Fp8ScaledMMOp(BaseFusedOp): out_dtype: torch.dtype, bias: Optional[torch.Tensor] = None, ) -> bool: + # The torch path is a measured SM90 win for some shapes, not a + # capability limit; elsewhere the AOT kernel always runs. if ( backend is KernelBackend.AOT - and super().backend_eligible( - KernelBackend.TORCH, - mat_a, - mat_b, - scales_a, - scales_b, - out_dtype, - bias, - ) + and capabilities_satisfied(_SM90, _platform()) and _prefer_torch_rowwise_fp8( mat_a, mat_b, scales_a, scales_b, out_dtype, bias ) @@ -173,11 +154,6 @@ register_kernel( backend=KernelBackend.CUTE_DSL, target="sglang.kernels.ops.gemm.cutedsl_dual_gemm:dual_gemm_swiglu", capabilities=_SM10X, - format_signature=FormatSignature( - supported_dtypes=("bfloat16", "float16"), - description="BF16/FP16 gate/up dual GEMM followed by SwiGLU", - ), - description="Blackwell TMA/tcgen05 fused dual GEMM and activation.", ) ) register_kernel( @@ -186,14 +162,6 @@ register_kernel( backend=KernelBackend.CUTE_DSL, target="sglang.kernels.ops.gemm.cutedsl_dual_gemm:dual_gemm_swiglu_fp8", capabilities=_SM10X, - format_signature=FormatSignature( - supported_dtypes=("float8_e4m3fn",), - description=( - "FP8 gate/up dual GEMM, SwiGLU, and static or dynamic " - "per-tensor or per-token FP8 activation quantization" - ), - ), - description="Blackwell TMA/tcgen05 fused dual GEMM and quantization.", ) ) register_kernel( @@ -201,12 +169,7 @@ register_kernel( op="gemm.bmm_fp8", backend=KernelBackend.FLASHINFER, target="sglang.srt.layers.quantization.fp8_utils:bmm_fp8", - capabilities=_CUDA, - format_signature=FormatSignature( - supported_dtypes=("float8_e4m3fn", "float8_e5m2"), - description="batched (3D) per-tensor-scale FP8 matmul: D = A_fp8 @ B_fp8 * A_scale * B_scale", - ), - description="Batched FP8 matmul (flashinfer cuBLAS backend, torch.compile-safe wrapper).", + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -214,11 +177,6 @@ register_kernel( op="gemm.dsv3_fused_a_gemm", backend=KernelBackend.AOT, target="sgl_kernel:dsv3_fused_a_gemm", - format_signature=FormatSignature( - supported_dtypes=("bfloat16",), - description="DeepSeek-V3 fused QKV-A GEMM", - ), - description="DeepSeek-V3 fused-A GEMM (sgl_kernel wheel).", ) ) register_kernel( @@ -226,12 +184,7 @@ register_kernel( op="gemm.dsv3_fused_a_gemm", backend=KernelBackend.JIT, target="sglang.kernels.ops.gemm.dsv3_fused_a_gemm:dsv3_fused_a_gemm", - capabilities=_CUDA, - format_signature=FormatSignature( - supported_dtypes=("bfloat16",), - description="DeepSeek-V3 fused QKV-A GEMM (drop-in with AOT signature)", - ), - description="DeepSeek-V3 fused-A GEMM (sglang.kernels.jit).", + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -239,12 +192,7 @@ register_kernel( op="gemm.tiny_gemm", backend=KernelBackend.JIT, target="sglang.kernels.ops.gemm.tiny_gemm:tiny_gemm_bf16", - capabilities=_CUDA, - format_signature=FormatSignature( - supported_dtypes=("bfloat16",), - description="skinny [m, k] @ [n, k].T with m in [1, max_m]; serves MoE routers and gate projections", - ), - description="Tiny bf16 GEMM (sglang.kernels.jit, JIT-only).", + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -252,12 +200,7 @@ register_kernel( op="gemm.n128k512", backend=KernelBackend.JIT, target="sglang.kernels.ops.gemm.small_gemm_bf16:n128k512_gemm_bf16", - capabilities=_CUDA, - format_signature=FormatSignature( - supported_dtypes=("bfloat16",), - description="[m, 512] @ [128, 512].T for decode batches m <= 32", - ), - description="bf16 GEMM specialised for N = 128, K = 512 (sglang.kernels.jit, JIT-only).", + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -265,12 +208,7 @@ register_kernel( op="gemm.n32k5120", backend=KernelBackend.JIT, target="sglang.kernels.ops.gemm.small_gemm_bf16:n32k5120_gemm_bf16", - capabilities=_CUDA, - format_signature=FormatSignature( - supported_dtypes=("bfloat16",), - description="[m, 5120] @ [32, 5120].T for decode batches m <= 32", - ), - description="bf16 GEMM specialised for N = 32, K = 5120 (sglang.kernels.jit, JIT-only).", + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -279,14 +217,6 @@ register_kernel( backend=KernelBackend.KDA, target=f"{_KDA_PACKAGE}.qwen3x_nvfp4_gemm:try_qwen3x_nvfp4_gemm", capabilities=_SM120, - format_signature=FormatSignature( - supported_dtypes=("uint8", "float8_e4m3fn", "bfloat16"), - description="Qwen3.x ModelOpt NVFP4 decode GEMM on SM120", - ), - description=( - "Shape-specialized Qwen3.x NVFP4 GEMM generated by Kernel Design " - "Agents and implemented with CuTe DSL." - ), ) ) register_kernel( @@ -295,13 +225,6 @@ register_kernel( backend=KernelBackend.KDA, target=f"{_KDA_PACKAGE}.sm120_fp8:try_sm120_fp8_linear", capabilities=_SM12X, - format_signature=FormatSignature( - supported_dtypes=("bfloat16", "float8_e4m3fn", "float32"), - description=( - "Adaptive static per-tensor FP8 linear for small batches on SM12x" - ), - ), - description=("SM12x streaming GEMV plus KDA CUTLASS skinny GEMM dispatch."), ) ) @@ -365,50 +288,6 @@ def bmm_fp8( ) -def dsv3_fused_a_gemm( - mat_a: torch.Tensor, - mat_b: torch.Tensor, - output: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """DeepSeek-V3 fused QKV-A GEMM.""" - return get_kernel("gemm.dsv3_fused_a_gemm", KernelBackend.AOT)(mat_a, mat_b, output) - - -# NOTE: named after the kernel entry, not the op -- a bare `tiny_gemm` here would -# be shadowed by the submodule of the same name as soon as it is imported. -def tiny_gemm_bf16( - x: torch.Tensor, - w: torch.Tensor, - out: Optional[torch.Tensor] = None, - *, - out_dtype: Optional[torch.dtype] = None, - max_m: int = 16, -) -> torch.Tensor: - """Tiny bf16 GEMM ``x[m, k] @ w[n, k].T``, support bf16/fp32 out""" - impl = get_kernel("gemm.tiny_gemm", KernelBackend.JIT) - return impl(x, w, out, out_dtype=out_dtype, max_m=max_m) - - -def n128k512_gemm_bf16( - x: torch.Tensor, - w: torch.Tensor, - out: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """bf16 GEMM ``x[m, 512] @ w[128, 512].T`` for decode batches (m <= 32).""" - impl = get_kernel("gemm.n128k512", KernelBackend.JIT) - return impl(x, w, out) - - -def n32k5120_gemm_bf16( - x: torch.Tensor, - w: torch.Tensor, - out: Optional[torch.Tensor] = None, -) -> torch.Tensor: - """bf16 GEMM ``x[m, 5120] @ w[32, 5120].T`` for decode batches (m <= 32).""" - impl = get_kernel("gemm.n32k5120", KernelBackend.JIT) - return impl(x, w, out) - - def try_qwen3x_nvfp4_gemm( input: torch.Tensor, weight: torch.Tensor, @@ -451,14 +330,10 @@ def kimi_k3_tiny_gemm(x: torch.Tensor, w: torch.Tensor) -> torch.Tensor: __all__ = [ "Fp8ScaledMMOp", "bmm_fp8", - "dsv3_fused_a_gemm", "dual_gemm_swiglu", "dual_gemm_swiglu_fp8", "fp8_scaled_mm", "kimi_k3_tiny_gemm", - "n128k512_gemm_bf16", - "n32k5120_gemm_bf16", - "tiny_gemm_bf16", "try_qwen3x_nvfp4_gemm", "try_sm120_fp8_linear", ] @@ -470,10 +345,6 @@ register_kernel( op="gemm.linear_bf16_fp32", backend=KernelBackend.TORCH, target="sglang.kernels.ops.gemm.bf16_fp32:_linear_bf16_fp32_cublas", - description=( - "Torch/cuBLAS implementation. The direct linear_bf16_fp32 entry also " - "retains its environment-controlled HPC-Ops, AITER and DeepGEMM dispatch." - ), ) ) register_kernel( @@ -481,7 +352,7 @@ register_kernel( op="gemm.wo_a_bf16_gemv", backend=KernelBackend.TRITON, target="sglang.kernels.ops.gemm.dsv4_wo_a:wo_a_bf16_gemv", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -489,7 +360,7 @@ register_kernel( op="gemm.wo_a_bf16_small_batch", backend=KernelBackend.TRITON, target="sglang.kernels.ops.gemm.dsv4_wo_a:wo_a_bf16_small_batch", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -497,7 +368,7 @@ register_kernel( op="gemm.wo_a_bf16_small_batch_mxfp8", backend=KernelBackend.TRITON, target="sglang.kernels.ops.gemm.dsv4_wo_a:wo_a_bf16_small_batch_mxfp8", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -531,11 +402,6 @@ register_kernel( backend=KernelBackend.JIT, target="sglang.kernels.ops.gemm.convrot_int8:convrot_rotate_quantize_activation", capabilities=_CONVROT_INT8, - format_signature=FormatSignature( - supported_dtypes=("bfloat16",), - description="group-wise regular Hadamard rotation + per-row INT8 quant of BF16 [M, K] -> (int8 [M, K], fp32 [M])", - ), - description="ConvRot INT8 activation / weight quantizer (sglang.kernels.jit).", ) ) register_kernel( @@ -544,11 +410,6 @@ register_kernel( backend=KernelBackend.JIT, target="sglang.kernels.ops.gemm.convrot_int8:convrot_int8_fused_linear", capabilities=_CONVROT_INT8, - format_signature=FormatSignature( - supported_dtypes=("bfloat16", "int8"), - description="BF16 [M, K] x int8 [N, K] -> BF16 [M, N] with in-kernel rotation, INT8 quant and fused per-row x per-column dequant (+ bias)", - ), - description="ConvRot INT8 W8A8 linear (sglang.kernels.jit).", ) ) register_kernel( @@ -557,11 +418,6 @@ register_kernel( backend=KernelBackend.JIT, target="sglang.kernels.ops.gemm.convrot_int8:convrot_int8_linear_prequant", capabilities=_CONVROT_INT8, - format_signature=FormatSignature( - supported_dtypes=("int8", "float32", "bfloat16"), - description="int8 [M, K] x int8 [N, K] with per-row / per-column fp32 scales -> BF16 [M, N] (+ bias); for linears sharing one rotated input", - ), - description="ConvRot INT8 GEMM on a pre-quantized activation (sglang.kernels.jit).", ) ) register_kernel( @@ -579,7 +435,7 @@ register_kernel( op="gemm.gptq_marlin_gemm", backend=KernelBackend.JIT, target="sglang.kernels.ops.gemm.gptq_marlin:gptq_marlin_gemm", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -587,7 +443,7 @@ register_kernel( op="gemm.rel_proj_small_t", backend=KernelBackend.JIT, target="sglang.kernels.ops.gemm.inkling_rel_proj:rel_proj_small_t", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -595,14 +451,7 @@ register_kernel( op="gemm.fused_hc_mix", backend=KernelBackend.TRITON, target="sglang.kernels.ops.gemm.hc_mix:fused_hc_mix", - capabilities=frozenset({CapabilityRequirement.CUDA}), - ) -) -register_kernel( - KernelSpec( - op="gemm.w8a8_block_fp8_matmul", - backend=KernelBackend.TRITON, - target="sglang.kernels.ops.gemm.fp8_kernel:w8a8_block_fp8_matmul", + capabilities=CUDA_ONLY, ) ) register_kernel( diff --git a/python/sglang/kernels/ops/gemm/cutedsl_bf16_gemm.py b/python/sglang/kernels/ops/gemm/cutedsl_bf16_gemm.py index 54b1b9b065..c27bcec6b9 100644 --- a/python/sglang/kernels/ops/gemm/cutedsl_bf16_gemm.py +++ b/python/sglang/kernels/ops/gemm/cutedsl_bf16_gemm.py @@ -82,14 +82,6 @@ _TGV_CUTE_EXT_TACTIC_CONFIGS: List[Tuple[int, int, int, bool]] = [ ] -def get_tgv_cute_ext_tactic_num() -> int: - return len(_TGV_CUTE_EXT_TACTIC_CONFIGS) - - -def get_tgv_cute_ext_default_tactic() -> int: - return _TGV_CUTE_EXT_DEFAULT_TACTIC - - class WorkTileInfo(NamedTuple): """Which output tile this CTA processes. Mirrors the original WorkTileInfo.""" diff --git a/python/sglang/kernels/ops/gemm/dense_bf16_gemm_sm100_splitk_epilogue.py b/python/sglang/kernels/ops/gemm/dense_bf16_gemm_sm100_splitk_epilogue.py index 015b942854..a61cd26100 100644 --- a/python/sglang/kernels/ops/gemm/dense_bf16_gemm_sm100_splitk_epilogue.py +++ b/python/sglang/kernels/ops/gemm/dense_bf16_gemm_sm100_splitk_epilogue.py @@ -165,39 +165,6 @@ def validate_tactic( ) -def autotune_tactics( - m: int, - n: int, - k: int, - *, - smem_capacity: int = _SMEM_CAPACITY_BYTES, -) -> list[SplitKTactic]: - """Return valid tactics in the shape-specific stage window.""" - tactics: list[SplitKTactic] = [] - for mma_m in _SUPPORTED_MMA_M: - for mma_n in _SUPPORTED_MMA_N: - for split_k in _SUPPORTED_SPLIT_K: - base = SplitKTactic(mma_m, mma_n, split_k, _MIN_AB_STAGES) - try: - validate_tactic(base, m, n, k, smem_capacity=smem_capacity) - except ValueError: - continue - max_stages = _max_ab_stages_for(base, smem_capacity) - # Short K favors shallow pipelines; long K stays near the cap. - tactics.extend( - dataclasses.replace(base, ab_stages=ab_stages) - for ab_stages in ( - range(_MIN_AB_STAGES, min(max_stages, 6) + 1) - if k <= 4 * _CTA_K - else range( - min(max(5, max_stages - 2), max_stages), - max_stages + 1, - ) - ) - ) - return tactics - - def default_tactic(m: int, n: int, k: int) -> SplitKTactic: """Choose the default occupancy-oriented tactic.""" if n <= 512: @@ -241,7 +208,6 @@ def default_tactic(m: int, n: int, k: int) -> SplitKTactic: __all__ = [ "SplitKTactic", - "autotune_tactics", "default_tactic", "run_splitk_dense", "run_splitk_dense_silu", diff --git a/python/sglang/kernels/ops/gemm/fp8_kernel.py b/python/sglang/kernels/ops/gemm/fp8_kernel.py index ff32551d33..2e7bdd7aaa 100644 --- a/python/sglang/kernels/ops/gemm/fp8_kernel.py +++ b/python/sglang/kernels/ops/gemm/fp8_kernel.py @@ -880,24 +880,6 @@ def w8a8_block_fp8_matmul_triton( return C -def w8a8_block_fp8_matmul( - A: torch.Tensor, - B: torch.Tensor, - As: torch.Tensor, - Bs: torch.Tensor, - block_size: List[int], - output_dtype: torch.dtype = torch.float16, -) -> torch.Tensor: - if output_dtype == torch.bfloat16 and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM: - return w8a8_block_fp8_matmul_deepgemm( - A, B, As, Bs, block_size, output_dtype=output_dtype - ) - - return w8a8_block_fp8_matmul_triton( - A, B, As, Bs, block_size, output_dtype=output_dtype - ) - - def is_weak_contiguous(x: torch.Tensor): strides = x.stride() sizes = x.shape diff --git a/python/sglang/kernels/ops/gemm/smallm_fp8_gfx950/__init__.py b/python/sglang/kernels/ops/gemm/smallm_fp8_gfx950/__init__.py index cd3c0ba7bd..87a1d7c27f 100644 --- a/python/sglang/kernels/ops/gemm/smallm_fp8_gfx950/__init__.py +++ b/python/sglang/kernels/ops/gemm/smallm_fp8_gfx950/__init__.py @@ -5,6 +5,7 @@ Qwen3.5-397B AttnFP8 TP4 and TP2 projection shapes below; SGLANG_ROCM_SMALLM_FP8 """ import ctypes +import logging import os import subprocess import tempfile @@ -18,6 +19,7 @@ from sglang.kernels.ops.moe.smallm_moe_gfx950 import ( _hipcc, _Kernel, ) +from sglang.srt.environ import envs _SRC = os.path.join(os.path.dirname(os.path.abspath(__file__)), "smallm_fp8_gemm.hip") # (N, K) -> ((max M, n-tiles per block, k steps per wave, waves, LDS-staged activations), ...), first match wins. @@ -34,6 +36,7 @@ SHAPES = { } _mod = None _kernels = {} +logger = logging.getLogger(__name__) class Args(ctypes.Structure): @@ -45,7 +48,7 @@ class Args(ctypes.Structure): def smallm_fp8_gemm_enabled() -> bool: global _mod - if os.environ.get("SGLANG_ROCM_SMALLM_FP8_PROJ", "1") == "0": + if not envs.SGLANG_ROCM_SMALLM_FP8_PROJ.get(): return False if _mod is None: _mod = False @@ -59,7 +62,7 @@ def smallm_fp8_gemm_enabled() -> bool: _check(_hip_lib().hipModuleLoad(ctypes.byref(mod), co.encode()), "load") _mod = mod except (OSError, subprocess.CalledProcessError, RuntimeError) as e: - print(f"[smallm_fp8_gemm] disabled: {e}", flush=True) + logger.warning("smallm_fp8_gemm disabled: %s", e) return _mod is not False diff --git a/python/sglang/kernels/ops/kvcache/__init__.py b/python/sglang/kernels/ops/kvcache/__init__.py index 07f2cb2b09..454870cae0 100644 --- a/python/sglang/kernels/ops/kvcache/__init__.py +++ b/python/sglang/kernels/ops/kvcache/__init__.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Optional from sglang.kernels.registry import register_kernel from sglang.kernels.selector import get_kernel -from sglang.kernels.spec import FormatSignature, KernelBackend, KernelSpec +from sglang.kernels.spec import KernelBackend, KernelSpec if TYPE_CHECKING: import torch @@ -21,11 +21,6 @@ register_kernel( op="kvcache.reshape_and_cache_flash", backend=KernelBackend.TRITON, target="sglang.kernels.ops.kvcache.cache_ops:launch_reshape_and_cache_flash", - format_signature=FormatSignature( - in_place=True, - description="write token-major K/V into paged KV cache layout", - ), - description="Reshape-and-cache (Triton launcher).", ) ) diff --git a/python/sglang/kernels/ops/layernorm/__init__.py b/python/sglang/kernels/ops/layernorm/__init__.py index 1e71b01c92..433bfbedf3 100644 --- a/python/sglang/kernels/ops/layernorm/__init__.py +++ b/python/sglang/kernels/ops/layernorm/__init__.py @@ -1,512 +1,12 @@ -"""Layer-normalization kernels. - -Each operator is a :class:`~sglang.kernels.fused_op.BaseFusedOp` with a -pure-``torch`` reference (``forward_native``) plus optimized per-device backends, -all behind one signature. The public module-level functions are thin wrappers -over module-level instances; auto-selection follows the production default for -the live device: AOT ``sgl_kernel`` on CUDA, ``aiter`` (or rocm-triton for -gemma) on ROCm, ``torch_npu`` on Ascend, native reference otherwise. -Pick a specific backend with e.g. -``_RMSNORM.forward(x, w, backend=KernelBackend.JIT)`` or globally via -``SGLANG_FORCE_FUSED_OP_BACKEND``. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Optional - -from sglang.kernels.fused_op import BaseFusedOp, register_fused_op -from sglang.kernels.spec import ( - CapabilityRequirement, - FormatSignature, - KernelBackend, -) - -if TYPE_CHECKING: - import torch - -_NORM_DTYPES = ("float16", "bfloat16") -_CUDA = frozenset({CapabilityRequirement.CUDA}) -_HIP = frozenset({CapabilityRequirement.HIP}) -_NPU = frozenset({CapabilityRequirement.NPU}) -# Unlike the gated-activation ops, sgl_kernel does *not* build the rmsnorm ops -# for ROCm (production: ``if _is_cuda or _is_xpu or _is_musa: from sgl_kernel -# import rmsnorm`` — HIP is absent), so AOT here is CUDA-only. ROCm instead has -# an ``aiter`` path, and Ascend a ``torch_npu`` path — a clean illustration that -# the same ``AOT`` provenance covers different devices per op. -# Priority (best -> fallback) is device-agnostic; per-op CapabilityRequirement -# decides eligibility, so on CUDA this resolves to AOT, on HIP to AITER, on NPU -# to TORCH_NPU, each matching the production default for that device. -_NORM_PRIORITY = ( - KernelBackend.AOT, - KernelBackend.JIT, - KernelBackend.AITER, - KernelBackend.TORCH_NPU, - KernelBackend.TORCH, -) - - -class RMSNormOp(BaseFusedOp): - """``out = (input / RMS(input)) * weight``; returns a tensor. - - ``enable_pdl`` is honored by the AOT backend only. - """ - - op = "layernorm.rmsnorm" - priority = _NORM_PRIORITY - capabilities = { - KernelBackend.AOT: _CUDA, - KernelBackend.JIT: _CUDA, - KernelBackend.AITER: _HIP, - KernelBackend.TORCH_NPU: _NPU, - } - format_signature = FormatSignature( - supported_dtypes=_NORM_DTYPES, - description="out = (x / RMS(x)) * weight; returns tensor", - ) - descriptions = { - KernelBackend.AOT: "RMS normalization (sgl_kernel wheel).", - KernelBackend.JIT: "RMS normalization (sglang.kernels.jit).", - KernelBackend.AITER: "RMS normalization (aiter rmsnorm2d_fwd, ROCm).", - KernelBackend.TORCH_NPU: "RMS normalization (torch_npu, Ascend).", - KernelBackend.TORCH: "RMS normalization (pure-torch reference).", - } - - def forward_native( - self, - input: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - out: Optional[torch.Tensor] = None, - enable_pdl: Optional[bool] = None, - ) -> torch.Tensor: - import torch - - x = input.to(torch.float32) - variance = x.pow(2).mean(dim=-1, keepdim=True) - x = x * torch.rsqrt(variance + eps) - result = (x * weight).to(input.dtype) - if out is None: - return result - out.copy_(result) - return out - - def forward_aot( - self, - input: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - out: Optional[torch.Tensor] = None, - enable_pdl: Optional[bool] = None, - ) -> torch.Tensor: - import sgl_kernel - - return sgl_kernel.rmsnorm(input, weight, eps, out, enable_pdl) - - def forward_jit( - self, - input: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - out: Optional[torch.Tensor] = None, - enable_pdl: Optional[bool] = None, - ) -> torch.Tensor: - import torch - - from sglang.kernels.ops.layernorm.norm import rmsnorm as jit_rmsnorm - - if out is None: - out = torch.empty_like(input) - jit_rmsnorm(input, weight, out, eps) - return out - - def forward_aiter( - self, - input: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - out: Optional[torch.Tensor] = None, - enable_pdl: Optional[bool] = None, - ) -> torch.Tensor: - import torch - from aiter import rmsnorm2d_fwd - - # Mirrors production srt/layers/layernorm.py: rmsnorm2d_fwd(out, x, w, eps) - # writes the normalized result in-place into ``out`` (ROCm path). - if out is None: - out = torch.empty_like(input) - rmsnorm2d_fwd(out, input, weight, eps) - return out - - def forward_torch_npu( - self, - input: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - out: Optional[torch.Tensor] = None, - enable_pdl: Optional[bool] = None, - ) -> torch.Tensor: - import torch_npu - - result = torch_npu.npu_rms_norm(input, weight, eps)[0] - if out is None: - return result - out.copy_(result) - return out - - -class FusedAddRMSNormOp(BaseFusedOp): - """In-place ``residual += input; input = RMSNorm(residual) * weight``. - - Writes the sum into ``residual`` and the normalized value into ``input``; - returns ``None``. ``enable_pdl`` is honored by the AOT backend only. - """ - - op = "layernorm.fused_add_rmsnorm" - priority = _NORM_PRIORITY - capabilities = { - KernelBackend.AOT: _CUDA, - KernelBackend.JIT: _CUDA, - KernelBackend.AITER: _HIP, - KernelBackend.TORCH_NPU: _NPU, - } - format_signature = FormatSignature( - supported_dtypes=_NORM_DTYPES, - in_place=True, - description="residual += x; x = RMSNorm(residual) * weight", - ) - descriptions = { - KernelBackend.AOT: ( - "Fused residual-add + RMS normalization (sgl_kernel wheel)." - ), - KernelBackend.JIT: ( - "Fused residual-add + RMS normalization (sglang.kernels.jit)." - ), - KernelBackend.AITER: ("Fused residual-add + RMS normalization (aiter, ROCm)."), - KernelBackend.TORCH_NPU: ( - "Fused residual-add + RMS normalization (torch_npu, Ascend)." - ), - KernelBackend.TORCH: ( - "Fused residual-add + RMS normalization (pure-torch reference)." - ), - } - - def forward_native( - self, - input: torch.Tensor, - residual: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - enable_pdl: Optional[bool] = None, - ) -> None: - import torch - - acc = input.to(torch.float32) + residual.to(torch.float32) - residual.copy_(acc.to(residual.dtype)) - variance = acc.pow(2).mean(dim=-1, keepdim=True) - normed = acc * torch.rsqrt(variance + eps) - input.copy_((normed * weight).to(input.dtype)) - - def forward_aot( - self, - input: torch.Tensor, - residual: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - enable_pdl: Optional[bool] = None, - ) -> None: - import sgl_kernel - - return sgl_kernel.fused_add_rmsnorm(input, residual, weight, eps, enable_pdl) - - def forward_jit( - self, - input: torch.Tensor, - residual: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - enable_pdl: Optional[bool] = None, - ) -> None: - from sglang.kernels.ops.layernorm.norm import ( - fused_add_rmsnorm as jit_fused_add_rmsnorm, - ) - - return jit_fused_add_rmsnorm(input, residual, weight, eps) - - def forward_aiter( - self, - input: torch.Tensor, - residual: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - enable_pdl: Optional[bool] = None, - ) -> None: - import torch - from aiter import rmsnorm2d_fwd_with_add - - # aiter writes the normalized value and the new residual into separate - # out buffers (production call order: out, x, residual_out, residual, w, - # eps); copy them back to honor this op's in-place contract. - out = torch.empty_like(input) - residual_out = torch.empty_like(residual) - rmsnorm2d_fwd_with_add(out, input, residual_out, residual, weight, eps) - input.copy_(out) - residual.copy_(residual_out) - - def forward_torch_npu( - self, - input: torch.Tensor, - residual: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - enable_pdl: Optional[bool] = None, - ) -> None: - import torch_npu - - # torch_npu.npu_add_rms_norm(residual, x, w, eps) -> (normed, _, new_sum) - out, _, residual_out = torch_npu.npu_add_rms_norm(residual, input, weight, eps) - input.copy_(out) - residual.copy_(residual_out) - - -class GemmaRMSNormOp(BaseFusedOp): - """``out = (input / RMS(input)) * (weight + 1)``; returns a tensor.""" - - op = "layernorm.gemma_rmsnorm" - priority = _NORM_PRIORITY - # AOT (sgl_kernel) on CUDA; JIT is the ROCm rocm-triton path - # (sglang.kernels.ops.moe.minimax_m3_swiglu) — a JIT provenance pinned to HIP, distinct - # from the CUDA-only JIT on the plain rmsnorm ops; torch_npu on Ascend. - capabilities = { - KernelBackend.AOT: _CUDA, - KernelBackend.JIT: _HIP, - KernelBackend.TORCH_NPU: _NPU, - } - format_signature = FormatSignature( - supported_dtypes=_NORM_DTYPES, - description="out = (x / RMS(x)) * (weight + 1); returns tensor", - ) - descriptions = { - KernelBackend.AOT: "Gemma-style RMS normalization (sgl_kernel wheel).", - KernelBackend.JIT: ( - "Gemma-style RMS normalization (rocm-triton, sglang.kernels.jit)." - ), - KernelBackend.TORCH_NPU: ("Gemma-style RMS normalization (torch_npu, Ascend)."), - KernelBackend.TORCH: "Gemma-style RMS normalization (pure-torch reference).", - } - - def forward_native( - self, - input: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - out: Optional[torch.Tensor] = None, - enable_pdl: Optional[bool] = None, - ) -> torch.Tensor: - import torch - - x = input.to(torch.float32) - variance = x.pow(2).mean(dim=-1, keepdim=True) - x = x * torch.rsqrt(variance + eps) - result = (x * (1.0 + weight.to(torch.float32))).to(input.dtype) - if out is None: - return result - out.copy_(result) - return out - - def forward_aot( - self, - input: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - out: Optional[torch.Tensor] = None, - enable_pdl: Optional[bool] = None, - ) -> torch.Tensor: - import sgl_kernel - - return sgl_kernel.gemma_rmsnorm(input, weight, eps, out, enable_pdl) - - def forward_jit( - self, - input: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - out: Optional[torch.Tensor] = None, - enable_pdl: Optional[bool] = None, - ) -> torch.Tensor: - from sglang.kernels.ops.layernorm.minimax_m3_rmsnorm import ( - gemma_rmsnorm as rocm_triton_gemma_rmsnorm, - ) - - result = rocm_triton_gemma_rmsnorm(input, weight, eps) - if out is None: - return result - out.copy_(result) - return out - - def forward_torch_npu( - self, - input: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - out: Optional[torch.Tensor] = None, - enable_pdl: Optional[bool] = None, - ) -> torch.Tensor: - import torch_npu - - result = torch_npu.npu_gemma_rms_norm(input, weight, eps)[0] - if out is None: - return result - out.copy_(result) - return out - - -class GemmaFusedAddRMSNormOp(BaseFusedOp): - """In-place ``residual += input; input = GemmaRMSNorm(residual) * (weight + 1)``.""" - - op = "layernorm.gemma_fused_add_rmsnorm" - priority = _NORM_PRIORITY - # AOT (sgl_kernel) on CUDA; JIT is the ROCm rocm-triton path on HIP. - # NPU here would use ``sgl_kernel_npu.add_gemma_rms_norm`` (a distinct AOT-npu - # wheel provenance, not torch_npu) — deferred until that provenance lands. - capabilities = { - KernelBackend.AOT: _CUDA, - KernelBackend.JIT: _HIP, - } - format_signature = FormatSignature( - supported_dtypes=_NORM_DTYPES, - in_place=True, - description="residual += x; x = GemmaRMSNorm(residual) * (weight + 1)", - ) - descriptions = { - KernelBackend.AOT: ("Gemma-style fused residual-add + RMS normalization."), - KernelBackend.JIT: ( - "Gemma-style fused residual-add + RMS normalization " - "(rocm-triton, sglang.kernels.jit)." - ), - KernelBackend.TORCH: ( - "Gemma-style fused residual-add + RMS normalization (pure-torch reference)." - ), - } - - def forward_native( - self, - input: torch.Tensor, - residual: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - enable_pdl: Optional[bool] = None, - ) -> None: - import torch - - acc = input.to(torch.float32) + residual.to(torch.float32) - residual.copy_(acc.to(residual.dtype)) - variance = acc.pow(2).mean(dim=-1, keepdim=True) - normed = acc * torch.rsqrt(variance + eps) - input.copy_((normed * (1.0 + weight.to(torch.float32))).to(input.dtype)) - - def forward_aot( - self, - input: torch.Tensor, - residual: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - enable_pdl: Optional[bool] = None, - ) -> None: - import sgl_kernel - - return sgl_kernel.gemma_fused_add_rmsnorm( - input, residual, weight, eps, enable_pdl - ) - - def forward_jit( - self, - input: torch.Tensor, - residual: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - enable_pdl: Optional[bool] = None, - ) -> None: - from sglang.kernels.ops.layernorm.minimax_m3_rmsnorm import ( - gemma_fused_add_rmsnorm as rocm_triton_gemma_fused_add_rmsnorm, - ) - - # rocm-triton returns (normed, new_residual); honor the in-place contract. - norm_out, residual_out = rocm_triton_gemma_fused_add_rmsnorm( - input, residual, weight, eps - ) - input.copy_(norm_out) - residual.copy_(residual_out) - - -_RMSNORM = register_fused_op(RMSNormOp(), __name__, "_RMSNORM") -_FUSED_ADD_RMSNORM = register_fused_op( - FusedAddRMSNormOp(), __name__, "_FUSED_ADD_RMSNORM" -) -_GEMMA_RMSNORM = register_fused_op(GemmaRMSNormOp(), __name__, "_GEMMA_RMSNORM") -_GEMMA_FUSED_ADD_RMSNORM = register_fused_op( - GemmaFusedAddRMSNormOp(), __name__, "_GEMMA_FUSED_ADD_RMSNORM" -) - - -def rmsnorm( - input: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - out: Optional[torch.Tensor] = None, - enable_pdl: Optional[bool] = None, -) -> torch.Tensor: - """RMS normalization: ``out = (input / RMS(input)) * weight``.""" - return _RMSNORM(input, weight, eps, out, enable_pdl) - - -def fused_add_rmsnorm( - input: torch.Tensor, - residual: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - enable_pdl: Optional[bool] = None, -) -> None: - """In-place fused residual add + RMS normalization.""" - return _FUSED_ADD_RMSNORM(input, residual, weight, eps, enable_pdl) - - -def gemma_rmsnorm( - input: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - out: Optional[torch.Tensor] = None, - enable_pdl: Optional[bool] = None, -) -> torch.Tensor: - """Gemma-style RMS normalization: ``out = (input / RMS(input)) * (weight + 1)``.""" - return _GEMMA_RMSNORM(input, weight, eps, out, enable_pdl) - - -def gemma_fused_add_rmsnorm( - input: torch.Tensor, - residual: torch.Tensor, - weight: torch.Tensor, - eps: float = 1e-6, - enable_pdl: Optional[bool] = None, -) -> None: - """In-place Gemma-style fused residual add + RMS normalization.""" - return _GEMMA_FUSED_ADD_RMSNORM(input, residual, weight, eps, enable_pdl) - - -__all__ = [ - "RMSNormOp", - "FusedAddRMSNormOp", - "GemmaRMSNormOp", - "GemmaFusedAddRMSNormOp", - "rmsnorm", - "fused_add_rmsnorm", - "gemma_rmsnorm", - "gemma_fused_add_rmsnorm", -] - +"""Layer-normalization kernels.""" from sglang.kernels.registry import register_kernel -from sglang.kernels.spec import KernelSpec +from sglang.kernels.spec import ( + CUDA_ONLY, + CapabilityRequirement, + KernelBackend, + KernelSpec, +) # Triton / TileLang kernels migrated from srt/layers top-level strays # (RFC #29630, Phase 2.5); registered for inventory. @@ -562,7 +62,7 @@ register_kernel( op="layernorm.rmsnorm_hf", backend=KernelBackend.JIT, target="sglang.kernels.ops.layernorm.rmsnorm_hf:rmsnorm_hf", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -570,14 +70,7 @@ register_kernel( op="layernorm.grouped_gemma_rmsnorm", backend=KernelBackend.JIT, target="sglang.kernels.ops.layernorm.grouped_gemma_rmsnorm:grouped_gemma_rmsnorm", - capabilities=frozenset({CapabilityRequirement.CUDA}), - ) -) -register_kernel( - KernelSpec( - op="layernorm.rms_normalize_triton", - backend=KernelBackend.TRITON, - target="sglang.kernels.ops.layernorm.rms_normalize_hip:rms_normalize_triton", + capabilities=CUDA_ONLY, ) ) @@ -589,9 +82,5 @@ register_kernel( capabilities=frozenset( {CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))} ), - format_signature=FormatSignature( - supported_dtypes=_NORM_DTYPES, - description="Three weighted RMSNorm outputs sharing a single-CTA row reduction.", - ), ) ) diff --git a/python/sglang/kernels/ops/layernorm/mhc.py b/python/sglang/kernels/ops/layernorm/mhc.py index bc7c7035c3..d6701a06fa 100644 --- a/python/sglang/kernels/ops/layernorm/mhc.py +++ b/python/sglang/kernels/ops/layernorm/mhc.py @@ -305,49 +305,6 @@ def hc_split_sinkhorn_kernel(hc: int, sinkhorn_iters: int, eps: float): return hc_split_sinkhorn_kernel_ -def _hc_split_sinkhorn_torch( - mixes: torch.Tensor, - hc_scale: torch.Tensor, - hc_base: torch.Tensor, - hc_mult: int = 4, - sinkhorn_iters: int = 20, - eps: float = 1e-6, -): - """Pure-torch equivalent of hc_split_sinkhorn_kernel. - - TileLang's CK-backed buffer addressing does not compile on gfx1250, so the - sinkhorn kernel is reimplemented here. Layout mirrors the kernel exactly: - the flattened ``mixes`` row holds ``pre`` (hc), ``post`` (hc) and the - ``comb`` matrix (hc * hc) consecutively. - """ - b, s, _ = mixes.size() - hc = hc_mult - flat = mixes.reshape(-1, (2 + hc) * hc).float() - scale = hc_scale.float() - base = hc_base.float() - - pre = torch.sigmoid(flat[:, :hc] * scale[0] + base[:hc]) + eps - post = 2 * torch.sigmoid(flat[:, hc : 2 * hc] * scale[1] + base[hc : 2 * hc]) - - comb = flat[:, 2 * hc :] * scale[2] + base[2 * hc :] - comb = comb.reshape(-1, hc, hc) - - # Initial row softmax (numerically stabilized) then column normalize. - row_max = comb.amax(dim=2, keepdim=True) - comb = torch.exp(comb - row_max) - comb = comb / comb.sum(dim=2, keepdim=True) + eps - comb = comb / (comb.sum(dim=1, keepdim=True) + eps) - - for _ in range(sinkhorn_iters - 1): - comb = comb / (comb.sum(dim=2, keepdim=True) + eps) - comb = comb / (comb.sum(dim=1, keepdim=True) + eps) - - pre = pre.reshape(b, s, hc).to(mixes.dtype) - post = post.reshape(b, s, hc).to(mixes.dtype) - comb = comb.reshape(b, s, hc, hc).to(mixes.dtype) - return pre, post, comb - - @triton.jit def _hc_split_sinkhorn_triton_kernel( mixes_ptr, @@ -486,7 +443,7 @@ def hc_split_sinkhorn( ) if is_gfx1250_supported(): # TileLang's CK-backed addressing doesn't compile on gfx1250; use the - # Triton port. _hc_split_sinkhorn_torch is kept as a reference fallback. + # Triton port. return _hc_split_sinkhorn_triton( mixes, hc_scale, hc_base, hc_mult, sinkhorn_iters, eps ) diff --git a/python/sglang/kernels/ops/layernorm/minimax_m3_rmsnorm.py b/python/sglang/kernels/ops/layernorm/minimax_m3_rmsnorm.py index 790c3141a0..44384590f2 100644 --- a/python/sglang/kernels/ops/layernorm/minimax_m3_rmsnorm.py +++ b/python/sglang/kernels/ops/layernorm/minimax_m3_rmsnorm.py @@ -15,6 +15,7 @@ import triton import triton.language as tl +# Gemma (1 + weight) RMSNorm over strided rows; no other Triton RMSNorm covers that. @triton.jit def _gemma_rmsnorm_kernel( x_ptr, diff --git a/python/sglang/kernels/ops/layernorm/norm.py b/python/sglang/kernels/ops/layernorm/norm.py index c3a8e948f3..6b961cb8eb 100644 --- a/python/sglang/kernels/ops/layernorm/norm.py +++ b/python/sglang/kernels/ops/layernorm/norm.py @@ -83,19 +83,6 @@ def _jit_fused_add_rmsnorm_module( ) -@cache_once -def _jit_qknorm_across_heads_module(dtype: torch.dtype) -> Module: - args = make_cpp_args(dtype) - return load_jit( - "qknorm_across_heads", - *args, - cuda_files=["elementwise/qknorm_across_heads.cuh"], - cuda_wrappers=[ - ("qknorm_across_heads", f"QKNormAcrossHeadsKernel<{args}>::run") - ], - ) - - @torch.compiler.assume_constant_result @cache_once def can_use_fused_inplace_qknorm(head_dim: int, dtype: torch.dtype) -> bool: @@ -155,25 +142,3 @@ def fused_add_rmsnorm( ) -> None: module = _jit_fused_add_rmsnorm_module(input.dtype, cast_x_before_out_mul) module.fused_add_rmsnorm(input, residual, weight, eps) - - -@debug_kernel_api -def fused_inplace_qknorm_across_heads( - q: torch.Tensor, - k: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, - eps: float = 1e-6, -) -> None: - """ - Fused inplace QK normalization across all heads. - - Args: - q: Query tensor of shape [batch_size, num_heads * head_dim] - k: Key tensor of shape [batch_size, num_heads * head_dim] - q_weight: Query weight tensor of shape [num_heads * head_dim] - k_weight: Key weight tensor of shape [num_heads * head_dim] - eps: Epsilon for numerical stability - """ - module = _jit_qknorm_across_heads_module(q.dtype) - module.qknorm_across_heads(q, k, q_weight, k_weight, eps) diff --git a/python/sglang/kernels/ops/layernorm/rms_normalize_hip.py b/python/sglang/kernels/ops/layernorm/rms_normalize_hip.py deleted file mode 100644 index bbd1e5476e..0000000000 --- a/python/sglang/kernels/ops/layernorm/rms_normalize_hip.py +++ /dev/null @@ -1,52 +0,0 @@ -"""RMS-normalize kernel used by the HIP DSV4 compressor. - -Migrated from ``sglang.srt.layers.attention.dsv4.compress_hip`` (RFC #29630, Phase 2.5). -""" - -import torch -import triton -import triton.language as tl - - -@triton.jit -def _rms_normalize_kernel( - x_ptr, - weight_ptr, - eps, - stride_row, - dim, - BLOCK_SIZE: tl.constexpr, - HAS_WEIGHT: tl.constexpr, -): - pid = tl.program_id(0) - offs = tl.arange(0, BLOCK_SIZE) - mask = offs < dim - base = pid * stride_row - x = tl.load(x_ptr + base + offs, mask=mask, other=0.0).to(tl.float32) - mean_sq = tl.sum(x * x, axis=0) / dim - rms_inv = tl.rsqrt(mean_sq + eps) - out = x * rms_inv - if HAS_WEIGHT: - weight = tl.load(weight_ptr + offs, mask=mask, other=0.0) - out = out * weight - tl.store(x_ptr + base + offs, out, mask=mask) - - -def rms_normalize_triton( - x: torch.Tensor, eps: float, weight: torch.Tensor = None -) -> torch.Tensor: - dim = x.shape[-1] - x_flat = x.view(-1, dim) - num_rows = x_flat.shape[0] - BLOCK_SIZE = triton.next_power_of_2(dim) - grid = (num_rows,) - _rms_normalize_kernel[grid]( - x_flat, - weight, - eps, - x_flat.stride(0), - dim, - BLOCK_SIZE=BLOCK_SIZE, - HAS_WEIGHT=(weight is not None), - ) - return x diff --git a/python/sglang/kernels/ops/lplb/cublasdx_solver.py b/python/sglang/kernels/ops/lplb/cublasdx_solver.py deleted file mode 100644 index d1f57182a4..0000000000 --- a/python/sglang/kernels/ops/lplb/cublasdx_solver.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Backwards-compatible shim. - -The Numba/nvmath-python fused IPM that used to live here has been replaced -by a CUDA C++ kernel JIT-compiled via sglang's ``load_jit`` infrastructure. -The new implementation lives in ``cuda_solver``. This module re-exports the -public API so any external import keeps working. -""" - -from sglang.kernels.ops.lplb.cuda_solver import ( # noqa: F401 - solve_ipm, - warmup, -) - -__all__ = ["solve_ipm", "warmup"] diff --git a/python/sglang/kernels/ops/lplb/cuda_solver.py b/python/sglang/kernels/ops/lplb/cuda_solver.py index 3a658cdf25..7c18c8ff75 100644 --- a/python/sglang/kernels/ops/lplb/cuda_solver.py +++ b/python/sglang/kernels/ops/lplb/cuda_solver.py @@ -1,6 +1,5 @@ """JIT-compiled CUDA Interior Point Method LP solver. -Replaces the Numba/nvmath-python implementation in ``cublasdx_solver.py``. The kernel is a single-block fused IPM defined in ``csrc/lplb/ipm.cuh`` and compiled per ``(NC, NV, BLOCK_DIM, SM_VER, NUM_ITERS)`` tuple via sglang's ``tvm-ffi`` ``load_jit``. @@ -222,8 +221,9 @@ def dispatch_probability( Samples a physical expert per (token, slot) via inverse-CDF on the per-row LP probabilities. Bit-equivalent to - :func:`dispatch_probability_torch_reference` when given the same - ``random_vals`` (modulo float rounding in the cumulative sum). + ``sglang.test.kernels.lplb.dispatch_probability_torch_reference`` when + given the same ``random_vals`` (modulo float rounding in the cumulative + sum). Args: topk_ids: (num_tokens, topk) int32, on CUDA. Logical expert ids from @@ -256,69 +256,3 @@ def dispatch_probability( module = _dispatch_module(max_copies, DISPATCH_BLOCK_DIM) module.dispatch_probability(out, flat_ids, log2phy_prob, map32, random_vals) return out.view(original_shape).to(topk_ids.dtype) - - -def dispatch_probability_torch_reference( - topk_ids: torch.Tensor, - log2phy_prob: torch.Tensor, - log2phy_map: torch.Tensor, - random_vals: torch.Tensor, -) -> torch.Tensor: - """Pure-torch reference of :func:`dispatch_probability`. - - Mirrors the CUDA kernel's algorithm exactly (inverse-CDF via cumsum - + threshold) so the two paths are bit-equivalent for identical - ``random_vals``, modulo floating-point rounding in the cumsum. Kept - for numerical comparison and testing — not on the production hot - path (it allocates and runs ~8 torch ops; the fused kernel collapses - them into one launch). - - Algorithm (matches ``csrc/lplb/dispatch_probability.cuh``): - - 1. Gather the per-row probability vector and physical-id map for - each logical id in ``topk_ids``. - 2. If the row sum is zero (LP gave no signal), fall back to - uniform over valid replicas (``log2phy_map != -1``). - 3. Sample: smallest ``c`` such that ``cumsum[0..c] > u * row_sum``, - where ``u = random_vals[i]``. Ties favor advancing ``c``, - matching the CUDA kernel. - 4. Return ``log2phy_map[logical_id, c]``. - - Args: - topk_ids: (num_tokens, topk) int, on CUDA or CPU. Logical expert ids. - log2phy_prob: (num_logical, max_copies) float32. LP solver output. - log2phy_map: (num_logical, max_copies) int. -1 = unused replica. - random_vals: (N,) float32, where N = ``topk_ids.numel()``. Uniform - samples in [0, 1) — same shape and semantics as the CUDA kernel. - - Returns: - Physical topk ids tensor with the same shape and dtype as ``topk_ids``. - """ - original_shape = topk_ids.shape - flat_ids = topk_ids.reshape(-1).long() - n = flat_ids.shape[0] - num_logical, max_copies = log2phy_prob.shape - assert log2phy_map.shape == (num_logical, max_copies) - assert random_vals.shape == (n,), ( - f"random_vals must be shape ({n},), got {tuple(random_vals.shape)}" - ) - - # Gather per-row probabilities and physical maps. - probs = log2phy_prob[flat_ids] # (N, max_copies), float32 - maps = log2phy_map[flat_ids] # (N, max_copies), same int dtype as input - - # Fallback when row_sum == 0: uniform over valid replicas. - row_sum = probs.sum(dim=-1, keepdim=True) # (N, 1) - fallback_probs = (maps >= 0).to(probs.dtype) # (N, max_copies) - probs = torch.where(row_sum > 0, probs, fallback_probs) - row_sum = probs.sum(dim=-1) # (N,) - - # Inverse-CDF sample: smallest c such that cumsum[..c] > u. - # ``(cum <= u).sum(dim=-1)`` counts how many slots are still below u, - # which equals the CUDA kernel's ``chosen`` after its for-loop. - u = (random_vals * row_sum).unsqueeze(-1) # (N, 1) - cum = probs.cumsum(dim=-1) # (N, max_copies) - chosen = (cum <= u).sum(dim=-1).clamp(max=max_copies - 1) # (N,) - - out = maps.gather(1, chosen.unsqueeze(-1)).squeeze(-1) - return out.view(original_shape).to(topk_ids.dtype) diff --git a/python/sglang/kernels/ops/lplb/shmem_budget.py b/python/sglang/kernels/ops/lplb/shmem_budget.py index 99b3d49466..f17e02d142 100644 --- a/python/sglang/kernels/ops/lplb/shmem_budget.py +++ b/python/sglang/kernels/ops/lplb/shmem_budget.py @@ -117,27 +117,6 @@ def assert_fits(nc: int, nv: int, gpu: str = "h100") -> None: ) -def max_nc_for_nv(nv: int, gpu: str = "h100") -> int: - """Largest NC that fits for a given NV. Solves - 4 * (NC^2 + (NV+1)*NC + 3*NV) + pad <= cap - via the quadratic formula (monotone in NC). Returns 0 if even NC=1 overflows. - """ - cap = gpu_budget_bytes(gpu) - b = _BYTES_PER_ELEM - # cap - pad >= b * (NC^2 + (NV+1)*NC + 3*NV) - rhs = (cap - _RUNTIME_PAD_BYTES) / b - 3 * nv - if rhs <= 0: - return 0 - # NC^2 + (NV+1)*NC - rhs <= 0 - import math - - disc = (nv + 1) ** 2 + 4 * rhs - nc_max = int((-(nv + 1) + math.sqrt(disc)) / 2.0) - while nc_max > 0 and shmem_bytes(nc_max, nv) > cap: - nc_max -= 1 - return max(nc_max, 0) - - def report(nc: int, nv: int, gpu: str = "h100") -> str: """Human-readable summary — used by kernels on init for logging.""" bd = breakdown(nc, nv) diff --git a/python/sglang/kernels/ops/lplb/torch_solver.py b/python/sglang/kernels/ops/lplb/torch_solver.py index 9c555ae718..ef484b11ba 100644 --- a/python/sglang/kernels/ops/lplb/torch_solver.py +++ b/python/sglang/kernels/ops/lplb/torch_solver.py @@ -134,59 +134,3 @@ def solve_ipm( f"LPLB fused solver requires float32; got A.dtype={A.dtype}." ) return _FUSED_SOLVE_IPM(A, b, c, num_iters=num_iters) - - -def solve_ipm_torch_reference( - A: torch.Tensor, - b: torch.Tensor, - c: torch.Tensor, - num_iters: int = 5, -) -> torch.Tensor: - """Pure-torch reference for the fused IPM kernel — testing only. - - Mirrors the barrier-method iteration in ``csrc/lplb/ipm.cuh`` - step-for-step so the two can be compared numerically: - - x <- 1 - for _ in range(num_iters): - ax2 = A * x^2 # (NC, NV) - ax2a = ax2 @ A^T # (NC, NC) KKT matrix - delta = solve(ax2a, ax2 @ c) - r = delta^T @ A # (NV,) - d = x * (c - r) - alpha = 0.999 / d_max (or 1.0 if d_max <= 1e-9) - x *= 1 - alpha * d - write 0.5 everywhere on non-convergence. - - NOT bit-equivalent to the kernel: the kernel factors the KKT system - with a hand-written block Cholesky while this uses - ``torch.linalg.solve`` (LU). The two agree to a small tolerance - (the numerical difference being the whole point of the comparison - test). This function is never on the production path — the fused - kernel is the only LP solver at runtime. - """ - nc, nv = A.shape - assert b.shape == (nc,), f"b shape mismatch: {b.shape} vs ({nc},)" - assert c.shape == (nv,), f"c shape mismatch: {c.shape} vs ({nv},)" - - x = torch.ones(nv, device=A.device, dtype=torch.float32) - d_max = torch.tensor(0.0, device=A.device, dtype=torch.float32) - for _ in range(num_iters): - ax2 = A * (x * x).unsqueeze(0) # (NC, NV) - ax2a = ax2 @ A.t() # (NC, NC) - ax2c = ax2 @ c # (NC,) - # Match the kernel's 1e-12 pivot clamp via a tiny diagonal jitter so - # a (near-)singular KKT system stays solvable instead of raising. - ax2a = ax2a + 1e-12 * torch.eye(nc, device=A.device, dtype=torch.float32) - delta = torch.linalg.solve(ax2a, ax2c) # (NC,) - r = A.t() @ delta # (NV,) - d = x * (c - r) # (NV,) - d_max = d.max() - alpha = 0.999 / d_max if d_max > 1e-9 else torch.tensor(1.0, device=A.device) - x = x * (1.0 - alpha * d) - - max_residual = (A @ x - b).abs().max() - converged = (d_max < 0.1) and (0 <= x[-1] < 1e-4) and (max_residual < 0.05) - if not converged: - return torch.full((nv,), 0.5, device=A.device, dtype=torch.float32) - return x diff --git a/python/sglang/kernels/ops/mamba/__init__.py b/python/sglang/kernels/ops/mamba/__init__.py index 7a55597e64..a7649e0148 100644 --- a/python/sglang/kernels/ops/mamba/__init__.py +++ b/python/sglang/kernels/ops/mamba/__init__.py @@ -6,17 +6,11 @@ from typing import TYPE_CHECKING, Optional from sglang.kernels.registry import register_kernel from sglang.kernels.selector import get_kernel -from sglang.kernels.spec import ( - CapabilityRequirement, - FormatSignature, - KernelBackend, - KernelSpec, -) +from sglang.kernels.spec import CUDA_ONLY, KernelBackend, KernelSpec if TYPE_CHECKING: import torch -_CUDA = frozenset({CapabilityRequirement.CUDA}) # JIT is the only backend: the AOT kernel it replaced was built for CUDA alone, # never by the ROCm / MUSA / Metal extensions. Non-CUDA resolves nothing here -- @@ -27,11 +21,7 @@ register_kernel( op="mamba.causal_conv1d_fwd", backend=KernelBackend.JIT, target="sglang.kernels.ops.mamba.causal_conv1d:causal_conv1d_fwd", - capabilities=_CUDA, - format_signature=FormatSignature( - in_place=True, description="causal depthwise conv1d forward (prefill)" - ), - description="Causal conv1d forward (sglang.kernels.jit).", + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -39,11 +29,7 @@ register_kernel( op="mamba.causal_conv1d_update", backend=KernelBackend.JIT, target="sglang.kernels.ops.mamba.causal_conv1d:causal_conv1d_update", - capabilities=_CUDA, - format_signature=FormatSignature( - in_place=True, description="causal depthwise conv1d update (decode)" - ), - description="Causal conv1d update (sglang.kernels.jit).", + capabilities=CUDA_ONLY, ) ) @@ -122,7 +108,7 @@ register_kernel( op="mamba.fused_qwen4_short_conv_state", backend=KernelBackend.TRITON, target="sglang.kernels.ops.mamba.qwen4_short_conv:fused_qwen4_short_conv_state", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -130,7 +116,7 @@ register_kernel( op="mamba.fused_qwen4_verify_conv", backend=KernelBackend.TRITON, target="sglang.kernels.ops.mamba.qwen4_short_conv:fused_qwen4_verify_conv", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -138,7 +124,7 @@ register_kernel( op="mamba.fused_lfm_short_conv_prefill", backend=KernelBackend.TRITON, target="sglang.kernels.ops.mamba.lfm_short_conv:fused_lfm_short_conv_prefill", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -146,7 +132,7 @@ register_kernel( op="mamba.fused_lfm_short_conv_decode", backend=KernelBackend.TRITON, target="sglang.kernels.ops.mamba.lfm_short_conv:fused_lfm_short_conv_decode", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) @@ -155,9 +141,6 @@ register_kernel( op="mamba.fill_track_conv_indices", backend=KernelBackend.TRITON, target="sglang.kernels.ops.mamba.sconv_tracking:fill_track_conv_indices", - capabilities=_CUDA, - format_signature=FormatSignature( - in_place=True, description="short-convolution checkpoint-window indices" - ), + capabilities=CUDA_ONLY, ) ) diff --git a/python/sglang/kernels/ops/mamba/qwen4_short_conv.py b/python/sglang/kernels/ops/mamba/qwen4_short_conv.py index d586d4f1e3..44f8302a8f 100644 --- a/python/sglang/kernels/ops/mamba/qwen4_short_conv.py +++ b/python/sglang/kernels/ops/mamba/qwen4_short_conv.py @@ -4,7 +4,7 @@ import torch import triton import triton.language as tl -from sglang.kernels.ops.elementwise.qwen4_gate import _round_bf16_to_fp32 +from sglang.kernels.numerics import round_bf16_to_fp32 _QWEN4_MAX_SHORT_CONV_STATE_LEN = 16 @@ -216,7 +216,7 @@ def _qwen4_verify_conv_finish_kernel( conv = tl.load( conv_ptr + (req * CHANNELS + channel) * WIDTH + step, mask, other=0 ).to(tl.float32) - activated = _round_bf16_to_fp32(conv * tl.sigmoid(conv)) + activated = round_bf16_to_fp32(conv * tl.sigmoid(conv)) residual = tl.load(residual_ptr + offsets, mask, other=0).to(tl.float32) valid = tl.load(valid_ptr + token, mask, other=0) tl.store(output_ptr + offsets, tl.where(valid, activated + residual, 0), mask) diff --git a/python/sglang/kernels/ops/memory/__init__.py b/python/sglang/kernels/ops/memory/__init__.py index bcb6618dee..216bd25bce 100644 --- a/python/sglang/kernels/ops/memory/__init__.py +++ b/python/sglang/kernels/ops/memory/__init__.py @@ -6,7 +6,7 @@ The Triton kernels migrated here live in this package """ from sglang.kernels.registry import register_kernel -from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec +from sglang.kernels.spec import CUDA_ONLY, KernelBackend, KernelSpec # (module, public_fn) migrated from mem_cache/triton_ops. _TRITON_KERNELS = [ @@ -51,28 +51,12 @@ register_kernel( # Public entry points inventoried by logical operator group (RFC #29630). -register_kernel( - KernelSpec( - op="memory.adler32_checksum", - backend=KernelBackend.JIT, - target="sglang.kernels.ops.memory.adler32:adler32_checksum", - capabilities=frozenset({CapabilityRequirement.CUDA}), - ) -) -register_kernel( - KernelSpec( - op="memory.adler32_regions_checksum", - backend=KernelBackend.JIT, - target="sglang.kernels.ops.memory.adler32:adler32_regions_checksum", - capabilities=frozenset({CapabilityRequirement.CUDA}), - ) -) register_kernel( KernelSpec( op="memory.adler32_strided_checksum", backend=KernelBackend.JIT, target="sglang.kernels.ops.memory.adler32:adler32_strided_checksum", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -80,6 +64,6 @@ register_kernel( op="memory.row_compact_bf16", backend=KernelBackend.JIT, target="sglang.kernels.ops.memory.row_compact:row_compact_bf16", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) diff --git a/python/sglang/kernels/ops/memory/adler32.py b/python/sglang/kernels/ops/memory/adler32.py index 36b48f72a9..782bde71bb 100644 --- a/python/sglang/kernels/ops/memory/adler32.py +++ b/python/sglang/kernels/ops/memory/adler32.py @@ -1,11 +1,7 @@ """Adler-32 checksum for GPU tensors. -Three entry points: - - adler32_checksum(tensor) -> int : checksum all bytes of a contiguous tensor - - adler32_regions_checksum(data_ptrs, lengths, device) -> int : checksum - multiple raw GPU regions in order - - adler32_strided_checksum(data_ptrs, strides, indices) -> int : checksum - selected items across multiple tensors in a single kernel launch +adler32_strided_checksum(data_ptrs, strides, indices) -> int checksums selected +items across multiple tensors in a single kernel launch. """ from __future__ import annotations @@ -26,7 +22,6 @@ def _jit_adler32_module() -> Module: "adler32_checksum", cuda_files=["elementwise/adler32_checksum.cuh"], cuda_wrappers=[ - ("adler32_whole", "Adler32WholeKernel::run"), ("adler32_strided", "Adler32StridedKernel::run"), ("adler32_reduce", "Adler32ReduceKernel::run"), ], @@ -34,8 +29,6 @@ def _jit_adler32_module() -> Module: _BLOCK_SIZE = 256 -_MAX_BLOCKS = 1024 -_REGION_CHUNK_SIZE = 512 * 1024 def _alloc_states(num_blocks: int, device: torch.device) -> torch.Tensor: @@ -49,51 +42,6 @@ def _reduce_states(module: Module, states: torch.Tensor, num_blocks: int) -> int return result.item() -def adler32_checksum(tensor: torch.Tensor) -> int: - """Compute Adler-32 checksum of entire tensor (all bytes).""" - assert tensor.is_contiguous(), "tensor must be contiguous" - - module = _jit_adler32_module() - num_bytes = tensor.numel() * tensor.element_size() - if num_bytes == 0: - return 1 - - total_threads = min(num_bytes, _BLOCK_SIZE * _MAX_BLOCKS) - chunk_size = max(1, (num_bytes + total_threads - 1) // total_threads) - num_threads_needed = (num_bytes + chunk_size - 1) // chunk_size - num_blocks = (num_threads_needed + _BLOCK_SIZE - 1) // _BLOCK_SIZE - - states = _alloc_states(num_blocks, tensor.device) - data_view = tensor.view(torch.uint8) - module.adler32_whole(data_view, states, num_bytes, chunk_size) - return _reduce_states(module, states, num_blocks) - - -def adler32_regions_checksum( - data_ptrs: list[int], lengths: list[int], device: torch.device -) -> int: - assert len(data_ptrs) == len(lengths) - module = _jit_adler32_module() - chunk_ptrs = [] - chunk_lengths = [] - for data_ptr, length in zip(data_ptrs, lengths): - assert data_ptr > 0 and length >= 0 - offset = 0 - while offset < length: - chunk_length = min(_REGION_CHUNK_SIZE, length - offset) - chunk_ptrs.append(data_ptr + offset) - chunk_lengths.append(chunk_length) - offset += chunk_length - if not chunk_ptrs: - return 1 - ptrs = torch.tensor(chunk_ptrs, dtype=torch.int64, device=device) - lens = torch.tensor(chunk_lengths, dtype=torch.int32, device=device) - num_blocks = (len(chunk_ptrs) + _BLOCK_SIZE - 1) // _BLOCK_SIZE - states = _alloc_states(num_blocks, torch.device(device)) - module.adler32_strided(ptrs, lens, states, len(chunk_ptrs)) - return _reduce_states(module, states, num_blocks) - - def adler32_strided_checksum( data_ptrs: list[int], strides: list[int], diff --git a/python/sglang/kernels/ops/mm/__init__.py b/python/sglang/kernels/ops/mm/__init__.py index c7569b8a3b..8f9881ecff 100644 --- a/python/sglang/kernels/ops/mm/__init__.py +++ b/python/sglang/kernels/ops/mm/__init__.py @@ -1,7 +1,7 @@ """Multimodal input-processing kernels; implementations are imported lazily.""" from sglang.kernels.registry import register_kernel -from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec +from sglang.kernels.spec import CUDA_ONLY, KernelBackend, KernelSpec __all__ = ["process"] @@ -12,7 +12,7 @@ register_kernel( op="mm.normalize_and_patchify", backend=KernelBackend.TRITON, target="sglang.kernels.ops.mm.process.image:normalize_and_patchify", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) diff --git a/python/sglang/kernels/ops/moe/__init__.py b/python/sglang/kernels/ops/moe/__init__.py index 1f75216749..c09ecb74fc 100644 --- a/python/sglang/kernels/ops/moe/__init__.py +++ b/python/sglang/kernels/ops/moe/__init__.py @@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, Optional from sglang.kernels.registry import register_kernel from sglang.kernels.selector import get_kernel from sglang.kernels.spec import ( + CUDA_ONLY, CapabilityRequirement, - FormatSignature, KernelBackend, KernelSpec, ) @@ -16,7 +16,6 @@ from sglang.kernels.spec import ( if TYPE_CHECKING: import torch -_CUDA = frozenset({CapabilityRequirement.CUDA}) _HIP = frozenset({CapabilityRequirement.HIP}) register_kernel( @@ -25,7 +24,6 @@ register_kernel( backend=KernelBackend.TRITON, target="sglang.kernels.ops.moe.shared_expert_gate:shared_expert_gate", capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))}), - description="FP32 sigmoid gate from a single BF16 row of width 2560, with PDL.", ) ) @@ -34,11 +32,6 @@ register_kernel( op="moe.moe_align_block_size", backend=KernelBackend.AOT, target="sgl_kernel:moe_align_block_size", - format_signature=FormatSignature( - in_place=True, - description="align/sort expert token ids into block-padded buffers", - ), - description="MoE align-block-size (sgl_kernel wheel).", ) ) register_kernel( @@ -46,12 +39,7 @@ register_kernel( op="moe.moe_align_block_size", backend=KernelBackend.JIT, target="sglang.kernels.ops.moe.moe_align:moe_align_block_size", - capabilities=_CUDA, - format_signature=FormatSignature( - in_place=True, - description="MoE align-block-size (JIT variant, AOT signature)", - ), - description="MoE align-block-size (sglang.kernels.jit).", + capabilities=CUDA_ONLY, ) ) register_kernel( @@ -60,11 +48,6 @@ register_kernel( backend=KernelBackend.AOT, target="sgl_kernel:topk_softmax", capabilities=_HIP, - format_signature=FormatSignature( - in_place=True, - description="top-k softmax routing weights/ids", - ), - description="MoE top-k softmax (sgl_kernel ROCm wheel).", ) ) register_kernel( @@ -72,12 +55,7 @@ register_kernel( op="moe.topk_softmax", backend=KernelBackend.JIT, target="sglang.kernels.ops.moe.moe_topk_softmax:topk_softmax", - capabilities=_CUDA, - format_signature=FormatSignature( - in_place=True, - description="top-k softmax routing weights/ids", - ), - description="MoE top-k softmax (sglang.kernels.jit).", + capabilities=CUDA_ONLY, ) ) @@ -206,12 +184,7 @@ register_kernel( op="moe.moe_align_small_numel", backend=KernelBackend.TRITON, target="sglang.kernels.ops.moe.moe_align_small_numel:moe_align_small_numel", - capabilities=_CUDA, - format_signature=FormatSignature( - in_place=True, - description="align/sort expert token ids into block-padded buffers", - ), - description="MoE align-block-size, single-launch triton variant.", + capabilities=CUDA_ONLY, ) ) @@ -223,8 +196,7 @@ register_kernel( op="moe.shuffle_rows_with_scales", backend=KernelBackend.TRITON, target="sglang.kernels.ops.moe.shuffle_rows_with_scales:shuffle_rows_with_scales", - capabilities=_CUDA, - description="Row gather of quantized values plus their scales, one launch.", + capabilities=CUDA_ONLY, ) ) @@ -235,7 +207,7 @@ register_kernel( op="moe.situ_and_mul_masked_post_quant", backend=KernelBackend.JIT, target="sglang.kernels.ops.moe._jit_situ_and_mul_quant:situ_and_mul_masked_post_quant", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) @@ -246,7 +218,7 @@ register_kernel( op="moe.gemma_routing_post_topk", backend=KernelBackend.TRITON, target="sglang.kernels.ops.moe.gemma4_routing:gemma_routing_post_topk", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) register_kernel( diff --git a/python/sglang/kernels/ops/moe/ep_moe_kernels.py b/python/sglang/kernels/ops/moe/ep_moe_kernels.py index ba4d737303..dec42b8167 100644 --- a/python/sglang/kernels/ops/moe/ep_moe_kernels.py +++ b/python/sglang/kernels/ops/moe/ep_moe_kernels.py @@ -972,48 +972,6 @@ def post_reorder_deepgemm( ) -@triton.jit -def post_reorder_triton_kernel( - down_output_ptr, - output_ptr, - src2dst_ptr, - topk_ids_ptr, - topk_weights_ptr, - topk, - hidden_size, - BLOCK_SIZE: tl.constexpr, -): - InDtype = down_output_ptr.dtype.element_ty - - src_idx_int32 = tl.program_id(0) - src_idx = src_idx_int32.to(tl.int64) - src2dst_ptr = src2dst_ptr + src_idx * topk - topk_ids_ptr = topk_ids_ptr + src_idx * topk - topk_weights_ptr = topk_weights_ptr + src_idx * topk - - store_ptr = output_ptr + src_idx * hidden_size - - vec = tl.arange(0, BLOCK_SIZE) - - for start_offset in tl.range(0, hidden_size, BLOCK_SIZE): - offset = start_offset + vec - mask = offset < hidden_size - - sum_vec = tl.zeros([BLOCK_SIZE], dtype=tl.float32) - for idx in range(topk): - expert_id = tl.load(topk_ids_ptr + idx) - if expert_id >= 0: - dst_idx_int32 = tl.load(src2dst_ptr + idx) - dst_idx = dst_idx_int32.to(tl.int64) - weigh_scale = tl.load(topk_weights_ptr + idx).to(tl.float32) - load_ptr = down_output_ptr + dst_idx * hidden_size - # accumulate expert outputs in fp32 for better precision - # before casting to the final output dtype. - in_data = tl.load(load_ptr + offset, mask=mask).to(tl.float32) - sum_vec += in_data * weigh_scale - tl.store(store_ptr + offset, sum_vec.to(InDtype), mask=mask) - - @triton.jit def _fwd_kernel_ep_scatter_1( num_recv_tokens_per_expert, diff --git a/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py b/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py index 592dcf3197..9a15978bbe 100644 --- a/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py +++ b/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py @@ -20,10 +20,10 @@ from sglang.kernels.ops.quantization.int8_kernel import ( sglang_per_token_group_quant_int8, ) from sglang.srt.batch_invariant_ops import is_batch_invariant_mode_enabled +from sglang.srt.environ import envs from sglang.srt.layers.moe.utils import get_moe_padding_size from sglang.srt.utils import ( cpu_has_amx_support, - get_bool_env_var, is_cpu, is_cuda, is_hip, @@ -41,7 +41,7 @@ _is_hip = is_hip() _is_cuda = is_cuda() _is_cpu_amx_available = cpu_has_amx_support() _is_cpu = is_cpu() -_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip +_use_aiter = envs.SGLANG_USE_AITER.get() and _is_hip if _is_cuda: pass diff --git a/python/sglang/kernels/ops/moe/inkling_gate_topk_renorm.py b/python/sglang/kernels/ops/moe/inkling_gate_topk_renorm.py index 452275c850..42b1c309a8 100644 --- a/python/sglang/kernels/ops/moe/inkling_gate_topk_renorm.py +++ b/python/sglang/kernels/ops/moe/inkling_gate_topk_renorm.py @@ -1,11 +1,10 @@ """Shape-specialized Inkling MoE gate top-k + renorm JIT kernels. -Three families, all specialized for the Inkling gate layout (logits +Two families, both specialized for the Inkling gate layout (logits ``[tokens, 258]`` fp32 = 256 routed + 2 shared experts, top-6 selection by ``sigmoid(logit) + bias``, logsigmoid renorm over selected ++ shared): -- ``inkling_gate_topk_renorm`` -- v1 warp-per-row gate (int64 indices). -- ``inkling_gate_topk_renorm_v2`` -- v2 gate: wide vector loads, int32 +- ``inkling_gate_topk_renorm_v2`` -- gate: wide vector loads, int32 indices, optional PDL, in-register raw-logit carry (no re-gather). - ``inkling_gate_gemv`` / ``inkling_gate_gemv_fused`` -- expert-per-block GEMV of the gate linear (x [tokens, 6144] bf16 @ W [264, 6144] bf16 -> fp32 @@ -41,8 +40,6 @@ def _jit_module() -> Module: "fast_math", cuda_files=["moe/inkling_gate_topk_renorm.cuh"], cuda_wrappers=[ - ("inkling_gate_topk_renorm", "inkling_gate_topk_renorm"), - ("inkling_gate_topk_renorm_packed", "inkling_gate_topk_renorm_packed"), ("inkling_gate_topk_renorm_v2", "inkling_gate_topk_renorm_v2"), ( "inkling_gate_topk_renorm_v2_packed", @@ -56,21 +53,6 @@ def _jit_module() -> Module: ) -def _launch_inkling_gate_topk_renorm( - logits: torch.Tensor, - bias: torch.Tensor, - global_scale: torch.Tensor, - routed_w: torch.Tensor, - shared_w: torch.Tensor, - indices: torch.Tensor, - route_scale: float, -) -> None: - module = _jit_module() - module.inkling_gate_topk_renorm( - logits, bias, global_scale, routed_w, shared_w, indices, float(route_scale) - ) - - def _check_gate_inputs( logits: torch.Tensor, bias: torch.Tensor, global_scale: torch.Tensor ) -> None: @@ -81,65 +63,6 @@ def _check_gate_inputs( assert global_scale.numel() == 1 -def inkling_gate_topk_renorm( - logits: torch.Tensor, - bias: torch.Tensor, - global_scale: torch.Tensor, - route_scale: float, - *, - return_packed: bool = False, -) -> ( - tuple[torch.Tensor, torch.Tensor, torch.Tensor] | tuple[torch.Tensor, torch.Tensor] -): - """Select top-6 routed experts from 256 and renorm with 2 shared experts. - - This is specialized for the Inkling fused gate layout: - ``logits`` is ``[tokens, 258]`` fp32, where columns ``0:256`` are routed - experts and columns ``256:258`` are shared experts. The top-k selection key is - ``sigmoid(logits[:, :256]) + bias``; renorm is over sigmoid(raw logits) for - the selected routed experts plus both shared experts. - - ``return_packed=True`` emits the FlashInfer routed-MoE pack instead of the - routed_w + indices pair: ``packed[t,6]`` int32 = ``(expert_id << 16) | bf16 - weight bits``. Returns ``(packed, shared_w)``. - """ - _check_gate_inputs(logits, bias, global_scale) - - tokens = logits.shape[0] - shared_w = torch.empty( - (tokens, _N_SHARED), dtype=torch.float32, device=logits.device - ) - if return_packed: - packed = torch.empty((tokens, _TOPK), dtype=torch.int32, device=logits.device) - if tokens == 0: - return packed, shared_w - _jit_module().inkling_gate_topk_renorm_packed( - logits, - bias.contiguous(), - global_scale.contiguous(), - packed, - shared_w, - float(route_scale), - ) - return packed, shared_w - - routed_w = torch.empty((tokens, _TOPK), dtype=torch.float32, device=logits.device) - indices = torch.empty((tokens, _TOPK), dtype=torch.int64, device=logits.device) - if tokens == 0: - return routed_w, shared_w, indices - - _launch_inkling_gate_topk_renorm( - logits, - bias.contiguous(), - global_scale.contiguous(), - routed_w, - shared_w, - indices, - route_scale, - ) - return routed_w, shared_w, indices - - def inkling_gate_topk_renorm_v2( logits: torch.Tensor, bias: torch.Tensor, @@ -150,7 +73,12 @@ def inkling_gate_topk_renorm_v2( enable_pdl: bool = False, warps_per_block: int = 0, ) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor, torch.Tensor | None]: - """v2 gate kernel; same math as v1 but int32 indices and optional PDL. + """Select top-6 routed experts from 256 and renorm with 2 shared experts. + + ``logits`` is ``[tokens, 258]`` fp32: columns ``0:256`` are routed experts + and ``256:258`` shared experts. The selection key is + ``sigmoid(logits[:, :256]) + bias``; renorm is over sigmoid(raw logits) of + the selected routed experts plus both shared experts. Returns ``(routed_w, indices, shared_w, packed)`` where the unused half is ``None`` depending on ``return_packed`` -- mirroring the triton diff --git a/python/sglang/kernels/ops/moe/moe_fused_gate.py b/python/sglang/kernels/ops/moe/moe_fused_gate.py index 2a7e17f643..8d554ed6fd 100644 --- a/python/sglang/kernels/ops/moe/moe_fused_gate.py +++ b/python/sglang/kernels/ops/moe/moe_fused_gate.py @@ -1,25 +1,16 @@ from __future__ import annotations -import logging -from typing import TYPE_CHECKING, Dict, Optional, Tuple +from typing import Dict, Optional, Tuple import torch import triton import triton.language as tl from triton.language.extra import libdevice -from sglang.kernels.jit.utils import ( - cache_once, - get_jit_cuda_arch, - is_arch_support_pdl, - load_jit, -) +from sglang.kernels.jit.utils import get_jit_cuda_arch, is_arch_support_pdl from sglang.kernels.kernel_api_logging import debug_kernel_api from sglang.kernels.ops.moe import moe_route_radix -if TYPE_CHECKING: - from tvm_ffi.module import Module - _SCORING_FUNC_MAP = { "sigmoid": 0, "sqrtsoftplus": 1, @@ -27,71 +18,6 @@ _SCORING_FUNC_MAP = { } -@cache_once -def _jit_moe_fused_gate_module() -> Module: - return load_jit( - "moe_fused_gate", - cuda_files=["moe/moe_fused_gate.cuh"], - cuda_wrappers=[("moe_fused_gate", "MoEFusedGateKernel::run")], - ) - - -@cache_once -def can_use_moe_fused_gate() -> bool: - logger = logging.getLogger(__name__) - try: - _jit_moe_fused_gate_module() - return True - except Exception as e: - logger.warning(f"Failed to load JIT MoE fused gate kernel: {e}") - return False - - -def moe_fused_gate_jit( - input: torch.Tensor, - bias: torch.Tensor, - topk: int, - scoring_func: str = "sigmoid", - num_fused_shared_experts: int = 0, - renormalize: bool = True, - routed_scaling_factor: float = 1.0, - apply_routed_scaling_factor_on_output: bool = False, -) -> Tuple[torch.Tensor, torch.Tensor]: - scoring_func_int = _SCORING_FUNC_MAP.get(scoring_func.lower()) - assert scoring_func_int is not None, ( - f"Unknown scoring_func '{scoring_func}', must be one of {list(_SCORING_FUNC_MAP.keys())}" - ) - - assert input.dtype == torch.float32, "input must be float32" - assert bias.dtype == torch.float32, "bias must be float32" - assert input.ndim == 2, "input must be 2D" - assert bias.ndim == 1, "bias must be 1D" - assert input.size(1) == bias.size(0), "input and bias must have same num_experts" - assert topk > num_fused_shared_experts, "topk must be > num_fused_shared_experts" - - num_rows, _ = input.shape - device = input.device - - output = torch.empty(num_rows, topk, dtype=torch.float32, device=device) - indices = torch.empty(num_rows, topk, dtype=torch.int32, device=device) - - module = _jit_moe_fused_gate_module() - module.moe_fused_gate( - input, - bias, - output, - indices, - topk, - scoring_func_int, - num_fused_shared_experts, - renormalize, - routed_scaling_factor, - apply_routed_scaling_factor_on_output, - ) - - return output, indices - - @triton.jit def _router_triton_kernel( scores_ptr, # [M, N] raw logits, fp32/fp16/bf16 (upcast to fp32 on load) @@ -357,8 +283,6 @@ def moe_fused_gate( ) -> Tuple[torch.Tensor, torch.Tensor]: """Triton fused router: scoring + bias + topk + (optional) renorm/scale. - Mirrors :func:`moe_fused_gate_jit` (the CUDA JIT kernel) for the shared - parameters; the keyword-only extras are Triton-only. With ``num_expert_group > 1`` it performs DeepSeek-V3 grouped routing (per-group top-2-sum group scores, keep ``topk_group`` groups, then top-k within). ``scores`` contains raw GEMM logits. diff --git a/python/sglang/kernels/ops/moe/moe_route_quant_fused.py b/python/sglang/kernels/ops/moe/moe_route_quant_fused.py index 58857df2f8..41e2d857d6 100644 --- a/python/sglang/kernels/ops/moe/moe_route_quant_fused.py +++ b/python/sglang/kernels/ops/moe/moe_route_quant_fused.py @@ -74,11 +74,16 @@ def covered( scores: torch.Tensor, bias: torch.Tensor, topk: int, x: torch.Tensor ) -> bool: """route_radix coverage plus the quant half: [M<=64, 3584] bf16 rows with - 32B-aligned starts (base and stride), same token count as the scores.""" + 32B-aligned starts (base and stride). + + Activations disagreeing with the scores on token count is a caller bug, not + an unsupported layout (the only staging site splits both from one GEMM + output), so it is left to the launcher, which reports it as "scores and + activations must have the same token count". + """ return ( moe_route_radix.covered(scores, bias, topk) and x.dim() == 2 - and x.shape[0] == scores.shape[0] and 0 < x.shape[0] <= _MAX_TOKENS and x.shape[1] == _HIDDEN and x.dtype in (torch.bfloat16, torch.float32) diff --git a/python/sglang/kernels/ops/moe/rocm_moe_utils.py b/python/sglang/kernels/ops/moe/rocm_moe_utils.py index 92f60e1cc5..ef6ba9a10e 100644 --- a/python/sglang/kernels/ops/moe/rocm_moe_utils.py +++ b/python/sglang/kernels/ops/moe/rocm_moe_utils.py @@ -1,122 +1,10 @@ # Adapted from https://github.com/vllm-project/vllm/blob/v0.9.1rc2/vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from enum import IntEnum -from typing import Optional - import torch import triton import triton.language as tl -from sglang.srt.utils import get_bool_env_var, is_hip -from sglang.srt.utils.custom_op import register_custom_op - -_is_hip = is_hip() -_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip - - -class ActivationMethod(IntEnum): - # This allows interfacing with AITER ActivationType enum - # without importing the ActivationType enum from AITER globally. - SILU = 0 - GELU = 1 - - -# NOTE: for non _use_aiter case, use lazy registration to avoid overhead -# (registration may not be trigger actually, since it will not be called) -@register_custom_op(out_shape="hidden_states", eager=_use_aiter) -def rocm_aiter_asm_moe_tkw1( - hidden_states: torch.Tensor, - w1: torch.Tensor, - w2: torch.Tensor, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - fc1_scale: Optional[torch.Tensor] = None, - fc2_scale: Optional[torch.Tensor] = None, - fc1_smooth_scale: Optional[torch.Tensor] = None, - fc2_smooth_scale: Optional[torch.Tensor] = None, - a16: bool = False, - per_tensor_quant_scale: Optional[torch.Tensor] = None, - expert_mask: Optional[torch.Tensor] = None, - activation_method: int = ActivationMethod.SILU.value, -) -> torch.Tensor: - - from aiter import ActivationType - from aiter.fused_moe_bf16_asm import asm_moe_tkw1 - - activation = ActivationType(activation_method) - - return asm_moe_tkw1( - hidden_states, - w1, - w2, - topk_weights, - topk_ids, - fc1_scale=fc1_scale, - fc2_scale=fc2_scale, - fc1_smooth_scale=fc1_smooth_scale, - fc2_smooth_scale=fc2_smooth_scale, - a16=a16, - per_tensor_quant_scale=per_tensor_quant_scale, - expert_mask=expert_mask, - activation=activation, - ) - - -def rocm_fused_experts_tkw1( - hidden_states: torch.Tensor, - w1: torch.Tensor, - w2: torch.Tensor, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - activation: str = "silu", - apply_router_weight_on_input: bool = False, - use_fp8_w8a8: bool = False, - per_channel_quant: bool = False, - w1_scale: Optional[torch.Tensor] = None, - w2_scale: Optional[torch.Tensor] = None, - a1_scale: Optional[torch.Tensor] = None, - a2_scale: Optional[torch.Tensor] = None, - block_shape: Optional[list[int]] = None, -) -> torch.Tensor: - - activation_method = ( - ActivationMethod.SILU if activation == "silu" else ActivationMethod.GELU - ) - # All AITER Fused MoE kernels are expecting the following datatypes - topk_weights = topk_weights.to(torch.float32) - topk_ids = topk_ids.to(torch.int32) - - # w8a8 per-channel quantization - if per_channel_quant and apply_router_weight_on_input and use_fp8_w8a8: - # AITER tkw1 kernel for FP8 models with `apply_router_weight_on_input` - # This applies topk_weights on the GEMM output of the first FC layer - # rather than the second FC. - assert topk_weights.dim() == 2, ( - "`topk_weights` should be in shape (num_tokens, topk)" - ) - assert topk_weights.shape[-1] == 1, ( - "Only support topk=1 when `apply_router_weight_on_input` is True" - ) - - return rocm_aiter_asm_moe_tkw1( - hidden_states, - w1, - w2, - topk_weights, - topk_ids, - fc1_scale=w1_scale, - fc2_scale=w2_scale, - fc1_smooth_scale=None, - fc2_smooth_scale=None, - a16=False, - per_tensor_quant_scale=None, - expert_mask=None, - activation_method=activation_method, - ) - else: - assert False, "This should not be called." - @triton.jit def upscale_kernel( diff --git a/python/sglang/kernels/ops/moe/smallm_moe_gfx950/__init__.py b/python/sglang/kernels/ops/moe/smallm_moe_gfx950/__init__.py index 6501b96968..97268acf9d 100644 --- a/python/sglang/kernels/ops/moe/smallm_moe_gfx950/__init__.py +++ b/python/sglang/kernels/ops/moe/smallm_moe_gfx950/__init__.py @@ -16,6 +16,7 @@ On by default; SGLANG_ROCM_SMALLM_MOE=0 turns it off. Any build/load failure dis from __future__ import annotations import ctypes +import logging import os import subprocess import tempfile @@ -25,10 +26,10 @@ import torch from sglang.srt.distributed.device_communicators.cuda_wrapper import ( find_loaded_library, ) +from sglang.srt.environ import envs _DIR = os.path.dirname(os.path.abspath(__file__)) _SRC = os.path.join(_DIR, "smallm_moe.hip") -_ENV = "SGLANG_ROCM_SMALLM_MOE" DIM = 4096 MAX_TOK = 64 # kernel list capacity (tokens per expert) and workspace size # Dispatch cap: above this token count aiter's flydsl path is faster (crossover ~44 @@ -43,6 +44,7 @@ _available = None _disabled_reason = None _build_dir = None _ws: dict = {} +logger = logging.getLogger(__name__) class SmallMMoeUnavailable(RuntimeError): @@ -166,7 +168,7 @@ def smallm_moe_enabled() -> bool: (hidden 4096, top-10 plus the optional fused shared expert). There is no model-name check; smallm_moe_supported() enforces the shape, and every other model or parallel layout keeps aiter fused_moe.""" global _available - if os.environ.get(_ENV, "1") == "0": + if not envs.SGLANG_ROCM_SMALLM_MOE.get(): return False if _available is None: arch = "" @@ -178,9 +180,11 @@ def smallm_moe_enabled() -> bool: and os.path.exists(_hipcc()) ) if not _available: - print( - f"[smallm_moe] not enabled (arch={arch!r}, hip={_hip_version()}, hipcc={os.path.exists(_hipcc())}); aiter fused_moe stays in use", - flush=True, + logger.info( + "smallm_moe not enabled (arch=%r, hip=%s, hipcc=%s); aiter fused_moe stays in use", + arch, + _hip_version(), + os.path.exists(_hipcc()), ) return _available and _disabled_reason is None @@ -188,9 +192,9 @@ def smallm_moe_enabled() -> bool: def _disable(reason): global _disabled_reason _disabled_reason = reason - print( - f"[smallm_moe] disabling the gfx950 small-M MoE kernel, falling back to aiter fused_moe: {reason}", - flush=True, + logger.warning( + "smallm_moe: disabling the gfx950 small-M MoE kernel, falling back to aiter fused_moe: %s", + reason, ) raise SmallMMoeUnavailable(reason) @@ -229,10 +233,7 @@ def _get_kernels(inter: int, pre: str): except RuntimeError as e: _disable(str(e)) _kernels["_mod"] = mod - print( - f"[smallm_moe] gfx950 small-M MoE kernels active (built {co})", - flush=True, - ) + logger.info("smallm_moe: gfx950 small-M MoE kernels active (built %s)", co) mod = _kernels["_mod"] try: kk = { diff --git a/python/sglang/kernels/ops/moe/smallm_router_gfx950/__init__.py b/python/sglang/kernels/ops/moe/smallm_router_gfx950/__init__.py index 13e6a9a8d6..1a360f28e5 100644 --- a/python/sglang/kernels/ops/moe/smallm_router_gfx950/__init__.py +++ b/python/sglang/kernels/ops/moe/smallm_router_gfx950/__init__.py @@ -3,6 +3,7 @@ softmax top-10 + renormalize + fused shared-expert slot, written as the [M, 11] smallm_moe_gfx950 consumes. SGLANG_ROCM_SMALLM_ROUTER=0 disables it; a build/load failure disables it too.""" import ctypes +import logging import os import subprocess import tempfile @@ -18,6 +19,7 @@ MAX_TOK = 8 # MAX_TOK in smallm_router.hip; from 9 tokens the unfused path is f _kernels: dict = {} _ws: dict = {} _disabled = False +logger = logging.getLogger(__name__) class Args(ctypes.Structure): @@ -67,7 +69,7 @@ def smallm_router(x, gate_weight, shared_gate_weight, scale=1.0): _check(_hip_lib().hipModuleLoad(ctypes.byref(mod), co.encode()), "load") _kernels.update({r: _Kernel(mod, f"smallm_router_r{r}") for r in (1, 2)}) except (OSError, subprocess.CalledProcessError, RuntimeError) as e: - print(f"[smallm_router] disabled: {e}", flush=True) + logger.warning("smallm_router disabled: %s", e) _disabled = True return None tok = x.shape[0] diff --git a/python/sglang/kernels/ops/quantization/__init__.py b/python/sglang/kernels/ops/quantization/__init__.py index ea7775c7fd..67a88fa4b0 100644 --- a/python/sglang/kernels/ops/quantization/__init__.py +++ b/python/sglang/kernels/ops/quantization/__init__.py @@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, Optional from sglang.kernels.registry import register_kernel from sglang.kernels.selector import get_kernel from sglang.kernels.spec import ( + CUDA_ONLY, CapabilityRequirement, - FormatSignature, KernelBackend, KernelSpec, ) @@ -16,24 +16,16 @@ from sglang.kernels.spec import ( if TYPE_CHECKING: import torch -_CUDA = frozenset({CapabilityRequirement.CUDA}) register_kernel( KernelSpec( op="quantization.sgl_per_token_quant_fp8", backend=KernelBackend.JIT, target="sglang.kernels.ops.quantization.per_token_quant_fp8:per_token_quant_fp8", - capabilities=_CUDA, - format_signature=FormatSignature( - supported_dtypes=("float8_e4m3fn",), - in_place=True, - description="per-token FP8 quantization into output_q/output_s", - ), - description="Per-token FP8 quantization (sglang.kernels.jit).", + capabilities=CUDA_ONLY, ) ) -# fp8 / int8 are legacy aliases of the same 8bit kernel in the wheel; register -# each public name so runtime imports resolve to a stable spec. +# fp8 / int8 are legacy aliases of the same 8bit kernel in the wheel. for _name in ( "sgl_per_token_group_quant_8bit", "sgl_per_token_group_quant_fp8", @@ -44,11 +36,6 @@ for _name in ( op=f"quantization.{_name}", backend=KernelBackend.AOT, target=f"sgl_kernel:{_name}", - format_signature=FormatSignature( - in_place=True, - description="per-token-group 8-bit quantization", - ), - description=f"{_name} (sgl_kernel wheel).", ) ) del _name @@ -58,17 +45,7 @@ register_kernel( op="quantization.per_token_group_quant", backend=KernelBackend.JIT, target="sglang.kernels.ops.quantization.per_token_group_quant:per_token_group_quant", - capabilities=_CUDA, - format_signature=FormatSignature( - supported_dtypes=("float8_e4m3fn", "int8"), - in_place=True, - description=( - "trait-driven per-token-group quantization: bf16/fp16 input, " - "group size 16..256, fp32 or packed-UE8M0 scales in row/col-major " - "layouts, optional fused silu_and_mul and masked EP-MoE schedule" - ), - ), - description="Unified per-token-group quantization (sglang.kernels.jit).", + capabilities=CUDA_ONLY, ) ) @@ -113,53 +90,9 @@ def sgl_per_token_group_quant_8bit( ) -# Legacy aliases kept for source compatibility with existing call sites. -sgl_per_token_group_quant_fp8 = sgl_per_token_group_quant_8bit -sgl_per_token_group_quant_int8 = sgl_per_token_group_quant_8bit - - -def per_token_group_quant( - input: torch.Tensor, - output_q: Optional[torch.Tensor] = None, - output_s: Optional[torch.Tensor] = None, - group_size: int = 128, - scale_ue8m0: bool = False, - fuse_silu_and_mul: bool = False, - masked_m: Optional[torch.Tensor] = None, - expected_m: Optional[int] = None, - *, - out_dtype: Optional[torch.dtype] = None, - column_major_scales: bool = False, -): - """Unified per-token-group quantization (JIT). Returns ``(x_q, x_s)``. - - bf16/fp16 input, fp8_e4m3/int8 output, group size 16..256, fp32 or - packed-UE8M0 scales in row-/col-major layouts, optional fused - ``silu_and_mul`` and masked EP-MoE schedule (``masked_m`` + - ``expected_m`` grid hint). Pass ``output_q``/``output_s`` to quantize into - caller-owned buffers (layout inferred from their dtype/strides), or omit - both to have them allocated. - """ - return get_kernel("quantization.per_token_group_quant", KernelBackend.JIT)( - input, - output_q, - output_s, - group_size, - scale_ue8m0, - fuse_silu_and_mul, - masked_m, - expected_m, - out_dtype=out_dtype, - column_major_scales=column_major_scales, - ) - - __all__ = [ "sgl_per_token_quant_fp8", "sgl_per_token_group_quant_8bit", - "sgl_per_token_group_quant_fp8", - "sgl_per_token_group_quant_int8", - "per_token_group_quant", ] @@ -167,20 +100,15 @@ __all__ = [ # srt/layers/quantization (RFC #29630, Phase 2.5); registered for inventory. # Import them from their modules. _TRITON_KERNELS = [ - ("fp8_kernel", "per_token_group_quant_8bit"), ("fp8_kernel", "sglang_per_token_group_quant_fp8"), - ("fp8_kernel", "sglang_per_token_group_quant_8bit"), ("fp8_kernel", "sglang_per_token_quant_fp8"), ("fp8_kernel", "static_quant_fp8"), ("fp8_kernel", "dequant_group_fp8_to_bf16"), ("fp8_kernel", "per_tensor_quant_mla_fp8"), ("fp8_kernel", "per_token_group_quant_mla_deep_gemm_masked_fp8"), - ("fp8_kernel", "per_token_group_quant_fp8_hopper_moe_mn_major"), - ("fp8_kernel", "per_group_transpose"), ("int8_kernel", "per_token_quant_int8"), ("int8_kernel", "per_token_group_quant_int8"), ("awq_triton", "awq_dequantize_triton"), - ("awq_triton", "awq_gemm_triton"), ("mxfp8_amd_gfx95", "mxfp8_e4m3_quantize"), ("mxfp8_swizzled_triton", "swiglu_oai_mxfp8"), ("mxfp8_swizzled_triton", "swiglu_oai_mxfp8_deepgemm"), @@ -204,7 +132,6 @@ register_kernel( ":nvfp4_gemm_swiglu_nvfp4_quant" ), capabilities=frozenset({CapabilityRequirement.cuda(min_sm=(10, 0))}), - description="Fused NVFP4 GEMM + SwiGLU + NVFP4 quant (CuTe DSL, SM100).", ) ) @@ -215,6 +142,6 @@ register_kernel( op="quantization.cvt_fp8_e4m3", backend=KernelBackend.JIT, target="sglang.kernels.ops.quantization.fp8_cvt:cvt_fp8_e4m3", - capabilities=frozenset({CapabilityRequirement.CUDA}), + capabilities=CUDA_ONLY, ) ) diff --git a/python/sglang/kernels/ops/quantization/awq_triton.py b/python/sglang/kernels/ops/quantization/awq_triton.py index b83dd79fbc..696a932a6b 100644 --- a/python/sglang/kernels/ops/quantization/awq_triton.py +++ b/python/sglang/kernels/ops/quantization/awq_triton.py @@ -107,128 +107,6 @@ def awq_dequantize_kernel( tl.store(result_ptr + result_offsets, iweights, result_masks) -@triton.jit -def awq_gemm_kernel( - a_ptr, - b_ptr, - c_ptr, - zeros_ptr, - scales_ptr, - M, - N, - K, - group_size, - BLOCK_SIZE_M: tl.constexpr, - BLOCK_SIZE_N: tl.constexpr, - BLOCK_SIZE_K: tl.constexpr, - SPLIT_K: tl.constexpr, -): - pid = tl.program_id(axis=0) - pid_z = tl.program_id(1) - - # NOTE: This doesn't work in TRITON_INTERPRET=1 mode. Use below instead. - # num_pid_n = (N + BLOCK_SIZE_N - 1) // BLOCK_SIZE_N - num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) - - pid_m = pid // num_pid_n - pid_n = pid % num_pid_n - - accumulator_dtype = c_ptr.type.element_ty - - # NOTE: This doesn't work in TRITON_INTERPRET=1 mode. Use below instead. - # accumulator = tl.arange(0, BLOCK_SIZE_N) - # accumulator = tl.broadcast_to(accumulator[None, :], - # (BLOCK_SIZE_M, BLOCK_SIZE_N)) - # accumulator = accumulator & 0x0 - # accumulator = accumulator.to(accumulator_dtype) - accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=accumulator_dtype) - - # Create reverse AWQ order as tensor: [0, 4, 1, 5, 2, 6, 3, 7] - # that will map given indices to the correct order. - reverse_awq_order_tensor = ( - (tl.arange(0, 2) * 4)[None, :] + tl.arange(0, 4)[:, None] - ).reshape(8) - - # Create the necessary shifts to use to unpack. - shifts = reverse_awq_order_tensor * 4 - shifts = tl.broadcast_to(shifts[None, :], (BLOCK_SIZE_K * (BLOCK_SIZE_N // 8), 8)) - shifts = tl.reshape(shifts, (BLOCK_SIZE_K, BLOCK_SIZE_N)) - - # Offsets and masks. - offsets_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) - masks_am = offsets_am < M - - offsets_bn = pid_n * (BLOCK_SIZE_N // 8) + tl.arange(0, BLOCK_SIZE_N // 8) - masks_bn = offsets_bn < N // 8 - - offsets_zn = pid_n * (BLOCK_SIZE_N // 8) + tl.arange(0, BLOCK_SIZE_N // 8) - masks_zn = offsets_zn < N // 8 - - offsets_sn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) - masks_sn = offsets_sn < N - - offsets_k = pid_z * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K) - offsets_a = K * offsets_am[:, None] + offsets_k[None, :] - offsets_b = (N // 8) * offsets_k[:, None] + offsets_bn[None, :] - - a_ptrs = a_ptr + offsets_a - b_ptrs = b_ptr + offsets_b - - # NOTE: Use this in TRITON_INTERPRET=1 mode instead of tl.cdiv - # block_offset = BLOCK_SIZE_K * SPLIT_K - # for k in range(0, (K + block_offset - 1) // (block_offset)): - for k in range(0, tl.cdiv(K, BLOCK_SIZE_K * SPLIT_K)): - masks_k = offsets_k < K - masks_a = masks_am[:, None] & masks_k[None, :] - a = tl.load(a_ptrs, mask=masks_a, other=0.0) - - masks_b = masks_k[:, None] & masks_bn[None, :] - b = tl.load(b_ptrs, mask=masks_b, other=0.0) - b = tl.interleave(b, b) - b = tl.interleave(b, b) - b = tl.interleave(b, b) - - # Dequantize b. - offsets_szk = ( - BLOCK_SIZE_K * SPLIT_K * k + pid_z * BLOCK_SIZE_K - ) // group_size + tl.arange(0, 1) - offsets_z = (N // 8) * offsets_szk[:, None] + offsets_zn[None, :] - masks_zk = offsets_szk < K // group_size - masks_z = masks_zk[:, None] & masks_zn[None, :] - zeros_ptrs = zeros_ptr + offsets_z - zeros = tl.load(zeros_ptrs, mask=masks_z, other=0.0) - zeros = tl.interleave(zeros, zeros) - zeros = tl.interleave(zeros, zeros) - zeros = tl.interleave(zeros, zeros) - zeros = tl.broadcast_to(zeros, (BLOCK_SIZE_K, BLOCK_SIZE_N)) - - offsets_s = N * offsets_szk[:, None] + offsets_sn[None, :] - masks_sk = offsets_szk < K // group_size - masks_s = masks_sk[:, None] & masks_sn[None, :] - scales_ptrs = scales_ptr + offsets_s - scales = tl.load(scales_ptrs, mask=masks_s, other=0.0) - scales = tl.broadcast_to(scales, (BLOCK_SIZE_K, BLOCK_SIZE_N)) - - b = (b >> shifts) & 0xF - zeros = (zeros >> shifts) & 0xF - b = (b - zeros) * scales - b = b.to(c_ptr.type.element_ty) - - # Accumulate results. - accumulator = tl.dot(a, b, accumulator, out_dtype=accumulator_dtype) - - offsets_k += BLOCK_SIZE_K * SPLIT_K - a_ptrs += BLOCK_SIZE_K * SPLIT_K - b_ptrs += BLOCK_SIZE_K * SPLIT_K * (N // 8) - - c = accumulator.to(c_ptr.type.element_ty) - offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) - offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) - c_ptrs = c_ptr + pid_z * N * M + N * offs_cm[:, None] + offs_cn[None, :] - c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) - tl.store(c_ptrs, c, mask=c_mask) - - # qweights - [K , M // 8], int32 # scales - [K // G, M ], float16 # zeros - [K // G, M // 8], int32 @@ -281,64 +159,6 @@ def awq_dequantize_triton( return result -# input - [M, K] -# qweight - [K, N // 8] -# qzeros - [K // G, N // 8] -# scales - [K // G, N] -# split_k_iters - parallelism along K-dimension, int, power of 2. -def awq_gemm_triton( - input: torch.Tensor, - qweight: torch.Tensor, - scales: torch.Tensor, - qzeros: torch.Tensor, - split_k_iters: int, - block_size_m: int = 32, - block_size_n: int = 32, - block_size_k: int = 32, -) -> torch.Tensor: - M, K = input.shape - N = qweight.shape[1] * 8 - group_size = qweight.shape[0] // qzeros.shape[0] - - assert N > 0 and K > 0 and M > 0 - assert qweight.shape[0] == K and qweight.shape[1] == N // 8 - assert qzeros.shape[0] == K // group_size and qzeros.shape[1] == N // 8 - assert scales.shape[0] == K // group_size and scales.shape[1] == N - assert split_k_iters & (split_k_iters - 1) == 0 and split_k_iters != 0 - assert split_k_iters <= 32 - assert group_size <= K - assert group_size in AWQ_TRITON_SUPPORTED_GROUP_SIZES or group_size == K - - grid = lambda META: ( - triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), - split_k_iters, - ) - - result = torch.zeros((split_k_iters, M, N), dtype=scales.dtype, device=input.device) - - # A = input, B = qweight, C = result - # A = M x K, B = K x N, C = M x N - awq_gemm_kernel[grid]( - input, - qweight, - result, - qzeros, - scales, - M, - N, - K, - group_size, - BLOCK_SIZE_M=block_size_m, - BLOCK_SIZE_N=block_size_n, - BLOCK_SIZE_K=block_size_k, - SPLIT_K=split_k_iters, - ) - - result = result.sum(0) - - return result - - def awq_dequantize_decomposition( qweight: torch.Tensor, scales: torch.Tensor, diff --git a/python/sglang/kernels/ops/quantization/fp8_kernel.py b/python/sglang/kernels/ops/quantization/fp8_kernel.py index 4e48b97e53..a927465468 100644 --- a/python/sglang/kernels/ops/quantization/fp8_kernel.py +++ b/python/sglang/kernels/ops/quantization/fp8_kernel.py @@ -22,12 +22,10 @@ import triton.language as tl from sglang.kernels.jit.utils import is_arch_support_pdl from sglang.kernels.ops.quantization.fp8_utils import fp8_dtype_to_triton +from sglang.srt.environ import envs from sglang.srt.utils import ( ceil_align, - get_bool_env_var, - is_cpu, is_cuda, - is_gfx1250_supported, is_hip, is_musa, is_xpu, @@ -36,11 +34,9 @@ from sglang.srt.utils.patch_torch import register_fake_if_exists _is_hip = is_hip() _is_cuda = is_cuda() -_is_cpu = is_cpu() _is_musa = is_musa() -_is_gfx1250 = is_gfx1250_supported() _is_xpu = is_xpu() -_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip +_use_aiter = envs.SGLANG_USE_AITER.get() and _is_hip if _is_cuda: from sglang.kernels.ops.quantization import sgl_per_token_quant_fp8 @@ -347,112 +343,6 @@ def _per_token_group_quant_8bit_raw( return x_q, x_s -def _per_token_group_quant_8bit_fuse_silu_and_mul( - x: torch.Tensor, - group_size: int, - dst_dtype: torch.dtype, - column_major_scales: bool, - scale_tma_aligned: bool, - scale_ue8m0: bool, - masked_m: Optional[torch.Tensor], -) -> Tuple[torch.Tensor, torch.Tensor]: - # Another way to implement (can be used in e.g. comparison tests) - # from sgl_kernel import silu_and_mul - # x_after_silu_and_mul = silu_and_mul(x) - # return per_token_group_quant_fp8( - # x_after_silu_and_mul, - # group_size=group_size, - # eps=eps, - # column_major_scales=column_major_scales, - # scale_tma_aligned=scale_tma_aligned, - # scale_ue8m0=scale_ue8m0, - # ) - - from deep_gemm import transform_sf_into_required_layout - - from sglang.kernels.ops.moe.ep_moe_kernels import silu_and_mul_masked_post_quant_fwd - - assert column_major_scales - assert scale_tma_aligned - assert scale_ue8m0 - - needs_unsqueeze = x.dim() == 2 - if needs_unsqueeze: - num_tokens, _ = x.shape - x = x.unsqueeze(0) - assert masked_m is None - masked_m = torch.tensor([num_tokens], device=x.device, dtype=torch.int32) - - # Use `zeros` for easier testing - output = torch.zeros( - (*x.shape[:-1], x.shape[-1] // 2), - device=x.device, - dtype=dst_dtype, - ) - # Use `zeros` for easier testing - output_scale_for_kernel = torch.zeros( - (*x.shape[:-1], x.shape[-1] // 2 // group_size), - device=x.device, - dtype=torch.float32, - ) - silu_and_mul_masked_post_quant_fwd( - input=x, - output=output, - output_scale=output_scale_for_kernel, - quant_group_size=group_size, - masked_m=masked_m, - scale_ue8m0=scale_ue8m0, - ) - - output_scale = transform_sf_into_required_layout( - output_scale_for_kernel, - num_groups=output.shape[0], - mn=output.shape[-2], - k=output.shape[-1], - recipe=(1, group_size, group_size), - is_sfa=True, - ) - - if needs_unsqueeze: - output = output.squeeze(0) - output_scale = output_scale.squeeze(0) - - return output, output_scale - - -def per_token_group_quant_8bit( - x: torch.Tensor, - group_size: int, - dst_dtype: torch.dtype, - eps: float = 1e-10, - column_major_scales: bool = False, - scale_tma_aligned: bool = False, - scale_ue8m0: bool = False, - fuse_silu_and_mul: bool = False, - masked_m: Optional[torch.Tensor] = None, -) -> Tuple[torch.Tensor, torch.Tensor]: - if fuse_silu_and_mul: - return _per_token_group_quant_8bit_fuse_silu_and_mul( - x=x, - group_size=group_size, - dst_dtype=dst_dtype, - column_major_scales=column_major_scales, - scale_tma_aligned=scale_tma_aligned, - scale_ue8m0=scale_ue8m0, - masked_m=masked_m, - ) - else: - return _per_token_group_quant_8bit_raw( - x=x, - group_size=group_size, - eps=eps, - column_major_scales=column_major_scales, - scale_tma_aligned=scale_tma_aligned, - scale_ue8m0=scale_ue8m0, - dtype=dst_dtype, - ) - - def create_per_token_group_quant_fp8_output_scale( x_shape, device, @@ -732,86 +622,6 @@ def sglang_per_token_group_quant_fp8_row_padded( return x_q, x_s -def sglang_per_token_group_quant_fp8_ue8m0( - x: torch.Tensor, - group_size: int, - eps: float = 1e-10, -) -> Tuple[torch.Tensor, torch.Tensor]: - assert x.shape[-1] % group_size == 0, ( - f"hidden ({x.shape[-1]}) must be divisible by group_size ({group_size})" - ) - assert x.is_contiguous(), "x must be contiguous" - - *x_batch, x_q_mn, x_q_k = x.shape - x_q = torch.empty(x.shape, device=x.device, dtype=fp8_dtype) - - x_s_mn = x_q_mn - x_s_k = x_q_k // group_size - aligned_mn = ceil_align(x_s_mn, 4) - aligned_k = ceil_align(x_s_k, 4) - x_s = torch.empty( - (*x_batch, aligned_k // 4, aligned_mn), - device=x.device, - dtype=torch.int, - ).transpose(-1, -2)[..., :x_s_mn, :] - - if x.shape[0] > 0: - _run_per_token_group_quant_8bit_kernel( - x, - x_q, - x_s, - group_size, - eps, - fp8_min, - fp8_max, - scale_ue8m0=True, - fuse_silu_and_mul=False, - masked_m=None, - ) - - return x_q, x_s - - -# TODO maybe unify int8 and fp8 code later -def sglang_per_token_group_quant_8bit( - x: torch.Tensor, - group_size: int, - dst_dtype: torch.dtype, - eps: float = 1e-10, - column_major_scales: bool = False, - scale_tma_aligned: bool = False, - scale_ue8m0: bool = False, - fuse_silu_and_mul: bool = False, - masked_m: Optional[torch.Tensor] = None, -): - from sglang.kernels.ops.quantization.int8_kernel import ( - sglang_per_token_group_quant_int8, - ) - - if dst_dtype == torch.int8: - assert not column_major_scales - assert not scale_tma_aligned - assert not fuse_silu_and_mul - assert masked_m is None - return sglang_per_token_group_quant_int8( - x=x, - group_size=group_size, - eps=eps, - dtype=dst_dtype, - ) - - return sglang_per_token_group_quant_fp8( - x=x, - group_size=group_size, - eps=eps, - column_major_scales=column_major_scales, - scale_tma_aligned=scale_tma_aligned, - scale_ue8m0=scale_ue8m0, - fuse_silu_and_mul=fuse_silu_and_mul, - masked_m=masked_m, - ) - - def sglang_per_token_quant_fp8( x: torch.Tensor, dtype: torch.dtype = fp8_dtype, @@ -1297,160 +1107,6 @@ else: return output, scale -fp8_autotune = triton.autotune( - configs=[ - triton.Config({"BLOCK_M": block_m}, num_warps=num_warps) - for block_m in [16, 32, 64, 128] - for num_warps in [2, 4, 8] - ], - key=["K", "BLOCK_K", "M_ALIGNMENT"], -) - - -@triton.jit -def _per_token_group_quant_fp8_hopper_moe_mn_major( - a, # (M, K):(K, 1) - expert_offsets, # (num_experts,) - problem_sizes, # (num_experts, 3) - a_fp8, # (M, K):(K, 1) - sfa, # (M, k) - K: tl.constexpr, - BLOCK_K: tl.constexpr, - M_ALIGNMENT: tl.constexpr, - FP8_DTYPE: tl.constexpr, - BLOCK_M: tl.constexpr, # tune -): - k_offset = tl.program_id(0) - expert_id = tl.program_id(1) - - m = tl.load(problem_sizes + expert_id * 3) - current_expert_offset = tl.load(expert_offsets + expert_id).to(tl.int64) - tl.multiple_of(m, M_ALIGNMENT) - tl.multiple_of(current_expert_offset, M_ALIGNMENT) - - coord_k = k_offset * BLOCK_K + tl.arange(0, BLOCK_K) - for i in tl.range(tl.cdiv(m, BLOCK_M)): - coord_m = i * BLOCK_M + tl.arange(0, BLOCK_M) - a_ptrs = a + current_expert_offset * K + coord_m[:, None] * K + coord_k[None, :] - a_mask = (coord_m < m)[:, None] & (coord_k < K)[None, :] - - inp = tl.load(a_ptrs, mask=a_mask).to(tl.float32) # [BLOCK_M, BLOCK_K] - inp_amax = tl.max(tl.abs(inp), axis=1) # [BLOCK_M,] - inp_amax = tl.clamp(inp_amax, min=1e-4, max=float("inf")) - inp_fp8 = (inp * (448.0 / inp_amax[:, None])).to(FP8_DTYPE) - - # Store fp8 - a_fp8_ptrs = ( - a_fp8 + current_expert_offset * K + coord_m[:, None] * K + coord_k[None, :] - ) - tl.store(a_fp8_ptrs, inp_fp8.to(tl.uint8, bitcast=True), mask=a_mask) - - # Store sfa - k = tl.cdiv(K, BLOCK_K) - sfa_ptrs = ( - sfa + current_expert_offset * k + k_offset * m + coord_m - ) # MN-Major with sfa - tl.store(sfa_ptrs, inp_amax / 448.0, mask=coord_m < m) - - -if not _is_cpu: - _per_token_group_quant_fp8_hopper_moe_mn_major = fp8_autotune( - _per_token_group_quant_fp8_hopper_moe_mn_major - ) - - -def per_token_group_quant_fp8_hopper_moe_mn_major( - A: torch.Tensor, - expert_offsets: torch.Tensor, - problem_sizes: torch.Tensor, - group_size: int, - expert_tokens_alignment: int = 1, -) -> Tuple[torch.Tensor, torch.Tensor]: - assert A.dim() == 2 - assert A.is_contiguous(), "`A` is not contiguous" - assert A.shape[-1] % group_size == 0, ( - "the last dimension of `A` cannot be divisible by `group_size`" - ) - - a_q = torch.empty_like(A, device=A.device, dtype=fp8_dtype) - M, K = A.shape[0], A.shape[1] - k = K // group_size - sfa = torch.empty((M, k), device=A.device, dtype=torch.float32) - num_experts = problem_sizes.shape[0] - grid = (k, num_experts) - _per_token_group_quant_fp8_hopper_moe_mn_major[grid]( - A, - expert_offsets, - problem_sizes, - a_q.view(torch.uint8), - sfa, - K, - group_size, - expert_tokens_alignment, - fp8_dtype_to_triton(fp8_dtype), - ) - return a_q, sfa - - -@triton.jit -def _per_group_transpose( - data_ptr: torch.Tensor, - trans_data_ptr: torch.Tensor, - expert_offsets: torch.Tensor, - k: int, - M_ALIGNMENT: tl.constexpr, - BLOCK_SIZE_M: tl.constexpr, - BLOCK_SIZE_K: tl.constexpr, -): - expert_id = tl.program_id(0) - m_id = tl.program_id(1) - k_id = tl.program_id(2) - - curr_expert_offset = tl.load(expert_offsets + expert_id) - next_expert_offset = tl.load(expert_offsets + expert_id + 1) - num_tokens_of_expert = next_expert_offset - curr_expert_offset - tl.multiple_of(curr_expert_offset, M_ALIGNMENT) - tl.multiple_of(next_expert_offset, M_ALIGNMENT) - - data_start_ptr = data_ptr + curr_expert_offset * k - trans_data_start_ptr = trans_data_ptr + curr_expert_offset * k - - k_coord = k_id * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K) - k_mask = k_coord < k - for start_m in tl.range(0, num_tokens_of_expert, BLOCK_SIZE_M * tl.num_programs(1)): - m_coord = start_m + m_id * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) - m_mask = m_coord < num_tokens_of_expert - off = m_coord[:, None] * k + k_coord[None, :] - trans_off = m_coord[:, None] + k_coord[None, :] * num_tokens_of_expert - mask = m_mask[:, None] & k_mask[None, :] - - data = tl.load(data_start_ptr + off, mask=mask) - tl.store(trans_data_start_ptr + trans_off, data, mask=mask) - - -def per_group_transpose( - a: torch.Tensor, - expert_offsets: torch.Tensor, - M_ALIGNMENT: int = 1, -) -> torch.Tensor: - assert a.dim() == 2 - assert a.is_contiguous(), "`a` is not contiguous" - - m, k = a.size() - trans_a = torch.empty_like(a) - num_experts = expert_offsets.size(0) - 1 - - grid = lambda META: ( - num_experts, - triton.cdiv((m + num_experts - 1) // num_experts, META["BLOCK_SIZE_M"]), - triton.cdiv(k, META["BLOCK_SIZE_K"]), - ) - _per_group_transpose[grid]( - a, trans_a, expert_offsets, k, M_ALIGNMENT, BLOCK_SIZE_M=16, BLOCK_SIZE_K=8 - ) - return trans_a - - @triton.jit def _dequant_group_fp8_kernel( X, S, Y, M, K: tl.constexpr, SS: tl.constexpr, BLOCK: tl.constexpr diff --git a/python/sglang/kernels/ops/quantization/int8_kernel.py b/python/sglang/kernels/ops/quantization/int8_kernel.py index c1daae04a4..c583734a91 100644 --- a/python/sglang/kernels/ops/quantization/int8_kernel.py +++ b/python/sglang/kernels/ops/quantization/int8_kernel.py @@ -89,6 +89,8 @@ def per_token_quant_int8(x, scale_dtype=torch.float32, cal_sum=False): return x_q, scales +# Not fp8_kernel._per_token_group_quant_8bit: that kernel multiplies by 1 / scale and +# clamps int8 to -127 on ROCm, the non-CUDA platforms this path serves. @triton.jit def _per_token_group_quant_int8( # Pointers to inputs and output diff --git a/python/sglang/kernels/ops/quantization/minimax_quant_ue8m0.py b/python/sglang/kernels/ops/quantization/minimax_quant_ue8m0.py index 75d2966e99..88a6d35fca 100644 --- a/python/sglang/kernels/ops/quantization/minimax_quant_ue8m0.py +++ b/python/sglang/kernels/ops/quantization/minimax_quant_ue8m0.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Tuple +from typing import TYPE_CHECKING import torch @@ -15,19 +15,6 @@ if TYPE_CHECKING: from tvm_ffi.module import Module -@cache_once -def _jit_module(group_size: int) -> Module: - args = make_cpp_args(group_size, is_arch_support_pdl()) - return load_jit( - "minimax_per_token_quant_ue8m0", - *args, - cuda_files=["minimax/per_token_quant_ue8m0.cuh"], - cuda_wrappers=[ - ("per_token_quant_ue8m0", f"per_token_quant_ue8m0<{args}>"), - ], - ) - - @cache_once def _jit_scatter_module(group_size: int, topk: int) -> Module: # topk is a template arg so the dst-row load/store loops fully unroll. @@ -45,32 +32,6 @@ def _jit_scatter_module(group_size: int, topk: int) -> Module: ) -def per_token_quant_fp8_ue8m0( - x: torch.Tensor, group_size: int = 128 -) -> Tuple[torch.Tensor, torch.Tensor]: - """Per-token group quant to FP8-e4m3 with a fused UE8M0 (int32-packed) scale. - - Returns ``(x_q, x_sf)`` where ``x_q`` is fp8_e4m3 ``[num_tokens, hidden]`` and - ``x_sf`` is the int32-packed UE8M0 scale ``[num_tokens, hidden//group_size//4]`` - (row-major). Byte-identical to ``per_token_group_quant_fp8(scale_ue8m0=True)`` - followed by ``transform_sf_into_required_layout`` (both ceil-round the scale), - but does it in a single kernel -- no separate transpose/pack launch. - """ - assert x.is_cuda and x.dtype == torch.bfloat16 and x.dim() == 2 - assert x.is_contiguous() - num_tokens, hidden = x.shape - assert hidden % group_size == 0 - num_groups = hidden // group_size - assert num_groups % 4 == 0, "num_groups must be a multiple of 4 for int32 packing" - - x_q = torch.empty_like(x, dtype=torch.float8_e4m3fn) - x_sf = torch.empty( - (num_tokens, num_groups // 4), dtype=torch.int32, device=x.device - ) - _jit_module(group_size).per_token_quant_ue8m0(x, x_q, x_sf) - return x_q, x_sf - - def per_token_quant_fp8_ue8m0_scatter( x: torch.Tensor, gateup_input: torch.Tensor, @@ -82,7 +43,7 @@ def per_token_quant_fp8_ue8m0_scatter( group_size: int = 128, ) -> None: """Fused per-token FP8/UE8M0 quant **and** scatter into the permuted grouped-GEMM - input -- a single kernel replacing ``per_token_quant_fp8_ue8m0`` + + input -- a single kernel replacing a UE8M0 per-token-group quant + ``fill_gateup_input_triton_kernel``. For each source token it computes the fp8 row + int32-packed UE8M0 scale once, diff --git a/python/sglang/kernels/ops/quantization/mxfp8_amd_gfx95.py b/python/sglang/kernels/ops/quantization/mxfp8_amd_gfx95.py index 7c727c6c8d..6bbb5267da 100644 --- a/python/sglang/kernels/ops/quantization/mxfp8_amd_gfx95.py +++ b/python/sglang/kernels/ops/quantization/mxfp8_amd_gfx95.py @@ -25,6 +25,7 @@ import triton import triton.language as tl from sglang.kernels.jit.utils import cache_once +from sglang.kernels.ops.quantization.mxfp8_quant import _mxfp8_quant_kernel # MXFP8 constants (OCP microscaling: 1x32 block, E8M0 shared scale). MXFP8_VALUE_DTYPE = torch.float8_e4m3fn @@ -69,47 +70,6 @@ def _mxfp8_e4m3_quantize_torch(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Ten return x_fp8, scales_uint8 -@triton.jit -def _mxfp8_quant_kernel( - x_ptr, - xq_ptr, - s_ptr, - M, - K, - sxm, - sxk, - sqm, - sqk, - ssm, - ssk, - BLOCK_M: tl.constexpr, -): - """Per-32-block E8M0 scale + FP8-E4M3 quant, one program per ``[BLOCK_M, 32]``.""" - pid_m = tl.program_id(0) - pid_b = tl.program_id(1) # which 32-element block along K - offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) - offs_k = pid_b * 32 + tl.arange(0, 32) - m_mask = offs_m < M - x = tl.load( - x_ptr + offs_m[:, None] * sxm + offs_k[None, :] * sxk, - mask=m_mask[:, None], - other=0.0, - ).to(tl.float32) - amax = tl.maximum(tl.max(tl.abs(x), axis=1), 1e-30) # [BLOCK_M] - # Round the E8M0 exponent up (ceil(log2(amax / e4m3_max))) so the block amax - # stays inside the e4m3 range and the full dynamic range is used. - sb = tl.ceil(tl.log2(amax / 448.0)) + 127.0 - sb = tl.minimum(tl.maximum(sb, 0.0), 254.0) - descale = tl.exp2(sb - 127.0) - xq = tl.clamp(x / descale[:, None], -448.0, 448.0).to(xq_ptr.dtype.element_ty) - tl.store( - xq_ptr + offs_m[:, None] * sqm + offs_k[None, :] * sqk, - xq, - mask=m_mask[:, None], - ) - tl.store(s_ptr + offs_m * ssm + pid_b * ssk, sb.to(tl.uint8), mask=m_mask) - - def _mxfp8_e4m3_quantize_triton( x: torch.Tensor, ) -> Tuple[torch.Tensor, torch.Tensor]: diff --git a/python/sglang/kernels/ops/quantization/mxfp8_quant.py b/python/sglang/kernels/ops/quantization/mxfp8_quant.py index a9a59b5459..a21916ce37 100644 --- a/python/sglang/kernels/ops/quantization/mxfp8_quant.py +++ b/python/sglang/kernels/ops/quantization/mxfp8_quant.py @@ -206,167 +206,3 @@ def quant_store_kv_mxfp8( D=D, PAGE=page_size, ) - - -@triton.jit -def _mxfp8_v_cache_update_kernel( - v_ptr, - loc_ptr, - cache_ptr, - sf_ptr, - N, - svn, - svh, - svd, - scp, - scs, - sch, - scd, - sfp, - sfh, - sfd, - sfb, - PAGE_SIZE: tl.constexpr, - HEAD_DIM: tl.constexpr, - SCAN_WINDOW: tl.constexpr, -): - n = tl.program_id(0) - h = tl.program_id(1) - myloc = tl.load(loc_ptr + n).to(tl.int64) - blk = myloc // 32 - - # Leader election: same-block tokens are <= SCAN_WINDOW consecutive loc - # entries (blocks never span pages; per-sequence slots are contiguous), - # so the program with the lowest index in that window owns the block. - is_leader = myloc >= 0 - j_lo = tl.maximum(n - SCAN_WINDOW, 0) - for j in range(j_lo, n): - lj = tl.load(loc_ptr + j).to(tl.int64) - is_leader = is_leader & ~((lj >= 0) & (lj // 32 == blk)) - - if is_leader: - page = blk * 32 // PAGE_SIZE - blk_in_page = (blk * 32) % PAGE_SIZE // 32 - d = tl.arange(0, HEAD_DIM) - sf_addr = sf_ptr + page * sfp + h * sfh + d * sfd + blk_in_page * sfb - e_old = tl.load(sf_addr).to(tl.int32) - - # Pass 1 over this call's tokens for the block: per-d exponent of new - # data, and whether the block's first slot is among them (fresh block - # -> ignore the stored exponent: kills the stale ratchet from page - # reuse and any garbage bytes). - e_new = tl.zeros((HEAD_DIM,), dtype=tl.int32) - has_start = False - j_hi = tl.minimum(n + SCAN_WINDOW, N) - for j in range(n, j_hi): - lj = tl.load(loc_ptr + j).to(tl.int64) - hit = (lj >= 0) & (lj // 32 == blk) - v = tl.load(v_ptr + j * svn + h * svh + d * svd, mask=hit, other=0.0).to( - tl.float32 - ) - amax = tl.maximum(tl.abs(v), 1e-30) - e_tok = (tl.ceil(tl.log2(amax / 448.0)) + 127.0).to(tl.int32) - e_tok = tl.minimum(tl.maximum(e_tok, 0), 254) - e_new = tl.maximum(e_new, tl.where(hit, e_tok, 0)) - has_start = has_start | (hit & (lj % 32 == 0)) - e_old = tl.where(has_start, 0, e_old) - e_blk = tl.maximum(e_old, e_new) - - # Rescale the existing payload where the exponent grew: an exact - # power-of-two shift (matches offline quantization up to subnormal - # double-rounding). Overwritten slots get fresh data below anyway. - s = tl.arange(0, 32) - cache_addr = ( - cache_ptr - + page * scp - + (blk_in_page * 32 + s)[:, None] * scs - + h * sch - + d[None, :] * scd - ) - old = tl.load(cache_addr).to(tl.float32) - old = old * tl.exp2((e_old - e_blk).to(tl.float32))[None, :] - tl.store(cache_addr, old.to(cache_ptr.dtype.element_ty)) - - # Pass 2: quantize and write the new tokens at the settled exponent. - descale = tl.exp2((e_blk - 127).to(tl.float32)) - for j in range(n, j_hi): - lj = tl.load(loc_ptr + j).to(tl.int64) - hit = (lj >= 0) & (lj // 32 == blk) - v = tl.load(v_ptr + j * svn + h * svh + d * svd, mask=hit, other=0.0).to( - tl.float32 - ) - q = tl.clamp(v / descale, -448.0, 448.0) - slot_in_page = lj % PAGE_SIZE - tl.store( - cache_ptr + page * scp + slot_in_page * scs + h * sch + d * scd, - q.to(cache_ptr.dtype.element_ty), - mask=hit, - ) - - tl.store(sf_addr, e_blk.to(tl.uint8)) - - -def update_mxfp8_v_cache_seqblocked( - v: torch.Tensor, - loc: torch.Tensor, - v_cache: torch.Tensor, - sfv: torch.Tensor, -) -> None: - """Append V rows to a legacy seq-blocked fp8 cache quantized per-32-token. - - Current paged FA4 v_dequant uses BlockScaledBasicChunk SFV laid out like - SFK: per token, per head-dim block. This helper keeps the older V-scale - layout where each head dim owns page_size / 32 token-block scales. - Incremental writes reproduce offline whole-block quantization -- scale - bytes exactly, payload up to subnormal double-rounding. - - v: (N, h_kv, head_dim) bf16/fp16 new rows - loc: (N,) int32/int64 destination slot ids; negative = skip (padding) - v_cache: (num_pages, page_size, h_kv, head_dim) float8_e4m3fn - sfv: (num_pages, h_kv, head_dim, page_size // 32) uint8 UE8M0 - - Requirements (asserted where cheap): page_size % 32 == 0; same-block - tokens occupy consecutive `loc` entries (true for sglang's per-sequence - contiguous allocation -- blocks never span pages). The sfv buffer must be - zero-initialized at pool allocation so never-written blocks can't hold - e8m0 NaN (0xFF). Fixed grid and no host syncs: CUDA-graph capturable. - """ - N, h_kv, head_dim = v.shape - num_pages, page_size = v_cache.shape[0], v_cache.shape[1] - assert page_size % MXFP8_BLOCK_SIZE == 0 - assert v_cache.shape[2:] == (h_kv, head_dim) - assert sfv.shape == (num_pages, h_kv, head_dim, page_size // MXFP8_BLOCK_SIZE) - assert sfv.dtype == torch.uint8 and v_cache.dtype == torch.float8_e4m3fn - if N == 0: - return - _mxfp8_v_cache_update_kernel[(N, h_kv)]( - v, - loc, - v_cache, - sfv, - N, - v.stride(0), - v.stride(1), - v.stride(2), - v_cache.stride(0), - v_cache.stride(1), - v_cache.stride(2), - v_cache.stride(3), - sfv.stride(0), - sfv.stride(1), - sfv.stride(2), - sfv.stride(3), - PAGE_SIZE=page_size, - HEAD_DIM=head_dim, - SCAN_WINDOW=MXFP8_BLOCK_SIZE, - ) - - -def from_mxfp8(x: MXFP8Tensor, out_dtype: torch.dtype = torch.bfloat16) -> torch.Tensor: - num_blocks = x.data.shape[-1] // MXFP8_BLOCK_SIZE - data = x.data.to(torch.float32).view( - *x.data.shape[:-1], num_blocks, MXFP8_BLOCK_SIZE - ) - scale_biased = x.scale.view(torch.uint8).to(torch.float32) - descale = torch.exp2(scale_biased - 127.0) - return (data * descale.unsqueeze(-1)).view_as(x.data).to(out_dtype) diff --git a/python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py b/python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py index f497baefb5..c93b9b987d 100644 --- a/python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py +++ b/python/sglang/kernels/ops/quantization/nvfp4_gemm_swiglu_nvfp4_quant.py @@ -2647,36 +2647,6 @@ class Sm100BlockScaledPersistentDenseGemmKernel: ) -@cute.jit -def cvt_sf_MKL_to_M32x4xrm_K4xrk_L( - sf_ref_tensor: cute.Tensor, - sf_mma_tensor: cute.Tensor, -): - """Convert scale factor tensor from MKL layout to mma specification M(32x4xrest_m)xK(4xrest_k)xL layout""" - # sf_mma_tensor has flatten shape (32, 4, rest_m, 4, rest_k, l) - # group to ((32, 4, rest_m), (4, rest_k), l) - sf_mma_tensor = cute.group_modes(sf_mma_tensor, 0, 3) - sf_mma_tensor = cute.group_modes(sf_mma_tensor, 1, 3) - for i in cutlass.range(cute.size(sf_ref_tensor)): - mkl_coord = sf_ref_tensor.layout.get_hier_coord(i) - sf_mma_tensor[mkl_coord] = sf_ref_tensor[mkl_coord] - - -@cute.jit -def cvt_sf_M32x4xrm_K4xrk_L_to_MKL( - sf_mma_tensor: cute.Tensor, - sf_ref_tensor: cute.Tensor, -): - """Convert scale factor tensor from mma specification M(32x4xrest_m)xK(4xrest_k)xL layout to MKL layout""" - # sf_mma_tensor has flatten shape (32, 4, rest_m, 4, rest_k, l) - # group to ((32, 4, rest_m), (4, rest_k), l) - sf_mma_tensor = cute.group_modes(sf_mma_tensor, 0, 3) - sf_mma_tensor = cute.group_modes(sf_mma_tensor, 1, 3) - for i in cutlass.range(cute.size(sf_ref_tensor)): - mkl_coord = sf_ref_tensor.layout.get_hier_coord(i) - sf_ref_tensor[mkl_coord] = sf_mma_tensor[mkl_coord] - - # --------------------------------------------------------------------------- # SGLang-side wrapper, helpers, and per-shape compile cache. # --------------------------------------------------------------------------- diff --git a/python/sglang/kernels/ops/quantization/per_token_group_quant_8bit_v2.py b/python/sglang/kernels/ops/quantization/per_token_group_quant_8bit_v2.py index 7ad66bfa1f..7a7d9bfcc5 100644 --- a/python/sglang/kernels/ops/quantization/per_token_group_quant_8bit_v2.py +++ b/python/sglang/kernels/ops/quantization/per_token_group_quant_8bit_v2.py @@ -1,7 +1,10 @@ -"""DEPRECATED: superseded by ``sglang.kernels.ops.quantization.per_token_group_quant`` (the -default CUDA path). No sglang runtime code may call this kernel; it is kept -only as the perf baseline for the per_token_group_quant benchmarks and its own -bit-parity tests, and will be deleted once those move to torch references. +"""JIT port of ``sgl_per_token_group_quant_8bit_v2``. + +``sglang.kernels.ops.quantization.per_token_group_quant`` is the default CUDA +path. This kernel still serves the row-major UE8M0 layout stored as fp32 +power-of-two scales, which per_token_group_quant does not produce (see +``fp8_kernel._run_per_token_group_quant_8bit_kernel``), and is the perf baseline +for the per_token_group_quant benchmarks. """ from __future__ import annotations diff --git a/python/sglang/kernels/ops/sampling/__init__.py b/python/sglang/kernels/ops/sampling/__init__.py index f8ab5828a7..f94ffc9880 100644 --- a/python/sglang/kernels/ops/sampling/__init__.py +++ b/python/sglang/kernels/ops/sampling/__init__.py @@ -1,14 +1,14 @@ -"""Sampling kernels (top-k / top-p probability renormalization).""" +"""Sampling kernels.""" from __future__ import annotations -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING from sglang.kernels.registry import register_kernel from sglang.kernels.selector import get_kernel from sglang.kernels.spec import ( + CUDA_ONLY, CapabilityRequirement, - FormatSignature, KernelBackend, KernelSpec, ) @@ -26,11 +26,6 @@ register_kernel( CapabilityRequirement.cuda(min_sm=(10, 3), max_sm=(10, 3)), } ), - format_signature=FormatSignature( - supported_dtypes=("float32",), - description="FP32 [batch, vocab] softmax probabilities", - ), - description="Cake Blackwell softmax distributed by FlashInfer.", ) ) @@ -62,45 +57,7 @@ def softmax( return torch.softmax(logits, dim=-1) -register_kernel( - KernelSpec( - op="sampling.top_k_renorm_probs", - backend=KernelBackend.AOT, - target="sgl_kernel.sampling:top_k_renorm_probs", - format_signature=FormatSignature( - description="renormalize probs by top-k thresholding; returns tensor" - ), - description="Top-k probability renormalization (sgl_kernel wheel).", - ) -) -register_kernel( - KernelSpec( - op="sampling.top_p_renorm_probs", - backend=KernelBackend.AOT, - target="sgl_kernel.sampling:top_p_renorm_probs", - format_signature=FormatSignature( - description="renormalize probs by top-p thresholding; returns tensor" - ), - description="Top-p probability renormalization (sgl_kernel wheel).", - ) -) - - -def top_k_renorm_probs( - probs: torch.Tensor, top_k: Union[torch.Tensor, int] -) -> torch.Tensor: - """Renormalize ``probs`` by top-k thresholding.""" - return get_kernel("sampling.top_k_renorm_probs", KernelBackend.AOT)(probs, top_k) - - -def top_p_renorm_probs( - probs: torch.Tensor, top_p: Union[torch.Tensor, float] -) -> torch.Tensor: - """Renormalize ``probs`` by top-p thresholding.""" - return get_kernel("sampling.top_p_renorm_probs", KernelBackend.AOT)(probs, top_p) - - -__all__ = ["cake_softmax", "softmax", "top_k_renorm_probs", "top_p_renorm_probs"] +__all__ = ["cake_softmax", "softmax"] # Migrated from srt/layers/utils/hash.py (RFC #29630, Phase 2.5). @@ -118,10 +75,6 @@ register_kernel( op="sampling.denoiser_statistics", backend=KernelBackend.TRITON, target="sglang.kernels.ops.sampling.denoiser_statistics:denoiser_statistics", - capabilities=frozenset({CapabilityRequirement.CUDA}), - format_signature=FormatSignature( - supported_dtypes=("float32",), - description="FP32 probabilities, entropy and raw-logit argmax for contiguous [B,M,V] logits; optional BF16 probabilities.", - ), + capabilities=CUDA_ONLY, ) ) diff --git a/python/sglang/kernels/ops/sampling/top_p_renorm_triton.py b/python/sglang/kernels/ops/sampling/top_p_renorm_triton.py deleted file mode 100644 index 3730e0b5d6..0000000000 --- a/python/sglang/kernels/ops/sampling/top_p_renorm_triton.py +++ /dev/null @@ -1,127 +0,0 @@ -"""ROCm-compatible top-p probability renormalization fallback.""" - -from __future__ import annotations - -from typing import Union - -import torch -import triton -import triton.language as tl - -_BLOCK_SIZE = 1024 - - -@triton.jit -def _mask_and_partial_sum_kernel( - probs_ptr, - pivots_ptr, - out_ptr, - partial_sums_ptr, - vocab_size: tl.constexpr, - num_chunks: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - row = tl.program_id(0) - chunk = tl.program_id(1) - offsets = chunk * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - mask = offsets < vocab_size - row_offsets = row * vocab_size + offsets - - probs = tl.load(probs_ptr + row_offsets, mask=mask, other=0.0).to(tl.float32) - pivot = tl.load(pivots_ptr + row) - kept = tl.where(mask & (probs >= pivot), probs, 0.0) - - tl.store(out_ptr + row_offsets, kept, mask=mask) - tl.store(partial_sums_ptr + row * num_chunks + chunk, tl.sum(kept, axis=0)) - - -@triton.jit -def _normalize_kernel( - out_ptr, - row_sums_ptr, - numel, - vocab_size: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - offsets = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - mask = offsets < numel - row = offsets // vocab_size - values = tl.load(out_ptr + offsets, mask=mask, other=0.0).to(tl.float32) - denominator = tl.load(row_sums_ptr + row, mask=mask, other=1.0) - tl.store(out_ptr + offsets, values / denominator, mask=mask) - - -def top_p_renorm_probs_triton( - probs: torch.Tensor, top_p: Union[torch.Tensor, float] -) -> torch.Tensor: - """Apply exact top-p thresholding and renormalize each probability row. - - Sorting and prefix sums use PyTorch's device kernels because a vocabulary-sized - in-register Triton sort does not scale to 100K+ vocabularies. Triton performs - the bandwidth-heavy masking, partial reduction, and normalization. - """ - if probs.ndim != 2: - raise ValueError(f"probs must be 2D, got shape={tuple(probs.shape)}") - if not probs.is_cuda: - raise ValueError("top_p_renorm_probs_triton requires a CUDA/HIP tensor") - - probs_fp32 = probs.float().contiguous() - batch_size, vocab_size = probs_fp32.shape - if batch_size == 0 or vocab_size == 0: - return probs_fp32 - - if isinstance(top_p, torch.Tensor): - top_ps = top_p.to(device=probs.device, dtype=torch.float32).reshape(-1) - if top_ps.numel() == 1: - top_ps = top_ps.expand(batch_size) - elif top_ps.numel() != batch_size: - raise ValueError( - f"top_p must be scalar or have one value per row, got " - f"{top_ps.numel()} values for {batch_size} rows" - ) - else: - if not 0.0 < float(top_p) <= 1.0: - raise ValueError("top_p values must be in (0, 1]") - top_ps = torch.full( - (batch_size,), float(top_p), device=probs.device, dtype=torch.float32 - ) - - # Match FlashInfer's threshold semantics: sort ascending, discard the prefix - # whose cumulative mass is below 1 - p, and retain all ties at the pivot. - sorted_probs = torch.sort(probs_fp32, dim=-1).values - cdf = torch.cumsum(sorted_probs, dim=-1) - cutoff = torch.searchsorted(cdf, (1.0 - top_ps).unsqueeze(1), right=False).squeeze( - 1 - ) - cutoff.clamp_(max=vocab_size - 1) - pivots = sorted_probs.gather(1, cutoff.unsqueeze(1)).squeeze(1).contiguous() - - num_chunks = triton.cdiv(vocab_size, _BLOCK_SIZE) - out = torch.empty_like(probs_fp32) - partial_sums = torch.empty( - (batch_size, num_chunks), device=probs.device, dtype=torch.float32 - ) - _mask_and_partial_sum_kernel[(batch_size, num_chunks)]( - probs_fp32, - pivots, - out, - partial_sums, - vocab_size=vocab_size, - num_chunks=num_chunks, - BLOCK_SIZE=_BLOCK_SIZE, - num_warps=8, - ) - - row_sums = partial_sums.sum(dim=1) - _normalize_kernel[(triton.cdiv(out.numel(), _BLOCK_SIZE),)]( - out, - row_sums, - out.numel(), - vocab_size=vocab_size, - BLOCK_SIZE=_BLOCK_SIZE, - num_warps=8, - ) - return out - - -__all__ = ["top_p_renorm_probs_triton"] diff --git a/python/sglang/kernels/ops/speculative/__init__.py b/python/sglang/kernels/ops/speculative/__init__.py index 9446e035aa..c662f80b20 100644 --- a/python/sglang/kernels/ops/speculative/__init__.py +++ b/python/sglang/kernels/ops/speculative/__init__.py @@ -6,12 +6,7 @@ The Triton kernels migrated here live in this package """ from sglang.kernels.registry import register_kernel -from sglang.kernels.spec import ( - CapabilityRequirement, - FormatSignature, - KernelBackend, - KernelSpec, -) +from sglang.kernels.spec import CUDA_ONLY, KernelBackend, KernelSpec # (module, public_fn) migrated from speculative/triton_ops. _TRITON_KERNELS = [ @@ -49,11 +44,7 @@ register_kernel( op="speculative.div_argmax", backend=KernelBackend.TRITON, target="sglang.kernels.ops.speculative.row_argmax:div_argmax", - capabilities=frozenset({CapabilityRequirement.CUDA}), - format_signature=FormatSignature( - supported_dtypes=("float32",), - description="Row-wise division and argmax on matching 2D tensors with unit column strides.", - ), + capabilities=CUDA_ONLY, ) ) diff --git a/python/sglang/kernels/ops/speculative/dspark/dspark_verify_window.py b/python/sglang/kernels/ops/speculative/dspark/dspark_verify_window.py index 88f091a554..93cbc131f0 100644 --- a/python/sglang/kernels/ops/speculative/dspark/dspark_verify_window.py +++ b/python/sglang/kernels/ops/speculative/dspark/dspark_verify_window.py @@ -236,84 +236,6 @@ def build_ragged_verify_window_triton( _SEARCH_NBITS = 11 -class CompactRowIndex: - @classmethod - def execute( - cls, *args, **kwargs - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - if inputs_on_cuda(*args, **kwargs): - return cls.triton(*args, **kwargs) - return cls.torch(*args, **kwargs) - - @classmethod - def torch( - cls, - *, - verify_lens: torch.Tensor, - padded_total: int, - device, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - return compact_row_index( - verify_lens=verify_lens, - padded_total=padded_total, - device=device, - ) - - @classmethod - def triton( - cls, - *, - verify_lens: torch.Tensor, - padded_total: int, - device, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - return compact_row_index_triton( - verify_lens=verify_lens, - padded_total=padded_total, - device=device, - ) - - -class CompactVerifyIds: - @classmethod - def execute(cls, *args, **kwargs) -> torch.Tensor: - if inputs_on_cuda(*args, **kwargs): - return cls.triton(*args, **kwargs) - return cls.torch(*args, **kwargs) - - @classmethod - def torch( - cls, - *, - draft_block_ids: torch.Tensor, - draft_tokens: torch.Tensor, - layout: RaggedVerifyLayout, - device: str, - ) -> torch.Tensor: - return compact_verify_ids( - draft_block_ids=draft_block_ids, - draft_tokens=draft_tokens, - layout=layout, - device=device, - ) - - @classmethod - def triton( - cls, - *, - draft_block_ids: torch.Tensor, - draft_tokens: torch.Tensor, - layout: RaggedVerifyLayout, - device: str, - ) -> torch.Tensor: - return compact_verify_ids_triton( - draft_block_ids=draft_block_ids, - draft_tokens=draft_tokens, - layout=layout, - device=device, - ) - - def compact_verify_ids( *, draft_block_ids: torch.Tensor, diff --git a/python/sglang/kernels/selector.py b/python/sglang/kernels/selector.py index 8f649fc6a7..3b01aa3503 100644 --- a/python/sglang/kernels/selector.py +++ b/python/sglang/kernels/selector.py @@ -96,8 +96,3 @@ def get_kernel(op: str, backend: Optional[KernelBackend] = None) -> Callable: call resolves and imports the backend; later calls hit the cache. """ return _resolve(op, backend) - - -def clear_cache() -> None: - """Drop the resolved-callable cache (used by tests).""" - _resolve.cache_clear() diff --git a/python/sglang/kernels/spec.py b/python/sglang/kernels/spec.py index b5054af676..18e0912d7e 100644 --- a/python/sglang/kernels/spec.py +++ b/python/sglang/kernels/spec.py @@ -85,21 +85,14 @@ class PlatformInfo(msgspec.Struct, frozen=True): def is_cuda(self) -> bool: return self.device_type == "cuda" - @property - def is_hip(self) -> bool: - return self.device_type == "hip" - @classmethod def detect(cls) -> PlatformInfo: """Build a :class:`PlatformInfo` from the current process. - Never raises: if ``torch`` is missing or no accelerator is visible the - default CPU platform is returned. + Never raises: if no accelerator is visible the default CPU platform is + returned. """ - try: - import torch - except Exception: - return cls() + import torch try: if torch.version.hip is not None and torch.cuda.is_available(): @@ -172,6 +165,8 @@ CapabilityRequirement.CUDA = CapabilityRequirement(device=DeviceType.CUDA) CapabilityRequirement.HIP = CapabilityRequirement(device=DeviceType.HIP) CapabilityRequirement.NPU = CapabilityRequirement(device=DeviceType.NPU) +CUDA_ONLY: FrozenSet[CapabilityRequirement] = frozenset({CapabilityRequirement.CUDA}) + def capabilities_satisfied( capabilities: Union[ @@ -191,18 +186,6 @@ def capabilities_satisfied( return (not capabilities) or any(c.is_satisfied_by(platform) for c in capabilities) -class FormatSignature(msgspec.Struct, frozen=True): - """A light description of a kernel's data contract. - - This is deliberately loose in the first version — enough to document intent - and support future inventory tooling, not a strict schema. - """ - - supported_dtypes: Tuple[str, ...] = () - in_place: bool = False - description: str = "" - - class KernelSpec(msgspec.Struct, frozen=True): """A single callable kernel implementation and its metadata. @@ -210,39 +193,25 @@ class KernelSpec(msgspec.Struct, frozen=True): ---------- op: Fully-qualified operator id, ``"."`` (e.g. - ``"layernorm.rmsnorm"``). This is the public lookup key. + ``"activation.silu_and_mul"``). This is the public lookup key. backend: Which :class:`KernelBackend` (provenance) provides this implementation. target: Import path of the callable in ``"module:attr"`` form, resolved lazily - by :meth:`load` (e.g. ``"sgl_kernel:rmsnorm"``). ``attr`` may be a + by :meth:`load` (e.g. ``"sgl_kernel:silu_and_mul"``). ``attr`` may be a dotted path into a module-level object, e.g. - ``"sglang.kernels.ops.layernorm:_RMSNORM.forward_aot"`` for a bound + ``"sglang.kernels.ops.activation:_SILU_AND_MUL.forward_aot"`` for a bound :class:`~sglang.kernels.fused_op.BaseFusedOp` backend method. capabilities: Set of :class:`CapabilityRequirement` (OR semantics; empty = runs on any device) used by the selector to skip backends unusable on the detected platform. - format_signature: - Optional data-contract description for inventory/documentation. - description: - Human-readable one-liner. """ op: str backend: KernelBackend target: str capabilities: FrozenSet[CapabilityRequirement] = frozenset() - format_signature: FormatSignature = msgspec.field(default_factory=FormatSignature) - description: str = "" - - @property - def group(self) -> str: - return self.op.split(".", 1)[0] - - @property - def name(self) -> str: - return self.op.split(".", 1)[1] if "." in self.op else self.op def is_available(self, platform: PlatformInfo) -> bool: """Whether this backend can run on ``platform`` (metadata-only check).""" diff --git a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md index e695950c08..5b7f51b127 100644 --- a/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md +++ b/python/sglang/multimodal_gen/.agents/skills/sglang-diffusion-benchmark-profile/existing-fast-paths.md @@ -403,7 +403,7 @@ framework-specific optimization workflow. - `can_use_fused_inplace_qknorm(head_dim, dtype)` returns true. - Supported head dims: `64, 128, 256, 512, 1024`. - Behavior: Fused path operates on `q` and `k` in place after reshaping to `[B, -1, head_dim]`. If preconditions fail, fall back to per-tensor RMSNorm. -- Validation: `test/registered/kernels/ops/layernorm/test_qknorm.py` and `test/registered/kernels/ops/layernorm/test_qknorm_across_heads.py`. +- Validation: `test/registered/kernels/ops/layernorm/test_qknorm.py`. **QK Norm + RoPE Optimization** @@ -520,7 +520,7 @@ framework-specific optimization workflow. `group_limited_topk` through the diffusion facade when the CUDA layout guard passes, otherwise it retains the eager reference chain. - LTX2 RMSNorm+modulate and FFN GELU epilogue under `quality="lossless"` and `quality="high"`: - `mark_ltx2_rms_norm_modulate_site` / `fused_ltx2_rms_norm_modulate` in + `LTX2_RMS_NORM_MODULATE.mark` / `fused_ltx2_rms_norm_modulate` in `kernels/ops/diffusion/sites/ltx2_rmsnorm_modulate_site.py` (mount-based `QualityGatedFusion`, not a first-sight `BitExactFusionGate` — the fused kernel is <=1 ULP off aten, so it is request-gated instead of verified), diff --git a/python/sglang/multimodal_gen/.claude/CLAUDE.md b/python/sglang/multimodal_gen/.claude/CLAUDE.md index e952cb2ac3..422d0ae9d8 100644 --- a/python/sglang/multimodal_gen/.claude/CLAUDE.md +++ b/python/sglang/multimodal_gen/.claude/CLAUDE.md @@ -121,6 +121,5 @@ Look for `Pixel data generated successfully in xxxx seconds` in console output. ## Env Vars Defined in `envs.py` (300+ vars). Key ones: -- `SGLANG_DIFFUSION_ATTENTION_BACKEND` — attention backend override - `SGLANG_CACHE_DIT_ENABLED` — enable Cache-DiT acceleration - `SGLANG_CLOUD_STORAGE_TYPE` — cloud output storage (s3, etc.) diff --git a/python/sglang/multimodal_gen/apps/realtime_webui/app.js b/python/sglang/multimodal_gen/apps/realtime_webui/app.js index ade87daf67..e4ba1e685a 100644 --- a/python/sglang/multimodal_gen/apps/realtime_webui/app.js +++ b/python/sglang/multimodal_gen/apps/realtime_webui/app.js @@ -1,10 +1,8 @@ const $ = (id) => document.getElementById(id); const RAW_RGB_CONTENT_TYPE = "application/x-raw-rgb"; -const RAW_RGB_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgb-delta-gzip"; -const RAW_RGBA_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgba-delta-gzip"; const WEBP_FRAME_CONTENT_TYPE = "image/webp"; const JPEG_FRAME_CONTENT_TYPE = "image/jpeg"; -const DECODER_WORKER_URL = "./decoder_worker.js?v=rgb-worker-v10"; +const DECODER_WORKER_URL = "./decoder_worker.js?v=rgb-worker-v11"; const DEFAULT_PREVIEW_OUTPUT_FORMAT = "webp"; const DEFAULT_PREVIEW_OUTPUT_QUALITY = 80; const MAX_WEBP_PREVIEW_OUTPUT_QUALITY = 80; @@ -213,7 +211,6 @@ let droppedDecodeFrames = 0; let lastDecodeDropAt = 0; let lastDecodeDropCount = 0; let nextEventId = 1; -let lastRawRgbFrame = null; let decoderWorker = null; let decodeWorkerUnavailable = false; let decodeRequestId = 1; @@ -299,7 +296,6 @@ function resetStreamStats() { encodedDecodeErrors = 0; renderedPreviewFrames = 0; controlStateController?.reset({ sendRelease: false }); - resetDecoderState(); updateStats(); $("renderFps").textContent = "0"; $("latencyText").textContent = "-"; @@ -348,11 +344,6 @@ function ensureDecoderWorker() { }; } -function resetDecoderState() { - lastRawRgbFrame = null; - if (decoderWorker) decoderWorker.postMessage({ type: "reset" }); -} - async function decodeFrameBatch(header, data) { const decodeStartedAt = performance.now(); if (!isWorkerDecodableContentType(header.content_type)) { @@ -435,11 +426,7 @@ function isWorkerDecodableContentType(contentType) { } function isWorkerDecodableRawContentType(contentType) { - return ( - contentType === RAW_RGB_CONTENT_TYPE || - contentType === RAW_RGB_DELTA_GZIP_CONTENT_TYPE || - contentType === RAW_RGBA_DELTA_GZIP_CONTENT_TYPE - ); + return contentType === RAW_RGB_CONTENT_TYPE; } function updateStats() { @@ -1040,83 +1027,17 @@ function rgbToImageData(header, payload) { return items; } -function rgbaToImageData(header, payload) { - const width = Number(header.width), height = Number(header.height); - const count = Number(header.num_frames); - const frameBytes = Number(header.bytes_per_frame); - const src = payload instanceof Uint8Array ? payload : new Uint8Array(payload); - const items = []; - for (let f = 0; f < count; f++) { - const offset = f * frameBytes; - const imageBytes = new Uint8ClampedArray( - src.buffer, - src.byteOffset + offset, - frameBytes, - ); - items.push({ image: new ImageData(imageBytes, width, height), chunk: header.chunk_index }); - } - return items; -} - -async function gunzipBytes(payload) { - if (typeof DecompressionStream === "undefined") { - throw new Error("This browser does not support gzip stream decoding"); - } - const stream = new Blob([payload]).stream().pipeThrough(new DecompressionStream("gzip")); - return new Uint8Array(await new Response(stream).arrayBuffer()); -} - -async function restoreDeltaGzipRawRgb(header, payload) { - const frameBytes = Number(header.bytes_per_frame); - const count = Number(header.num_frames); - const expectedSize = frameBytes * count; - const restored = await gunzipBytes(payload); - if (restored.length !== expectedSize) { - throw new Error(`delta payload size mismatch: expected ${expectedSize}, got ${restored.length}`); - } - let previous = header.delta_reference === "previous-frame" ? lastRawRgbFrame : null; - if (header.delta_reference === "previous-frame" && !previous) { - throw new Error("Missing previous frame for delta payload"); - } - for (let f = 0; f < count; f++) { - const current = f * frameBytes; - if (previous) { - for (let i = 0; i < frameBytes; i++) { - restored[current + i] ^= previous[i]; - } - } - previous = restored.slice(current, current + frameBytes); - } - return restored; -} - async function framePayloadToImageData(header, payload) { - let rawPayload; - const isRgba = header.content_type === RAW_RGBA_DELTA_GZIP_CONTENT_TYPE; if ( header.content_type === WEBP_FRAME_CONTENT_TYPE || header.content_type === JPEG_FRAME_CONTENT_TYPE ) { return encodedImageToImageData(header, payload); } else if (header.content_type === RAW_RGB_CONTENT_TYPE) { - rawPayload = payload instanceof Uint8Array ? payload : new Uint8Array(payload); - } else if (header.content_type === RAW_RGB_DELTA_GZIP_CONTENT_TYPE) { - rawPayload = await restoreDeltaGzipRawRgb(header, payload); - } else if (isRgba) { - rawPayload = await restoreDeltaGzipRawRgb(header, payload); - } else { - throw new Error(`Unsupported content type ${header.content_type}`); + const rawPayload = payload instanceof Uint8Array ? payload : new Uint8Array(payload); + return rgbToImageData(header, rawPayload); } - const frameBytes = Number(header.bytes_per_frame); - const frameCount = Number(header.num_frames); - if (frameCount > 0) { - const offset = (frameCount - 1) * frameBytes; - lastRawRgbFrame = rawPayload.slice(offset, offset + frameBytes); - } - if (isRgba) { - return rgbaToImageData(header, rawPayload); - } - return rgbToImageData(header, rawPayload); + throw new Error(`Unsupported content type ${header.content_type}`); } function isEncodedPreviewContentType(contentType) { @@ -1344,7 +1265,6 @@ function abortCurrentSession(reason = "session closed by client", { controlStateController?.reset({ sendRelease: false }); pendingHeader = null; rejectPendingDecodes("session aborted"); - resetDecoderState(); if (clearFrames) { clearFrameQueue(); updateStats(); @@ -1881,7 +1801,6 @@ function selectedTransportLabel() { function shortPayloadMode(contentType) { if (contentType === WEBP_FRAME_CONTENT_TYPE) return "webp"; if (contentType === JPEG_FRAME_CONTENT_TYPE) return "jpeg"; - if (contentType === RAW_RGB_DELTA_GZIP_CONTENT_TYPE) return "delta-gzip"; if (contentType === RAW_RGB_CONTENT_TYPE) return "raw RGB"; return contentType; } diff --git a/python/sglang/multimodal_gen/apps/realtime_webui/decoder_worker.js b/python/sglang/multimodal_gen/apps/realtime_webui/decoder_worker.js index 5695f310d4..1fe9d12bed 100644 --- a/python/sglang/multimodal_gen/apps/realtime_webui/decoder_worker.js +++ b/python/sglang/multimodal_gen/apps/realtime_webui/decoder_worker.js @@ -1,51 +1,7 @@ const RAW_RGB_CONTENT_TYPE = "application/x-raw-rgb"; -const RAW_RGB_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgb-delta-gzip"; -const RAW_RGBA_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgba-delta-gzip"; const WEBP_FRAME_CONTENT_TYPE = "image/webp"; const JPEG_FRAME_CONTENT_TYPE = "image/jpeg"; -let lastFrame = null; - -function reset() { - lastFrame = null; -} - -async function gunzipBytes(payload) { - if (typeof DecompressionStream === "undefined") { - throw new Error("This browser does not support gzip stream decoding"); - } - const stream = new Blob([payload]).stream().pipeThrough(new DecompressionStream("gzip")); - return new Uint8Array(await new Response(stream).arrayBuffer()); -} - -async function restoreDeltaGzipFrames(header, payload) { - const frameBytes = Number(header.bytes_per_frame); - const count = Number(header.num_frames); - const expectedSize = frameBytes * count; - const restored = await gunzipBytes(payload); - if (restored.length !== expectedSize) { - throw new Error(`delta payload size mismatch: expected ${expectedSize}, got ${restored.length}`); - } - - let previous = header.delta_reference === "previous-frame" ? lastFrame : null; - if (header.delta_reference === "previous-frame") { - if (!previous) throw new Error("Missing previous frame for delta payload"); - if (previous.byteLength !== frameBytes) { - throw new Error("Previous frame size does not match current delta payload"); - } - } - - for (let f = 0; f < count; f++) { - const offset = f * frameBytes; - if (previous) { - for (let i = 0; i < frameBytes; i++) restored[offset + i] ^= previous[i]; - } - previous = restored.slice(offset, offset + frameBytes); - } - lastFrame = previous; - return restored; -} - function rawFramesToRgbaBuffers(header, payload) { const width = Number(header.width); const height = Number(header.height); @@ -130,16 +86,6 @@ async function decode(header, payload) { }; } else if (header.content_type === RAW_RGB_CONTENT_TYPE) { rawPayload = new Uint8Array(payload); - const frameBytes = Number(header.bytes_per_frame); - const count = Number(header.num_frames); - lastFrame = count > 0 - ? rawPayload.slice((count - 1) * frameBytes, count * frameBytes) - : null; - } else if ( - header.content_type === RAW_RGB_DELTA_GZIP_CONTENT_TYPE || - header.content_type === RAW_RGBA_DELTA_GZIP_CONTENT_TYPE - ) { - rawPayload = await restoreDeltaGzipFrames(header, payload); } else { throw new Error(`Unsupported content type ${header.content_type}`); } @@ -156,10 +102,6 @@ async function decode(header, payload) { self.onmessage = async (event) => { const message = event.data; try { - if (message.type === "reset") { - reset(); - return; - } const result = await decode(message.header, message.payload); self.postMessage({ type: "decoded", ...result }, result.frames); } catch (error) { diff --git a/python/sglang/multimodal_gen/apps/realtime_webui/index.html b/python/sglang/multimodal_gen/apps/realtime_webui/index.html index 3489af8f36..74404062a9 100644 --- a/python/sglang/multimodal_gen/apps/realtime_webui/index.html +++ b/python/sglang/multimodal_gen/apps/realtime_webui/index.html @@ -45,7 +45,7 @@ diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/__init__.py b/python/sglang/multimodal_gen/configs/pipeline_configs/__init__.py index 5021ca2e02..827196cb9f 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/__init__.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/__init__.py @@ -2,7 +2,6 @@ from sglang.multimodal_gen.configs.pipeline_configs.base import ( PipelineConfig, - SlidingTileAttnConfig, ) from sglang.multimodal_gen.configs.pipeline_configs.cosmos3 import Cosmos3Config from sglang.multimodal_gen.configs.pipeline_configs.cosmos3_multiview import ( @@ -109,7 +108,6 @@ __all__ = [ "SanaPipelineConfig", "SanaVideoPipelineConfig", "SenseNovaU1PipelineConfig", - "SlidingTileAttnConfig", "MOVAPipelineConfig", "Pi05PipelineConfig", "StableDiffusion3PipelineConfig", diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/base.py b/python/sglang/multimodal_gen/configs/pipeline_configs/base.py index 2fb698d945..e6b66ff26f 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/base.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/base.py @@ -1350,24 +1350,6 @@ class SpatialImagePipelineConfig(ImagePipelineConfig): return self._gather_sp_tensor(latents, dim=2) -@dataclass -class SlidingTileAttnConfig(PipelineConfig): - """Configuration for sliding tile attention.""" - - # Override any BaseConfig defaults as needed - # Add sliding tile specific parameters - window_size: int = 16 - stride: int = 8 - - # You can provide custom defaults for inherited fields - height: int = 576 - width: int = 1024 - - # Additional configuration specific to sliding tile attention - pad_to_square: bool = False - use_overlap_optimization: bool = True - - def parse_int_list(value: str) -> list[int]: """Parse a comma-separated string of integers into a list.""" if not value: diff --git a/python/sglang/multimodal_gen/configs/sample/hunyuan_image3.py b/python/sglang/multimodal_gen/configs/sample/hunyuan_image3.py index f11ad64dc6..90909a8003 100644 --- a/python/sglang/multimodal_gen/configs/sample/hunyuan_image3.py +++ b/python/sglang/multimodal_gen/configs/sample/hunyuan_image3.py @@ -6,8 +6,6 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger logger = init_logger(__name__) -HUNYUAN_IMAGE3_RESOLUTION_ALIGNMENT = 16 - VALID_BOT_TASKS = { "auto", "image", @@ -103,18 +101,3 @@ class HunyuanImage3SamplingParams(SamplingParams): or not 0.0 <= (self.output_pad_value) <= 1.0 ): raise ValueError("output_pad_value must be a finite value in [0, 1]") - - -def align_hunyuan_image3_dimension(value: int) -> int: - """Round a HunyuanImage-3 dimension up to a supported multiple.""" - return max( - HUNYUAN_IMAGE3_RESOLUTION_ALIGNMENT, - (value + HUNYUAN_IMAGE3_RESOLUTION_ALIGNMENT - 1) - // HUNYUAN_IMAGE3_RESOLUTION_ALIGNMENT - * HUNYUAN_IMAGE3_RESOLUTION_ALIGNMENT, - ) - - -def align_hunyuan_image3_resolution(width: int, height: int) -> tuple[int, int]: - """Align both width and height to supported multiples.""" - return align_hunyuan_image3_dimension(width), align_hunyuan_image3_dimension(height) diff --git a/python/sglang/multimodal_gen/envs.py b/python/sglang/multimodal_gen/envs.py index d882a9c220..51102ab850 100644 --- a/python/sglang/multimodal_gen/envs.py +++ b/python/sglang/multimodal_gen/envs.py @@ -22,13 +22,8 @@ if "SGLANG_DIFFUSION_TRACE_FUNCTION" in os.environ: if TYPE_CHECKING: SGLANG_DIFFUSION_NCCL_SO_PATH: str | None = None - LD_LIBRARY_PATH: str | None = None LOCAL_RANK: int = 0 - CUDA_VISIBLE_DEVICES: str | None = None SGLANG_DIFFUSION_CACHE_ROOT: str = os.path.expanduser("~/.cache/sgl_diffusion") - SGLANG_DIFFUSION_CONFIG_ROOT: str = os.path.expanduser("~/.config/sgl_diffusion") - SGLANG_DIFFUSION_LOGGING_LEVEL: str = "INFO" - SGLANG_DIFFUSION_LOGGING_PREFIX: str = "" SGLANG_DIFFUSION_DISABLE_EARLY_VAE_DECODER_CAST: bool = False SGLANG_DIFFUSION_DISABLE_VAE_DECODER_STORE: bool = False SGLANG_DIFFUSION_DISABLE_MAPPED_WILLNEED: bool = False @@ -36,15 +31,9 @@ if TYPE_CHECKING: SGLANG_DIFFUSION_DEBUG_HOST_MEMORY: bool = False SGLANG_DIFFUSION_DEBUG_LAYERWISE_TIMING: bool = False SGLANG_DIFFUSION_DISABLE_LORA_MERGE_CACHE: bool = False - SGLANG_DIFFUSION_TARGET_DEVICE: str = "cuda" SGLANG_DIFFUSION_PLATFORM_OVERRIDE: str = "" SGLANG_EXTERNAL_MODEL_PACKAGE: str = "" - MAX_JOBS: str | None = None - NVCC_THREADS: str | None = None - CMAKE_BUILD_TYPE: str | None = None - VERBOSE: bool = False SGLANG_DIFFUSION_DISABLE_SP_PAD_MASK: bool = False - SGLANG_DIFFUSION_SERVER_DEV_MODE: bool = False SGLANG_DIFFUSION_DISABLE_MAPPED_COURIER: bool = False SGLANG_DIFFUSION_HOST_SPILL_DIR: str = os.path.expanduser( "~/.cache/sglang/diffusion/host_spill" @@ -125,6 +114,10 @@ if TYPE_CHECKING: SGLANG_LINGBOT_LAZY_VAE_ENCODE_BLACK_FRAMES: int | None = None SGLANG_DIFFUSION_FLASHINFER_FP4_GEMM_BACKEND: str | None = None SGLANG_DIFFUSION_ENABLE_W8A8_FP8_GEMM: bool = False + SGLANG_ENABLE_FUSED_QKNORM_ROPE: bool = True + MINIMAX_H3_VAE_DECODER_FUSED_NORM: bool = True + MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM: bool = True + MINIMAX_H3_VAE_DECODER_STREAM_TEMPORAL_CAT: bool = True SGLANG_DIFFUSION_MXFP8_FA_HEAD_CHUNK_SIZE: int = 4 SGLANG_DIFFUSION_FP8_WEIGHT_DEQUANT_CACHE: bool = True SGLANG_DIFFUSION_ENABLE_COSMOS3_STEP_MIXED_PRECISION: bool = True @@ -145,13 +138,6 @@ def get_default_cache_root() -> str: ) -def get_default_config_root() -> str: - return os.getenv( - "XDG_CONFIG_HOME", - os.path.join(os.path.expanduser("~"), ".config"), - ) - - # helpers for environment variable definitions def _lazy_str(key: str, default: str | None = None) -> Callable[[], str | None]: return lambda: os.getenv(key, default) @@ -203,34 +189,6 @@ def _lazy_path( # begin-env-vars-definition environment_variables: dict[str, Callable[[], Any]] = { - # ================== Installation Time Env Vars ================== - # Target device of sglang-diffusion, supporting [cuda (by default), - # rocm, neuron, cpu, openvino] - "SGLANG_DIFFUSION_TARGET_DEVICE": _lazy_str( - "SGLANG_DIFFUSION_TARGET_DEVICE", "cuda" - ), - # Maximum number of compilation jobs to run in parallel. - # By default this is the number of CPUs - "MAX_JOBS": _lazy_str("MAX_JOBS"), - # Number of threads to use for nvcc - # By default this is 1. - # If set, `MAX_JOBS` will be reduced to avoid oversubscribing the CPU. - "NVCC_THREADS": _lazy_str("NVCC_THREADS"), - # CMake build type - # If not set, defaults to "Debug" or "RelWithDebInfo" - # Available options: "Debug", "Release", "RelWithDebInfo" - "CMAKE_BUILD_TYPE": _lazy_str("CMAKE_BUILD_TYPE"), - # If set, sgl_diffusion will print verbose logs during installation - "VERBOSE": _lazy_bool("VERBOSE"), - # Root directory for SGL-diffusion configuration files - # Defaults to `~/.config/sgl_diffusion` unless `XDG_CONFIG_HOME` is set - # Note that this not only affects how sgl_diffusion finds its configuration files - # during runtime, but also affects how sgl_diffusion installs its configuration - # files during **installation**. - "SGLANG_DIFFUSION_CONFIG_ROOT": _lazy_path( - "SGLANG_DIFFUSION_CONFIG_ROOT", - lambda: os.path.join(get_default_config_root(), "sgl_diffusion"), - ), # ================== Runtime Env Vars ================== # Root directory for SGL-diffusion cache files # Defaults to `~/.cache/sgl_diffusion` unless `XDG_CACHE_HOME` is set @@ -241,29 +199,14 @@ environment_variables: dict[str, Callable[[], Any]] = { # Path to the NCCL library file. It is needed because nccl>=2.19 brought # by PyTorch contains a bug: https://github.com/NVIDIA/nccl/issues/1234 "SGLANG_DIFFUSION_NCCL_SO_PATH": _lazy_str("SGLANG_DIFFUSION_NCCL_SO_PATH"), - # when `SGLANG_DIFFUSION_NCCL_SO_PATH` is not set, sgl_diffusion will try to find the nccl - # library file in the locations specified by `LD_LIBRARY_PATH` - "LD_LIBRARY_PATH": _lazy_str("LD_LIBRARY_PATH"), # local rank of the process in the distributed setting, used to determine # the GPU device id "LOCAL_RANK": _lazy_int("LOCAL_RANK", 0), - # used to control the visible devices in the distributed setting - "CUDA_VISIBLE_DEVICES": _lazy_str("CUDA_VISIBLE_DEVICES"), - # this is used for configuring the default logging level - "SGLANG_DIFFUSION_LOGGING_LEVEL": _lazy_str( - "SGLANG_DIFFUSION_LOGGING_LEVEL", "INFO" - ), - # if set, SGLANG_DIFFUSION_LOGGING_PREFIX will be prepended to all log messages - "SGLANG_DIFFUSION_LOGGING_PREFIX": _lazy_str("SGLANG_DIFFUSION_LOGGING_PREFIX", ""), # Path to the attention configuration file. Only used for sliding tile # attention for now. "SGLANG_DIFFUSION_ATTENTION_CONFIG": _lazy_path( "SGLANG_DIFFUSION_ATTENTION_CONFIG" ), - # Optional override to force a specific attention backend (e.g. "aiter") - "SGLANG_DIFFUSION_ATTENTION_BACKEND": _lazy_str( - "SGLANG_DIFFUSION_ATTENTION_BACKEND" - ), # Cosmos3 Multiview-AV attention backend for masked exports: "auto" (default; # FA4 block-sparse on Hopper and Blackwell, FlexAttention Triton elsewhere), "triton" or # "fa4". An explicit pipeline-config multiview_attention_backend wins over it. @@ -314,10 +257,6 @@ environment_variables: dict[str, Callable[[], Any]] = { "SGLANG_DIFFUSION_TORCH_PROFILER_DIR": _lazy_path( "SGLANG_DIFFUSION_TORCH_PROFILER_DIR" ), - # If set, sgl_diffusion will run in development mode, which will enable - # some additional endpoints for developing and debugging, - # e.g. `/reset_prefix_cache` - "SGLANG_DIFFUSION_SERVER_DEV_MODE": _lazy_bool("SGLANG_DIFFUSION_SERVER_DEV_MODE"), # Kill-switch for the courier thread that ships checkpoint-mapped layers to # the device off the compute thread. The courier already falls back to the # synchronous copy on any failure; this forces that path up front. @@ -521,6 +460,30 @@ environment_variables: dict[str, Callable[[], Any]] = { "SGLANG_DIFFUSION_ENABLE_W8A8_FP8_GEMM": _lazy_bool( "SGLANG_DIFFUSION_ENABLE_W8A8_FP8_GEMM" ), + # Kill-switch for the fused QK RMSNorm + RoPE kernels in diffusion DiTs. + # Only 0/false/off/no disable it; any other value keeps it on. + "SGLANG_ENABLE_FUSED_QKNORM_ROPE": lambda: ( + os.getenv("SGLANG_ENABLE_FUSED_QKNORM_ROPE", "1").lower() + not in {"0", "false", "off", "no"} + ), + # MiniMax-H3 VAE decoder: fused RMSNorm / QK RMSNorm+RoPE kernels in the + # ViT blocks (NVIDIA only). Only 1/true/yes/on keep it enabled. + "MINIMAX_H3_VAE_DECODER_FUSED_NORM": lambda: ( + os.getenv("MINIMAX_H3_VAE_DECODER_FUSED_NORM", "1").strip().lower() + in ("1", "true", "yes", "on") + ), + # MiniMax-H3 VAE decoder: run the ViT norms in FP32 (the released recipe). + # Only 1/true/yes/on keep it enabled. + "MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM": lambda: ( + os.getenv("MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM", "1").strip().lower() + in ("1", "true", "yes", "on") + ), + # MiniMax-H3 VAE: stitch temporal decode chunks into a preallocated output + # as they finish instead of concatenating at the end. + "MINIMAX_H3_VAE_DECODER_STREAM_TEMPORAL_CAT": lambda: ( + os.getenv("MINIMAX_H3_VAE_DECODER_STREAM_TEMPORAL_CAT", "1").strip().lower() + not in ("0", "false", "no", "off", "disable", "disabled") + ), # Dequantize storage-only FP8 linear weights to the compute dtype once, # at first use (bit-identical outputs; trades weight VRAM for skipping # the per-forward dequant pass). Weights are kept FP8-resident when free diff --git a/python/sglang/multimodal_gen/registry.py b/python/sglang/multimodal_gen/registry.py index c06eadb178..8115ff88c3 100644 --- a/python/sglang/multimodal_gen/registry.py +++ b/python/sglang/multimodal_gen/registry.py @@ -368,24 +368,36 @@ def _normalize_hf_cache_path(path: str) -> str: return os.path.normpath(path).lower().replace("\\", "/") -def has_registered_diffusion_model_path(model_path: str) -> bool: - _ensure_registry_initialized() +def _match_registered_hf_path(model_path: str) -> Optional[str]: + """The registered HF id ``model_path`` names: exactly, by short name, or as + a local HuggingFace cache path.""" all_model_hf_paths = sorted(_MODEL_HF_PATH_TO_NAME.keys(), key=len, reverse=True) - if is_sensenova_u1_model(model_path): - return True - + # 1. Exact match if model_path in _MODEL_HF_PATH_TO_NAME: - return True + logger.debug(f"Resolved model path '{model_path}' from exact path match.") + return model_path + # 2. Partial match: find the best (longest) match against all registered model hf paths. model_short_name = get_model_short_name(model_path.lower()) for registered_model_hf_id in all_model_hf_paths: if registered_model_hf_id.lower() in SENSENOVA_U1_MODEL_IDS: continue registered_model_name = get_model_short_name(registered_model_hf_id.lower()) - if registered_model_name in model_short_name: - return True + if registered_model_name in model_short_name: + logger.debug( + f"Resolved model name '{registered_model_hf_id}' from partial path match." + ) + return registered_model_hf_id + + # 2b. Match local HuggingFace cache snapshot/blob paths such as: + # .../models--org--repo/snapshots/ + # This lets users pass a local HF cache snapshot directory directly even + # when its basename is only the snapshot hash. + # Example: + # /xxx/models--black-forest-labs--FLUX.2-dev-NVFP4/snapshots/142b87e70bc3006937b7093d89ff287b5f59f071 + # -> models--black-forest-labs--flux.2-dev-nvfp4 (to match with cache_repo_fragment) normalized_model_path = _normalize_hf_cache_path(model_path) for registered_model_hf_id in all_model_hf_paths: if registered_model_hf_id.lower() in SENSENOVA_U1_MODEL_IDS: @@ -394,9 +406,22 @@ def has_registered_diffusion_model_path(model_path: str) -> bool: f"models--{registered_model_hf_id.lower().replace('/', '--')}" ) if cache_repo_fragment in normalized_model_path: - return True + logger.debug( + "Resolved HuggingFace cache path '%s' to registered model '%s'.", + model_path, + registered_model_hf_id, + ) + return registered_model_hf_id - return False + return None + + +def has_registered_diffusion_model_path(model_path: str) -> bool: + _ensure_registry_initialized() + return ( + is_sensenova_u1_model(model_path) + or _match_registered_hf_path(model_path) is not None + ) @lru_cache(maxsize=1) @@ -436,48 +461,10 @@ def _get_config_info( if is_flux3_image_package(model_path): return _CONFIG_REGISTRY.get(_MODEL_HF_PATH_TO_NAME[FLUX3_IMAGE_HF_PATHS[0]]) - # 1. Exact match - if model_path in _MODEL_HF_PATH_TO_NAME: - model_id = _MODEL_HF_PATH_TO_NAME[model_path] - logger.debug(f"Resolved model path '{model_path}' from exact path match.") - return _CONFIG_REGISTRY.get(model_id) - - # 2. Partial match: find the best (longest) match against all registered model hf paths. - model_short_name = get_model_short_name(model_path.lower()) - for registered_model_hf_id in all_model_hf_paths: - if registered_model_hf_id.lower() in SENSENOVA_U1_MODEL_IDS: - continue - registered_model_name = get_model_short_name(registered_model_hf_id.lower()) - - if registered_model_name in model_short_name: - logger.debug( - f"Resolved model name '{registered_model_hf_id}' from partial path match." - ) - model_id = _MODEL_HF_PATH_TO_NAME[registered_model_hf_id] - return _CONFIG_REGISTRY.get(model_id) - - # 2b. Match local HuggingFace cache snapshot/blob paths such as: - # .../models--org--repo/snapshots/ - # This lets users pass a local HF cache snapshot directory directly even - # when its basename is only the snapshot hash. - # Example: - # /xxx/models--black-forest-labs--FLUX.2-dev-NVFP4/snapshots/142b87e70bc3006937b7093d89ff287b5f59f071 - # -> models--black-forest-labs--flux.2-dev-nvfp4 (to match with cache_repo_fragment) - normalized_model_path = _normalize_hf_cache_path(model_path) - for registered_model_hf_id in all_model_hf_paths: - if registered_model_hf_id.lower() in SENSENOVA_U1_MODEL_IDS: - continue - cache_repo_fragment = ( - f"models--{registered_model_hf_id.lower().replace('/', '--')}" - ) - if cache_repo_fragment in normalized_model_path: - logger.debug( - "Resolved HuggingFace cache path '%s' to registered model '%s'.", - model_path, - registered_model_hf_id, - ) - model_id = _MODEL_HF_PATH_TO_NAME[registered_model_hf_id] - return _CONFIG_REGISTRY.get(model_id) + # 1/2/2b. Exact, short-name, or local HuggingFace cache path match + registered_hf_id = _match_registered_hf_path(model_path) + if registered_hf_id is not None: + return _CONFIG_REGISTRY.get(_MODEL_HF_PATH_TO_NAME[registered_hf_id]) # 3. Use detectors pipeline_name = get_non_diffusers_pipeline_name(model_path) diff --git a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/runner.py b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/runner.py index a6be42a84e..56dddd4a0e 100644 --- a/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/runner.py +++ b/python/sglang/multimodal_gen/runtime/breakable_cuda_graph/runner.py @@ -64,17 +64,6 @@ def _env_int(name: str, default: int) -> int: return default -def _env_float(name: str, default: float) -> float: - raw = os.environ.get(name) - if raw is None: - return default - try: - return float(raw) - except ValueError: - logger.warning("[BCG] ignoring invalid float %s=%r", name, raw) - return default - - def _map_tensors(obj, fn): """Rebuild ``obj`` applying ``fn`` to every tensor leaf, recursing into list/tuple/dict containers; everything else passes through unchanged.""" diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py b/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py index 1a081f97f7..167ffcad5b 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py @@ -694,55 +694,6 @@ class DiffGenerator: # _send_lora_request already raises on error, so output.error is always None here return output.output or {} - def _ensure_lora_state( - self, - lora_path: str | None, - lora_nickname: str | None = None, - merge_lora: bool = True, - ) -> None: - """ - Ensure the LoRA state matches the desired configuration. - - Note: This method does not cache client-side state. The server handles - idempotent operations, so redundant calls are safe but may have minor overhead. - """ - if lora_path is None: - # Unmerge all LoRA weights when no lora_path is provided - self.unmerge_lora_weights() - return - - lora_nickname = lora_nickname or self.server_args.lora_nickname - - # Set the LoRA adapter (server handles idempotent logic) - self.set_lora(lora_nickname, lora_path) - - # Merge or unmerge based on the merge_lora flag - if merge_lora: - self.merge_lora_weights() - else: - self.unmerge_lora_weights() - - def generate_with_lora( - self, - prompt: str | list[str] | None = None, - sampling_params: SamplingParams | None = None, - *, - lora_path: str | None = None, - lora_nickname: str | None = None, - merge_lora: bool = True, - **kwargs, - ): - self._ensure_lora_state( - lora_path=lora_path, lora_nickname=lora_nickname, merge_lora=merge_lora - ) - return self.generate( - sampling_params_kwargs=dict( - prompt=prompt, - sampling_params=sampling_params, - **kwargs, - ) - ) - def shutdown(self): """ Shutdown the generator. diff --git a/python/sglang/multimodal_gen/runtime/layers/flux3_fused_ops.py b/python/sglang/multimodal_gen/runtime/layers/flux3_fused_ops.py index 88bef517e6..e36d22269b 100644 --- a/python/sglang/multimodal_gen/runtime/layers/flux3_fused_ops.py +++ b/python/sglang/multimodal_gen/runtime/layers/flux3_fused_ops.py @@ -18,7 +18,7 @@ import torch from torch import nn from sglang.kernels.fused_op import BaseFusedOp, register_fused_op -from sglang.kernels.spec import FormatSignature, KernelBackend +from sglang.kernels.spec import KernelBackend from sglang.multimodal_gen.runtime.layers.layernorm import apply_qk_norm_rope from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA from sglang.multimodal_gen.runtime.layers.quantization.flux3_prims_ts import ( @@ -79,10 +79,6 @@ def _unpack_qkv(qkv: torch.Tensor, local_heads: int): class _Flux3PrimsTsOp(BaseFusedOp): priority = (KernelBackend.FLASHINFER, KernelBackend.TORCH) capabilities = {KernelBackend.FLASHINFER: FLUX3_PRIMS_TS_CAPABILITIES} - format_signature = FormatSignature( - supported_dtypes=("bfloat16",), - description="FLUX.3 projection; layer-prepared PrimsTS weights", - ) def _flashinfer_eligible( self, layer: nn.Module, x: torch.Tensor, role: Flux3PrimsTsRole @@ -98,10 +94,6 @@ class Flux3LinearOp(_Flux3PrimsTsOp): """``layer(x)`` for an attention, MLP-down, or unfused QKV projection.""" op = "gemm.flux3_linear" - descriptions = { - KernelBackend.FLASHINFER: "PrimsTS FP8/NVFP4 dense GEMM.", - KernelBackend.TORCH: "Generic SGLang linear (layer quant method).", - } def backend_eligible(self, backend, layer, x, quantized=None) -> bool: if backend is KernelBackend.FLASHINFER: @@ -125,10 +117,6 @@ class Flux3QKVQKNormRopeOp(_Flux3PrimsTsOp): """Packed QKV projection, Q/K RMSNorm, and RoPE -> ``(B, L, H, D)`` q, k, v.""" op = "gemm.flux3_qkv_qknorm_rope" - descriptions = { - KernelBackend.FLASHINFER: "PrimsTS QKV GEMM with QK-norm/RoPE epilogue.", - KernelBackend.TORCH: "FLUX.3 linear, then fused QK-norm/RoPE kernel.", - } def backend_eligible( self, @@ -197,10 +185,6 @@ class Flux3GatedMLPOp(_Flux3PrimsTsOp): """``mlp_out(silu(gate) * value)`` with ``[gate | value] = mlp_in(x)``.""" op = "gemm.flux3_gated_mlp" - descriptions = { - KernelBackend.FLASHINFER: "PrimsTS MLP-up with SwiGLU epilogue.", - KernelBackend.TORCH: "Generic MLP-up, silu_and_mul, FLUX.3 linear.", - } def backend_eligible(self, backend, mlp_in, mlp_out, x, quantized=None) -> bool: if backend is KernelBackend.FLASHINFER: diff --git a/python/sglang/multimodal_gen/runtime/layers/layernorm.py b/python/sglang/multimodal_gen/runtime/layers/layernorm.py index 362fdf2784..b9c9ed372d 100755 --- a/python/sglang/multimodal_gen/runtime/layers/layernorm.py +++ b/python/sglang/multimodal_gen/runtime/layers/layernorm.py @@ -4,7 +4,6 @@ # Adapted from vllm: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/layers/layernorm.py """Custom normalization layers.""" -import os from typing import Optional, Tuple, Union import torch @@ -30,6 +29,7 @@ from sglang.kernels.ops.layernorm.rmsnorm_hf import ( is_supported_rmsnorm_hf_hidden_size, rmsnorm_hf, ) +from sglang.multimodal_gen import envs from sglang.multimodal_gen.runtime.distributed.parallel_state import ( get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, @@ -1155,12 +1155,7 @@ def apply_qk_norm_rope( raise ValueError( f"cos_sin_cache width must be even and <= head_dim, got {rope_dim} vs {head_dim}" ) - fused_enabled = os.getenv("SGLANG_ENABLE_FUSED_QKNORM_ROPE", "1").lower() not in { - "0", - "false", - "off", - "no", - } + fused_enabled = envs.SGLANG_ENABLE_FUSED_QKNORM_ROPE q_has_supported_layout = q.is_contiguous() k_has_supported_layout = k.is_contiguous() if allow_strided_qk: diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py b/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py index bc68dccf70..2ca566b686 100644 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/__init__.py @@ -27,37 +27,6 @@ QuantizationMethods = Literal[ QUANTIZATION_METHODS: list[str] = list(get_args(QuantizationMethods)) -# The customized quantization methods which will be added to this dict. -_CUSTOMIZED_METHOD_TO_QUANT_CONFIG: dict[str, type[QuantizationConfig]] = {} - - -def register_quantization_config(quantization: str): - """Register a customized vllm quantization config. - - When a quantization method is not supported by vllm, you can register a customized - quantization config to support it. - - Args: - quantization (str): The quantization method name. - - - """ # noqa: E501 - - def _wrapper(quant_config_cls): - if quantization in QUANTIZATION_METHODS: - raise ValueError( - f"The quantization method `{quantization}` is already exists." - ) - if not issubclass(quant_config_cls, QuantizationConfig): - raise ValueError( - "The quantization config must be a subclass of `QuantizationConfig`." - ) - _CUSTOMIZED_METHOD_TO_QUANT_CONFIG[quantization] = quant_config_cls - QUANTIZATION_METHODS.append(quantization) - return quant_config_cls - - return _wrapper - def get_quantization_config(quantization: str) -> type[QuantizationConfig]: quantization = canonical_quantization_method(quantization) @@ -104,8 +73,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]: "mxfp4_npu": NPUMXFP4Config, "convrot_int8": ConvRotInt8Config, } - # Update the `method_to_config` with customized quantization methods. - method_to_config.update(_CUSTOMIZED_METHOD_TO_QUANT_CONFIG) return method_to_config[quantization] diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/bitsandbytes.py b/python/sglang/multimodal_gen/runtime/layers/quantization/bitsandbytes.py index bcc53f635f..e13aa012be 100644 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/bitsandbytes.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/bitsandbytes.py @@ -168,83 +168,6 @@ def apply_bnb_4bit( current_index += output_size -class BitsAndBytes4BitLinear(nn.Module): - """Storage-only bitsandbytes 4-bit linear for nn.Linear-based encoders.""" - - def __init__( - self, - in_features: int, - out_features: int, - bias: bool = True, - compute_dtype: torch.dtype | None = None, - ) -> None: - super().__init__() - require_bitsandbytes() - self.in_features = in_features - self.out_features = out_features - self.compute_dtype = compute_dtype - quant_ratio = calculate_quant_ratio(compute_dtype or torch.get_default_dtype()) - total_size = in_features * out_features - if total_size % quant_ratio != 0: - raise ValueError( - "The input size is not aligned with the quantized weight shape." - ) - - self.weight = nn.Parameter( - torch.empty(total_size // quant_ratio, 1, dtype=torch.uint8), - requires_grad=False, - ) - set_weight_attrs( - self.weight, - { - "pack_factor": quant_ratio, - "use_bitsandbytes_4bit": True, - }, - ) - if bias: - self.bias = nn.Parameter( - torch.empty( - out_features, dtype=compute_dtype or torch.get_default_dtype() - ), - requires_grad=False, - ) - else: - self.register_parameter("bias", None) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - original_type = x.dtype - original_shape = x.shape - if x.ndim > 2: - x = x.reshape(-1, x.size(-1)) - - out = torch.empty( - x.shape[0], self.out_features, dtype=torch.bfloat16, device=x.device - ) - apply_bnb_4bit(x.to(torch.bfloat16), self.weight, out) - out = out.to(original_type) - - if len(original_shape) > 2: - out = out.view(*original_shape[:-1], out.size(-1)) - - if self.bias is not None: - out = out + self.bias - return out - - -def swap_linears_to_bitsandbytes_4bit(module: nn.Module) -> None: - for name, child in list(module.named_children()): - if isinstance(child, nn.Linear): - replacement = BitsAndBytes4BitLinear( - child.in_features, - child.out_features, - bias=child.bias is not None, - compute_dtype=child.weight.dtype, - ) - setattr(module, name, replacement) - else: - swap_linears_to_bitsandbytes_4bit(child) - - _BNB_4BIT_STATE_SUFFIXES = { "absmax", "quant_map", diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py b/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py index 34fb45b8dd..d5d1c93be4 100755 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/modelopt_quant.py @@ -194,17 +194,6 @@ def _rebase_partition_weight_scales( return rebased.to(torch.float8_e4m3fn) -def _nvfp4_uses_tma_scale_layout( - flashinfer_backend: str | None, *, uses_flux1_scale_layout: bool = False -) -> bool: - return uses_flux1_scale_layout or flashinfer_backend in { - None, - "auto", - "cudnn", - "cutlass", - } - - def _dynamic_nvfp4_input_scale(layer: torch.nn.Module, x: torch.Tensor) -> torch.Tensor: """Compute the reference tensorwise activation scale. diff --git a/python/sglang/multimodal_gen/runtime/layers/quantization/weight_only_fp8.py b/python/sglang/multimodal_gen/runtime/layers/quantization/weight_only_fp8.py index 99ed9a9480..12c232f17f 100644 --- a/python/sglang/multimodal_gen/runtime/layers/quantization/weight_only_fp8.py +++ b/python/sglang/multimodal_gen/runtime/layers/quantization/weight_only_fp8.py @@ -490,24 +490,3 @@ def _maybe_promote_fp8_weight(module: nn.Module, x_dtype: torch.dtype) -> None: "FP8-resident)." ) _dequant_cache_logged = True - - -def swap_linears_to_weight_only_fp8(module: nn.Module) -> None: - """Recursively replace nn.Linear with WeightOnlyFP8Linear. - - Ideogram FP8 checkpoints provide ``.weight_scale`` for every - quantized linear. Swapping before load lets strict state-dict checks verify - both the FP8 weight and its row-wise scale. - """ - - for name, child in list(module.named_children()): - if isinstance(child, nn.Linear): - replacement = WeightOnlyFP8Linear( - child.in_features, - child.out_features, - bias=child.bias is not None, - compute_dtype=child.weight.dtype, - ) - setattr(module, name, replacement) - else: - swap_linears_to_weight_only_fp8(child) diff --git a/python/sglang/multimodal_gen/runtime/loader/readonly_safetensors.py b/python/sglang/multimodal_gen/runtime/loader/readonly_safetensors.py index c2284d1f36..47182e6b33 100644 --- a/python/sglang/multimodal_gen/runtime/loader/readonly_safetensors.py +++ b/python/sglang/multimodal_gen/runtime/loader/readonly_safetensors.py @@ -76,11 +76,6 @@ def _tensor(mapped: mmap.mmap, base: int, meta: dict) -> torch.Tensor: return flat.view(shape) -def safetensors_keys(path: str) -> list[str]: - header, _ = _header(_mapping(path)) - return [name for name in header if name != "__metadata__"] - - def iter_safetensors_readonly(path: str) -> Iterator[tuple[str, torch.Tensor]]: """(name, tensor) for every tensor in the file, as views of a read-only mapping.""" mapped = _mapping(path) diff --git a/python/sglang/multimodal_gen/runtime/loader/utils.py b/python/sglang/multimodal_gen/runtime/loader/utils.py index d53c0cb0fc..f38f006126 100644 --- a/python/sglang/multimodal_gen/runtime/loader/utils.py +++ b/python/sglang/multimodal_gen/runtime/loader/utils.py @@ -630,26 +630,6 @@ def _load_safetensors_file(path: str) -> dict[str, torch.Tensor]: return safetensors_load_file(path) -def load_safetensors_state_dict(model_path: str) -> dict[str, torch.Tensor]: - """Load one safetensors checkpoint, including an indexed sharded set.""" - index_path = _select_safetensors_index_file(model_path, _DEFAULT_SAFETENSORS_INDEX) - safetensors_files = _list_safetensors_files(model_path) - if index_path is not None: - state_dict: dict[str, torch.Tensor] = {} - for path in safetensors_files: - state_dict.update(_load_safetensors_file(path)) - return state_dict - - if not safetensors_files: - raise ValueError(f"No safetensors files found in {model_path}") - if len(safetensors_files) != 1: - raise ValueError( - f"Found {len(safetensors_files)} safetensors files in {model_path} " - "and no index to disambiguate them." - ) - return _load_safetensors_file(safetensors_files[0]) - - BYTES_PER_GB = 1024**3 diff --git a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py index fe4c366f63..061ce945d9 100644 --- a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py +++ b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py @@ -66,7 +66,6 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager im peek_global_component_residency_manager, ) from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( - LayerwiseUsageTracker, configure_layerwise_offload_modules, ) from sglang.multimodal_gen.runtime.managers.memory_managers.memory_occupation_controller import ( @@ -708,16 +707,11 @@ class GPUWorker(GPUWorkerPostTrainingMixin): int(req.width or 0), int(req.height or 0), int(req.num_frames or 1), - max(1, int(req.num_inference_steps or 1)), ) if measure_server_warmup else None ) warmup_baseline_allocated_bytes = 0 - layerwise_usage_tracker: LayerwiseUsageTracker | None = None - layerwise_layer_uses_by_stage: dict[ - str, dict[str, dict[str, tuple[int, ...]]] - ] = {} try: if measure_server_warmup: # Drop the previous request's allocator pool so each probe @@ -731,20 +725,6 @@ class GPUWorker(GPUWorkerPostTrainingMixin): warmup_baseline_allocated_bytes = ( torch.get_device_module().memory_allocated() ) - if ( - self.server_args.performance_mode == "auto" - and self.pipeline is not None - ): - layerwise_usage_tracker = LayerwiseUsageTracker( - self.pipeline.modules, - stage_name_provider=( - lambda: ( - req.metrics.active_stage_name - if req.metrics is not None - else None - ) - ), - ) start_time = ( execution_start_time @@ -841,20 +821,10 @@ class GPUWorker(GPUWorkerPostTrainingMixin): # or the estimator would plan from the remaining partial data if measure_server_warmup: assert warmup_workload is not None - if layerwise_usage_tracker is not None: - ( - layerwise_layer_uses, - layerwise_layer_uses_by_stage, - ) = layerwise_usage_tracker.finish_with_stages() - else: - layerwise_layer_uses = {} self._record_server_warmup_memory( - req=req, workload=warmup_workload, baseline_allocated_bytes=warmup_baseline_allocated_bytes, succeeded=output_batch is not None and output_batch.error is None, - layerwise_layer_uses=layerwise_layer_uses, - layerwise_layer_uses_by_stage=layerwise_layer_uses_by_stage, ) return output_batch @@ -970,46 +940,23 @@ class GPUWorker(GPUWorkerPostTrainingMixin): def _record_server_warmup_memory( self, *, - req: Req, - workload: tuple[int, int, int, int], + workload: tuple[int, int, int], baseline_allocated_bytes: int, succeeded: bool, - layerwise_layer_uses: dict[str, dict[str, tuple[int, ...]]] | None = None, - layerwise_layer_uses_by_stage: ( - dict[str, dict[str, dict[str, tuple[int, ...]]]] | None - ) = None, ) -> None: - phase_allocated_peaks: dict[str, int] = {} - phase_components: dict[str, tuple[str, ...]] = {} - phase_used_components: dict[str, tuple[str, ...]] = {} - phase_full_weight_transition_components: dict[str, tuple[str, ...]] = {} - untracked_active_components: tuple[str, ...] = () + # Warmup phases reset the device peak at every boundary, so the request + # peak is the largest per-phase peak or what ran after the last phase. residency_manager = peek_global_component_residency_manager() - if residency_manager is not None: - for phase_name, peak in residency_manager.take_warmup_phase_peaks().items(): - phase_allocated_peaks[phase_name] = peak.allocated_bytes - phase_components[phase_name] = peak.active_components - phase_used_components[phase_name] = peak.used_components - phase_full_weight_transition_components[phase_name] = ( - peak.full_weight_transition_components - ) - untracked_active_components = residency_manager.current_device_components() + phase_allocated_peaks = ( + residency_manager.take_warmup_phase_peaks() + if residency_manager is not None + else {} + ) request_allocated_peak = max( int(torch.get_device_module().max_memory_allocated()), max(phase_allocated_peaks.values(), default=0), ) - if request_allocated_peak > max(phase_allocated_peaks.values(), default=0): - # Work after the residency-managed stage timeline (for example, - # output materialization) must remain a placement constraint. - # A reserved-only increase is not a second live placement. Record - # it separately so post-placement validation can still require - # allocator headroom without charging cache to candidate deltas. - phase_allocated_peaks["request:untracked"] = request_allocated_peak - phase_components["request:untracked"] = untracked_active_components - phase_used_components["request:untracked"] = () - phase_full_weight_transition_components["request:untracked"] = () - metrics = req.metrics - width, height, num_frames, num_inference_steps = workload + width, height, num_frames = workload self._auto_residency_warmup_records.append( WarmupMemoryRecord( width=width, @@ -1018,34 +965,6 @@ class GPUWorker(GPUWorkerPostTrainingMixin): baseline_allocated_bytes=int(baseline_allocated_bytes), peak_allocated_bytes=request_allocated_peak, succeeded=succeeded, - peak_reserved_bytes=int( - torch.get_device_module().max_memory_reserved() - ), - phase_peak_allocated_bytes=phase_allocated_peaks, - phase_active_components=phase_components, - phase_used_components=phase_used_components, - phase_full_weight_transition_components=( - phase_full_weight_transition_components - ), - layerwise_layer_uses=layerwise_layer_uses or {}, - layerwise_layer_uses_by_stage=layerwise_layer_uses_by_stage or {}, - num_inference_steps=num_inference_steps, - total_duration_ms=( - float(metrics.total_duration_ms) if metrics is not None else 0.0 - ), - stage_duration_ms=(dict(metrics.stages) if metrics is not None else {}), - step_duration_ms=(tuple(metrics.steps) if metrics is not None else ()), - step_duration_ms_by_stage=( - { - stage_name: tuple(durations) - for stage_name, durations in metrics.steps_by_stage.items() - } - if metrics is not None - else {} - ), - stage_iterations=( - dict(metrics.stage_iterations) if metrics is not None else {} - ), ) ) diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/auto_residency.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/auto_residency.py index 22f3700bfa..73f6b66ce0 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/auto_residency.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/auto_residency.py @@ -1,22 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 -"""Warmup memory, timing and layer-use estimates for the default workload. +"""Warmup memory estimates for the default workload. Peak-memory extrapolation separates persistent weights from workload-scaled -activations. Phase and layer-use estimates account for stage-specific iteration -counts rather than scaling every stage by the request's denoising steps. +activations. """ from __future__ import annotations -import statistics -from typing import Iterable, Mapping +from typing import Iterable import msgspec -from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import ( - is_dit_component_name, -) - GIB_BYTES = 1024**3 # Activation memory rarely scales perfectly linearly with workload units; @@ -33,254 +27,11 @@ class WarmupMemoryRecord(msgspec.Struct, frozen=True): baseline_allocated_bytes: int peak_allocated_bytes: int succeeded: bool - peak_reserved_bytes: int = 0 - phase_peak_allocated_bytes: dict[str, int] = {} - phase_active_components: dict[str, tuple[str, ...]] = {} - phase_used_components: dict[str, tuple[str, ...]] = {} - phase_full_weight_transition_components: dict[str, tuple[str, ...]] = {} - layerwise_layer_uses: dict[str, dict[str, tuple[int, ...]]] = {} - layerwise_layer_uses_by_stage: dict[str, dict[str, dict[str, tuple[int, ...]]]] = {} - num_inference_steps: int = 1 - total_duration_ms: float = 0.0 - stage_duration_ms: dict[str, float] = {} - step_duration_ms: tuple[float, ...] = () - step_duration_ms_by_stage: dict[str, tuple[float, ...]] = {} - stage_iterations: dict[str, tuple[int, int]] = {} def workload_units(self) -> int: return max(1, self.width) * max(1, self.height) * max(1, self.num_frames) -def estimate_layerwise_layer_uses( - *, - records: Iterable[WarmupMemoryRecord], - target_units: int | None, - target_num_inference_steps: int, -) -> dict[str, dict[str, tuple[int, ...]]]: - """Estimate per-request layer calls from the same calibration forward. - - A full-shape memory probe deliberately runs only a few denoise steps. - Stage-attributed calls use that stage's measured and target iteration - counts, so independent shape, paint, refiner, and chunk loops are not all - multiplied by one request-wide ratio. Legacy records retain the repeated - DiT-layer heuristic. - """ - successful = [record for record in records if record.succeeded] - if target_units is not None: - covering = [ - record for record in successful if record.workload_units() >= target_units - ] - if covering: - successful = covering - - estimated: dict[str, dict[str, list[int]]] = {} - for record in successful: - source_steps = max(1, record.num_inference_steps) - component_stages = _component_stages(record) - repeated_stages = _repeated_stages(record, component_stages) - for component_name, groups in record.layerwise_layer_uses.items(): - component = estimated.setdefault(component_name, {}) - for layer_name, counts in groups.items(): - target = component.setdefault(layer_name, [0] * len(counts)) - if len(target) != len(counts): - continue - stage_counts = [ - ( - stage_name, - stage_components[component_name][layer_name], - ) - for stage_name, stage_components in ( - record.layerwise_layer_uses_by_stage.items() - ) - if component_name in stage_components - and layer_name in stage_components[component_name] - and len(stage_components[component_name][layer_name]) == len(counts) - ] - for layer_index, count in enumerate(counts): - if stage_counts: - measured_by_stage = sum( - per_layer_counts[layer_index] - for _, per_layer_counts in stage_counts - ) - scaled = max(0, count - measured_by_stage) - for stage_name, per_layer_counts in stage_counts: - measured_iterations, target_iterations = _stage_iterations( - record, - stage_name, - repeated_stages=repeated_stages, - target_num_inference_steps=(target_num_inference_steps), - ) - stage_count = per_layer_counts[layer_index] - if stage_count <= 1: - scaled += stage_count - else: - scaled += ( - stage_count * target_iterations - + measured_iterations - - 1 - ) // measured_iterations - else: - scaled = count - component_is_repeated = is_dit_component_name( - component_name - ) or any( - stage_name in repeated_stages - for stage_name in component_stages.get(component_name, ()) - ) - if ( - component_is_repeated - and count > 1 - and target_num_inference_steps > source_steps - ): - scaled = ( - count * target_num_inference_steps + source_steps - 1 - ) // source_steps - target[layer_index] = max(target[layer_index], scaled) - return { - component_name: { - layer_name: tuple(counts) for layer_name, counts in groups.items() - } - for component_name, groups in estimated.items() - } - - -def _component_stages( - record: WarmupMemoryRecord, - *, - timed_stage_names: set[str] | None = None, -) -> dict[str, set[str]]: - component_stages: dict[str, set[str]] = {} - phase_components = record.phase_used_components or record.phase_active_components - for phase_name, components in phase_components.items(): - fields = phase_name.split(":", 2) - if len(fields) < 2 or not fields[0].isdigit(): - continue - stage_name = fields[1] - if timed_stage_names is not None and stage_name not in timed_stage_names: - continue - for component_name in components: - component_stages.setdefault(component_name, set()).add(stage_name) - return component_stages - - -def _repeated_stages( - record: WarmupMemoryRecord, - component_stages: Mapping[str, set[str]], -) -> set[str]: - stages = { - stage_name - for component_name, stage_names in component_stages.items() - if is_dit_component_name(component_name) - for stage_name in stage_names - } - stages.update(record.stage_iterations) - stages.update( - stage_name - for stage_name in set(record.stage_duration_ms).union( - *(stage_names for stage_names in component_stages.values()) - ) - if stage_name.endswith("DenoisingStage") - and not stage_name.endswith("BeforeDenoisingStage") - ) - return stages - - -def _stage_iterations( - record: WarmupMemoryRecord, - stage_name: str, - *, - repeated_stages: set[str], - target_num_inference_steps: int, -) -> tuple[int, int]: - explicit = record.stage_iterations.get(stage_name) - if explicit is not None: - return max(1, explicit[0]), max(1, explicit[1]) - measured = max(1, record.num_inference_steps) - target = ( - max(1, target_num_inference_steps) - if stage_name in repeated_stages - else measured - ) - return measured, target - - -def estimate_default_workload_timing( - *, - records: Iterable[WarmupMemoryRecord], - target_units: int | None, - target_num_inference_steps: int, -) -> tuple[int, dict[str, int], dict[str, tuple[str, ...]]]: - """Estimate full-request and stage durations from the warmup workload. - - Repeated stages scale by their own measured/default iteration counts. The - full-shape probe intentionally executes only a few steps, so using its raw - total would make every one-shot encoder transfer look important relative - to a long video request. - """ - successful = [record for record in records if record.succeeded] - if not successful: - return 0, {}, {} - if target_units is not None: - at_target = [ - record for record in successful if record.workload_units() >= target_units - ] - if at_target: - successful = at_target - representative = max( - successful, - key=lambda record: ( - record.workload_units(), - record.total_duration_ms, - ), - ) - if representative.total_duration_ms <= 0 or not representative.stage_duration_ms: - return 0, {}, {} - - component_stages = _component_stages( - representative, - timed_stage_names=set(representative.stage_duration_ms), - ) - repeated_stages = _repeated_stages(representative, component_stages) - - stage_duration_ns: dict[str, int] = {} - for stage_name, duration_ms in representative.stage_duration_ms.items(): - measured_iterations, target_iterations = _stage_iterations( - representative, - stage_name, - repeated_stages=repeated_stages, - target_num_inference_steps=target_num_inference_steps, - ) - step_durations = representative.step_duration_ms_by_stage.get(stage_name, ()) - if not step_durations and len(repeated_stages) == 1: - step_durations = representative.step_duration_ms - if stage_name in repeated_stages and step_durations: - steady_steps = ( - step_durations[1:] if len(step_durations) > 1 else step_durations - ) - non_step_ms = max(0.0, duration_ms - sum(step_durations)) - target_duration_ms = non_step_ms + ( - statistics.median(steady_steps) * target_iterations - ) - else: - target_duration_ms = duration_ms * target_iterations / measured_iterations - stage_duration_ns[stage_name] = max(0, int(target_duration_ms * 1_000_000)) - - measured_stage_ms = sum(representative.stage_duration_ms.values()) - untracked_ms = max(0.0, representative.total_duration_ms - measured_stage_ms) - request_duration_ns = sum(stage_duration_ns.values()) + int( - untracked_ms * 1_000_000 - ) - - return ( - request_duration_ns, - stage_duration_ns, - { - component_name: tuple(sorted(stage_names)) - for component_name, stage_names in component_stages.items() - }, - ) - - def estimate_default_workload_peak_bytes( *, records: Iterable[WarmupMemoryRecord], @@ -368,103 +119,3 @@ def estimate_default_workload_peak_bytes( baseline + int(activation * ratio * ACTIVATION_EXTRAPOLATION_MARGIN) ) return max(estimates) - - -def estimate_workload_phase_peaks( - *, - records: Iterable[WarmupMemoryRecord], - target_units: int | None, - component_weight_bytes: Mapping[str, int], -) -> tuple[ - dict[str, int], - dict[str, tuple[str, ...]], - dict[str, tuple[str, ...]], - dict[str, tuple[str, ...]], -]: - """Estimate each measured execution phase at the target workload. - - A component already active in a phase is part of that phase's measured - peak. Keeping a component resident therefore adds no weight bytes to its - own component-offload phase, while it adds the full footprint to phases - where the component was absent. Measurements with different active layouts - remain separate constraints; combining one layout's peak with another - layout's component set would describe a state that never occurred. - """ - successful = [record for record in records if record.succeeded] - if target_units is not None: - covering = [ - record for record in successful if record.workload_units() >= target_units - ] - if covering: - successful = covering - - grouped: dict[ - tuple[str, tuple[str, ...], tuple[str, ...], tuple[str, ...]], - list[WarmupMemoryRecord], - ] = {} - for record in successful: - used_by_phase = record.phase_used_components or record.phase_active_components - for phase_name in record.phase_peak_allocated_bytes: - active = tuple(sorted(record.phase_active_components.get(phase_name, ()))) - used = tuple(sorted(used_by_phase.get(phase_name, ()))) - full_weight_transitions = tuple( - sorted( - record.phase_full_weight_transition_components.get(phase_name, ()) - ) - ) - grouped.setdefault( - (phase_name, active, used, full_weight_transitions), [] - ).append(record) - - layouts_per_phase: dict[str, int] = {} - for phase_name, _, _, _ in grouped: - layouts_per_phase[phase_name] = layouts_per_phase.get(phase_name, 0) + 1 - - estimated_peaks: dict[str, int] = {} - active_components: dict[str, tuple[str, ...]] = {} - used_components: dict[str, tuple[str, ...]] = {} - full_weight_transition_components: dict[str, tuple[str, ...]] = {} - layout_indices: dict[str, int] = {} - for ( - phase_name, - active, - used, - full_weight_transitions, - ), phase_records in sorted(grouped.items()): - output_name = phase_name - if layouts_per_phase[phase_name] > 1: - index = layout_indices.get(phase_name, 0) - layout_indices[phase_name] = index + 1 - output_name = f"{phase_name}:layout:{index}" - weight_floor = sum(component_weight_bytes.get(name, 0) for name in active) - phase_measurements = [ - WarmupMemoryRecord( - width=record.width, - height=record.height, - num_frames=record.num_frames, - baseline_allocated_bytes=min( - record.baseline_allocated_bytes, - record.phase_peak_allocated_bytes[phase_name], - ), - peak_allocated_bytes=record.phase_peak_allocated_bytes[phase_name], - succeeded=True, - ) - for record in phase_records - ] - estimate = estimate_default_workload_peak_bytes( - records=phase_measurements, - target_units=target_units, - constant_weight_bytes=weight_floor, - ) - if estimate is None: - continue - estimated_peaks[output_name] = estimate - active_components[output_name] = active - used_components[output_name] = used - full_weight_transition_components[output_name] = full_weight_transitions - return ( - estimated_peaks, - active_components, - used_components, - full_weight_transition_components, - ) diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py index 1bbc18bfa4..314fbdc656 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py @@ -1,4 +1,4 @@ -from collections.abc import Iterable, Iterator +from collections.abc import Iterator from contextlib import contextmanager from dataclasses import dataclass from typing import Mapping, MutableMapping, Protocol, Sequence @@ -78,14 +78,6 @@ class ResidencyState: batch_is_warmup: bool = False -@dataclass(frozen=True, slots=True) -class WarmupPhasePeak: - active_components: tuple[str, ...] - allocated_bytes: int - used_components: tuple[str, ...] = () - full_weight_transition_components: tuple[str, ...] = () - - class ResidencyBatch(Protocol): is_warmup: bool @@ -167,11 +159,9 @@ class ComponentResidencyManager: self._modules_seen: dict[str, nn.Module] = {} self._track_warmup_memory = False self._warmup_phase_key: str | None = None - self._warmup_phase_components: tuple[str, ...] = () - self._warmup_phase_used_components: tuple[str, ...] = () - self._warmup_phase_full_weight_transition_components: tuple[str, ...] = () - self._warmup_phase_peaks: dict[str, WarmupPhasePeak] = {} - self._completed_warmup_phase_peaks: dict[str, WarmupPhasePeak] = {} + # phase key -> peak allocated bytes + self._warmup_phase_peaks: dict[str, int] = {} + self._completed_warmup_phase_peaks: dict[str, int] = {} @property def host_pin_budget(self) -> HostPinBudget: @@ -236,9 +226,6 @@ class ComponentResidencyManager: and torch.get_device_module().is_available() ) self._warmup_phase_key = None - self._warmup_phase_components = () - self._warmup_phase_used_components = () - self._warmup_phase_full_weight_transition_components = () self._warmup_phase_peaks = {} self._completed_warmup_phase_peaks = {} if self._track_warmup_memory: @@ -246,7 +233,6 @@ class ComponentResidencyManager: # Start the first interval without another reset so preprocessing # before stage 0 remains part of the placement constraints. self._warmup_phase_key = "request:before-stage" - self._warmup_phase_components = self._warmup_active_components() self._validate_explicit_nonresident_components() def _validate_explicit_nonresident_components(self) -> None: @@ -302,48 +288,7 @@ class ComponentResidencyManager: self.state.stage_name = self.stage_name(stage) self.state.next_stage_name = self._next_stage_name(stage_index) if self._track_warmup_memory: - self._begin_warmup_phase( - key=f"{stage_index}:{self.state.stage_name}:setup", - components=self._warmup_active_components(), - used_components=(), - ) - - @contextmanager - def full_weight_transition(self, component_names: Iterable[str]) -> Iterator[None]: - """Measure request logic that temporarily materializes complete weights.""" - names = tuple(sorted(set(component_names))) - if not self._track_warmup_memory or not names: - yield - return - previous_phase = ( - self._warmup_phase_key, - self._warmup_phase_components, - self._warmup_phase_used_components, - self._warmup_phase_full_weight_transition_components, - ) - self._begin_warmup_phase( - key=( - f"{self.state.stage_index}:{self.state.stage_name}:" - f"full-weight-transition:{','.join(names)}" - ), - components=self._warmup_active_components(), - used_components=(), - full_weight_transition_components=names, - ) - try: - yield - finally: - previous_key, components, used_components, transitions = previous_phase - if previous_key is None: - self._record_warmup_phase_peak() - self._warmup_phase_key = None - else: - self._begin_warmup_phase( - key=previous_key, - components=components, - used_components=used_components, - full_weight_transition_components=transitions, - ) + self._begin_warmup_phase(f"{stage_index}:{self.state.stage_name}:setup") def begin_stage(self) -> None: """Prepare a stage that declares one uninterrupted component use.""" @@ -700,14 +645,7 @@ class ComponentResidencyManager: ) strategy = self.strategy_for(component_name, module) if self._track_warmup_memory: - will_prepare = self.state.batch_is_warmup and preferred - self._begin_warmup_phase( - key=f"request:cleanup:{component_name}", - components=self._warmup_active_components( - (use,) if will_prepare else () - ), - used_components=(component_name,) if will_prepare else (), - ) + self._begin_warmup_phase(f"request:cleanup:{component_name}") was_on_supported_device = self._module_on_supported_device(module) strategy.finish_request(module, use, self.state, preferred=preferred) self._empty_cache_after_large_release( @@ -715,10 +653,8 @@ class ComponentResidencyManager: ) if self._track_warmup_memory: self._record_warmup_phase_peak() - self._warmup_phase_peaks["idle"] = WarmupPhasePeak( - active_components=self._warmup_active_components(), - allocated_bytes=int(torch.get_device_module().memory_allocated()), - used_components=(), + self._warmup_phase_peaks["idle"] = int( + torch.get_device_module().memory_allocated() ) self._completed_warmup_phase_peaks = dict(self._warmup_phase_peaks) self._track_warmup_memory = False @@ -755,23 +691,11 @@ class ComponentResidencyManager: use.component_name, ) - def _begin_warmup_phase( - self, - *, - key: str, - components: tuple[str, ...], - used_components: tuple[str, ...], - full_weight_transition_components: tuple[str, ...] = (), - ) -> None: + def _begin_warmup_phase(self, key: str) -> None: if not self._track_warmup_memory: return self._record_warmup_phase_peak() self._warmup_phase_key = key - self._warmup_phase_components = tuple(sorted(set(components))) - self._warmup_phase_used_components = tuple(sorted(set(used_components))) - self._warmup_phase_full_weight_transition_components = tuple( - sorted(set(full_weight_transition_components)) - ) torch.get_device_module().reset_peak_memory_stats() def _begin_warmup_transition( @@ -782,104 +706,42 @@ class ComponentResidencyManager: previous_name = previous.component_name if previous is not None else "idle" upcoming_name = upcoming.component_name if upcoming is not None else "idle" self._begin_warmup_phase( - key=( - f"{self.state.stage_index}:{self.state.stage_name}:transition:" - f"{previous_name}->{upcoming_name}" - ), - components=self._warmup_active_components( - tuple(use for use in (previous, upcoming) if use is not None) - ), - used_components=tuple( - use.component_name for use in (previous, upcoming) if use is not None - ), + f"{self.state.stage_index}:{self.state.stage_name}:transition:" + f"{previous_name}->{upcoming_name}" ) def _begin_warmup_use(self, use: ComponentUse) -> None: phase = use.phase or use.component_name self._begin_warmup_phase( - key=f"{self.state.stage_index}:{self.state.stage_name}:use:{phase}", - components=self._warmup_active_components((use,)), - used_components=(use.component_name,), + f"{self.state.stage_index}:{self.state.stage_name}:use:{phase}" ) def _begin_warmup_between_uses(self) -> None: self._begin_warmup_phase( - key=f"{self.state.stage_index}:{self.state.stage_name}:between", - components=self._warmup_active_components(), - used_components=(), + f"{self.state.stage_index}:{self.state.stage_name}:between" ) def _begin_warmup_prefetch(self, use: ComponentUse) -> None: phase = use.phase or use.component_name self._begin_warmup_phase( - key=f"{self.state.stage_index}:{self.state.stage_name}:prefetch:{phase}", - components=self._warmup_active_components((use,)), - used_components=(use.component_name,), + f"{self.state.stage_index}:{self.state.stage_name}:prefetch:{phase}" ) - def _warmup_active_components( - self, active_uses: Sequence[ComponentUse] = () - ) -> tuple[str, ...]: - components = {use.component_name for use in active_uses} - for component_name, module in self.pipeline.modules.items(): - if not isinstance(module, nn.Module): - continue - if is_layerwise_offloaded_module(module): - continue - if self._module_on_supported_device(module): - components.add(component_name) - return tuple(sorted(components)) - def _record_warmup_phase_peak(self) -> None: if not self._track_warmup_memory or self._warmup_phase_key is None: return - peak = WarmupPhasePeak( - active_components=self._warmup_phase_components, - allocated_bytes=int(torch.get_device_module().max_memory_allocated()), - used_components=self._warmup_phase_used_components, - full_weight_transition_components=( - self._warmup_phase_full_weight_transition_components - ), - ) + peak = int(torch.get_device_module().max_memory_allocated()) previous = self._warmup_phase_peaks.get(self._warmup_phase_key) - if previous is None: - self._warmup_phase_peaks[self._warmup_phase_key] = peak - else: - self._warmup_phase_peaks[self._warmup_phase_key] = WarmupPhasePeak( - active_components=tuple( - sorted( - set(previous.active_components) & set(peak.active_components) - ) - ), - allocated_bytes=max(previous.allocated_bytes, peak.allocated_bytes), - used_components=tuple( - sorted(set(previous.used_components) & set(peak.used_components)) - ), - full_weight_transition_components=tuple( - sorted( - set(previous.full_weight_transition_components) - & set(peak.full_weight_transition_components) - ) - ), - ) + self._warmup_phase_peaks[self._warmup_phase_key] = ( + peak if previous is None else max(previous, peak) + ) - def take_warmup_phase_peaks( - self, - ) -> dict[str, WarmupPhasePeak]: + def take_warmup_phase_peaks(self) -> dict[str, int]: """Return and clear the most recently completed warmup phase peaks.""" peaks = self._completed_warmup_phase_peaks self._completed_warmup_phase_peaks = {} return peaks - def current_device_components(self) -> tuple[str, ...]: - """Components whose complete module is currently on the device. - - Dormant layerwise-managed modules are excluded because only their - resident window is present. Active layerwise uses are attributed by - the managed phase timeline instead. - """ - return self._warmup_active_components() - def stage_name(self, stage: ComponentResidencyStage) -> str: return self._stage_names_by_id.get(id(stage), stage.__class__.__name__) diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py index 5f37175c5e..a305c1f055 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py @@ -3699,99 +3699,3 @@ def configure_layerwise_offload_modules( elif warn_missing: logger.debug("No selected pipeline component enabled layerwise offload") return configured_component_names - - -class LayerwiseUsageTracker: - """Temporary per-layer call counters for one calibration request. - - Managers cannot provide this information when layerwise offload has not - been configured yet or was disabled by a resident placement. Observe the - declared layer groups directly and remove every hook before returning, so - ordinary serving requests pay no counter or hook-dispatch cost. - """ - - def __init__( - self, - modules: Mapping[str, object], - *, - stage_name_provider: Callable[[], str | None] | None = None, - ) -> None: - self._handles: list[Any] = [] - self._counts: dict[str, dict[str, list[int]]] = {} - self._counts_by_stage: dict[str, dict[str, dict[str, list[int]]]] = {} - self._stage_name_provider = stage_name_provider - for component_name, module in modules.items(): - if not isinstance(module, LayerwiseOffloadableModuleMixin): - continue - named_modules = dict(module.named_modules()) - component_counts: dict[str, list[int]] = {} - for layer_name in module.layer_names: - layers = named_modules.get(layer_name) - if not isinstance(layers, (torch.nn.ModuleList, torch.nn.Sequential)): - continue - counts = [0] * len(layers) - if not counts: - continue - component_counts[layer_name] = counts - for layer_index, layer in enumerate(layers): - - def record_use( - _module, - _inputs, - *, - target_counts=counts, - target_index=layer_index, - target_component_name=component_name, - target_layer_name=layer_name, - target_layer_count=len(layers), - ) -> None: - target_counts[target_index] += 1 - if self._stage_name_provider is None: - return - stage_name = self._stage_name_provider() - if stage_name is None: - return - stage_counts = self._counts_by_stage.setdefault(stage_name, {}) - component_stage_counts = stage_counts.setdefault( - target_component_name, {} - ) - layer_stage_counts = component_stage_counts.setdefault( - target_layer_name, [0] * target_layer_count - ) - layer_stage_counts[target_index] += 1 - - self._handles.append(layer.register_forward_pre_hook(record_use)) - if component_counts: - self._counts[component_name] = component_counts - - def finish(self) -> dict[str, dict[str, tuple[int, ...]]]: - counts, _ = self.finish_with_stages() - return counts - - def finish_with_stages( - self, - ) -> tuple[ - dict[str, dict[str, tuple[int, ...]]], - dict[str, dict[str, dict[str, tuple[int, ...]]]], - ]: - for handle in self._handles: - handle.remove() - self._handles.clear() - counts = { - component_name: { - layer_name: tuple(counts) - for layer_name, counts in component_counts.items() - } - for component_name, component_counts in self._counts.items() - } - counts_by_stage = { - stage_name: { - component_name: { - layer_name: tuple(counts) - for layer_name, counts in component_counts.items() - } - for component_name, component_counts in stage_counts.items() - } - for stage_name, stage_counts in self._counts_by_stage.items() - } - return counts, counts_by_stage diff --git a/python/sglang/multimodal_gen/runtime/models/decoders/ltx_2_5_diffusion_decoder.py b/python/sglang/multimodal_gen/runtime/models/decoders/ltx_2_5_diffusion_decoder.py index 443d35c8bf..e71111c930 100644 --- a/python/sglang/multimodal_gen/runtime/models/decoders/ltx_2_5_diffusion_decoder.py +++ b/python/sglang/multimodal_gen/runtime/models/decoders/ltx_2_5_diffusion_decoder.py @@ -325,17 +325,10 @@ class LTX2VideoVaeRotaryPosEmbed3D(nn.Module): ) -> tuple[torch.Tensor, torch.Tensor]: """Rotate Q/K together, with a bit-exact eager fallback.""" tables = self._tables(query) - verified = _LTX25_DECODER_ROPE.verified - if ( - not _LTX25_DECODER_ROPE.disabled - and _is_cuda - and query.is_cuda - and query.dtype is torch.bfloat16 - and (verified or _LTX25_DECODER_ROPE.can_attempt_once()) - ): + if _is_cuda and query.is_cuda and query.dtype is torch.bfloat16: dim_t, dim_h, _ = self.rope_dim_split - try: - fused = fused_ltx25_decoder_rope( + return _LTX25_DECODER_ROPE.run( + lambda: fused_ltx25_decoder_rope( query, key, *tables[0], @@ -343,23 +336,14 @@ class LTX2VideoVaeRotaryPosEmbed3D(nn.Module): *tables[2], dim_t, dim_h, - ) - except Exception as exc: - _LTX25_DECODER_ROPE.on_exception(exc, logger=logger) - else: - if verified: - return fused - eager = self._apply_rope(query, tables), self._apply_rope(key, tables) - return _LTX25_DECODER_ROPE.accept_or_fallback( - fused, - eager, - equal=tensors_equal, - logger=logger, - mismatch_msg=( - "LTX-2.5 decoder fused RoPE is not bit-exact on this " - "platform; falling back to eager" - ), - ) + ), + lambda: ( + self._apply_rope(query, tables), + self._apply_rope(key, tables), + ), + equal=tensors_equal, + logger=logger, + ) return self._apply_rope(query, tables), self._apply_rope(key, tables) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/anima.py b/python/sglang/multimodal_gen/runtime/models/dits/anima.py index a58857c4b9..cc2e947651 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/anima.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/anima.py @@ -64,23 +64,13 @@ def _anima_rope(q, k, cos, sin): ) if can_use_fused_rope_rotate_half_fp32(q, k, cos, sin): - sig = (q.device, q.dtype, q.shape) - verified = _ANIMA_ROPE.is_verified(sig) - if verified or not torch.cuda.is_current_stream_capturing(): - try: - out = fused_rope_rotate_half_fp32(q, k, cos, sin) - except Exception as exc: - _ANIMA_ROPE.on_exception(exc, logger=logger) - else: - if verified: - return out - return _ANIMA_ROPE.accept_or_fallback( - out, - _anima_rope_eager(q, k, cos, sin), - sig=sig, - equal=tensors_equal, - logger=logger, - ) + return _ANIMA_ROPE.run( + lambda: fused_rope_rotate_half_fp32(q, k, cos, sin), + lambda: _anima_rope_eager(q, k, cos, sin), + sig=(q.device, q.dtype, q.shape), + equal=tensors_equal, + logger=logger, + ) return _anima_rope_eager(q, k, cos, sin) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py b/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py index 8e58f39c3d..50590421ec 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/causal_wanvideo.py @@ -294,8 +294,7 @@ class CausalWanTransformerBlock(nn.Module): self.norm_q = RMSNorm(dim, eps=eps) self.norm_k = RMSNorm(dim, eps=eps) else: - print("QK Norm type not supported") - raise Exception + raise ValueError(f"QK Norm type not supported: {qk_norm}") self.tp_rmsnorm = ( use_megatron_tp and qk_norm == "rms_norm_across_heads" and tp_size > 1 ) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/common.py b/python/sglang/multimodal_gen/runtime/models/dits/common.py index 7629acad0b..9cb6d87e0c 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/common.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/common.py @@ -2,15 +2,21 @@ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Callable, Sequence from typing import Any import torch +from torch import nn +from sglang.kernels.ops import diffusion as diffusion_ops +from sglang.kernels.ops.diffusion import BitExactFusionGate from sglang.multimodal_gen.runtime.layers.quantization.convrot_int8_jit import ( apply_convrot_int8_shared_input, convrot_int8_shares_input, ) +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) def modulate( @@ -28,6 +34,39 @@ def modulate( return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) +def layernorm_modulate_bitexact( + gate: BitExactFusionGate, + norm: nn.Module, + x: torch.Tensor, + scale: torch.Tensor, + shift: torch.Tensor, + reference: Callable[[], torch.Tensor], + *, + sig: Any = None, +) -> torch.Tensor: + """Single-kernel ``LN(x) * (1 + scale) + shift`` for an affine-free LayerNorm. + + ``scale`` / ``shift`` are the ``(B, D)`` modulation rows and ``reference`` + is the model's eager chain. The kernel replicates the BF16 aten LayerNorm + reduction, so ``gate`` verifies it against ``reference`` per its mode. + """ + if not ( + x.is_cuda + and diffusion_ops.is_plain_layer_norm(norm, x.shape[-1]) + and diffusion_ops.can_use_fused_layernorm_modulate(x.dtype, x.shape[-1]) + ): + return reference() + # The raw launch skips the custom-op dispatch, which compile tracing needs. + kernel = ( + diffusion_ops.fused_layernorm_modulate + if torch.compiler.is_compiling() + else diffusion_ops.fused_layernorm_modulate_raw + ) + return gate.run( + lambda: kernel(x, scale, shift, norm.eps), reference, sig=sig, logger=logger + ) + + def _project_shared_input( x: torch.Tensor, layers: Sequence[torch.nn.Module] ) -> list[torch.Tensor]: diff --git a/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py b/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py index 5015ecb190..bd224a40ae 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/cosmos3video.py @@ -289,6 +289,8 @@ def _apply_qwen3_qk_norm_rope( ) +# Also writes K/V into one packed buffer in the same launch, which +# apply_qk_norm_rope cannot. def _apply_qwen3_qk_norm_rope_pack_kv( q: torch.Tensor, k: torch.Tensor, diff --git a/python/sglang/multimodal_gen/runtime/models/dits/ernie_image.py b/python/sglang/multimodal_gen/runtime/models/dits/ernie_image.py index 856bed68d8..db92bb3a0c 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/ernie_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/ernie_image.py @@ -87,34 +87,19 @@ def _ernie_norm_scale_shift( first call verifies ``torch.equal`` against the eager chain and disables the fast path permanently on any mismatch. """ - verified = _ERNIE_NORM.verified if ( - not _ERNIE_NORM.disabled - and x.is_cuda + x.is_cuda and norm.variance_size_override is None and can_use_fused_rmsnorm_scale_shift(x.dtype, x.shape[-1]) - and (verified or _ERNIE_NORM.can_attempt_once()) ): - try: - out = fused_rmsnorm_scale_shift_bitexact( + return _ERNIE_NORM.run( + lambda: fused_rmsnorm_scale_shift_bitexact( x, norm.weight, scale, shift, norm.variance_epsilon - ) - except Exception as exc: - _ERNIE_NORM.on_exception(exc, logger=logger) - else: - if verified: - return out - return _ERNIE_NORM.accept_or_fallback( - out, - _eager_norm_scale_shift(norm, x, scale, shift), - logger=logger, - mismatch_msg=( - "ERNIE fused-norm fast path is not bit-exact against this " - "platform's rmsnorm dispatch; falling back to eager" - ), - diagnostic_hint=flashinfer_rmsnorm_diagnostic_hint, - ) - + ), + lambda: _eager_norm_scale_shift(norm, x, scale, shift), + logger=logger, + diagnostic_hint=flashinfer_rmsnorm_diagnostic_hint, + ) return _eager_norm_scale_shift(norm, x, scale, shift) @@ -162,10 +147,6 @@ def _ernie_gated_norm_scale_shift( (ref, res_ref), equal=tensors_equal, logger=logger, - mismatch_msg=( - "ERNIE fused gated-norm fast path is not bit-exact against " - "this platform's rmsnorm dispatch; falling back to eager" - ), diagnostic_hint=flashinfer_rmsnorm_diagnostic_hint, ) @@ -456,29 +437,12 @@ def _ernie_rope( rounding boundary exactly; the first call still verifies ``torch.equal`` against the eager chain and disables the fast path on any mismatch. """ - verified = _ERNIE_ROPE.verified - if ( - not _ERNIE_ROPE.disabled - and x.is_cuda - and x.dtype is torch.bfloat16 - and (verified or _ERNIE_ROPE.can_attempt_once()) - ): - try: - out = fused_rope_rotate_half_bitexact(x, cos_, sin_) - except Exception as exc: - _ERNIE_ROPE.on_exception(exc, logger=logger) - else: - if verified: - return out - return _ERNIE_ROPE.accept_or_fallback( - out, - _apply_rotary_bshd_eager(x, cos_, sin_), - logger=logger, - mismatch_msg=( - "ERNIE fused RoPE fast path is not bit-exact on this " - "platform; falling back to eager" - ), - ) + if x.is_cuda and x.dtype is torch.bfloat16: + return _ERNIE_ROPE.run( + lambda: fused_rope_rotate_half_bitexact(x, cos_, sin_), + lambda: _apply_rotary_bshd_eager(x, cos_, sin_), + logger=logger, + ) return _apply_rotary_bshd_eager(x, cos_, sin_) @@ -554,10 +518,6 @@ def _ernie_qknorm_rope( ref, equal=tensors_equal, logger=logger, - mismatch_msg=( - "ERNIE fused QKNorm+RoPE fast path is not bit-exact on " - "this platform; falling back to split kernels" - ), ) return _ernie_qknorm_rope_reference( @@ -577,31 +537,17 @@ def _ernie_geglu(gate_up: torch.Tensor) -> torch.Tensor: to bf16 before the multiply exactly like the eager two-step; first call self-verifies like :func:`_ernie_rope`. """ - verified = _ERNIE_GEGLU.verified if ( - not _ERNIE_GEGLU.disabled - and gate_up.dtype in (torch.bfloat16, torch.float16) + gate_up.dtype in (torch.bfloat16, torch.float16) and gate_up.is_cuda and gate_up.is_contiguous() and gate_up.shape[-1] % 2 == 0 - and (verified or _ERNIE_GEGLU.can_attempt_once()) ): - try: - out = gelu_and_mul_with_activation_rounding(gate_up) - except Exception as exc: - _ERNIE_GEGLU.on_exception(exc, logger=logger) - else: - if verified: - return out - return _ERNIE_GEGLU.accept_or_fallback( - out, - _eager_geglu(gate_up), - logger=logger, - mismatch_msg=( - "ERNIE fused GELU-mul fast path is not bit-exact on this " - "platform; falling back to eager" - ), - ) + return _ERNIE_GEGLU.run( + lambda: gelu_and_mul_with_activation_rounding(gate_up), + lambda: _eager_geglu(gate_up), + logger=logger, + ) return _eager_geglu(gate_up) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux.py b/python/sglang/multimodal_gen/runtime/models/dits/flux.py index 10c3645db2..d734cb5236 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux.py @@ -29,18 +29,16 @@ from diffusers.models.normalization import ( from torch.nn import LayerNorm as LayerNorm from sglang.kernels.ops.diffusion import ( + FUSED_LINEAR_GELU, + FUSED_LN_MODULATE, BitExactFusionGate, can_use_fused_layernorm_modulate, can_use_linear_gelu, can_use_ln_modulate, - fused_gelu_active, fused_layernorm_modulate, fused_linear_gelu_tanh, fused_ln_modulate, - fused_ln_modulate_active, is_plain_layer_norm, - mark_fused_gelu_site, - mark_fused_ln_modulate_site, modulate_scale_shift, residual_gate_add, ) @@ -155,10 +153,6 @@ def _flux_fused_ln_modulate( ref, sig=sig, logger=logger, - mismatch_msg=( - "FLUX fused LN+modulate fast path is not bit-exact against this " - "platform's LayerNorm dispatch; falling back to eager" - ), ) @@ -182,7 +176,7 @@ def _flux_norm_modulate( out = _flux_fused_ln_modulate(norm, x, scale, shift) if out is not None: return out - if fused_ln_modulate_active(site) and can_use_ln_modulate(x, scale, shift): + if FUSED_LN_MODULATE.is_enabled(site) and can_use_ln_modulate(x, scale, shift): return fused_ln_modulate(x, scale, shift, norm.eps) return modulate_scale_shift(norm(x), scale, shift) @@ -193,7 +187,7 @@ class FluxAdaLayerNormZero(AdaLayerNormZero): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - mark_fused_ln_modulate_site(self) + FUSED_LN_MODULATE.mark(self) def forward( self, @@ -219,7 +213,7 @@ class FluxAdaLayerNormZeroSingle(AdaLayerNormZeroSingle): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - mark_fused_ln_modulate_site(self) + FUSED_LN_MODULATE.mark(self) def forward( self, @@ -410,10 +404,12 @@ class FluxGELU(nn.Module): self.gelu = nn.GELU(approximate="tanh") # lossless/high fusion site: up-proj GEMM + tanh-GELU in the cublasLt # epilogue. Off by default; mounted per batch by the denoising stage. - mark_fused_gelu_site(self, "proj") + FUSED_LINEAR_GELU.mark(self, "proj") def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - if fused_gelu_active(self) and can_use_linear_gelu(self.proj, hidden_states): + if FUSED_LINEAR_GELU.is_enabled(self) and can_use_linear_gelu( + self.proj, hidden_states + ): return fused_linear_gelu_tanh( hidden_states, self.proj.weight, self.proj.bias ) @@ -434,10 +430,12 @@ class FluxFusedGELUProj(nn.Module): def __init__(self, proj: nn.Linear): super().__init__() self.proj = proj - mark_fused_gelu_site(self, "proj") + FUSED_LINEAR_GELU.mark(self, "proj") def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - if fused_gelu_active(self) and can_use_linear_gelu(self.proj, hidden_states): + if FUSED_LINEAR_GELU.is_enabled(self) and can_use_linear_gelu( + self.proj, hidden_states + ): return fused_linear_gelu_tanh( hidden_states, self.proj.weight, self.proj.bias ) @@ -877,7 +875,7 @@ class FluxSingleTransformerBlock(nn.Module): self.act_mlp = nn.GELU(approximate="tanh") # lossless/high fusion site: proj_mlp GEMM + tanh-GELU in the # cublasLt epilogue (mounted per batch by the denoising stage). - mark_fused_gelu_site(self, "proj_mlp") + FUSED_LINEAR_GELU.mark(self, "proj_mlp") proj_out_cls = ( RowParallelLinear if shard_single_block else ColumnParallelLinear ) @@ -979,7 +977,7 @@ class FluxSingleTransformerBlock(nn.Module): hidden_states = gate * hidden_states hidden_states = residual + hidden_states else: - if fused_gelu_active(self) and can_use_linear_gelu( + if FUSED_LINEAR_GELU.is_enabled(self) and can_use_linear_gelu( self.proj_mlp, norm_hidden_states ): mlp_hidden_states = fused_linear_gelu_tanh( @@ -1044,7 +1042,7 @@ class FluxTransformerBlock(nn.Module): self.norm2_context = LayerNorm(dim, eps=1e-6, elementwise_affine=False) # lossless/high site: norm2/norm2_context modulate folds into the # LN affine when mounted. - mark_fused_ln_modulate_site(self) + FUSED_LN_MODULATE.mark(self) nunchaku_enabled = ( quant_config is not None diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux3.py b/python/sglang/multimodal_gen/runtime/models/dits/flux3.py index bc9645f60f..3d5af0a02b 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux3.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux3.py @@ -36,7 +36,6 @@ request. from __future__ import annotations import math -import os from typing import Any import msgspec @@ -48,15 +47,13 @@ from sglang.kernels.ops.diffusion import ( BitExactFusionGate, can_use_fp8_rowwise, can_use_fused_inplace_qknorm_rope, - can_use_fused_layernorm_modulate, fp8_rowwise, fused_inplace_qknorm_rope, - fused_layernorm_modulate_raw, fused_packed_silu_mul_bitexact, - is_plain_layer_norm, residual_gate_add, ) from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz +from sglang.multimodal_gen import envs from sglang.multimodal_gen.configs.models.dits.flux3 import ( Flux3ArchConfig, Flux3DiTConfig, @@ -88,6 +85,9 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im LayerwiseOffloadableModuleMixin, ) from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT +from sglang.multimodal_gen.runtime.models.dits.common import ( + layernorm_modulate_bitexact, +) from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger @@ -139,71 +139,46 @@ def _fast_path_allowed(x: torch.Tensor) -> bool: return x.is_cuda and x.dtype is torch.bfloat16 and not torch.compiler.is_compiling() -def _can_verify() -> bool: - # First-sight verification runs the eager chain and a host sync. - return not torch.cuda.is_current_stream_capturing() - - def _norm_modulate( norm: nn.LayerNorm, x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor ) -> torch.Tensor: """``(1 + scale) * LN(x) + shift`` with ``(B, 1, D)`` modulation.""" - scale_row, shift_row = scale[:, 0], shift[:, 0] - if ( - _LN_MODULATE.disabled - or not _fast_path_allowed(x) - or not is_plain_layer_norm(norm, x.shape[-1]) - or not can_use_fused_layernorm_modulate(x.dtype, x.shape[-1]) - ): + + def reference(): return (1 + scale) * norm(x) + shift - sig = (x.device, x.shape[0], x.shape[-1], norm.eps) - if not _LN_MODULATE.is_verified(sig) and not _can_verify(): - return (1 + scale) * norm(x) + shift - try: - out = fused_layernorm_modulate_raw(x, scale_row, shift_row, norm.eps) - except Exception as exc: - _LN_MODULATE.on_exception(exc, logger=logger) - return (1 + scale) * norm(x) + shift - if _LN_MODULATE.is_verified(sig): - return out - return _LN_MODULATE.accept_or_fallback( - out, - (1 + scale) * norm(x) + shift, - sig=sig, - logger=logger, - mismatch_msg="FLUX 3 fused LN+modulate is not bit-exact here; using eager", + + if not _fast_path_allowed(x): + return reference() + return layernorm_modulate_bitexact( + _LN_MODULATE, + norm, + x, + scale[:, 0], + shift[:, 0], + reference, + sig=(x.device, x.shape[0], x.shape[-1], norm.eps), ) def _swiglu(packed: torch.Tensor) -> torch.Tensor: """``silu(gate) * value`` of a packed ``[gate | value]`` projection.""" gate, value = packed.chunk(2, dim=-1) - if _SWIGLU.disabled or not _fast_path_allowed(packed): + if not _fast_path_allowed(packed): return F.silu(gate) * value - sig = (packed.device, packed.shape[-1], packed.stride(-2)) - if not _SWIGLU.is_verified(sig) and not _can_verify(): - return F.silu(gate) * value - try: - out = fused_packed_silu_mul_bitexact(packed) - except Exception as exc: - _SWIGLU.on_exception(exc, logger=logger) - return F.silu(gate) * value - if _SWIGLU.is_verified(sig): - return out - return _SWIGLU.accept_or_fallback( - out, - F.silu(gate) * value, - sig=sig, + return _SWIGLU.run( + lambda: fused_packed_silu_mul_bitexact(packed), + lambda: F.silu(gate) * value, + sig=(packed.device, packed.shape[-1], packed.stride(-2)), logger=logger, - mismatch_msg="FLUX 3 fused SwiGLU is not bit-exact here; using eager", ) +# Not apply_qk_norm_rope: Flux3RMSNorm has neither RMSNorm's weight nor its eps, +# and the eager fallback is Flux3's own fp32 pair rotation. def _fused_qknorm_rope_enabled(q: torch.Tensor, head_dim: int) -> bool: return ( _fast_path_allowed(q) - and os.getenv("SGLANG_ENABLE_FUSED_QKNORM_ROPE", "1").lower() - not in ("0", "false", "off", "no") + and envs.SGLANG_ENABLE_FUSED_QKNORM_ROPE and can_use_fused_inplace_qknorm_rope( head_dim=head_dim, rope_dim=head_dim, diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux3_image.py b/python/sglang/multimodal_gen/runtime/models/dits/flux3_image.py index 228e7a3ee1..7f28eeb5ed 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux3_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux3_image.py @@ -211,21 +211,6 @@ def pack( ) -def pack_cfg( - text: list[tuple[Tensor, Tensor]], - condition: tuple[Tensor, Tensor] | None, - image: tuple[Tensor, Tensor], - channels: dict[str, int], -) -> Packed: - return pack( - [ - {TEXT: text_tokens, "image_cond": condition, "image": image} - for text_tokens in text - ], - channels, - ) - - def flux3_dit_layer_names(module: nn.Module) -> list[str]: """Layerwise ModuleList paths. Expand each ModuleDict group per key.""" names = ["single_blocks", "txt_mode_blocks"] diff --git a/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py b/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py index 24bf3b1ed5..af79838191 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/flux_2.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os from typing import Any, Dict, Optional, Tuple import torch @@ -23,25 +22,25 @@ from diffusers.models.embeddings import TimestepEmbedding, Timesteps from diffusers.models.normalization import AdaLayerNormContinuous from sglang.kernels.ops.diffusion import ( + FLUX2_NVFP4_SWIGLU_QUANT, BitExactFusionGate, can_defer_flux2_gated_residual, can_use_flux2_gated_resnorm, can_use_flux2_strided_qknorm_rope, can_use_fused_layernorm_modulate, flux2_gated_resnorm_raw, - flux2_nvfp4_swiglu_quant_active, flux2_strided_qknorm_rope, fused_layernorm_modulate_fp8_quant_raw, fused_layernorm_modulate_raw, fused_packed_silu_mul_bitexact, is_plain_layer_norm, - mark_flux2_nvfp4_swiglu_quant_site, residual_gate_add, try_flux2_token_cat_fp8, try_flux2_token_cat_nvfp4, try_fused_flux2_qkv_epilogue, ) from sglang.kernels.ops.quantization.fp8_kernel import static_quant_fp8 +from sglang.multimodal_gen import envs from sglang.multimodal_gen.configs.models.dits.flux import FluxConfig from sglang.multimodal_gen.runtime.distributed import ( divide, @@ -126,10 +125,8 @@ def _flux2_single_qk_rope(q, k, q_norm, k_norm, head_dim, cache, complex_freqs): allow_strided_qk=current_platform.is_rocm(), ) - gate = _FLUX2_STRIDED_QK_ROPE if ( - gate.disabled - or head_dim != 128 + head_dim != 128 or type(q_norm) is not RMSNorm or type(k_norm) is not RMSNorm or q_norm.variance_epsilon != k_norm.variance_epsilon @@ -139,30 +136,18 @@ def _flux2_single_qk_rope(q, k, q_norm, k_norm, head_dim, cache, complex_freqs): is not RMSNorm.forward_cuda for norm in (q_norm, k_norm) ) - or os.getenv("SGLANG_ENABLE_FUSED_QKNORM_ROPE", "1").lower() - in {"0", "false", "off", "no"} + or not envs.SGLANG_ENABLE_FUSED_QKNORM_ROPE or not can_use_flux2_strided_qknorm_rope( q, k, q_norm.weight, k_norm.weight, cache ) ): return reference() - sig = (q.device, q.shape, q.stride(), cache.stride(), q_norm.variance_epsilon) - verified = gate.is_verified(sig) - if not verified and torch.cuda.is_current_stream_capturing(): - return reference() - try: - out = flux2_strided_qknorm_rope( + return _FLUX2_STRIDED_QK_ROPE.run( + lambda: flux2_strided_qknorm_rope( q, k, q_norm.weight, k_norm.weight, cache, q_norm.variance_epsilon - ) - except Exception as exc: - gate.on_exception(exc, logger=logger) - return reference() - if verified: - return out - return gate.accept_or_fallback( - out, - reference(), - sig=sig, + ), + reference, + sig=(q.device, q.shape, q.stride(), cache.stride(), q_norm.variance_epsilon), equal=lambda a, b: all( torch.equal(x.view(torch.int16), y.view(torch.int16)) for x, y in zip(a, b) ), @@ -245,10 +230,6 @@ def _try_flux2_norm_modulate_fp8( reference, sig=sig, logger=logger, - mismatch_msg=( - "FLUX.2 fused LN+modulate+FP8 quant fast path is not bit-exact " - "on this platform; falling back to the split path" - ), ) @@ -410,10 +391,6 @@ def _flux2_norm_modulate( ref, sig=sig, logger=logger, - mismatch_msg=( - "FLUX.2 fused LN+modulate fast path is not bit-exact on this " - "platform; falling back to eager" - ), ) @@ -460,10 +437,6 @@ def _flux2_swiglu(x: torch.Tensor) -> torch.Tensor: F.silu(x[..., :half]) * x[..., half:], sig=sig, logger=logger, - mismatch_msg=( - "FLUX.2 fused SwiGLU fast path is not bit-exact on this " - "platform; falling back to eager" - ), ) return F.silu(x[..., :half]) * x[..., half:] @@ -528,11 +501,11 @@ class Flux2FeedForward(nn.Module): # available. self.linear_in._interleave_for_swiglu_fusion = True self.linear_out._accepts_prequantized_fp4 = True - mark_flux2_nvfp4_swiglu_quant_site(self) + FLUX2_NVFP4_SWIGLU_QUANT.mark(self) def forward(self, x: torch.Tensor) -> torch.Tensor: if ( - flux2_nvfp4_swiglu_quant_active(self) + FLUX2_NVFP4_SWIGLU_QUANT.is_enabled(self) and getattr(self.linear_in, "_swiglu_fusion_ready", False) and not torch.compiler.is_compiling() and not torch.cuda.is_current_stream_capturing() diff --git a/python/sglang/multimodal_gen/runtime/models/dits/glm_image.py b/python/sglang/multimodal_gen/runtime/models/dits/glm_image.py index 8a101dded9..a3320194d3 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/glm_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/glm_image.py @@ -19,16 +19,13 @@ import torch.nn as nn import torch.nn.functional as F from sglang.kernels.ops.diffusion import ( + FUSED_LINEAR_GELU, BitExactFusionGate, - can_use_fused_layernorm_modulate, can_use_fused_qk_head_layernorm, can_use_linear_gelu, - fused_gelu_active, - fused_layernorm_modulate, fused_linear_gelu_tanh, fused_qk_head_layernorm, is_plain_layer_norm, - mark_fused_gelu_site, residual_gate_add, tensors_equal, ) @@ -59,6 +56,9 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im LayerwiseOffloadableModuleMixin, ) from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT +from sglang.multimodal_gen.runtime.models.dits.common import ( + layernorm_modulate_bitexact, +) from sglang.multimodal_gen.runtime.platforms import ( AttentionBackendEnum, current_platform, @@ -99,33 +99,15 @@ def _glm_ln_modulate( the first call verifies ``torch.equal`` against the eager chain and disables the fast path permanently on any mismatch. """ - verified = _GLM_LN_MOD.verified - if ( - not _GLM_LN_MOD.disabled - and _is_cuda - and x.is_cuda - and dtype is x.dtype - and is_plain_layer_norm(norm, x.shape[-1]) - and can_use_fused_layernorm_modulate(x.dtype, x.shape[-1]) - and (verified or _GLM_LN_MOD.can_attempt_once()) - ): - try: - out = fused_layernorm_modulate(x, scale, shift, norm.eps) - except Exception as exc: - _GLM_LN_MOD.on_exception(exc, logger=logger) - else: - if verified: - return out - return _GLM_LN_MOD.accept_or_fallback( - out, - _eager_ln_modulate(norm, x, scale, shift, dtype), - logger=logger, - mismatch_msg=( - "GLM fused LN+modulate fast path is not bit-exact against " - "this platform's LayerNorm dispatch; falling back to eager" - ), - ) - + if _is_cuda and dtype is x.dtype: + return layernorm_modulate_bitexact( + _GLM_LN_MOD, + norm, + x, + scale, + shift, + lambda: _eager_ln_modulate(norm, x, scale, shift, dtype), + ) return _eager_ln_modulate(norm, x, scale, shift, dtype) @@ -141,10 +123,8 @@ def _glm_qk_layernorm( First call verifies ``torch.equal`` against the eager pair and falls back permanently on any mismatch. """ - verified = _GLM_QK_LN.verified if ( - not _GLM_QK_LN.disabled - and _is_cuda + _is_cuda and query.is_cuda and dtype is query.dtype and dtype is key.dtype @@ -152,30 +132,13 @@ def _glm_qk_layernorm( and is_plain_layer_norm(norm_k, key.shape[-1]) and norm_q.eps == norm_k.eps and can_use_fused_qk_head_layernorm(query.dtype, query.shape[-1]) - and (verified or _GLM_QK_LN.can_attempt_once()) ): - try: - out = fused_qk_head_layernorm(query, key, norm_q.eps) - except Exception as exc: - _GLM_QK_LN.on_exception(exc, logger=logger) - else: - if verified: - return out - ref = ( - norm_q(query).to(dtype=dtype), - norm_k(key).to(dtype=dtype), - ) - return _GLM_QK_LN.accept_or_fallback( - out, - ref, - equal=tensors_equal, - logger=logger, - mismatch_msg=( - "GLM fused qk-LayerNorm fast path is not bit-exact against " - "this platform's LayerNorm dispatch; falling back to eager" - ), - ) - + return _GLM_QK_LN.run( + lambda: fused_qk_head_layernorm(query, key, norm_q.eps), + lambda: (norm_q(query).to(dtype=dtype), norm_k(key).to(dtype=dtype)), + equal=tensors_equal, + logger=logger, + ) return norm_q(query).to(dtype=dtype), norm_k(key).to(dtype=dtype) @@ -450,10 +413,12 @@ class GlmImageGELU(nn.Module): ) # lossless/high fusion site: up-proj GEMM + tanh-GELU in cublasLt # epilogue. Off by default; mounted per batch by the denoising stage. - mark_fused_gelu_site(self, "proj") + FUSED_LINEAR_GELU.mark(self, "proj") def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - if fused_gelu_active(self) and can_use_linear_gelu(self.proj, hidden_states): + if FUSED_LINEAR_GELU.is_enabled(self) and can_use_linear_gelu( + self.proj, hidden_states + ): return fused_linear_gelu_tanh( hidden_states, self.proj.weight, self.proj.bias ) @@ -645,14 +610,18 @@ class GlmImageAttention(torch.nn.Module): q_img = query[:, text_seq_length:, :, :] k_img = key[:, text_seq_length:, :, :] - q_img, k_img = self.rotary_emb( + q_rot, k_rot = self.rotary_emb( query=q_img, key=k_img, cos=cos, sin=sin, ) - query[:, text_seq_length:, :, :] = q_img - key[:, text_seq_length:, :, :] = k_img + # The CUDA RoPE rotates contiguous views in place and returns them; + # torch >= 2.14 rejects copying a tensor onto its own memory. + if q_rot.data_ptr() != q_img.data_ptr(): + q_img.copy_(q_rot) + if k_rot.data_ptr() != k_img.data_ptr(): + k_img.copy_(k_rot) if kv_cache is not None: if kv_cache.mode == "write": diff --git a/python/sglang/multimodal_gen/runtime/models/dits/helios.py b/python/sglang/multimodal_gen/runtime/models/dits/helios.py index 02958a0c45..abbdd61bea 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/helios.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/helios.py @@ -17,9 +17,9 @@ import torch.nn as nn import torch.nn.functional as F from sglang.kernels.ops.diffusion import ( + HELIOS_GATED_RESIDUAL, fused_inplace_helios_qk_rope, - mark_helios_gated_residual_site, - try_helios_gated_residual, + try_gated_residual, ) from sglang.multimodal_gen.configs.models.dits.helios import HeliosConfig from sglang.multimodal_gen.configs.models.fsdp import is_block @@ -497,7 +497,7 @@ class HeliosTransformerBlock(nn.Module): # 4. Guidance cross-attention flag self.guidance_cross_attn = guidance_cross_attn - mark_helios_gated_residual_site(self) + HELIOS_GATED_RESIDUAL.mark(self) def forward( self, @@ -528,7 +528,9 @@ class HeliosTransformerBlock(nn.Module): attn_output = self.attn1( norm_hidden_states, rotary_emb, original_context_length ) - fused = try_helios_gated_residual(self, hidden_states, attn_output, gate_msa) + fused = try_gated_residual( + HELIOS_GATED_RESIDUAL, self, hidden_states, attn_output, gate_msa + ) hidden_states = ( fused if fused is not None @@ -567,7 +569,9 @@ class HeliosTransformerBlock(nn.Module): # 3. Feed-forward norm_hidden_states = self.norm3(hidden_states, c_shift_msa, c_scale_msa) ff_output = self.ffn(norm_hidden_states) - fused = try_helios_gated_residual(self, hidden_states, ff_output, c_gate_msa) + fused = try_gated_residual( + HELIOS_GATED_RESIDUAL, self, hidden_states, ff_output, c_gate_msa + ) hidden_states = ( fused if fused is not None diff --git a/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py b/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py index 04288d6412..1516dc4267 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/hunyuanvideo.py @@ -9,13 +9,12 @@ import torch import torch.nn as nn from sglang.kernels.ops.diffusion import ( + FUSED_LINEAR_GELU, + HUNYUAN_QKNORM, BitExactFusionGate, can_use_linear_gelu, - fused_gelu_active, fused_linear_gelu_tanh, hunyuan_qkv_rope_pack, - mark_fused_gelu_site, - mark_hunyuan_qknorm_site, tensors_equal, try_hunyuan_qknorm, ) @@ -106,10 +105,10 @@ class HunyuanMLP(MLP): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - mark_fused_gelu_site(self, "fc_in") + FUSED_LINEAR_GELU.mark(self, "fc_in") def forward(self, x: torch.Tensor) -> torch.Tensor: - if fused_gelu_active(self) and can_use_linear_gelu(self.fc_in, x): + if FUSED_LINEAR_GELU.is_enabled(self) and can_use_linear_gelu(self.fc_in, x): x = fused_linear_gelu_tanh(x, self.fc_in.weight, self.fc_in.bias) else: x, _ = self.fc_in(x) @@ -173,9 +172,6 @@ def _hunyuan_pack_qkv( sig=sig, equal=tensors_equal, logger=logger, - mismatch_msg=( - "HunyuanVideo fused QKV RoPE pack is not bit-exact on this platform" - ), ) @@ -364,7 +360,7 @@ class MMDoubleStreamBlock(nn.Module): supported_attention_backends=supported_attention_backends, prefix=f"{prefix}.attn", ) - mark_hunyuan_qknorm_site(self) + HUNYUAN_QKNORM.mark(self) def forward( self, @@ -562,7 +558,7 @@ class MMSingleStreamBlock(nn.Module): supported_attention_backends=supported_attention_backends, prefix=f"{prefix}.attn", ) - mark_hunyuan_qknorm_site(self) + HUNYUAN_QKNORM.mark(self) def forward( self, diff --git a/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py b/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py index 63b584ca55..90fd43ee13 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/ideogram.py @@ -8,13 +8,12 @@ import torch.nn as nn import torch.nn.functional as F from sglang.kernels.ops.diffusion import ( + FUSED_GATE_RMSNORM, BitExactFusionGate, - fused_gate_rmsnorm_active, fused_rmsnorm_scale, fused_rmsnorm_tanh_residual, fused_rope_rotate_half_bitexact, fused_silu_mul_bitexact, - mark_fused_gate_rmsnorm_site, modulate_scale_shift, residual_gate_add, tensors_equal, @@ -85,62 +84,32 @@ def _ideogram_rope( equals the eager ``round(round(q1*cos1) - round(q2*sin1))`` exactly), so it mounts by default with first-call self-verification. """ - verified = _IDEOGRAM_ROPE.verified if ( - not _IDEOGRAM_ROPE.disabled - and q.is_cuda + q.is_cuda and q.dtype is torch.bfloat16 and q.is_contiguous() and k.is_contiguous() - and (verified or _IDEOGRAM_ROPE.can_attempt_once()) ): - try: - q_fused = fused_rope_rotate_half_bitexact(q, cos, sin) - k_fused = fused_rope_rotate_half_bitexact(k, cos, sin) - except Exception as exc: - _IDEOGRAM_ROPE.on_exception(exc, logger=logger) - else: - if verified: - return q_fused, k_fused - return _IDEOGRAM_ROPE.accept_or_fallback( - (q_fused, k_fused), - qwen3_apply_rotary_pos_emb(q, k, cos, sin), - equal=tensors_equal, - logger=logger, - mismatch_msg=( - "Ideogram fused RoPE fast path is not bit-exact on this " - "platform; falling back to eager" - ), - ) + return _IDEOGRAM_ROPE.run( + lambda: ( + fused_rope_rotate_half_bitexact(q, cos, sin), + fused_rope_rotate_half_bitexact(k, cos, sin), + ), + lambda: qwen3_apply_rotary_pos_emb(q, k, cos, sin), + equal=tensors_equal, + logger=logger, + ) return qwen3_apply_rotary_pos_emb(q, k, cos, sin) def _ideogram_swiglu(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: """``silu(a) * b`` in one kernel, bit-exact vs the eager pair.""" - verified = _IDEOGRAM_SWIGLU.verified - if ( - not _IDEOGRAM_SWIGLU.disabled - and _is_cuda - and a.is_cuda - and a.dtype is torch.bfloat16 - and (verified or _IDEOGRAM_SWIGLU.can_attempt_once()) - ): - try: - out = fused_silu_mul_bitexact(a, b) - except Exception as exc: - _IDEOGRAM_SWIGLU.on_exception(exc, logger=logger) - else: - if verified: - return out - return _IDEOGRAM_SWIGLU.accept_or_fallback( - out, - F.silu(a) * b, - logger=logger, - mismatch_msg=( - "Ideogram fused SiLU-mul fast path is not bit-exact on " - "this platform; falling back to eager" - ), - ) + if _is_cuda and a.is_cuda and a.dtype is torch.bfloat16: + return _IDEOGRAM_SWIGLU.run( + lambda: fused_silu_mul_bitexact(a, b), + lambda: F.silu(a) * b, + logger=logger, + ) return F.silu(a) * b @@ -493,7 +462,7 @@ class Ideogram4TransformerBlock(nn.Module): # collapses into one Triton kernel (Z-Image bf16-native suite). Off by # default (bit-exact reference path); mounted per batch by the # denoising stage. - mark_fused_gate_rmsnorm_site( + FUSED_GATE_RMSNORM.mark( self, ("attention_norm1", "attention_norm2", "ffn_norm1", "ffn_norm2") ) self.adaln_modulation = _linear( @@ -510,7 +479,7 @@ class Ideogram4TransformerBlock(nn.Module): adaln_input ).chunk(4, dim=-1) enable_fused = ( - fused_gate_rmsnorm_active(self) and not torch.compiler.is_compiling() + FUSED_GATE_RMSNORM.is_enabled(self) and not torch.compiler.is_compiling() ) attn_out = self.attention( _norm_scale(x, scale_msa, self.attention_norm1, enable_fused), diff --git a/python/sglang/multimodal_gen/runtime/models/dits/joy_image.py b/python/sglang/multimodal_gen/runtime/models/dits/joy_image.py index c1e91c5db3..fb75e56bc0 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/joy_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/joy_image.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 import math -import os from functools import lru_cache from typing import Any, Optional, Tuple @@ -11,6 +10,7 @@ from einops import rearrange from sglang.kernels.ops import diffusion as diffusion_kernels from sglang.kernels.ops.diffusion.sites.bitexact_gate import BitExactFusionGate +from sglang.multimodal_gen import envs from sglang.multimodal_gen.configs.models.dits.joy_image import JoyImageDiTConfig from sglang.multimodal_gen.configs.models.fsdp import is_blocks_or_double_blocks from sglang.multimodal_gen.runtime.distributed import ( @@ -78,10 +78,10 @@ def _joy_image_qk_rope( ) # Read the packed projection directly, using the same CUDA arithmetic as - # the original contiguous in-place operation. Keep unmeasured paths native. + # the original contiguous in-place operation; apply_qk_norm_rope only fuses + # in place, so it needs the contiguous copies. Keep unmeasured paths native. if ( - _JOY_IMAGE_QK_ROPE.disabled - or not q.is_cuda + not q.is_cuda or torch.version.hip or torch.compiler.is_compiling() or q.ndim != 4 @@ -113,19 +113,15 @@ def _joy_image_qk_rope( x.requires_grad for x in (q, k, q_norm.weight, k_norm.weight, cache) ) ) - or os.getenv("SGLANG_ENABLE_FUSED_QKNORM_ROPE", "1").lower() - in {"0", "false", "off", "no"} + or not envs.SGLANG_ENABLE_FUSED_QKNORM_ROPE ): return reference() - sig = (q.device, tuple(q.shape), tuple(q.stride()), q_norm.variance_epsilon) - verified = _JOY_IMAGE_QK_ROPE.is_verified(sig) - if not verified and torch.cuda.is_current_stream_capturing(): - return reference() if not diffusion_kernels.can_use_fused_inplace_qknorm_rope( 128, 128, False, q.dtype, cache.dtype ): return reference() - try: + + def fused(): q_out = torch.empty(q.shape, device=q.device, dtype=q.dtype) k_out = torch.empty_like(q_out) positions = torch.arange(q.shape[1], device=q.device, dtype=torch.int64) @@ -145,18 +141,14 @@ def _joy_image_qk_rope( head_dim=128, rope_dim=128, ) - except Exception as exc: - # The out-of-place operation leaves packed Q/K/V pristine, including - # when it has written part of an output before raising. - _JOY_IMAGE_QK_ROPE.on_exception(exc, logger=logger) - return reference() - out = (q_out, k_out) - if verified: - return out - return _JOY_IMAGE_QK_ROPE.accept_or_fallback( - out, - reference(), - sig=sig, + return q_out, k_out + + # The out-of-place kernel leaves packed Q/K/V pristine, including when it + # has written part of an output before raising, so a failure falls back. + return _JOY_IMAGE_QK_ROPE.run( + fused, + reference, + sig=(q.device, tuple(q.shape), tuple(q.stride()), q_norm.variance_epsilon), equal=lambda actual, expected: all( torch.equal(a.view(torch.int16), b.view(torch.int16)) for a, b in zip(actual, expected, strict=True) @@ -172,8 +164,7 @@ def _joy_joint_qkv(*inputs: torch.Tensor) -> tuple[torch.Tensor, ...]: # Below 32 MiB per image tensor, eager dispatch costs more than the copy # saves. Keep small resolutions and short sequence-parallel shards native. if ( - _JOY_QKV_CAT.disabled - or inputs[0].numel() < 16 * 1024 * 1024 + inputs[0].numel() < 16 * 1024 * 1024 or not inputs[0].is_cuda or torch.version.hip or torch.compiler.is_compiling() @@ -182,23 +173,11 @@ def _joy_joint_qkv(*inputs: torch.Tensor) -> tuple[torch.Tensor, ...]: or any(x.requires_grad for x in inputs) ): return reference() - sig = (inputs[0].device, inputs[0].dtype) + tuple( - (tuple(x.shape), tuple(x.stride())) for x in inputs - ) - verified = _JOY_QKV_CAT.is_verified(sig) - if not verified and torch.cuda.is_current_stream_capturing(): - return reference() - try: - out = diffusion_kernels.joint_qkv_cat(*inputs) - except Exception as exc: - _JOY_QKV_CAT.on_exception(exc, logger=logger) - return reference() - if verified: - return out - return _JOY_QKV_CAT.accept_or_fallback( - out, - reference(), - sig=sig, + return _JOY_QKV_CAT.run( + lambda: diffusion_kernels.joint_qkv_cat(*inputs), + reference, + sig=(inputs[0].device, inputs[0].dtype) + + tuple((tuple(x.shape), tuple(x.stride())) for x in inputs), equal=lambda actual, expected: all( torch.equal(a.view(torch.int16), b.view(torch.int16)) for a, b in zip(actual, expected, strict=True) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/krea2.py b/python/sglang/multimodal_gen/runtime/models/dits/krea2.py index 2c0dc91a13..576c35a0c1 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/krea2.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/krea2.py @@ -8,7 +8,6 @@ names follow the released K2 checkpoint, so weights load without remapping. """ import math -import os from typing import Any, Optional import torch @@ -17,6 +16,7 @@ import torch.nn.functional as F from einops import rearrange from torch import Tensor +from sglang.multimodal_gen import envs from sglang.multimodal_gen.configs.models.dits.krea2 import Krea2DitConfig from sglang.multimodal_gen.runtime.distributed import ( get_sp_world_size, @@ -64,12 +64,7 @@ def ropeapply(xq: Tensor, xk: Tensor, freqs: Tensor) -> tuple[Tensor, Tensor]: def _fused_qknorm_rope_enabled() -> bool: - return os.getenv("SGLANG_ENABLE_FUSED_QKNORM_ROPE", "1").lower() not in ( - "0", - "false", - "off", - "no", - ) + return envs.SGLANG_ENABLE_FUSED_QKNORM_ROPE def _can_use_fused_qknorm_rope(head_dim: int, dtype: torch.dtype) -> bool: @@ -346,7 +341,8 @@ class Attention(nn.Module): # Fast path: fuse RMSNorm(q), RMSNorm(k) and RoPE into one in-place kernel on # the [B, S, H, D] layout USPAttention consumes (also skips the [B, H, L, D] # transpose round-trip the eager path needs). Eager fallback below preserves - # parity off CUDA / for unsupported dtypes. + # parity off CUDA / for unsupported dtypes. Not apply_qk_norm_rope: Krea's + # norms scale by (1 + weight) and its eager rotation is the fp32 ropeapply. if ( freqs is not None and q.is_cuda diff --git a/python/sglang/multimodal_gen/runtime/models/dits/lingbot_video_moe.py b/python/sglang/multimodal_gen/runtime/models/dits/lingbot_video_moe.py index 9edd001e51..5603dbb59e 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/lingbot_video_moe.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/lingbot_video_moe.py @@ -12,12 +12,11 @@ from diffusers.models.embeddings import TimestepEmbedding, Timesteps from torch import nn from sglang.kernels.ops.diffusion import ( + LINGBOT_VIDEO_GATED_RESIDUAL, + LINGBOT_VIDEO_RMSNORM, can_use_rmsnorm_scale_shift_per_token, - lingbot_video_gated_residual_active, - mark_lingbot_video_gated_residual_site, - mark_lingbot_video_rmsnorm_site, rmsnorm_scale_shift_per_token, - try_lingbot_video_gated_residual, + try_gated_residual, try_lingbot_video_rmsnorm, ) from sglang.multimodal_gen.configs.models.dits.lingbot_video_moe import ( @@ -88,7 +87,7 @@ class LingBotVideoRMSNorm(nn.Module): super().__init__() self.weight = nn.Parameter(torch.ones(dim)) self.variance_epsilon = eps - mark_lingbot_video_rmsnorm_site(self) + LINGBOT_VIDEO_RMSNORM.mark(self) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: fused = try_lingbot_video_rmsnorm( @@ -116,7 +115,9 @@ def _lingbot_gated_residual( block's quality-gated site is enabled; otherwise the reference FP32 multiply form (bit-exact for ``quality="exact"``). """ - fused = try_lingbot_video_gated_residual(block, residual, update, gate) + fused = try_gated_residual( + LINGBOT_VIDEO_GATED_RESIDUAL, block, residual, update, gate + ) if fused is not None: return fused return residual + (gate * update).to(residual.dtype) @@ -136,7 +137,7 @@ def _lingbot_norm_modulate( quality-gated site is enabled; otherwise the reference chain. ``scale`` is the raw ``scale_msa/mlp``; the kernel applies ``x * (1 + scale)``. """ - if lingbot_video_gated_residual_active( + if LINGBOT_VIDEO_GATED_RESIDUAL.is_enabled( block ) and can_use_rmsnorm_scale_shift_per_token(x, norm.weight, scale, shift): try: @@ -337,7 +338,7 @@ class LingBotVideoBlock(nn.Module): self.layer_idx = layer_idx h = hidden_size self.scale_shift_table = nn.Parameter(torch.zeros(1, 6 * h)) - mark_lingbot_video_gated_residual_site(self) + LINGBOT_VIDEO_GATED_RESIDUAL.mark(self) self.norm1 = LingBotVideoRMSNorm(h, norm_eps) self.attn = LingBotVideoAttention( h, diff --git a/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py b/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py index 9103b600d7..51b93a2a5e 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/longcat_image.py @@ -34,12 +34,11 @@ from diffusers.models.normalization import ( from sglang.kernels.ops import diffusion as diffusion_ops from sglang.kernels.ops.diffusion import ( + FUSED_LINEAR_GELU, BitExactFusionGate, can_use_fused_inplace_qknorm_rope, can_use_linear_gelu, - fused_gelu_active, fused_linear_gelu_tanh, - mark_fused_gelu_site, residual_gate_add, tensors_equal, ) @@ -58,6 +57,9 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im LayerwiseOffloadableModuleMixin, ) from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT +from sglang.multimodal_gen.runtime.models.dits.common import ( + layernorm_modulate_bitexact, +) from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger @@ -76,24 +78,17 @@ def _longcat_gelu_cat( if ( type(activation) is nn.GELU and activation.approximate == "tanh" - and _LONGCAT_GELU_CAT.can_attempt_once() and _is_cuda and attn.is_cuda and attn.dtype is torch.bfloat16 and not torch.is_grad_enabled() and not torch.compiler.is_compiling() ): - try: - output = diffusion_ops.fused_gelu_tanh_cat(attn, mlp) - except Exception as exc: - _LONGCAT_GELU_CAT.on_exception(exc, logger=logger) - else: - if _LONGCAT_GELU_CAT.verified: - return output - reference = torch.cat((attn, activation(mlp)), dim=-1) - return _LONGCAT_GELU_CAT.accept_or_fallback( - output, reference, logger=logger - ) + return _LONGCAT_GELU_CAT.run( + lambda: diffusion_ops.fused_gelu_tanh_cat(attn, mlp), + lambda: torch.cat((attn, activation(mlp)), dim=-1), + logger=logger, + ) return torch.cat((attn, activation(mlp)), dim=-1) @@ -103,17 +98,22 @@ def _longcat_norm_modulate( scale: torch.Tensor, shift: torch.Tensor, ) -> torch.Tensor: - if torch.is_grad_enabled() or torch.compiler.is_compiling(): + + def reference(): return norm(x) * (1 + scale[:, None]) + shift[:, None] - # LayerNorm's reduction depends on the live aten dispatch. Verify each - # shape/stride before using the bit-exact fused kernel, outside capture. - if ( - not _LONGCAT_LN_MOD.disabled - and x.is_cuda - and diffusion_ops.is_plain_layer_norm(norm, x.shape[-1]) - and diffusion_ops.can_use_fused_layernorm_modulate(x.dtype, x.shape[-1]) - ): - sig = ( + + if torch.is_grad_enabled() or torch.compiler.is_compiling(): + return reference() + # LayerNorm's reduction depends on the live aten dispatch, so each + # shape/stride is verified separately. + return layernorm_modulate_bitexact( + _LONGCAT_LN_MOD, + norm, + x, + scale, + shift, + reference, + sig=( x.shape, x.stride(), scale.shape, @@ -123,21 +123,8 @@ def _longcat_norm_modulate( norm.eps, x.dtype, x.device, - ) - verified = _LONGCAT_LN_MOD.is_verified(sig) - if verified or not torch.cuda.is_current_stream_capturing(): - try: - out = diffusion_ops.fused_layernorm_modulate(x, scale, shift, norm.eps) - except Exception as exc: - _LONGCAT_LN_MOD.on_exception(exc, logger=logger) - else: - if verified: - return out - reference = norm(x) * (1 + scale[:, None]) + shift[:, None] - return _LONGCAT_LN_MOD.accept_or_fallback( - out, reference, sig=sig, logger=logger - ) - return norm(x) * (1 + scale[:, None]) + shift[:, None] + ), + ) class _LongCatAdaLayerNormZero(AdaLayerNormZero): @@ -266,10 +253,6 @@ def _apply_longcat_qknorm_rope( ref, equal=tensors_equal, logger=logger, - mismatch_msg=( - "LongCat fused QKNorm+RoPE is not bit-exact on this " - "platform; falling back to the Diffusers chain" - ), ) return _longcat_qknorm_rope_reference( @@ -327,11 +310,13 @@ class _LongCatFFN(nn.Module): # lossless/high site: up-proj GEMM + tanh-GELU epilogue. Off by # default; the denoising stage mounts it per batch. The ModuleDict holds # `proj` in _modules, so getattr resolves it for the fusion helper. - mark_fused_gelu_site(self.net[0], "proj") + FUSED_LINEAR_GELU.mark(self.net[0], "proj") def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: proj = self.net[0]["proj"] - if fused_gelu_active(self.net[0]) and can_use_linear_gelu(proj, hidden_states): + if FUSED_LINEAR_GELU.is_enabled(self.net[0]) and can_use_linear_gelu( + proj, hidden_states + ): hidden_states = fused_linear_gelu_tanh( hidden_states, proj.weight, proj.bias ) @@ -615,7 +600,7 @@ class _SingleTransformerBlock(nn.Module): self.act_mlp = nn.GELU(approximate="tanh") # lossless/high site: proj_mlp GEMM + tanh-GELU epilogue, # mounted per batch by the denoising stage; off (bit-exact) by default. - mark_fused_gelu_site(self, "proj_mlp") + FUSED_LINEAR_GELU.mark(self, "proj_mlp") # proj_out: RowParallelLinear reduces sharded [attn | mlp] concat via # all-reduce, matching Flux2SingleTransformerBlockAttention.to_out. self.proj_out = RowParallelLinear( @@ -680,7 +665,7 @@ class _SingleTransformerBlock(nn.Module): residual = hidden_states norm_hidden_states, gate = self.norm(hidden_states, emb=temb) - gelu_in_projection = fused_gelu_active(self) and can_use_linear_gelu( + gelu_in_projection = FUSED_LINEAR_GELU.is_enabled(self) and can_use_linear_gelu( self.proj_mlp, norm_hidden_states ) if gelu_in_projection: diff --git a/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py b/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py index ec4eef61ab..0374d65f93 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/ltx_2.py @@ -11,20 +11,17 @@ import torch.nn as nn import torch.nn.functional as F from sglang.kernels.ops.diffusion import ( + FUSED_LINEAR_GELU, + LTX2_QKNORM_SPLIT_ROPE, + LTX2_RMS_NORM_MODULATE, BitExactFusionGate, can_use_fused_rmsnorm_modulation, can_use_linear_gelu, can_use_ltx2_qknorm_split_rope_cuda, can_use_modulate_scale_shift_cuda, - fused_gelu_active, fused_linear_gelu_tanh, fused_ltx2_rms_norm_modulate, - ltx2_qknorm_split_rope_active, ltx2_qknorm_split_rope_cuda, - ltx2_rms_norm_modulate_active, - mark_fused_gelu_site, - mark_ltx2_qknorm_split_rope_site, - mark_ltx2_rms_norm_modulate_site, modulate_scale_shift_cuda, residual_gate_add, ) @@ -184,10 +181,6 @@ def _ltx2_modulate( out, x * (1 + scale) + shift, logger=logger, - mismatch_msg=( - "LTX-2 fused modulate is not bit-exact on this " - "platform; falling back to eager" - ), ) return x * (1 + scale) + shift @@ -208,7 +201,7 @@ def _ltx2_rms_norm_modulate( default). The fused kernel is not bit-exact (<=1 bf16 ULP) so it is gated on the request-scoped mount rather than a runtime self-check. """ - if ltx2_rms_norm_modulate_active(block) and can_use_fused_rmsnorm_modulation( + if LTX2_RMS_NORM_MODULATE.is_enabled(block) and can_use_fused_rmsnorm_modulation( x, scale, shift ): return fused_ltx2_rms_norm_modulate(x, scale, shift, eps) @@ -755,7 +748,7 @@ class LTX2Attention(nn.Module): self.apply_gated_attention = bool(apply_gated_attention) self.enable_packed_qkv_input_a2a = bool(enable_packed_qkv_input_a2a) self.prefix = prefix - mark_ltx2_qknorm_split_rope_site(self) + LTX2_QKNORM_SPLIT_ROPE.mark(self) tp_size = get_tp_world_size() if tp_size <= 0: @@ -919,7 +912,7 @@ class LTX2Attention(nn.Module): eps=self.norm_eps, num_heads=self.local_heads, head_dim=self.dim_head, - allow_sm90=ltx2_qknorm_split_rope_active(self), + allow_sm90=LTX2_QKNORM_SPLIT_ROPE.is_enabled(self), ) if fused_qk is not None: @@ -1083,10 +1076,10 @@ class LTX2FeedForward(nn.Module): input_is_parallel=True, quant_config=quant_config, ) - mark_fused_gelu_site(self, "proj_in") + FUSED_LINEAR_GELU.mark(self, "proj_in") def forward(self, x: torch.Tensor) -> torch.Tensor: - if fused_gelu_active(self) and can_use_linear_gelu(self.proj_in, x): + if FUSED_LINEAR_GELU.is_enabled(self) and can_use_linear_gelu(self.proj_in, x): x = fused_linear_gelu_tanh(x, self.proj_in.weight, self.proj_in.bias) else: x, _ = self.proj_in(x) @@ -1224,7 +1217,7 @@ class LTX2TransformerBlock(nn.Module): self.ff = LTX2FeedForward( dim, dim_out=dim, bias=ff_bias, quant_config=quant_config ) - mark_ltx2_rms_norm_modulate_site(self) + LTX2_RMS_NORM_MODULATE.mark(self) self.audio_ff = LTX2FeedForward( audio_dim, dim_out=audio_dim, bias=audio_ff_bias, quant_config=quant_config ) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/ming_image.py b/python/sglang/multimodal_gen/runtime/models/dits/ming_image.py index a893b8657b..3c6b0c9932 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/ming_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/ming_image.py @@ -45,8 +45,7 @@ class MingSiluAndMul(nn.Module): return _eager_silu_and_mul(x) can_fuse = ( - not _MING_SWIGLU_FUSION.disabled - and x.is_cuda + x.is_cuda and x.dtype is torch.bfloat16 and x.dim() == 3 and x.stride(-1) == 1 @@ -58,28 +57,19 @@ class MingSiluAndMul(nn.Module): if not can_fuse: return _eager_silu_and_mul(x) - # The first tensor of each layout is compared bit-for-bit against - # eager. Avoid a host synchronization inside CUDA graph capture. - sig = (x.dtype, x.device, x.shape[0], x.shape[-1], x.stride(-2), x.stride(-1)) - verified = _MING_SWIGLU_FUSION.is_verified(sig) - if not verified and torch.cuda.is_current_stream_capturing(): - return _eager_silu_and_mul(x) - try: - out = fused_packed_silu_mul_bitexact(x) - except Exception as exc: - _MING_SWIGLU_FUSION.on_exception(exc, logger=logger) - return _eager_silu_and_mul(x) - if verified: - return out - return _MING_SWIGLU_FUSION.accept_or_fallback( - out, - _eager_silu_and_mul(x), - sig=sig, - logger=logger, - mismatch_msg=( - "Ming-Image packed SiLU-mul is not bit-exact on this platform; " - "falling back to eager" + # The first tensor of each layout is compared bit-for-bit against eager. + return _MING_SWIGLU_FUSION.run( + lambda: fused_packed_silu_mul_bitexact(x), + lambda: _eager_silu_and_mul(x), + sig=( + x.dtype, + x.device, + x.shape[0], + x.shape[-1], + x.stride(-2), + x.stride(-1), ), + logger=logger, ) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py index ebc75569a6..e2f4cfc8c4 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/minimax_h3.py @@ -23,6 +23,7 @@ from sglang.kernels.ops.activation.activation import ( silu_and_mul_with_activation_rounding_, ) from sglang.kernels.ops.diffusion import ( + MINIMAX_H3_NORM_MODULATE, can_use_fused_inplace_qknorm_rope, can_use_mxfp8_swizzled, can_use_rmsnorm_indexed_scale_shift, @@ -36,8 +37,6 @@ from sglang.kernels.ops.diffusion import ( indexed_scale_shift_bf16_, indexed_scale_shift_fp32_, indexed_scale_shift_mxfp8_, - mark_minimax_h3_norm_modulate_site, - minimax_h3_norm_modulate_active, rmsnorm_indexed_scale_shift, silu_mul_mxfp8, ) @@ -1019,6 +1018,8 @@ class MiniMaxH3Attention(nn.Module): self.k_norm = _norm(arch.attention_head_dim, eps=arch.qk_norm_eps) # cache width covers cos/sin for temporal, height, and width frequencies rope_dim = 6 * arch.rope_inv_freq_len + # Not apply_qk_norm_rope: Q/K are [tokens, heads, head_dim] and the norms + # are torch nn.RMSNorm, so the fused kernel is called directly. self._use_fused_qknorm_rope = ( current_platform.is_cuda() and can_use_fused_inplace_qknorm_rope( @@ -1634,7 +1635,7 @@ class MiniMaxH3DiTBlock(nn.Module): super().__init__() self.norm1 = _norm(arch.hidden_size, eps=arch.norm_eps) self.norm2 = _norm(arch.hidden_size, eps=arch.norm_eps) - mark_minimax_h3_norm_modulate_site(self) + MINIMAX_H3_NORM_MODULATE.mark(self) self.attn = MiniMaxH3Attention( arch, quant_config, @@ -1707,7 +1708,7 @@ class MiniMaxH3DiTBlock(nn.Module): scale_msa, combined_indices, dtype=_BF16_DTYPE, - fused=minimax_h3_norm_modulate_active(self), + fused=MINIMAX_H3_NORM_MODULATE.is_enabled(self), ) h = self.attn( h, @@ -1750,7 +1751,7 @@ class MiniMaxH3DiTBlock(nn.Module): scale_mlp, combined_indices, dtype=_BF16_DTYPE, - fused=minimax_h3_norm_modulate_active(self), + fused=MINIMAX_H3_NORM_MODULATE.is_enabled(self), ) h = self.mlp(h) # `residual` is block-local here (see above), so this stays in-place diff --git a/python/sglang/multimodal_gen/runtime/models/dits/moss_image.py b/python/sglang/multimodal_gen/runtime/models/dits/moss_image.py index 89de04ef20..44df278dfe 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/moss_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/moss_image.py @@ -79,27 +79,6 @@ _NORM_MOD_FUSION = BitExactFusionGate("MOSSImage RMSNorm + modulation") _NORM_ROPE_FUSION = BitExactFusionGate("MOSSImage RMSNorm + RoPE") -def _fused_or_native(gate, eligible, fused, native, sig=None): - if eligible and not gate.disabled: - verified = gate.is_verified(sig) - if verified or not ( - torch.compiler.is_compiling() or torch.cuda.is_current_stream_capturing() - ): - try: - out = fused() - except ValueError: - raise - except Exception as exc: - gate.on_exception(exc, logger=logger) - else: - return ( - out - if verified - else gate.accept_or_fallback(out, native(), sig=sig, logger=logger) - ) - return native() - - def validate_skip_layers(skip_layers, num_layers): try: skip_layers = tuple(skip_layers) @@ -142,14 +121,16 @@ class _RMSNorm(nn.Module): self.weight = nn.Parameter(torch.ones(dim)) def forward(self, x): - return _fused_or_native( - _RMSNORM_FUSION, - can_use_rmsnorm_cast_before_weight( - device=x.device, dtype=x.dtype, contiguous=x.is_contiguous() - ), + if not can_use_rmsnorm_cast_before_weight( + device=x.device, dtype=x.dtype, contiguous=x.is_contiguous() + ): + return self.forward_native(x) + return _RMSNORM_FUSION.run( lambda: rmsnorm_cast_before_weight_bitexact(x, self.weight), lambda: self.forward_native(x), sig=(x.device.index, x.dtype, x.shape[-1], x.numel() // x.shape[-1] >= 16), + logger=logger, + reraise=(ValueError,), ) def forward_native(self, x): @@ -164,11 +145,13 @@ def _situ_native(hidden, gate): def _situ(hidden, gate): - return _fused_or_native( - _SITU_FUSION, - can_use_fused_situ_softcap_mul(device=hidden.device, dtype=hidden.dtype), + if not can_use_fused_situ_softcap_mul(device=hidden.device, dtype=hidden.dtype): + return _situ_native(hidden, gate) + return _SITU_FUSION.run( lambda: fused_situ_softcap_mul_bitexact(hidden, gate), lambda: _situ_native(hidden, gate), + logger=logger, + reraise=(ValueError,), ) @@ -186,11 +169,13 @@ def _rotate_native(x, rope): def _rotate(x, rope): sin, cos = (part.flatten(0, 1) for part in rope) - return _fused_or_native( - _ROPE_FUSION, - x.is_cuda and x.dtype is torch.bfloat16, + if not (x.is_cuda and x.dtype is torch.bfloat16): + return _rotate_native(x, rope) + return _ROPE_FUSION.run( lambda: fused_rope_rotate_half_bitexact(x, cos, sin), lambda: _rotate_native(x, rope), + logger=logger, + reraise=(ValueError,), ) @@ -206,34 +191,38 @@ def _attention(q, k, v, rope, heads, attention, attn_mask_meta=None): def _norm_modulate(norm, x, scale, shift): - return _fused_or_native( - _NORM_MOD_FUSION, - can_use_rmsnorm_cast_before_weight_modulate( - device=x.device, - dtype=x.dtype, - dim=x.shape[-1], - rows=x.numel() // x.shape[-1], - ), + if not can_use_rmsnorm_cast_before_weight_modulate( + device=x.device, + dtype=x.dtype, + dim=x.shape[-1], + rows=x.numel() // x.shape[-1], + ): + return modulate_scale_shift(norm(x), scale, shift) + return _NORM_MOD_FUSION.run( lambda: rmsnorm_cast_before_weight_modulate_bitexact( x, norm.weight, scale, shift ), lambda: modulate_scale_shift(norm(x), scale, shift), + logger=logger, + reraise=(ValueError,), ) def _norm_rope(norm, x, rope, heads): x = x.unflatten(-1, (heads, -1)) sin, cos = (part.flatten(0, 1) for part in rope) - return _fused_or_native( - _NORM_ROPE_FUSION, - can_use_rmsnorm_cast_before_weight_rope( - device=x.device, - dtype=x.dtype, - dim=x.shape[-1], - rows=x.numel() // x.shape[-1], - ), + if not can_use_rmsnorm_cast_before_weight_rope( + device=x.device, + dtype=x.dtype, + dim=x.shape[-1], + rows=x.numel() // x.shape[-1], + ): + return _rotate(norm(x), rope).flatten(2) + return _NORM_ROPE_FUSION.run( lambda: rmsnorm_cast_before_weight_rope_bitexact(x, norm.weight, cos, sin), lambda: _rotate(norm(x), rope), + logger=logger, + reraise=(ValueError,), ).flatten(2) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py index a159e42516..f647916968 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/qwen_image.py @@ -3,7 +3,6 @@ # SPDX-License-Identifier: Apache-2.0 import functools -import os from dataclasses import dataclass from typing import Any, Dict, List, Optional, Tuple, Union @@ -17,16 +16,14 @@ from diffusers.models.modeling_outputs import Transformer2DModelOutput from diffusers.models.normalization import AdaLayerNormContinuous from sglang.kernels.ops.diffusion import ( + FUSED_LINEAR_GELU, + QWEN_IMAGE_ADDED_QKV, BitExactFusionGate, can_use_fused_layernorm_modulate, can_use_linear_gelu, - fused_gelu_active, fused_layernorm_modulate_raw, fused_linear_gelu_tanh, is_plain_layer_norm, - mark_fused_gelu_site, - mark_qwen_image_added_qkv_site, - qwen_image_added_qkv_active, try_fused_bias_mul_add, try_fused_bias_scale_residual_norm_scale_shift, try_fused_norm_scale_shift_fp8, @@ -34,6 +31,7 @@ from sglang.kernels.ops.diffusion import ( try_fused_scale_residual_norm_scale_shift_fp8, try_fused_scale_residual_norm_scale_shift_nvfp4, ) +from sglang.multimodal_gen import envs from sglang.multimodal_gen.configs.models.dits.qwenimage import QwenImageDitConfig from sglang.multimodal_gen.configs.models.fsdp import is_transformer_block from sglang.multimodal_gen.runtime.distributed import ( @@ -180,10 +178,6 @@ def _qwen_norm_out( reference, sig=sig, logger=logger, - mismatch_msg=( - "Qwen-Image fused norm_out is not bit-exact on this platform; " - "falling back to eager" - ), ) @@ -771,8 +765,7 @@ class QwenImageCrossAttention(nn.Module): and current_platform.is_cuda() and capability is not None and capability.major == 9 - and os.getenv("SGLANG_ENABLE_FUSED_QKNORM_ROPE", "1").lower() - not in {"0", "false", "off", "no"} + and envs.SGLANG_ENABLE_FUSED_QKNORM_ROPE ) self.use_fused_qkv = ( isinstance(quant_config, NunchakuConfig) or quant_name == "modelopt_fp8" @@ -847,7 +840,7 @@ class QwenImageCrossAttention(nn.Module): if self._unquantized_added_qkv_is_packed: # Packing changes BF16 GEMM reduction association, so it # is off at exact and mounts at lossless or high. - mark_qwen_image_added_qkv_site(self) + QWEN_IMAGE_ADDED_QKV.mark(self) else: self.add_q_proj = ColumnParallelLinear( added_kv_proj_dim, @@ -949,7 +942,7 @@ class QwenImageCrossAttention(nn.Module): if self.use_fused_added_qkv: if ( self._unquantized_added_qkv_is_packed - and not qwen_image_added_qkv_active(self) + and not QWEN_IMAGE_ADDED_QKV.is_enabled(self) and isinstance(self.to_added_qkv, MergedColumnParallelLinear) ): return _split_unquantized_merged_linear( @@ -1020,8 +1013,9 @@ class QwenImageCrossAttention(nn.Module): num_heads=self.local_num_heads, head_dim=self.head_dim, ) - elif self._unquantized_added_qkv_is_packed and not qwen_image_added_qkv_active( - self + elif ( + self._unquantized_added_qkv_is_packed + and not QWEN_IMAGE_ADDED_QKV.is_enabled(self) ): img_query, img_key, img_value, _, _, _ = _get_qkv_projections( self, @@ -1280,7 +1274,7 @@ class QwenImageGELU(nn.Module): ) # Extra-high-or-higher fusion site: up-proj GEMM + tanh-GELU in cublasLt # epilogue. Off by default; mounted per batch by the denoising stage. - mark_fused_gelu_site(self, "proj") + FUSED_LINEAR_GELU.mark(self, "proj") def forward( self, @@ -1293,7 +1287,9 @@ class QwenImageGELU(nn.Module): output_dtype=self.proj.params_dtype, bias=self.proj.bias, ) - elif fused_gelu_active(self) and can_use_linear_gelu(self.proj, hidden_states): + elif FUSED_LINEAR_GELU.is_enabled(self) and can_use_linear_gelu( + self.proj, hidden_states + ): return fused_linear_gelu_tanh( hidden_states, self.proj.weight, self.proj.bias ) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/sana.py b/python/sglang/multimodal_gen/runtime/models/dits/sana.py index b3768d35cb..ea07f9d336 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/sana.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/sana.py @@ -117,10 +117,6 @@ def sana_ln_modulate( _eager_ln_modulate(norm, x, scale, shift), sig=sig, logger=logger, - mismatch_msg=( - "Sana fused LN+modulate fast path is not bit-exact against " - "this platform's LayerNorm dispatch; falling back to eager" - ), ) _SANA_LN_MOD_DISABLED = _SANA_LN_MOD.disabled return result @@ -199,10 +195,6 @@ def sana_conv_bias_silu( out, F.silu(raw + conv.bias[None, :, None, None]), logger=logger, - mismatch_msg=( - "Sana fused conv bias-SiLU path is not bit-exact on this " - "platform; falling back to eager" - ), ) @@ -240,10 +232,6 @@ def sana_conv_bias_glu( out, hidden_states * F.silu(gate), logger=logger, - mismatch_msg=( - "Sana fused conv bias-GLU path is not bit-exact on this " - "platform; falling back to eager" - ), ) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/sana_video.py b/python/sglang/multimodal_gen/runtime/models/dits/sana_video.py index b347799381..d4aed8d85b 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/sana_video.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/sana_video.py @@ -10,9 +10,9 @@ import torch.nn.functional as F from diffusers.models.embeddings import PixArtAlphaTextProjection from sglang.kernels.ops.diffusion import ( + SANA_VIDEO_LINEAR_ATTENTION, BitExactFusionGate, fused_interleaved_rope_fp64, - mark_sana_video_linear_attention_site, tensors_equal, try_sana_video_linear_attention, ) @@ -65,39 +65,21 @@ def apply_interleaved_rotary_emb_pair( freqs_sin: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: """Apply paired interleaved RoPE, with a bit-exact eager fallback.""" - verified = _SANA_VIDEO_ROPE.verified - if ( - not _SANA_VIDEO_ROPE.disabled - and _is_cuda - and query.is_cuda - and query.dtype is torch.bfloat16 - and (verified or _SANA_VIDEO_ROPE.can_attempt_once()) - ): - try: - fused = fused_interleaved_rope_fp64(query, key, freqs_cos, freqs_sin) - except Exception as exc: - _SANA_VIDEO_ROPE.on_exception(exc, logger=logger) - else: - if verified: - return fused - eager = ( - apply_interleaved_rotary_emb(query, freqs_cos, freqs_sin), - apply_interleaved_rotary_emb(key, freqs_cos, freqs_sin), - ) - return _SANA_VIDEO_ROPE.accept_or_fallback( - fused, - eager, - equal=tensors_equal, - logger=logger, - mismatch_msg=( - "SANA-Video fused RoPE is not bit-exact on this platform; " - "falling back to eager" - ), - ) - return ( - apply_interleaved_rotary_emb(query, freqs_cos, freqs_sin), - apply_interleaved_rotary_emb(key, freqs_cos, freqs_sin), - ) + + def reference(): + return ( + apply_interleaved_rotary_emb(query, freqs_cos, freqs_sin), + apply_interleaved_rotary_emb(key, freqs_cos, freqs_sin), + ) + + if _is_cuda and query.is_cuda and query.dtype is torch.bfloat16: + return _SANA_VIDEO_ROPE.run( + lambda: fused_interleaved_rope_fp64(query, key, freqs_cos, freqs_sin), + reference, + equal=tensors_equal, + logger=logger, + ) + return reference() class SanaVideoRotaryPosEmbed(nn.Module): @@ -252,7 +234,7 @@ class SanaVideoLinearAttention(nn.Module): self.to_out = nn.ModuleList( [nn.Linear(self.inner_dim, query_dim, bias=True), nn.Identity()] ) - mark_sana_video_linear_attention_site(self) + SANA_VIDEO_LINEAR_ATTENTION.mark(self) def forward( self, diff --git a/python/sglang/multimodal_gen/runtime/models/dits/sana_wm.py b/python/sglang/multimodal_gen/runtime/models/dits/sana_wm.py index aac01d8ef0..3263dffbf3 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/sana_wm.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/sana_wm.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -import os from typing import Callable, List, Optional, Tuple import torch @@ -13,7 +12,6 @@ from sglang.multimodal_gen.configs.models.fsdp import ( from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( LayerwiseOffloadableModuleMixin, ) -from sglang.multimodal_gen.runtime.models.dits import sana_wm_parity as parity_probe from sglang.multimodal_gen.runtime.models.dits.base import CachableDiT # Re-exported for back-compat: callers import these names from this module path. @@ -838,34 +836,6 @@ class SanaWMTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin): if not self.use_chunk_plucker_post_attn: plucker_emb = None - # parity harness (env-gated, no-op in prod): on the FIRST sink-path call - # (frame_index not None), checksum the pre-block tensors and x after every - # block to localize where the two execution paths first diverge. - _probe_path = os.environ.get(parity_probe.ENV_BLOCK_PROBE) - _probe = None - if ( - _probe_path - and frame_index is not None - and not getattr(self, "_block_probe_done", False) - ): - _ck = parity_probe.checksum - _probe = { - "x_embed": _ck(x), - "t6": _ck(t6), - "y": _ck(y), - "freqs": ( - ( - tuple(freqs.shape), - float(freqs.real.detach().double().sum().item()), - float(freqs.imag.detach().double().sum().item()), - ) - if freqs is not None - else None - ), - "plucker_emb": _ck(plucker_emb), - "frame_index": frame_index.tolist(), - } - HW = (T, H, W) new_cache = [] for i, block in enumerate(self.blocks): @@ -882,11 +852,6 @@ class SanaWMTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin): save_kv_cache=save_kv_cache, ) new_cache.append(block_cache) - if _probe is not None: - _probe[f"x_after_block_{i:02d}"] = parity_probe.checksum(x) - if _probe is not None: - torch.save(_probe, _probe_path) - self._block_probe_done = True x = self.final_layer(x, t_emb) x = x.reshape(B, T, H, W, p_t, p_h, p_w, self.out_channels) diff --git a/python/sglang/multimodal_gen/runtime/models/dits/sana_wm_parity.py b/python/sglang/multimodal_gen/runtime/models/dits/sana_wm_parity.py deleted file mode 100644 index b1f25f3361..0000000000 --- a/python/sglang/multimodal_gen/runtime/models/dits/sana_wm_parity.py +++ /dev/null @@ -1,82 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""Env-gated parity dump/probe helpers (no-op in production). - -Single home for the debug harness that localized the realtime<->batch parity -bugs (previously copy-pasted across streaming/realtime/refiner/sana_wm). - -Env vars: -- ``SANAWM_FORK_DUMP_DIR`` — batch-path dumps (init_noise, conditioning, - per-chunk stage-1, kv/refiner probes, weights fingerprint) -- ``SANAWM_RT_DUMP_DIR`` — realtime-path dumps (same set) -- ``SANAWM_BLOCK_PROBE`` — file path: per-block forward_long checksums on - the first sink-path call -- ``SANAWM_INJECT_DIR`` — batch-path injection inputs (read, not written) - -All checksums reduce in float64 (bf16/fp32 upcasts are lossless, so the -formula is bitwise-stable across both paths). -""" - -from __future__ import annotations - -import os -from pathlib import Path - -import torch - -ENV_FORK_DUMP = "SANAWM_FORK_DUMP_DIR" -ENV_RT_DUMP = "SANAWM_RT_DUMP_DIR" -ENV_BLOCK_PROBE = "SANAWM_BLOCK_PROBE" -ENV_INJECT = "SANAWM_INJECT_DIR" - - -def probe_dir(*env_names: str) -> Path | None: - """First configured dump dir among ``env_names`` (created on demand).""" - for name in env_names: - value = os.environ.get(name) - if value: - path = Path(value) - path.mkdir(parents=True, exist_ok=True) - return path - return None - - -def dump_tensor(dirpath: Path | str | None, name: str, tensor) -> None: - """Save ``tensor`` as float32 CPU under ``dirpath/name.pt`` (None-safe).""" - if dirpath is None or tensor is None: - return - torch.save(tensor.detach().float().cpu(), Path(dirpath) / f"{name}.pt") - - -def dump_obj(dirpath: Path | str | None, name: str, obj) -> None: - """Save a picklable object (e.g. a checksum dict) under ``dirpath/name.pt``.""" - if dirpath is None or obj is None: - return - torch.save(obj, Path(dirpath) / f"{name}.pt") - - -def checksum(tensor) -> tuple[tuple[int, ...], float] | None: - """(shape, float64 sum) of a tensor; None passes through.""" - if tensor is None: - return None - return ( - tuple(tensor.shape), - float(tensor.detach().float().double().sum().item()), - ) - - -def kv_cache_checksums(chunk_kv, sink_num: int) -> dict: - """Per-block, per-slot checksums of an accumulated KV cache.""" - probe: dict = {"sink_num": sink_num} - for block_id, slots in enumerate(chunk_kv): - for slot_id, tensor in enumerate(slots): - if tensor is not None: - probe[f"b{block_id:02d}s{slot_id}"] = checksum(tensor) - return probe - - -def weights_fingerprint(module: torch.nn.Module) -> dict[str, float]: - """Per-parameter |.|-sum fingerprint (catches wrong/mutated weights).""" - return { - name: float(param.detach().float().abs().sum().item()) - for name, param in module.named_parameters() - } diff --git a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py index 39088fca35..b569305607 100755 --- a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py @@ -10,14 +10,12 @@ import torch import torch.nn as nn from sglang.kernels.ops.diffusion import ( + FUSED_LINEAR_GELU, + NVFP4_BIAS_GELU, BitExactFusionGate, can_use_linear_gelu, - fused_gelu_active, fused_linear_gelu_tanh, fused_temb_table_slices, - mark_fused_gelu_site, - mark_nvfp4_bias_gelu_site, - nvfp4_bias_gelu_active, tensors_equal, ) from sglang.multimodal_gen.configs.models.dits import WanVideoConfig @@ -102,22 +100,22 @@ class _WanGELUMLP(MLP): quant_config=quant_config, fuse_bias_gelu_tanh=False, ) - mark_fused_gelu_site(self, "fc_in") + FUSED_LINEAR_GELU.mark(self, "fc_in") self.fuse_bias_gelu_tanh = isinstance( self.fc_in.quant_method, ModelOptFp4LinearMethod ) if self.fuse_bias_gelu_tanh: - mark_nvfp4_bias_gelu_site(self) + NVFP4_BIAS_GELU.mark(self) def forward(self, x: torch.Tensor) -> torch.Tensor: - if fused_gelu_active(self) and can_use_linear_gelu(self.fc_in, x): + if FUSED_LINEAR_GELU.is_enabled(self) and can_use_linear_gelu(self.fc_in, x): x = fused_linear_gelu_tanh(x, self.fc_in.weight, self.fc_in.bias) else: x, bias = self.fc_in(x) x = self._apply_activation( x, bias, - use_fused_bias_gelu=nvfp4_bias_gelu_active(self), + use_fused_bias_gelu=NVFP4_BIAS_GELU.is_enabled(self), ) x, _ = self.fc_out(x) return x @@ -405,31 +403,18 @@ def _wan_temb_table_slices( so the result is bit-identical to the eager chain; the first call still verifies ``torch.equal`` and falls back permanently on mismatch. """ - verified = _WAN_TEMB_SLICES.verified - if ( - not _WAN_TEMB_SLICES.disabled - and _is_cuda - and temb.is_cuda - and (verified or _WAN_TEMB_SLICES.can_attempt_once()) - ): - try: - buf = fused_temb_table_slices(table, temb) - except Exception as exc: - _WAN_TEMB_SLICES.on_exception(exc, logger=logger) - else: - out = tuple(buf[j] for j in range(6)) - if verified: - return out - return _WAN_TEMB_SLICES.accept_or_fallback( - out, - _eager_temb_table_slices(table, temb), - equal=tensors_equal, - logger=logger, - mismatch_msg=( - "Wan fused temb-table slices are not bit-exact on this " - "platform; falling back to eager" - ), - ) + + def fused(): + buf = fused_temb_table_slices(table, temb) + return tuple(buf[j] for j in range(6)) + + if _is_cuda and temb.is_cuda: + return _WAN_TEMB_SLICES.run( + fused, + lambda: _eager_temb_table_slices(table, temb), + equal=tensors_equal, + logger=logger, + ) return _eager_temb_table_slices(table, temb) diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py b/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py index a4d4c3edf1..2059333271 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/gemma_3.py @@ -282,17 +282,6 @@ class Gemma3Attention(nn.Module): is_neox_style=True, ) - # Local Attention not support attention mask, we use global attention instead. - # self.attn = LocalAttention( - # self.num_heads, - # self.head_dim, - # self.num_kv_heads, - # softmax_scale=self.scaling, - # causal=True, - # supported_attention_backends=config._supported_attention_backends, - # window_size=self.window_size, - # ) - # Gemma3 adds normalization for q and k self.q_norm = Gemma3RMSNorm( dim=self.head_dim, eps=config.text_config.rms_norm_eps diff --git a/python/sglang/multimodal_gen/runtime/models/registry.py b/python/sglang/multimodal_gen/runtime/models/registry.py index 0330acc75c..14b54d8f6f 100644 --- a/python/sglang/multimodal_gen/runtime/models/registry.py +++ b/python/sglang/multimodal_gen/runtime/models/registry.py @@ -6,17 +6,12 @@ import ast import importlib import os -import pickle -import subprocess -import sys -import tempfile from abc import ABC, abstractmethod -from collections.abc import Callable, Set +from collections.abc import Set from dataclasses import dataclass, field from functools import lru_cache -from typing import NoReturn, TypeVar, cast +from typing import NoReturn, cast -import cloudpickle from torch import nn from sglang.multimodal_gen.runtime.utils.external_model_package import ( @@ -156,31 +151,8 @@ def _discover_and_register_models() -> dict[str, tuple[str, str, str]]: _SGLANG_DIFFUSION_MODELS = _discover_and_register_models() -_SUBPROCESS_COMMAND = [ - sys.executable, - "-m", - "sglang.multimodal_gen.runtime.models.dits.registry", -] - -_T = TypeVar("_T") - - -@dataclass(frozen=True) -class _ModelInfo: - architecture: str - - @staticmethod - def from_model_cls(model: type[nn.Module]) -> "_ModelInfo": - return _ModelInfo( - architecture=model.__name__, - ) - class _BaseRegisteredModel(ABC): - @abstractmethod - def inspect_model_cls(self) -> _ModelInfo: - raise NotImplementedError - @abstractmethod def load_model_cls(self) -> type[nn.Module]: raise NotImplementedError @@ -192,51 +164,12 @@ class _RegisteredModel(_BaseRegisteredModel): Represents a model that has already been imported in the main process. """ - interfaces: _ModelInfo model_cls: type[nn.Module] - @staticmethod - def from_model_cls(model_cls: type[nn.Module]): - return _RegisteredModel( - interfaces=_ModelInfo.from_model_cls(model_cls), - model_cls=model_cls, - ) - - def inspect_model_cls(self) -> _ModelInfo: - return self.interfaces - def load_model_cls(self) -> type[nn.Module]: return self.model_cls -def _run_in_subprocess(fn: Callable[[], _T]) -> _T: - # NOTE: We use a temporary directory instead of a temporary file to avoid - # issues like https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file - with tempfile.TemporaryDirectory() as tempdir: - output_filepath = os.path.join(tempdir, "registry_output.tmp") - - # `cloudpickle` allows pickling lambda functions directly - input_bytes = cloudpickle.dumps((fn, output_filepath)) - - # cannot use `sys.executable __file__` here because the script - # contains relative imports - returned = subprocess.run( - _SUBPROCESS_COMMAND, input=input_bytes, capture_output=True - ) - - # check if the subprocess is successful - try: - returned.check_returncode() - except Exception as e: - # wrap raised exception to provide more information - raise RuntimeError( - f"Error raised in subprocess:\n{returned.stderr.decode()}" - ) from e - - with open(output_filepath, "rb") as f: - return cast(_T, pickle.load(f)) - - @dataclass(frozen=True) class _LazyRegisteredModel(_BaseRegisteredModel): """ @@ -246,12 +179,6 @@ class _LazyRegisteredModel(_BaseRegisteredModel): module_name: str class_name: str - # Performed in another process to avoid initializing CUDA - def inspect_model_cls(self) -> _ModelInfo: - return _run_in_subprocess( - lambda: _ModelInfo.from_model_cls(self.load_model_cls()) - ) - def load_model_cls(self) -> type[nn.Module]: mod = importlib.import_module(self.module_name) return cast(type[nn.Module], getattr(mod, self.class_name)) @@ -272,18 +199,6 @@ def _try_load_model_cls( return None -@lru_cache(maxsize=128) -def _try_inspect_model_cls( - model_arch: str, - model: _BaseRegisteredModel, -) -> _ModelInfo | None: - try: - return model.inspect_model_cls() - except Exception: - logger.exception("Error in inspecting model architecture '%s'", model_arch) - return None - - @dataclass class _ModelRegistry: # Keyed by model_arch @@ -335,7 +250,7 @@ class _ModelRegistry: module_name=split_str[0], class_name=split_str[1] ) else: - model = _RegisteredModel.from_model_cls(model_cls) + model = _RegisteredModel(model_cls=model_cls) self.registered_models[model_arch] = model @@ -359,12 +274,6 @@ class _ModelRegistry: return _try_load_model_cls(model_arch, self.registered_models[model_arch]) - def _try_inspect_model_cls(self, model_arch: str) -> _ModelInfo | None: - if model_arch not in self.registered_models: - return None - - return _try_inspect_model_cls(model_arch, self.registered_models[model_arch]) - def _normalize_archs( self, architectures: str | list[str], @@ -392,19 +301,6 @@ class _ModelRegistry: normalized_arch.append(arch) return normalized_arch - def inspect_model_cls( - self, - architectures: str | list[str], - ) -> tuple[_ModelInfo, str]: - architectures = self._normalize_archs(architectures) - - for arch in architectures: - model_info = self._try_inspect_model_cls(arch) - if model_info is not None: - return (model_info, arch) - - return self._raise_for_unsupported(architectures) - def resolve_model_cls( self, architectures: str | list[str], diff --git a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_minimax_h3_euler_ancestral.py b/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_minimax_h3_euler_ancestral.py deleted file mode 100644 index 81629fe147..0000000000 --- a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_minimax_h3_euler_ancestral.py +++ /dev/null @@ -1,213 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -from __future__ import annotations - -import math -from typing import Any - -import torch - - -def _require_finite_tensor(tensor: torch.Tensor, name: str) -> None: - if not bool(torch.isfinite(tensor).all().item()): - raise ValueError(f"{name} must be finite") - - -def _validate_unit_timestep(timestep: torch.Tensor, name: str) -> None: - if not isinstance(timestep, torch.Tensor): - raise ValueError(f"{name} must be a torch.Tensor") - if not torch.is_floating_point(timestep): - raise ValueError(f"{name} must be a floating point tensor") - _require_finite_tensor(timestep, name) - out_of_range = (timestep < 0) | (timestep > 1) - if bool(out_of_range.any().item()): - raise ValueError(f"{name} must be in [0, 1]") - - -def _validate_sigma(value: float, name: str) -> float: - sigma = float(value) - if not math.isfinite(sigma): - raise ValueError(f"{name} must be finite") - if sigma < 0.0: - raise ValueError(f"{name} must be non-negative") - return sigma - - -def _validate_timestep_sigma_pair( - timestep: torch.Tensor, - sigma_curr: float, - name: str, -) -> float: - _validate_unit_timestep(timestep, f"{name}_timestep") - sigma = _validate_sigma(sigma_curr, f"{name}_sigma_curr") - expected = 1.0 - timestep.detach().to(dtype=torch.float32) - actual = torch.full_like(expected, sigma) - if not torch.allclose(actual, expected, rtol=1e-5, atol=1e-5): - raise ValueError(f"{name}_sigma_curr must equal 1 - {name}_timestep") - return sigma - - -def minimax_h3_rf_v_to_x0( - xt: torch.Tensor, - v: torch.Tensor, - timestep: torch.Tensor, -) -> torch.Tensor: - if xt.shape != v.shape: - raise ValueError(f"xt and v shapes must match, got {xt.shape} vs {v.shape}") - if not torch.is_floating_point(xt): - raise ValueError("xt must be a floating point tensor") - if not torch.is_floating_point(v): - raise ValueError("v must be a floating point tensor") - _require_finite_tensor(xt, "xt") - _require_finite_tensor(v, "v") - _validate_unit_timestep(timestep, "timestep") - x0 = _minimax_h3_rf_v_to_x0(xt, v, timestep) - _require_finite_tensor(x0, "x0") - return x0 - - -def _minimax_h3_rf_v_to_x0( - xt: torch.Tensor, - v: torch.Tensor, - timestep: torch.Tensor, -) -> torch.Tensor: - cond_t = timestep.to(device=xt.device, dtype=xt.dtype) - while cond_t.ndim < xt.ndim: - cond_t = cond_t.unsqueeze(-1) - sigma_t = 1 - cond_t - return xt + sigma_t * v - - -def minimax_h3_euler_eta0_step( - state: torch.Tensor, - denoised: torch.Tensor, - *, - sigma_curr: float, - sigma_next: float, -) -> torch.Tensor: - if state.shape != denoised.shape: - raise ValueError( - f"state and denoised shapes must match, got {state.shape} vs " - f"{denoised.shape}" - ) - if not torch.is_floating_point(state): - raise ValueError("state must be a floating point tensor") - if not torch.is_floating_point(denoised): - raise ValueError("denoised must be a floating point tensor") - _require_finite_tensor(state, "state") - _require_finite_tensor(denoised, "denoised") - sigma_curr = _validate_sigma(sigma_curr, "sigma_curr") - sigma_next = _validate_sigma(sigma_next, "sigma_next") - if sigma_curr == 0.0 and sigma_next != 0.0: - raise ValueError("sigma_next must be 0 when sigma_curr is 0") - out = _minimax_h3_euler_eta0_step( - state, - denoised, - sigma_curr=sigma_curr, - sigma_next=sigma_next, - ) - _require_finite_tensor(out, "euler_eta0_step output") - return out - - -def _minimax_h3_euler_eta0_step( - state: torch.Tensor, - denoised: torch.Tensor, - *, - sigma_curr: float, - sigma_next: float, - sigma_ratio: torch.Tensor | None = None, -) -> torch.Tensor: - if sigma_curr == 0.0: - return state - compute_dtype = torch.float32 - if state.dtype not in (torch.float16, torch.bfloat16): - compute_dtype = state.dtype - if sigma_ratio is None: - sigma_curr_t = state.new_tensor(sigma_curr, dtype=compute_dtype) - sigma_next_t = state.new_tensor(sigma_next, dtype=compute_dtype) - ratio = sigma_next_t / sigma_curr_t - else: - ratio = sigma_ratio.to(device=state.device, dtype=compute_dtype) - out = ratio * state.to(dtype=compute_dtype) + (1.0 - ratio) * denoised.to( - dtype=compute_dtype - ) - return out.to(dtype=state.dtype) - - -class MiniMaxH3EulerAncestralEta0SchedulerAdapter: - def __init__(self, **config: Any) -> None: - if config: - raise ValueError( - f"{type(self).__name__} does not accept config fields: {sorted(config)}" - ) - - def set_shift(self, _flow_shift: float) -> None: - """Ignore flow shift, matching the previous loader-specific path.""" - - def step_denoising( - self, - *, - input_visual_latent: torch.Tensor, - input_audio_latent: torch.Tensor, - timestep: torch.Tensor, - noise_pred_visual: torch.Tensor, - noise_pred_audio: torch.Tensor, - sigma_curr: float, - sigma_next: float, - video_timestep: torch.Tensor | None = None, - audio_timestep: torch.Tensor | None = None, - video_sigma_curr: float | None = None, - video_sigma_next: float | None = None, - audio_sigma_curr: float | None = None, - audio_sigma_next: float | None = None, - ) -> dict[str, torch.Tensor]: - visual_timestep = timestep if video_timestep is None else video_timestep - audio_timestep = timestep if audio_timestep is None else audio_timestep - visual_sigma_curr = sigma_curr if video_sigma_curr is None else video_sigma_curr - visual_sigma_next = sigma_next if video_sigma_next is None else video_sigma_next - audio_sigma_curr = sigma_curr if audio_sigma_curr is None else audio_sigma_curr - audio_sigma_next = sigma_next if audio_sigma_next is None else audio_sigma_next - visual_sigma_curr = _validate_timestep_sigma_pair( - visual_timestep, - visual_sigma_curr, - "video", - ) - audio_sigma_curr = _validate_timestep_sigma_pair( - audio_timestep, - audio_sigma_curr, - "audio", - ) - - denoised_visual = minimax_h3_rf_v_to_x0( - input_visual_latent, - noise_pred_visual, - visual_timestep, - ) - denoised_audio = minimax_h3_rf_v_to_x0( - input_audio_latent, - noise_pred_audio, - audio_timestep, - ) - return { - "output_visual_latent": minimax_h3_euler_eta0_step( - input_visual_latent, - denoised_visual, - sigma_curr=visual_sigma_curr, - sigma_next=visual_sigma_next, - ), - "output_audio_latent": minimax_h3_euler_eta0_step( - input_audio_latent, - denoised_audio, - sigma_curr=audio_sigma_curr, - sigma_next=audio_sigma_next, - ), - } - - -EntryClass = MiniMaxH3EulerAncestralEta0SchedulerAdapter - -__all__ = [ - "MiniMaxH3EulerAncestralEta0SchedulerAdapter", - "minimax_h3_euler_eta0_step", - "minimax_h3_rf_v_to_x0", -] diff --git a/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_fm_modules.py b/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_fm_modules.py index e9ff876475..088a0c645b 100644 --- a/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_fm_modules.py +++ b/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_fm_modules.py @@ -1,15 +1,10 @@ # Modified for SGLang; see this directory's README.md for upstream source. -import logging import math -from functools import lru_cache -import numpy as np import torch import torch.nn as nn -logger = logging.getLogger(__name__) - def modulate(x, shift, scale=None): if shift is None: @@ -17,17 +12,6 @@ def modulate(x, shift, scale=None): return x * (1 + scale) + shift -class RMSNorm(nn.Module): - def __init__(self, dim: int, eps: float = 1e-5): - super().__init__() - self.eps = eps - self.weight = nn.Parameter(torch.ones(dim)) - - def forward(self, x: torch.Tensor) -> torch.Tensor: - output = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) - return output * self.weight - - class TimestepEmbedder(nn.Module): """ Embeds scalar timesteps into vector representations. @@ -96,88 +80,6 @@ class ResBlock(nn.Module): return x + gate_mlp * h -# class FinalLayer(nn.Module): - -# def __init__(self, model_channels, out_channels): -# super().__init__() -# self.norm_final = nn.LayerNorm(model_channels, elementwise_affine=False, eps=1e-6) -# self.linear = nn.Linear(model_channels, out_channels, bias=True) -# self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(model_channels, 2 * model_channels, bias=True)) - -# def forward(self, x, c): -# shift, scale = self.adaLN_modulation(c).chunk(2, dim=-1) -# x = modulate(self.norm_final(x), shift, scale) -# x = self.linear(x) -# return x - -# class SimpleMLPAdaLN(nn.Module): - -# def __init__(self, input_dim, out_dim, dim=1536, layers=12, mlp_ratio=1.0): -# super().__init__() -# self.input_dim = input_dim -# self.out_dim = out_dim -# self.dim = dim -# self.layers = layers -# self.mlp_ratio = mlp_ratio - -# self.time_embed = TimestepEmbedder(dim) -# self.input_proj = nn.Linear(input_dim, dim) - -# res_blocks = [] -# for _ in range(layers): -# res_blocks.append(ResBlock(dim, mlp_ratio)) -# self.res_blocks = nn.ModuleList(res_blocks) - -# self.final_layer = FinalLayer(dim, out_dim) - -# self.grad_checkpointing = False - -# self.initialize_weights() - -# def initialize_weights(self): -# def _basic_init(module): -# if isinstance(module, nn.Linear): -# torch.nn.init.xavier_uniform_(module.weight) -# if module.bias is not None: -# nn.init.constant_(module.bias, 0) - -# self.apply(_basic_init) - -# # Initialize timestep embedding MLP -# nn.init.normal_(self.time_embed.mlp[0].weight, std=0.02) -# nn.init.normal_(self.time_embed.mlp[2].weight, std=0.02) - -# # Zero-out adaLN modulation layers -# for block in self.res_blocks: -# nn.init.constant_(block.adaLN_modulation[-1].weight, 0) -# nn.init.constant_(block.adaLN_modulation[-1].bias, 0) - -# # Zero-out output layers -# nn.init.constant_(self.final_layer.adaLN_modulation[-1].weight, 0) -# nn.init.constant_(self.final_layer.adaLN_modulation[-1].bias, 0) -# nn.init.constant_(self.final_layer.linear.weight, 0) -# nn.init.constant_(self.final_layer.linear.bias, 0) - -# def forward(self, x, t): -# """ -# x.shape = (bsz, input_dim) -# t.shape = (bsz,) -# """ - -# x = self.input_proj(x) -# t = self.time_embed(t) - -# y = t - -# for block in self.res_blocks: -# if self.grad_checkpointing and self.training: -# x = checkpoint(block, x, y, use_reentrant=True) -# else: -# x = block(x, y) - -# return self.final_layer(x, y) - - class FlowMatchingHead(nn.Module): def __init__(self, input_dim, out_dim, dim=1536, layers=12, mlp_ratio=1.0): super(FlowMatchingHead, self).__init__() @@ -202,140 +104,25 @@ class FlowMatchingHead(nn.Module): return x -def precompute_freqs_cis_2d( - dim: int, height: int, width: int, theta: float = 10000.0, scale=16.0 -): - # assert H * H == end - # flat_patch_pos = torch.linspace(-1, 1, end) # N = end - x_pos = torch.linspace(0, scale, width) - y_pos = torch.linspace(0, scale, height) - y_pos, x_pos = torch.meshgrid(y_pos, x_pos, indexing="ij") - y_pos = y_pos.reshape(-1) - x_pos = x_pos.reshape(-1) - freqs = 1.0 / ( - theta ** (torch.arange(0, dim, 4)[: (dim // 4)].float() / dim) - ) # Hc/4 - x_freqs = torch.outer(x_pos, freqs).float() # N Hc/4 - y_freqs = torch.outer(y_pos, freqs).float() # N Hc/4 - x_cis = torch.polar(torch.ones_like(x_freqs), x_freqs) - y_cis = torch.polar(torch.ones_like(y_freqs), y_freqs) - freqs_cis = torch.cat( - [x_cis.unsqueeze(dim=-1), y_cis.unsqueeze(dim=-1)], dim=-1 - ) # N,Hc/4,2 - freqs_cis = freqs_cis.reshape(height * width, -1) - return freqs_cis - - -class NerfEmbedder(nn.Module): - def __init__(self, in_channels, hidden_size_input, max_freqs): - super().__init__() - self.max_freqs = max_freqs - self.hidden_size_input = hidden_size_input - self.embedder = nn.Sequential( - nn.Linear(in_channels + max_freqs**2, hidden_size_input, bias=True), - ) - - @lru_cache - def fetch_pos(self, patch_size, device, dtype): - pos = precompute_freqs_cis_2d( - self.max_freqs**2 * 2, patch_size, patch_size - ).real - pos = pos[None, :, :].to(device=device, dtype=dtype) - return pos - - def forward(self, inputs): - B, P2, C = inputs.shape - patch_size = int(P2**0.5) - device = inputs.device - dtype = inputs.dtype - dct = self.fetch_pos(patch_size, device, dtype) - dct = dct.repeat(B, 1, 1) - inputs = torch.cat([inputs, dct], dim=-1) - inputs = self.embedder(inputs) - return inputs - - class SimpleMLPAdaLN(nn.Module): - """ - The MLP for Diffusion Loss. - :param in_channels: channels in the input Tensor. - :param model_channels: base channel count for the model. - :param out_channels: channels in the output Tensor. - :param z_channels: channels in the condition. - :param num_res_blocks: number of residual blocks per downsample. - """ + """Timestep-conditioned MLP that maps LLM hidden states to flow-matching targets.""" - def __init__( - self, - in_channels, - model_channels, - out_channels, - z_channels, - num_res_blocks, - patch_size, - grad_checkpointing=False, - ): + def __init__(self, input_dim, out_dim, dim=1536, layers=12, mlp_ratio=1.0): super().__init__() + self.time_embed = TimestepEmbedder(dim) + self.input_proj = nn.Linear(input_dim, dim) + self.res_blocks = nn.ModuleList( + [ResBlock(dim, mlp_ratio) for _ in range(layers)] + ) + self.final_layer = FinalLayer(dim, out_dim) - self.in_channels = in_channels - self.model_channels = model_channels - self.out_channels = out_channels - self.num_res_blocks = num_res_blocks - self.grad_checkpointing = grad_checkpointing - self.patch_size = patch_size - - self.cond_embed = nn.Linear(z_channels, patch_size**2 * model_channels) - - self.input_proj = nn.Linear(in_channels, model_channels) - - res_blocks = [] - for i in range(num_res_blocks): - res_blocks.append( - ResBlock( - model_channels, - ) - ) - - self.res_blocks = nn.ModuleList(res_blocks) - self.final_layer = FinalLayer(model_channels, out_channels) - - self.initialize_weights() - - def initialize_weights(self): - def _basic_init(module): - if isinstance(module, nn.Linear): - torch.nn.init.xavier_uniform_(module.weight) - if module.bias is not None: - nn.init.constant_(module.bias, 0) - - self.apply(_basic_init) - - # Zero-out adaLN modulation layers - for block in self.res_blocks: - nn.init.constant_(block.adaLN_modulation[-1].weight, 0) - nn.init.constant_(block.adaLN_modulation[-1].bias, 0) - - # Zero-out output layers - nn.init.constant_(self.final_layer.linear.weight, 0) - nn.init.constant_(self.final_layer.linear.bias, 0) - - def forward(self, x, c): - """ - Apply the model to an input batch. - :param x: an [N x C] Tensor of inputs. - :param t: a 1-D batch of timesteps. - :param c: conditioning from AR transformer. - :return: an [N x C] Tensor of outputs. - """ + def forward(self, x, t): + # x: (N, input_dim), t: (N,) x = self.input_proj(x) - c = self.cond_embed(c) - - y = c.reshape(-1, self.patch_size**2, self.model_channels) - + y = self.time_embed(t) for block in self.res_blocks: x = block(x, y) - - return self.final_layer(x) + return self.final_layer(x, y) class FinalLayer(nn.Module): @@ -349,282 +136,14 @@ class FinalLayer(nn.Module): model_channels, elementwise_affine=False, eps=1e-6 ) self.linear = nn.Linear(model_channels, out_channels, bias=True) - - def forward(self, x): - x = self.norm_final(x) - x = self.linear(x) - return x - - -################################################################################# -# Sine/Cosine Positional Embedding Functions # -################################################################################# -# https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py - - -def get_2d_sincos_pos_embed( - embed_dim, grid_size, cls_token=False, extra_tokens=0, pe_interpolation=1.0 -): - """ - grid_size: int of the grid height and width - return: - pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token) - """ - grid_h = np.arange(grid_size, dtype=np.float32) / pe_interpolation - grid_w = np.arange(grid_size, dtype=np.float32) / pe_interpolation - grid = np.meshgrid(grid_w, grid_h) # here w goes first - grid = np.stack(grid, axis=0) - - grid = grid.reshape([2, 1, grid_size, grid_size]) - pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) - if cls_token and extra_tokens > 0: - pos_embed = np.concatenate( - [np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0 - ) - return pos_embed - - -def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): - assert embed_dim % 2 == 0 - - # use half of dimensions to encode grid_h - emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2) - emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2) - - emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) - return emb - - -def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): - """ - embed_dim: output dimension for each position - pos: a list of positions to be encoded: size (M,) - out: (M, D) - """ - assert embed_dim % 2 == 0 - omega = np.arange(embed_dim // 2, dtype=np.float64) - omega /= embed_dim / 2.0 - omega = 1.0 / 10000**omega # (D/2,) - - pos = pos.reshape(-1) # (M,) - out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product - - emb_sin = np.sin(out) # (M, D/2) - emb_cos = np.cos(out) # (M, D/2) - - emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) - return emb - - -# -------------------------------------------------------- -# Interpolate position embeddings for high-resolution -# References: -# DeiT: https://github.com/facebookresearch/deit -# -------------------------------------------------------- -def interpolate_pos_embed( - model_path, pe_key: str = "gen_pos_embed", new_len: int = 4096 -): - state_dict = torch.load(model_path, map_location="cpu") - - pos_embed_1d = state_dict[pe_key] - _, ori_len, embed_dim = pos_embed_1d.shape - - ori_size = int(ori_len**0.5) - new_size = int(new_len**0.5) - - if ori_size != new_size: - logger.info( - "Position interpolate from %dx%d to %dx%d" - % (ori_size, ori_size, new_size, new_size) - ) - pos_embed_2d = pos_embed_1d.reshape(-1, ori_size, ori_size, embed_dim).permute( - 0, 3, 1, 2 - ) - pos_embed_2d = torch.nn.functional.interpolate( - pos_embed_2d, size=(new_size, new_size), mode="bicubic", align_corners=False - ) - pos_embed_1d = pos_embed_2d.permute(0, 2, 3, 1).flatten(1, 2) - state_dict[pe_key] = pos_embed_1d - - torch.save(state_dict, model_path) - - -class PositionEmbedding(nn.Module): - def __init__(self, max_num_patch_per_side, hidden_size): - super().__init__() - self.max_num_patch_per_side = max_num_patch_per_side - self.hidden_size = hidden_size - self.pos_embed = nn.Parameter( - torch.zeros(max_num_patch_per_side**2, hidden_size), requires_grad=False - ) - self._init_weights() - - def _init_weights(self): - # Initialize (and freeze) pos_embed by sin-cos embedding: - pos_embed = get_2d_sincos_pos_embed( - self.hidden_size, self.max_num_patch_per_side - ) - self.pos_embed.data.copy_(torch.from_numpy(pos_embed).float()) - - def forward(self, position_ids): - return self.pos_embed[position_ids] - - -class ResidualConvBlock(nn.Module): - def __init__(self, channels: int): - super().__init__() - self.block = nn.Sequential( - nn.Conv2d(channels, channels, kernel_size=3, padding=1), - nn.SiLU(), - nn.Conv2d(channels, channels, kernel_size=3, padding=1), - ) - nn.init.zeros_(self.block[2].weight) - nn.init.zeros_(self.block[2].bias) - - def forward(self, x): - return x + self.block(x) - - -class PostConvSmoother(nn.Module): - def __init__(self, in_channels=3, hidden_channels=64, num_blocks=3): - super().__init__() - self.in_proj = nn.Conv2d(in_channels, hidden_channels, kernel_size=3, padding=1) - self.blocks = nn.Sequential( - *[ResidualConvBlock(hidden_channels) for _ in range(num_blocks)] - ) - self.out_proj = nn.Conv2d(hidden_channels, in_channels, kernel_size=1) - - nn.init.zeros_(self.out_proj.weight) - nn.init.zeros_(self.out_proj.bias) - - def forward(self, x): - h = self.in_proj(x) - h = self.blocks(h) - return x + self.out_proj(h) - - -class ProgressiveConvDecoder(nn.Module): - def __init__(self, hidden_dim=4096, out_channels=3): - super().__init__() - - # self.proj = nn.Linear(hidden_dim, 1024) - # self.act = nn.SiLU() - - self.up_blocks = nn.ModuleList( - [ - nn.Sequential( - nn.Upsample(scale_factor=2, mode="nearest"), - nn.Conv2d(hidden_dim, 512, kernel_size=3, padding=1), - nn.GroupNorm(32, 512), - nn.SiLU(), - ), - nn.Sequential( - nn.Upsample(scale_factor=2, mode="nearest"), - nn.Conv2d(512, 256, kernel_size=3, padding=1), - nn.GroupNorm(32, 256), - nn.SiLU(), - ), - nn.Sequential( - nn.Upsample(scale_factor=2, mode="nearest"), - nn.Conv2d(256, 64, kernel_size=3, padding=1), - nn.GroupNorm(32, 64), - nn.SiLU(), - ), - nn.Sequential( - nn.Upsample(scale_factor=2, mode="nearest"), - nn.Conv2d(64, 32, kernel_size=3, padding=1), - nn.GroupNorm(16, 32), - nn.SiLU(), - ), - nn.Sequential( - nn.Upsample(scale_factor=2, mode="nearest"), - nn.Conv2d(32, 16, kernel_size=3, padding=1), - nn.SiLU(), - ), - ] + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), nn.Linear(model_channels, 2 * model_channels, bias=True) ) - self.out_conv = nn.Conv2d(16, out_channels, kernel_size=3, padding=1) - - def forward(self, x_2d): - # B, C, H, W = x_2d.shape - # x = x_2d.permute(0, 2, 3, 1).contiguous() # (B, H, W, C) - # x = self.proj(x) - # x = self.act(x) - # x = x.permute(0, 3, 1, 2).contiguous() # (B, 512, H, W) - x = x_2d - for block in self.up_blocks: - x = block(x) - - out = self.out_conv(x) - return out - - -class PatchDecoder_postps(nn.Module): - def __init__(self): - super().__init__() - # layer 1: H/32 -> H/8 (4x upscale) - - self.conv1 = nn.Conv2d(4096, 4096, kernel_size=3, padding=1) - self.ps1 = nn.PixelShuffle(4) - self.act1 = nn.GELU() - - # layer 2: H/8 -> H (8x upscale) - self.conv2 = nn.Conv2d(256, 192, kernel_size=3, padding=1) - self.ps2 = nn.PixelShuffle(8) - - def forward(self, x): - # x shape: [B, 4096, H/32, W/32] - x = self.ps1(self.act1(self.conv1(x))) # -> [B, 256, H/8, W/8] - x = self.ps2(self.conv2(x)) # -> [B, 3, H, W] - return x - - -class PatchDecoder_preps(nn.Module): - def __init__(self): - super().__init__() - # layer 1: H/32 -> H/16 (2x upscale) - self.ps1 = nn.PixelShuffle(2) - self.conv1 = nn.Conv2d(1024, 1024, kernel_size=3, padding=1) - self.act1 = nn.GELU() - - # layer 2: H/16 -> H/8 (2x upscale) - self.ps2 = nn.PixelShuffle(2) - self.conv2 = nn.Conv2d(256, 256, kernel_size=3, padding=1) - self.act2 = nn.GELU() - - # layer 3: H/8 -> H (8x upscale) - self.ps3 = nn.PixelShuffle(8) - self.conv3 = nn.Conv2d(4, 3, kernel_size=3, padding=1) - - def forward(self, x): - # x shape: [B, 4096, H/32, W/32] - x = self.act1(self.conv1(self.ps1((x)))) # -> [B, 256, H/16, W/16] - x = self.act2(self.conv2(self.ps2((x)))) # -> [B, 256, H/8, W/8] - x = self.conv3(self.ps3((x))) # -> [B, 3, H, W] - return x - - -class PatchDecoder_preps1(nn.Module): - def __init__(self): - super().__init__() - # layer 1: H/32 -> H/16 (2x upscale) - self.ps1 = nn.PixelShuffle(2) - self.conv1 = nn.Conv2d(1024, 1024, kernel_size=3, padding=1) - self.act1 = nn.GELU() - - # layer 2: H/16 -> H/8 (2x upscale) - self.ps2 = nn.PixelShuffle(2) - self.conv2 = nn.Conv2d(256, 192, kernel_size=3, padding=1) - - # layer 3: H/8 -> H (8x upscale) - self.ps3 = nn.PixelShuffle(8) - - def forward(self, x): - # x shape: [B, 4096, H/32, W/32] - x = self.act1(self.conv1(self.ps1((x)))) # -> [B, 256, H/16, W/16] - x = self.ps3(self.conv2(self.ps2((x)))) # -> [B, 256, H/8, W/8] - return x + def forward(self, x, c): + shift, scale = self.adaLN_modulation(c).chunk(2, dim=-1) + x = modulate(self.norm_final(x), shift, scale) + return self.linear(x) class ConvDecoder(nn.Module): diff --git a/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_neo_chat.py b/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_neo_chat.py index 04fc74888d..5c3838468d 100644 --- a/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_neo_chat.py +++ b/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_neo_chat.py @@ -7,14 +7,10 @@ from typing import List, Optional, Tuple, Union import torch.utils.checkpoint import transformers from torch import nn -from torch.nn import CrossEntropyLoss -from transformers import GenerationConfig from transformers.modeling_outputs import CausalLMOutputWithPast from transformers.modeling_utils import PreTrainedModel from transformers.utils import logging -from sglang.multimodal_gen.runtime.cache.conditioning import cached_conditioning - from .configuration_neo_chat import NEOChatConfig, NEOMoELLMConfig from .conversation import get_conv_template from .modeling_fm_modules import ( @@ -22,14 +18,14 @@ from .modeling_fm_modules import ( FlowMatchingHead, TimestepEmbedder, ) -from .modeling_neo_vit import NEOVisionModel, build_abs_positions_from_grid_hw +from .modeling_neo_vit import NEOVisionModel from .modeling_qwen3 import ( Qwen3ForCausalLM, create_block_causal_mask, npu_fia_available, ) from .modeling_qwen3_moe import Qwen3MoeForCausalLM -from .utils import SYSTEM_MESSAGE_FOR_GEN, load_image_native +from .utils import SYSTEM_MESSAGE_FOR_GEN logger = logging.get_logger(__name__) @@ -335,8 +331,6 @@ class NEOChatModel(PreTrainedModel): noise_scale_embedder = TimestepEmbedder(llm_hidden_size) self.fm_modules["noise_scale_embedder"] = noise_scale_embedder - self.img_context_token_id = None - self.img_start_token_id = 151670 self.last_think_content = "" self.conv_template = get_conv_template(self.template) self.system_message = self.conv_template.system_message @@ -367,164 +361,15 @@ class NEOChatModel(PreTrainedModel): return_dict: Optional[bool] = None, ) -> Union[Tuple, CausalLMOutputWithPast]: raise NotImplementedError("forward") - return_dict = ( - return_dict if return_dict is not None else self.config.use_return_dict - ) - image_flags = image_flags.squeeze(-1) - input_embeds = self.language_model.get_input_embeddings()(input_ids).clone() - - vit_embeds = self.extract_feature(pixel_values) - vit_embeds = vit_embeds[image_flags == 1] - - B, N, C = input_embeds.shape - input_embeds = input_embeds.reshape(B * N, C) - - # if torch.distributed.is_initialized() and torch.distributed.get_rank() == 0: - # print(f'dynamic ViT batch size: {vit_batch_size}, images per sample: {vit_batch_size / B}, dynamic token length: {N}') - - input_ids = input_ids.reshape(B * N) - selected = input_ids == self.img_context_token_id - try: - input_embeds[selected] = input_embeds[selected] * 0.0 + vit_embeds.reshape( - -1, C - ) - except Exception as e: - vit_embeds = vit_embeds.reshape(-1, C) - print( - f"warning: {e}, input_embeds[selected].shape={input_embeds[selected].shape}, " - f"vit_embeds.shape={vit_embeds.shape}" - ) - n_token = min(selected.sum(), vit_embeds.size(0)) - input_embeds[selected][:n_token] = ( - input_embeds[selected][:n_token] * 0.0 + vit_embeds[:n_token] - ) - - input_embeds = input_embeds.reshape(B, N, C) - - outputs = self.language_model( - inputs_embeds=input_embeds, - attention_mask=attention_mask, - position_ids=position_ids, - past_key_values=past_key_values, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - logits = outputs.logits - - loss = None - if labels is not None: - # Shift so that tokens < n predict n - shift_logits = logits[..., :-1, :].contiguous() - shift_labels = labels[..., 1:].contiguous() - # Flatten the tokens - loss_fct = CrossEntropyLoss() - shift_logits = shift_logits.view(-1, self.language_model.config.vocab_size) - shift_labels = shift_labels.view(-1) - # Enable model parallelism - shift_labels = shift_labels.to(shift_logits.device) - loss = loss_fct(shift_logits, shift_labels) - - if not return_dict: - output = (logits,) + outputs[1:] - return (loss,) + output if loss is not None else output - - return CausalLMOutputWithPast( - loss=loss, - logits=logits, - past_key_values=outputs.past_key_values, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - ) - - def extract_feature(self, pixel_values, gen_model=False, grid_hw=None): - if gen_model: - return self.fm_modules["vision_model_mot_gen"]( - pixel_values=pixel_values, - output_hidden_states=False, - return_dict=True, - grid_hw=grid_hw, - ).last_hidden_state - else: - return self._reference_image_features(pixel_values, grid_hw) - - @cached_conditioning - def _reference_image_features(self, pixel_values, grid_hw): - return self.vision_model( + def extract_feature(self, pixel_values, grid_hw=None): + return self.fm_modules["vision_model_mot_gen"]( pixel_values=pixel_values, output_hidden_states=False, return_dict=True, grid_hw=grid_hw, ).last_hidden_state - def batch_chat( - self, - tokenizer, - pixel_values, - questions, - generation_config, - num_patches_list=None, - history=None, - return_history=False, - IMG_START_TOKEN="", - IMG_END_TOKEN="", - IMG_CONTEXT_TOKEN="", - verbose=False, - image_counts=None, - ): - raise NotImplementedError("batch_chat") - if history is not None or return_history: - print("Now multi-turn chat is not supported in batch_chat.") - raise NotImplementedError - - if image_counts is not None: - num_patches_list = image_counts - print( - "Warning: `image_counts` is deprecated. Please use `num_patches_list` instead." - ) - - img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN) - self.img_context_token_id = img_context_token_id - - if verbose and pixel_values is not None: - image_bs = pixel_values.shape[0] - print(f"dynamic ViT batch size: {image_bs}") - - queries = [] - for idx, num_patches in enumerate(num_patches_list): - question = questions[idx] - if pixel_values is not None and "" not in question: - question = "\n" + question - template = get_conv_template(self.template) - template.system_message = self.system_message - template.append_message(template.roles[0], question) - template.append_message(template.roles[1], None) - query = template.get_prompt() - - image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN + IMG_END_TOKEN - query = query.replace("", image_tokens, 1) - queries.append(query) - - tokenizer.padding_side = "left" - model_inputs = tokenizer(queries, return_tensors="pt", padding=True) - input_ids = model_inputs["input_ids"].to(self.device) - attention_mask = model_inputs["attention_mask"].to(self.device) - eos_token_id = tokenizer.convert_tokens_to_ids(template.sep.strip()) - generation_config["eos_token_id"] = eos_token_id - generation_output = self.generate( - pixel_values=pixel_values, - input_ids=input_ids, - attention_mask=attention_mask, - **generation_config, - ) - responses = tokenizer.batch_decode(generation_output, skip_special_tokens=True) - responses = [ - response.split(template.sep.strip())[0].strip() for response in responses - ] - return responses - def patchify(self, images, patch_size, channel_first=False): """ images: (N, 3, H, W) @@ -556,10 +401,6 @@ class NEOChatModel(PreTrainedModel): images = x.reshape(shape=(x.shape[0], 3, h * patch_size, w * patch_size)) return images - def _euler_step(self, v_pred, z, t, t_next): - z_next = z + (t_next - t) * v_pred - return z_next - def _calculate_dynamic_mu(self, image_seq_len: int) -> float: denom = self.max_image_seq_len - self.base_image_seq_len if denom == 0: @@ -675,20 +516,6 @@ class NEOChatModel(PreTrainedModel): ) return out.past_key_values, out.last_hidden_state - def _it2i_prefix_forward( - self, input_imbeds, indexes, attention_mask, gen_indicators=None - ): - out = self.language_model.model( - inputs_embeds=input_imbeds, - indexes=indexes, - attention_mask=attention_mask, - use_cache=True, - image_gen_indicators=( - gen_indicators.view(1, -1) if gen_indicators is not None else None - ), - ) - return out.past_key_values, out.last_hidden_state - def _think_prefix_forward(self, **kwargs): """Build the Think prefix cache without materialising per-token logits.""" return self.language_model(use_cache=True, logits_to_keep=1, **kwargs) @@ -849,1531 +676,6 @@ class NEOChatModel(PreTrainedModel): v_pred = (x_pred - z) / (1 - t).clamp_min(self.config.t_eps) return v_pred - def _build_it2i_inputs(self, tokenizer, query, pixel_values=None, grid_hw=None): - model_inputs = tokenizer(query, return_tensors="pt") - input_ids = model_inputs["input_ids"].to(self.device) - - indexes = self.get_thw_indexes(input_ids[0], grid_hw) - - attention_mask = {"full_attention": create_block_causal_mask(indexes[0])} - - input_embeds = self.language_model.get_input_embeddings()(input_ids) - B, N, C = input_embeds.shape - if pixel_values is not None: - vit_embeds = self.extract_feature(pixel_values, grid_hw=grid_hw) - input_embeds = input_embeds.reshape(B * N, C) - input_ids = input_ids.reshape(B * N) - selected = input_ids == self.img_context_token_id - assert selected.sum() != 0 - input_embeds[selected] = vit_embeds.reshape(-1, C).to(input_embeds.device) - input_embeds = input_embeds.reshape(B, N, C) - - return input_embeds, indexes, attention_mask - - @torch.no_grad() - def interleave_gen_image_only( - self, - tokenizer, - prompt, - gt_text, - images=None, - gt_images=None, - cfg_scale=1.0, - img_cfg_scale=1.0, - cfg_norm="none", - max_images=10, - enable_timestep_shift=True, - timestep_shift=1.0, - image_size=(256, 256), - num_steps=30, - IMG_START_TOKEN="", - IMG_END_TOKEN="", - IMG_CONTEXT_TOKEN="", - method="euler", - cfg_interval=(0, 1), - t_eps=0.02, - verbose=False, - system_message="", - ): - self.img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN) - self.img_start_token_id = tokenizer.convert_tokens_to_ids(IMG_START_TOKEN) - self.config.t_eps = t_eps - - if isinstance(image_size, tuple): - image_size_list = [image_size] * max_images - elif isinstance(image_size, list) and isinstance(image_size[0], tuple): - image_size_list = image_size - if len(image_size) < max_images: - image_size_list += [image_size_list[-1]] * ( - max_images - len(image_size_list) - ) - else: - assert False, "image size should be a tuple or a list of tuple" - - if images is None: - images = [] - - image_token_count = prompt.count("") - assert len(images) >= image_token_count - if len(images) > image_token_count: - prompt = "\n" * (len(images) - image_token_count) + prompt - - pixel_values = [] - grid_hw = [] - for image in images: - cur_pixel_values, cur_grid_hw = load_image_native( - image, - self.patch_size, - self.downsample_ratio, - min_pixels=512 * 512, - max_pixels=min(2048 * 2048, (4096 * 4096) // max(1, len(images))), - upscale=False, - ) - grid_hw.append(cur_grid_hw.to(self.device)) - pixel_values.append(cur_pixel_values.to(self.device).to(torch.bfloat16)) - - merge_size = int(1 / self.downsample_ratio) - pv_tensor = torch.cat(pixel_values) if pixel_values else None - ghw_tensor = torch.cat(grid_hw) if grid_hw else None - - # Condition Initial Cache - template_cond = get_conv_template(self.template) - template_cond.system_message = system_message - template_cond.append_message(template_cond.roles[0], prompt) - template_cond.append_message(template_cond.roles[1], None) - query_cond = template_cond.get_prompt() + "\n\n\n\n" - - def replace_image_tokens(query, grid_hw_list): - for i in range(len(grid_hw_list)): - num_patch_token = int( - grid_hw_list[i][0, 0] - * grid_hw_list[i][0, 1] - * self.downsample_ratio**2 - ) - image_tokens = ( - IMG_START_TOKEN - + IMG_CONTEXT_TOKEN * num_patch_token - + IMG_END_TOKEN - ) - query = query.replace("", image_tokens, 1) - return query - - query_cond = replace_image_tokens(query_cond, grid_hw) - input_embeds_cond, indexes_cond, attention_mask_cond = self._build_it2i_inputs( - tokenizer, query_cond, pv_tensor, ghw_tensor - ) - - outputs_cond = self.language_model( - inputs_embeds=input_embeds_cond, - indexes=indexes_cond, - attention_mask=attention_mask_cond, - use_cache=True, - ) - past_key_values_cond = outputs_cond.past_key_values - t_index_cond = indexes_cond[0].max().item() - - # Text Uncondition Cache Initial - question_text_uncondition = "" * len(images) - template_tu = get_conv_template(self.template) - template_tu.system_message = self.system_message - template_tu.append_message(template_tu.roles[0], question_text_uncondition) - template_tu.append_message(template_tu.roles[1], None) - query_text_uncond = template_tu.get_prompt() - query_text_uncond = replace_image_tokens(query_text_uncond, grid_hw) - - input_embeds_tu, indexes_tu, attention_mask_tu = self._build_it2i_inputs( - tokenizer, query_text_uncond, pv_tensor, ghw_tensor - ) - outputs_tu = self.language_model( - inputs_embeds=input_embeds_tu, - indexes=indexes_tu, - attention_mask=attention_mask_tu, - use_cache=True, - ) - past_key_values_tu = outputs_tu.past_key_values - t_index_tu = indexes_tu[0].max().item() - - # Img Uncondition Cache Initial - query_img_uncond = self._build_t2i_query("", append_text=IMG_START_TOKEN) - input_embeds_iu, indexes_iu, attention_mask_iu = self._build_it2i_inputs( - tokenizer, query_img_uncond - ) - outputs_iu = self.language_model( - inputs_embeds=input_embeds_iu, - indexes=indexes_iu, - attention_mask=attention_mask_iu, - use_cache=True, - ) - past_key_values_iu = outputs_iu.past_key_values - - generated_images = [] - img_count = 0 - device = self.device - - def append_ids_to_cache(cache, t_idx, input_ids): - if input_ids.shape[1] == 0: - return t_idx - seq_len = input_ids.shape[1] - inputs_embeds = self.language_model.get_input_embeddings()(input_ids) - - t_indexes = torch.arange( - t_idx + 1, t_idx + 1 + seq_len, dtype=torch.long, device=device - ) - h_indexes = torch.zeros(seq_len, dtype=torch.long, device=device) - w_indexes = torch.zeros(seq_len, dtype=torch.long, device=device) - indexes = torch.stack([t_indexes, h_indexes, w_indexes], dim=0) - - past_len = cache.get_seq_length() - mask = torch.zeros(1, 1, seq_len, past_len + seq_len, device=device) - causal_mask = torch.tril(torch.ones(seq_len, seq_len, device=device)) - causal_mask = torch.where(causal_mask == 1, 0.0, float("-inf")) - mask[:, :, :, past_len:] = causal_mask - attention_mask_dict = {"full_attention": mask} - - self.language_model( - inputs_embeds=inputs_embeds, - indexes=indexes, - attention_mask=attention_mask_dict, - past_key_values=cache, - use_cache=True, - ) - return t_idx + seq_len - - def append_image_to_cache( - cache, t_idx, inputs_embeds_img, N_img_tokens, abs_pos_w, abs_pos_h - ): - past_len = cache.get_seq_length() - tgt_len = N_img_tokens + 1 - - t_indexes = torch.zeros(tgt_len, dtype=torch.long, device=device) - t_indexes[:N_img_tokens] = t_idx + 1 - t_indexes[N_img_tokens] = t_idx + 2 - - h_indexes = torch.zeros(tgt_len, dtype=torch.long, device=device) - w_indexes = torch.zeros(tgt_len, dtype=torch.long, device=device) - h_indexes[:N_img_tokens] = abs_pos_h - w_indexes[:N_img_tokens] = abs_pos_w - - indexes = torch.stack([t_indexes, h_indexes, w_indexes], dim=0) - - mask = torch.zeros(1, 1, tgt_len, past_len + tgt_len, device=device) - mask[0, 0, :N_img_tokens, past_len + N_img_tokens] = float("-inf") - attention_mask_dict = {"full_attention": mask} - - self.language_model( - inputs_embeds=inputs_embeds_img, - indexes=indexes, - attention_mask=attention_mask_dict, - past_key_values=cache, - use_cache=True, - ) - return t_idx + 2 - - parts = gt_text.split("") - img_start_id_tensor = torch.tensor([[self.img_start_token_id]], device=device) - - for i, part in enumerate(parts): - if len(part) > 0: - if verbose: - print(part, end="", flush=True) - part_ids = tokenizer( - part, return_tensors="pt", add_special_tokens=False - )["input_ids"].to(device) - t_index_cond = append_ids_to_cache( - past_key_values_cond, t_index_cond, part_ids - ) - - if i < len(parts) - 1: - if img_count >= max_images: - break - - if verbose: - print("", end="", flush=True) - - t_index_cond = append_ids_to_cache( - past_key_values_cond, t_index_cond, img_start_id_tensor - ) - t_index_tu = append_ids_to_cache( - past_key_values_tu, t_index_tu, img_start_id_tensor - ) - - cur_image_size = image_size_list[img_count] - token_h = cur_image_size[1] // (self.patch_size * merge_size) - token_w = cur_image_size[0] // (self.patch_size * merge_size) - - indexes_image_condition = self._build_t2i_image_indexes( - token_h, token_w, t_index_cond + 1, device=device - ) - indexes_image_text_uncondition = self._build_t2i_image_indexes( - token_h, token_w, t_index_tu + 1, device=device - ) - indexes_image_img_uncondition = self._build_t2i_image_indexes( - token_h, token_w, indexes_iu[0].max() + 1, device=device - ) - - grid_h = cur_image_size[1] // self.patch_size - grid_w = cur_image_size[0] // self.patch_size - gen_grid_hw = torch.tensor([[grid_h, grid_w]], device=device) - - noise_scale = self.noise_scale - if self.noise_scale_mode in ("resolution", "dynamic", "dynamic_sqrt"): - noise_scale = math.sqrt( - (grid_h * grid_w) - / (merge_size**2) - / self.noise_scale_base_image_seq_len - ) - base = float(self.noise_scale_base_image_seq_len) - noise_scale = math.sqrt( - (grid_h * grid_w) / (merge_size**2) / base - ) * float(self.noise_scale) - if self.noise_scale_mode == "dynamic_sqrt": - noise_scale = math.sqrt(noise_scale) - noise_scale = min(noise_scale, self.noise_scale_max_value) - image_prediction = noise_scale * torch.randn( - (1, 3, cur_image_size[1], cur_image_size[0]), - device=device, - dtype=outputs_cond.logits.dtype, - ) - - past_key_values_cond_cfg = past_key_values_cond - past_key_values_tu_cfg = past_key_values_tu - past_key_values_iu_cfg = past_key_values_iu - - # attention_mask_condition = {"full_attention": torch.zeros(1, 1, token_h*token_w, past_key_values_cond.get_seq_length() + token_h*token_w, device=device)} - # attention_mask_text_uncondition = {"full_attention": torch.zeros(1, 1, token_h*token_w, past_key_values_tu.get_seq_length() + token_h*token_w, device=device)} - # attention_mask_img_uncondition = {"full_attention": torch.zeros(1, 1, token_h*token_w, past_key_values_iu.get_seq_length() + token_h*token_w, device=device)} - attention_mask_condition = {"full_attention": None} - attention_mask_text_uncondition = {"full_attention": None} - attention_mask_img_uncondition = {"full_attention": None} - - prepare_flash_kv_cache( - past_key_values_cond_cfg, - current_len=token_h * token_w, - batch_size=1, - ) - prepare_flash_kv_cache( - past_key_values_tu_cfg, - current_len=token_h * token_w, - batch_size=1, - ) - prepare_flash_kv_cache( - past_key_values_iu_cfg, - current_len=token_h * token_w, - batch_size=1, - ) - - timesteps = torch.linspace(0.0, 1.0, num_steps + 1, device=device) - if enable_timestep_shift: - timesteps = self._apply_time_schedule( - timesteps, token_h * token_w, timestep_shift - ) - - step_iter = range(num_steps) - if verbose: - try: - from tqdm import tqdm as _tqdm - - step_iter = _tqdm( - step_iter, - desc=f"image {img_count + 1} ({image_size[0]}x{image_size[1]})", - total=num_steps, - leave=False, - ) - except ImportError: - pass - for step_i in step_iter: - t = timesteps[step_i] - t_next = timesteps[step_i + 1] - - z = self.patchify(image_prediction, self.patch_size * merge_size) - image_input = self.patchify( - image_prediction, self.patch_size, channel_first=True - ) - image_embeds = self.extract_feature( - image_input.view(1 * grid_h * grid_w, -1), - gen_model=True, - grid_hw=gen_grid_hw, - ).view(1, token_h * token_w, -1) - t_expanded = t.expand(token_h * token_w) - timestep_embeddings = self.fm_modules["timestep_embedder"]( - t_expanded - ).view(1, token_h * token_w, -1) - if self.add_noise_scale_embedding: - noise_scale_tensor = torch.full_like( - t_expanded, noise_scale / self.noise_scale_max_value - ) - noise_embeddings = self.fm_modules["noise_scale_embedder"]( - noise_scale_tensor - ).view(1, token_h * token_w, -1) - timestep_embeddings += noise_embeddings - image_embeds = image_embeds + timestep_embeddings - - use_cfg = ( - t > cfg_interval[0] and t < cfg_interval[1] - ) or cfg_interval[0] == 0 - out_cond = self._t2i_predict_v( - image_embeds, - indexes_image_condition, - attention_mask_condition, - past_key_values_cond_cfg, - t, - z, - image_token_num=token_h * token_w, - timestep_embeddings=timestep_embeddings, - image_size=cur_image_size, - ) - if not use_cfg: - v_pred = out_cond - elif cfg_scale == 1 and img_cfg_scale == 1: - v_pred = out_cond - elif img_cfg_scale == 1: - out_img_cond = self._t2i_predict_v( - image_embeds, - indexes_image_text_uncondition, - attention_mask_text_uncondition, - past_key_values_tu_cfg, - t, - z, - image_token_num=token_h * token_w, - timestep_embeddings=timestep_embeddings, - image_size=cur_image_size, - ) - v_pred = out_img_cond + cfg_scale * (out_cond - out_img_cond) - elif cfg_scale == img_cfg_scale: - out_uncond = self._t2i_predict_v( - image_embeds, - indexes_image_img_uncondition, - attention_mask_img_uncondition, - past_key_values_iu_cfg, - t, - z, - image_token_num=token_h * token_w, - timestep_embeddings=timestep_embeddings, - image_size=cur_image_size, - ) - v_pred = out_uncond + cfg_scale * (out_cond - out_uncond) - else: - out_img_cond = self._t2i_predict_v( - image_embeds, - indexes_image_text_uncondition, - attention_mask_text_uncondition, - past_key_values_tu_cfg, - t, - z, - image_token_num=token_h * token_w, - timestep_embeddings=timestep_embeddings, - image_size=cur_image_size, - ) - out_uncond = self._t2i_predict_v( - image_embeds, - indexes_image_img_uncondition, - attention_mask_img_uncondition, - past_key_values_iu_cfg, - t, - z, - image_token_num=token_h * token_w, - timestep_embeddings=timestep_embeddings, - image_size=cur_image_size, - ) - v_pred = ( - out_uncond - + cfg_scale * (out_cond - out_img_cond) - + img_cfg_scale * (out_img_cond - out_uncond) - ) - if (cfg_scale > 1 or img_cfg_scale > 1) and use_cfg: - if cfg_norm == "global": - norm_v_condition = torch.norm( - out_cond, dim=(1, 2), keepdim=True - ) - norm_v_cfg = torch.norm(v_pred, dim=(1, 2), keepdim=True) - scale = (norm_v_condition / (norm_v_cfg + 1e-8)).clamp( - min=0, max=1.0 - ) - v_pred = v_pred * scale - elif cfg_norm == "channel": - norm_v_condition = torch.norm( - out_cond, dim=-1, keepdim=True - ) - norm_v_cfg = torch.norm(v_pred, dim=-1, keepdim=True) - scale = (norm_v_condition / (norm_v_cfg + 1e-8)).clamp( - min=0, max=1.0 - ) - v_pred = v_pred * scale - - z = z + (t_next - t) * v_pred - image_prediction = self.unpatchify( - z, - self.patch_size * merge_size, - cur_image_size[1], - cur_image_size[0], - ) - - generated_images.append(image_prediction) - - clear_flash_kv_cache(past_key_values_cond_cfg) - clear_flash_kv_cache(past_key_values_tu_cfg) - clear_flash_kv_cache(past_key_values_iu_cfg) - - if gt_images is not None and img_count < len(gt_images): - gt_img_pil = gt_images[img_count] - gt_pixel_values, gt_grid_hw = load_image_native( - gt_img_pil, - self.patch_size, - self.downsample_ratio, - min_pixels=512 * 512, - max_pixels=(2048 * 2048), - upscale=False, - ) - gt_pixel_values = gt_pixel_values.to(device).to(torch.bfloat16) - - flatten_pixel_values = gt_pixel_values - gen_grid_hw_und = gt_grid_hw - else: - pred_img = image_prediction[0].unsqueeze(0).to(torch.bfloat16) - raw_img = pred_img * 0.5 + 0.5 - img_mean = torch.tensor( - [0.485, 0.456, 0.406], dtype=raw_img.dtype, device=device - ).view(1, 3, 1, 1) - img_std = torch.tensor( - [0.229, 0.224, 0.225], dtype=raw_img.dtype, device=device - ).view(1, 3, 1, 1) - und_img = (raw_img - img_mean) / img_std - - c, h, w = und_img[0].shape - ps = self.patch_size - p_grid_h = h // ps - p_grid_w = w // ps - flatten_pixel_values = ( - und_img[0] - .view(c, p_grid_h, ps, p_grid_w, ps) - .permute(1, 3, 0, 2, 4) - .reshape(p_grid_h * p_grid_w, c * ps**2) - ) - gen_grid_hw_und = torch.tensor( - [[p_grid_h, p_grid_w]], device=device - ) - - vit_embeds = self.extract_feature( - flatten_pixel_values, grid_hw=gen_grid_hw_und[:1] - ).unsqueeze(0) - - img_end_id = tokenizer.convert_tokens_to_ids(IMG_END_TOKEN) - img_end_embed = self.language_model.get_input_embeddings()( - torch.tensor([[img_end_id]], device=device) - ) - inputs_embeds_img = torch.cat( - [vit_embeds, img_end_embed], dim=1 - ) # (1, N + 1, C) - - N_img_tokens = vit_embeds.shape[1] - abs_pos_w, abs_pos_h = build_abs_positions_from_grid_hw( - gen_grid_hw_und[:1] // int(1 / self.downsample_ratio), device=device - ) - - t_index_cond = append_image_to_cache( - past_key_values_cond, - t_index_cond, - inputs_embeds_img, - N_img_tokens, - abs_pos_w, - abs_pos_h, - ) - t_index_tu = append_image_to_cache( - past_key_values_tu, - t_index_tu, - inputs_embeds_img, - N_img_tokens, - abs_pos_w, - abs_pos_h, - ) - - img_count += 1 - - return generated_images - - @torch.no_grad() - def interleave_gen( - self, - tokenizer, - prompt, - images=None, - generation_config=None, - cfg_scale=1.0, - img_cfg_scale=1.0, - cfg_norm="none", - max_images=10, - enable_timestep_shift=True, - timestep_shift=1.0, - image_size=(256, 256), - num_steps=30, - IMG_START_TOKEN="", - IMG_END_TOKEN="", - IMG_CONTEXT_TOKEN="", - method="euler", - cfg_interval=(0, 1), - t_eps=0.02, - verbose=False, - system_message="", - think_mode=False, - seed=0, - ): - self.img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN) - self.img_start_token_id = tokenizer.convert_tokens_to_ids(IMG_START_TOKEN) - self.config.t_eps = t_eps - - if isinstance(image_size, tuple): - image_size_list = [image_size] * max_images - elif isinstance(image_size, list) and isinstance(image_size[0], tuple): - image_size_list = image_size - if len(image_size) < max_images: - image_size_list += [image_size_list[-1]] * ( - max_images - len(image_size_list) - ) - else: - assert False, "image size should be a tuple or a list of tuple" - - if ( - generation_config - and hasattr(generation_config, "max_new_tokens") - and generation_config.max_new_tokens is not None - ): - max_new_tokens = generation_config.max_new_tokens - else: - max_new_tokens = 8192 - - current_generated_tokens = 0 - - if images is None: - images = [] - - template = get_conv_template(self.template) - template.system_message = self.system_message - eos_token_id = tokenizer.convert_tokens_to_ids(template.sep.strip()) - - image_token_count = prompt.count("") - assert len(images) >= image_token_count - if len(images) > image_token_count: - prompt = "\n" * (len(images) - image_token_count) + prompt - - pixel_values = [] - grid_hw = [] - for image in images: - cur_pixel_values, cur_grid_hw = load_image_native( - image, - self.patch_size, - self.downsample_ratio, - min_pixels=512 * 512, - max_pixels=min(2048 * 2048, (4096 * 4096) // max(1, len(images))), - upscale=False, - ) - grid_hw.append(cur_grid_hw.to(self.device)) - pixel_values.append(cur_pixel_values.to(self.device).to(torch.bfloat16)) - - merge_size = int(1 / self.downsample_ratio) - pv_tensor = torch.cat(pixel_values) if pixel_values else None - ghw_tensor = torch.cat(grid_hw) if grid_hw else None - - # Condition - template_cond = get_conv_template(self.template) - template_cond.system_message = system_message - template_cond.append_message(template_cond.roles[0], prompt) - template_cond.append_message(template_cond.roles[1], None) - query_cond = template_cond.get_prompt() - - if not think_mode: - query_cond = query_cond + "\n\n\n\n" - - def replace_image_tokens(query, grid_hw_list): - for i in range(len(grid_hw_list)): - num_patch_token = int( - grid_hw_list[i][0, 0] - * grid_hw_list[i][0, 1] - * self.downsample_ratio**2 - ) - image_tokens = ( - IMG_START_TOKEN - + IMG_CONTEXT_TOKEN * num_patch_token - + IMG_END_TOKEN - ) - query = query.replace("", image_tokens, 1) - return query - - query_cond = replace_image_tokens(query_cond, grid_hw) - input_embeds_cond, indexes_cond, attention_mask_cond = self._build_it2i_inputs( - tokenizer, query_cond, pv_tensor, ghw_tensor - ) - - outputs_cond = self.language_model( - inputs_embeds=input_embeds_cond, - indexes=indexes_cond, - attention_mask=attention_mask_cond, - use_cache=True, - ) - past_key_values_cond = outputs_cond.past_key_values - t_index_cond = indexes_cond[0].max().item() - - # Initialize Text Uncondition Cache - question_text_uncondition = "" * len(images) - template_tu = get_conv_template(self.template) - template_tu.system_message = self.system_message - template_tu.append_message(template_tu.roles[0], question_text_uncondition) - template_tu.append_message(template_tu.roles[1], None) - query_text_uncond = template_tu.get_prompt() - query_text_uncond = replace_image_tokens(query_text_uncond, grid_hw) - - input_embeds_tu, indexes_tu, attention_mask_tu = self._build_it2i_inputs( - tokenizer, query_text_uncond, pv_tensor, ghw_tensor - ) - outputs_tu = self.language_model( - inputs_embeds=input_embeds_tu, - indexes=indexes_tu, - attention_mask=attention_mask_tu, - use_cache=True, - ) - past_key_values_tu = outputs_tu.past_key_values - t_index_tu = indexes_tu[0].max().item() - - # Initialize Img (ALL) Uncondition Cache - query_img_uncond = self._build_t2i_query("", append_text=IMG_START_TOKEN) - input_embeds_iu, indexes_iu, attention_mask_iu = self._build_it2i_inputs( - tokenizer, query_img_uncond - ) - outputs_iu = self.language_model( - inputs_embeds=input_embeds_iu, - indexes=indexes_iu, - attention_mask=attention_mask_iu, - use_cache=True, - ) - past_key_values_iu = outputs_iu.past_key_values - - generated_text = "" - generated_images = [] - max_images = 10 - img_count = 0 - - next_token = torch.argmax(outputs_cond.logits[:, -1, :], dim=-1) - - generator = torch.Generator(self.device).manual_seed(seed) - while True: - # text generation - gen_tokens = [] - hit_max_tokens = False - last_decoded = 0 - while True: - token_item = next_token.item() - if token_item == eos_token_id or token_item == self.img_start_token_id: - break - gen_tokens.append(token_item) - current_generated_tokens += 1 - - self.language_model.model.current_index = t_index_cond - outputs_cond = self.language_model( - input_ids=next_token.unsqueeze(0), - past_key_values=past_key_values_cond, - use_cache=True, - ) - past_key_values_cond = outputs_cond.past_key_values - t_index_cond += 1 - next_token = torch.argmax(outputs_cond.logits[:, -1, :], dim=-1) - - # Stream partial text so users see liveness during long runs - # (e.g. low VRAM offload). Decode in 16-token chunks. - if verbose and len(gen_tokens) - last_decoded >= 16: - partial = tokenizer.decode( - gen_tokens[last_decoded:], skip_special_tokens=True - ) - print(partial, end="", flush=True) - last_decoded = len(gen_tokens) - - if current_generated_tokens >= max_new_tokens: - hit_max_tokens = True - break - - if len(gen_tokens) > 0: - chunk_text = tokenizer.decode(gen_tokens, skip_special_tokens=True) - generated_text += chunk_text - if verbose: - remaining = tokenizer.decode( - gen_tokens[last_decoded:], skip_special_tokens=True - ) - if remaining: - print(remaining, end="", flush=True) - - if next_token.item() == eos_token_id or hit_max_tokens: - break - - if next_token.item() == self.img_start_token_id: - if img_count >= max_images: - break - - generated_text += "" - if verbose: - print( - f"\n[image {img_count + 1}] preparing diffusion...", flush=True - ) - - # Add the img_start_token for condition and text_uncondition branch - self.language_model.model.current_index = t_index_cond - outputs_cond = self.language_model( - input_ids=next_token.unsqueeze(0), - past_key_values=past_key_values_cond, - use_cache=True, - ) - past_key_values_cond = outputs_cond.past_key_values - t_index_cond += 1 - - self.language_model.model.current_index = t_index_tu - outputs_tu = self.language_model( - input_ids=next_token.unsqueeze(0), - past_key_values=past_key_values_tu, - use_cache=True, - ) - past_key_values_tu = outputs_tu.past_key_values - t_index_tu += 1 - - image_size = image_size_list[img_count] - # Image Generation - token_h = image_size[1] // (self.patch_size * merge_size) - token_w = image_size[0] // (self.patch_size * merge_size) - device = self.device - - indexes_image_condition = self._build_t2i_image_indexes( - token_h, token_w, t_index_cond + 1, device=device - ) - indexes_image_text_uncondition = self._build_t2i_image_indexes( - token_h, token_w, t_index_tu + 1, device=device - ) - indexes_image_img_uncondition = self._build_t2i_image_indexes( - token_h, token_w, indexes_iu[0].max() + 1, device=device - ) - - grid_h = image_size[1] // self.patch_size - grid_w = image_size[0] // self.patch_size - gen_grid_hw = torch.tensor([[grid_h, grid_w]], device=device) - - noise_scale = self.noise_scale - if self.noise_scale_mode in ("resolution", "dynamic", "dynamic_sqrt"): - base = float(self.noise_scale_base_image_seq_len) - noise_scale = math.sqrt( - (grid_h * grid_w) / (merge_size**2) / base - ) * float(self.noise_scale) - if self.noise_scale_mode == "dynamic_sqrt": - noise_scale = math.sqrt(noise_scale) - noise_scale = min(noise_scale, self.noise_scale_max_value) - image_prediction = noise_scale * torch.randn( - (1, 3, image_size[1], image_size[0]), - device=device, - dtype=outputs_cond.logits.dtype, - generator=generator, - ) - - past_key_values_cond_cfg = past_key_values_cond - past_key_values_tu_cfg = past_key_values_tu - past_key_values_iu_cfg = past_key_values_iu - - attention_mask_condition = {"full_attention": None} - attention_mask_text_uncondition = {"full_attention": None} - attention_mask_img_uncondition = {"full_attention": None} - - prepare_flash_kv_cache( - past_key_values_cond_cfg, - current_len=token_h * token_w, - batch_size=1, - ) - prepare_flash_kv_cache( - past_key_values_tu_cfg, - current_len=token_h * token_w, - batch_size=1, - ) - prepare_flash_kv_cache( - past_key_values_iu_cfg, - current_len=token_h * token_w, - batch_size=1, - ) - - timesteps = torch.linspace(0.0, 1.0, num_steps + 1, device=device) - if enable_timestep_shift: - timesteps = self._apply_time_schedule( - timesteps, token_h * token_w, timestep_shift - ) - - step_iter = range(num_steps) - if verbose: - try: - from tqdm import tqdm as _tqdm - - step_iter = _tqdm( - step_iter, - desc=f"image {img_count + 1} ({image_size[0]}x{image_size[1]})", - total=num_steps, - leave=False, - ) - except ImportError: - pass - for step_i in step_iter: - t = timesteps[step_i] - t_next = timesteps[step_i + 1] - - z = self.patchify(image_prediction, self.patch_size * merge_size) - image_input = self.patchify( - image_prediction, self.patch_size, channel_first=True - ) - image_embeds = self.extract_feature( - image_input.view(1 * grid_h * grid_w, -1), - gen_model=True, - grid_hw=gen_grid_hw, - ).view(1, token_h * token_w, -1) - t_expanded = t.expand(token_h * token_w) - timestep_embeddings = self.fm_modules["timestep_embedder"]( - t_expanded - ).view(1, token_h * token_w, -1) - if self.add_noise_scale_embedding: - noise_scale_tensor = torch.full_like( - t_expanded, noise_scale / self.noise_scale_max_value - ) - noise_embeddings = self.fm_modules["noise_scale_embedder"]( - noise_scale_tensor - ).view(1, token_h * token_w, -1) - timestep_embeddings += noise_embeddings - image_embeds = image_embeds + timestep_embeddings - - use_cfg = ( - t > cfg_interval[0] and t < cfg_interval[1] - ) or cfg_interval[0] == 0 - out_cond = self._t2i_predict_v( - image_embeds, - indexes_image_condition, - attention_mask_condition, - past_key_values_cond_cfg, - t, - z, - image_token_num=token_h * token_w, - timestep_embeddings=timestep_embeddings, - image_size=image_size, - ) - if not use_cfg: - v_pred = out_cond - elif cfg_scale == 1 and img_cfg_scale == 1: - v_pred = out_cond - elif img_cfg_scale == 1: - out_img_cond = self._t2i_predict_v( - image_embeds, - indexes_image_text_uncondition, - attention_mask_text_uncondition, - past_key_values_tu_cfg, - t, - z, - image_token_num=token_h * token_w, - timestep_embeddings=timestep_embeddings, - image_size=image_size, - ) - v_pred = out_img_cond + cfg_scale * (out_cond - out_img_cond) - elif cfg_scale == img_cfg_scale: - out_uncond = self._t2i_predict_v( - image_embeds, - indexes_image_img_uncondition, - attention_mask_img_uncondition, - past_key_values_iu_cfg, - t, - z, - image_token_num=token_h * token_w, - timestep_embeddings=timestep_embeddings, - image_size=image_size, - ) - v_pred = out_uncond + cfg_scale * (out_cond - out_uncond) - else: - out_img_cond = self._t2i_predict_v( - image_embeds, - indexes_image_text_uncondition, - attention_mask_text_uncondition, - past_key_values_tu_cfg, - t, - z, - image_token_num=token_h * token_w, - timestep_embeddings=timestep_embeddings, - image_size=image_size, - ) - out_uncond = self._t2i_predict_v( - image_embeds, - indexes_image_img_uncondition, - attention_mask_img_uncondition, - past_key_values_iu_cfg, - t, - z, - image_token_num=token_h * token_w, - timestep_embeddings=timestep_embeddings, - image_size=image_size, - ) - v_pred = ( - out_uncond - + cfg_scale * (out_cond - out_img_cond) - + img_cfg_scale * (out_img_cond - out_uncond) - ) - if cfg_scale > 1 or img_cfg_scale > 1 and use_cfg: - if cfg_norm == "global": - norm_v_condition = torch.norm( - out_cond, dim=(1, 2), keepdim=True - ) - norm_v_cfg = torch.norm(v_pred, dim=(1, 2), keepdim=True) - scale = (norm_v_condition / (norm_v_cfg + 1e-8)).clamp( - min=0, max=1.0 - ) - v_pred = v_pred * scale - elif cfg_norm == "channel": - norm_v_condition = torch.norm( - out_cond, dim=-1, keepdim=True - ) - norm_v_cfg = torch.norm(v_pred, dim=-1, keepdim=True) - scale = (norm_v_condition / (norm_v_cfg + 1e-8)).clamp( - min=0, max=1.0 - ) - v_pred = v_pred * scale - - z = z + (t_next - t) * v_pred - image_prediction = self.unpatchify( - z, self.patch_size * merge_size, image_size[1], image_size[0] - ) - - generated_images.append(image_prediction) - - clear_flash_kv_cache(past_key_values_cond_cfg) - clear_flash_kv_cache(past_key_values_tu_cfg) - clear_flash_kv_cache(past_key_values_iu_cfg) - - img_count += 1 - - # re-encode the generated image using the und-branch - pred_img = image_prediction[0].unsqueeze(0).to(torch.bfloat16) - # re-normalize the image - raw_img = pred_img * 0.5 + 0.5 - img_mean = torch.tensor( - [0.485, 0.456, 0.406], dtype=raw_img.dtype, device=device - ).view(1, 3, 1, 1) - img_std = torch.tensor( - [0.229, 0.224, 0.225], dtype=raw_img.dtype, device=device - ).view(1, 3, 1, 1) - und_img = (raw_img - img_mean) / img_std - c, h, w = und_img[0].shape - ps = self.patch_size - p_grid_h = h // ps - p_grid_w = w // ps - flatten_pixel_values = ( - und_img[0] - .view(c, p_grid_h, ps, p_grid_w, ps) - .permute( - 1, 3, 0, 2, 4 - ) # [grid_h, grid_w, c, patch_size, patch_size] - .reshape(p_grid_h * p_grid_w, c * ps**2) - ) - vit_embeds = self.extract_feature( - flatten_pixel_values, grid_hw=gen_grid_hw[:1] - ).unsqueeze(0) - - img_end_id = tokenizer.convert_tokens_to_ids(IMG_END_TOKEN) - img_end_embed = self.language_model.get_input_embeddings()( - torch.tensor([[img_end_id]], device=device) - ) - inputs_embeds_img = torch.cat( - [vit_embeds, img_end_embed], dim=1 - ) # (1, N + 1, C) - - N_img_tokens = vit_embeds.shape[1] - abs_pos_w, abs_pos_h = build_abs_positions_from_grid_hw( - gen_grid_hw[:1] // int(1 / self.downsample_ratio), device=device - ) - - def append_image_to_cache(cache, t_idx): - past_len = cache.get_seq_length() - tgt_len = N_img_tokens + 1 - - t_indexes = torch.zeros(tgt_len, dtype=torch.long, device=device) - t_indexes[:N_img_tokens] = t_idx + 1 - t_indexes[N_img_tokens] = t_idx + 2 - - h_indexes = torch.zeros(tgt_len, dtype=torch.long, device=device) - w_indexes = torch.zeros(tgt_len, dtype=torch.long, device=device) - h_indexes[:N_img_tokens] = abs_pos_h - w_indexes[:N_img_tokens] = abs_pos_w - - indexes = torch.stack([t_indexes, h_indexes, w_indexes], dim=0) - - mask = torch.zeros(1, 1, tgt_len, past_len + tgt_len, device=device) - mask[0, 0, :N_img_tokens, past_len + N_img_tokens] = float("-inf") - attention_mask_dict = {"full_attention": mask} - - outputs = self.language_model( - inputs_embeds=inputs_embeds_img, - indexes=indexes, - attention_mask=attention_mask_dict, - past_key_values=cache, - use_cache=True, - ) - return outputs, t_idx + 2 - - outputs_cond, t_index_cond = append_image_to_cache( - past_key_values_cond, t_index_cond - ) - outputs_tu, t_index_tu = append_image_to_cache( - past_key_values_tu, t_index_tu - ) - - next_token = torch.argmax(outputs_cond.logits[:, -1, :], dim=-1) - - return generated_text, generated_images - - @torch.no_grad() - def it2i_generate( - self, - tokenizer, - prompt, - images, - cfg_scale=1, - img_cfg_scale=1, - cfg_norm="none", - enable_timestep_shift=True, - timestep_shift=1, - image_size=(256, 256), - num_steps=30, - IMG_START_TOKEN="", - IMG_END_TOKEN="", - IMG_CONTEXT_TOKEN="", - method="euler", - cfg_interval=(0, 1), - batch_size=1, - t_eps=0.02, - think_mode=False, - seed=0, - ): - assert cfg_norm in ["none", "global", "channel"] - self._notify_layer_offload_phase("prefix") - - self.img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN) - self.config.t_eps = t_eps - - image_token_count = prompt.count("") - assert len(images) >= image_token_count - if len(images) > image_token_count: - if image_token_count == 0 and len(images) > 1: - prompt = ( - "".join(f"Image-{i + 1}:\n" for i in range(len(images))) - + prompt - ) - else: - prompt = "\n" * (len(images) - image_token_count) + prompt - - pixel_values = [] - grid_hw = [] - for image in images: - cur_pixel_values, cur_grid_hw = load_image_native( - image, - self.patch_size, - self.downsample_ratio, - min_pixels=512 * 512, - max_pixels=min(2048 * 2048, (4096 * 4096) // len(images)), - upscale=False, - ) - cur_grid_hw = cur_grid_hw.to(self.device) - cur_pixel_values = cur_pixel_values.to(self.device).to(torch.bfloat16) - pixel_values.append(cur_pixel_values) - grid_hw.append(cur_grid_hw) - pixel_values = torch.cat(pixel_values) - grid_hw = torch.cat(grid_hw) - - merge_size = int(1 / self.downsample_ratio) - question_condition = f"{prompt}" - think_text = "" - needs_cfg = not (cfg_scale == 1 and img_cfg_scale == 1) - needs_img_condition = needs_cfg and ( - img_cfg_scale == 1 or cfg_scale != img_cfg_scale - ) - needs_uncondition = needs_cfg and img_cfg_scale != 1 - - think_content = ( - "\n" if think_mode else "\n\n\n\n" + IMG_START_TOKEN - ) - query_condition = self._build_t2i_query( - question_condition, - system_message=SYSTEM_MESSAGE_FOR_GEN, - append_text=think_content, - ) - query_img_condition = ( - self._build_t2i_query("" * len(images), append_text=IMG_START_TOKEN) - if needs_img_condition - else None - ) - query_uncondition = ( - self._build_t2i_query("", append_text=IMG_START_TOKEN) - if needs_uncondition - else None - ) - - for i in range(grid_hw.shape[0]): - num_patch_token = int( - grid_hw[i, 0] * grid_hw[i, 1] * self.downsample_ratio**2 - ) - image_tokens = ( - IMG_START_TOKEN + IMG_CONTEXT_TOKEN * num_patch_token + IMG_END_TOKEN - ) - query_condition = query_condition.replace("", image_tokens, 1) - if query_img_condition is not None: - query_img_condition = query_img_condition.replace( - "", image_tokens, 1 - ) - - input_embeds_condition, indexes_condition, attention_mask_condition_prefix = ( - self._build_it2i_inputs(tokenizer, query_condition, pixel_values, grid_hw) - ) - if query_img_condition is not None: - ( - input_embeds_img_condition, - indexes_img_condition, - attention_mask_img_condition_prefix, - ) = self._build_it2i_inputs( - tokenizer, query_img_condition, pixel_values, grid_hw - ) - else: - input_embeds_img_condition = indexes_img_condition = ( - attention_mask_img_condition_prefix - ) = None - if query_uncondition is not None: - ( - input_embeds_uncondition, - indexes_uncondition, - attention_mask_uncondition_prefix, - ) = self._build_it2i_inputs(tokenizer, query_uncondition) - else: - input_embeds_uncondition = indexes_uncondition = ( - attention_mask_uncondition_prefix - ) = None - - token_h = image_size[1] // (self.patch_size * merge_size) - token_w = image_size[0] // (self.patch_size * merge_size) - - indexes_image_condition = self._build_t2i_image_indexes( - token_h, - token_w, - indexes_condition[0].max() + 1, - device=input_embeds_condition.device, - ) - indexes_image_img_condition = ( - self._build_t2i_image_indexes( - token_h, - token_w, - indexes_img_condition[0].max() + 1, - device=input_embeds_img_condition.device, - ) - if indexes_img_condition is not None - else None - ) - indexes_image_uncondition = ( - self._build_t2i_image_indexes( - token_h, - token_w, - indexes_uncondition[0].max() + 1, - device=input_embeds_uncondition.device, - ) - if indexes_uncondition is not None - else None - ) - - if think_mode: - outputs_condition = self._think_prefix_forward( - inputs_embeds=input_embeds_condition, - indexes=indexes_condition, - attention_mask=attention_mask_condition_prefix, - ) - past_key_values_condition = outputs_condition.past_key_values - device = outputs_condition.logits.device - dtype = outputs_condition.logits.dtype - t_index_condition = indexes_condition[0].max().item() - past_key_values_condition, t_index_condition, think_text = ( - self._generate_think( - tokenizer, - outputs_condition, - past_key_values_condition, - t_index_condition, - IMG_START_TOKEN, - ) - ) - indexes_image_condition = self._build_t2i_image_indexes( - token_h, - token_w, - t_index_condition + 1, - device=input_embeds_condition.device, - ) - del outputs_condition - else: - past_key_values_condition, prefix_hidden_states = self._it2i_prefix_forward( - input_embeds_condition, - indexes_condition, - attention_mask_condition_prefix, - ) - device = prefix_hidden_states.device - dtype = prefix_hidden_states.dtype - del prefix_hidden_states - past_key_values_img_condition = None - if input_embeds_img_condition is not None: - past_key_values_img_condition, _ = self._it2i_prefix_forward( - input_embeds_img_condition, - indexes_img_condition, - attention_mask_img_condition_prefix, - ) - past_key_values_uncondition = None - if input_embeds_uncondition is not None: - past_key_values_uncondition, _ = self._it2i_prefix_forward( - input_embeds_uncondition, - indexes_uncondition, - attention_mask_uncondition_prefix, - ) - - del pixel_values, grid_hw - del input_embeds_condition, indexes_condition, attention_mask_condition_prefix - if input_embeds_img_condition is not None: - del ( - input_embeds_img_condition, - indexes_img_condition, - attention_mask_img_condition_prefix, - ) - if input_embeds_uncondition is not None: - del ( - input_embeds_uncondition, - indexes_uncondition, - attention_mask_uncondition_prefix, - ) - self._notify_layer_offload_phase("denoise") - - for layer_idx in range(len(past_key_values_condition.layers)): - past_key_values_condition.layers[ - layer_idx - ].keys = past_key_values_condition.layers[layer_idx].keys.expand( - batch_size, *past_key_values_condition.layers[layer_idx].keys.shape[1:] - ) - past_key_values_condition.layers[ - layer_idx - ].values = past_key_values_condition.layers[layer_idx].values.expand( - batch_size, - *past_key_values_condition.layers[layer_idx].values.shape[1:], - ) - if past_key_values_img_condition is not None: - past_key_values_img_condition.layers[ - layer_idx - ].keys = past_key_values_img_condition.layers[layer_idx].keys.expand( - batch_size, - *past_key_values_img_condition.layers[layer_idx].keys.shape[1:], - ) - past_key_values_img_condition.layers[ - layer_idx - ].values = past_key_values_img_condition.layers[ - layer_idx - ].values.expand( - batch_size, - *past_key_values_img_condition.layers[layer_idx].values.shape[1:], - ) - if past_key_values_uncondition is not None: - past_key_values_uncondition.layers[ - layer_idx - ].keys = past_key_values_uncondition.layers[layer_idx].keys.expand( - batch_size, - *past_key_values_uncondition.layers[layer_idx].keys.shape[1:], - ) - past_key_values_uncondition.layers[ - layer_idx - ].values = past_key_values_uncondition.layers[layer_idx].values.expand( - batch_size, - *past_key_values_uncondition.layers[layer_idx].values.shape[1:], - ) - - prepare_flash_kv_cache( - past_key_values_condition, - current_len=token_h * token_w, - batch_size=batch_size, - ) - if past_key_values_img_condition is not None: - prepare_flash_kv_cache( - past_key_values_img_condition, - current_len=token_h * token_w, - batch_size=batch_size, - ) - if past_key_values_uncondition is not None: - prepare_flash_kv_cache( - past_key_values_uncondition, - current_len=token_h * token_w, - batch_size=batch_size, - ) - - grid_h = image_size[1] // self.patch_size - grid_w = image_size[0] // self.patch_size - grid_hw = torch.tensor([[grid_h, grid_w]] * batch_size, device=device) - - noise_scale = self.noise_scale - if self.noise_scale_mode in ("resolution", "dynamic", "dynamic_sqrt"): - base = float(self.noise_scale_base_image_seq_len) - scale = math.sqrt((grid_h * grid_w) / (merge_size**2) / base) - noise_scale = scale * float(self.noise_scale) - if self.noise_scale_mode == "dynamic_sqrt": - noise_scale = math.sqrt(noise_scale) - noise_scale = min(noise_scale, self.noise_scale_max_value) - image_prediction = noise_scale * _randn_with_seed( - (batch_size, 3, image_size[1], image_size[0]), - device=device, - dtype=dtype, - seed=seed, - ) - - attention_mask_condition = {"full_attention": None} - attention_mask_img_condition = {"full_attention": None} - attention_mask_uncondition = {"full_attention": None} - - timesteps = torch.linspace(0.0, 1.0, num_steps + 1, device=device) - if enable_timestep_shift: - timesteps = self._apply_time_schedule( - timesteps, token_h * token_w, timestep_shift - ) - denoise_embeddings = None - if device.type == "npu": - denoise_embeddings = self.fm_modules["timestep_embedder"](timesteps[:-1]) - if self.add_noise_scale_embedding: - noise_level = timesteps.new_tensor( - [noise_scale / self.noise_scale_max_value] - ) - denoise_embeddings = denoise_embeddings + self.fm_modules[ - "noise_scale_embedder" - ](noise_level) - - for step_i in range(num_steps): - t = timesteps[step_i] - t_next = timesteps[step_i + 1] - use_cfg = (t > cfg_interval[0] and t < cfg_interval[1]) or cfg_interval[ - 0 - ] == 0 - - z = self.patchify(image_prediction, self.patch_size * merge_size) - image_input = self.patchify( - image_prediction, self.patch_size, channel_first=True - ) - image_embeds = self.extract_feature( - image_input.view(batch_size * grid_h * grid_w, -1), - gen_model=True, - grid_hw=grid_hw, - ).view(batch_size, token_h * token_w, -1) - if denoise_embeddings is not None: - timestep_embeddings = denoise_embeddings[step_i].view(1, 1, -1) - else: - t_expanded = t.expand(batch_size * token_h * token_w) - timestep_embeddings = self.fm_modules["timestep_embedder"]( - t_expanded - ).view(batch_size, token_h * token_w, -1) - if self.add_noise_scale_embedding: - noise_scale_tensor = torch.full_like( - t_expanded, noise_scale / self.noise_scale_max_value - ) - timestep_embeddings += self.fm_modules["noise_scale_embedder"]( - noise_scale_tensor - ).view(batch_size, token_h * token_w, -1) - image_embeds = image_embeds + timestep_embeddings - - out_cond = self._t2i_predict_v( - image_embeds, - indexes_image_condition, - attention_mask_condition, - past_key_values_condition, - t, - z, - image_token_num=token_h * token_w, - timestep_embeddings=timestep_embeddings, - image_size=image_size, - ) - - if not use_cfg: - v_pred = out_cond - elif cfg_scale == 1 and img_cfg_scale == 1: - v_pred = out_cond - elif img_cfg_scale == 1: - out_img_cond = self._t2i_predict_v( - image_embeds, - indexes_image_img_condition, - attention_mask_img_condition, - past_key_values_img_condition, - t, - z, - image_token_num=token_h * token_w, - timestep_embeddings=timestep_embeddings, - image_size=image_size, - ) - v_pred = out_img_cond + cfg_scale * (out_cond - out_img_cond) - elif cfg_scale == img_cfg_scale: - out_uncond = self._t2i_predict_v( - image_embeds, - indexes_image_uncondition, - attention_mask_uncondition, - past_key_values_uncondition, - t, - z, - image_token_num=token_h * token_w, - timestep_embeddings=timestep_embeddings, - image_size=image_size, - ) - v_pred = out_uncond + cfg_scale * (out_cond - out_uncond) - else: - out_img_cond = self._t2i_predict_v( - image_embeds, - indexes_image_img_condition, - attention_mask_img_condition, - past_key_values_img_condition, - t, - z, - image_token_num=token_h * token_w, - timestep_embeddings=timestep_embeddings, - image_size=image_size, - ) - out_uncond = self._t2i_predict_v( - image_embeds, - indexes_image_uncondition, - attention_mask_uncondition, - past_key_values_uncondition, - t, - z, - image_token_num=token_h * token_w, - timestep_embeddings=timestep_embeddings, - image_size=image_size, - ) - v_pred = ( - out_uncond - + cfg_scale * (out_cond - out_img_cond) - + img_cfg_scale * (out_img_cond - out_uncond) - ) - if (cfg_scale > 1 or img_cfg_scale > 1) and use_cfg: - if cfg_norm == "global": - norm_v_condition = torch.norm(out_cond, dim=(1, 2), keepdim=True) - norm_v_cfg = torch.norm(v_pred, dim=(1, 2), keepdim=True) - scale = (norm_v_condition / (norm_v_cfg + 1e-8)).clamp( - min=0, max=1.0 - ) - v_pred = v_pred * scale - elif cfg_norm == "channel": - norm_v_condition = torch.norm(out_cond, dim=-1, keepdim=True) - norm_v_cfg = torch.norm(v_pred, dim=-1, keepdim=True) - scale = (norm_v_condition / (norm_v_cfg + 1e-8)).clamp( - min=0, max=1.0 - ) - v_pred = v_pred * scale - - z = z + (t_next - t) * v_pred - image_prediction = self.unpatchify( - z, self.patch_size * merge_size, image_size[1], image_size[0] - ) - - clear_flash_kv_cache(past_key_values_condition) - if past_key_values_img_condition is not None: - clear_flash_kv_cache(past_key_values_img_condition) - if past_key_values_uncondition is not None: - clear_flash_kv_cache(past_key_values_uncondition) - - self.last_think_content = think_text - if think_mode: - return image_prediction, think_text - return image_prediction - @torch.no_grad() def t2i_generate( self, @@ -2638,7 +940,6 @@ class NEOChatModel(PreTrainedModel): ) image_embeds = self.extract_feature( image_input.view(batch_size * grid_h * grid_w, -1), - gen_model=True, grid_hw=grid_hw, ).view(batch_size, token_h * token_w, -1) if denoise_embeddings is not None: @@ -2737,125 +1038,6 @@ class NEOChatModel(PreTrainedModel): return image_prediction, think_text return image_prediction - def chat( - self, - tokenizer, - pixel_values, - question, - generation_config, - history=None, - return_history=False, - grid_hw=None, - IMG_START_TOKEN="", - IMG_END_TOKEN="", - IMG_CONTEXT_TOKEN="", - verbose=False, - ): - - if history is None and pixel_values is not None and "" not in question: - question = "\n" + question - - img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN) - self.img_context_token_id = img_context_token_id - self.img_start_token_id = tokenizer.convert_tokens_to_ids(IMG_START_TOKEN) - - template = get_conv_template(self.template) - template.system_message = self.system_message - eos_token_id = tokenizer.convert_tokens_to_ids(template.sep.strip()) - - history = [] if history is None else history - for old_question, old_answer in history: - template.append_message(template.roles[0], old_question) - template.append_message(template.roles[1], old_answer) - template.append_message(template.roles[0], question) - template.append_message(template.roles[1], None) - query = template.get_prompt() - - if verbose and pixel_values is not None: - print(f"dynamic image size: {grid_hw[0] * self.patch_size}") - - for i in range(grid_hw.shape[0]): - num_patch_token = int( - grid_hw[i, 0] * grid_hw[i, 1] * self.downsample_ratio**2 - ) - image_tokens = ( - IMG_START_TOKEN + IMG_CONTEXT_TOKEN * num_patch_token + IMG_END_TOKEN - ) - query = query.replace("", image_tokens, 1) - - model_inputs = tokenizer(query, return_tensors="pt") - input_ids = model_inputs["input_ids"].to(self.device) - attention_mask = model_inputs["attention_mask"].to(self.device) - generation_config["eos_token_id"] = eos_token_id - generation_output = self.generate( - pixel_values=pixel_values, - input_ids=input_ids, - grid_hw=grid_hw, - attention_mask=attention_mask, - **generation_config, - ) - response = tokenizer.batch_decode(generation_output, skip_special_tokens=True)[ - 0 - ] - response = response.split(template.sep.strip())[0].strip() - history.append((question, response)) - if return_history: - return response, history - else: - query_to_print = query.replace(IMG_CONTEXT_TOKEN, "") - query_to_print = query_to_print.replace( - f"{IMG_START_TOKEN}{IMG_END_TOKEN}", "" - ) - if verbose: - print(query_to_print, response) - return response - - @torch.no_grad() - def generate( - self, - pixel_values: Optional[torch.FloatTensor] = None, - input_ids: Optional[torch.FloatTensor] = None, - grid_hw: Optional[torch.LongTensor] = None, - attention_mask: Optional[torch.LongTensor] = None, - visual_features: Optional[torch.FloatTensor] = None, - generation_config: Optional[GenerationConfig] = None, - output_hidden_states: Optional[bool] = None, - **generate_kwargs, - ) -> torch.LongTensor: - assert input_ids.shape[0] == 1 - assert self.img_context_token_id is not None - indexes = self.get_thw_indexes(input_ids[0], grid_hw) - if pixel_values is not None: - if visual_features is not None: - vit_embeds = visual_features - else: - vit_embeds = self.extract_feature(pixel_values, grid_hw=grid_hw) - - input_embeds = self.language_model.get_input_embeddings()(input_ids) - B, N, C = input_embeds.shape - input_embeds = input_embeds.reshape(B * N, C) - - input_ids = input_ids.reshape(B * N) - selected = input_ids == self.img_context_token_id - assert selected.sum() != 0 - input_embeds[selected] = vit_embeds.reshape(-1, C).to(input_embeds.device) - - input_embeds = input_embeds.reshape(B, N, C) - else: - input_embeds = self.language_model.get_input_embeddings()(input_ids) - - outputs = self.language_model.generate( - inputs_embeds=input_embeds, - indexes=indexes, - attention_mask=attention_mask, - generation_config=generation_config, - output_hidden_states=output_hidden_states, - use_cache=True, - **generate_kwargs, - ) - - return outputs - @property def lm_head(self): return self.language_model.get_output_embeddings() @@ -2871,26 +1053,3 @@ class NEOChatModel(PreTrainedModel): def set_output_embeddings(self, value): return self.language_model.set_output_embeddings(value) - - def get_thw_indexes(self, input_ids, grid_hw=None): - img_start_shift = torch.cat( - [ - torch.zeros(1, dtype=torch.long).to(input_ids.device), - (input_ids == self.img_start_token_id).long(), - ], - dim=0, - )[:-1] - not_img_token = (input_ids != self.img_context_token_id).long() - t_indexes = (img_start_shift + not_img_token).cumsum(0) - 1 - h_indexes = torch.zeros_like(t_indexes).to(t_indexes.device) - w_indexes = torch.zeros_like(t_indexes).to(t_indexes.device) - - if grid_hw is not None: - selected = input_ids == self.img_context_token_id - if selected.long().sum() > 0: - abs_pos_w, abs_pos_h = build_abs_positions_from_grid_hw( - grid_hw // int(1 / self.downsample_ratio), device=t_indexes.device - ) - h_indexes[selected] = abs_pos_h.to(t_indexes.device, t_indexes.dtype) - w_indexes[selected] = abs_pos_w.to(t_indexes.device, t_indexes.dtype) - return torch.stack([t_indexes, h_indexes, w_indexes], dim=0) diff --git a/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_qwen3.py b/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_qwen3.py index 7be0a93a55..7abc3300d3 100644 --- a/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_qwen3.py +++ b/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/modeling_qwen3.py @@ -13,12 +13,7 @@ from transformers.cache_utils import Cache, DynamicCache from transformers.generation import GenerationMixin from transformers.masking_utils import create_causal_mask from transformers.modeling_flash_attention_utils import FlashAttentionKwargs -from transformers.modeling_layers import ( - GenericForQuestionAnswering, - GenericForSequenceClassification, - GenericForTokenClassification, - GradientCheckpointingLayer, -) +from transformers.modeling_layers import GradientCheckpointingLayer from transformers.modeling_outputs import ( BaseModelOutputWithPast, CausalLMOutputWithPast, @@ -331,16 +326,6 @@ def create_block_causal_mask( return output.unsqueeze(1) -def visualize_mask(mask: torch.Tensor, i: int = 0, j: int = 12): - """ - mask: (1,1, L, L) - """ - submask = torch.where(mask[0, 0, :, :] == 0, torch.tensor(1.0), torch.tensor(0.0)) - submask = mask[i:j, i:j].int().cpu().numpy() - for row in submask: - print(" ".join(map(str, row))) - - def make_qwen3_rms_norm(hidden_size: int, eps: float) -> RMSNorm: return RMSNorm( hidden_size, @@ -1569,27 +1554,8 @@ class Qwen3ForCausalLM(Qwen3PreTrainedModel, GenerationMixin): ) -class Qwen3ForSequenceClassification( - GenericForSequenceClassification, Qwen3PreTrainedModel -): - pass - - -class Qwen3ForTokenClassification(GenericForTokenClassification, Qwen3PreTrainedModel): - pass - - -class Qwen3ForQuestionAnswering(GenericForQuestionAnswering, Qwen3PreTrainedModel): - base_model_prefix = ( - "transformer" # For BC, where `transformer` was used instead of `model` - ) - - __all__ = [ "Qwen3ForCausalLM", - "Qwen3ForQuestionAnswering", "Qwen3PreTrainedModel", "Qwen3Model", - "Qwen3ForSequenceClassification", - "Qwen3ForTokenClassification", ] diff --git a/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/transformers_compat.py b/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/transformers_compat.py index e7b60f1716..f0dd12b636 100644 --- a/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/transformers_compat.py +++ b/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/transformers_compat.py @@ -57,11 +57,6 @@ def causal_mask_kwargs( return {name: value for name, value in candidates.items() if name in parameters} -def pretrained_dtype_kwargs(dtype: torch.dtype) -> dict[str, torch.dtype]: - """Use the public dtype keyword supported throughout Transformers 4.57+.""" - return {"dtype": dtype} - - def tied_weights_keys(output_key: str, input_key: str) -> list[str] | dict[str, str]: """Return the `_tied_weights_keys` shape expected by Transformers 4 or 5.""" import transformers diff --git a/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/utils.py b/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/utils.py index d5001c02b4..35751907c2 100644 --- a/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/utils.py +++ b/python/sglang/multimodal_gen/runtime/models/sensenova_u1/neo_unify/utils.py @@ -1,13 +1,5 @@ # Modified for SGLang; see this directory's README.md for upstream source. -from __future__ import annotations - -import math - -import torch -import torchvision.transforms as T -from PIL import Image - SYSTEM_MESSAGE_FOR_GEN = ( "You are an image generation and editing assistant that accurately understands and executes " "user intent.\n\nYou support two modes:\n\n1. Think Mode:\nIf the task requires reasoning, you " @@ -27,139 +19,3 @@ SYSTEM_MESSAGE_FOR_GEN = ( "the user's description, not the language of the prompt. If no language is specified, use the " "user's input language." ) - -IMAGENET_MEAN = (0.485, 0.456, 0.406) -IMAGENET_STD = (0.229, 0.224, 0.225) - - -def round_by_factor(number: float, factor: int) -> int: - """Returns the closest integer to `number` that is divisible by `factor`.""" - return round(number / factor) * factor - - -def ceil_by_factor(number: float, factor: int) -> int: - """Returns the smallest integer >= `number` that is divisible by `factor`.""" - return math.ceil(number / factor) * factor - - -def floor_by_factor(number: float, factor: int) -> int: - """Returns the largest integer <= `number` that is divisible by `factor`.""" - return math.floor(number / factor) * factor - - -def smart_resize( - height: int, - width: int, - factor: int = 32, - min_pixels: int = 65536, - max_pixels: int = 4194304, -) -> tuple[int, int]: - """Rescale so that H/W are divisible by `factor` and total pixels ∈ [min, max]. - - Copied from https://github.com/QwenLM/Qwen2.5-VL/blob/main/qwen-vl-utils/src/qwen_vl_utils/vision_process.py#L60 - """ - if max(height, width) / min(height, width) > 200: - raise ValueError( - f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}" - ) - h_bar = max(factor, round_by_factor(height, factor)) - w_bar = max(factor, round_by_factor(width, factor)) - if h_bar * w_bar > max_pixels: - beta = math.sqrt((height * width) / max_pixels) - h_bar = max(factor, floor_by_factor(height / beta, factor)) - w_bar = max(factor, floor_by_factor(width / beta, factor)) - elif h_bar * w_bar < min_pixels: - beta = math.sqrt(min_pixels / (height * width)) - h_bar = ceil_by_factor(height * beta, factor) - w_bar = ceil_by_factor(width * beta, factor) - return h_bar, w_bar - - -def dynamic_preprocess_native_resolution( - image: Image.Image, - size_factor: int = 32, - min_pixels: int = 65536, - max_pixels: int = 4194304, - **_kwargs, -) -> Image.Image: - width, height = image.size - resized_height, resized_width = smart_resize( - height, - width, - factor=size_factor, - min_pixels=min_pixels, - max_pixels=max_pixels, - ) - return image.resize((resized_width, resized_height)) - - -def preprocess_pixel_values(pixel_values: torch.Tensor, patch_size: int = 16): - c, h, w = pixel_values.shape - grid_h = h // patch_size - grid_w = w // patch_size - - flatten_pixel_values = ( - pixel_values.view(c, grid_h, patch_size, grid_w, patch_size) - .permute(1, 3, 0, 2, 4) # [grid_h, grid_w, c, patch_size, patch_size] - .reshape(grid_h * grid_w, c * patch_size**2) - ) - - grid_hw = torch.tensor([[grid_h, grid_w]], device=pixel_values.device) - return flatten_pixel_values, grid_hw - - -def get_contrasting_background(image: Image.Image): - """Return a background color for RGBA->RGB conversion, or ``None`` to use default. - - The original Neo_Unify implementation computed a contrasting background - from the alpha channel. For this open-source release we fall back to a - plain white background; callers that need the smarter behavior can override - this function. - """ - del image - return (255, 255, 255) - - -def load_image_native( - image, - patch_size: int = 16, - downsample_ratio: float = 0.5, - min_pixels: int = 65536, - max_pixels: int = 4194304, - upscale: bool = False, -): - """Load and preprocess an image: RGB convert, smart-resize, normalize, patchify.""" - if not isinstance(image, Image.Image): - image = Image.open(image) - if image.mode == "RGBA": - bg_color = get_contrasting_background(image) - if bg_color: - background = Image.new("RGB", image.size, bg_color) - background.paste(image, mask=image.split()[3]) - image = background.convert("RGB") - else: - image = image.convert("RGB") - else: - image = image.convert("RGB") - - if upscale: - image = image.resize((image.width * 2, image.height * 2), Image.BILINEAR) - - transform = T.Compose( - [ - T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img), - T.ToTensor(), - T.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD), - ] - ) - - new_image = dynamic_preprocess_native_resolution( - image, - size_factor=int(patch_size // downsample_ratio), - min_pixels=min_pixels, - max_pixels=max_pixels, - ) - pixel_values, grid_hw = preprocess_pixel_values( - transform(new_image).to(torch.float32), patch_size=patch_size - ) - return pixel_values, grid_hw diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_dc.py b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_dc.py index 5941d32d70..6bf5f344d8 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_dc.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_dc.py @@ -68,6 +68,24 @@ class AutoencoderDC(nn.Module, LayerwiseOffloadableModuleMixin): return arg.dtype return None + def _load_inner_state(self, state_dict: dict[str, torch.Tensor]) -> None: + missing, unexpected = self._inner_model.load_state_dict( + state_dict, strict=False + ) + if missing: + logger.warning( + "AutoencoderDC missing keys when loading: %d keys", len(missing) + ) + if len(missing) > 10: + logger.debug("First 10 missing keys: %s", list(missing)[:10]) + else: + logger.debug("Missing keys: %s", list(missing)) + if unexpected: + logger.debug( + "AutoencoderDC unexpected keys when loading: %d keys", + len(unexpected), + ) + def _ensure_inner_model(self, state_dict: dict[str, torch.Tensor] | None = None): if self._inner_model is not None: return @@ -94,22 +112,7 @@ class AutoencoderDC(nn.Module, LayerwiseOffloadableModuleMixin): self._inner_model = DiffusersAutoencoderDC.from_config(hf_config) if state_to_load: - missing, unexpected = self._inner_model.load_state_dict( - state_to_load, strict=False - ) - if missing: - logger.warning( - "AutoencoderDC missing keys when loading: %d keys", len(missing) - ) - if len(missing) > 10: - logger.debug("First 10 missing keys: %s", list(missing)[:10]) - else: - logger.debug("Missing keys: %s", list(missing)) - if unexpected: - logger.debug( - "AutoencoderDC unexpected keys when loading: %d keys", - len(unexpected), - ) + self._load_inner_state(state_to_load) if state_dict is None: self._loaded_state_dict.clear() @@ -190,7 +193,11 @@ class AutoencoderDC(nn.Module, LayerwiseOffloadableModuleMixin): assign: bool = False, ): """Intercept load_state_dict to route weights into the inner diffusers model.""" - self._ensure_inner_model(state_dict=state_dict) + if self._inner_model is None: + self._ensure_inner_model(state_dict=state_dict) + else: + # The loader reads state_dict() first, which builds the inner model. + self._load_inner_state(state_dict) def state_dict(self, *args, **kwargs) -> dict[str, torch.Tensor]: self._ensure_inner_model() diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py index 3a729a22cc..37b402075b 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage.py @@ -562,14 +562,6 @@ class QwenImageEncoder3d(nn.Module): super().__init__() dim_mult = list(dim_mult) temperal_downsample = list(temperal_downsample) - # dim = config.arch_config.dim - # z_dim = config.arch_config.z_dim - # dim_mult = config.arch_config.dim_mult - # num_res_blocks = config.arch_config.num_res_blocks - # attn_scales = config.arch_config.attn_scales - # temperal_downsample = config.arch_config.temperal_downsample - # dropout = config.arch_config.dropout - # non_linearity = config.arch_config.non_linearity self.dim = dim self.z_dim = z_dim self.dim_mult = dim_mult diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_vae_cuda_opt.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_vae_cuda_opt.py index 105cfba393..98f1c1d83f 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_vae_cuda_opt.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_vae_cuda_opt.py @@ -21,6 +21,7 @@ from sglang.kernels.ops.diffusion import ( can_use_fused_inplace_qknorm_rope, fused_inplace_qknorm_rope, ) +from sglang.multimodal_gen import envs from sglang.multimodal_gen.runtime.models.vaes.fast_path_gate import ( VaeFastPathGate, register_vae_fast_path_gate, @@ -31,7 +32,6 @@ from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.attention im _sdpa_attention, ) from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vit_utils import ( - _env_flag, apply_rotary_pos_emb_qk, native_rope_cache, ) @@ -155,7 +155,7 @@ def maybe_optimize_minimax_h3_vae(vae: nn.Module) -> nn.Module: decoder = vae.decoder if type(decoder) is not ViT3DDecoder: return vae - if not _env_flag("MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM", "1"): + if not envs.MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM: logger.info("MiniMax-H3 VAE: fp32 QK norm disabled; skipping fast path.") return vae diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/attention.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/attention.py index 23fe4bec7a..cf4dffc92e 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/attention.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/attention.py @@ -10,13 +10,14 @@ import torch.nn.functional as F from diffusers.utils import logging from torch.nn.attention import SDPBackend, sdpa_kernel +from sglang.multimodal_gen import envs from sglang.multimodal_gen.runtime.layers.attention import USPAttention from sglang.multimodal_gen.runtime.platforms import ( AttentionBackendEnum, current_platform, ) -from .vit_utils import _env_flag, apply_rotary_pos_emb_qk +from .vit_utils import apply_rotary_pos_emb_qk logger = logging.get_logger(__name__) # pylint: disable=invalid-name _FORCE_ROCM_MATH_SDPA = current_platform.is_rocm() and "gfx95" in str( @@ -36,52 +37,15 @@ def _sdpa_attention(query, key, value): def _vit_norm_input(module, hidden_states): - if _env_flag("MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM", "1"): + if envs.MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM: return hidden_states.float() weight = module.weight return hidden_states.to(weight.dtype if weight is not None else hidden_states.dtype) -def _try_fused_qk_rmsnorm_rope(norm_q, norm_k, query, key, rotary_pos_emb): - if not _env_flag("MINIMAX_H3_VAE_DECODER_FUSED_NORM", "1"): - return None - # NVIDIA only: see _fused_blocks_available in vae_vit.py. - if not current_platform.is_cuda(): - return None - if ( - rotary_pos_emb is None - or torch.is_grad_enabled() - or torch.compiler.is_compiling() - or not isinstance(norm_q, nn.RMSNorm) - or not isinstance(norm_k, nn.RMSNorm) - or norm_q.weight is not None - or norm_k.weight is not None - or norm_q.eps != norm_k.eps - ): - return None - cos, sin = rotary_pos_emb[:2] - if cos.dim() != 4 or sin.shape != cos.shape or cos.shape[2] != 1: - return None - # Stacked tiles broadcast one rotary table. A real batch stride would mean - # each tile has its own positions, which this kernel does not index. - if cos.shape[0] != 1 and cos.stride(0) != 0: - return None - if sin.shape[0] != 1 and sin.stride(0) != 0: - return None - from sglang.kernels.ops.diffusion import h3_vae_qk_rmsnorm_rope - - return h3_vae_qk_rmsnorm_rope( - query, - key, - norm_q.eps, - cos[0, :, 0, :], - sin[0, :, 0, :], - ) - - def _apply_qk_norm(module, hidden_states): if ( - _env_flag("MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM", "1") + envs.MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM and isinstance(module, (nn.LayerNorm, nn.RMSNorm)) and module.weight is None and (not isinstance(module, nn.LayerNorm) or module.bias is None) @@ -175,20 +139,12 @@ class Attention(nn.Module): qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head) query, key, value = torch.chunk(qkv, 3, dim=-1) - fused_qk = None - if self.norm_q is not None and self.norm_k is not None: - fused_qk = _try_fused_qk_rmsnorm_rope( - self.norm_q, self.norm_k, query, key, rotary_pos_emb - ) - if fused_qk is not None: - query, key = fused_qk - else: - if self.norm_q is not None: - query = _apply_qk_norm(self.norm_q, query) - if self.norm_k is not None: - key = _apply_qk_norm(self.norm_k, key) - if rotary_pos_emb is not None: - query, key = apply_rotary_pos_emb_qk(query, key, rotary_pos_emb) + if self.norm_q is not None: + query = _apply_qk_norm(self.norm_q, query) + if self.norm_k is not None: + key = _apply_qk_norm(self.norm_k, key) + if rotary_pos_emb is not None: + query, key = apply_rotary_pos_emb_qk(query, key, rotary_pos_emb) if self.attn is not None and query.dtype in (torch.float16, torch.bfloat16): hidden_states = self.attn(query, key, value) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/base_module.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/base_module.py index b258bb8951..bac7ae2dd9 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/base_module.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/base_module.py @@ -7,24 +7,21 @@ from typing import Optional import torch import torch.nn as nn -from diffusers.utils import logging from diffusers.utils.torch_utils import maybe_allow_in_graph from sglang.kernels.ops.activation.activation import ( silu_and_mul_with_activation_rounding, ) from sglang.kernels.ops.diffusion import try_fused_scaled_residual_add_exact +from sglang.multimodal_gen import envs from sglang.multimodal_gen.runtime.layers.activation import SiluAndMul from sglang.multimodal_gen.runtime.platforms import current_platform from .attention import Attention -from .vit_utils import _env_flag, _vit_torch_compile_kwargs - -logger = logging.get_logger(__name__) # pylint: disable=invalid-name def _vit_norm_input(module, hidden_states): - if _env_flag("MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM", "1"): + if envs.MINIMAX_H3_VAE_DECODER_VIT_FP32_NORM: return hidden_states.float() return hidden_states.to(getattr(module.weight, "dtype", hidden_states.dtype)) @@ -99,15 +96,8 @@ class FeedForward(nn.Module): ) self.w2 = nn.Linear(inner_dim, dim_out, bias=bias) - self._compile_forward_enabled = _env_flag( - "MINIMAX_H3_VAE_DECODER_VIT_FF_TORCH_COMPILE", "0" - ) - self._compile_forward_fatal = _env_flag( - "MINIMAX_H3_VAE_DECODER_VIT_FF_TORCH_COMPILE_FATAL", "0" - ) - self._compiled_forward = None - def _forward_impl(self, hidden_states: torch.Tensor) -> torch.Tensor: + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: hidden_states = self.w1(hidden_states) if self.use_gated: @@ -130,59 +120,6 @@ class FeedForward(nn.Module): hidden_states = _unfused_bias_linear(self.w2, hidden_states) return hidden_states - def _get_forward_impl(self): - if not self._compile_forward_enabled: - return self._forward_impl - if self._compiled_forward is not None: - return self._compiled_forward - if not hasattr(torch, "compile"): - message = ( - "torch.compile is unavailable; falling back to eager ViT FeedForward" - ) - if self._compile_forward_fatal: - raise RuntimeError(message) - logger.warning(f"[ViTFeedForward] {message}") - self._compile_forward_enabled = False - return self._forward_impl - - kwargs = _vit_torch_compile_kwargs( - "MINIMAX_H3_VAE_DECODER_VIT_FF_TORCH_COMPILE" - ) - try: - self._compiled_forward = torch.compile(self._forward_impl, **kwargs) - logger.info(f"[ViTFeedForward] torch.compile enabled kwargs={kwargs}") - except Exception as exc: - if self._compile_forward_fatal: - raise - logger.warning( - f"[ViTFeedForward] torch.compile setup failed: {type(exc).__name__}: {exc}; " - "falling back to eager" - ) - self._compile_forward_enabled = False - self._compiled_forward = None - return self._forward_impl - return self._compiled_forward - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - forward_impl = self._get_forward_impl() - try: - return forward_impl(hidden_states) - except Exception as exc: - if ( - self._compile_forward_enabled - and self._compiled_forward is not None - and forward_impl is self._compiled_forward - and not self._compile_forward_fatal - ): - logger.warning( - f"[ViTFeedForward] compiled forward failed: {type(exc).__name__}: {exc}; " - "disabling compile and retrying eager" - ) - self._compile_forward_enabled = False - self._compiled_forward = None - return self._forward_impl(hidden_states) - raise - class RotaryEmbeddingND(nn.Module): def __init__(self, dim, rotary_base=10000, n_dim=3, use_angle=False): diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py index eab7ee373a..bd7a21a862 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # MiniMax H3 visual VAE: 3D causal CNN encoder + ViT3D decoder (inference-only bundle). import math -import os from typing import List, Union import numpy as np @@ -14,6 +13,7 @@ from diffusers.models import ModelMixin from diffusers.utils import logging from PIL import Image +from sglang.multimodal_gen import envs from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode from sglang.multimodal_gen.runtime.distributed import ( get_decode_parallel_group_coordinator, @@ -33,38 +33,6 @@ from .vae_vit import ViT3DDecoder logger = logging.get_logger(__name__) # pylint: disable=invalid-name -def _resolve_temporal_cat_dtype(): - raw = ( - os.environ.get("MINIMAX_H3_VAE_DECODER_TEMPORAL_CAT_DTYPE", "").strip().lower() - ) - if raw in ("", "0", "false", "no", "off", "none", "keep", "default"): - return None - mapping = { - "fp16": torch.float16, - "float16": torch.float16, - "half": torch.float16, - "bf16": torch.bfloat16, - "bfloat16": torch.bfloat16, - "fp32": torch.float32, - "float32": torch.float32, - } - if raw not in mapping: - raise ValueError( - "MINIMAX_H3_VAE_DECODER_TEMPORAL_CAT_DTYPE must be one of " - "fp16|bf16|fp32|keep, got %r" % raw - ) - return mapping[raw] - - -def _resolve_temporal_stream_cat(): - raw = ( - os.environ.get("MINIMAX_H3_VAE_DECODER_STREAM_TEMPORAL_CAT", "1") - .strip() - .lower() - ) - return raw not in ("0", "false", "no", "off", "disable", "disabled") - - def get_tile_parallel_state(): if not dist.is_initialized() or not model_parallel_is_initialized(): return 0, 1 @@ -664,7 +632,7 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin): return int(total_frames), int(pad_frames), int(total_frames - pad_frames) def _decode_temporal_streaming( - self, z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype, on_frames + self, z, z_head, z_tail, num_chunks, pad_tokens, on_frames ): total_frames, pad_frames, output_frames = ( self._decode_temporal_output_frame_plan( @@ -721,8 +689,6 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin): clip_dec = self._adaptive_decode(clip_z) decoded_count += 1 - if temporal_cat_dtype is not None and clip_dec.dtype != temporal_cat_dtype: - clip_dec = clip_dec.to(temporal_cat_dtype) if clip_dec.device != z.device: clip_dec = clip_dec.to(z.device) @@ -817,10 +783,9 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin): pad_z = z[:, :, -1:, :, :].expand(-1, -1, pad_tokens, -1, -1) z = torch.cat([z, pad_z], dim=2) - temporal_cat_dtype = _resolve_temporal_cat_dtype() - if not self.training and _resolve_temporal_stream_cat(): + if not self.training and envs.MINIMAX_H3_VAE_DECODER_STREAM_TEMPORAL_CAT: return self._decode_temporal_streaming( - z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype, on_frames + z, z_head, z_tail, num_chunks, pad_tokens, on_frames ) decoded_tasks = [] @@ -836,8 +801,6 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin): clip_z = torch.cat([clip_z, z_tail], dim=2) clip_dec = self._adaptive_decode(clip_z) - if temporal_cat_dtype is not None and clip_dec.dtype != temporal_cat_dtype: - clip_dec = clip_dec.to(temporal_cat_dtype) decoded_tasks.append((i, clip_dec)) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/norm.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/norm.py index c1ad34d89b..1496a2a9b0 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/norm.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/norm.py @@ -1,32 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # Torch-native normalization for the MiniMax H3 visual VAE. import math -import os -import torch import torch.nn as nn import torch.nn.functional as F from .conv import BaseConv3d -def _validate_activation(activation): - valid_activations = {"identity", "silu", "relu"} - if activation not in valid_activations: - raise ValueError( - f"Unsupported activation: {activation}. Supported: {valid_activations}" - ) - - -def _apply_activation(x, activation): - _validate_activation(activation) - if activation == "identity": - return x - if activation == "silu": - return F.silu(x) - return F.relu(x) - - def _merge_time_to_batch(x): batch, channels, depth, height, width = x.shape return ( @@ -46,142 +27,6 @@ def _split_time_from_batch(x, batch): ) -def fused_group_norm(x, num_groups, weight, bias, eps=1e-5, activation="silu"): - out = F.group_norm(x, num_groups, weight=weight, bias=bias, eps=eps) - return _apply_activation(out, activation) - - -def fused_spatial_norm( - f, - num_groups, - norm_weight, - norm_bias, - dynamic_scale, - dynamic_bias, - eps=1e-5, - activation="silu", -): - norm_f = F.group_norm( - f, - num_groups, - weight=norm_weight, - bias=norm_bias, - eps=eps, - ) - out = norm_f * dynamic_scale + dynamic_bias - return _apply_activation(out, activation) - - -class DummyAffine(torch.nn.Module): - def __init__(self, num_channels, affine=True): - super().__init__() - if affine: - self.weight = torch.nn.Parameter(torch.ones(num_channels)) - self.bias = torch.nn.Parameter(torch.zeros(num_channels)) - else: - self.register_parameter("weight", None) - self.register_parameter("bias", None) - - def forward(self, input): - if self.weight is None: - return input - shape = [1, -1] + [1] * (input.dim() - 2) - return input * self.weight.view(*shape) + self.bias.view(*shape) - - -class FusedGroupNorm3D(torch.nn.Module): - """Compatibility wrapper implemented with native PyTorch ops.""" - - def __init__( - self, - num_groups, - num_channels, - eps=1e-5, - affine=True, - activation="silu", - cond_channels=None, - use_t_isolated_gn=False, - padding_mode="zeros", - padding_mode_t=None, - causal=True, - ): - super().__init__() - _validate_activation(activation) - self.num_groups = num_groups - self.num_channels = num_channels - self.eps = eps - self.affine = affine - self.activation = activation - self.use_t_isolated_gn = use_t_isolated_gn - - if cond_channels is not None: - self.use_spatial_affine = True - self.norm_layer = DummyAffine(num_channels, affine=affine) - self.conv_y = BaseConv3d( - cond_channels, - num_channels, - kernel_size=1, - padding_mode=padding_mode, - padding_mode_t=padding_mode_t, - causal=causal, - ) - self.conv_b = BaseConv3d( - cond_channels, - num_channels, - kernel_size=1, - padding_mode=padding_mode, - padding_mode_t=padding_mode_t, - causal=causal, - ) - else: - self.use_spatial_affine = False - if self.affine: - self.weight = torch.nn.Parameter(torch.ones(num_channels)) - self.bias = torch.nn.Parameter(torch.zeros(num_channels)) - else: - self.register_parameter("weight", None) - self.register_parameter("bias", None) - - def forward(self, f, cond=None): - need_reshape = self.use_t_isolated_gn and f.dim() == 5 - batch = f.shape[0] if need_reshape else None - f_size = f.shape[-3:] - if need_reshape: - f = _merge_time_to_batch(f) - - if self.use_spatial_affine: - scale = self.conv_y(cond) - bias = self.conv_b(cond) - if math.prod(scale.shape[-3:]) * math.prod(bias.shape[-3:]) > 1: - scale = F.interpolate(scale, size=f_size, mode="nearest") - bias = F.interpolate(bias, size=f_size, mode="nearest") - if need_reshape: - scale = _merge_time_to_batch(scale) - bias = _merge_time_to_batch(bias) - out = fused_spatial_norm( - f, - self.num_groups, - self.norm_layer.weight, - self.norm_layer.bias, - scale, - bias, - self.eps, - self.activation, - ) - else: - if cond is not None: - raise NotImplementedError("Dynamic affine is not defined") - weight = self.weight if self.affine else None - bias = self.bias if self.affine else None - out = fused_group_norm( - f, self.num_groups, weight, bias, self.eps, self.activation - ) - - if need_reshape: - out = _split_time_from_batch(out, batch) - return out - - class TemporalIsolatedGroupNorm(nn.GroupNorm): def forward(self, input): if input.dim() == 5: @@ -247,18 +92,6 @@ def get_spatial_norm_3d( causal=True, use_t_isolated_gn=False, ): - if os.environ.get("MINIMAX_H3_USE_FUSED_NORM", "false").lower() == "true": - return FusedGroupNorm3D( - num_groups=32, - num_channels=num_channels, - eps=1e-6, - affine=True, - cond_channels=cond_channels, - use_t_isolated_gn=use_t_isolated_gn, - padding_mode=padding_mode, - padding_mode_t=padding_mode_t, - causal=causal, - ) return SpatialNorm3D( num_channels, cond_channels, @@ -270,14 +103,5 @@ def get_spatial_norm_3d( def get_group_norm_3d(num_channels, use_t_isolated_gn=False): - if os.environ.get("MINIMAX_H3_USE_FUSED_NORM", "false").lower() == "true": - return FusedGroupNorm3D( - num_groups=32, - num_channels=num_channels, - eps=1e-6, - affine=True, - use_t_isolated_gn=use_t_isolated_gn, - ) - norm_cls = TemporalIsolatedGroupNorm if use_t_isolated_gn else nn.GroupNorm return norm_cls(num_groups=32, num_channels=num_channels, eps=1e-6, affine=True) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_cnn.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_cnn.py index 9de936e653..a0efd489c6 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_cnn.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_cnn.py @@ -1,7 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # 3D causal CNN encoder for the MiniMax H3 visual VAE (inference-only bundle). -import os - import torch.nn as nn import torch.nn.functional as F @@ -74,10 +72,6 @@ class ResnetBlock3D(nn.Module): out_channels = in_channels if out_channels is None else out_channels self.out_channels = out_channels - self.use_fused_norm = ( - os.environ.get("MINIMAX_H3_USE_FUSED_NORM", "false").lower() == "true" - ) - if zq_ch is None: self.norm1 = get_group_norm_3d( in_channels, use_t_isolated_gn=use_t_isolated_gn @@ -136,17 +130,11 @@ class ResnetBlock3D(nn.Module): def forward(self, x, zq=None): h = x - if self.use_fused_norm: - h = self.norm1(h, zq) - else: - h = norm_silu(h, self.norm1, zq) + h = norm_silu(h, self.norm1, zq) h = self.conv1(h) - if self.use_fused_norm: - h = self.norm2(h, zq) - else: - h = norm_silu(h, self.norm2, zq) + h = norm_silu(h, self.norm2, zq) h = self.conv2(h) @@ -184,10 +172,6 @@ class EncoderFCN3D(nn.Module): self.in_channels = in_channels - self.use_fused_norm = ( - os.environ.get("MINIMAX_H3_USE_FUSED_NORM", "false").lower() == "true" - ) - block_mid = [ch * ch_mult[i] for i in range(self.num_levels)] block_in = [block_mid[0]] + block_mid[:-1] block_out = block_mid @@ -265,10 +249,7 @@ class EncoderFCN3D(nn.Module): if hasattr(self.down[i_level], "downsample"): h = self.down[i_level].downsample(h) - if self.use_fused_norm: - h = self.norm_out(h, zq) - else: - h = norm_silu(h, self.norm_out, zq) + h = norm_silu(h, self.norm_out, zq) h = self.conv_out(h) return h diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_vit.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_vit.py index 8c570c65d5..aaeba436a7 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_vit.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vae_vit.py @@ -9,10 +9,11 @@ from diffusers.configuration_utils import ConfigMixin, register_to_config from diffusers.models.modeling_utils import ModelMixin from diffusers.utils import logging +from sglang.multimodal_gen import envs from sglang.multimodal_gen.runtime.platforms import current_platform from .base_module import RotaryEmbeddingND, TransformerBlock, _scaled_residual_add -from .vit_utils import _env_flag, create_token_ids, prepare_rotary_pos_emb +from .vit_utils import create_token_ids, prepare_rotary_pos_emb logger = logging.get_logger(__name__) @@ -31,7 +32,7 @@ def _cuda_autocast_disabled(tensor: torch.Tensor): def _fused_blocks_available(blocks, hidden_states) -> bool: - if not _env_flag("MINIMAX_H3_VAE_DECODER_FUSED_NORM", "1"): + if not envs.MINIMAX_H3_VAE_DECODER_FUSED_NORM: return False # The fused Triton norms decode to all-NaN on ROCm (MI300), where # tensor.is_cuda is also true; keep ROCm on the unfused fp32-norm path. diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vit_utils.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vit_utils.py index e777fa8d27..b8b5e89704 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vit_utils.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/vit_utils.py @@ -1,41 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # ViT runtime helpers for the MiniMax H3 visual VAE. -import os from collections.abc import Sequence from typing import Tuple import torch -from diffusers.utils import logging - - -def _env_flag(name, default="0"): - value = os.environ.get(name, default) - return str(value).strip().lower() in ("1", "true", "yes", "on") - - -def _env_optional_bool(name, default=""): - value = str(os.environ.get(name, default)).strip().lower() - if value in ("", "default", "auto", "none", "unset"): - return None - return value not in ("0", "false", "no", "off", "disabled") - - -def _vit_torch_compile_kwargs(prefix): - kwargs = {} - backend = os.environ.get(f"{prefix}_BACKEND", "inductor").strip() - mode = os.environ.get(f"{prefix}_MODE", "reduce-overhead").strip() - if backend and backend.lower() not in ("default", "none"): - kwargs["backend"] = backend - if mode and mode.lower() not in ("default", "none"): - kwargs["mode"] = mode - kwargs["fullgraph"] = _env_flag(f"{prefix}_FULLGRAPH", "0") - dynamic = _env_optional_bool(f"{prefix}_DYNAMIC") - if dynamic is not None: - kwargs["dynamic"] = dynamic - return kwargs - - -logger = logging.get_logger(__name__) # pylint: disable=invalid-name def create_token_ids( @@ -128,7 +96,6 @@ def prepare_rotary_pos_emb( or cos.shape[0] != 1 or cos.shape[2] != 1 or cos.shape[-1] % 2 - or _env_flag("MINIMAX_H3_VAE_DECODER_VIT_ROPE_TORCH_COMPILE", "0") ): return cos, sin @@ -149,66 +116,10 @@ def prepare_rotary_pos_emb( return cos, sin, cache, positions -_COMPILED_APPLY_ROTARY_POS_EMB = None -_APPLY_ROTARY_POS_EMB_COMPILE_DISABLED = False - - -def _get_apply_rotary_pos_emb_impl(): - global _COMPILED_APPLY_ROTARY_POS_EMB, _APPLY_ROTARY_POS_EMB_COMPILE_DISABLED - if _APPLY_ROTARY_POS_EMB_COMPILE_DISABLED or not _env_flag( - "MINIMAX_H3_VAE_DECODER_VIT_ROPE_TORCH_COMPILE", "0" - ): - return _apply_rotary_pos_emb_impl - if _COMPILED_APPLY_ROTARY_POS_EMB is not None: - return _COMPILED_APPLY_ROTARY_POS_EMB - if not hasattr(torch, "compile"): - message = ( - "torch.compile is unavailable; falling back to eager ViT rotary embedding" - ) - if _env_flag("MINIMAX_H3_VAE_DECODER_VIT_ROPE_TORCH_COMPILE_FATAL", "0"): - raise RuntimeError(message) - logger.warning(f"[ViTRope] {message}") - _APPLY_ROTARY_POS_EMB_COMPILE_DISABLED = True - return _apply_rotary_pos_emb_impl - - kwargs = _vit_torch_compile_kwargs("MINIMAX_H3_VAE_DECODER_VIT_ROPE_TORCH_COMPILE") - try: - _COMPILED_APPLY_ROTARY_POS_EMB = torch.compile( - _apply_rotary_pos_emb_impl, **kwargs - ) - logger.info(f"[ViTRope] torch.compile enabled kwargs={kwargs}") - except Exception as exc: - if _env_flag("MINIMAX_H3_VAE_DECODER_VIT_ROPE_TORCH_COMPILE_FATAL", "0"): - raise - logger.warning( - f"[ViTRope] torch.compile setup failed: {type(exc).__name__}: {exc}; " - "falling back to eager" - ) - _APPLY_ROTARY_POS_EMB_COMPILE_DISABLED = True - _COMPILED_APPLY_ROTARY_POS_EMB = None - return _apply_rotary_pos_emb_impl - return _COMPILED_APPLY_ROTARY_POS_EMB - - def apply_rotary_pos_emb( t: torch.Tensor, rotary_pos_emb: Sequence[torch.Tensor] ) -> torch.Tensor: - global _COMPILED_APPLY_ROTARY_POS_EMB, _APPLY_ROTARY_POS_EMB_COMPILE_DISABLED - fn = _get_apply_rotary_pos_emb_impl() - try: - return fn(t, rotary_pos_emb) - except Exception as exc: - if fn is _COMPILED_APPLY_ROTARY_POS_EMB and not _env_flag( - "MINIMAX_H3_VAE_DECODER_VIT_ROPE_TORCH_COMPILE_FATAL", "0" - ): - logger.warning( - f"[ViTRope] compiled call failed: {type(exc).__name__}: {exc}; " - "disabling compile and retrying eager" - ) - _APPLY_ROTARY_POS_EMB_COMPILE_DISABLED = True - _COMPILED_APPLY_ROTARY_POS_EMB = None - return _apply_rotary_pos_emb_impl(t, rotary_pos_emb) - raise + return _apply_rotary_pos_emb_impl(t, rotary_pos_emb) def native_rope_cache( diff --git a/python/sglang/multimodal_gen/runtime/pipelines/flux3_image_weights.py b/python/sglang/multimodal_gen/runtime/pipelines/flux3_image_weights.py index 9203023faf..9d6e4d6454 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/flux3_image_weights.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/flux3_image_weights.py @@ -50,18 +50,6 @@ def flux3_image_dit_config_from_package(package: Flux3ImagePackage) -> Flux3DiTC ) -def flux3_image_quantization_name( - quant_config: QuantizationConfig | None, -) -> str | None: - if quant_config is None: - return None - name = quant_config.get_name() - return { - "modelopt_fp4": "nvfp4", - "rowwise_fp8": "rowwise-fp8", - }.get(name, name) - - def resolve_flux3_image_dit_quant_config( dit_path: str, quantization: str | None = None, diff --git a/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py index 96862da90a..d8f80d7853 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py +++ b/python/sglang/multimodal_gen/runtime/pipelines/minimax_h3_pipeline.py @@ -49,7 +49,7 @@ class MiniMaxH3Pipeline(LoRAPipeline, ComposedPipelineBase): # scheduler intentionally absent: model_index carries scheduler=null; # per-modality sigma schedules are generated in TimestepPreparation # from the task profile, and the loop scheduler math lives in - # scheduling_minimax_h3_euler_ancestral (stages accept scheduler=None). + # minimax_h3/denoise_loop.py (stages accept scheduler=None). "transformer", ] @@ -209,7 +209,7 @@ class FastH3Pipeline(MiniMaxH3Pipeline): """FastH3: 8-step DMD2-distilled MiniMax-H3 (V2, t2va only). The flat single-partition repo is materialized into the base-H3 layout by - the bundled model overlay (see model_overlays/), so every stage, loader, + its registered model overlay (see runtime/utils/model_overlay.py), so every stage, loader, and admission path below is exactly the MiniMax-H3 one. There is no FL2VA/Ref2VA partition layout to default into. """ diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py b/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py index c3b175ae5d..ab3a64f468 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/schedule_batch.py @@ -349,7 +349,6 @@ class Req: self.suppress_logs = True self.metrics.suppress_stage_breakdown = True self.extra["cache_dit_num_inference_steps"] = self.num_inference_steps - self.extra["warmup_target_num_inference_steps"] = self.num_inference_steps floor = getattr(type(self.sampling_params), "min_num_inference_steps", 1) self.num_inference_steps = max(warmup_steps, floor) @@ -358,34 +357,6 @@ class Req: req.set_as_warmup(warmup_steps) return req - def record_stage_iterations( - self, - measured_iterations: int, - target_iterations: int | None = None, - ) -> None: - """Record a stage loop against its full default-request work. - - Most stages declare the count as a formula of the step count - (``PipelineStage.default_workload_iterations``) and never call this. - It is for loops whose length is only known inside them (chunked or - block-wise schedules); ``target_iterations`` defaults to scaling the - measured count from the probe's steps to the default workload's. - """ - if not self.is_warmup or self.metrics is None: - return - measured = max(1, int(measured_iterations)) - if target_iterations is None: - measured_request_steps = max(1, int(self.num_inference_steps)) - target_request_steps = int( - self.extra.get( - "warmup_target_num_inference_steps", measured_request_steps - ) - ) - target_iterations = ( - measured * max(1, target_request_steps) + measured_request_steps - 1 - ) // measured_request_steps - self.metrics.record_stage_iterations(measured, target_iterations) - def validate(self): """Initialize dependent fields after dataclass initialization.""" if getattr(self.sampling_params, "data_type", None) == DataType.ACTION: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/__init__.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/__init__.py index d0bd576d50..38732f8998 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/__init__.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/__init__.py @@ -24,7 +24,6 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import Denois from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising_dmd import ( DmdDenoisingStage, ) -from sglang.multimodal_gen.runtime.pipelines_core.stages.encoding import EncodingStage from sglang.multimodal_gen.runtime.pipelines_core.stages.image_encoding import ( ImageEncodingStage, ImageVAEEncodingStage, @@ -59,7 +58,6 @@ __all__ = [ "DenoisingStage", "DmdDenoisingStage", "CausalDMDDenoisingStage", - "EncodingStage", "ConditionEncodingStage", "AuxiliaryConditionEncodingStage", "DecodingStage", diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py index 3a9851a24d..2451034162 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/base.py @@ -55,31 +55,6 @@ class StageVerificationError(Exception): pass -def record_default_workload_iterations(stage, batch) -> None: - """Record how often a stage's repeated unit ran, once per warmup request. - - A stage whose loop length is a formula of the request's step count - declares it in ``default_workload_iterations``; the same formula at the - default workload's step count is the target. A stage that recorded - explicitly inside its loop (a count only known there) is left alone. - """ - metrics = batch.metrics - if metrics is None or metrics.active_stage_name is None: - return - if metrics.active_stage_name in metrics.stage_iterations: - return - measured = stage.default_workload_iterations(batch, int(batch.num_inference_steps)) - if measured is None: - return - target_steps = int( - batch.extra.get("warmup_target_num_inference_steps", batch.num_inference_steps) - ) - target = stage.default_workload_iterations(batch, target_steps) - metrics.record_stage_iterations( - max(1, int(measured)), max(1, int(measured if target is None else target)) - ) - - class PipelineStage(StageDedupMixin, ABC): """ Abstract base class for all pipeline stages. @@ -179,14 +154,6 @@ class PipelineStage(StageDedupMixin, ABC): def set_profile_stage_name(self, stage_name: str) -> None: self._profile_stage_name = stage_name - def default_workload_iterations( - self, batch: Req, num_inference_steps: int - ) -> int | None: - """How many times this stage's repeated unit runs for a request with - ``num_inference_steps`` steps; ``None`` means the stage is not repeated - (or records its count itself with ``batch.record_stage_iterations``).""" - return None - def _component_stage_name(self, stage_name: str | None = None) -> str: return stage_name or self._registered_stage_name or self.__class__.__name__ @@ -407,13 +374,7 @@ class PipelineStage(StageDedupMixin, ABC): metrics = batch.metrics if metrics is not None and self.role_affinity == RoleType.DENOISER: metrics.denoising_stages.add(stage_name) - warmup_metrics = metrics if batch.is_warmup else None - previous_active_stage = ( - warmup_metrics.active_stage_name if warmup_metrics is not None else None - ) self._current_batch_is_warmup = batch.is_warmup - if warmup_metrics is not None: - warmup_metrics.active_stage_name = self._component_stage_name() try: with StageProfiler( stage_name, @@ -424,11 +385,7 @@ class PipelineStage(StageDedupMixin, ABC): perf_dump_path_provided=batch.perf_dump_path is not None, ): result = self.forward(batch, server_args) - if warmup_metrics is not None: - record_default_workload_iterations(self, batch) finally: - if warmup_metrics is not None: - warmup_metrics.active_stage_name = previous_active_stage self._current_batch_is_warmup = previous_batch_is_warmup self._current_use_nvtx = False diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/causal_denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/causal_denoising.py index ff9aeba24a..0a9db62d79 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/causal_denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/causal_denoising.py @@ -145,12 +145,6 @@ class CausalDMDRealtimeCacheContext: class CausalDMDDenoisingStage(DenoisingStage): - def default_workload_iterations( - self, batch: Req, num_inference_steps: int - ) -> int | None: - # blocks x fixed DMD steps, known only once the block sizes are laid out - return None - """ Denoising stage for causal diffusion. """ @@ -1259,9 +1253,6 @@ class CausalDMDDenoisingStage(DenoisingStage): block_sizes = [1] + [self.num_frames_per_block] * num_blocks start_index = 0 - total_iterations = len(block_sizes) * len(timesteps) - batch.record_stage_iterations(total_iterations, total_iterations) - def prepare_context_input(current_latents): return current_latents diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py index 4ee2e65f7d..7604f562eb 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py @@ -21,34 +21,21 @@ import torch import torch.nn as nn from sglang.kernels.ops.diffusion import ( - mount_flux2_nvfp4_swiglu_quant, - mount_fused_gate_rmsnorm, - mount_fused_linear_gelu, - mount_fused_ln_modulate, - mount_helios_gated_residual, - mount_hunyuan_qknorm, - mount_lingbot_video_gated_residual, - mount_lingbot_video_rmsnorm, - mount_ltx2_qknorm_split_rope, - mount_ltx2_rms_norm_modulate, - mount_minimax_h3_norm_modulate, - mount_nvfp4_bias_gelu, - mount_qwen_image_added_qkv, - mount_sana_video_linear_attention, - unmount_flux2_nvfp4_swiglu_quant, - unmount_fused_gate_rmsnorm, - unmount_fused_linear_gelu, - unmount_fused_ln_modulate, - unmount_helios_gated_residual, - unmount_hunyuan_qknorm, - unmount_lingbot_video_gated_residual, - unmount_lingbot_video_rmsnorm, - unmount_ltx2_qknorm_split_rope, - unmount_ltx2_rms_norm_modulate, - unmount_minimax_h3_norm_modulate, - unmount_nvfp4_bias_gelu, - unmount_qwen_image_added_qkv, - unmount_sana_video_linear_attention, + FLUX2_NVFP4_SWIGLU_QUANT, + FUSED_GATE_RMSNORM, + FUSED_LINEAR_GELU, + FUSED_LN_MODULATE, + HELIOS_GATED_RESIDUAL, + HUNYUAN_QKNORM, + LINGBOT_VIDEO_GATED_RESIDUAL, + LINGBOT_VIDEO_RMSNORM, + LTX2_QKNORM_SPLIT_ROPE, + LTX2_RMS_NORM_MODULATE, + MINIMAX_H3_NORM_MODULATE, + NVFP4_BIAS_GELU, + QWEN_IMAGE_ADDED_QKV, + SANA_VIDEO_LINEAR_ATTENTION, + QualityGatedFusion, ) from sglang.multimodal_gen import envs from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType, STA_Mode @@ -186,101 +173,43 @@ logger = init_logger(__name__) # to "high". No entry is tier "exact": the default must keep running the same # kernels it runs today, so a by-construction bit-exact fast path has to come # with a BitExactFusionGate before it can move there. -_QUALITY_FUSION_HANDLERS: tuple[ - tuple[str, str, Callable[[nn.Module], bool], Callable[[nn.Module], None]], ... -] = ( - ( - # quantizes FC2's input before the reference BF16 intermediate exists - "high", - "FLUX.2 NVFP4 FC1+SwiGLU+quant", - mount_flux2_nvfp4_swiglu_quant, - unmount_flux2_nvfp4_swiglu_quant, - ), - ( - "lossless", - "fused linear+GELU (cublasLt epilogue)", - mount_fused_linear_gelu, - unmount_fused_linear_gelu, - ), - ( - "lossless", - "Wan NVFP4 fused bias+GELU", - mount_nvfp4_bias_gelu, - unmount_nvfp4_bias_gelu, - ), - ( - "lossless", - "Qwen-Image fused added-QKV", - mount_qwen_image_added_qkv, - unmount_qwen_image_added_qkv, - ), - ( - "lossless", - "fused LN+modulate (affine folding)", - mount_fused_ln_modulate, - unmount_fused_ln_modulate, - ), - ( - "lossless", - "LTX-2 Hopper QKNorm+split-RoPE", - mount_ltx2_qknorm_split_rope, - unmount_ltx2_qknorm_split_rope, - ), - ( - "lossless", - "LTX-2 fused RMSNorm+modulate", - mount_ltx2_rms_norm_modulate, - unmount_ltx2_rms_norm_modulate, - ), - ( - # Ideogram's reference keeps the norm statistics in fp32; the - # BF16-native kernel rounds them, which lowers accumulation precision - "high", - "fused gate RMSNorm (BF16-native Triton)", - mount_fused_gate_rmsnorm, - unmount_fused_gate_rmsnorm, - ), - ( - "lossless", - "HunyuanVideo strided QK RMSNorm", - mount_hunyuan_qknorm, - unmount_hunyuan_qknorm, - ), - ( - "lossless", - "LingBot Video fused RMSNorm", - mount_lingbot_video_rmsnorm, - unmount_lingbot_video_rmsnorm, - ), - ( - "lossless", - "LingBot Video per-token gated residual", - mount_lingbot_video_gated_residual, - unmount_lingbot_video_gated_residual, - ), - ( - "lossless", - "Helios per-token gated residual", - mount_helios_gated_residual, - unmount_helios_gated_residual, - ), - ( - "lossless", - "MiniMax-H3 fused RMSNorm + AdaLN", - mount_minimax_h3_norm_modulate, - unmount_minimax_h3_norm_modulate, - ), - ( - # the first attention GEMM takes BF16 inputs where the reference - # promotes them to FP32 - "high", - "SANA-Video BF16-input linear attention", - mount_sana_video_linear_attention, - unmount_sana_video_linear_attention, - ), +_QUALITY_FUSION_HANDLERS: tuple[tuple[str, QualityGatedFusion], ...] = ( + # quantizes FC2's input before the reference BF16 intermediate exists + ("high", FLUX2_NVFP4_SWIGLU_QUANT), + ("lossless", FUSED_LINEAR_GELU), + ("lossless", NVFP4_BIAS_GELU), + ("lossless", QWEN_IMAGE_ADDED_QKV), + ("lossless", FUSED_LN_MODULATE), + ("lossless", LTX2_QKNORM_SPLIT_ROPE), + ("lossless", LTX2_RMS_NORM_MODULATE), + # Ideogram's reference keeps the norm statistics in fp32; the + # BF16-native kernel rounds them, which lowers accumulation precision + ("high", FUSED_GATE_RMSNORM), + ("lossless", HUNYUAN_QKNORM), + ("lossless", LINGBOT_VIDEO_RMSNORM), + ("lossless", LINGBOT_VIDEO_GATED_RESIDUAL), + ("lossless", HELIOS_GATED_RESIDUAL), + ("lossless", MINIMAX_H3_NORM_MODULATE), + # the first attention GEMM takes BF16 inputs where the reference + # promotes them to FP32 + ("high", SANA_VIDEO_LINEAR_ATTENTION), ) +def toggle_quality_fusions(transformers: list[nn.Module | None], quality: str) -> None: + """Mount the fusions ``quality`` allows on each transformer, unmount the rest.""" + mounted_fusions: set[str] = set() + for transformer in filter(None, transformers): + for tier, fusion in _QUALITY_FUSION_HANDLERS: + if quality_allows(quality, tier): + if fusion.mount(transformer): + mounted_fusions.add(fusion.name) + else: + fusion.unmount(transformer) + for name in sorted(mounted_fusions): + logger.debug("Mounted %s for quality=%s", name, quality) + + def _ensure_tensor_model_output(model_output): sample = getattr(model_output, "sample", None) if isinstance(sample, torch.Tensor): @@ -373,11 +302,6 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): def role_affinity(self): return RoleType.DENOISER - def default_workload_iterations( - self, batch: Req, num_inference_steps: int - ) -> int | None: - return num_inference_steps - def __init__( self, transformer, scheduler, pipeline=None, transformer_2=None, vae=None ) -> None: @@ -836,18 +760,8 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): quality = getattr(batch.sampling_params, "quality", "lossless") if quality == self._mounted_quality: return - mounted_fusions: set[str] = set() - for transformer in filter(None, [self.transformer, self.transformer_2]): - for tier, description, mount, unmount in _QUALITY_FUSION_HANDLERS: - if quality_allows(quality, tier): - if mount(transformer): - mounted_fusions.add(description) - else: - unmount(transformer) - + toggle_quality_fusions([self.transformer, self.transformer_2], quality) self._mounted_quality = quality - for description in sorted(mounted_fusions): - logger.debug("Mounted %s for quality=%s", description, quality) def _cache_dit_dual_model_name(self) -> str: return "wan2.2" diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising_dmd.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising_dmd.py index 2e7848571e..5cefe0a7f8 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising_dmd.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising_dmd.py @@ -25,12 +25,6 @@ logger = init_logger(__name__) class DmdDenoisingStage(DenoisingStage): - def default_workload_iterations( - self, batch: Req, num_inference_steps: int - ) -> int | None: - # a fixed distilled schedule: the same count at any requested step count - return len(self.server_args.pipeline_config.dmd_denoising_steps) - """ Denoising stage for DMD. """ diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encoding.py deleted file mode 100644 index b73b879933..0000000000 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encoding.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo - -# SPDX-License-Identifier: Apache-2.0 -""" -Encoding stage for diffusion pipelines. -""" - -import torch - -from sglang.multimodal_gen.runtime.distributed import get_local_torch_device -from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( - ComponentUse, -) -from sglang.multimodal_gen.runtime.models.vaes.common import ParallelTiledVAE -from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req -from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage -from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( - V, # Import validators -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( - VerificationResult, -) -from sglang.multimodal_gen.runtime.server_args import ServerArgs -from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger -from sglang.multimodal_gen.runtime.utils.precision import ( - autocast_context, - autocast_enabled, - resolve_precision, - temporary_module_dtype, -) - -logger = init_logger(__name__) - - -class EncodingStage(PipelineStage): - """ - Stage for encoding pixel space representations into latent space. - - This stage handles the encoding of pixel-space video/images into latent - representations for further processing in the diffusion pipeline. - """ - - def __init__(self, vae: ParallelTiledVAE) -> None: - super().__init__() - self.vae: ParallelTiledVAE = vae - - def component_uses( - self, server_args: ServerArgs, stage_name: str | None = None - ) -> list[ComponentUse]: - vae_dtype = resolve_precision( - server_args, "vae", precision_attr="vae_precision" - ) - stage_name = self._component_stage_name(stage_name) - return [ - ComponentUse( - stage_name, - "vae", - target_dtype=vae_dtype, - ) - ] - - @torch.no_grad() - def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: - """Verify encoding stage inputs.""" - result = VerificationResult() - # Input video/images for VAE encoding: [batch_size, channels, frames, height, width] - result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) - return result - - def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: - """Verify encoding stage outputs.""" - result = VerificationResult() - # Encoded latents: [batch_size, channels, frames, height_latents, width_latents] - result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)]) - return result - - def forward( - self, - batch: Req, - server_args: ServerArgs, - ) -> Req: - """ - Encode pixel space representations into latent space. - - - - Returns: - The batch with encoded latents. - """ - assert batch.latents is not None and isinstance(batch.latents, torch.Tensor) - - # Setup VAE precision from user policy. - vae_dtype = resolve_precision( - server_args, "vae", precision_attr="vae_precision" - ) - vae_autocast_enabled = autocast_enabled(vae_dtype, server_args.disable_autocast) - - # Normalize input to [-1, 1] range (reverse of decoding normalization) - latents = (batch.latents * 2.0 - 1.0).clamp(-1, 1) - - # Move to appropriate device and dtype - latents = latents.to(get_local_torch_device()) - - with self.use_declared_component(component_name="vae", module=self.vae) as vae: - assert vae is not None - self.vae = vae - - # Encode image to latents - with autocast_context(vae_dtype, server_args.disable_autocast): - if server_args.pipeline_config.vae_tiling: - self.vae.enable_tiling() - # if server_args.vae_sp: - # self.vae.enable_parallel() - should_cast_vae = not vae_autocast_enabled - if not vae_autocast_enabled: - latents = latents.to(vae_dtype) - with temporary_module_dtype( - self.vae, vae_dtype, enabled=should_cast_vae - ) as vae: - latents = vae.encode(latents).mean - - # Update batch with encoded latents - batch.latents = latents - - return batch diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/cosmos3.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/cosmos3.py index 7c3eda3933..35084f3c81 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/cosmos3.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/cosmos3.py @@ -1060,11 +1060,6 @@ class Cosmos3TimestepPreparationStage(PipelineStage): class Cosmos3DenoisingStage(PipelineStage, RolloutDenoisingMixin): - def default_workload_iterations( - self, batch: Req, num_inference_steps: int - ) -> int | None: - return num_inference_steps - """Cosmos3 denoise loop, including CFG and the parallelism modes. The UND pathway runs once and its K/V is cached per cache_key (``cond`` / diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py index 91ef3fd9ef..2b6dcd2604 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/helios_denoising.py @@ -13,13 +13,6 @@ import numpy as np import torch import torch.nn.functional as F -from sglang.kernels.ops.diffusion import ( - mount_helios_gated_residual, - unmount_helios_gated_residual, -) -from sglang.multimodal_gen.configs.sample.sampling_params import ( - quality_allows, -) from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( @@ -30,6 +23,9 @@ from sglang.multimodal_gen.runtime.pipelines_core.diffusion_scheduler_utils impo ) from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import ( + toggle_quality_fusions, +) from sglang.multimodal_gen.runtime.server_args import ServerArgs from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.runtime.utils.perf_logger import StageProfiler @@ -110,27 +106,14 @@ class HeliosChunkedDenoisingStage(PipelineStage): super().__init__() self.transformer = transformer self.scheduler = scheduler - self._quality_fusions_mounted = False + self._mounted_quality = "exact" def _maybe_toggle_quality_fusions(self, batch: Req) -> None: - # The gated-residual fusion rounds the FP32 gate and update to BF16 - # before multiplying, which moves a rounding without lowering the - # reference's own operand precision: tier "lossless". The "exact" - # default keeps the reference FP32-multiply form bit-for-bit. quality = getattr(batch.sampling_params, "quality", "lossless") - want = quality_allows(quality, "lossless") - if want == self._quality_fusions_mounted: + if quality == self._mounted_quality: return - self._quality_fusions_mounted = want - if self.transformer is None: - return - if want: - if mount_helios_gated_residual(self.transformer): - logger.debug( - "Mounted Helios per-token gated residual for quality=%s", quality - ) - else: - unmount_helios_gated_residual(self.transformer) + toggle_quality_fusions([self.transformer], quality) + self._mounted_quality = quality @property def role_affinity(self) -> RoleType: @@ -770,9 +753,5 @@ class HeliosChunkedDenoisingStage(PipelineStage): # separately to avoid temporal artifacts at chunk boundaries. batch.latent_chunks = chunk_latents_list batch.latents = history_latents[:, :, -total_generated_latent_frames:] - batch.record_stage_iterations( - global_step_offset, - global_step_offset if is_enable_stage2 else None, - ) return batch diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_world/lingbot_world_causal_denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_world/lingbot_world_causal_denoising.py index 4df75705bb..b3285730f8 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_world/lingbot_world_causal_denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_world/lingbot_world_causal_denoising.py @@ -730,7 +730,6 @@ class LingBotWorldCausalDMDDenoisingStage(CausalDMDDenoisingStage): "Ensure ImageVAEEncodingStage runs before this stage." ) ctx = self._prepare_causal_dmd_forward_context(batch, server_args) - batch.record_stage_iterations(len(ctx.timesteps), len(ctx.timesteps)) latents = ctx.latents cache_ctx = self._prepare_realtime_causal_caches(batch, server_args, ctx) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/longlive2.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/longlive2.py index 59e85ff0fc..c58bc10706 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/longlive2.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/longlive2.py @@ -396,9 +396,6 @@ class LongLive2CausalDenoisingStage(CausalDMDDenoisingStage): num_blocks = (t - 1) // self.num_frames_per_block block_sizes = [1] + [self.num_frames_per_block] * num_blocks - total_iterations = len(block_sizes) * len(timesteps) - batch.record_stage_iterations(total_iterations, total_iterations) - start_index = 0 self._validate_block_prompt_count(batch, block_sizes) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising_av.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising_av.py index 498f291665..1cfeaeb58f 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising_av.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising_av.py @@ -107,12 +107,6 @@ class LTX2AVDenoisingStage(LTX2DenoisingStage): class LTX2RefinementStage(LTX2AVDenoisingStage): - def default_workload_iterations( - self, batch: Req, num_inference_steps: int - ) -> int | None: - # the refiner runs its distilled sigma schedule regardless of the request's steps - return max(1, len(self.distilled_sigmas) - 1) - """Stage-2 refinement wrapper that re-noises distilled LTX latents once.""" def __init__( diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py index 328359e124..d4c9e3d71f 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py @@ -443,11 +443,6 @@ def _precompute_rope_cache( class MiniMaxH3DenoisingStage(DenoisingStage): - def default_workload_iterations( - self, batch: Req, num_inference_steps: int - ) -> int | None: - return num_inference_steps - def __init__(self, transformer, pipeline=None) -> None: super().__init__( transformer=transformer, diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py index b1f9c39e06..d3657c99f6 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/mova.py @@ -153,11 +153,6 @@ class MOVATimestepPreparationStage(PipelineStage): class MOVADenoisingStage(PipelineStage): - def default_workload_iterations( - self, batch: Req, num_inference_steps: int - ) -> int | None: - return num_inference_steps - """Run MOVA dual-tower denoising loop.""" def __init__(self, video_dit, video_dit_2, audio_dit, dual_tower_bridge, scheduler): diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_video2.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_video2.py index 7df4778975..8e73563b65 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_video2.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_video2.py @@ -221,9 +221,6 @@ class SanaVideo2DenoisingStage(PipelineStage): def component_uses(self, server_args, stage_name=None): return [ComponentUse(self._component_stage_name(stage_name), "transformer")] - def default_workload_iterations(self, batch, num_inference_steps): - return num_inference_steps - @torch.no_grad() def forward(self, batch, server_args): self.begin_declared_component_use( diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/realtime_chain.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/realtime_chain.py index a271a427b6..81d1ba54a5 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/realtime_chain.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/realtime_chain.py @@ -26,7 +26,6 @@ import torch from sglang.multimodal_gen.runtime.distributed import get_local_torch_device from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context -from sglang.multimodal_gen.runtime.models.dits import sana_wm_parity as parity_probe from sglang.multimodal_gen.runtime.models.dits.sana_wm_components import ( compute_chunk_plucker, ) @@ -254,11 +253,6 @@ class SanaWMRealtimeLatentPrepStage(SanaWMRealtimeStage): ) buf[:, :, :1] = first_latent noise.noise_buffer = buf - parity_probe.dump_tensor( # parity harness (no-op in prod) - parity_probe.probe_dir(parity_probe.ENV_RT_DUMP), - "noise_buffer", - buf, - ) # Chunk plan: enough stage-1 chunks for >=1 refiner block. stage1_cur = cache.chunk_indices[-1] diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/refiner.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/refiner.py index 205a9e1c20..adbd290077 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/refiner.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/refiner.py @@ -728,8 +728,6 @@ class SanaWMLTX2RefinerStage(PipelineStage): return batch batch_size = int(batch.latents.shape[0]) - total_iterations = batch_size * (len(STAGE_2_DISTILLED_SIGMA_VALUES) - 1) - batch.record_stage_iterations(total_iterations, total_iterations) prompts = self._prompts_for_batch(batch, batch_size) fps = float(getattr(batch, "fps", 16) or 16) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/streaming.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/streaming.py index f1614b8e4d..9ba8df15c9 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/streaming.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/streaming.py @@ -14,8 +14,6 @@ Gotchas vs the dense ``SanaWMDenoisingStage`` (one-shot bidirectional): from __future__ import annotations -# --- debug parity harness (gated by env; no-op in production) --- -import os as _os import time from contextlib import nullcontext from types import SimpleNamespace @@ -27,7 +25,6 @@ from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_c from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( ComponentUse, ) -from sglang.multimodal_gen.runtime.models.dits import sana_wm_parity as parity_probe from sglang.multimodal_gen.runtime.models.dits.sana_wm import ( _NUM_STREAM_CACHE_SLOTS, ) @@ -58,8 +55,6 @@ from .base import ( log_sana_wm_tensor_stats, ) -_SANAWM_INJECT_DIR = _os.environ.get(parity_probe.ENV_INJECT) - def self_forcing_denoise_chunk( *, @@ -82,7 +77,6 @@ def self_forcing_denoise_chunk( target_dtype: torch.dtype, device, forward_ctx=None, - on_noise_pred=None, ) -> list: """Self-forcing denoise of ONE chunk + the clean t=0 KV pass; returns the chunk's updated per-block KV cache. @@ -126,9 +120,6 @@ def self_forcing_denoise_chunk( noise_uncond, noise_text = noise_pred.chunk(2) noise_pred = noise_uncond + cfg_scale * (noise_text - noise_uncond) - if on_noise_pred is not None: - on_noise_pred(int(t.item()), noise_pred) - denoised = scheduler.step( -noise_pred.reshape(B, C, -1).transpose(1, 2), t, @@ -284,15 +275,6 @@ class SanaWMStreamingDenoisingStage(CausalDMDDenoisingStage): # the DiT stays device-resident for the session's lifetime. transformer = self.transformer.to(device=device).eval() num_blocks = len(transformer.blocks) - _dump_dir = parity_probe.probe_dir(parity_probe.ENV_RT_DUMP) - if _dump_dir and state.chunk_idx == 0: # parity harness - parity_probe.dump_tensor(_dump_dir, "cond_embeds", sc.embeds) - parity_probe.dump_tensor(_dump_dir, "cond_mask", sc.mask) - parity_probe.dump_obj( - _dump_dir, - "dit_fingerprint", - parity_probe.weights_fingerprint(transformer), - ) offset = 0 for n in plan: @@ -330,18 +312,6 @@ class SanaWMStreamingDenoisingStage(CausalDMDDenoisingStage): sampler_cfg.num_cached_blocks, num_blocks, ) - if _dump_dir: # parity harness - parity_probe.dump_obj( - _dump_dir, - f"kv_probe_{chunk_idx:03d}", - parity_probe.kv_cache_checksums(chunk_kv, sink_num), - ) - parity_probe.dump_tensor( - _dump_dir, f"cond_camera_{chunk_idx:03d}", sc.camera - ) - parity_probe.dump_tensor( - _dump_dir, f"cond_plucker_{chunk_idx:03d}", sc.plucker - ) frame_index = ( torch.arange(start_f, end_f, device=device, dtype=torch.long) if sink_num > 0 @@ -375,12 +345,6 @@ class SanaWMStreamingDenoisingStage(CausalDMDDenoisingStage): lat[:, :, loc] = init_chunk[:, :, loc] _local["lat"] = lat - def _on_noise_pred(ts_int, noise_pred, chunk_idx=chunk_idx): - if _dump_dir and chunk_idx == 0: # parity harness - parity_probe.dump_tensor( - _dump_dir, f"noise_pred_c0_t{ts_int}", noise_pred - ) - # Same forward context the mega-stage held across the tick (the # per-chunk denoise core relies on an ambient context here; the # offline path supplies per-step contexts instead). @@ -408,7 +372,6 @@ class SanaWMStreamingDenoisingStage(CausalDMDDenoisingStage): cfg_scale=sc.cfg_scale, target_dtype=target_dtype, device=device, - on_noise_pred=_on_noise_pred, ) chunk_lat = _local["lat"] state.latents = ( @@ -416,9 +379,6 @@ class SanaWMStreamingDenoisingStage(CausalDMDDenoisingStage): if state.latents is None else torch.cat([state.latents, chunk_lat], dim=2) ) - parity_probe.dump_tensor( # parity harness - _dump_dir, f"stage1_{chunk_idx:03d}_{start_f}_{end_f}", chunk_lat - ) state.chunk_idx += 1 state.current_chunk_start_frame = end_f @@ -433,7 +393,6 @@ class SanaWMStreamingDenoisingStage(CausalDMDDenoisingStage): *, device: torch.device, target_dtype: torch.dtype, - iload=None, ): """Resolve sampler config + text/camera conditioning (shared by the offline loop and realtime path).""" pcfg = server_args.pipeline_config @@ -484,12 +443,6 @@ class SanaWMStreamingDenoisingStage(CausalDMDDenoisingStage): ) embeds_in = torch.cat([neg_embeds, pos_embeds], dim=0) if do_cfg else pos_embeds mask_in = _cat_optional_tensors(neg_mask, pos_mask) if do_cfg else pos_mask - if _SANAWM_INJECT_DIR and iload is not None and not do_cfg: - _cond = iload("cond").to(device=device, dtype=target_dtype) - while _cond.dim() > embeds_in.dim(): - _cond = _cond.squeeze(1) - embeds_in = _cond - mask_in = iload("cond_mask").to(device=device) # --- camera / plücker (FULL length; forward_long windows internally) --- extra = batch.extra or {} @@ -509,12 +462,6 @@ class SanaWMStreamingDenoisingStage(CausalDMDDenoisingStage): if do_cfg and chunk_plucker is not None else chunk_plucker ) - if _SANAWM_INJECT_DIR and iload is not None: - cam_in = iload("raymap").to(device=device, dtype=target_dtype) - plk_in = iload("chunk_plucker").to(device=device, dtype=target_dtype) - if do_cfg: - cam_in = torch.cat([cam_in, cam_in], dim=0) - plk_in = torch.cat([plk_in, plk_in], dim=0) return SimpleNamespace( sampler_cfg=sampler_cfg, @@ -554,29 +501,8 @@ class SanaWMStreamingDenoisingStage(CausalDMDDenoisingStage): init_latents = latents.clone() B, C, total_frames, H, W = latents.shape - def _iload(_name): - return torch.load(f"{_SANAWM_INJECT_DIR}/{_name}.pt", map_location=device) - - _dump_dir = parity_probe.probe_dir(parity_probe.ENV_FORK_DUMP) - - def _fdump(_name, _t): - parity_probe.dump_tensor(_dump_dir, _name, _t) - - if ( - _SANAWM_INJECT_DIR - ): # parity harness: run the OFFICIAL's exact stage-1 inputs - latents = ( - _iload("z_full_initial").to(device=device, dtype=target_dtype).clone() - ) - init_latents = latents.clone() - B, C, total_frames, H, W = latents.shape - - _fdump( - "init_noise", init_latents - ) # parity harness: seeded pre-noise (cond @ frame 0) - sc = self._resolve_stream_conditioning( - batch, server_args, device=device, target_dtype=target_dtype, iload=_iload + batch, server_args, device=device, target_dtype=target_dtype ) sampler_cfg = sc.sampler_cfg num_frame_per_block = sampler_cfg.num_frame_per_block @@ -587,13 +513,6 @@ class SanaWMStreamingDenoisingStage(CausalDMDDenoisingStage): do_cfg = sc.do_cfg embeds_in, mask_in, cam_in, plk_in = sc.embeds, sc.mask, sc.camera, sc.plucker - # parity harness: full-length conditioning fed to forward_long (windowed - # internally per chunk via [start_f:end_f]). - _fdump("cond_embeds", embeds_in) - _fdump("cond_mask", mask_in) - _fdump("cond_camera", cam_in) - _fdump("cond_plucker", plk_in) - scheduler = FlowMatchEulerDiscreteScheduler(shift=1.0) chunk_indices = self._autoregressive_segments(total_frames, num_frame_per_block) @@ -610,12 +529,6 @@ class SanaWMStreamingDenoisingStage(CausalDMDDenoisingStage): assert transformer is not None self.transformer = transformer num_blocks = len(transformer.blocks) - if _dump_dir: # parity harness: weights fingerprint - parity_probe.dump_obj( - _dump_dir, - "dit_fingerprint", - parity_probe.weights_fingerprint(transformer), - ) kv_cache = [ [[None] * _NUM_STREAM_CACHE_SLOTS for _ in range(num_blocks)] for _ in range(num_chunks) @@ -630,8 +543,6 @@ class SanaWMStreamingDenoisingStage(CausalDMDDenoisingStage): len(explicit_sigmas), do_cfg, ) - total_iterations = num_chunks * len(explicit_sigmas) - batch.record_stage_iterations(total_iterations, total_iterations) for chunk_idx in self.progress_bar(range(num_chunks), batch=batch): chunk_kv, sink_num = self._accumulate_kv_cache( @@ -642,12 +553,6 @@ class SanaWMStreamingDenoisingStage(CausalDMDDenoisingStage): sink_token, num_blocks, ) - if _dump_dir: # parity harness: accumulated-KV checksums - parity_probe.dump_obj( - _dump_dir, - f"kv_probe_{chunk_idx:03d}", - parity_probe.kv_cache_checksums(chunk_kv, sink_num), - ) start_f = chunk_indices[chunk_idx] end_f = chunk_indices[chunk_idx + 1] chunk_frames = end_f - start_f @@ -685,10 +590,6 @@ class SanaWMStreamingDenoisingStage(CausalDMDDenoisingStage): forward_batch=batch, ) - def _on_noise_pred(ts_int, noise_pred, chunk_idx=chunk_idx): - if chunk_idx == 0: # parity harness: per-step model output - _fdump(f"noise_pred_c0_t{ts_int}", noise_pred) - kv_cache[chunk_idx] = self_forcing_denoise_chunk( transformer=transformer, scheduler=scheduler, @@ -709,11 +610,6 @@ class SanaWMStreamingDenoisingStage(CausalDMDDenoisingStage): target_dtype=target_dtype, device=device, forward_ctx=_forward_ctx, - on_noise_pred=_on_noise_pred, - ) - _fdump( - f"stage1_{chunk_idx:03d}_{start_f}_{end_f}", - latents[:, :, start_f:end_f], ) log_sana_wm_tensor_stats("stream.output_latents", latents) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/streaming_refiner.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/streaming_refiner.py index 7ba671187f..1e1dba5c8f 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/streaming_refiner.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/streaming_refiner.py @@ -19,7 +19,6 @@ import torch from torch import nn from sglang.multimodal_gen.runtime.distributed import get_local_torch_device -from sglang.multimodal_gen.runtime.models.dits import sana_wm_parity as parity_probe from sglang.multimodal_gen.runtime.models.dits.sana_wm_refiner_transformer import ( pack_latents, unpack_latents, @@ -445,32 +444,6 @@ class RefinerChunkRunner: def refine_block( self, *, block_idx, clean_block, block_start, block_end, sink_seed_frames=None ): - # parity harness (env-gated, no-op in prod): per-block input/config/output - # checksums; both the batch stage and the realtime stage run through here. - _probe_dir = parity_probe.probe_dir( - parity_probe.ENV_RT_DUMP, parity_probe.ENV_FORK_DUMP - ) - _probe = None - if _probe_dir: - _ck = parity_probe.checksum - _probe = { - "block_idx": int(block_idx), - "block_start": int(block_start), - "block_end": int(block_end), - "clean_block": _ck(clean_block), - "sink_seed_frames": _ck(sink_seed_frames), - "prompt_embeds": _ck(self.prompt_embeds), - "prompt_attention_mask": _ck(self.prompt_attention_mask), - "fps": self.fps, - "sigmas": [float(s) for s in self.sigmas], - "source_sink_frames": self.source_sink_frames, - "block_size": self.block_size, - "kv_max_frames": self.kv_max_frames, - "history_frames": self.history_frames, - "generator_state": float( - self.generator.get_state().double().sum().item() - ), - } del block_idx refiner = self.refiner if block_start < self.source_sink_frames: @@ -580,11 +553,6 @@ class RefinerChunkRunner: old[1][:, -keep_tokens:], ) self.history_frames = self.max_history_frames - if _probe is not None: - _probe["refined"] = parity_probe.checksum(x_t) - parity_probe.dump_obj( - _probe_dir, f"refiner_probe_{_probe['block_idx']:03d}", _probe - ) return x_t @@ -634,8 +602,6 @@ class SanaWMStreamingRefinerStage(SanaWMLTX2RefinerStage): ) return batch n_blocks = math.ceil(n_active / self.block_size) - total_iterations = n_blocks * (len(STAGE_2_DISTILLED_SIGMA_VALUES) - 1) - batch.record_stage_iterations(total_iterations, total_iterations) self.log_info( "SANA-WM streaming refiner: latent=%s, sink=%d, block=%d, blocks=%d, kv_max=%d, seed=%d", tuple(latents.shape), diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/validators.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/validators.py index 1ca9e992d7..b10b8c5901 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/validators.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/validators.py @@ -32,11 +32,6 @@ class StageValidators: """Check if value is a non-negative float.""" return isinstance(value, int | float) and value >= 0 - @staticmethod - def positive_float(value: Any) -> bool: - """Check if value is a positive float.""" - return isinstance(value, int | float) and value > 0 - @staticmethod def non_negative_float(value: Any) -> bool: """Check if value is a non-negative float.""" @@ -72,43 +67,11 @@ class StageValidators: return False return not torch.isnan(value).any().item() - @staticmethod - def tensor_shape_matches(value: Any, expected_shape: tuple) -> bool: - """Check if tensor shape matches expected shape (None for any size) and no NaN values.""" - if not isinstance(value, torch.Tensor): - return False - if len(value.shape) != len(expected_shape): - return False - for actual, expected in zip(value.shape, expected_shape, strict=True): - if expected is not None and actual != expected: - return False - return not torch.isnan(value).any().item() - @staticmethod def list_not_empty(value: Any) -> bool: """Check if value is a non-empty list.""" return isinstance(value, list) and len(value) > 0 - @staticmethod - def list_length(value: Any, length: int) -> bool: - """Check if list has specific length.""" - return isinstance(value, list) and len(value) == length - - @staticmethod - def list_min_length(value: Any, min_length: int) -> bool: - """Check if list has at least min_length items.""" - return isinstance(value, list) and len(value) >= min_length - - @staticmethod - def string_not_empty(value: Any) -> bool: - """Check if value is a non-empty string.""" - return isinstance(value, str) and len(value.strip()) > 0 - - @staticmethod - def string_not_none(value: Any) -> bool: - """Check if value is a non-empty string.""" - return isinstance(value, str) and len(value) > 0 - @staticmethod def string_or_list_strings(value: Any) -> bool: """Check if value is a string or list of strings.""" diff --git a/python/sglang/multimodal_gen/runtime/platforms/cpu.py b/python/sglang/multimodal_gen/runtime/platforms/cpu.py index bc493f3d40..8ea044f797 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/cpu.py +++ b/python/sglang/multimodal_gen/runtime/platforms/cpu.py @@ -60,10 +60,6 @@ class CpuPlatform(Platform): return psutil.virtual_memory().total - @classmethod - def is_async_output_supported(cls, enforce_eager: bool | None) -> bool: - return True - @classmethod def get_current_memory_usage( cls, device: torch.types.Device | None = None diff --git a/python/sglang/multimodal_gen/runtime/platforms/cuda.py b/python/sglang/multimodal_gen/runtime/platforms/cuda.py index d2c5a660fd..0c24b84951 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/cuda.py +++ b/python/sglang/multimodal_gen/runtime/platforms/cuda.py @@ -602,17 +602,6 @@ class CudaPlatformBase(Platform): def get_device_capability(cls, device_id: int = 0) -> DeviceCapability | None: raise NotImplementedError - @classmethod - def is_async_output_supported(cls, enforce_eager: bool | None) -> bool: - if enforce_eager: - logger.warning( - "To see benefits of async output processing, enable CUDA " - "graph. Since, enforce-eager is enabled, async output " - "processor cannot be used" - ) - return False - return True - @classmethod @lru_cache(maxsize=1) def get_modelopt_fp4_quantize_op(cls) -> Callable | None: diff --git a/python/sglang/multimodal_gen/runtime/platforms/interface.py b/python/sglang/multimodal_gen/runtime/platforms/interface.py index 5026ffacb4..5b7d2d4f27 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/interface.py +++ b/python/sglang/multimodal_gen/runtime/platforms/interface.py @@ -130,8 +130,6 @@ class Platform: # compilation strategy. simple_compile_backend: str = "inductor" - supported_quantization: list[str] = [] - def init_backend(self) -> None: """One-time backend initialization, in each worker; raising aborts startup. @@ -410,13 +408,6 @@ class Platform: or self.is_xpu() ) - @classmethod - def is_async_output_supported(cls, enforce_eager: bool | None) -> bool: - """ - Check if the current platform supports async output. - """ - raise NotImplementedError - @classmethod def inference_mode(cls): """A device-specific wrapper of `torch.inference_mode`. @@ -453,16 +444,6 @@ class Platform: """ pass - @classmethod - def verify_quantization(cls, quant: str) -> None: - """ - Verify whether the quantization is supported by the current platform. - """ - if cls.supported_quantization and quant not in cls.supported_quantization: - raise ValueError( - f"{quant} quantization is currently not supported in {cls.device_name}." - ) - @classmethod def get_current_memory_usage( cls, device: torch.types.Device | None = None diff --git a/python/sglang/multimodal_gen/runtime/platforms/mps.py b/python/sglang/multimodal_gen/runtime/platforms/mps.py index 31f793cb67..de8f10723e 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/mps.py +++ b/python/sglang/multimodal_gen/runtime/platforms/mps.py @@ -57,17 +57,6 @@ class MpsPlatform(Platform): return psutil.virtual_memory().total - @classmethod - def is_async_output_supported(cls, enforce_eager: bool | None) -> bool: - if enforce_eager: - logger.warning( - "To see benefits of async output processing, enable MPS " - "graph. Since, enforce-eager is enabled, async output " - "processor cannot be used" - ) - return False - return True - @classmethod def get_current_memory_usage( cls, device: torch.types.Device | None = None diff --git a/python/sglang/multimodal_gen/runtime/platforms/musa.py b/python/sglang/multimodal_gen/runtime/platforms/musa.py index af8ad1ce53..ca77e3bcdc 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/musa.py +++ b/python/sglang/multimodal_gen/runtime/platforms/musa.py @@ -84,17 +84,6 @@ class MusaPlatformBase(Platform): def get_device_capability(cls, device_id: int = 0) -> DeviceCapability | None: raise NotImplementedError - @classmethod - def is_async_output_supported(cls, enforce_eager: bool | None) -> bool: - if enforce_eager: - logger.warning( - "To see benefits of async output processing, enable MUSA " - "graph. Since, enforce-eager is enabled, async output " - "processor cannot be used" - ) - return False - return True - @classmethod def is_full_mtlink(cls, device_ids: list[int]) -> bool: raise NotImplementedError @@ -375,10 +364,3 @@ try: MusaPlatform.log_warnings() except ModuleNotFoundError: MusaPlatform.log_warnings() - -if __name__ == "__main__": - print(MusaPlatform.__name__) - print(MusaPlatform.get_device_name()) - print(MusaPlatform.get_device_capability()) - print(MusaPlatform.get_device_total_memory()) - print(MusaPlatform.is_full_mtlink([0, 1, 2, 3, 4, 5, 6, 7])) diff --git a/python/sglang/multimodal_gen/runtime/platforms/npu.py b/python/sglang/multimodal_gen/runtime/platforms/npu.py index 11aaa2d082..e2d2e955a1 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/npu.py +++ b/python/sglang/multimodal_gen/runtime/platforms/npu.py @@ -61,17 +61,6 @@ class NPUPlatformBase(Platform): device_props = torch.npu.get_device_properties(device_id) return int(device_props.total_memory) - @classmethod - def is_async_output_supported(cls, enforce_eager: bool | None) -> bool: - if enforce_eager: - logger.warning( - "To see benefits of async output processing, enable NPU " - "graph. Since, enforce-eager is enabled, async output " - "processor cannot be used" - ) - return False - return True - @classmethod def inference_mode(cls): # npu kernels in diffusion paths may need tensor version counters diff --git a/python/sglang/multimodal_gen/runtime/platforms/rocm.py b/python/sglang/multimodal_gen/runtime/platforms/rocm.py index d617b41ebd..0fba6bba88 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/rocm.py +++ b/python/sglang/multimodal_gen/runtime/platforms/rocm.py @@ -68,16 +68,6 @@ class RocmPlatform(Platform): def get_device_total_memory(cls, device_id: int = 0) -> int: return torch.cuda.get_device_properties(device_id).total_memory - @classmethod - def is_async_output_supported(cls, enforce_eager: bool | None) -> bool: - if enforce_eager: - logger.warning( - "To see benefits of async output processing, enable CUDA graph. " - "Since enforce-eager is enabled, async output processor cannot be used" - ) - return False - return True - @classmethod def get_current_memory_usage(cls, device: torch.device | None = None) -> float: torch.cuda.reset_peak_memory_stats(device) diff --git a/python/sglang/multimodal_gen/runtime/platforms/xpu.py b/python/sglang/multimodal_gen/runtime/platforms/xpu.py index d893a1cf73..ce8b39f2c4 100644 --- a/python/sglang/multimodal_gen/runtime/platforms/xpu.py +++ b/python/sglang/multimodal_gen/runtime/platforms/xpu.py @@ -51,17 +51,6 @@ class XpuPlatform(Platform): props = torch.xpu.get_device_properties(device_id) return props.total_memory - @classmethod - def is_async_output_supported(cls, enforce_eager: bool | None) -> bool: - """Check if async output is supported on Intel XPU.""" - if enforce_eager: - logger.warning( - "To see benefits of async output processing, disable enforce-eager. " - "Since enforce-eager is enabled, async output processor cannot be used" - ) - return False - return True - @classmethod def get_current_memory_usage( cls, device: torch.types.Device | None = None diff --git a/python/sglang/multimodal_gen/runtime/realtime/video.py b/python/sglang/multimodal_gen/runtime/realtime/video.py index 5782ed9e7a..ed61d70b34 100644 --- a/python/sglang/multimodal_gen/runtime/realtime/video.py +++ b/python/sglang/multimodal_gen/runtime/realtime/video.py @@ -3,7 +3,6 @@ from __future__ import annotations import time -import zlib from collections.abc import Callable, Sequence from typing import TYPE_CHECKING, Any @@ -21,84 +20,9 @@ if TYPE_CHECKING: logger = init_logger(__name__) RAW_RGB_CONTENT_TYPE = "application/x-raw-rgb" -RAW_RGB_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgb-delta-gzip" -RAW_RGBA_DELTA_GZIP_CONTENT_TYPE = "application/x-raw-rgba-delta-gzip" WEBP_FRAME_CONTENT_TYPE = "image/webp" JPEG_FRAME_CONTENT_TYPE = "image/jpeg" RAW_RGB_CHANNELS = 3 -_RAW_RGB_DELTA_GZIP_LEVEL = 0 - - -def build_delta_gzip_raw_rgb_payload( - frames: list[bytes], - *, - reference_frame: bytes | None = None, -) -> bytes: - if not frames: - return b"" - - frame_size = len(frames[0]) - if reference_frame is not None and len(reference_frame) != frame_size: - raise ValueError("raw RGB delta gzip reference frame size mismatch") - - previous = ( - np.frombuffer(reference_frame, dtype=np.uint8) - if reference_frame is not None - else None - ) - # keep gzip framing for lossless transport without spending realtime budget on compression - compressor = zlib.compressobj( - level=_RAW_RGB_DELTA_GZIP_LEVEL, method=zlib.DEFLATED, wbits=31 - ) - compressed_chunks = [] - for frame in frames: - if len(frame) != frame_size: - raise ValueError("raw RGB delta gzip requires fixed-size frames") - current = np.frombuffer(frame, dtype=np.uint8) - if previous is None: - delta_frame = frame - else: - delta_frame = np.bitwise_xor(current, previous).tobytes() - compressed_chunks.append(compressor.compress(delta_frame)) - previous = current - - compressed_chunks.append(compressor.flush()) - return b"".join(compressed_chunks) - - -def restore_delta_gzip_raw_rgb_payload( - payload: bytes, - *, - bytes_per_frame: int, - num_frames: int, - reference_frame: bytes | None = None, -) -> bytes: - if reference_frame is not None and len(reference_frame) != bytes_per_frame: - raise ValueError("delta gzip reference frame size mismatch") - - delta_payload = zlib.decompress(payload, wbits=31) - expected_size = bytes_per_frame * num_frames - if len(delta_payload) != expected_size: - raise ValueError( - "delta gzip payload size mismatch: " - f"expected {expected_size}, got {len(delta_payload)}" - ) - - restored = bytearray(delta_payload) - previous = ( - np.frombuffer(reference_frame, dtype=np.uint8) - if reference_frame is not None - else None - ) - for frame_idx in range(num_frames): - offset = frame_idx * bytes_per_frame - current = np.frombuffer( - restored, dtype=np.uint8, count=bytes_per_frame, offset=offset - ) - if previous is not None: - current ^= previous - previous = current - return bytes(restored) def build_raw_rgb_frame_batches( diff --git a/python/sglang/multimodal_gen/runtime/server_args/disagg.py b/python/sglang/multimodal_gen/runtime/server_args/disagg.py index 0240ca9f57..7b9742a2a1 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/disagg.py +++ b/python/sglang/multimodal_gen/runtime/server_args/disagg.py @@ -61,19 +61,6 @@ class DisaggServerArgsMixin: def derive_pool_work_endpoint(self) -> str: return format_tcp_endpoint("0.0.0.0", self.scheduler_port, "pool_work_endpoint") - def derive_pool_control_endpoint(self) -> str: - return format_tcp_endpoint( - "0.0.0.0", self.scheduler_port + 1, "pool_control_endpoint" - ) - - def derive_pool_control_advertised_endpoint(self) -> str: - host = self.host or self.disagg_p2p_hostname or "127.0.0.1" - if host == "0.0.0.0": - host = self.disagg_p2p_hostname or "127.0.0.1" - return format_tcp_endpoint( - host, self.scheduler_port + 1, "pool_control_advertised_endpoint" - ) - @classmethod def add_disagg_cli_args(cls, parser: FlexibleArgumentParser) -> None: role_default = ( diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index d66ce49fdc..53764d6d49 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -486,9 +486,6 @@ class ServerArgs(DisaggServerArgsMixin): pin_cpu_memory: bool = True ltx2_two_stage_device_mode: str | None = None _explicit_arg_names: set[str] = field(default_factory=set, repr=False) - _automatic_component_attention_backend_keys: set[str] = field( - default_factory=set, init=False, repr=False - ) _required_resident_components: set[str] = field( default_factory=set, init=False, repr=False ) @@ -636,8 +633,6 @@ class ServerArgs(DisaggServerArgsMixin): decoder_sp: int | None = None pool_work_endpoint: str | None = None pool_result_endpoint: str | None = None - pool_control_endpoint: str | None = None - pool_control_advertised_endpoint: str | None = None # Logging log_level: str = "info" @@ -1183,7 +1178,6 @@ class ServerArgs(DisaggServerArgsMixin): logger.info( "Automatically set torch_sdpa backend for component text_encoder to preserve LTX2 official attention semantics" ) - self._automatic_component_attention_backend_keys.add("text_encoder") else: logger.warning( "Overriding %s backend with torch_sdpa for component text_encoder to preserve LTX2 official attention semantics", @@ -1222,7 +1216,6 @@ class ServerArgs(DisaggServerArgsMixin): "encoder; laser_attn applies to the transformer" ) self.component_attention_backends["text_encoder"] = "torch_sdpa" - self._automatic_component_attention_backend_keys.add("text_encoder") if self.ring_degree > 1: if ( @@ -1463,15 +1456,6 @@ class ServerArgs(DisaggServerArgsMixin): ) -> None: """Select a component's backend on the user's behalf.""" self.component_attention_backends[component_name] = backend - self._automatic_component_attention_backend_keys.add(component_name) - - def is_component_attention_backend_automatic( - self, component_name: str | None - ) -> bool: - return ( - component_name is not None - and component_name in self._automatic_component_attention_backend_keys - ) def resolve_component_backend_by_role( self, *component_names: str | None diff --git a/python/sglang/multimodal_gen/runtime/utils/model_overlay.py b/python/sglang/multimodal_gen/runtime/utils/model_overlay.py index 8c049c3e07..f72775b025 100644 --- a/python/sglang/multimodal_gen/runtime/utils/model_overlay.py +++ b/python/sglang/multimodal_gen/runtime/utils/model_overlay.py @@ -97,35 +97,10 @@ def _compute_overlay_fingerprint(overlay_dir: str) -> str: return hasher.hexdigest() -def _resolve_bundled_overlay_dir(overlay_spec: dict[str, Any]) -> str | None: - bundled_overlay_subdir = overlay_spec.get("bundled_overlay_subdir") - if not bundled_overlay_subdir: - return None - bundled_overlay_dir = os.path.abspath( - os.path.join( - os.path.dirname(__file__), - "..", - "..", - "model_overlays", - str(bundled_overlay_subdir), - ) - ) - if not os.path.isdir(bundled_overlay_dir): - return None - if load_overlay_manifest_if_present(bundled_overlay_dir) is None: - return None - return bundled_overlay_dir - - def get_diffusion_cache_root() -> str: return envs.SGLANG_DIFFUSION_CACHE_ROOT -def clear_model_overlay_registry_cache() -> None: - global _MODEL_OVERLAY_REGISTRY_CACHE - _MODEL_OVERLAY_REGISTRY_CACHE = None - - def _load_model_overlay_registry() -> dict[str, dict[str, Any]]: global _MODEL_OVERLAY_REGISTRY_CACHE if _MODEL_OVERLAY_REGISTRY_CACHE is not None: @@ -446,15 +421,6 @@ def download_overlay_metadata( *, snapshot_download_fn: Callable[..., str], ) -> str: - bundled_overlay_dir = _resolve_bundled_overlay_dir(overlay_spec) - if bundled_overlay_dir is not None: - logger.info( - "Using bundled overlay metadata for %s from %s", - source_model_id, - bundled_overlay_dir, - ) - return bundled_overlay_dir - overlay_repo_id = str(overlay_spec["overlay_repo_id"]) if os.path.exists(overlay_repo_id): logger.info( diff --git a/python/sglang/multimodal_gen/runtime/utils/perf_logger.py b/python/sglang/multimodal_gen/runtime/utils/perf_logger.py index 46b65262aa..e6b95023fb 100644 --- a/python/sglang/multimodal_gen/runtime/utils/perf_logger.py +++ b/python/sglang/multimodal_gen/runtime/utils/perf_logger.py @@ -60,9 +60,6 @@ class RequestMetrics: self.stages: Dict[str, float] = {} self.denoising_stages: set[str] = set() self.steps: list[float] = [] - self.steps_by_stage: Dict[str, list[float]] = {} - self.stage_iterations: Dict[str, tuple[int, int]] = {} - self.active_stage_name: str | None = None self.total_duration_ms: float = 0.0 self.suppress_stage_breakdown: bool = False # Set on the first request after a failed request-based warmup: its @@ -85,26 +82,7 @@ class RequestMetrics: """Records the duration of a denoising step in execution order.""" if self.suppress_stage_breakdown: return - duration_ms = duration_s * 1000 - self.steps.append(duration_ms) - if self.active_stage_name is not None: - self.steps_by_stage.setdefault(self.active_stage_name, []).append( - duration_ms - ) - - def record_stage_iterations( - self, measured_iterations: int, target_iterations: int - ) -> None: - """Record calibration and default-workload iterations for this stage.""" - if self.suppress_stage_breakdown or self.active_stage_name is None: - return - measured = max(1, int(measured_iterations)) - target = max(1, int(target_iterations)) - previous = self.stage_iterations.get(self.active_stage_name, (0, 0)) - self.stage_iterations[self.active_stage_name] = ( - previous[0] + measured, - previous[1] + target, - ) + self.steps.append(duration_s * 1000) def record_memory_snapshot(self, checkpoint_name: str, snapshot: MemorySnapshot): if self.suppress_stage_breakdown: diff --git a/python/sglang/multimodal_gen/runtime/utils/profiler.py b/python/sglang/multimodal_gen/runtime/utils/profiler.py index 4595a2102f..f51deb12ef 100644 --- a/python/sglang/multimodal_gen/runtime/utils/profiler.py +++ b/python/sglang/multimodal_gen/runtime/utils/profiler.py @@ -5,6 +5,7 @@ from collections.abc import Iterator import torch +from sglang.multimodal_gen import envs from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.utils.logging_utils import CYAN, RESET, init_logger from sglang.srt.utils.torch_npu_patch_utils import apply_torch_npu_patches @@ -40,7 +41,7 @@ def _resolve_profiler_log_dir(log_dir: str | None) -> str: if log_dir is not None: return log_dir - diffusion_profiler_dir = os.getenv("SGLANG_DIFFUSION_TORCH_PROFILER_DIR") + diffusion_profiler_dir = envs.SGLANG_DIFFUSION_TORCH_PROFILER_DIR if diffusion_profiler_dir: return diffusion_profiler_dir diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index 3e566c08b8..b8707da5be 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -1614,22 +1614,29 @@ STANDALONE_FILES = { "../single_test_file/test_pynccl_a2a_capture_2_gpu.py", "../single_test_file/test_usp_replicated_parity_2_gpu.py", "../single_test_file/test_vdn_ulysses_exchange_2_gpu.py", + "../unit/test_async_output_save.py", + "../unit/test_kandinsky6_sequence_parallel.py", + "../unit/test_kandinsky6_sr_sequence_parallel.py", + "../unit/test_qwen_image21_distributed.py", ], } # test_update_weights_from_disk fails deterministically on ROCm: the diffusion # weight-update-from-disk reload path does not reconcile the diffusers # checkpoint layout with the sglang transformer parameters (shape mismatch -> -# HTTP 400). Tracked in #31924. NVIDIA does not run standalone files, so this -# test is AMD-only; skip it on ROCm until the reload path is fixed. Remove -# this block (keeping the STANDALONE_FILES / est-time entry above) to -# re-enable once #31924 lands. +# HTTP 400). Tracked in #31924. Skip it on ROCm until the reload path is +# fixed. Remove this block (keeping the STANDALONE_FILES / est-time entry +# above) to re-enable once #31924 lands. if current_platform.is_hip(): STANDALONE_FILES["1-gpu"] = [ f for f in STANDALONE_FILES["1-gpu"] if f != "../single_test_file/test_update_weights_from_disk.py" ] + # The 2-GPU unit/ cases are only validated on CUDA. + STANDALONE_FILES["2-gpu"] = [ + f for f in STANDALONE_FILES["2-gpu"] if not f.startswith("../unit/") + ] # New standalone files may omit an estimate once to learn the real CI runtime. # CI will use a fallback estimate for sharding, run the test, then print a @@ -1639,6 +1646,7 @@ STANDALONE_FILE_EST_TIMES = { "../single_test_file/test_diffusion_bcg_zimage_turbo.py": 420.0, }, "1-gpu": { + "../single_test_file/test_generate_zimage_turbo_cli.py": 180.0, "../single_test_file/test_update_weights_from_disk.py": 1200.0, }, "2-gpu": { @@ -1665,6 +1673,11 @@ STANDALONE_FILE_EST_TIMES = { "../single_test_file/test_usp_replicated_parity_2_gpu.py": 180.0, # no model load; two small all-to-alls "../single_test_file/test_vdn_ulysses_exchange_2_gpu.py": 60.0, + # the unit/ suite runs on one GPU; these files carry its 2-GPU cases + "../unit/test_async_output_save.py": 30.0, + "../unit/test_kandinsky6_sequence_parallel.py": 150.0, + "../unit/test_kandinsky6_sr_sequence_parallel.py": 150.0, + "../unit/test_qwen_image21_distributed.py": 90.0, }, } diff --git a/python/sglang/multimodal_gen/test/server/realtime_consistency.py b/python/sglang/multimodal_gen/test/server/realtime_consistency.py index 5eb3d4bf33..1be775614d 100644 --- a/python/sglang/multimodal_gen/test/server/realtime_consistency.py +++ b/python/sglang/multimodal_gen/test/server/realtime_consistency.py @@ -17,12 +17,7 @@ import numpy as np import pytest from openai import Client -from sglang.multimodal_gen.runtime.realtime.video import ( - RAW_RGB_CONTENT_TYPE, - RAW_RGB_DELTA_GZIP_CONTENT_TYPE, - RAW_RGBA_DELTA_GZIP_CONTENT_TYPE, - restore_delta_gzip_raw_rgb_payload, -) +from sglang.multimodal_gen.runtime.realtime.video import RAW_RGB_CONTENT_TYPE from sglang.multimodal_gen.test.server.testcase_configs import DiffusionSamplingParams from sglang.multimodal_gen.test.test_utils import is_image_url @@ -252,14 +247,9 @@ def pop_realtime_key_frames(case_id: str) -> list[np.ndarray] | None: def decode_realtime_raw_rgb_frames( header: dict[str, Any], payload: bytes, - previous_frame: bytes | None = None, ) -> list[np.ndarray]: content_type = header.get("content_type") - if content_type not in ( - RAW_RGB_CONTENT_TYPE, - RAW_RGB_DELTA_GZIP_CONTENT_TYPE, - RAW_RGBA_DELTA_GZIP_CONTENT_TYPE, - ): + if content_type != RAW_RGB_CONTENT_TYPE: raise ValueError(f"Unsupported realtime frame content type: {content_type}") width = int(header["width"]) @@ -268,18 +258,6 @@ def decode_realtime_raw_rgb_frames( num_frames = int(header["num_frames"]) bytes_per_frame = int(header["bytes_per_frame"]) expected_size = num_frames * bytes_per_frame - if content_type in ( - RAW_RGB_DELTA_GZIP_CONTENT_TYPE, - RAW_RGBA_DELTA_GZIP_CONTENT_TYPE, - ): - if header.get("delta_reference") != "previous-frame": - previous_frame = None - payload = restore_delta_gzip_raw_rgb_payload( - payload, - bytes_per_frame=bytes_per_frame, - num_frames=num_frames, - reference_frame=previous_frame, - ) if len(payload) != expected_size: raise ValueError( f"Realtime payload size mismatch: expected {expected_size}, got {len(payload)}" @@ -371,7 +349,6 @@ async def collect_realtime_output( await send_events_for_boundary(ws, -1) received_chunks: set[int] = set() - previous_frame: bytes | None = None while len(received_chunks) < num_chunks or ( require_chunk_stats and len(chunk_stats) < len(received_chunks) ): @@ -397,14 +374,8 @@ async def collect_realtime_output( if not isinstance(raw_payload, bytes): raise ValueError("Realtime frame payload must be bytes") - chunk_frames = decode_realtime_raw_rgb_frames( - header, - raw_payload, - previous_frame, - ) + chunk_frames = decode_realtime_raw_rgb_frames(header, raw_payload) frames.extend(chunk_frames) - if chunk_frames: - previous_frame = chunk_frames[-1].tobytes() chunk_index = int(header["chunk_index"]) if header.get("is_final_frame_batch", True): received_chunks.add(chunk_index) diff --git a/python/sglang/multimodal_gen/test/single_test_file/test_diffusion_bcg_tp2_zimage_turbo.py b/python/sglang/multimodal_gen/test/single_test_file/test_diffusion_bcg_tp2_zimage_turbo.py index 36dc3d1534..8c2c921534 100644 --- a/python/sglang/multimodal_gen/test/single_test_file/test_diffusion_bcg_tp2_zimage_turbo.py +++ b/python/sglang/multimodal_gen/test/single_test_file/test_diffusion_bcg_tp2_zimage_turbo.py @@ -42,6 +42,9 @@ BCG_CAPTURE_MARKER = "[Diffusion BCG] captured" # capture() context the fix enters. Its absence is the precise signature of the # regression: capture ran, registration did not. GRAPH_BUFFER_REGISTRATION_MARKER = "cuda graph addresses" +# CustomAllReduceV2 registers graph inputs only for its zero-copy pull algorithms, +# which this workload does not select, so the marker only applies to v1. +CUSTOM_ALL_REDUCE_V2_MARKER = "All Reduce config:" ILLEGAL_MEMORY_MARKER = "illegal memory access" # The regression fails during warmup within ~1 min; the timeout only has to be @@ -146,13 +149,14 @@ class TestDiffusionBCGTP2ZImageTurbo(CustomTestCase): f"BCG never captured, so this run did not exercise TP+BCG. " f"Log: {log_path}\n{tail}", ) - self.assertIn( - GRAPH_BUFFER_REGISTRATION_MARKER, - result.stdout, - f"custom all-reduce graph buffers were never registered, so capture " - f"did not go through the TP group's graph_capture(). " - f"Log: {log_path}\n{tail}", - ) + if CUSTOM_ALL_REDUCE_V2_MARKER not in result.stdout: + self.assertIn( + GRAPH_BUFFER_REGISTRATION_MARKER, + result.stdout, + f"custom all-reduce graph buffers were never registered, so capture " + f"did not go through the TP group's graph_capture(). " + f"Log: {log_path}\n{tail}", + ) self.assertIn("Pixel data generated successfully", result.stdout) diff --git a/python/sglang/multimodal_gen/test/unit/modelopt_fp8_stub.py b/python/sglang/multimodal_gen/test/unit/modelopt_fp8_stub.py new file mode 100644 index 0000000000..11d9e4a9ff --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/modelopt_fp8_stub.py @@ -0,0 +1,21 @@ +"""A ModelOpt FP8 linear stand-in for the norm+quant activation-gate tests.""" + +import torch +import torch.nn as nn + +from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import ( + ModelOptFp8LinearMethod, +) + + +def fp8_linear(input_scale: float) -> nn.Module: + linear = nn.Module() + linear.quant_method = object.__new__(ModelOptFp8LinearMethod) + linear.register_parameter( + "input_scale", + nn.Parameter( + torch.tensor(input_scale, dtype=torch.float32, device="cuda"), + requires_grad=False, + ), + ) + return linear diff --git a/python/sglang/multimodal_gen/test/unit/realtime/test_realtime_consistency_harness.py b/python/sglang/multimodal_gen/test/unit/realtime/test_realtime_consistency_harness.py index 48f6d85f03..e8318666a4 100644 --- a/python/sglang/multimodal_gen/test/unit/realtime/test_realtime_consistency_harness.py +++ b/python/sglang/multimodal_gen/test/unit/realtime/test_realtime_consistency_harness.py @@ -8,12 +8,7 @@ import msgspec.msgpack import numpy as np import pytest -from sglang.multimodal_gen.runtime.realtime.video import ( - RAW_RGB_CONTENT_TYPE, - RAW_RGB_DELTA_GZIP_CONTENT_TYPE, - RAW_RGBA_DELTA_GZIP_CONTENT_TYPE, - build_delta_gzip_raw_rgb_payload, -) +from sglang.multimodal_gen.runtime.realtime.video import RAW_RGB_CONTENT_TYPE from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord from sglang.multimodal_gen.test.server import test_server_common from sglang.multimodal_gen.test.server.realtime_consistency import ( @@ -40,9 +35,6 @@ from sglang.multimodal_gen.test.server.testcase_configs import ( DiffusionSamplingParams, DiffusionServerArgs, DiffusionTestCase, - LONGLIVE2_I2V_CI_sampling_params, - LONGLIVE2_T2V_CI_sampling_params, - REALTIME_MODEL_sampling_params, ScenarioConfig, ) @@ -185,58 +177,6 @@ def test_decode_realtime_raw_rgb_frames_rejects_truncated_payload(): decode_realtime_raw_rgb_frames(header, b"too-short") -def test_decode_realtime_delta_gzip_raw_rgb_frames_roundtrips(): - first = np.arange(12, dtype=np.uint8).reshape(2, 2, 3) - second = (np.arange(12, dtype=np.uint8) + 1).reshape(2, 2, 3) - header = { - "type": "frame_batch_header", - "content_type": RAW_RGB_DELTA_GZIP_CONTENT_TYPE, - "num_frames": 2, - "width": 2, - "height": 2, - "channels": 3, - "bytes_per_frame": 12, - } - - frames = decode_realtime_raw_rgb_frames( - header, - build_delta_gzip_raw_rgb_payload([first.tobytes(), second.tobytes()]), - ) - - assert len(frames) == 2 - np.testing.assert_array_equal(frames[0], first) - np.testing.assert_array_equal(frames[1], second) - - -def test_decode_realtime_rgba_delta_gzip_strips_alpha(): - first = np.array( - [[[1, 2, 3, 255], [4, 5, 6, 255]]], - dtype=np.uint8, - ) - second = np.array( - [[[1, 2, 4, 255], [4, 6, 6, 255]]], - dtype=np.uint8, - ) - header = { - "type": "frame_batch_header", - "content_type": RAW_RGBA_DELTA_GZIP_CONTENT_TYPE, - "num_frames": 2, - "width": 2, - "height": 1, - "channels": 4, - "bytes_per_frame": 8, - } - - frames = decode_realtime_raw_rgb_frames( - header, - build_delta_gzip_raw_rgb_payload([first.tobytes(), second.tobytes()]), - ) - - assert len(frames) == 2 - np.testing.assert_array_equal(frames[0], first[:, :, :3]) - np.testing.assert_array_equal(frames[1], second[:, :, :3]) - - # Stream collection and realtime performance stats @@ -633,72 +573,3 @@ def test_realtime_sampling_params_route_to_realtime_video_generator(): ) assert generate_fn.__name__ == "generate_realtime_video" - - -def test_realtime_model_params_are_lossless_gt_ready(): - params = REALTIME_MODEL_sampling_params - - assert "floating island hotel" in params.prompt - assert "825646291038" in str(params.image_path) - assert params.output_size == "832x480" - assert params.realtime_num_chunks == 4 - assert params.realtime_output_format is None - assert params.realtime_perf_ignore_initial_chunks == 2 - assert params.realtime_perf_thresholds["p95_chunk_total_ms"] == 5000.0 - assert params.realtime_perf_thresholds["p95_scheduler_forward_ms"] == 4500.0 - assert params.realtime_events == [] - assert params.extras["condition_inputs"]["camera_actions"] == [ - ["w"], - ["w"], - ["w"], - ["w"], - ["w"], - ["w"], - [], - [], - [], - [], - [], - [], - ] - - -def test_longlive2_cases_share_realtime_model_sampling_profile(): - for params in ( - LONGLIVE2_T2V_CI_sampling_params, - LONGLIVE2_I2V_CI_sampling_params, - ): - assert params.prompt == REALTIME_MODEL_sampling_params.prompt - assert params.fps == REALTIME_MODEL_sampling_params.fps - assert params.extras == { - "seed": 42, - "num_inference_steps": 4, - "guidance_scale": 1.0, - } - assert params.realtime_num_chunks is None - assert params.realtime_perf_thresholds == {} - - assert LONGLIVE2_T2V_CI_sampling_params.image_path is None - assert ( - LONGLIVE2_T2V_CI_sampling_params.output_size - == REALTIME_MODEL_sampling_params.output_size - ) - assert ( - LONGLIVE2_I2V_CI_sampling_params.image_path - == REALTIME_MODEL_sampling_params.image_path - ) - assert LONGLIVE2_I2V_CI_sampling_params.output_size == "960x928" - - -def test_lingbot_realtime_case_is_registered_by_default(): - from sglang.multimodal_gen.test.server.gpu_cases import ONE_GPU_CASES - - case = next( - item - for item in ONE_GPU_CASES - if item.id == "lingbot_world_realtime_plastic_beach" - ) - assert case.id == "lingbot_world_realtime_plastic_beach" - assert case.run_consistency_check is True - assert case.run_perf_check is True - assert case.sampling_params.realtime_output_format is None diff --git a/python/sglang/multimodal_gen/test/unit/realtime/test_realtime_output_transport.py b/python/sglang/multimodal_gen/test/unit/realtime/test_realtime_output_transport.py index 9bcd595057..8ec11d3484 100644 --- a/python/sglang/multimodal_gen/test/unit/realtime/test_realtime_output_transport.py +++ b/python/sglang/multimodal_gen/test/unit/realtime/test_realtime_output_transport.py @@ -19,9 +19,7 @@ from sglang.multimodal_gen.runtime.realtime.video import ( JPEG_FRAME_CONTENT_TYPE, RAW_RGB_CONTENT_TYPE, WEBP_FRAME_CONTENT_TYPE, - build_delta_gzip_raw_rgb_payload, build_raw_rgb_frame_batches, - restore_delta_gzip_raw_rgb_payload, ) @@ -148,23 +146,6 @@ def test_output_batch_uses_raw_frame_transport_names(): assert output_batch.raw_frame_metadata == {"format": "rgb24"} -def test_delta_gzip_raw_rgb_payload_roundtrips_exactly(): - frames = [ - bytes([1, 2, 3, 4, 5, 6]), - bytes([1, 2, 4, 4, 6, 6]), - bytes([2, 2, 4, 5, 6, 7]), - ] - - payload = build_delta_gzip_raw_rgb_payload(frames) - restored = restore_delta_gzip_raw_rgb_payload( - payload, - bytes_per_frame=6, - num_frames=3, - ) - - assert restored == b"".join(frames) - - @pytest.mark.parametrize( "request_id,output_options", [("req-1", {}), ("req-raw", {"realtime_output_format": "raw"})], diff --git a/python/sglang/multimodal_gen/test/unit/sana_wm/test_pipeline_config.py b/python/sglang/multimodal_gen/test/unit/sana_wm/test_pipeline_config.py index 4f957107d3..003320a98f 100644 --- a/python/sglang/multimodal_gen/test/unit/sana_wm/test_pipeline_config.py +++ b/python/sglang/multimodal_gen/test/unit/sana_wm/test_pipeline_config.py @@ -10,8 +10,6 @@ from unittest.mock import patch import torch -from sglang.multimodal_gen.configs.models.dits.sana_wm import SanaWMConfig -from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType from sglang.multimodal_gen.configs.pipeline_configs.sana_wm import SanaWMPipelineConfig from sglang.multimodal_gen.configs.sample.sana_wm import SanaWMSamplingParams from sglang.multimodal_gen.registry import _get_config_info, get_pipeline_config_classes @@ -48,7 +46,6 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.s ) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_wm.refiner import ( OfficialDiffusersLTX2RefinerModule, - OfficialGemma3TextEncoderModule, SanaWMLTX2RefinerStage, SanaWMRefinerDecodingStage, _refiner_config_value, @@ -90,9 +87,6 @@ class TestSanaWMPipelineConfig(unittest.TestCase): def setUp(self) -> None: self.config = SanaWMPipelineConfig() - def test_task_type_is_ti2v(self) -> None: - self.assertEqual(self.config.task_type, ModelTaskType.TI2V) - def test_adjust_num_frames_rounds_to_temporal_stride(self) -> None: self.assertEqual(self.config.adjust_num_frames(50), 49) self.assertEqual(self.config.adjust_num_frames(49), 49) @@ -136,31 +130,6 @@ class TestSanaWMPipelineConfig(unittest.TestCase): self.assertIs(kwargs["camera_conditions"], camera_conditions) self.assertIs(kwargs["chunk_plucker"], chunk_plucker) - def test_deployment_config_enables_memory_mode_dit_layerwise_offload( - self, - ) -> None: - deployment = self.config.get_model_deployment_config() - self.assertEqual(deployment.dit_layerwise_offload_modes, ("memory",)) - self.assertEqual(deployment.fsdp_auto_min_available_memory_gb, 60) - - def test_text_encoder_padding_matches_cfg_concat_contract(self) -> None: - self.assertEqual( - self.config.text_encoder_extra_args[0]["padding"], "max_length" - ) - self.assertTrue(self.config.text_encoder_extra_args[0]["return_attention_mask"]) - self.assertTrue(self.config.chi_prompt) - - def test_inference_flow_shift_matches_official_reference(self) -> None: - self.assertEqual(self.config.flow_shift, 9.95) - self.assertEqual(self.config.inference_flow_shift, 9.8) - - def test_dit_arch_text_norm_defaults_match_upstream(self) -> None: - arch = SanaWMConfig().arch_config - self.assertTrue(arch.y_norm) - self.assertEqual(arch.y_norm_scale_factor, 0.01) - self.assertEqual(arch.y_norm_eps, 1e-5) - self.assertEqual(arch.timestep_norm_scale_factor, 1.0) - def test_prepare_neg_cond_kwargs_keeps_camera_for_cfg(self) -> None: camera_conditions = torch.zeros(1, 7, 20) chunk_plucker = torch.zeros(1, 48, 7, 22, 40) @@ -296,15 +265,6 @@ class TestSanaWMPipelineConfig(unittest.TestCase): class TestSanaWMSamplingParams(unittest.TestCase): - def test_defaults_match_video_ti2v_contract(self) -> None: - params = SanaWMSamplingParams() - self.assertEqual(params.height, 704) - self.assertEqual(params.width, 1280) - self.assertEqual(params.num_frames, 49) - self.assertEqual(params.num_inference_steps, 20) - self.assertEqual(params.guidance_scale, 4.5) - self.assertEqual(params.negative_prompt, "") - def test_build_request_extra_omits_camera_by_default(self) -> None: params = SanaWMSamplingParams() extra = params.build_request_extra() @@ -411,17 +371,6 @@ class TestSanaWMTwoStagePipeline(unittest.TestCase): self.assertEqual(refiner_root, "/custom/refiner") self.assertEqual(refiner_gemma_root, "/custom/refiner/text_encoder") - def test_refiner_modules_are_loaded_from_official_subtrees(self) -> None: - self.assertEqual( - SanaWMTwoStagePipeline._REFINER_SUB_MODULES, - ( - ("transformer_2", "refiner/transformer"), - ("connectors", "refiner/connectors"), - ("text_encoder_2", "refiner/text_encoder"), - ("tokenizer_2", "refiner/text_encoder"), - ), - ) - class TestSanaWMPipeline(unittest.TestCase): def test_validate_parallelism_rejects_tensor_parallelism(self) -> None: @@ -1122,16 +1071,6 @@ class TestSanaWMDenoisingStage(unittest.TestCase): class TestSanaWMNativeDiTChunking(unittest.TestCase): - def test_softmax_chunking_is_disabled_by_default_for_upstream_parity(self) -> None: - arch = SanaWMConfig().arch_config - - self.assertFalse(arch.use_chunked_softmax_attention) - - def test_main_gdn_backend_defaults_to_auto(self) -> None: - arch = SanaWMConfig().arch_config - - self.assertEqual(arch.gdn_backend, "auto") - def test_first_chunk_plus_one_chunk_indices_match_upstream(self) -> None: self.assertEqual( _sana_wm_chunk_index_from_chunk_size( @@ -1275,16 +1214,6 @@ class TestSanaWMRefinerStage(_GlobalStageArgsMixin, unittest.TestCase): self.assertEqual(_refiner_config_value(module, "patch_size"), 1) - def test_official_refiner_wrappers_expose_layerwise_blocks(self) -> None: - self.assertEqual( - OfficialDiffusersLTX2RefinerModule.layer_names, - ["module.transformer_blocks"], - ) - self.assertIn( - "module.model.language_model.layers", - OfficialGemma3TextEncoderModule.layer_names, - ) - def test_refiner_component_uses_follow_execution_order(self) -> None: stage = SanaWMLTX2RefinerStage( transformer=torch.nn.Identity(), diff --git a/python/sglang/multimodal_gen/test/unit/sana_wm/test_streaming_cached.py b/python/sglang/multimodal_gen/test/unit/sana_wm/test_streaming_cached.py index c96087527b..031bf6498a 100644 --- a/python/sglang/multimodal_gen/test/unit/sana_wm/test_streaming_cached.py +++ b/python/sglang/multimodal_gen/test/unit/sana_wm/test_streaming_cached.py @@ -3,25 +3,15 @@ `_gdn_scan_cached` / `_single_path_delta_scan_cached` are the per-chunk combiners the streaming path calls: a FORWARD (inclusive) pass that carries recurrent state -across chunks plus a BACKWARD (exclusive) pass recomputed intra-chunk. Two -invariants pin them down: - -1. **Reduce-to-bidirectional** (#26153-only): a single chunk with no carried - state must equal the validated dense `_gdn_scan_bidirectional` / - `_single_path_delta_scan_bidirectional`. -2. **Port-fidelity** (vs the reference `minimal-sanawm`): output AND carried state - must match `recurrent_gdn_cached` / `recurrent_delta_cached` chunk-for-chunk, - including a 2-chunk state-carry run. +across chunks plus a BACKWARD (exclusive) pass recomputed intra-chunk. A +single chunk with no carried state must equal the validated dense +`_gdn_scan_bidirectional` / `_single_path_delta_scan_bidirectional`. FP64, CPU. """ from __future__ import annotations -import pathlib -import sys - -import pytest import torch from sglang.multimodal_gen.runtime.models.dits.sana_wm import ( @@ -53,11 +43,6 @@ def _inputs(): return q, k, v, q_rot, k_rot, beta, decay -def _frame_slice(x, f0, f1): - # x is (B, H, D, N=T*S), frame-major (T outer, S inner) - return x[..., f0 * S : f1 * S].contiguous() - - # --------------------------------------------------------------------------- # # 1. Reduce-to-bidirectional (uses only #26153 functions) # --------------------------------------------------------------------------- # @@ -80,110 +65,7 @@ def test_cam_cached_single_chunk_reduces_to_bidirectional(): # --------------------------------------------------------------------------- # -# 2. Port-fidelity vs reference (gold standard; skip if reference absent) -# --------------------------------------------------------------------------- # - -_REF_DIR = pathlib.Path("/sgl-workspace/sglang/myUtils/sana/minimal-sanawm") - - -def _load_reference(): - if not (_REF_DIR / "components.py").exists(): - pytest.skip(f"reference impl not found at {_REF_DIR}") - if str(_REF_DIR) not in sys.path: - sys.path.insert(0, str(_REF_DIR)) - try: - import components as ref # type: ignore - except Exception as exc: # pragma: no cover - environment dependent - pytest.skip(f"reference components.py not importable: {exc}") - return ref - - -def test_gdn_cached_matches_reference_single_and_chunked(): - ref = _load_reference() - q, k, v, q_rot, k_rot, beta, decay = _inputs() - - # Single chunk, no carried state. - mine, (kv_m, z_m) = _gdn_scan_cached(q, k, v, q_rot, k_rot, beta, decay) - r_out, kv_r, z_r, _ = ref.recurrent_gdn_cached( - q, k, v, q_rot, k_rot, beta, decay, eps=1e-6, kv_state=None, z_state=None - ) - torch.testing.assert_close(mine, r_out, atol=1e-9, rtol=0) - torch.testing.assert_close(kv_m, kv_r, atol=1e-9, rtol=0) - torch.testing.assert_close(z_m, z_r, atol=1e-9, rtol=0) - - # Two chunks carrying state — mine vs reference, chunk-for-chunk. - split = 2 - - def chunk(x, f0, f1): - return _frame_slice(x, f0, f1) - - args0 = ( - chunk(q, 0, split), - chunk(k, 0, split), - chunk(v, 0, split), - chunk(q_rot, 0, split), - chunk(k_rot, 0, split), - beta[:, :, :split], - decay[:, :, :split], - ) - args1 = ( - chunk(q, split, T), - chunk(k, split, T), - chunk(v, split, T), - chunk(q_rot, split, T), - chunk(k_rot, split, T), - beta[:, :, split:], - decay[:, :, split:], - ) - - m0, (mkv, mz) = _gdn_scan_cached(*args0) - m1, _ = _gdn_scan_cached(*args1, init_state_kv=mkv, init_state_z=mz) - - r0, rkv, rz, _ = ref.recurrent_gdn_cached( - *args0, eps=1e-6, kv_state=None, z_state=None - ) - r1, _, _, _ = ref.recurrent_gdn_cached(*args1, eps=1e-6, kv_state=rkv, z_state=rz) - - torch.testing.assert_close(m0, r0, atol=1e-9, rtol=0) - torch.testing.assert_close(m1, r1, atol=1e-9, rtol=0) - - -def test_cam_cached_matches_reference_single_and_chunked(): - ref = _load_reference() - _, _, v, q_rot, k_rot, beta, decay = _inputs() - - mine, kv_m = _single_path_delta_scan_cached(q_rot, k_rot, v, beta, decay) - r_out, kv_r = ref.recurrent_delta_cached(q_rot, k_rot, v, beta, decay, state=None) - torch.testing.assert_close(mine, r_out, atol=1e-9, rtol=0) - torch.testing.assert_close(kv_m, kv_r, atol=1e-9, rtol=0) - - split = 3 - a0 = ( - _frame_slice(q_rot, 0, split), - _frame_slice(k_rot, 0, split), - _frame_slice(v, 0, split), - beta[:, :, :split], - decay[:, :, :split], - ) - a1 = ( - _frame_slice(q_rot, split, T), - _frame_slice(k_rot, split, T), - _frame_slice(v, split, T), - beta[:, :, split:], - decay[:, :, split:], - ) - - m0, mkv = _single_path_delta_scan_cached(*a0) - m1, _ = _single_path_delta_scan_cached(*a1, init_state_kv=mkv) - r0, rkv = ref.recurrent_delta_cached(*a0, state=None) - r1, _ = ref.recurrent_delta_cached(*a1, state=rkv) - - torch.testing.assert_close(m0, r0, atol=1e-9, rtol=0) - torch.testing.assert_close(m1, r1, atol=1e-9, rtol=0) - - -# --------------------------------------------------------------------------- # -# 3. Cached short conv on K (cache slot 4) +# 2. Cached short conv on K (cache slot 4) # --------------------------------------------------------------------------- # C_CONV, KERN = 5, 4 @@ -232,7 +114,7 @@ def test_short_conv_cached_forward_continuity_across_chunks(): # --------------------------------------------------------------------------- # -# 4. Cached FFN temporal tail (GLUMBConvTemp, cache slot 9) +# 3. Cached FFN temporal tail (GLUMBConvTemp, cache slot 9) # --------------------------------------------------------------------------- # C_FFN, HID = 6, 8 diff --git a/python/sglang/multimodal_gen/test/unit/test_attention_backend_selector.py b/python/sglang/multimodal_gen/test/unit/test_attention_backend_selector.py index 125299db2e..b451d3743a 100644 --- a/python/sglang/multimodal_gen/test/unit/test_attention_backend_selector.py +++ b/python/sglang/multimodal_gen/test/unit/test_attention_backend_selector.py @@ -114,6 +114,7 @@ _FAKE_BACKENDS = { class TestAttentionBackendFallback(unittest.TestCase): def setUp(self) -> None: _cached_get_attn_backend.cache_clear() + self.addCleanup(_cached_get_attn_backend.cache_clear) _FakePlatform.selected_backend = None def _resolve( diff --git a/python/sglang/multimodal_gen/test/unit/test_auto_residency.py b/python/sglang/multimodal_gen/test/unit/test_auto_residency.py index 774ee2c084..31299282eb 100644 --- a/python/sglang/multimodal_gen/test/unit/test_auto_residency.py +++ b/python/sglang/multimodal_gen/test/unit/test_auto_residency.py @@ -12,9 +12,6 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.auto_residency impor GIB_BYTES, WarmupMemoryRecord, estimate_default_workload_peak_bytes, - estimate_default_workload_timing, - estimate_layerwise_layer_uses, - estimate_workload_phase_peaks, ) from sglang.multimodal_gen.runtime.warmup_request_builder import ( SERVER_WARMUP_MAX_VIDEO_FRAMES, @@ -30,19 +27,7 @@ def _record( num_frames=17, baseline_gib=10, peak_gib=12, - peak_reserved_gib=0, succeeded=True, - num_inference_steps=1, - total_duration_ms=0.0, - stage_duration_ms=None, - step_duration_ms=(), - step_duration_ms_by_stage=None, - stage_iterations=None, - phase_active_components=None, - phase_used_components=None, - phase_full_weight_transition_components=None, - layerwise_layer_uses=None, - layerwise_layer_uses_by_stage=None, ) -> WarmupMemoryRecord: return WarmupMemoryRecord( width=width, @@ -51,225 +36,9 @@ def _record( baseline_allocated_bytes=baseline_gib * GIB_BYTES, peak_allocated_bytes=peak_gib * GIB_BYTES, succeeded=succeeded, - peak_reserved_bytes=peak_reserved_gib * GIB_BYTES, - num_inference_steps=num_inference_steps, - total_duration_ms=total_duration_ms, - stage_duration_ms=stage_duration_ms or {}, - step_duration_ms=step_duration_ms, - step_duration_ms_by_stage=step_duration_ms_by_stage or {}, - stage_iterations=stage_iterations or {}, - phase_active_components=phase_active_components or {}, - phase_used_components=phase_used_components or {}, - phase_full_weight_transition_components=( - phase_full_weight_transition_components or {} - ), - layerwise_layer_uses=layerwise_layer_uses or {}, - layerwise_layer_uses_by_stage=layerwise_layer_uses_by_stage or {}, ) -class TestEstimateDefaultWorkloadTiming: - def test_scales_only_denoising_with_target_steps(self): - record = _record( - num_inference_steps=2, - total_duration_ms=1_200, - stage_duration_ms={ - "TextEncodingStage": 100, - "DenoisingStage": 1_000, - }, - phase_active_components={ - "0:TextEncodingStage:use:text_encoder": ("text_encoder",), - "1:DenoisingStage:use:transformer": ("transformer",), - }, - ) - - request_ns, stage_ns, component_stages = estimate_default_workload_timing( - records=[record], - target_units=record.workload_units(), - target_num_inference_steps=40, - ) - - assert stage_ns == { - "TextEncodingStage": 100_000_000, - "DenoisingStage": 20_000_000_000, - } - assert request_ns == 20_200_000_000 - assert component_stages == { - "text_encoder": ("TextEncodingStage",), - "transformer": ("DenoisingStage",), - } - - def test_uses_steady_step_instead_of_scaling_first_step_setup(self): - record = _record( - num_inference_steps=2, - total_duration_ms=720, - stage_duration_ms={ - "TextEncodingStage": 100, - "DenoisingStage": 620, - }, - step_duration_ms=(500, 100), - phase_active_components={ - "0:TextEncodingStage:use:text_encoder": ("text_encoder",), - "1:DenoisingStage:use:transformer": ("transformer",), - }, - ) - - request_ns, stage_ns, _ = estimate_default_workload_timing( - records=[record], - target_units=record.workload_units(), - target_num_inference_steps=10, - ) - - assert stage_ns["DenoisingStage"] == 1_020_000_000 - assert request_ns == 1_120_000_000 - - def test_scales_standard_and_nonstandard_denoising_stages_together(self): - record = _record( - num_inference_steps=2, - total_duration_ms=1_200, - stage_duration_ms={ - "TextEncodingStage": 100, - "DenoisingStage": 500, - "CustomDenoisingStage": 600, - }, - phase_active_components={ - "0:TextEncodingStage:use:text_encoder": ("text_encoder",), - "1:DenoisingStage:use:transformer": ("transformer",), - "2:CustomDenoisingStage:use:custom_refiner": ("custom_refiner",), - }, - ) - - request_ns, stage_ns, _ = estimate_default_workload_timing( - records=[record], - target_units=record.workload_units(), - target_num_inference_steps=10, - ) - - assert stage_ns == { - "TextEncodingStage": 100_000_000, - "DenoisingStage": 2_500_000_000, - "CustomDenoisingStage": 3_000_000_000, - } - assert request_ns == 5_600_000_000 - - def test_uses_each_stage_iteration_target(self): - record = _record( - num_inference_steps=4, - total_duration_ms=1_100, - stage_duration_ms={ - "ShapeDenoisingStage": 500, - "PaintStage": 600, - }, - step_duration_ms_by_stage={ - "ShapeDenoisingStage": (200, 100, 100, 100), - }, - stage_iterations={ - "ShapeDenoisingStage": (4, 50), - "PaintStage": (4, 30), - }, - phase_active_components={ - "0:ShapeDenoisingStage:use:hy3dshape_model": ("hy3dshape_model",), - "1:PaintStage:use:paint_transformer": ("paint_transformer",), - }, - ) - - request_ns, stage_ns, _ = estimate_default_workload_timing( - records=[record], - target_units=record.workload_units(), - target_num_inference_steps=50, - ) - - assert stage_ns == { - "ShapeDenoisingStage": 5_000_000_000, - "PaintStage": 4_500_000_000, - } - assert request_ns == 9_500_000_000 - - -class TestEstimateLayerwiseLayerUses: - def test_scales_repeated_groups_but_not_one_shot_groups(self): - record = _record( - num_inference_steps=2, - layerwise_layer_uses={ - "transformer": { - "token_refiner.blocks": (1, 1), - "blocks": (2, 2, 2), - }, - "vae": { - "encoder.down_blocks": (0, 0), - "decoder.up_blocks": (1, 1), - }, - }, - ) - - uses = estimate_layerwise_layer_uses( - records=[record], - target_units=record.workload_units(), - target_num_inference_steps=10, - ) - - assert uses["transformer"] == { - "token_refiner.blocks": (1, 1), - "blocks": (10, 10, 10), - } - assert uses["vae"] == { - "encoder.down_blocks": (0, 0), - "decoder.up_blocks": (1, 1), - } - - def test_stage_role_scales_nonstandard_denoiser_but_not_encoder_repeats(self): - record = _record( - num_inference_steps=2, - phase_used_components={ - "0:TextEncodingStage:use:custom_encoder": ("custom_encoder",), - "1:CustomDenoisingStage:use:custom_refiner": ("custom_refiner",), - }, - layerwise_layer_uses={ - "custom_encoder": {"layers": (2, 2)}, - "custom_refiner": {"blocks": (2, 2)}, - }, - ) - - uses = estimate_layerwise_layer_uses( - records=[record], - target_units=record.workload_units(), - target_num_inference_steps=10, - ) - - assert uses["custom_encoder"]["layers"] == (2, 2) - assert uses["custom_refiner"]["blocks"] == (10, 10) - - def test_scales_each_stage_calls_with_its_own_iteration_target(self): - record = _record( - num_inference_steps=4, - phase_used_components={ - "0:ShapeStage:use:transformer": ("transformer",), - "1:PaintStage:use:transformer": ("transformer",), - }, - stage_iterations={ - "ShapeStage": (4, 50), - "PaintStage": (4, 30), - }, - layerwise_layer_uses={ - "transformer": {"blocks": (9, 9), "one_shot": (2, 2)} - }, - layerwise_layer_uses_by_stage={ - "ShapeStage": {"transformer": {"blocks": (4, 4), "one_shot": (1, 1)}}, - "PaintStage": {"transformer": {"blocks": (4, 4), "one_shot": (1, 1)}}, - }, - ) - - uses = estimate_layerwise_layer_uses( - records=[record], - target_units=record.workload_units(), - target_num_inference_steps=50, - ) - - # one untracked one-shot call + 50 shape calls + 30 paint calls - assert uses["transformer"]["blocks"] == (81, 81) - assert uses["transformer"]["one_shot"] == (2, 2) - - class TestEstimateDefaultWorkloadPeak: def test_same_shape_uses_measured_peak(self): record = _record() @@ -428,154 +197,6 @@ class TestEstimateDefaultWorkloadPeak: estimate_default_workload_peak_bytes(records=[], target_units=None) is None ) - def test_phase_estimation_preserves_active_component_membership(self): - small = _record(num_frames=9, peak_gib=30) - large = _record(num_frames=17, peak_gib=32) - small = WarmupMemoryRecord( - width=small.width, - height=small.height, - num_frames=small.num_frames, - baseline_allocated_bytes=small.baseline_allocated_bytes, - peak_allocated_bytes=small.peak_allocated_bytes, - succeeded=small.succeeded, - phase_peak_allocated_bytes={"denoise": 30 * GIB_BYTES}, - phase_active_components={"denoise": ("transformer",)}, - ) - large = WarmupMemoryRecord( - width=large.width, - height=large.height, - num_frames=large.num_frames, - baseline_allocated_bytes=large.baseline_allocated_bytes, - peak_allocated_bytes=large.peak_allocated_bytes, - succeeded=large.succeeded, - phase_peak_allocated_bytes={"denoise": 32 * GIB_BYTES}, - phase_active_components={"denoise": ("transformer",)}, - ) - - peaks, active, used, _ = estimate_workload_phase_peaks( - records=[small, large], - target_units=832 * 480 * 81, - component_weight_bytes={"transformer": 28 * GIB_BYTES}, - ) - - assert peaks["denoise"] >= 32 * GIB_BYTES - assert active == {"denoise": ("transformer",)} - assert used == active - - def test_phase_estimation_uses_allocated_peak(self): - record = WarmupMemoryRecord( - width=1024, - height=1024, - num_frames=1, - baseline_allocated_bytes=5 * GIB_BYTES, - peak_allocated_bytes=12 * GIB_BYTES, - succeeded=True, - phase_peak_allocated_bytes={"denoise": 11 * GIB_BYTES}, - phase_active_components={"denoise": ("transformer",)}, - ) - - peaks, _, _, _ = estimate_workload_phase_peaks( - records=[record], - target_units=record.workload_units(), - component_weight_bytes={"transformer": 10 * GIB_BYTES}, - ) - - assert peaks["denoise"] == 11 * GIB_BYTES - - def test_phase_estimation_preserves_full_weight_transition_components(self): - record = WarmupMemoryRecord( - width=1024, - height=1024, - num_frames=1, - baseline_allocated_bytes=2 * GIB_BYTES, - peak_allocated_bytes=4 * GIB_BYTES, - succeeded=True, - phase_peak_allocated_bytes={"lora_switch": 4 * GIB_BYTES}, - phase_full_weight_transition_components={"lora_switch": ("transformer",)}, - ) - - peaks, _, _, transitions = estimate_workload_phase_peaks( - records=[record], - target_units=record.workload_units(), - component_weight_bytes={"transformer": 2 * GIB_BYTES}, - ) - - assert peaks == {"lora_switch": 4 * GIB_BYTES} - assert transitions == {"lora_switch": ("transformer",)} - - def test_phase_estimation_prefers_target_layout_over_smaller_warmup(self): - small = WarmupMemoryRecord( - width=256, - height=256, - num_frames=9, - baseline_allocated_bytes=2 * GIB_BYTES, - peak_allocated_bytes=30 * GIB_BYTES, - succeeded=True, - phase_peak_allocated_bytes={"denoise": 30 * GIB_BYTES}, - phase_active_components={"denoise": ()}, - ) - target = WarmupMemoryRecord( - width=768, - height=512, - num_frames=25, - baseline_allocated_bytes=2 * GIB_BYTES, - peak_allocated_bytes=48 * GIB_BYTES, - succeeded=True, - phase_peak_allocated_bytes={"denoise": 48 * GIB_BYTES}, - phase_active_components={"denoise": ("transformer",)}, - ) - - peaks, active, used, _ = estimate_workload_phase_peaks( - records=[small, target], - target_units=target.workload_units(), - component_weight_bytes={"transformer": 40 * GIB_BYTES}, - ) - - assert peaks == {"denoise": 48 * GIB_BYTES} - assert active == {"denoise": ("transformer",)} - assert used == active - - def test_phase_estimation_keeps_distinct_active_layouts_separate(self): - transformer_phase = WarmupMemoryRecord( - width=768, - height=512, - num_frames=25, - baseline_allocated_bytes=2 * GIB_BYTES, - peak_allocated_bytes=48 * GIB_BYTES, - succeeded=True, - phase_peak_allocated_bytes={"denoise": 48 * GIB_BYTES}, - phase_active_components={"denoise": ("transformer",)}, - ) - encoder_phase = WarmupMemoryRecord( - width=768, - height=512, - num_frames=25, - baseline_allocated_bytes=2 * GIB_BYTES, - peak_allocated_bytes=20 * GIB_BYTES, - succeeded=True, - phase_peak_allocated_bytes={"denoise": 20 * GIB_BYTES}, - phase_active_components={"denoise": ("text_encoder",)}, - ) - - peaks, active, used, _ = estimate_workload_phase_peaks( - records=[transformer_phase, encoder_phase], - target_units=transformer_phase.workload_units(), - component_weight_bytes={ - "transformer": 40 * GIB_BYTES, - "text_encoder": 10 * GIB_BYTES, - }, - ) - - assert peaks == { - "denoise:layout:0": 20 * GIB_BYTES, - "denoise:layout:1": 48 * GIB_BYTES, - } - assert active == { - "denoise:layout:0": ("text_encoder",), - "denoise:layout:1": ("transformer",), - } - assert used == active - class TestWarmupFrameAdjustment: def _server_args(self, *, bcg: bool = False, num_gpus: int = 1) -> SimpleNamespace: diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py index 98a8a376b5..1853dcdc8e 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py @@ -951,11 +951,6 @@ def test_ref2va_image_forward_kwargs_are_supported(): assert out_audio.shape == audio.shape -@pytest.mark.skipif(not torch.cuda.is_available(), reason="optional GPU smoke") -def test_optional_gpu_step_output_shapes(): - pytest.skip("no tiny H3 checkpoint on this machine; do not download weights") - - # --- profile --- from sglang.multimodal_gen.runtime.pipelines.minimax_h3_pipeline import ( diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_model_patcher.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_model_patcher.py deleted file mode 100644 index d38cd33b50..0000000000 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_model_patcher.py +++ /dev/null @@ -1,28 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""ComfyUI must keep tracking the model when a temporary SGLD clone expires.""" - -import gc -import weakref - -import pytest -import torch - -model_management = pytest.importorskip("comfy.model_management") - -from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core.model_patcher import ( - SGLDModelPatcher, -) - - -def test_expired_clone_returns_loaded_model_to_live_parent(): - model = torch.nn.Linear(1, 1) - device = torch.device("cpu") - parent = SGLDModelPatcher(model, device, device, model_type="minimax_h3") - tracked = model_management.LoadedModel(parent.clone().clone()) - tracked.real_model = weakref.ref(model) - - gc.collect() - - assert not tracked.is_dead() - assert tracked.model is parent - assert tracked.real_model() is model diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_qwen_image21.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_qwen_image21.py index 70f7101dce..0c60c8825c 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_qwen_image21.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_qwen_image21.py @@ -75,53 +75,6 @@ def test_int8_marker_of_fused_gate_up_covers_both_halves(monkeypatch) -> None: assert set(markers) == {"transformer_blocks.0.attn.to_q"} -@pytest.mark.parametrize( - "target_hw,ref_hws,slots", - [ - ((4, 6), [(4, 4), (2, 6)], [2, 6]), - # Odd parity: ComfyUI centres the reference by half a token. - ((5, 6), [(4, 4), (3, 6)], [2, 6]), - # A reference without a slot follows the text. - ((4, 6), [(4, 4), (2, 6)], [2]), - ], -) -def test_layout_follows_comfyui_token_order(target_hw, ref_hws, slots) -> None: - """The prefix hidden states, RoPE and segments match ComfyUI's own model.""" - model_mod = pytest.importorskip("comfy.ldm.qwen_image21.model") - ops = pytest.importorskip("comfy.ops") - torch.manual_seed(0) - model = model_mod.QwenImage21Transformer2DModel( - in_channels=4, - out_channels=4, - num_layers=1, - attention_head_dim=128, - num_attention_heads=1, - context_in_dim=8, - mlp_ratio=1, - dtype=torch.float32, - operations=ops.disable_weight_init, - ) - for p in model.parameters(): - torch.nn.init.normal_(p) - x = torch.randn(1, 4, *target_hw) - context = torch.randn(1, 11, 8) - refs = [torch.randn(1, 4, *hw) for hw in ref_hws] - with torch.no_grad(): - hidden, pe, segments = model.build_sequence(x, context, refs, slots) - ctx, flags = stage_mod.insert_image_placeholders(context, slots, len(refs)) - shapes = [(1, *hw) for hw in ref_hws] + [(1, *target_hw)] - layout = stage_mod.comfyui_layout(flags, shapes, (16, 56, 56), "cpu") - prefix = model.txt_in(ctx).index_select(1, layout["text_indices"]) - cond = torch.cat([r.flatten(2).transpose(1, 2) for r in refs], dim=1) - prefix[:, layout["image_indices"]] = model.img_in(cond) - prefix_len = layout["prefix_rope"].shape[0] - torch.testing.assert_close(prefix, hidden[:, :prefix_len], rtol=1e-5, atol=1e-4) - rope = torch.cat([layout["prefix_rope"], layout["target_rope"]]) - torch.testing.assert_close(rope.real, pe[0, :, 0, :, 0, 0], rtol=0, atol=1e-4) - torch.testing.assert_close(rope.imag, pe[0, :, 0, :, 1, 0], rtol=0, atol=1e-4) - assert list(layout["segments"]) == [(s, e, m is None) for s, e, m in segments[:-1]] - - class _RecordingExecutor(QwenImage21Executor): """Real forward and cond-cache bookkeeping; the worker echoes the latents.""" diff --git a/python/sglang/multimodal_gen/test/unit/test_component_residency.py b/python/sglang/multimodal_gen/test/unit/test_component_residency.py index b54f1ea7a4..f64b415ed7 100644 --- a/python/sglang/multimodal_gen/test/unit/test_component_residency.py +++ b/python/sglang/multimodal_gen/test/unit/test_component_residency.py @@ -18,7 +18,6 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager im ComponentResidencyManager, ComponentUse, ResidencyState, - WarmupPhasePeak, build_component_residency_strategy, ) from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency import ( @@ -721,23 +720,14 @@ def test_warmup_records_use_and_transition_peaks(monkeypatch): manager.finish_request() peaks = manager.take_warmup_phase_peaks() - inactive_peak = WarmupPhasePeak((), 7) - transformer_peak = WarmupPhasePeak( - ("transformer",), 7, used_components=("transformer",) - ) - assert peaks["request:before-stage"] == inactive_peak - assert peaks["0:denoise:setup"] == inactive_peak - assert peaks["0:denoise:transition:idle->transformer"] == transformer_peak - assert peaks["0:denoise:use:transformer"] == transformer_peak - assert peaks["0:denoise:transition:transformer->idle"] == transformer_peak - assert peaks["0:denoise:between"] == inactive_peak - # A non-preferred component is being released during cleanup, so it is no - # longer part of the placement that follows this transition. - assert peaks["request:cleanup:transformer"] == inactive_peak - assert peaks["idle"] == WarmupPhasePeak( - active_components=(), - allocated_bytes=2, - ) + assert peaks["request:before-stage"] == 7 + assert peaks["0:denoise:setup"] == 7 + assert peaks["0:denoise:transition:idle->transformer"] == 7 + assert peaks["0:denoise:use:transformer"] == 7 + assert peaks["0:denoise:transition:transformer->idle"] == 7 + assert peaks["0:denoise:between"] == 7 + assert peaks["request:cleanup:transformer"] == 7 + assert peaks["idle"] == 2 def test_warmup_skips_memory_tracking_for_unsupported_pipeline(monkeypatch): @@ -766,44 +756,6 @@ def test_warmup_skips_memory_tracking_for_unsupported_pipeline(monkeypatch): device_module.reset_peak_memory_stats.assert_not_called() -def test_warmup_records_full_weight_transition_without_preparing(monkeypatch): - device_module = SimpleNamespace( - is_available=lambda: True, - reset_peak_memory_stats=Mock(), - max_memory_allocated=lambda: 7, - memory_allocated=lambda: 2, - ) - monkeypatch.setattr(torch, "get_device_module", lambda: device_module) - monkeypatch.setattr(current_platform, "is_cuda", lambda: True) - - stage = _Stage() - module = torch.nn.Linear(2, 2) - pipeline = SimpleNamespace( - modules={"transformer": module}, - _stage_name_mapping={"lora_switch": stage}, - component_residency_strategies={}, - ) - server_args = _server_args() - manager = ComponentResidencyManager(pipeline, server_args) - manager.strategy_for = Mock() - manager.refresh_pipeline(pipeline) - manager.begin_request([stage], SimpleNamespace(is_warmup=True), server_args) - - manager.before_stage(stage, 0, SimpleNamespace(is_warmup=True), server_args) - with manager.full_weight_transition(("transformer",)): - pass - - assert manager._warmup_phase_peaks[ - "0:lora_switch:full-weight-transition:transformer" - ] == WarmupPhasePeak( - (), - 7, - full_weight_transition_components=("transformer",), - ) - assert manager._warmup_phase_key == "0:lora_switch:setup" - manager.strategy_for.assert_not_called() - - def test_warmup_records_same_component_dtype_prepare_as_transition(monkeypatch): device_module = SimpleNamespace( is_available=lambda: True, @@ -835,9 +787,9 @@ def test_warmup_records_same_component_dtype_prepare_as_transition(monkeypatch): manager.begin_use(second, module=module) manager._record_warmup_phase_peak() - assert manager._warmup_phase_peaks[ - "0:stage:transition:transformer->transformer" - ] == WarmupPhasePeak(("transformer",), 7, used_components=("transformer",)) + assert ( + manager._warmup_phase_peaks["0:stage:transition:transformer->transformer"] == 7 + ) assert strategy.prepare_for_use.call_count == 2 @@ -884,10 +836,8 @@ def test_warmup_attributes_prefetch_peak_to_prefetched_component(monkeypatch): manager.end_stage() manager.before_stage(denoise_stage, 1, SimpleNamespace(is_warmup=True), server_args) - assert manager._warmup_phase_peaks[ - "0:encode:prefetch:transformer" - ] == WarmupPhasePeak(("transformer",), 7, used_components=("transformer",)) - assert manager._warmup_phase_peaks["0:encode:between"] == WarmupPhasePeak((), 7) + assert manager._warmup_phase_peaks["0:encode:prefetch:transformer"] == 7 + assert manager._warmup_phase_peaks["0:encode:between"] == 7 def test_warmup_splits_sequential_component_transition(monkeypatch): @@ -923,12 +873,8 @@ def test_warmup_splits_sequential_component_transition(monkeypatch): manager.begin_use(second) manager._record_warmup_phase_peak() - assert manager._warmup_phase_peaks["0:stage:transition:text_encoder->idle"] == ( - WarmupPhasePeak(("text_encoder",), 7, used_components=("text_encoder",)) - ) - assert manager._warmup_phase_peaks["0:stage:transition:idle->transformer"] == ( - WarmupPhasePeak(("transformer",), 7, used_components=("transformer",)) - ) + assert manager._warmup_phase_peaks["0:stage:transition:text_encoder->idle"] == 7 + assert manager._warmup_phase_peaks["0:stage:transition:idle->transformer"] == 7 def _manager_for_stage(stage, modules): diff --git a/python/sglang/multimodal_gen/test/unit/test_conditioning_model_boundaries.py b/python/sglang/multimodal_gen/test/unit/test_conditioning_model_boundaries.py index d493d1bf77..4003129d8b 100644 --- a/python/sglang/multimodal_gen/test/unit/test_conditioning_model_boundaries.py +++ b/python/sglang/multimodal_gen/test/unit/test_conditioning_model_boundaries.py @@ -11,9 +11,6 @@ from sglang.multimodal_gen.runtime.cache.conditioning import ( invalidate_conditioning_caches, ) from sglang.multimodal_gen.runtime.models.dits.cosmos3video import Cosmos3LanguageModel -from sglang.multimodal_gen.runtime.models.sensenova_u1.neo_unify.modeling_neo_chat import ( - NEOChatModel, -) from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req from sglang.multimodal_gen.runtime.pipelines_core.stages.realtime.text_encoding import ( RealtimeTextEncodingStage, @@ -63,32 +60,6 @@ def test_cosmos_understanding_cache_restores_private_kv(): assert layer.calls == 3 -class Vision(torch.nn.Module): - def __init__(self): - super().__init__() - self.calls = 0 - - def forward(self, pixel_values, **kwargs): - self.calls += 1 - return SimpleNamespace(last_hidden_state=pixel_values * 2) - - -@torch.no_grad() -def test_sensenova_caches_references_but_runs_noisy_vision_each_step(): - model = NEOChatModel.__new__(NEOChatModel) - torch.nn.Module.__init__(model) - model.vision_model = Vision() - model.fm_modules = torch.nn.ModuleDict({"vision_model_mot_gen": Vision()}) - model.eval() - pixels = torch.ones(2, 4) - with ConditioningCache(4096).scope(): - for _ in range(2): - model.extract_feature(pixels) - model.extract_feature(pixels, gen_model=True) - assert model.vision_model.calls == 1 - assert model.fm_modules["vision_model_mot_gen"].calls == 2 - - @pytest.mark.parametrize("disabled, capacity", [(False, 512), (True, 512), (False, 0)]) def test_realtime_text_respects_disable_and_weight_invalidation( monkeypatch, disabled, capacity diff --git a/python/sglang/multimodal_gen/test/unit/test_convrot_int8_backends.py b/python/sglang/multimodal_gen/test/unit/test_convrot_int8_backends.py index 49b85dd321..20cb22c0b5 100644 --- a/python/sglang/multimodal_gen/test/unit/test_convrot_int8_backends.py +++ b/python/sglang/multimodal_gen/test/unit/test_convrot_int8_backends.py @@ -662,42 +662,6 @@ def test_every_no_kernel_branch_raises_instead_of_falling_back(): config.get_quant_method(LinearBase(256, 4), "gate") -@patch(_LOAD_COMFY_KITCHEN) -@patch(_LOAD_JIT_KERNEL) -def test_serialized_layers_log_the_backend_split_once_loaded(_jit, _comfy, caplog): - """Serialized layers skip online quantization, so the backend split must be - reported when their weights are in place instead.""" - import logging - - markers = { - "fc1": {"format": "int8_tensorwise", "convrot": True, "convrot_groupsize": 256}, - "small": { - "format": "int8_tensorwise", - "convrot": True, - "convrot_groupsize": 16, - }, - } - with ( - patch(_JIT_AVAILABLE, return_value=True), - patch(_COMFY_AVAILABLE, return_value=True), - caplog.at_level(logging.INFO), - ): - config = ConvRotInt8Config(layer_markers=markers) - fc1 = ReplicatedLinear(256, 16, bias=False, quant_config=config, prefix="fc1") - small = ReplicatedLinear( - 64, 16, bias=False, quant_config=config, prefix="small" - ) - fc1.quant_method.process_weights_after_loading(fc1) - assert not [r for r in caplog.records if "serialized INT8" in r.getMessage()] - small.quant_method.process_weights_after_loading(small) - lines = [ - r.getMessage() for r in caplog.records if "serialized INT8" in r.getMessage() - ] - assert lines == [ - "convrot_int8: loaded 2 serialized INT8 linear layers (jit 1, comfy_kitchen 1)" - ] - - def test_kitchen_int8_is_a_deprecated_alias_of_convrot_int8(caplog): """Existing `--quantization kitchen_int8` commands must keep working under the new name, with one deprecation warning per process.""" diff --git a/python/sglang/multimodal_gen/test/unit/test_cosmos3.py b/python/sglang/multimodal_gen/test/unit/test_cosmos3.py index 8b1bc7c56a..270f2a9894 100644 --- a/python/sglang/multimodal_gen/test/unit/test_cosmos3.py +++ b/python/sglang/multimodal_gen/test/unit/test_cosmos3.py @@ -1932,24 +1932,6 @@ class TestCosmos3BatchedActionStages(unittest.TestCase): self.assertEqual(tuple(image.shape), (2, 3, 16, 16)) -class TestCosmos3ModalitySamplingParams(unittest.TestCase): - """Sound / V2V / action sampling-param fields and defaults.""" - - def test_sound_duration_default_and_set(self): - self.assertEqual(Cosmos3SamplingParams(prompt="t").sound_duration, 0.0) - self.assertEqual( - Cosmos3SamplingParams(prompt="t", sound_duration=3.0).sound_duration, 3.0 - ) - - def test_v2v_fields(self): - sp = Cosmos3SamplingParams( - prompt="t", video_path="in.mp4", condition_frame_indexes=[0, 1] - ) - self.assertEqual(sp.video_path, "in.mp4") - self.assertEqual(sp.condition_frame_indexes, [0, 1]) - self.assertEqual(sp.condition_video_keep, "first") - - class TestCosmos3CaptionMetadata(unittest.TestCase): """Structured captions get generation metadata; prose prompts opt out.""" diff --git a/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py b/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py index 6c329d039b..7666092d03 100644 --- a/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py +++ b/python/sglang/multimodal_gen/test/unit/test_diffusion_bcg_padding.py @@ -128,9 +128,11 @@ class TestQualityFusionBCGCompatibility(unittest.TestCase): handlers = ( ( "lossless", - "test fusion", - lambda t: mounted.append(t) or True, - lambda _: None, + SimpleNamespace( + name="test fusion", + mount=lambda t: mounted.append(t) or True, + unmount=lambda _: None, + ), ), ) @@ -141,7 +143,14 @@ class TestQualityFusionBCGCompatibility(unittest.TestCase): self.assertEqual(self.stage._mounted_quality, "lossless") def test_allows_high_when_model_has_no_dit_quality_fusions(self): - handlers = (("lossless", "test fusion", lambda _: False, lambda _: None),) + handlers = ( + ( + "lossless", + SimpleNamespace( + name="test fusion", mount=lambda _: False, unmount=lambda _: None + ), + ), + ) with patch.object(denoising_module, "_QUALITY_FUSION_HANDLERS", handlers): self.stage._maybe_toggle_quality_fusions(self._batch("high")) @@ -154,9 +163,11 @@ class TestQualityFusionBCGCompatibility(unittest.TestCase): handlers = ( ( "lossless", - "test fusion", - lambda t: mounted.append(t) or True, - lambda t: unmounted.append(t), + SimpleNamespace( + name="test fusion", + mount=lambda t: mounted.append(t) or True, + unmount=lambda t: unmounted.append(t), + ), ), ) @@ -177,15 +188,19 @@ class TestQualityFusionBCGCompatibility(unittest.TestCase): handlers = ( ( "lossless", - "equivalent fusion", - lambda _: mounted.append("equivalent") or True, - lambda _: None, + SimpleNamespace( + name="equivalent fusion", + mount=lambda _: mounted.append("equivalent") or True, + unmount=lambda _: None, + ), ), ( "high", - "approximate fusion", - lambda _: mounted.append("approximate") or True, - lambda _: None, + SimpleNamespace( + name="approximate fusion", + mount=lambda _: mounted.append("approximate") or True, + unmount=lambda _: None, + ), ), ) @@ -666,63 +681,6 @@ class TestDiffusionBCGPadding(unittest.TestCase): self.assertEqual(out["x"].shape, (1, 4, 16)) self.assertIsInstance(out["attn_mask_meta"], dict) - def test_ideogram_is_registered_as_bcg_supported(self): - self.assertIn( - "ideogram-ai/ideogram-4-fp8", - BREAKABLE_CUDA_GRAPH_SUPPORTED_MODEL_IDS, - ) - self.assertIn( - "comfy-org/ideogram-4", - BREAKABLE_CUDA_GRAPH_SUPPORTED_MODEL_IDS, - ) - self.assertIn( - "fal/ideogram-v4-fast", - BREAKABLE_CUDA_GRAPH_SUPPORTED_MODEL_IDS, - ) - self.assertIn( - "fal/ideogram-v4-instant", - BREAKABLE_CUDA_GRAPH_SUPPORTED_MODEL_IDS, - ) - self.assertIn( - "Ideogram4PipelineConfig", - BREAKABLE_CUDA_GRAPH_SUPPORTED_PIPELINE_CONFIGS, - ) - - def test_image_generation_models_are_registered_as_bcg_supported(self): - for model_id in ( - "meituan-longcat/longcat-image", - "meituan-longcat/longcat-image-edit-turbo", - "qwen/qwen-image", - "qwen/qwen-image-2512", - "tongyi-mai/z-image", - "tongyi-mai/z-image-turbo", - "zai-org/glm-image", - ): - self.assertIn(model_id, BREAKABLE_CUDA_GRAPH_SUPPORTED_MODEL_IDS) - - for config_name in ( - "GlmImagePipelineConfig", - "LongCatImagePipelineConfig", - "LongCatImageEditPipelineConfig", - "QwenImagePipelineConfig", - "ZImagePipelineConfig", - ): - self.assertIn(config_name, BREAKABLE_CUDA_GRAPH_SUPPORTED_PIPELINE_CONFIGS) - - def test_ltx_models_are_registered_as_bcg_supported(self): - for model_id in ( - "lightricks/ltx-2", - "lightricks/ltx-2.3", - ): - self.assertIn(model_id, BREAKABLE_CUDA_GRAPH_SUPPORTED_MODEL_IDS) - - self.assertIn( - "LTX2PipelineConfig", BREAKABLE_CUDA_GRAPH_SUPPORTED_PIPELINE_CONFIGS - ) - self.assertIn( - "LTX23PipelineConfig", BREAKABLE_CUDA_GRAPH_SUPPORTED_PIPELINE_CONFIGS - ) - def test_sana_video_keeps_its_fixed_prompt_shape(self): kwargs = { "encoder_hidden_states": torch.zeros(1, 300, 2304), diff --git a/python/sglang/multimodal_gen/test/unit/test_flux2_fp8_norm_quant_gate.py b/python/sglang/multimodal_gen/test/unit/test_flux2_fp8_norm_quant_gate.py index 4faf97e46c..dc5c129a38 100644 --- a/python/sglang/multimodal_gen/test/unit/test_flux2_fp8_norm_quant_gate.py +++ b/python/sglang/multimodal_gen/test/unit/test_flux2_fp8_norm_quant_gate.py @@ -3,47 +3,31 @@ import unittest from types import SimpleNamespace -import torch import torch.nn as nn -from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import ( - ModelOptFp8LinearMethod, -) from sglang.multimodal_gen.runtime.models.dits.flux_2 import ( Flux2SingleTransformerBlock, Flux2TransformerBlock, ) +from sglang.multimodal_gen.test.unit.modelopt_fp8_stub import fp8_linear from sglang.test.test_utils import CustomTestCase -def _fp8_linear(input_scale: float) -> nn.Module: - linear = nn.Module() - linear.quant_method = object.__new__(ModelOptFp8LinearMethod) - linear.register_parameter( - "input_scale", - nn.Parameter( - torch.tensor(input_scale, dtype=torch.float32, device="cuda"), - requires_grad=False, - ), - ) - return linear - - def _double_block(scales: tuple[float, float, float]) -> Flux2TransformerBlock: block = object.__new__(Flux2TransformerBlock) nn.Module.__init__(block) block.attn = SimpleNamespace( use_fused_qkv=False, use_fused_added_qkv=False, - to_q=_fp8_linear(scales[0]), - to_k=_fp8_linear(scales[1]), - to_v=_fp8_linear(scales[2]), - add_q_proj=_fp8_linear(scales[0]), - add_k_proj=_fp8_linear(scales[1]), - add_v_proj=_fp8_linear(scales[2]), + to_q=fp8_linear(scales[0]), + to_k=fp8_linear(scales[1]), + to_v=fp8_linear(scales[2]), + add_q_proj=fp8_linear(scales[0]), + add_k_proj=fp8_linear(scales[1]), + add_v_proj=fp8_linear(scales[2]), ) - block.ff = SimpleNamespace(linear_in=_fp8_linear(scales[0])) - block.ff_context = SimpleNamespace(linear_in=_fp8_linear(scales[0])) + block.ff = SimpleNamespace(linear_in=fp8_linear(scales[0])) + block.ff_context = SimpleNamespace(linear_in=fp8_linear(scales[0])) block._fp8_img_attn_norm_quant = False block._fp8_txt_attn_norm_quant = False block._fp8_img_ff_norm_quant = False @@ -67,7 +51,7 @@ class TestFlux2Fp8NormQuantGate(CustomTestCase): def test_single_block_uses_merged_projection_scale(self) -> None: block = object.__new__(Flux2SingleTransformerBlock) nn.Module.__init__(block) - block.attn = SimpleNamespace(to_qkv_mlp_proj=_fp8_linear(0.25)) + block.attn = SimpleNamespace(to_qkv_mlp_proj=fp8_linear(0.25)) block._fp8_norm_quant = False block.configure_fp8_norm_quant() diff --git a/python/sglang/multimodal_gen/test/unit/test_flux3_image_served.py b/python/sglang/multimodal_gen/test/unit/test_flux3_image_served.py index e4d4fe3e4a..1085b79c90 100644 --- a/python/sglang/multimodal_gen/test/unit/test_flux3_image_served.py +++ b/python/sglang/multimodal_gen/test/unit/test_flux3_image_served.py @@ -580,7 +580,6 @@ class TestFlux3RequestControls(unittest.TestCase): ) params.prepare_synthetic_warmup_request_for_queue(req, server_args) self.assertEqual(req.num_inference_steps, 1) - self.assertEqual(req.extra["warmup_target_num_inference_steps"], 28) self.assertEqual(req.extra["cache_dit_num_inference_steps"], 28) def test_image_api_accepts_the_request_mode(self): diff --git a/python/sglang/multimodal_gen/test/unit/test_glm_image_attention.py b/python/sglang/multimodal_gen/test/unit/test_glm_image_attention.py new file mode 100644 index 0000000000..db281cbf14 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_glm_image_attention.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +"""GLM-Image attention rotates its image tokens on every CUDA layout.""" + +import pytest +import torch + +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.models.dits.glm_image import ( + GlmImageAttention, + GlmImageRotaryPosEmbed, +) +from sglang.multimodal_gen.runtime.platforms import current_platform + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="needs the CUDA RoPE") +@torch.no_grad() +def test_in_place_rope_on_contiguous_views_matches_the_copy_back_path( + single_process_model_parallel, monkeypatch +): + """The CUDA RoPE rotates a contiguous single-request Q/K view in place and + returns that view; writing it back onto itself must not fail the request.""" + torch.manual_seed(0) + attn = ( + GlmImageAttention( + query_dim=64, + heads=2, + dim_head=32, + out_dim=64, + bias=True, + qk_norm="layer_norm", + elementwise_affine=False, + eps=1e-5, + ) + .cuda() + .to(torch.bfloat16) + ) + for param in attn.parameters(): + param.normal_(0, 0.05) + hidden = torch.randn(1, 16, 64, device="cuda", dtype=torch.bfloat16) + encoder = torch.randn(1, 4, 64, device="cuda", dtype=torch.bfloat16) + rope = GlmImageRotaryPosEmbed(32, 1)(torch.empty(1, 4, 4, 4, device="cuda")) + + with set_forward_context(current_timestep=0, attn_metadata=None): + in_place = attn(hidden, encoder, image_rotary_emb=rope) + rotate = attn.rotary_emb.forward + monkeypatch.setattr( + attn.rotary_emb, + "forward", + lambda **kwargs: tuple(t.clone() for t in rotate(**kwargs)), + ) + copied = attn(hidden, encoder, image_rotary_emb=rope) + + for actual, expected in zip(in_place, copied, strict=True): + assert torch.equal(actual, expected) diff --git a/python/sglang/multimodal_gen/test/unit/test_hunyuan_image3_tp_broadcast.py b/python/sglang/multimodal_gen/test/unit/test_hunyuan_image3_tp_broadcast.py index 28665dde95..268bfd48cf 100644 --- a/python/sglang/multimodal_gen/test/unit/test_hunyuan_image3_tp_broadcast.py +++ b/python/sglang/multimodal_gen/test/unit/test_hunyuan_image3_tp_broadcast.py @@ -1,8 +1,5 @@ """Unit tests for HunyuanImage-3 tensor-parallel input broadcasts.""" -import ast -import inspect -import textwrap import unittest from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -36,36 +33,6 @@ class _FakeTPGroup: class TestHunyuanImage3TPBroadcast(CustomTestCase): - def test_static_broadcast_happens_outside_denoising_loop(self): - source = textwrap.dedent( - inspect.getsource(ar_stage.HunyuanImage3AR._forward_batched) - ) - function = ast.parse(source).body[0] - static_broadcast_calls = [ - node - for node in ast.walk(function) - if isinstance(node, ast.Call) - and isinstance(node.func, ast.Attribute) - and node.func.attr == "_broadcast_static_inputs" - ] - self.assertEqual(len(static_broadcast_calls), 1) - - denoising_loops = [ - node - for node in ast.walk(function) - if isinstance(node, ast.For) - and any( - isinstance(child, ast.Name) and child.id == "timesteps" - for child in ast.walk(node.iter) - ) - ] - self.assertEqual(len(denoising_loops), 1) - self.assertLess( - static_broadcast_calls[0].lineno, - denoising_loops[0].lineno, - "request-static inputs must be broadcast before the denoising loop", - ) - def test_static_inputs_are_broadcast_once(self): stage = ar_stage.HunyuanImage3AR.__new__(ar_stage.HunyuanImage3AR) tp_group = _FakeTPGroup() diff --git a/python/sglang/multimodal_gen/test/unit/test_ipc_cuda.py b/python/sglang/multimodal_gen/test/unit/test_ipc_cuda.py index 448a912746..b7e997f3ac 100644 --- a/python/sglang/multimodal_gen/test/unit/test_ipc_cuda.py +++ b/python/sglang/multimodal_gen/test/unit/test_ipc_cuda.py @@ -20,12 +20,6 @@ def _require_cuda() -> None: pytest.skip("CUDA is required for CUDA IPC tests") -def test_pickle_copies_cuda_tensor_through_host() -> None: - _require_cuda() - tensor = torch.ones(16, 90, 160, device="cuda", dtype=torch.bfloat16) - assert len(pickle.dumps(tensor)) > tensor.nbytes - - def test_ipc_handle_pickle_is_much_smaller_than_tensor() -> None: _require_cuda() tensor = torch.ones(16, 90, 160, device="cuda", dtype=torch.bfloat16) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_pipeline_contract.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_pipeline_contract.py index 45db98a742..457cb602b0 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_pipeline_contract.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_pipeline_contract.py @@ -30,47 +30,12 @@ from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import ( from sglang.multimodal_gen.runtime.pipelines.kandinsky6_pipeline import ( Kandinsky6TI2VAPipeline, ) -from sglang.multimodal_gen.runtime.pipelines_core.stages import InputValidationStage from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6 import ( - Kandinsky6AudioDecodingStage, - Kandinsky6DecodingStage, Kandinsky6DenoisingStage, - Kandinsky6ImageEncodingStage, - Kandinsky6LatentPreparationStage, denoising, ) -from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import ( - TextEncodingStage, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.timestep_preparation import ( - TimestepPreparationStage, -) from sglang.multimodal_gen.runtime.server_args import get_global_server_args -EXPECTED_REQUIRED_MODULES = [ - "scheduler", - "text_encoder", - "text_encoder_2", - "tokenizer", - "tokenizer_2", - "transformer", - "vae", - "audio_vae", -] - -EXPECTED_STAGE_ORDER = [ - ("input_validation_stage", InputValidationStage), - ("text_encoding_stage", TextEncodingStage), - ("timestep_preparation_stage", TimestepPreparationStage), - ("latent_preparation_stage", Kandinsky6LatentPreparationStage), - ("image_encoding_stage", Kandinsky6ImageEncodingStage), - ("denoising_stage", Kandinsky6DenoisingStage), - # Audio decodes before video, matching the diffusers reference -- see - # Kandinsky6TI2VAPipeline.create_pipeline_stages's comment. - ("audio_decoding_stage", Kandinsky6AudioDecodingStage), - ("decoding_stage", Kandinsky6DecodingStage), -] - def _make_pipeline() -> Kandinsky6TI2VAPipeline: pipeline = object.__new__(Kandinsky6TI2VAPipeline) @@ -96,33 +61,6 @@ def test_disagg_role_rejects_non_monolithic_deployment(): pipeline.validate_disagg_role(RoleType.MONOLITHIC) -def test_create_pipeline_stages_produces_documented_order(): - pipeline = _make_pipeline() - assert pipeline.pipeline_name == "Kandinsky6TI2VAPipeline" - assert pipeline.is_video_pipeline is True - assert pipeline._required_config_modules == EXPECTED_REQUIRED_MODULES - pipeline.create_pipeline_stages(MagicMock()) - assert [type(stage) for stage in pipeline.stages] == [ - stage_cls for _, stage_cls in EXPECTED_STAGE_ORDER - ] - assert [ - (name, type(stage)) for name, stage in pipeline._stage_name_mapping.items() - ] == EXPECTED_STAGE_ORDER - - # get_module wiring: each stage received the mocked module it asked for. - denoising_stage = pipeline._stage_name_mapping["denoising_stage"] - assert denoising_stage.transformer is pipeline.modules["transformer"] - - audio_decoding_stage = pipeline._stage_name_mapping["audio_decoding_stage"] - assert audio_decoding_stage.audio_vae is pipeline.modules["audio_vae"] - - decoding_stage = pipeline._stage_name_mapping["decoding_stage"] - assert decoding_stage.vae is pipeline.modules["vae"] - - image_encoding_stage = pipeline._stage_name_mapping["image_encoding_stage"] - assert image_encoding_stage.vae is pipeline.modules["vae"] - - def _pro_distill_scheduler() -> PiflowScheduler: """The scheduler of kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers.""" scheduler = PiflowScheduler( diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_rope.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_rope.py index 77c86fb27f..721eb298ed 100644 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_rope.py +++ b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_rope.py @@ -107,17 +107,3 @@ def test_rotary_cast_preserves_rounding_and_autograd(device, dtype, strided): torch.testing.assert_close( kandinsky6._apply_rotary(x, rope, dtype), expected, rtol=0, atol=0 ) - - -@torch.no_grad() -@pytest.mark.skipif( - not torch.cuda.is_available() or torch.version.hip is not None, - reason="CUDA required", -) -def test_gate_sum_preserves_fp32_intermediates(): - x = torch.randn(2, 257, 128, dtype=torch.bfloat16, device="cuda") - update = torch.randn_like(x) - gate = torch.randn(2, 1, 1152, dtype=x.dtype, device=x.device)[..., 256:384] - expected = (x.float() + gate.float() * update.float()).to(x.dtype) - assert not torch.equal(expected, x + gate * update) - assert torch.equal(kandinsky6.residual_gate_fp32(x, update, gate), expected) diff --git a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_reference_parity.py b/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_reference_parity.py deleted file mode 100644 index abf617fa70..0000000000 --- a/python/sglang/multimodal_gen/test/unit/test_kandinsky6_sr_reference_parity.py +++ /dev/null @@ -1,763 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -"""CPU parity of the Kandinsky 6 SR port against the k6_video reference (``kandinsky_sr``). - -The reference package is not part of this repository. Point -``KANDINSKY_SR_REFERENCE_SRC`` at the directory that contains ``kandinsky_sr`` (for -example ``k6_video/src``); without it the whole module is skipped. The reference needs -``flash_attn`` and ``loguru``, which are stubbed here (per-sequence SDPA for the dense -varlen attention), and its base resolutions are shrunk so that tiles stay tiny. All -components use tiny random weights (every parameter re-randomized: the reference -zero-initializes most modulation / output layers, which would hide bugs) and float32 -math on CPU. The port loads the reference ``state_dict()`` (unprefixed, i.e. exactly the official -Diffusers layout) with ``strict=True`` through its own name mapping. -""" - -import copy -import importlib.machinery -import os -import re -import sys -import types -from functools import partial -from types import SimpleNamespace - -import pytest -import torch -from kandinsky6_sr_tiny_components import ( - TINY_KVAE, - TINY_LU_MODEL, - TINY_PIFLOW, - super_resolve, -) - -from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import ( - Kandinsky6SRDitConfig, -) -from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import ( - Kandinsky6SRVAEConfig, -) -from sglang.multimodal_gen.runtime.loader.utils import ( - get_param_names_mapping, - hf_to_custom_state_dict, -) -from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context -from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import ( - Kandinsky6SRTransformer3DModel, -) -from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import ( - PiflowScheduler, -) -from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import ( - FlowMatchEulerDiscreteScheduler, -) -from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import ( - Kandinsky6SRLatentUpscalerBank, -) -from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr import ( - tiling, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import ( - build_dit_spec, - build_sampling_spec, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import ( - SamplingSpec, - denoise_chunks, - module_dtype, -) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import ( - plan_tiles, - stitch_tiles, -) - -REFERENCE_ENV = "KANDINSKY_SR_REFERENCE_SRC" -_reference_dir = os.environ.get(REFERENCE_ENV) -if not _reference_dir or not os.path.isdir( - os.path.join(_reference_dir, "kandinsky_sr") -): - pytest.skip( - f"set {REFERENCE_ENV} to the directory containing the kandinsky_sr package", - allow_module_level=True, - ) - -# --------------------------------------------------------------------------- # -# Reference harness (test-only): stubs, tiny resolutions, tiny random components -# --------------------------------------------------------------------------- # -TINY_RESOLUTIONS = {512: [(64, 64), (64, 96), (96, 64)]} -TINY_KVAE_ENC = dict(TINY_KVAE, in_channels=3) -TINY_KVAE_DEC = dict(TINY_KVAE, out_ch=3) -TINY_SCALING_FACTOR = 0.5 -TINY_DIT_CFG = dict( - in_visual_dim=4, - in_text_dim=8, - in_text_dim2=8, - time_dim=16, - out_visual_dim=4, - patch_size=(1, 2, 2), - model_dim=32, - ff_dim=64, - num_text_blocks=0, - num_visual_blocks=2, - axes_dims=(8, 4, 4), - visual_cond=False, - instruct_type="noise", - attention_params={"512": {"type": "flash"}}, - use_text=False, -) -TINY_SR_PARAMS = dict( - scale_factor={512: [1.0, 1.0, 1.0]}, - visual_size=[512], - scheduler_scale=5.0, - lq_noise_scale=0.7, - lq_noise_type="ddpm", - lq_channel_noise_scale=0.0, - cap_noise_timestep=False, - fps=24, -) - - -def _varlen_sdpa(q, k, v, cu_q, cu_k): - outs = [] - for i in range(len(cu_q) - 1): - qs, qe = int(cu_q[i]), int(cu_q[i + 1]) - ks, ke = int(cu_k[i]), int(cu_k[i + 1]) - qi = q[qs:qe].transpose(0, 1).unsqueeze(0) - ki = k[ks:ke].transpose(0, 1).unsqueeze(0) - vi = v[ks:ke].transpose(0, 1).unsqueeze(0) - out = torch.nn.functional.scaled_dot_product_attention(qi, ki, vi) - outs.append(out.squeeze(0).transpose(0, 1)) - return torch.cat(outs, dim=0) - - -def _flash_attn_stub() -> types.ModuleType: - module = types.ModuleType("flash_attn") - - def varlen( - q, - k, - v, - cu_seqlens_q, - cu_seqlens_k, - max_seqlen_q, - max_seqlen_k, - return_attn_probs=False, - **_, - ): - out = _varlen_sdpa(q, k, v, cu_seqlens_q, cu_seqlens_k) - return (out, torch.zeros(1), None) if return_attn_probs else out - - def qkvpacked(qkv, cu_seqlens, max_seqlen, return_attn_probs=False, **_): - q, k, v = qkv.unbind(dim=1) - out = _varlen_sdpa(q, k, v, cu_seqlens, cu_seqlens) - return (out, torch.zeros(1), None) if return_attn_probs else out - - module.flash_attn_varlen_func = varlen - module.flash_attn_varlen_qkvpacked_func = qkvpacked - # transformers probes flash_attn with importlib.util.find_spec - module.__spec__ = importlib.machinery.ModuleSpec("flash_attn", loader=None) - return module - - -def _loguru_stub() -> types.ModuleType: - module = types.ModuleType("loguru") - - class _Logger: - def __getattr__(self, name): - return lambda *args, **kwargs: self - - module.logger = _Logger() - module.__spec__ = importlib.machinery.ModuleSpec("loguru", loader=None) - return module - - -def _install_reference() -> None: - os.environ.setdefault("TORCH_COMPILE_DISABLE", "1") - os.environ.setdefault("TORCHDYNAMO_DISABLE", "1") - if _reference_dir not in sys.path: - sys.path.insert(0, _reference_dir) - if "flash_attn" not in sys.modules: - sys.modules["flash_attn"] = _flash_attn_stub() - if "loguru" not in sys.modules: - sys.modules["loguru"] = _loguru_stub() - from kandinsky_sr import constants - - constants.RESOLUTIONS.clear() - constants.RESOLUTIONS.update(TINY_RESOLUTIONS) - - -_install_reference() - -from kandinsky_sr.core.components.model.dit import get_dit # noqa: E402 -from kandinsky_sr.core.components.model.dx_dit import DXDiTWrapper # noqa: E402 -from kandinsky_sr.core.components.video_kvae.cached_model import ( # noqa: E402 - CachedCausalVAE, -) -from omegaconf import OmegaConf # noqa: E402 - -pytestmark = pytest.mark.usefixtures("single_process_model_parallel") - - -def randomize(module: torch.nn.Module, seed: int, std: float = 0.05) -> None: - generator = torch.Generator().manual_seed(seed) - with torch.no_grad(): - for param in module.parameters(): - param.copy_(torch.randn(param.shape, generator=generator) * std) - - -def build_reference_dit(*, piflow: dict | None, seed: int = 2, cfg: dict | None = None): - cfg = dict(cfg or TINY_DIT_CFG) - if piflow is not None: - dit = DXDiTWrapper( - cfg, out_visual_dim=cfg["out_visual_dim"], n_grid=piflow["n_grid"] - ) - dit.piflow_params = dict(piflow) - else: - dit = get_dit(cfg) - dit.eval() - randomize(dit, seed) - return dit - - -def build_port_dit( - reference_dit, *, cfg: dict | None = None, piflow: dict | None, overrides: dict -) -> Kandinsky6SRTransformer3DModel: - """Port DiT loaded with the reference weights (``model.`` prefix), strictly.""" - cfg = dict(cfg or TINY_DIT_CFG) - flat = dict( - cfg, - out_visual_dim=cfg["out_visual_dim"] * (piflow["n_grid"] if piflow else 1), - attribute_overrides=dict(overrides), - sr_params=dict(TINY_SR_PARAMS), - ) - dit_config = Kandinsky6SRDitConfig() - dit_config.update_model_arch(flat) - model = Kandinsky6SRTransformer3DModel(dit_config, flat) - official = dict(reference_dit.state_dict()) # unprefixed = the official layout - mapped, _ = hf_to_custom_state_dict( - official, - get_param_names_mapping(model.param_names_mapping), - valid_target_names=set(model.state_dict()), - ) - model.load_state_dict(mapped, strict=True) - return model.eval() - - -# --------------------------------------------------------------------------- # -# DiT forward parity -# --------------------------------------------------------------------------- # -def _reference_forward(dit, x, time, *, scale_factor, motion_score=None): - """Run the packed reference DiT on a batched ``[B, T, H, W, C]`` input.""" - batch, frames, height, width, channels = x.shape - packed = x.reshape(batch * frames, height, width, channels) - visual_cu = frames * torch.arange(batch + 1, dtype=torch.int32) - text_cu = torch.zeros(batch + 1, dtype=torch.int32) - rope_pos = [ - torch.cat([torch.arange(frames) for _ in range(batch)]), - torch.arange(height // dit.patch_size[1]), - torch.arange(width // dit.patch_size[2]), - ] - out = dit( - packed, - torch.zeros(0, 1), - torch.zeros(batch, 1), - time, - visual_cu, - text_cu, - rope_pos, - torch.zeros(0, dtype=torch.long), - scale_factor=scale_factor, - motion_score=motion_score, - ) - return out.reshape(batch, frames, *out.shape[1:]) if out.dim() == 4 else out - - -def _port_forward(model, x, time, *, scale_factor, motion_score=None): - batch, frames, height, width, _ = x.shape - rope_pos = [ - torch.arange(frames), - torch.arange(height // model.patch_size[1]), - torch.arange(width // model.patch_size[2]), - ] - with set_forward_context(current_timestep=0, attn_metadata=None): - return model( - x, - time, - rope_pos, - scale_factor=scale_factor, - motion_score=motion_score, - ) - - -def _random_latent( - width_channels: int, *, batch=2, frames=3, height=4, width=6, seed=0 -): - generator = torch.Generator().manual_seed(seed) - return torch.randn( - batch, frames, height, width, width_channels, generator=generator - ) - - -@pytest.mark.parametrize("piflow", [TINY_PIFLOW, None], ids=["dx_head", "plain_head"]) -def test_dit_forward_matches_reference(piflow): - """Guards the block math, RoPE, modulation, time bias and the DX head layout. - - The port is batched ``[B, T, H, W, C]`` while the reference packs samples with - ``cu_seqlens``; different per-sample times make a wrong batching/broadcast visible. - """ - reference = build_reference_dit(piflow=piflow) - model = build_port_dit( - reference, piflow=piflow, overrides={"instruct_type": "noise"} - ) - x = _random_latent(4) - time = torch.tensor([300.0, 850.0]) - scale = (1.0, 2.0, 2.0) - with torch.no_grad(): - expected = _reference_forward(reference, x, time, scale_factor=scale) - actual = _port_forward(model, x, time, scale_factor=scale) - n_grid = 1 if piflow is None else piflow["n_grid"] - if piflow is not None: - # reference DX wrapper already splits the grids: [B*T, n_grid, H, W, C] - expected = expected.movedim(1, -2).reshape(*actual.shape) - assert actual.shape[-1] == 4 * n_grid - torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5) - - -def test_dit_forward_matches_reference_with_motion_score_and_wide_input(): - """Guards the optional motion embedding and the ``2 * in + 1`` conditioned input.""" - cfg = dict( - TINY_DIT_CFG, - use_motion_score=True, - instruct_type="hybrid_anchor", - use_lq_noise_cond=True, - ) - reference = build_reference_dit(piflow=TINY_PIFLOW, cfg=cfg) - model = build_port_dit( - reference, - cfg=cfg, - piflow=TINY_PIFLOW, - overrides={"instruct_type": "noise", "visual_cond": True}, - ) - reference.instruct_type = "noise" - reference.visual_cond = True - x = _random_latent(2 * 4 + 1) - time = torch.tensor([120.0, 999.0]) - motion = torch.full((1,), 900.0) - with torch.no_grad(): - expected = _reference_forward( - reference, x, time, scale_factor=(1.0, 1.0, 1.0), motion_score=motion - ) - actual = _port_forward( - model, x, time, scale_factor=(1.0, 1.0, 1.0), motion_score=motion - ) - without_motion = _port_forward(model, x, time, scale_factor=(1.0, 1.0, 1.0)) - expected = expected.movedim(1, -2).reshape(*actual.shape) - torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5) - assert not torch.allclose(actual, without_motion) - - -def test_state_dict_keys_equal_mapped_reference_keys(): - """Guards the module tree / key mapping: extra or missing keys would break real checkpoints.""" - reference = build_reference_dit(piflow=TINY_PIFLOW) - model = build_port_dit(reference, piflow=TINY_PIFLOW, overrides={}) - mapped, _ = hf_to_custom_state_dict( - dict(reference.state_dict()), - get_param_names_mapping(model.param_names_mapping), - ) - assert set(mapped) == set(model.state_dict()) - - -# --------------------------------------------------------------------------- # -# Sampler loops (pi-Flow / Euler) against the reference generate functions -# --------------------------------------------------------------------------- # -def _reference_loop_inputs(x): - batch, frames, height, width, _ = x.shape - visual_cu = frames * torch.arange(batch + 1, dtype=torch.int32) - text = {"text_embeds": torch.zeros(0, 1), "pooled_embed": torch.zeros(batch, 1)} - text_cu = torch.zeros(batch + 1, dtype=torch.int32) - rope_pos = [ - torch.cat([torch.arange(frames) for _ in range(batch)]), - torch.arange(height // 2), - torch.arange(width // 2), - ] - return visual_cu, text, text_cu, rope_pos, torch.zeros(0, dtype=torch.long) - - -def _port_sample(model, x, scale, scheduler, steps, capped=False): - spec = SamplingSpec( - tiling_scale=2, - tiles_batch_size=1, - seed=42, - num_steps=steps, - is_piflow=isinstance(scheduler, PiflowScheduler), - tile_min_overlap=0.2, - visual_size=512, - scale_factor=scale, - lq_noise_scale=0.7, - lq_noise_type="ddpm", - lq_channel_noise_scale=0.0, - cap_noise_timestep=capped, - ) - return denoise_chunks( - [x.clone()], - model, - scheduler, - dit_spec=build_dit_spec(model), - spec=spec, - device=x.device, - step_context=lambda step: set_forward_context( - current_timestep=step, attn_metadata=None - ), - )[0] - - -def test_piflow_loop_matches_reference_piflow_generate(): - """Compare the packed reference loop with the batched native PiflowScheduler.""" - from kandinsky_sr.core.algo.piflow_sampler import piflow_generate - - reference = build_reference_dit(piflow=TINY_PIFLOW) - model = build_port_dit( - reference, piflow=TINY_PIFLOW, overrides={"instruct_type": "noise"} - ) - x = _random_latent(4, seed=5) - scale = (1.0, 2.0, 2.0) - visual_cu, text, text_cu, rope_pos, text_rope = _reference_loop_inputs(x) - packed = x.reshape(-1, *x.shape[2:]).clone() - scheduler = PiflowScheduler(**TINY_PIFLOW) - with torch.no_grad(): - expected = piflow_generate( - packed, - reference, - text, - visual_cu, - text_cu, - rope_pos, - text_rope, - scale, - out_dim=4, - start_timestep=1.0, - device="cpu", - **TINY_PIFLOW, - ) - actual = _port_sample(model, x, scale, scheduler, TINY_PIFLOW["nfe"]) - torch.testing.assert_close( - actual, expected.reshape(*x.shape[:4], 4), rtol=1e-5, atol=1e-5 - ) - - -@pytest.mark.parametrize( - "capped", [False, True], ids=["full_range", "capped_noise_start"] -) -def test_euler_loop_matches_reference_generate(capped): - """Compare noise caps and channel updates with the reference's five-point grid - (four DiT calls); both schedulers append the same terminal zero.""" - from kandinsky_sr.core.algo.utils import generate - - cfg = dict(TINY_DIT_CFG, use_motion_score=True, instruct_type="hybrid_anchor") - reference = build_reference_dit(piflow=None, cfg=cfg) - model = build_port_dit( - reference, - cfg=cfg, - piflow=None, - overrides={"instruct_type": "noise", "visual_cond": True}, - ) - reference.instruct_type, reference.visual_cond = "noise", True - x = _random_latent(2 * 4 + 1, seed=9) - scale = (1.0, 1.5, 1.5) - start = 0.7 if capped else 1.0 - visual_cu, text, text_cu, rope_pos, text_rope = _reference_loop_inputs(x) - packed = x.reshape(-1, *x.shape[2:]).clone() - num_inference_steps = 4 # the reference's 5 grid points == 4 actual Euler steps - scheduler = FlowMatchEulerDiscreteScheduler(shift=5.0) - with torch.no_grad(): - expected = generate( - packed, - reference, - "cpu", - 5, - text, - text, - visual_cu, - text_cu, - text_cu, - rope_pos, - text_rope, - text_rope, - scale, - 1.0, - 5.0, - start_timestep=start, - ) - actual = _port_sample(model, x, scale, scheduler, num_inference_steps, capped) - torch.testing.assert_close( - actual, expected.reshape(*x.shape[:4], 4), rtol=1e-5, atol=1e-5 - ) - - -# --------------------------------------------------------------------------- # -# Tiling / stitching against the reference helpers -# --------------------------------------------------------------------------- # -@pytest.mark.parametrize( - "frame_hw,scale", [((64, 96), 2), ((64, 64), 4), ((96, 64), 2), ((80, 144), 2)] -) -def test_tile_plan_and_stitch_match_reference(frame_hw, scale): - """Guards the tile grid (aspect -> base resolution -> even grid) and that the - frame-chunked stitch equals one reference stitch over the whole clip (T=19, not a - multiple of the chunk size).""" - from kandinsky_sr.core.algo.tiling_utils import stitch_tiles_hanning - from kandinsky_sr.pipeline.sr_pipeline import _tile_geometry - - height, width = frame_hw - plan = plan_tiles( - frame_hw=frame_hw, - visual_size=512, - tiling_scale=scale, - tile_min_overlap=0.2, - resolutions=TINY_RESOLUTIONS, - ) - base, tile_hw, grid = _tile_geometry( - height, width, 512, scale, 0.25, 16, "even", 0.2 - ) - assert (plan.base_hw, plan.tile_hw) == (base, tile_hw) - assert tuple(plan.pixel_grid) == tuple(grid) - generator = torch.Generator().manual_seed(3) - tiles = [ - torch.randint(0, 256, (3, 19, *base), dtype=torch.uint8, generator=generator) - for _ in range(grid.total_tiles) - ] - expected = ( - stitch_tiles_hanning( - [t.float() for t in tiles], grid, height, width, scale=scale - ) - .clamp(0, 255) - .to(torch.uint8) - ) - assert torch.equal(stitch_tiles(tiles, plan), expected) - - -# --------------------------------------------------------------------------- # -# End to end: pure orchestration vs kandinsky_sr.pipeline.stages.run_tiled_sr* -# --------------------------------------------------------------------------- # -def build_reference_lu_bank(target_scales=("2x", "4x"), seed=0): - """Reference LU bank with random weights and its bundle-style ``models`` config.""" - import pydantic - from kandinsky_sr.core.algo.latent_upscaler import LatentUpscalerBank - from kandinsky_sr.core.components.latent_upscaler.config import ModelConfig - from kandinsky_sr.core.components.latent_upscaler.model.factory import ( - build_upsampler, - ) - - models = [ - {"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)} - for scale in target_scales - ] - adapter = pydantic.TypeAdapter(ModelConfig) - bank = LatentUpscalerBank() - for index, spec in enumerate(models): - upsampler = build_upsampler(adapter.validate_python(spec["model"])).eval() - randomize(upsampler, seed + 10 + index) - upsampler.target_scale = spec["target_scale"] - upsampler.scaling_factor = TINY_SCALING_FACTOR - bank[spec["target_scale"]] = upsampler - return bank, models - - -def build_stacks(*, piflow, use_lu): - """(reference SRComponents, port dict) sharing identical random weights.""" - from kandinsky_sr.pipeline.sr_pipeline import SRComponents, SRParams - - torch.manual_seed(0) - ref_vae = CachedCausalVAE( - encoder_conf=OmegaConf.create(dict(TINY_KVAE_ENC)), - decoder_conf=OmegaConf.create(dict(TINY_KVAE_DEC)), - ).eval() - randomize(ref_vae, 1, std=0.1) - ref_vae.config = SimpleNamespace(scaling_factor=TINY_SCALING_FACTOR) - ref_dit = build_reference_dit(piflow=piflow, seed=2) - ref_bank, models = build_reference_lu_bank() if use_lu else (None, None) - reference = SRComponents( - dit=ref_dit, - vae=ref_vae, - latent_upscaler=ref_bank, - sr_params=SRParams(**TINY_SR_PARAMS), - spatial_factor=16, - ) - - vae_config = Kandinsky6SRVAEConfig() - vae_config.update_model_arch( - dict( - vae_type="video-kvae", - encoder_config=dict(TINY_KVAE_ENC), - decoder_config=dict(TINY_KVAE_DEC), - scaling_factor=TINY_SCALING_FACTOR, - spatial_factor=16, - temporal_factor=4, - ) - ) - vae = Kandinsky6SRVAE(vae_config) - # unprefixed reference keys = the official layout, mapped by the wrapper's load hook - vae.load_state_dict(dict(ref_vae.state_dict()), strict=True) - port = { - "vae": vae.eval(), - "scheduler": PiflowScheduler(**piflow) - if piflow - else FlowMatchEulerDiscreteScheduler(shift=TINY_SR_PARAMS["scheduler_scale"]), - "dit": build_port_dit( - ref_dit, piflow=piflow, overrides={"instruct_type": "noise"} - ), - "bank": None, - } - if use_lu: - bank = Kandinsky6SRLatentUpscalerBank(models, TINY_SCALING_FACTOR) - converted = {} - scale_indices = {"2x": 0, "4x": 1} - for key, value in ref_bank.state_dict().items(): - scale, suffix = key.split(".", 1) - suffix = suffix.replace("private_", "") - suffix = re.sub(r"(output_proj\.)0\.", r"\1norm.", suffix) - suffix = re.sub(r"(output_proj\.)2\.", r"\1conv.", suffix) - converted[f"_models.{scale_indices[scale]}.{suffix}"] = value - bank.load_state_dict( - converted, - strict=True, - ) - port["bank"] = bank.eval() - return reference, port - - -def run_reference(video, reference, *, scale, seed, tiles_batch_size, num_steps): - from kandinsky_sr.core.algo.latent_upscaler import latent_upscaler_for_scale - from kandinsky_sr.pipeline.sr_pipeline import ( - RunConfig, - encode_lq_video_to_lr_latent, - ) - from kandinsky_sr.pipeline.stages import run_tiled_sr, run_tiled_sr_from_pixels - from kandinsky_sr.pipeline.upscale_utils import ( - pre_upscale_video, - resolve_scale_request, - ) - - tiling_scale, pre = resolve_scale_request(float(scale)) - if pre != 1.0: - video = pre_upscale_video(video, pre, 16) - run_config = RunConfig( - device="cpu", - num_steps=num_steps, - seed=seed, - overlap=0.25, - tiles_batch_size=tiles_batch_size, - resolution_scale=tiling_scale, - ) - if latent_upscaler_for_scale(reference.latent_upscaler, tiling_scale) is not None: - lr_latent = encode_lq_video_to_lr_latent(video, reference.vae, "cpu") - return run_tiled_sr(lr_latent, reference, run_config) - return run_tiled_sr_from_pixels(video, reference, run_config) - - -def run_port(video, port, *, scale, seed, tiles_batch_size, num_steps): - """The stage logic (scale request, pre-upscale, specs) around the pure function. - - ``num_steps`` here is the reference's convention (timestep grid points); for a - flow-matching (non pi-Flow) bundle the port's own ``num_inference_steps`` is - ``num_steps - 1`` (DiT calls), so it is translated once, here, at the one remaining - boundary between the two conventions. A pi-Flow bundle ignores it either way (its - scheduler's own ``nfe`` always wins), so no translation is needed for that case. - """ - tiling_scale, pre = tiling.resolve_scale_request(float(scale)) - if pre != 1.0: - video = tiling.pre_upscale_video(video, pre, 16) - arch = port["dit"].config - scheduler = port["scheduler"] - port_num_steps = ( - num_steps if isinstance(scheduler, PiflowScheduler) else num_steps - 1 - ) - spec = build_sampling_spec( - arch=arch, - tiling_scale=tiling_scale, - seed=seed, - num_steps=port_num_steps, - tiles_batch_size=tiles_batch_size, - tile_min_overlap=0.2, - scheduler=scheduler, - ) - bank = port["bank"] - use_lu = bank is not None and bank.for_scale(tiling_scale) is not None - return super_resolve( - video, - vae=port["vae"], - scaling_factor=port["vae"].scaling_factor, - dit=port["dit"], - dit_spec=build_dit_spec(port["dit"]), - spec=spec, - scheduler=scheduler, - device="cpu", - upscale_fn=partial(bank.upscale, scale=tiling_scale) if use_lu else None, - lu_dtype=module_dtype(bank) if use_lu else None, - resolutions=TINY_RESOLUTIONS, - ) - - -def _random_video(frames, height, width, seed=0): - generator = torch.Generator().manual_seed(seed) - return torch.randint( - 0, 256, (frames, 3, height, width), dtype=torch.uint8, generator=generator - ) - - -E2E_CASES = [ - # scale, (H, W), tiles_batch_size - (2, (64, 96), 1), - (2, (64, 96), 3), - (4, (64, 64), 2), - (2.25, (64, 96), 1), -] - - -def _assert_uint8_close(actual, expected, *, max_diff: int = 1): - assert actual.shape == expected.shape and actual.dtype == expected.dtype - diff = (actual.int() - expected.int()).abs() - exact = (diff == 0).float().mean().item() - assert diff.max().item() <= max_diff and exact > 0.999, (diff.max().item(), exact) - - -@pytest.mark.parametrize("use_lu", [False, True], ids=["pixel_path", "lu_path"]) -@pytest.mark.parametrize("scale,hw,tiles_batch_size", E2E_CASES) -def test_tiled_piflow_super_resolution_matches_reference( - use_lu, scale, hw, tiles_batch_size -): - """Compare tiled pixel/LU paths with identical random weights and seeds. - - The reference ends at raw_dst=0, whereas PiflowScheduler ends at eps (1e-6). - Allow two uint8 levels of error while requiring more than 99.9% exact pixels. - """ - reference, port = build_stacks(piflow=TINY_PIFLOW, use_lu=use_lu) - video = _random_video(9, *hw) - kwargs = dict(scale=scale, seed=42, tiles_batch_size=tiles_batch_size, num_steps=5) - expected = run_reference(video, reference, **kwargs) - actual = run_port(video, port, **kwargs) - _assert_uint8_close(actual, expected, max_diff=2) - - -@pytest.mark.parametrize("use_lu", [False, True], ids=["pixel_path", "lu_path"]) -def test_tiled_euler_super_resolution_matches_reference(use_lu): - """Same as above for a plain (non pi-Flow) DiT: the Euler loop with ``num_steps``.""" - reference, port = build_stacks(piflow=None, use_lu=use_lu) - video = _random_video(9, 64, 96, seed=1) - kwargs = dict(scale=2, seed=7, tiles_batch_size=2, num_steps=4) - expected = run_reference(video, reference, **kwargs) - actual = run_port(video, port, **kwargs) - _assert_uint8_close(actual, expected) - - -def test_seed_and_chunking_change_the_result_like_the_reference(): - """The initial noise of chunk k is seeded with ``seed + chunk start``: another seed or - another chunk size must change the output (a port that ignored them would not).""" - _, port = build_stacks(piflow=TINY_PIFLOW, use_lu=False) - video = _random_video(9, 64, 96, seed=2) - base = run_port(video, port, scale=2, seed=42, tiles_batch_size=1, num_steps=5) - assert not torch.equal( - base, run_port(video, port, scale=2, seed=43, tiles_batch_size=1, num_steps=5) - ) - assert not torch.equal( - base, run_port(video, port, scale=2, seed=42, tiles_batch_size=4, num_steps=5) - ) diff --git a/python/sglang/multimodal_gen/test/unit/test_latent_upsampler_group_norm_silu.py b/python/sglang/multimodal_gen/test/unit/test_latent_upsampler_group_norm_silu.py index 2a9700930d..53f6d17ea1 100644 --- a/python/sglang/multimodal_gen/test/unit/test_latent_upsampler_group_norm_silu.py +++ b/python/sglang/multimodal_gen/test/unit/test_latent_upsampler_group_norm_silu.py @@ -6,7 +6,6 @@ import pytest import torch import sglang.multimodal_gen.runtime.models.upsampler.latent_upsampler as lu_mod -from sglang.kernels.ops.diffusion import apply_group_norm_silu from sglang.multimodal_gen.runtime.models.upsampler.latent_upsampler import ( LatentUpsampler, ResBlock, @@ -146,54 +145,3 @@ def test_resblock_actually_uses_triton_kernel_cuda(): with torch.no_grad(): block(x) assert spy.call_count >= 1 - - -@requires_cuda -@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) -@pytest.mark.parametrize( - "dims,latent_shape,mid_channels", - [ - (2, (1, 32, 2, 16, 16), 64), - (3, (1, 32, 2, 16, 16), 64), - ], -) -def test_initial_groupnorm_silu_parity_cuda_local( - dtype, dims, latent_shape, mid_channels -): - # Sharp parity at the fused initial-norm boundary, before downstream convs - # can amplify drift; uses kernel-level tolerance instead of the looser e2e. - torch.manual_seed(2) - device = torch.device("cuda") - upsampler = ( - LatentUpsampler( - in_channels=latent_shape[1], - mid_channels=mid_channels, - num_blocks_per_stage=2, - dims=dims, - spatial_upsample=True, - temporal_upsample=False, - spatial_scale=2.0, - rational_resampler=False, - ) - .to(device=device, dtype=dtype) - .eval() - ) - torch.manual_seed(3) - latent = torch.randn(*latent_shape, device=device, dtype=dtype) - - with torch.no_grad(): - if dims == 2: - from einops import rearrange - - b, _, f, _, _ = latent.shape - x_in = rearrange(latent, "b c f h w -> (b f) c h w") - else: - x_in = latent - x_after_conv = upsampler.initial_conv(x_in) - out_fused = apply_group_norm_silu( - x_after_conv, upsampler.initial_norm, upsampler.initial_activation - ) - out_eager = upsampler.initial_activation(upsampler.initial_norm(x_after_conv)) - - atol, rtol = _RESBLOCK_TOL[dtype] - torch.testing.assert_close(out_fused, out_eager, atol=atol, rtol=rtol) diff --git a/python/sglang/multimodal_gen/test/unit/test_ltx2_5_config.py b/python/sglang/multimodal_gen/test/unit/test_ltx2_5_config.py index 256344885a..356eccc55d 100644 --- a/python/sglang/multimodal_gen/test/unit/test_ltx2_5_config.py +++ b/python/sglang/multimodal_gen/test/unit/test_ltx2_5_config.py @@ -20,7 +20,6 @@ from sglang.multimodal_gen.configs.models.dits.ltx_2_5 import LTX25ArchConfig from sglang.multimodal_gen.configs.models.vaes.ltx_2_5_video import ( LTX25VideoVAEArchConfig, ) -from sglang.multimodal_gen.configs.models.vaes.ltx_video import LTXVideoVAEArchConfig from sglang.multimodal_gen.configs.models.vocoder.ltx_vocoder import LTXVocoderConfig from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import LTX2PipelineConfig from sglang.multimodal_gen.configs.pipeline_configs.ltx_2_5 import ( @@ -30,23 +29,6 @@ from sglang.multimodal_gen.configs.pipeline_configs.ltx_2_5 import ( class TestLTX25DiTConfig(unittest.TestCase): - def test_inherits_ltx23_audio_video_base(self): - arch = LTX25ArchConfig() - self.assertTrue(arch.apply_gated_attention) - self.assertTrue(arch.cross_attention_adaln) - # `use_prompt_embeddings: false` upstream -- the caption projection - # lives in the connector, not the DiT. - self.assertTrue(arch.caption_proj_before_connector) - self.assertEqual(arch.rope_type.value, "split") - self.assertTrue(arch.double_precision_rope) - - def test_feed_forward_bias_is_video_only(self): - # LTX-2.5 checkpoints have no `ff.net.*.bias` for the video branch but - # do for the audio one. - arch = LTX25ArchConfig() - self.assertFalse(arch.ff_bias) - self.assertTrue(arch.audio_ff_bias) - def test_param_names_mapping_extends_ltx2(self): # Regression: read off a class attribute pinned to LTX2ArchConfig, the # LTX-2.5 renames never reached the loader. @@ -67,13 +49,6 @@ class TestLTX25DiTConfig(unittest.TestCase): self.assertTrue(mapped.startswith(key.split(".")[0] + "_single."), mapped) self.assertEqual(reverse(mapped)[0], key) - def test_ltx2_defaults_unchanged(self): - # The shared LTX-2 config must keep its original behaviour. - arch = LTX2ArchConfig() - self.assertTrue(arch.ff_bias) - self.assertTrue(arch.audio_ff_bias) - self.assertFalse(arch.use_keyframes_abs_pos_embedding) - class TestLTX25VAEConfig(unittest.TestCase): # Reversed like its sibling lists, this would give the wrong strides. @@ -90,10 +65,6 @@ class TestLTX25VAEConfig(unittest.TestCase): ["spatiotemporal", "spatiotemporal", "temporal", "spatial"], ) - def test_ltx2_defaults_to_all_spatiotemporal(self): - # `None` must keep LTX-2 bit-identical. - self.assertIsNone(LTXVideoVAEArchConfig().upsample_type) - def test_decoder_builds_expected_upsampler_strides(self): import torch @@ -228,9 +199,6 @@ class TestLTX25PipelineConfig(unittest.TestCase): ) ) - def test_ltx2_has_no_pinned_schedule(self): - self.assertIsNone(LTX2PipelineConfig().default_sigmas) - def test_stays_an_ltx2_variant(self): # LTX-2.5 uses the LTX-2 linspace sigma path, so it must NOT be marked # as an LTX-2.3 native variant even though it shares 2.3's architecture. @@ -397,11 +365,6 @@ class TestLTX25DiffusionDecoder(unittest.TestCase): # The remaining spatial factor is the pixel patch size. self.assertEqual(spatial * arch.patch_size, arch.spatial_compression_ratio) - def test_ships_as_a_single_step_x0_decoder(self): - arch = self._config().arch_config - self.assertEqual(arch.decoder_num_inference_steps, 1) - self.assertEqual(arch.decoder_model_output_type, "x0") - def test_builds_and_reports_expected_context_width(self): import torch diff --git a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_denoise_loop.py b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_denoise_loop.py index e21deabe5f..fdfe72a02e 100644 --- a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_denoise_loop.py +++ b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_denoise_loop.py @@ -11,10 +11,6 @@ from sglang.multimodal_gen.configs.models.dits.minimax_h3 import ( MINIMAX_H3_ADALN_MODALITY_NUM, ) from sglang.multimodal_gen.runtime.managers.forward_context import get_forward_context -from sglang.multimodal_gen.runtime.models.schedulers.scheduling_minimax_h3_euler_ancestral import ( - _minimax_h3_euler_eta0_step, - _minimax_h3_rf_v_to_x0, -) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.denoise_loop import ( MiniMaxH3DenoiseBranch, _build_local_embedding_layout, @@ -35,6 +31,37 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.m ) +def _minimax_h3_rf_v_to_x0( + xt: torch.Tensor, + v: torch.Tensor, + timestep: torch.Tensor, +) -> torch.Tensor: + cond_t = timestep.to(device=xt.device, dtype=xt.dtype) + while cond_t.ndim < xt.ndim: + cond_t = cond_t.unsqueeze(-1) + sigma_t = 1 - cond_t + return xt + sigma_t * v + + +def _minimax_h3_euler_eta0_step( + state: torch.Tensor, + denoised: torch.Tensor, + *, + sigma_curr: float, + sigma_ratio: torch.Tensor, +) -> torch.Tensor: + if sigma_curr == 0.0: + return state + compute_dtype = torch.float32 + if state.dtype not in (torch.float16, torch.bfloat16): + compute_dtype = state.dtype + ratio = sigma_ratio.to(device=state.device, dtype=compute_dtype) + out = ratio * state.to(dtype=compute_dtype) + (1.0 - ratio) * denoised.to( + dtype=compute_dtype + ) + return out.to(dtype=state.dtype) + + def _branch( mode: str, token_tags: torch.Tensor | None = None ) -> MiniMaxH3DenoiseBranch: @@ -166,7 +193,6 @@ def test_inplace_target_update_matches_scheduler_math(): state, denoised, sigma_curr=sigma_curr, - sigma_next=sigma_next, sigma_ratio=ratio, ) diff --git a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_vae_parallel_modes.py b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_vae_parallel_modes.py index 352cd5a267..d3d811d717 100644 --- a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_vae_parallel_modes.py +++ b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_vae_parallel_modes.py @@ -332,18 +332,12 @@ def test_fused_decoder_matches_unfused_forward(): "h3_vae_scale_add_rmsnorm", wraps=diffusion_ops.h3_vae_scale_add_rmsnorm, ) as scale_add, - mock.patch.object( - diffusion_ops, - "h3_vae_qk_rmsnorm_rope", - wraps=diffusion_ops.h3_vae_qk_rmsnorm_rope, - ) as qk_rope, ): os.environ["MINIMAX_H3_VAE_DECODER_FUSED_NORM"] = "0" eager = decoder(latent) os.environ["MINIMAX_H3_VAE_DECODER_FUSED_NORM"] = "1" fused = decoder(latent) assert scale_add.call_count == 3 - assert qk_rope.call_count == 2 finally: if previous is None: os.environ.pop("MINIMAX_H3_VAE_DECODER_FUSED_NORM", None) @@ -368,10 +362,7 @@ def test_fused_decoder_is_disabled_off_nvidia(): set_forward_context(current_timestep=0, attn_metadata=None), mock.patch.dict(os.environ, {"MINIMAX_H3_VAE_DECODER_FUSED_NORM": "1"}), mock.patch(f"{vae_dir}.vae_vit.current_platform.is_cuda", return_value=False), - mock.patch(f"{vae_dir}.attention.current_platform.is_cuda", return_value=False), mock.patch.object(diffusion_ops, "h3_vae_scale_add_rmsnorm") as scale_add, - mock.patch.object(diffusion_ops, "h3_vae_qk_rmsnorm_rope") as qk_rope, ): decoder(latent) scale_add.assert_not_called() - qk_rope.assert_not_called() diff --git a/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py b/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py index 6ebd03de24..283e25b5cc 100644 --- a/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py +++ b/python/sglang/multimodal_gen/test/unit/test_performance_metrics.py @@ -12,9 +12,6 @@ from sglang.multimodal_gen.runtime.disaggregation.orchestrator import ( _deserialize_request_metrics, ) from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker -from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( - WarmupPhasePeak, -) from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage @@ -120,27 +117,6 @@ def test_worker_perf_dump_has_one_writer_per_replica( worker._record_replica_peak_memory.assert_called_once_with([metrics]) -def test_request_metrics_attributes_steps_and_iterations_to_active_stage(): - metrics = RequestMetrics("request") - metrics.active_stage_name = "ShapeStage" - - metrics.record_step(0.1) - metrics.record_stage_iterations(4, 50) - metrics.active_stage_name = "PaintStage" - metrics.record_step(0.2) - metrics.record_stage_iterations(4, 30) - - assert metrics.steps == [100.0, 200.0] - assert metrics.steps_by_stage == { - "ShapeStage": [100.0], - "PaintStage": [200.0], - } - assert metrics.stage_iterations == { - "ShapeStage": (4, 50), - "PaintStage": (4, 30), - } - - @pytest.mark.parametrize("roundtrip", [False, True]) @pytest.mark.parametrize( "stage_class,profile_name,is_denoising", @@ -291,20 +267,11 @@ def test_server_warmup_preserves_peak_after_managed_stage_timeline(): worker._auto_residency_warmup_records = [] residency_manager = Mock() residency_manager.take_warmup_phase_peaks.return_value = { - "0:denoise:use:transformer": WarmupPhasePeak(("transformer",), 8) + "0:denoise:use:transformer": 8 } - residency_manager.current_device_components.return_value = ("transformer",) device_module = Mock() device_module.max_memory_allocated.return_value = 9 device_module.max_memory_reserved.return_value = 12 - req = SimpleNamespace( - width=64, - height=64, - num_frames=1, - num_inference_steps=1, - metrics=None, - ) - with ( patch.object(torch, "get_device_module", return_value=device_module), patch.object( @@ -314,19 +281,14 @@ def test_server_warmup_preserves_peak_after_managed_stage_timeline(): ), ): worker._record_server_warmup_memory( - req=req, - workload=(128, 96, 9, 2), + workload=(128, 96, 9), baseline_allocated_bytes=3, succeeded=True, ) record = worker._auto_residency_warmup_records[0] assert record.peak_allocated_bytes == 9 - assert record.peak_reserved_bytes == 12 assert (record.width, record.height, record.num_frames) == (128, 96, 9) - assert record.num_inference_steps == 2 - assert record.phase_peak_allocated_bytes["request:untracked"] == 9 - assert record.phase_active_components["request:untracked"] == ("transformer",) def test_server_warmup_does_not_treat_allocator_cache_as_untracked_live_memory(): @@ -334,20 +296,11 @@ def test_server_warmup_does_not_treat_allocator_cache_as_untracked_live_memory() worker._auto_residency_warmup_records = [] residency_manager = Mock() residency_manager.take_warmup_phase_peaks.return_value = { - "0:denoise:use:transformer": WarmupPhasePeak(("transformer",), 8) + "0:denoise:use:transformer": 8 } - residency_manager.current_device_components.return_value = ("transformer",) device_module = Mock() device_module.max_memory_allocated.return_value = 8 device_module.max_memory_reserved.return_value = 12 - req = SimpleNamespace( - width=64, - height=64, - num_frames=1, - num_inference_steps=1, - metrics=None, - ) - with ( patch.object(torch, "get_device_module", return_value=device_module), patch.object( @@ -357,14 +310,13 @@ def test_server_warmup_does_not_treat_allocator_cache_as_untracked_live_memory() ), ): worker._record_server_warmup_memory( - req=req, - workload=(64, 64, 1, 1), + workload=(64, 64, 1), baseline_allocated_bytes=3, succeeded=True, ) record = worker._auto_residency_warmup_records[0] - assert "request:untracked" not in record.phase_peak_allocated_bytes + assert record.peak_allocated_bytes == 8 def test_baseline_config_loads_per_scenario_peak_vram(tmp_path): @@ -589,65 +541,6 @@ def test_results_json_merges_retry_sessions(tmp_path): assert {item["test_name"] for item in results} == {"first", "second"} -def _warmup_batch_for_iterations(steps: int, target_steps: int): - from sglang.multimodal_gen.runtime.utils.perf_logger import RequestMetrics - - metrics = RequestMetrics(request_id="probe") - metrics.suppress_stage_breakdown = False - metrics.active_stage_name = "DenoisingStage" - return SimpleNamespace( - metrics=metrics, - num_inference_steps=steps, - extra={"warmup_target_num_inference_steps": target_steps}, - is_warmup=True, - ) - - -def test_stage_formula_records_probe_and_default_iterations(): - from sglang.multimodal_gen.runtime.pipelines_core.stages.base import ( - record_default_workload_iterations, - ) - - batch = _warmup_batch_for_iterations(steps=4, target_steps=50) - stage = SimpleNamespace(default_workload_iterations=lambda batch, steps: steps - 1) - record_default_workload_iterations(stage, batch) - assert batch.metrics.stage_iterations == {"DenoisingStage": (3, 49)} - - -def test_fixed_schedule_formula_records_the_same_count_twice(): - from sglang.multimodal_gen.runtime.pipelines_core.stages.base import ( - record_default_workload_iterations, - ) - - batch = _warmup_batch_for_iterations(steps=4, target_steps=50) - stage = SimpleNamespace(default_workload_iterations=lambda batch, steps: 8) - record_default_workload_iterations(stage, batch) - assert batch.metrics.stage_iterations == {"DenoisingStage": (8, 8)} - - -def test_explicit_loop_record_wins_over_the_formula(): - from sglang.multimodal_gen.runtime.pipelines_core.stages.base import ( - record_default_workload_iterations, - ) - - batch = _warmup_batch_for_iterations(steps=4, target_steps=50) - batch.metrics.record_stage_iterations(12, 12) - stage = SimpleNamespace(default_workload_iterations=lambda batch, steps: steps) - record_default_workload_iterations(stage, batch) - assert batch.metrics.stage_iterations == {"DenoisingStage": (12, 12)} - - -def test_stage_without_a_formula_records_nothing(): - from sglang.multimodal_gen.runtime.pipelines_core.stages.base import ( - record_default_workload_iterations, - ) - - batch = _warmup_batch_for_iterations(steps=4, target_steps=50) - stage = SimpleNamespace(default_workload_iterations=lambda batch, steps: None) - record_default_workload_iterations(stage, batch) - assert batch.metrics.stage_iterations == {} - - def test_warmup_peak_stays_out_of_the_runtime_peak(): from unittest import mock diff --git a/python/sglang/multimodal_gen/test/unit/test_qwen_image21.py b/python/sglang/multimodal_gen/test/unit/test_qwen_image21.py index 8e281623e6..7b2e1759be 100644 --- a/python/sglang/multimodal_gen/test/unit/test_qwen_image21.py +++ b/python/sglang/multimodal_gen/test/unit/test_qwen_image21.py @@ -50,6 +50,7 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.component_residency_ from sglang.multimodal_gen.runtime.managers.scheduler import Scheduler from sglang.multimodal_gen.runtime.models.dits.qwen_image21 import ( QwenImage21Attention, + QwenImage21TransformerBlock, build_layout, ) from sglang.multimodal_gen.runtime.models.encoders.base import ( @@ -497,19 +498,27 @@ def test_condition_slots_expand_to_actual_latent_grid(): torch.testing.assert_close(collapsed[slots][0], hidden[2]) -class _RecordingBlock(torch.nn.Module): - def __init__(self, layer_id): - super().__init__() - self._layer_id = layer_id - self.seen = None +class _RecordingAttention(torch.nn.Module): + def forward(self, x, ropes, prefixes, layouts, caches, prefix_pad): + self.seen = caches + return torch.zeros_like(x), [None] * len(caches) - def forward(self, hidden_states, *args): - caches = [cache[self._layer_id] for cache in args[-1]] - self.seen = caches[0] - return hidden_states + +def _block_recording_caches(layer_id, dim=4): + # The real block forward with its attention swapped for a recorder. + block = QwenImage21TransformerBlock.__new__(QwenImage21TransformerBlock) + torch.nn.Module.__init__(block) + block._layer_id = layer_id + block.img_norm1 = torch.nn.LayerNorm(dim, elementwise_affine=False) + block.img_norm2 = torch.nn.LayerNorm(dim, elementwise_affine=False) + block.attn = _RecordingAttention() + block.img_mlp = torch.nn.Identity() + return block class _UnifiedBlocks(torch.nn.Module): + """Cache-DiT's wrapper: every wrapped block receives the same extras.""" + def __init__(self, blocks): super().__init__() self.transformer_blocks = torch.nn.ModuleList(blocks) @@ -521,49 +530,21 @@ class _UnifiedBlocks(torch.nn.Module): return x -def _run_blocks(blocks, prefix_caches): - x = torch.zeros(1) - for block in blocks: - x = block(x, prefix_caches) - return x - - -def test_cache_dit_wrapper_keeps_per_layer_prefix_kv(): - inner = [_RecordingBlock(0), _RecordingBlock(1), _RecordingBlock(2)] - wrapped = torch.nn.ModuleList([_UnifiedBlocks(inner)]) +@pytest.mark.parametrize("cache_dit_wrapped", [False, True]) +def test_blocks_keep_per_layer_prefix_kv(cache_dit_wrapped): + blocks = [_block_recording_caches(layer_id) for layer_id in range(3)] + runners = [_UnifiedBlocks(blocks)] if cache_dit_wrapped else blocks prefix_caches = [[{"layer": 0}, {"layer": 1}, {"layer": 2}]] - - _run_blocks(wrapped, prefix_caches) - assert [block.seen for block in inner] == prefix_caches[0] - - -def test_plain_blocks_still_get_per_layer_prefix_kv(): - blocks = torch.nn.ModuleList([_RecordingBlock(0), _RecordingBlock(1)]) - prefix_caches = [[{"layer": 0}, {"layer": 1}]] - - _run_blocks(blocks, prefix_caches) - assert [block.seen for block in blocks] == prefix_caches[0] - - -class _FirstSlotBlock(torch.nn.Module): - """Old loop body: take caches[0] and broadcast it to every layer.""" - - def __init__(self): - super().__init__() - self.seen = None - - def forward(self, hidden_states, *args): - self.seen = args[-1][0] - return hidden_states - - -def test_first_slot_only_caches_are_shared_across_layers(): - inner = [_FirstSlotBlock(), _FirstSlotBlock()] - unified = _UnifiedBlocks(inner) - prefix_caches = [[{"layer": 0}, {"layer": 1}]] - - unified(torch.zeros(1), [cache[0] for cache in prefix_caches]) - assert [block.seen for block in inner] == [prefix_caches[0][0], prefix_caches[0][0]] + x = torch.zeros(1, 2, 4) + modulation = (torch.zeros(1, 1, 4),) * 4 + prefix_states = [{"hidden_states": None}] + for runner in runners: + x = runner( + x, modulation, prefix_states, None, [None], [None], prefix_caches, None + ) + assert [block.attn.seen for block in blocks] == [ + [cache] for cache in prefix_caches[0] + ] def test_adjacent_image_slots_stay_distinct(): diff --git a/python/sglang/multimodal_gen/test/unit/test_qwen_image21_distributed.py b/python/sglang/multimodal_gen/test/unit/test_qwen_image21_distributed.py index 38a38e408c..310478e3d1 100644 --- a/python/sglang/multimodal_gen/test/unit/test_qwen_image21_distributed.py +++ b/python/sglang/multimodal_gen/test/unit/test_qwen_image21_distributed.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 -"""Run with torchrun --standalone --nproc-per-node=2 -m pytest -q .""" +"""Two-rank TP / spatial-shard checks; ``test_two_ranks`` torchruns this file.""" import os +import subprocess +import sys from types import SimpleNamespace import pytest @@ -30,13 +32,23 @@ from sglang.multimodal_gen.runtime.models.vaes.autoencoder_kl_qwenimage21 import from sglang.multimodal_gen.runtime.pipelines_core import Req from sglang.multimodal_gen.runtime.server_args import ServerArgs, set_global_server_args -pytestmark = pytest.mark.skipif( - not torch.cuda.is_available() or int(os.environ.get("WORLD_SIZE", "1")) != 2, +_IS_RANK = int(os.environ.get("WORLD_SIZE", "1")) == 2 +requires_ranks = pytest.mark.skipif( + not torch.cuda.is_available() or not _IS_RANK, reason="requires two CUDA ranks launched by torchrun", ) -@pytest.fixture(scope="module", autouse=True) +@pytest.mark.skipif( + _IS_RANK or torch.cuda.device_count() < 2, reason="requires two CUDA GPUs" +) +def test_two_ranks(): + cmd = [sys.executable, "-m", "torch.distributed.run", "--standalone"] + cmd += ["--nproc-per-node=2", "-m", "pytest", "-q", __file__] + subprocess.run(cmd, check=True, timeout=1200) + + +@pytest.fixture(scope="module") def distributed(): args = ServerArgs( model_path="Qwen/Qwen-Image-2.1", @@ -56,6 +68,8 @@ def distributed(): torch.backends.cudnn.allow_tf32 = cudnn_tf32 +@requires_ranks +@pytest.mark.usefixtures("distributed") @pytest.mark.parametrize("edit", [False, True]) @torch.no_grad() def test_encoder_tp_shards_weights_and_preserves_conditioning(edit): @@ -126,6 +140,8 @@ def test_encoder_tp_shards_weights_and_preserves_conditioning(edit): torch.testing.assert_close(actual, expected, atol=2e-5, rtol=2e-5) +@requires_ranks +@pytest.mark.usefixtures("distributed") @pytest.mark.parametrize("height", [4, 5]) @pytest.mark.parametrize("residual", [False, True]) @pytest.mark.parametrize("dtype", [torch.float32, torch.float64]) diff --git a/python/sglang/multimodal_gen/test/unit/test_qwen_image_fp8_norm_quant.py b/python/sglang/multimodal_gen/test/unit/test_qwen_image_fp8_norm_quant.py index ac47f2500d..62bccb7d7c 100644 --- a/python/sglang/multimodal_gen/test/unit/test_qwen_image_fp8_norm_quant.py +++ b/python/sglang/multimodal_gen/test/unit/test_qwen_image_fp8_norm_quant.py @@ -4,43 +4,27 @@ import unittest from types import SimpleNamespace from unittest.mock import patch -import torch import torch.nn as nn -from sglang.multimodal_gen.runtime.layers.quantization.modelopt_quant import ( - ModelOptFp8LinearMethod, -) from sglang.multimodal_gen.runtime.models.dits.qwen_image import ( QwenImageTransformerBlock, ) +from sglang.multimodal_gen.test.unit.modelopt_fp8_stub import fp8_linear from sglang.test.test_utils import CustomTestCase -def _fp8_linear(input_scale: float) -> nn.Module: - linear = nn.Module() - linear.quant_method = object.__new__(ModelOptFp8LinearMethod) - linear.register_parameter( - "input_scale", - nn.Parameter( - torch.tensor(input_scale, dtype=torch.float32, device="cuda"), - requires_grad=False, - ), - ) - return linear - - def _attention(*, fused: bool, scales: tuple[float, ...]) -> SimpleNamespace: if fused: return SimpleNamespace( use_fused_qkv=True, - to_qkv=_fp8_linear(scales[0]), + to_qkv=fp8_linear(scales[0]), added_kv_proj_dim=None, ) return SimpleNamespace( use_fused_qkv=False, - to_q=_fp8_linear(scales[0]), - to_k=_fp8_linear(scales[1]), - to_v=_fp8_linear(scales[2]), + to_q=fp8_linear(scales[0]), + to_k=fp8_linear(scales[1]), + to_v=fp8_linear(scales[2]), added_kv_proj_dim=None, ) diff --git a/python/sglang/multimodal_gen/test/unit/test_readonly_safetensors.py b/python/sglang/multimodal_gen/test/unit/test_readonly_safetensors.py index 05f7c1b996..368da9616e 100644 --- a/python/sglang/multimodal_gen/test/unit/test_readonly_safetensors.py +++ b/python/sglang/multimodal_gen/test/unit/test_readonly_safetensors.py @@ -7,7 +7,6 @@ from safetensors.torch import load_file, save_file from sglang.multimodal_gen.runtime.loader.readonly_safetensors import ( iter_safetensors_readonly, load_safetensors_readonly, - safetensors_keys, ) @@ -33,7 +32,6 @@ def test_readonly_load_matches_safetensors(tmp_path): assert ours[name].dtype == theirs[name].dtype assert ours[name].shape == theirs[name].shape assert torch.equal(ours[name], theirs[name]) - assert safetensors_keys(str(path)) == list(theirs) @pytest.mark.skipif(sys.platform != "linux", reason="/proc/self/maps") diff --git a/python/sglang/multimodal_gen/test/unit/test_sensenova_u1.py b/python/sglang/multimodal_gen/test/unit/test_sensenova_u1.py index 9678e8ffb4..3c4dcc88ea 100644 --- a/python/sglang/multimodal_gen/test/unit/test_sensenova_u1.py +++ b/python/sglang/multimodal_gen/test/unit/test_sensenova_u1.py @@ -37,9 +37,6 @@ from sglang.multimodal_gen.runtime.models.sensenova_u1.neo_unify.configuration_n from sglang.multimodal_gen.runtime.models.sensenova_u1.neo_unify.configuration_neo_vit import ( NEOVisionConfig, ) -from sglang.multimodal_gen.runtime.models.sensenova_u1.neo_unify.conversation import ( - get_conv_template, -) from sglang.multimodal_gen.runtime.models.sensenova_u1.neo_unify.modeling_neo_chat import ( NEOChatModel, _copy_right_aligned_prefix_bnsd, @@ -52,8 +49,6 @@ from sglang.multimodal_gen.runtime.models.sensenova_u1.neo_unify.modeling_qwen3 _flash_or_sdpa, _sdpa_attn_func, create_block_causal_mask, - make_qwen3_rms_norm, - npu_fia_available, position_ids_from_indexes, ) from sglang.multimodal_gen.runtime.pipelines_core.executors.pipeline_executor import ( @@ -69,7 +64,6 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.s from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.server_args.server_args import ServerArgs from sglang.multimodal_gen.runtime.utils.perf_logger import MemorySnapshot -from sglang.srt.layers.layernorm import RMSNorm class _FakeSenseNovaModel: @@ -172,33 +166,6 @@ class _WorkerBackedSchedulerClient: return next(self.worker.execute_forward_sequentially(batches)) -@pytest.mark.parametrize( - ("template_name", "expected_system_message"), - [ - ( - "Hermes-2", - "\u4f60\u662f\u7531\u4e0a\u6d77\u4eba\u5de5\u667a\u80fd\u5b9e\u9a8c\u5ba4\u8054\u5408\u5546\u6c64\u79d1\u6280\u5f00\u53d1\u7684\u4e66\u751f\u591a\u6a21\u6001\u5927\u6a21\u578b\uff0c\u82f1\u6587\u540d\u53ebInternVL, \u662f\u4e00\u4e2a\u6709\u7528\u65e0\u5bb3\u7684\u4eba\u5de5\u667a\u80fd\u52a9\u624b\u3002", - ), - ( - "internlm2-chat", - "\u4f60\u662f\u7531\u4e0a\u6d77\u4eba\u5de5\u667a\u80fd\u5b9e\u9a8c\u5ba4\u8054\u5408\u5546\u6c64\u79d1\u6280\u5f00\u53d1\u7684\u4e66\u751f\u591a\u6a21\u6001\u5927\u6a21\u578b\uff0c\u82f1\u6587\u540d\u53ebInternVL, \u662f\u4e00\u4e2a\u6709\u7528\u65e0\u5bb3\u7684\u4eba\u5de5\u667a\u80fd\u52a9\u624b\u3002", - ), - ( - "phi3-chat", - "\u4f60\u662f\u7531\u4e0a\u6d77\u4eba\u5de5\u667a\u80fd\u5b9e\u9a8c\u5ba4\u8054\u5408\u5546\u6c64\u79d1\u6280\u5f00\u53d1\u7684\u4e66\u751f\u591a\u6a21\u6001\u5927\u6a21\u578b\uff0c\u82f1\u6587\u540d\u53ebInternVL, \u662f\u4e00\u4e2a\u6709\u7528\u65e0\u5bb3\u7684\u4eba\u5de5\u667a\u80fd\u52a9\u624b\u3002", - ), - ( - "internvl2_5", - "\u4f60\u662f\u4e66\u751f\xb7\u4e07\u8c61\uff0c\u82f1\u6587\u540d\u662fInternVL\uff0c\u662f\u7531\u4e0a\u6d77\u4eba\u5de5\u667a\u80fd\u5b9e\u9a8c\u5ba4\u3001\u6e05\u534e\u5927\u5b66\u53ca\u591a\u5bb6\u5408\u4f5c\u5355\u4f4d\u8054\u5408\u5f00\u53d1\u7684\u591a\u6a21\u6001\u5927\u8bed\u8a00\u6a21\u578b\u3002", - ), - ], -) -def test_sensenova_u1_conversation_preserves_upstream_system_prompt( - template_name, expected_system_message -): - assert get_conv_template(template_name).system_message == expected_system_message - - def _force_generator_fallback(monkeypatch, device_type): original_generator = torch.Generator @@ -370,24 +337,6 @@ def test_sensenova_u1_right_aligns_bnsd_prefix_for_npu_fia(): assert destination[:, :, 5:].eq(0).all() -@pytest.mark.parametrize("available", [False, True]) -def test_sensenova_u1_npu_fia_checks_operator_availability(monkeypatch, available): - namespace = SimpleNamespace() - if available: - namespace.npu_fused_infer_attention_score = object() - monkeypatch.setattr(torch.ops, "npu", namespace, raising=False) - - assert npu_fia_available() is available - - -def test_sensenova_u1_shared_rmsnorm_uses_framework_dispatch(): - norm = make_qwen3_rms_norm(64, eps=1e-6) - - assert isinstance(norm, RMSNorm) - assert norm.cast_x_before_out_mul - assert norm._forward_method != norm.forward_native - - @torch.no_grad() def test_sensenova_u1_fused_dense_mlp_matches_original(monkeypatch): config = SimpleNamespace( @@ -658,13 +607,6 @@ def test_sensenova_u1_accepts_openai_image_api_num_frames(): assert params.data_type == DataType.IMAGE -def test_sensenova_u1_scheduler_capabilities(): - config = SenseNovaU1PipelineConfig() - - assert config.supports_dynamic_batching() - assert config.supports_sequential_multi_output_inference() - - def _make_sensenova_u1_scheduler_request( request_id: str, prompt: str, seed: int | list[int], **sampling_overrides ) -> Req: diff --git a/python/sglang/multimodal_gen/test/unit/test_sensenova_u1_backbone.py b/python/sglang/multimodal_gen/test/unit/test_sensenova_u1_backbone.py index d4348aa3e6..7dd885257e 100644 --- a/python/sglang/multimodal_gen/test/unit/test_sensenova_u1_backbone.py +++ b/python/sglang/multimodal_gen/test/unit/test_sensenova_u1_backbone.py @@ -7,6 +7,9 @@ from sglang.multimodal_gen.runtime.models.sensenova_u1.neo_unify.configuration_n NEOLLMConfig, NEOMoELLMConfig, ) +from sglang.multimodal_gen.runtime.models.sensenova_u1.neo_unify.modeling_fm_modules import ( + FlowMatchingHead, +) from sglang.multimodal_gen.runtime.models.sensenova_u1.neo_unify.modeling_qwen3 import ( Qwen3ForCausalLM, ) @@ -83,3 +86,19 @@ def test_sensenova_backbone_prefix_cache_and_generation(moe, dtype): kwargs["image_gen_indicators"][0, 0] = False with pytest.raises(NotImplementedError, match="Mixed und/gen"): model.model(**kwargs) + + +@torch.inference_mode() +def test_sensenova_deep_fm_head_maps_tokens_with_timesteps(): + """Checkpoints with fm_head_layers > 2 build FlowMatchingHead, which must + construct and map (tokens, hidden) plus per-token timesteps to pixels.""" + head = FlowMatchingHead(input_dim=8, out_dim=12, dim=16, layers=3) + assert { + "net.time_embed.mlp.0.weight", + "net.input_proj.weight", + "net.res_blocks.2.adaLN_modulation.1.weight", + "net.final_layer.adaLN_modulation.1.weight", + "net.final_layer.linear.weight", + } <= set(head.state_dict()) + out = head(torch.randn(5, 8), torch.rand(()).repeat(5)) + assert out.shape == (5, 12) diff --git a/python/sglang/multimodal_gen/test/unit/test_server_args.py b/python/sglang/multimodal_gen/test/unit/test_server_args.py index fcbb26b54b..9834854011 100644 --- a/python/sglang/multimodal_gen/test/unit/test_server_args.py +++ b/python/sglang/multimodal_gen/test/unit/test_server_args.py @@ -25,9 +25,6 @@ from sglang.multimodal_gen.configs.pipeline_configs.flux import ( Flux2PipelineConfig, FluxPipelineConfig, ) -from sglang.multimodal_gen.configs.pipeline_configs.helios import ( - HeliosDistilledConfig, -) from sglang.multimodal_gen.configs.pipeline_configs.hunyuan import FastHunyuanConfig from sglang.multimodal_gen.configs.pipeline_configs.lingbot_world import ( LingBotWorldCausalDMDConfig, @@ -40,7 +37,6 @@ from sglang.multimodal_gen.configs.pipeline_configs.longlive2 import ( ) from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import ( LTX2PipelineConfig, - LTX23PipelineConfig, ) from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3 import ( FastH3PipelineConfig, @@ -129,6 +125,10 @@ def _mock_cuda_platform( "sglang.multimodal_gen.runtime.platforms.current_platform.is_cuda", return_value=True, ), + patch( + "sglang.multimodal_gen.runtime.platforms.current_platform.is_rocm", + return_value=False, + ), patch( "sglang.multimodal_gen.runtime.platforms.current_platform.get_device_total_memory", return_value=memory_gb * 1024**3, @@ -196,6 +196,8 @@ class _CudaPlatformTestCase(unittest.TestCase): class TestSchedulerEndpoints(unittest.TestCase): def test_host_normalization_preserves_replica_ports(self): + """The scheduler ingress is an unauthenticated pickle-RPC endpoint; it must + never follow a wildcard --host (see ServerArgs.scheduler_endpoint_for).""" args = _from_dict_without_model_resolution({"model_path": "test/model"}) args.dp_size = 2 args.scheduler_port = 23000 @@ -380,28 +382,6 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase): self.assertEqual(backend.name, "TORCH_SDPA") self.assertEqual(matched_key, "text_encoder") - def test_ltx_automatic_text_encoder_backend_is_not_explicit(self): - args = _from_dict_without_model_resolution( - {"model_path": "Lightricks/LTX-2.3"}, - pipeline_config=LTX2PipelineConfig(), - ) - - self.assertEqual( - args.component_attention_backends, {"text_encoder": "torch_sdpa"} - ) - self.assertTrue(args.is_component_attention_backend_automatic("text_encoder")) - - def test_ltx_explicit_text_encoder_backend_remains_explicit(self): - args = _from_dict_without_model_resolution( - { - "model_path": "Lightricks/LTX-2.3", - "component_attention_backends": {"text_encoder": "torch_sdpa"}, - }, - pipeline_config=LTX2PipelineConfig(), - ) - - self.assertFalse(args.is_component_attention_backend_automatic("text_encoder")) - def _fasth3_args(self, capability=(9, 0), **kwargs): with patch( "sglang.multimodal_gen.runtime.platforms.current_platform.get_device_capability", @@ -414,14 +394,27 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase): def test_fasth3_selects_vsa_h3_for_the_transformer(self): for capability in ((8, 0), (9, 0), (10, 0), (10, 3)): - with self.subTest(capability=capability): + # Check configuration selection without importing a GPU backend. + with ( + self.subTest(capability=capability), + patch( + "sglang.multimodal_gen.runtime.layers.attention.selector.get_attn_backend" + ) as get_attn_backend, + ): args = self._fasth3_args(capability) self.assertEqual( args.component_attention_backends, {"transformer": "video_sparse_attn_h3"}, ) + get_attn_backend.assert_called_once() + self.assertEqual( + get_attn_backend.call_args.kwargs["selected_attention_backend"], + AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3, + ) self.assertTrue( - args.is_component_attention_backend_automatic("transformer") + get_attn_backend.call_args.kwargs[ + "attention_requirements" + ].packed_varlen ) def test_fasth3_keeps_a_requested_or_incompatible_transformer_backend(self): @@ -442,9 +435,6 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase): args.component_attention_backends, kwargs.get("component_attention_backends", {}), ) - self.assertFalse( - args.is_component_attention_backend_automatic("transformer") - ) def test_invalid_component_attention_backend_raises(self): with self.assertRaises(ValueError): @@ -1280,28 +1270,6 @@ class TestMiniMaxH3Routing(unittest.TestCase): class TestOffloadDefaults(_CudaPlatformTestCase): - def test_wan_decode_precision_defaults(self): - for pipeline_config in ( - WanT2V480PConfig(), - WanI2V480PConfig(), - ): - with self.subTest(pipeline_config=pipeline_config.__class__.__name__): - self.assertEqual(pipeline_config.vae_precision, "fp32") - self.assertEqual(pipeline_config.vae_decode_precision, "bf16") - - for pipeline_config in ( - FastWan2_2_TI2V_5B_Config(), - Wan2_2_T2V_A14B_Config(), - Wan2_2_I2V_A14B_Config(), - ): - with self.subTest(pipeline_config=pipeline_config.__class__.__name__): - self.assertEqual(pipeline_config.vae_precision, "fp32") - self.assertEqual(pipeline_config.vae_decode_precision, "fp32") - - generic_config = PipelineConfig() - self.assertEqual(generic_config.vae_precision, "fp32") - self.assertIsNone(generic_config.vae_decode_precision) - def _from_dict_with_pipeline_config( self, pipeline_config, @@ -1971,108 +1939,21 @@ class TestOffloadDefaults(_CudaPlatformTestCase): self.assertEqual(args.layerwise_offload_components, ["dit"]) self.assertEqual(args.residency_mode("transformer"), LAYERWISE_OFFLOAD) - def test_pipeline_configs_declare_auto_tune_hints(self): - qwen_deployment = QwenImagePipelineConfig().get_model_deployment_config() - cosmos3_deployment = Cosmos3Config( - model_path="nvidia/Cosmos3-Nano" - ).get_model_deployment_config() - cosmos3_super_deployment = Cosmos3Config( - model_path="nvidia/Cosmos3-Super" - ).get_model_deployment_config() - local_cosmos3_nano_deployment = Cosmos3Config( - model_path="/models/custom-checkpoint", is_nano=True - ).get_model_deployment_config() - wan_deployment = WanT2V480PConfig().get_model_deployment_config() - mova_deployment = MOVAPipelineConfig().get_model_deployment_config() - zimage_deployment = ZImagePipelineConfig().get_model_deployment_config() - lingbot_deployment = LingBotWorldCausalDMDConfig().get_model_deployment_config() - ltx_deployment = LTX2PipelineConfig().get_model_deployment_config() - ltx23_config = LTX23PipelineConfig() - longlive_deployment = LongLive2T2VConfig().get_model_deployment_config() - sana_wm_deployment = SanaWMPipelineConfig().get_model_deployment_config() + def test_cosmos3_resident_threshold_follows_checkpoint_size(self): + def threshold(**kwargs): + deployment = Cosmos3Config(**kwargs).get_model_deployment_config() + return deployment.keep_resident_min_available_gb - self.assertIsNone(qwen_deployment.fsdp_auto_min_available_memory_gb) - self.assertEqual(qwen_deployment.dit_layerwise_offload_modes, ()) - - self.assertEqual(cosmos3_deployment.keep_resident_min_available_gb, 90) - self.assertEqual(cosmos3_deployment.keep_resident_components, ("dit", "vae")) - self.assertEqual(cosmos3_super_deployment.keep_resident_min_available_gb, 120) + nano = threshold(model_path="nvidia/Cosmos3-Nano") self.assertEqual( - local_cosmos3_nano_deployment.keep_resident_min_available_gb, 90 + threshold(model_path="/models/custom-checkpoint", is_nano=True), nano ) + self.assertGreater(threshold(model_path="nvidia/Cosmos3-Super"), nano) - self.assertIsNone(wan_deployment.fsdp_auto_min_available_memory_gb) - self.assertEqual(wan_deployment.dit_layerwise_offload_modes, ("memory",)) - self.assertEqual(wan_deployment.keep_resident_min_available_gb, 60) - self.assertEqual(wan_deployment.keep_resident_components, ("dit",)) - - self.assertIsNone(mova_deployment.fsdp_auto_min_available_memory_gb) - self.assertEqual( - mova_deployment.dit_layerwise_offload_modes, ("auto", "memory") - ) - self.assertEqual(mova_deployment.keep_resident_min_available_gb, 130) - self.assertEqual(mova_deployment.keep_resident_components, ("dit", "vae")) - - self.assertEqual(zimage_deployment.fsdp_auto_min_available_memory_gb, 40) - self.assertEqual(zimage_deployment.keep_resident_min_available_gb, 30) - self.assertTrue(zimage_deployment.fsdp_auto_requires_cfg) - self.assertEqual(zimage_deployment.dit_layerwise_offload_modes, ()) - - self.assertEqual(lingbot_deployment.dit_layerwise_offload_modes, ("memory",)) - self.assertEqual(lingbot_deployment.keep_resident_min_available_gb, 70) - self.assertEqual(lingbot_deployment.keep_resident_components, ("dit",)) - - self.assertEqual(ltx_deployment.keep_resident_min_available_gb, 70) - self.assertEqual(ltx_deployment.keep_resident_components, ("dit",)) - self.assertEqual( - ltx_deployment.auto_cfg_parallel_degree_by_num_gpus, ((4, 1), (8, 1)) - ) - self.assertEqual(ltx_deployment.get_auto_cfg_parallel_degree(4), 1) - self.assertEqual(ltx_deployment.get_auto_cfg_parallel_degree(8), 1) - self.assertEqual(ltx_deployment.get_auto_cfg_parallel_degree(2), 2) - self.assertEqual(longlive_deployment.keep_resident_min_available_gb, 60) - self.assertEqual( - longlive_deployment.keep_resident_components, - ("dit", "text_encoder", "vae"), - ) - self.assertFalse( - LTX2PipelineConfig().dit_config.arch_config.enable_packed_qkv_input_a2a - ) - self.assertFalse( - ltx23_config.dit_config.arch_config.enable_packed_qkv_input_a2a - ) - - self.assertEqual(sana_wm_deployment.fsdp_auto_min_available_memory_gb, 60) - self.assertEqual(sana_wm_deployment.dit_layerwise_offload_modes, ("memory",)) - self.assertEqual(sana_wm_deployment.keep_resident_min_available_gb, 120) - self.assertEqual(sana_wm_deployment.keep_resident_components, ("dit", "vae")) - - helios_deployment = HeliosDistilledConfig().get_model_deployment_config() - self.assertEqual(helios_deployment.keep_resident_min_available_gb, 120) - self.assertEqual(helios_deployment.keep_resident_components, ("dit", "vae")) - self.assertEqual(helios_deployment.dit_layerwise_offload_modes, ("memory",)) - - fast_hunyuan_deployment = FastHunyuanConfig().get_model_deployment_config() - self.assertEqual(fast_hunyuan_deployment.keep_resident_min_available_gb, 60) - self.assertEqual( - fast_hunyuan_deployment.keep_resident_components, ("dit", "vae") - ) - - fast_wan_deployment = FastWan2_2_TI2V_5B_Config().get_model_deployment_config() - self.assertEqual(fast_wan_deployment.keep_resident_min_available_gb, 60) - self.assertEqual(fast_wan_deployment.keep_resident_components, ("dit",)) - - for dual_dit_config in ( - Wan2_2_T2V_A14B_Config(), - Wan2_2_I2V_A14B_Config(), - ): - dual_dit_deployment = dual_dit_config.get_model_deployment_config() - self.assertIsNone(dual_dit_deployment.keep_resident_min_available_gb) - self.assertEqual(dual_dit_deployment.keep_resident_components, ("vae",)) - - # default keeps only vae resident (encoders are large, dit owned by FSDP) - self.assertEqual(qwen_deployment.keep_resident_components, ("vae",)) - self.assertIsNone(qwen_deployment.keep_resident_min_available_gb) + def test_auto_cfg_parallel_degree_falls_back_to_two(self): + deployment = LTX2PipelineConfig().get_model_deployment_config() + self.assertEqual(deployment.get_auto_cfg_parallel_degree(4), 1) + self.assertEqual(deployment.get_auto_cfg_parallel_degree(2), 2) def test_longlive_residency_scales_with_available_memory(self): high_memory_args = self._from_dict_with_pipeline_config( @@ -3476,13 +3357,6 @@ class TestModelIdResolution(unittest.TestCase): self.assertIs(info.pipeline_config_cls, QwenImagePipelineConfig) - def test_model_id_works_after_tilde_expansion(self): - # simulate the full flow: user passes ~/..., engine expands and resolves - expanded = os.path.expanduser("~/.cache/huggingface/hub/bbb/snapshots/ccc") - _get_config_info.cache_clear() - info = _get_config_info(expanded, model_id="Qwen-Image") - self.assertIsNotNone(info) - def test_hf_cache_snapshot_path_resolves_registered_nvfp4_model(self): path = ( "/root/.cache/huggingface/hub/" @@ -3612,14 +3486,6 @@ class TestPerRoleParallelism(unittest.TestCase): args.derive_pool_work_endpoint(), f"tcp://0.0.0.0:{args.scheduler_port}", ) - self.assertEqual( - args.derive_pool_control_endpoint(), - f"tcp://0.0.0.0:{args.scheduler_port + 1}", - ) - self.assertEqual( - args.derive_pool_control_advertised_endpoint(), - f"tcp://10.0.0.7:{args.scheduler_port + 1}", - ) def test_pool_result_endpoint_validates_addr_and_role(self): args = self._from_dict({"model_path": "/fake", "disagg_server_addr": "bad"}) @@ -3713,12 +3579,6 @@ class TestPipelineResolutionCliOverride(unittest.TestCase): class TestDisaggTimeoutArgs(unittest.TestCase): - def test_disagg_defaults_match_reviewed_values(self): - args = _from_dict_without_model_resolution({"model_path": "/fake"}) - self.assertEqual(args.disagg_max_slots_per_instance, 8) - self.assertEqual(args.disagg_downstream_wait_timeout, 1800) - self.assertEqual(args.disagg_timeout, 3600) - def test_downstream_wait_timeout_cli_arg_is_parsed(self): parser = FlexibleArgumentParser() ServerArgs.add_cli_args(parser) @@ -3732,14 +3592,6 @@ class TestDisaggTimeoutArgs(unittest.TestCase): args, _unknown = parser.parse_known_args(argv) self.assertEqual(args.disagg_downstream_wait_timeout, 45) - def test_disagg_timeout_help_uses_current_defaults(self): - parser = FlexibleArgumentParser() - ServerArgs.add_cli_args(parser) - help_text = parser.format_help() - - self.assertIn("Default: 3600.", help_text) - self.assertIn("Default: 1800.", help_text) - def test_disagg_role_alias_cli_arg_is_accepted(self): parser = FlexibleArgumentParser() ServerArgs.add_cli_args(parser) @@ -3760,10 +3612,6 @@ class TestDisaggTimeoutArgs(unittest.TestCase): class TestSchedulerRpcTimeoutArgs(unittest.TestCase): - def test_scheduler_rpc_timeout_defaults_to_unbounded(self): - args = _from_dict_without_model_resolution({"model_path": "/fake"}) - self.assertIsNone(args.scheduler_rpc_timeout) - def test_scheduler_rpc_timeout_cli_arg_is_parsed_in_seconds(self): parser = FlexibleArgumentParser() ServerArgs.add_cli_args(parser) @@ -3794,10 +3642,6 @@ class TestSchedulerRpcTimeoutArgs(unittest.TestCase): class TestDisaggTransferBackendArgs(unittest.TestCase): - def test_transfer_backend_defaults_to_auto(self): - args = _from_dict_without_model_resolution({"model_path": "/fake"}) - self.assertEqual(args.disagg_transfer_backend, "auto") - def test_transfer_backend_cli_arg_is_parsed(self): parser = FlexibleArgumentParser() ServerArgs.add_cli_args(parser) @@ -3901,87 +3745,6 @@ class TestDirectGpuWeightLoading(_CudaPlatformTestCase): tp_args._validate_direct_gpu_weight_loading() -class TestSchedulerEndpointBinding(unittest.TestCase): - """The scheduler ingress is an unauthenticated pickle-RPC endpoint; it must - never follow a wildcard --host (see ServerArgs.scheduler_endpoint_for).""" - - def _args(self, kwargs): - # Endpoint formatting does not need model resolution or port allocation. - args = ServerArgs.__new__(ServerArgs) - for key, value in kwargs.items(): - setattr(args, key, value) - return args - - def test_wildcard_host_pinned_to_loopback(self): - args = self._args( - {"model_path": "/fake/model", "host": "0.0.0.0", "scheduler_port": 5555} - ) - self.assertEqual(args.scheduler_endpoint_for(0), "tcp://127.0.0.1:5555") - - def test_ipv6_wildcard_host_pinned_to_loopback(self): - args = self._args( - {"model_path": "/fake/model", "host": "::", "scheduler_port": 5555} - ) - self.assertEqual(args.scheduler_endpoint_for(0), "tcp://127.0.0.1:5555") - - def test_default_host_stays_loopback(self): - args = self._args({"model_path": "/fake/model", "scheduler_port": 5555}) - self.assertEqual(args.scheduler_endpoint_for(0), "tcp://127.0.0.1:5555") - - def test_explicit_host_is_honored(self): - args = self._args( - { - "model_path": "/fake/model", - "host": "10.1.2.3", - "scheduler_port": 5555, - } - ) - self.assertEqual(args.scheduler_endpoint_for(0), "tcp://10.1.2.3:5555") - - def test_explicit_ipv6_host_uses_ipv4_loopback(self): - args = self._args( - { - "model_path": "/fake/model", - "host": "::1", - "scheduler_port": 5555, - } - ) - self.assertEqual(args.scheduler_endpoint_for(0), "tcp://127.0.0.1:5555") - - def test_replica_ports_increment_on_loopback(self): - args = self._args( - {"model_path": "/fake/model", "host": "0.0.0.0", "scheduler_port": 5555} - ) - self.assertEqual(args.scheduler_endpoint_for(1), "tcp://127.0.0.1:5556") - - def test_explicit_scheduler_ports_on_loopback(self): - args = self._args( - { - "model_path": "/fake/model", - "host": "0.0.0.0", - "scheduler_port": 5555, - "dp_size": 2, - } - ) - args.scheduler_ports = [6100, 6200] - self.assertEqual(args.scheduler_endpoint, "tcp://127.0.0.1:6100") - self.assertEqual( - args.scheduler_endpoints, - ["tcp://127.0.0.1:6100", "tcp://127.0.0.1:6200"], - ) - - def test_explicit_scheduler_ports_with_ipv6(self): - args = self._args( - { - "model_path": "/fake/model", - "host": "::1", - "scheduler_port": 5555, - } - ) - args.scheduler_ports = [6100, 6200] - self.assertEqual(args.scheduler_endpoint_for(1), "tcp://127.0.0.1:6200") - - class TestLayerwiseResidencyLifetime(unittest.TestCase): def _help_by_option(self): parser = FlexibleArgumentParser() diff --git a/python/sglang/multimodal_gen/test/unit/test_storage.py b/python/sglang/multimodal_gen/test/unit/test_storage.py index 18f0c9ef1d..49233268d5 100644 --- a/python/sglang/multimodal_gen/test/unit/test_storage.py +++ b/python/sglang/multimodal_gen/test/unit/test_storage.py @@ -5,12 +5,10 @@ Tests verify file upload, cleanup, URL generation, and error handling. """ import asyncio -import importlib import os +import sys from types import SimpleNamespace -import pytest - import sglang.multimodal_gen.runtime.entrypoints.openai.storage as storage_mod from sglang.multimodal_gen.runtime.entrypoints.openai.storage import CloudStorage @@ -188,45 +186,45 @@ def test_content_type_detection(tmp_path): assert called["content_type"] == expected_type -# requires moto and boto3 -has_moto = ( - importlib.util.find_spec("moto") is not None - and importlib.util.find_spec("boto3") is not None -) +def test_env_configured_storage_uploads_and_cleans_up(tmp_path, monkeypatch): + """CloudStorage() builds its boto3 client from the SGLANG_S3_* env vars.""" + monkeypatch.setenv("SGLANG_CLOUD_STORAGE_TYPE", "S3") + monkeypatch.setenv("SGLANG_S3_BUCKET_NAME", "integration-test") + monkeypatch.setenv("SGLANG_S3_REGION_NAME", "us-east-1") + monkeypatch.setenv("SGLANG_S3_ACCESS_KEY_ID", "key-id") + monkeypatch.setenv("SGLANG_S3_SECRET_ACCESS_KEY", "secret") + monkeypatch.delenv("SGLANG_S3_ENDPOINT_URL", raising=False) + uploads = [] + client_kwargs = {} + def fake_client(service, **kwargs): + client_kwargs.update(kwargs, service=service) + return SimpleNamespace( + upload_file=lambda *args, **kw: uploads.append((args, kw)) + ) -@pytest.mark.skipif(not has_moto, reason="moto/boto3 not installed") -def test_integration_with_moto(tmp_path): - """Integration test using moto to mock real S3 service.""" - import boto3 - from moto import mock_aws + monkeypatch.setitem(sys.modules, "boto3", SimpleNamespace(client=fake_client)) - os.environ["SGLANG_CLOUD_STORAGE_TYPE"] = "s3" - os.environ["SGLANG_S3_BUCKET_NAME"] = "integration-test" - os.environ["SGLANG_S3_REGION_NAME"] = "us-east-1" + storage = CloudStorage() + assert storage.is_enabled() + assert client_kwargs == { + "service": "s3", + "aws_access_key_id": "key-id", + "aws_secret_access_key": "secret", + "endpoint_url": None, + "region_name": "us-east-1", + } - with mock_aws(): - s3 = boto3.client("s3", region_name="us-east-1") - s3.create_bucket(Bucket="integration-test") + file_path = _create_temp_file(tmp_path, "integration.png", b"test_data") + url = asyncio.run(storage.upload_and_cleanup(file_path)) - storage = CloudStorage() - assert storage.is_enabled() - - file_path = _create_temp_file(tmp_path, "integration.png", b"test_data") - - url = asyncio.run(storage.upload_and_cleanup(file_path)) - - assert url is not None - assert "integration-test" in url - assert "integration.png" in url - assert not os.path.exists(file_path) - - obj = s3.get_object(Bucket="integration-test", Key="integration.png") - assert obj["Body"].read() == b"test_data" - - for key in [ - "SGLANG_CLOUD_STORAGE_TYPE", - "SGLANG_S3_BUCKET_NAME", - "SGLANG_S3_REGION_NAME", - ]: - os.environ.pop(key, None) + assert url == ( + "https://integration-test.s3.us-east-1.amazonaws.com/integration.png" + ) + assert not os.path.exists(file_path) + assert uploads == [ + ( + (file_path, "integration-test", "integration.png"), + {"ExtraArgs": {"ContentType": "image/png"}}, + ) + ] diff --git a/python/sglang/multimodal_gen/test/unit/test_usp_ring_replicated.py b/python/sglang/multimodal_gen/test/unit/test_usp_ring_replicated.py index 2a557d35cc..03dbccde9f 100644 --- a/python/sglang/multimodal_gen/test/unit/test_usp_ring_replicated.py +++ b/python/sglang/multimodal_gen/test/unit/test_usp_ring_replicated.py @@ -49,6 +49,14 @@ def _ring_pair_via_impl(impl, q, k_shard, v_shard): class RingReplicatedBase(unittest.TestCase): B, S_SHARD, REP, H, D = 1, 6, 3, 2, 8 + def setUp(self): + self.enterContext( + patch( + f"{_LAYER}._usp_input_all_to_all_qkv", + side_effect=lambda q, k, v: (q, k, v), + ) + ) + def _attn(self): obj = USPAttention.__new__(USPAttention) obj.skip_sequence_parallel = False diff --git a/python/sglang/multimodal_gen/test/unit/test_vae_loader.py b/python/sglang/multimodal_gen/test/unit/test_vae_loader.py index 2a1f87cbd7..415996fb50 100644 --- a/python/sglang/multimodal_gen/test/unit/test_vae_loader.py +++ b/python/sglang/multimodal_gen/test/unit/test_vae_loader.py @@ -10,6 +10,7 @@ from safetensors.torch import save_file as safetensors_save_file from sglang.multimodal_gen.configs.models.vaes.minimax_h3_audio import ( MiniMaxH3AudioVAEConfig, ) +from sglang.multimodal_gen.configs.models.vaes.sana import SanaVAEConfig from sglang.multimodal_gen.configs.pipeline_configs.cosmos3 import Cosmos3Config from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import LTX2PipelineConfig from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import ( @@ -44,6 +45,7 @@ from sglang.multimodal_gen.runtime.managers.memory_managers import ( host_memory_budget, ) from sglang.multimodal_gen.runtime.models.vaes import wanvae +from sglang.multimodal_gen.runtime.models.vaes.autoencoder_dc import AutoencoderDC from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.decoding_av import ( LTX2AVDecodingStage, ) @@ -182,6 +184,31 @@ class TestPlainWeightNormCheckpoint(CustomTestCase): self.assertIn("0.parametrizations.weight.original0", module.state_dict()) + def test_lazy_dc_ae_keeps_the_checkpoint_after_state_dict_builds_it(self): + """SANA's DC-AE must load its checkpoint even though the loader reads + ``state_dict()`` (which builds the lazy inner model) first.""" + config = SanaVAEConfig() + for key, value in dict( + latent_channels=4, + attention_head_dim=4, + encoder_block_types=("ResBlock", "EfficientViTBlock"), + decoder_block_types=("ResBlock", "EfficientViTBlock"), + encoder_block_out_channels=(8, 16), + decoder_block_out_channels=(8, 16), + encoder_layers_per_block=(1, 1), + decoder_layers_per_block=(1, 1), + encoder_qkv_multiscales=((), (5,)), + decoder_qkv_multiscales=((), (5,)), + ).items(): + setattr(config.arch_config, key, value) + vae = AutoencoderDC(config) + checkpoint = {k: torch.randn_like(v) for k, v in vae.state_dict().items()} + + load_model_state_dict(vae, dict(checkpoint), strict=False) + + state = vae.state_dict() + self.assertTrue(all(torch.equal(state[k], v) for k, v in checkpoint.items())) + def test_moves_checkpoint_latent_stats_into_arch_config(self): config = MiniMaxH3AudioVAEConfig() loaded = { diff --git a/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_config.py b/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_config.py index 8cb3af6d0b..906969871c 100644 --- a/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_config.py +++ b/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_config.py @@ -347,20 +347,6 @@ def test_block_refuses_a_checkpoint_with_offline_qk_rotation(monkeypatch): # WanAnimate2ArchConfig / WanAnimate2Config -def test_in_out_channels_match_in_context_io(): - # in_channels = 16 noise + 20 conditioning; out_channels = 16 latent. - arch = WanAnimate2ArchConfig() - assert arch.in_channels == 36 - assert arch.out_channels == 16 - - -def test_backbone_dims_reuse_wan22_i2v_14b(): - arch = WanAnimate2ArchConfig() - assert arch.num_layers == 40 - assert arch.image_dim == 1280 - assert arch.added_kv_proj_dim == 5120 - - def test_post_init_derives_hidden_size_and_latent_channels(): arch = WanAnimate2ArchConfig() assert arch.num_attention_heads == 40 @@ -492,13 +478,6 @@ def test_official_checkpoint_config_names_update_the_arch_config(): assert config.text_len == 512 -def test_text_len_defaults_to_the_checkpoint_value(): - """The pipeline pads prompts to ``arch_config.text_len``: the checkpoint's - transformer/config.json sets it and the default covers a config without the key.""" - assert WanAnimate2ArchConfig().text_len == 512 - assert Wan_Animate_2_14B_Config().dit_config.arch_config.text_len == 512 - - def test_checkpoint_config_without_image_embedding_is_rejected(): # The model always builds the image-embedding cross-attention; a checkpoint that # declares use_img_emb false would load and run the wrong architecture silently. @@ -1336,13 +1315,6 @@ _FLOW_MATCHING_SCHEDULER_KWARGS = { } -def test_scheduler_is_a_loaded_component(): - assert "scheduler" in WanAnimate2Pipeline._required_config_modules - assert len(WanAnimate2Pipeline._required_config_modules) == 7 - # No pipeline-side construction: the base class hook is inherited unchanged. - assert "initialize_pipeline" not in vars(WanAnimate2Pipeline) - - def test_loader_keeps_the_checkpoint_scheduler_settings(): kwargs = _supported_init_kwargs( DPMSolverMultistepScheduler, dict(_CHECKPOINT_SCHEDULER_CONFIG) diff --git a/python/sglang/multimodal_gen/test/unit/test_wan_gelu_mlp.py b/python/sglang/multimodal_gen/test/unit/test_wan_gelu_mlp.py index c6bc413251..22d330dc9b 100644 --- a/python/sglang/multimodal_gen/test/unit/test_wan_gelu_mlp.py +++ b/python/sglang/multimodal_gen/test/unit/test_wan_gelu_mlp.py @@ -2,12 +2,8 @@ import pytest import torch from sglang.kernels.ops.diffusion import ( - fused_gelu_active, - mount_fused_linear_gelu, - mount_nvfp4_bias_gelu, - nvfp4_bias_gelu_active, - unmount_fused_linear_gelu, - unmount_nvfp4_bias_gelu, + FUSED_LINEAR_GELU, + NVFP4_BIAS_GELU, ) from sglang.multimodal_gen.runtime.distributed.parallel_state import ( maybe_init_distributed_environment_and_model_parallel, @@ -48,13 +44,13 @@ def test_wan_gelu_mlp_quality_path_and_lossless_restore(): x = torch.randn(2, 129, 64, device="cuda", dtype=torch.bfloat16) reference = mlp(x) - assert not fused_gelu_active(mlp) + assert not FUSED_LINEAR_GELU.is_enabled(mlp) - assert mount_fused_linear_gelu(mlp) + assert FUSED_LINEAR_GELU.mount(mlp) torch.testing.assert_close(mlp(x), reference, atol=2e-2, rtol=2e-2) - unmount_fused_linear_gelu(mlp) - assert not fused_gelu_active(mlp) + FUSED_LINEAR_GELU.unmount(mlp) + assert not FUSED_LINEAR_GELU.is_enabled(mlp) assert torch.equal(mlp(x), reference) @@ -71,13 +67,13 @@ def test_wan_nvfp4_mlp_defers_bias_for_gelu_fusion(): assert mlp.fuse_bias_gelu_tanh assert not mlp.fc_in.skip_bias_add assert not mlp.fc_out.skip_bias_add - assert not nvfp4_bias_gelu_active(mlp) + assert not NVFP4_BIAS_GELU.is_enabled(mlp) - assert mount_nvfp4_bias_gelu(mlp) - assert nvfp4_bias_gelu_active(mlp) + assert NVFP4_BIAS_GELU.mount(mlp) + assert NVFP4_BIAS_GELU.is_enabled(mlp) assert mlp.fc_in.skip_bias_add - unmount_nvfp4_bias_gelu(mlp) - assert not nvfp4_bias_gelu_active(mlp) + NVFP4_BIAS_GELU.unmount(mlp) + assert not NVFP4_BIAS_GELU.is_enabled(mlp) assert not mlp.fc_in.skip_bias_add @@ -94,4 +90,4 @@ def test_wan_nvfp4_mlp_does_not_mark_excluded_linear(): assert not mlp.fuse_bias_gelu_tanh assert not mlp.fc_in.skip_bias_add - assert not mount_nvfp4_bias_gelu(mlp) + assert not NVFP4_BIAS_GELU.mount(mlp) diff --git a/python/sglang/multimodal_gen/test/unit/test_warmup_step_floor.py b/python/sglang/multimodal_gen/test/unit/test_warmup_step_floor.py index 262a1a7772..c7546bf8f0 100644 --- a/python/sglang/multimodal_gen/test/unit/test_warmup_step_floor.py +++ b/python/sglang/multimodal_gen/test/unit/test_warmup_step_floor.py @@ -28,14 +28,12 @@ def _req(sampling_params: SamplingParams, steps: int) -> Req: def test_generic_warmup_keeps_the_requested_steps(): warmup = _req(SamplingParams(prompt="p"), 50).copy_as_warmup(1) assert warmup.num_inference_steps == 1 - assert warmup.extra["warmup_target_num_inference_steps"] == 50 def test_h3_warmup_keeps_one_denoise_step(): assert MiniMaxH3SamplingParams.min_num_inference_steps == 1 warmup = _req(MiniMaxH3SamplingParams(prompt="p"), 50).copy_as_warmup(1) assert warmup.num_inference_steps == 1 - assert warmup.extra["warmup_target_num_inference_steps"] == 50 def test_explicit_steps_above_the_floor_are_kept(): diff --git a/python/sglang/multimodal_gen/test/unit/test_weight_utils.py b/python/sglang/multimodal_gen/test/unit/test_weight_utils.py index bef5cf9068..3591f62848 100644 --- a/python/sglang/multimodal_gen/test/unit/test_weight_utils.py +++ b/python/sglang/multimodal_gen/test/unit/test_weight_utils.py @@ -16,7 +16,6 @@ from sglang.multimodal_gen.runtime.loader.utils import ( _list_safetensors_files, checkpoint_bytes, dit_parameter_count, - load_safetensors_state_dict, ) from sglang.multimodal_gen.runtime.loader.weight_utils import ( _disable_runai_streamer_rank_discovery_collective, @@ -44,18 +43,6 @@ class TestPrecisionVariantSelection(unittest.TestCase): ["/tmp/model.safetensors", "/tmp/other.bf16.safetensors"], ) - def test_shared_state_dict_loader_uses_canonical_file(self): - with tempfile.TemporaryDirectory() as tmpdir: - model_dir = Path(tmpdir) - canonical = model_dir / "model.safetensors" - variant = model_dir / "model.fp16.safetensors" - safetensors_save_file({"weight": torch.tensor([1.0])}, canonical) - safetensors_save_file({"weight": torch.tensor([2.0])}, variant) - - state_dict = load_safetensors_state_dict(str(model_dir)) - - self.assertTrue(torch.equal(state_dict["weight"], torch.tensor([1.0]))) - def test_index_selection_precedes_canonical_fallback(self): with tempfile.TemporaryDirectory() as tmpdir: model_dir = Path(tmpdir) diff --git a/python/sglang/multimodal_gen/tools/convert_hf_to_fp8.py b/python/sglang/multimodal_gen/tools/convert_hf_to_fp8.py deleted file mode 100644 index 50de4df3c4..0000000000 --- a/python/sglang/multimodal_gen/tools/convert_hf_to_fp8.py +++ /dev/null @@ -1,320 +0,0 @@ -# copied and adapted from Slime -""" -Convert HuggingFace safetensors model to FP8 format for efficient inference. - -Example usage: - # convert FLUX.1-dev transformer to FP8 - python -m sglang.multimodal_gen.tools.convert_hf_to_fp8 \ - --model-dir /path/to/FLUX.1-dev/transformer \ - --save-dir /path/to/FLUX.1-dev/transformer-FP8 \ - --strategy block \ - --block-size 128 128 - -Options: - --model-dir MODEL_DIR - path to the directory of the HF safetensors model (e.g., transformer subfolder) - --save-dir SAVE_DIR - path to the directory to save the converted FP8 model - --strategy {block,channel,tensor} - quantization strategy (default: block) - --block-size [BLOCK_SIZE ...] - block size for block quantization, e.g., --block-size 128 128 - --max-workers MAX_WORKERS - number of worker threads for parallel processing (default: 1) -""" - -import argparse -import gc -import json -import os -import shutil -import threading -from concurrent.futures import ThreadPoolExecutor - -import safetensors -import safetensors.torch -import torch -import torch.nn.functional as F -from tqdm import tqdm - -FP8_INFO = torch.finfo(torch.float8_e4m3fn) -FP8_MAX, FP8_MIN = FP8_INFO.max, FP8_INFO.min - - -def ceildiv(a, b): - return -(-a // b) - - -def block_fp8(weight, block_size): - - # per block quant - block_n, block_k = block_size[0], block_size[1] - - shape_0, shape_1 = weight.shape - - n_tiles = ceildiv(shape_0, block_n) - k_tiles = ceildiv(shape_1, block_k) - - q_weight = F.pad( - weight, - (0, k_tiles * block_k - shape_1, 0, n_tiles * block_n - shape_0), - mode="constant", - value=0.0, - ) - - qweight = q_weight.reshape(n_tiles, block_n, k_tiles, block_k) - block_max = torch.max(torch.abs(qweight), dim=1, keepdim=True)[0] - block_max = torch.max(block_max, dim=3, keepdim=True)[0] - - scale = block_max.to(torch.float32) / FP8_MAX - qweight = ( - (qweight / scale) - .clamp(min=FP8_MIN, max=FP8_MAX) - .reshape((n_tiles * block_n, k_tiles * block_k)) - .to(torch.float8_e4m3fn) - ) - qweight = qweight[:shape_0, :shape_1].clone().detach() - scale = scale.squeeze() - - return qweight, scale - - -def channel_fp8(weight): - channel_max = torch.max(weight.abs(), dim=-1, keepdim=True)[0] - scale = channel_max.clamp(min=1e-12).to(torch.float32) / FP8_MAX - qweight = (weight / scale).clamp(min=FP8_MIN, max=FP8_MAX) - qweight = qweight.to(torch.float8_e4m3fn) - return qweight, scale - - -def tensor_fp8(weight): - scale = weight.abs().max().clamp(min=1e-12).to(torch.float32) / FP8_MAX - qweight = (weight / scale).clamp(min=FP8_MIN, max=FP8_MAX) - qweight = qweight.to(torch.float8_e4m3fn) - scale = scale.view(1) - return qweight, scale - - -def quant_fp8(weight, strategy, block_size=None): - if strategy == "tensor": - return tensor_fp8(weight) - elif strategy == "channel": - return channel_fp8(weight) - else: - return block_fp8(weight, block_size) - - -class ConversionResult: - def __init__(self): - self.lock = threading.Lock() - self.weight_map = {} - self.param_count = 0 - self.modules_to_not_convert = [] - - def add_result(self, filename, q_weights, module_names): - with self.lock: - for k, v in q_weights.items(): - self.weight_map[k] = filename - self.param_count += v.numel() - self.modules_to_not_convert.extend(module_names) - - -def process_file( - input_path, output_path, filename, strategy, block_size, result_collector -): - if not filename.endswith(".safetensors"): - return - - print(f"Processing {filename}, memory usage: {torch.cuda.memory_allocated()}") - weights = {} - q_weights = {} - - with safetensors.safe_open( - os.path.join(input_path, filename), framework="pt", device="cuda" - ) as f: - for k in f.keys(): - weights[k] = f.get_tensor(k) - - modules_to_not_convert = [] - for key in weights.keys(): - if ( - "weight" in key - and "layernorm" not in key - and "embed" not in key - and "router" not in key - and "mlp.gate." not in key - and "norm" not in key - and "lm_head" not in key - and "eh_proj" not in key - and "net" not in key - and "txt_mod" not in key - and "img_mod" not in key - and "modulation" not in key - and "img_in" not in key - and "txt_in" not in key - and "time_in" not in key - and "vector_in" not in key - and "adaLN_modulation" not in key - and "all_final_layer" not in key - and "feed_forward" not in key - and "proj_out.weight" != key - ): - qw, s = quant_fp8(weights[key], strategy, block_size) - q_weights[key] = qw - if block_size: - scale_name = key.replace(".weight", ".weight_scale_inv") - else: - scale_name = key.replace(".weight", ".weight_scale") - q_weights[scale_name] = s - else: - modules_to_not_convert.append(key.replace(".weight", "")) - q_weights[key] = weights[key] - - safetensors.torch.save_file( - q_weights, os.path.join(output_path, filename), metadata={"format": "pt"} - ) - - result_collector.add_result(filename, q_weights, modules_to_not_convert) - - -def convert_fp8(input_path, output_path, strategy, block_size=None, max_workers=4): - input_path = os.path.abspath(input_path) - os.makedirs(output_path, exist_ok=True) - - for filename in os.listdir(input_path): - if not filename.endswith(".safetensors") and not os.path.isdir( - os.path.join(input_path, filename) - ): - shutil.copyfile( - os.path.join(input_path, filename), os.path.join(output_path, filename) - ) - - safetensors_files = [ - f for f in os.listdir(input_path) if f.endswith(".safetensors") - ] - - result_collector = ConversionResult() - - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = [] - for filename in safetensors_files: - future = executor.submit( - process_file, - input_path, - output_path, - filename, - strategy, - block_size, - result_collector, - ) - futures.append(future) - - for future in tqdm(futures, desc="Processing files"): - future.result() - - if strategy == "block" or strategy == "tensor": - quantization_config = { - "activation_scheme": "dynamic", - "fmt": "e4m3", - "quant_method": "fp8", - } - if block_size: - quantization_config["weight_block_size"] = block_size - if len(result_collector.modules_to_not_convert) > 0: - quantization_config["modules_to_not_convert"] = list( - set(result_collector.modules_to_not_convert) - ) - else: - quant_group = { - "group_0": { - "input_activations": { - "actorder": None, - "block_structure": None, - "dynamic": True, - "group_size": None, - "num_bits": 8, - "observer": None, - "observer_kwargs": {}, - "strategy": "token", - "symmetric": True, - "type": "float", - }, - "output_activations": None, - "targets": ["Linear"], - "weights": { - "actorder": None, - "block_structure": None, - "dynamic": False, - "group_size": None, - "num_bits": 8, - "observer": "minmax", - "observer_kwargs": {}, - "strategy": strategy, - "symmetric": True, - "type": "float", - }, - }, - } - quantization_config = { - "config_groups": quant_group, - "format": "float-quantized", - "ignore": list(set(result_collector.modules_to_not_convert)), - "quant_method": "compressed-tensors", - "quantization_status": "compressed", - } - - config_path = os.path.join(input_path, "config.json") - if os.path.exists(config_path): - cfg = json.load(open(config_path)) - cfg["quantization_config"] = quantization_config - json.dump(cfg, open(os.path.join(output_path, "config.json"), "w"), indent=2) - - index_dict = { - "weight_map": result_collector.weight_map, - "metadata": {"total_size": result_collector.param_count}, - } - json.dump( - index_dict, - open(os.path.join(output_path, "model.safetensors.index.json"), "w"), - indent=2, - ) - - gc.collect() - torch.cuda.empty_cache() - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--model-dir", - type=str, - help="Path to the directory of the HF safetensors model.", - ) - parser.add_argument( - "--save-dir", - type=str, - help="Path to the directory to save the converted model.", - ) - parser.add_argument( - "--strategy", type=str, default="block", choices=["block", "channel", "tensor"] - ) - parser.add_argument( - "--block-size", type=int, nargs="*", default=None, help="eg. --block-size 32 32" - ) - parser.add_argument( - "--max-workers", - type=int, - default=8, - help="Number of worker threads for parallel processing", - ) - args = parser.parse_args() - - if not os.path.exists(args.save_dir): - print(f"Creating directory {args.save_dir}") - os.makedirs(args.save_dir) - elif not os.path.isdir(args.save_dir): - raise ValueError("The save_dir should be a directory.") - - convert_fp8( - args.model_dir, args.save_dir, args.strategy, args.block_size, args.max_workers - ) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index dfdb32bc56..8b73503ecf 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -971,6 +971,8 @@ class Envs: # ROCm/AITER path. Requires GPU_MAX_HW_QUEUES>=5 to avoid HW-queue serialization. SGLANG_ROCM_USE_MULTI_STREAM = EnvBool(False) SGLANG_ROCM_SMALLM_ROUTER = EnvBool(True) + SGLANG_ROCM_SMALLM_MOE = EnvBool(True) + SGLANG_ROCM_SMALLM_FP8_PROJ = EnvBool(True) # Fold the KDA [f_a|b] tail into the wide [q,k,v,g] projection so the whole # in-proj is one GEMM. Decode is bandwidth bound there, so the 144 extra # output columns ride along nearly free. @@ -1022,6 +1024,10 @@ class Envs: # go back to the unfused chain on the verify path. SGLANG_OPT_FUSED_QK_NORM_ROPE_VERIFY = EnvBool(True) SGLANG_OPT_USE_AITER_INDEXER = EnvBool(False) + # Pooled prefill-logits budget of the HIP FP4 indexer, in MiB. + SGLANG_DSV4_FP4_LOGITS_BUDGET_MB = EnvInt(2048) + # Split-K for HCA streams in the HIP unified-KV paged decode; 0 keeps the heuristic. + SGLANG_MLA_HCA_KV_SPLITS = EnvInt(4) # =================================================================== # Apple Silicon and MLX @@ -1080,7 +1086,6 @@ class Envs: # =================================================================== SGLANG_INT4_WEIGHT = EnvBool(False) SGLANG_CPU_QUANTIZATION = EnvBool(False) - SGLANG_USE_DYNAMIC_MXFP4_LINEAR = EnvBool(False) SGLANG_FORCE_FP8_MARLIN = EnvBool(False) # Cache BF16 expansions of Hopper group32 FP8 weights for larger GEMMs. # Disable to save the additional weight memory or allow online updates. @@ -1268,7 +1273,6 @@ class Envs: SGLANG_OPT_MASK_DP_PAD_MOE = EnvBool(False) SGLANG_JIT_DEEPGEMM_PRECOMPILE = EnvBool(True) SGLANG_JIT_DEEPGEMM_FAST_WARMUP = EnvBool(False) - SGLANG_JIT_DEEPGEMM_COMPILE_WORKERS = EnvInt(4) SGLANG_IN_DEEPGEMM_PRECOMPILE_STAGE = EnvBool(False) # Resolved lazily so it tracks SGLANG_CACHE_DIR, which is defined below. SGLANG_DG_CACHE_DIR = EnvStr(lambda: _default_cache_subdir("deep_gemm")) @@ -1333,10 +1337,6 @@ class Envs: SGLANG_DEEPEP_V2_NUM_SMS = EnvInt(0) SGLANG_DEEPEP_V2_ENABLE_PREFILL_EXPAND = EnvBool(None) SGLANG_DEEPEP_LL_COMBINE_SEND_NUM_SMS = EnvInt(32) - # A5 DSV4 FP4 + DeepEP low-latency dispatch wire format. This is read only - # by the model-specific dispatcher configuration; all other paths retain - # their existing behavior. - SGLANG_NPU_DSV4_DEEPEP_LL_DISPATCH_QUANT_MODE = EnvStr("mxfp8") SGLANG_BLACKWELL_OVERLAP_SHARED_EXPERTS_OUTSIDE_SBO = EnvBool(False) SGLANG_ENABLE_QWEN_DEEPEP_SHARED_OVERLAP = EnvBool(True) # Force dynamic Waterfill with runtime EP all-reduce instead of the default diff --git a/python/sglang/srt/layers/attention/dsv4/compress_hip.py b/python/sglang/srt/layers/attention/dsv4/compress_hip.py deleted file mode 100644 index 6c43d91e2f..0000000000 --- a/python/sglang/srt/layers/attention/dsv4/compress_hip.py +++ /dev/null @@ -1,451 +0,0 @@ -from __future__ import annotations - -import os -from typing import TYPE_CHECKING, Any - -import torch -import torch.nn as nn - -from sglang.kernels.ops.attention.deepseek_v4_rope import ( - fused_norm_rope_inplace_triton, - fused_softmax_pool_triton, -) -from sglang.srt.layers.attention.dsa.dsa_indexer import rotate_activation -from sglang.srt.layers.attention.dsv4.compressor import Compressor as _CompressorBase -from sglang.srt.layers.attention.nsa.nsa_indexer import rotate_activation - -try: - from sglang.kernels.ops.attention.deepseek_v4_rope import fused_softmax_pool_triton -except ImportError: - fused_softmax_pool_triton = None -from sglang.srt.mem_cache.deepseek_v4_compress_state import ( - CompressStatePool, - KVAndScore, -) -from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool - -if TYPE_CHECKING: - from sglang.srt.layers.attention.base_attn_backend import AttentionBackend - from sglang.srt.layers.attention.deepseek_v4_backend_hip_radix import ( - DeepseekV4HipRadixBackend, - ) - from sglang.srt.model_executor.forward_batch_info import ForwardBatch - -from sglang.kernels.ops.layernorm.rms_normalize_hip import rms_normalize_triton - - -class DeepseekRefRMSNorm(nn.Module): - def __init__(self, dim: int, eps: float = 1e-6): - super().__init__() - self.dim = dim - self.eps = eps - self.weight = nn.Parameter(torch.ones(dim, dtype=torch.float32)) - - def forward(self, x: torch.Tensor): - return rms_normalize_triton(x, self.eps, self.weight) - - -class CompressorHip(_CompressorBase): - """HIP (ROCm) specific Compressor implementation.""" - - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.norm = DeepseekRefRMSNorm(self.head_dim, eps=self.norm.variance_epsilon) - self._freqs_cis_real: torch.Tensor | None = None - - def _get_states( - self, - forward_batch: ForwardBatch, - attn_backend: AttentionBackend, - ) -> KVAndScore: - token_to_kv_pool = attn_backend.token_to_kv_pool - assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) - if self.is_in_indexer: - return token_to_kv_pool.get_indexer_compress_states(self.layer_id) - else: - return token_to_kv_pool.get_attention_compress_states(self.layer_id) - - def _get_state_pool(self, attn_backend: AttentionBackend) -> CompressStatePool: - token_to_kv_pool = attn_backend.token_to_kv_pool - assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) - if self.is_in_indexer: - ret = token_to_kv_pool.get_indexer_compress_states(self.layer_id) - else: - ret = token_to_kv_pool.get_attention_compress_states(self.layer_id) - assert isinstance(ret, CompressStatePool) - return ret - - def overlap_transform(self, tensor: torch.Tensor, fill_value: Any) -> torch.Tensor: - assert tensor.dim() == 3 - assert tensor.shape[1:] == (self.ratio, 2 * self.head_dim) - - s, r, d = tensor.size(0), self.ratio, self.head_dim - new_tensor = tensor.new_full((s, 2 * r, d), fill_value) - new_tensor[:, r:] = tensor[:, :, d:] - new_tensor[1:, :r] = tensor[:-1, :, :d] - return new_tensor - - def overlap_transform_decode(self, tensor: torch.Tensor) -> torch.Tensor: - assert tensor.dim() == 3 - assert tensor.shape[1:] == (2 * self.ratio, 2 * self.head_dim) - r, d = self.ratio, self.head_dim - ret = torch.cat((tensor[:, :r, :d], tensor[:, r:, d:]), dim=1) - return ret - - @staticmethod - def compute_state_len(seq_len: int, ratio: int): - return seq_len % ratio + (ratio == 4) * ratio - - @staticmethod - def compute_state_len_indices(seq_len: int, ratio: int): - state_len = seq_len % ratio + (ratio == 4) * ratio - return torch.arange(seq_len - state_len, seq_len).clamp(min=-1) - - def print_tensor(self, y: torch.Tensor, name: str): - enable = int(os.environ.get("SGLANG_ENABLE_PRINT_TENSOR", 0)) - if enable: - print(f"[sgl] {name}: shape={y.shape}, dtype={y.dtype}, device={y.device}") - print(f"{y.flatten()[:10]}...{y.flatten()[-10:]}") - - def compress_extend_paged( - self, - kv_and_scores: KVAndScore, - forward_batch: ForwardBatch, - attn_backend: AttentionBackend, - ): - backend = attn_backend - if TYPE_CHECKING: - assert isinstance(backend, DeepseekV4HipRadixBackend) - token_to_kv_pool = backend.token_to_kv_pool - assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) - req_ring_state = self.ratio == 128 or ( - self.ratio == 4 and token_to_kv_pool._unified_kv - ) - - state_pool = self._get_state_pool(backend) - prefix_lens = forward_batch.extend_prefix_lens_cpu - extend_lens = forward_batch.extend_seq_lens_cpu - req_pool_indices = forward_batch.req_pool_indices - req_to_token = backend.req_to_token_pool.req_to_token - assert not self.forward_mode.is_target_verify() - - assert extend_lens is not None and prefix_lens is not None - device = kv_and_scores.kv.device - - assert kv_and_scores.kv.shape[-1] == self.head_dim * self.coff - compressed_kv_output = torch.full( - (kv_and_scores.kv.size(0), self.head_dim), - fill_value=10000.0, - dtype=kv_and_scores.kv.dtype, - device=device, - ) - - bs = forward_batch.batch_size - pt = 0 - for i in range(bs): - kv_and_score = kv_and_scores[pt : pt + extend_lens[i]] - pre_state_indices = self.compute_state_len_indices( - seq_len=prefix_lens[i], ratio=self.ratio - ).to(device) - if req_ring_state: - state_loc = state_pool.translate_from_req_position_to_state_loc( - req_pool_indices[i], pre_state_indices - ) - else: - raw_loc = torch.where( - pre_state_indices < 0, - -1, - req_to_token[req_pool_indices[i], pre_state_indices], - ) - swa_loc = token_to_kv_pool.translate_loc_from_full_to_swa(raw_loc) - state_loc = state_pool.translate_from_swa_loc_to_state_loc(swa_loc) - pre_kv_state = state_pool.get_state_by_state_loc(state_loc) - kv_and_score_buffer = KVAndScore.cat([pre_kv_state, kv_and_score], dim=0) - valid_kv_len = kv_and_score_buffer.kv.size(0) - - post_state_indices = self.compute_state_len_indices( - seq_len=prefix_lens[i] + extend_lens[i], ratio=self.ratio - ).to(device) - post_state_len = post_state_indices.size(0) - - assert post_state_len <= valid_kv_len - if req_ring_state: - post_state_loc = state_pool.translate_from_req_position_to_state_loc( - req_pool_indices[i], post_state_indices - ) - else: - post_raw_loc = torch.where( - post_state_indices < 0, - -1, - req_to_token[req_pool_indices[i], post_state_indices], - ) - post_swa_loc = token_to_kv_pool.translate_loc_from_full_to_swa( - post_raw_loc - ) - post_state_loc = state_pool.translate_from_swa_loc_to_state_loc( - post_swa_loc - ) - post_state_to_set = kv_and_score_buffer[valid_kv_len - post_state_len :] - state_pool.set_state_by_state_loc(post_state_loc, post_state_to_set) - - compress_len = valid_kv_len // self.ratio * self.ratio - if compress_len == 0: - pt += extend_lens[i] - continue - - kv_and_score_to_compress = kv_and_score_buffer[:compress_len].view( - compress_len // self.ratio, self.ratio, -1 - ) - kv_and_score_to_compress.score.add_(self.ape.unsqueeze(0)) - - if self.overlap: - new_kv = self.overlap_transform( - kv_and_score_to_compress.kv, fill_value=0 - ) - new_score = self.overlap_transform( - kv_and_score_to_compress.score, fill_value=float("-inf") - ) - kv_and_score_to_compress = KVAndScore.from_kv_score( - kv=new_kv, score=new_score - ) - del new_kv, new_score - kv_and_score_to_compress = kv_and_score_to_compress[1:] - - if kv_and_score_to_compress.kv.size(0) == 0: - pt += extend_lens[i] - continue - - beg_idx = prefix_lens[i] // self.ratio * self.ratio - end_idx = (prefix_lens[i] + extend_lens[i]) // self.ratio * self.ratio - - kv_compressed = fused_softmax_pool_triton( - kv_and_score_to_compress.kv_score, - kv_and_score_to_compress._item_size, - ) - - assert kv_compressed.dtype == torch.float32 - - freqs_cis = self.freqs_cis[beg_idx : end_idx : self.ratio] - assert freqs_cis.size(0) == kv_compressed.size(0), ( - f"{freqs_cis.shape=} {kv_compressed.shape=}" - ) - fused_norm_rope_inplace_triton( - kv_compressed, self.norm.weight, self.norm.eps, freqs_cis - ) - del beg_idx, end_idx - - if self.rotate: - kv_compressed = rotate_activation(kv_compressed) - - start = prefix_lens[i] - start = start + self.ratio - 1 - start % self.ratio - indices_in_seq = torch.arange( - start, - prefix_lens[i] + extend_lens[i], - self.ratio, - device=kv_and_scores.kv.device, - ) - assert indices_in_seq.size(0) == kv_compressed.size(0) - compressed_kv_output[indices_in_seq - prefix_lens[i] + pt] = kv_compressed - - pt += extend_lens[i] - - return compressed_kv_output - - def compress_decode_paged( - self, - kv_and_scores: KVAndScore, - forward_batch: ForwardBatch, - attn_backend: AttentionBackend, - ): - """Paged and cudagraph compatible version of compress_decode""" - assert self.ape_converted - state_pool = self._get_state_pool(attn_backend) - token_to_kv_pool = attn_backend.token_to_kv_pool - assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) - req_ring_state = self.ratio == 128 or ( - self.ratio == 4 and token_to_kv_pool._unified_kv - ) - req_pool_indices = forward_batch.req_pool_indices - req_to_token = attn_backend.req_to_token_pool.req_to_token - seq_lens = forward_batch.seq_lens - - if forward_batch.forward_mode.is_target_verify(): - draft_tokens = attn_backend.speculative_num_draft_tokens - offsets = torch.arange(1, draft_tokens + 1, device=seq_lens.device) - seq_lens_2d = seq_lens[:, None] + offsets[None, :] - seq_lens = seq_lens_2d.view(-1) - req_pool_indices = req_pool_indices.repeat_interleave(draft_tokens) - - if req_ring_state: - state_locs = state_pool.translate_from_req_position_to_state_loc( - req_pool_indices, seq_lens - 1 - ) - else: - raw_locs = req_to_token[req_pool_indices, seq_lens - 1] - swa_locs = token_to_kv_pool.translate_loc_from_full_to_swa(raw_locs) - state_locs = state_pool.translate_from_swa_loc_to_state_loc(swa_locs) - state_pool.set_state_by_state_loc(state_locs, kv_and_scores) - - compress_bulk_len = self.ratio * self.coff - compress_indices = seq_lens[:, None] + torch.arange( - -compress_bulk_len, 0, device=seq_lens.device - ) - compress_indices.clamp_(min=-1) - if req_ring_state: - compress_indices_state = ( - state_pool.translate_from_req_position_to_state_loc( - req_pool_indices[:, None], compress_indices - ) - ) - else: - compress_indices_raw = torch.where( - compress_indices < 0, - -1, - req_to_token[req_pool_indices[:, None], compress_indices], - ) - compress_indices_swa = token_to_kv_pool.translate_loc_from_full_to_swa( - compress_indices_raw - ) - compress_indices_state = state_pool.translate_from_swa_loc_to_state_loc( - compress_indices_swa - ) - kv_and_score_to_compress = state_pool.get_state_by_state_loc( - compress_indices_state.view(-1) - ).view(-1, self.ratio, self.coff * self.head_dim) - bs = seq_lens.size(0) - - # Unfused reference path - kv_and_score_to_compress.score.add_(self.ape.unsqueeze(0)) - - if self.overlap: - kv_and_score_to_compress = kv_and_score_to_compress.view( - bs, self.coff * self.ratio, self.coff * self.head_dim - ) - kv_and_score_to_compress = KVAndScore.from_kv_score( - kv=self.overlap_transform_decode(kv_and_score_to_compress.kv), - score=self.overlap_transform_decode(kv_and_score_to_compress.score), - ) - - kv_and_score_to_compress = kv_and_score_to_compress.view( - bs, self.ratio * self.coff, self.head_dim - ) - - kv_compressed = fused_softmax_pool_triton( - kv_and_score_to_compress.kv_score, - kv_and_score_to_compress._item_size, - ) - freqs_cis = self._init_freqs_cis_per_decode_step(forward_batch, seq_lens) - fused_norm_rope_inplace_triton( - kv_compressed, self.norm.weight, self.norm.eps, freqs_cis - ) - if self.rotate: - kv_compressed = rotate_activation(kv_compressed) - - return kv_compressed - - def compress_fused( - self, - kv_score: torch.Tensor, - forward_batch: ForwardBatch, - attn_backend: AttentionBackend, - ) -> torch.Tensor: - backend = attn_backend - if TYPE_CHECKING: - assert isinstance(backend, DeepseekV4HipRadixBackend) - kv_score_buffer = self._get_state_pool(backend) - kv_score_buffer = kv_score_buffer.kv_score_buffer.kv_score - - return backend.forward_compress( - kv_score_buffer=kv_score_buffer, - kv_score_input=kv_score, - ape=self.ape.view(-1, self.head_dim), - head_dim=self.head_dim, - norm=self.norm, - freqs_cis_cache=self.freqs_cis, - rotate=self.rotate, - compress_ratio=self.ratio, - forward_batch=forward_batch, - is_paged=True, - ) - - def _get_freqs_cis_real(self) -> torch.Tensor: - """Cache the float32 view of freqs_cis (complex64 -> real interleaved).""" - if self._freqs_cis_real is None: - if self.freqs_cis.is_complex(): - self._freqs_cis_real = ( - torch.view_as_real(self.freqs_cis).flatten(-2).contiguous() - ) - else: - self._freqs_cis_real = self.freqs_cis.contiguous() - return self._freqs_cis_real - - def compress_dispatch( - self, - kv_score: torch.Tensor, - forward_batch: ForwardBatch, - attn_backend: AttentionBackend, - ) -> torch.Tensor: - if ( - forward_batch.forward_mode.is_decode() - or forward_batch.forward_mode.is_extend_without_speculative() - ): - return self.compress_fused( - kv_score, forward_batch, attn_backend=attn_backend - ) - - self.compress_decode = self.compress_decode_paged - self.compress_extend = self.compress_extend_paged - kv_and_scores = KVAndScore(kv_score) - - if TYPE_CHECKING: - assert isinstance(kv_and_scores, KVAndScore) - - if ( - forward_batch.forward_mode.is_decode() - or forward_batch.forward_mode.is_target_verify() - ): - result = self.compress_decode( - kv_and_scores=kv_and_scores, - forward_batch=forward_batch, - attn_backend=attn_backend, - ) - elif forward_batch.forward_mode.is_extend(): - result = self.compress_extend( - kv_and_scores=kv_and_scores, - forward_batch=forward_batch, - attn_backend=attn_backend, - ) - else: - msg = f"Forward mode {forward_batch.forward_mode} not supported in Compressor." - raise NotImplementedError(msg) - - return result - - def _init_freqs_cis_per_decode_step( - self, - forward_batch: ForwardBatch, - seq_lens: torch.Tensor, - ) -> torch.Tensor: - attr = f"freqs_cis_c{self.ratio}" - cached = getattr(forward_batch, attr, None) - if cached is not None: - return cached - decoded = self.freqs_cis[(seq_lens - 1) // self.ratio * self.ratio] - setattr(forward_batch, attr, decoded) - return decoded - - def forward( - self, - x: torch.Tensor, - forward_batch: ForwardBatch, - attn_backend: AttentionBackend, - ) -> torch.Tensor: - if forward_batch.forward_mode.is_idle(): - assert x.shape[0] == 0 - return x.new_empty(0, self.head_dim) - kv_score = self.compute_kv_score(x, forward_batch) - self.forward_mode = forward_batch.forward_mode - return self.compress_dispatch( - kv_score, forward_batch, attn_backend=attn_backend - ) diff --git a/python/sglang/srt/layers/attention/dsv4/compressor.py b/python/sglang/srt/layers/attention/dsv4/compressor.py index d005a1b355..a74b6e3bf5 100644 --- a/python/sglang/srt/layers/attention/dsv4/compressor.py +++ b/python/sglang/srt/layers/attention/dsv4/compressor.py @@ -1,21 +1,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List, Literal, NamedTuple, Optional, Union +from typing import TYPE_CHECKING, Literal, Optional import torch import torch.nn as nn from sglang.kernels.fused_op import BaseFusedOp -from sglang.kernels.ops.attention.dsv4 import triton_create_paged_compress_data -from sglang.kernels.ops.attention.dsv4.compress_old import ( - CompressorDecodePlan, - CompressorPrefillPlan, - compress_forward, - compress_fused_norm_rope_inplace, -) from sglang.kernels.ops.gemm.bf16_fp32 import linear_bf16_fp32 from sglang.srt.configs.deepseek_v4 import DeepSeekV4Config -from sglang.srt.environ import envs from sglang.srt.layers.attention.dsa.utils import dsa_use_prefill_cp from sglang.srt.layers.cp.utils import cp_materialize_global_token_order from sglang.srt.layers.layernorm import RMSNorm @@ -42,295 +34,6 @@ if TYPE_CHECKING: from sglang.srt.model_executor.forward_batch_info import ForwardBatch -class FusedCompressMetadata(NamedTuple): - write_loc: torch.Tensor - extra_data: Optional[torch.Tensor] - plan: Union[CompressorDecodePlan, CompressorPrefillPlan] - - def copy_(self, other: FusedCompressMetadata) -> None: - from .metadata import maybe_copy_inplace - - self.write_loc.copy_(other.write_loc) - maybe_copy_inplace(self.extra_data, src=other.extra_data) - self.plan.copy_(other.plan) - - -class CompressorBackendMixin: - def get_paged_compress_metadata(self, compress_ratio: int) -> FusedCompressMetadata: - attr_name = f"c{compress_ratio}_compress_metadata" - metadata = getattr(self.forward_metadata, attr_name) - assert isinstance(metadata, FusedCompressMetadata) - return metadata - - def forward_compress( - self, - *, - kv_score_buffer: torch.Tensor, - kv_score_input: torch.Tensor, - ape: torch.Tensor, - head_dim: int, - norm: RMSNorm, - freqs_cis_cache: torch.Tensor, - rotate: bool, - forward_batch: ForwardBatch, - compress_ratio: int, - is_paged: bool = False, - ) -> torch.Tensor: - from sglang.srt.layers.attention.dsa.dsa_indexer import rotate_activation - - assert compress_ratio in ( - 4, - 128, - ), f"DSV4 supports CSA(4x) and HCA(128x) only, got {compress_ratio=}" - if is_paged: - metadata = self.get_paged_compress_metadata(compress_ratio) - coff = 2 if is_overlap_compress(compress_ratio) else 1 - if compress_ratio == 128 and envs.SGLANG_OPT_USE_ONLINE_COMPRESS.get(): - kv_score_buffer = kv_score_buffer.view(-1, 1, head_dim * 3) - else: - last_dim = 2 * head_dim * coff - assert kv_score_buffer.shape[-1] == last_dim - kv_score_buffer = kv_score_buffer.view(-1, compress_ratio, last_dim) - else: - plan = make_compressor_plan(compress_ratio, forward_batch) - metadata = (forward_batch.req_pool_indices.to(torch.int32), None, plan) - indices, extra_data, plan = metadata - - if _is_hip: - if not is_paged: - raise NotImplementedError("HIP fused compressor expects paged metadata") - - from sglang.kernels.ops.attention.dsv4.fused_compress_triton import ( - hip_compress_forward, - hip_compress_fused_norm_rope_hadamard_inplace, - hip_compress_fused_norm_rope_inplace, - ) - - kv_compressed = hip_compress_forward( - kv_score_buffer=kv_score_buffer, - kv_score_input=kv_score_input, - ape=ape, - indices=indices, - plan=plan, - compress_ratio=compress_ratio, - head_dim=head_dim, - extra_data=extra_data, - ) - norm_eps = ( - norm.variance_epsilon if hasattr(norm, "variance_epsilon") else norm.eps - ) - if rotate: - hip_compress_fused_norm_rope_hadamard_inplace( - kv_compressed, - norm.weight, - norm_eps, - freqs_cis_cache, - plan, - head_dim, - ) - else: - hip_compress_fused_norm_rope_inplace( - kv_compressed, - norm.weight, - norm_eps, - freqs_cis_cache, - plan, - ) - return kv_compressed - - kv_compressed = compress_forward( - kv_score_buffer=kv_score_buffer, - kv_score_input=kv_score_input, - ape=ape, - indices=indices, - plan=plan, - compress_ratio=compress_ratio, - head_dim=head_dim, - extra_data=extra_data, - ) - compress_fused_norm_rope_inplace( - kv_compressed, - norm.weight, - norm.variance_epsilon, - freqs_cis_cache, - plan, - ) - return rotate_activation(kv_compressed) if rotate else kv_compressed - - def forward_core_compressor( - self, - x: torch.Tensor, - forward_batch: ForwardBatch, - layer_id: int, - compressor: Compressor, - ) -> None: - if forward_batch.forward_mode.is_idle(): - return - token_to_kv_pool = self.token_to_kv_pool - if TYPE_CHECKING: - assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) - - new_compressed_kv = compressor(x, forward_batch, attn_backend=self) - core_metadata = self.forward_metadata.core_metadata - out_loc = ( - core_metadata.c4_out_loc - if compressor.ratio == 4 - else core_metadata.c128_out_loc - ) - if out_loc.shape[0] > new_compressed_kv.shape[0]: - out_loc = out_loc[: new_compressed_kv.shape[0]] - - token_to_kv_pool.set_extra_key_buffer_fused( - layer_id=layer_id, - loc=out_loc, - cache_k=new_compressed_kv, - ) - - def forward_indexer_compressor( - self, - x: torch.Tensor, - forward_batch: ForwardBatch, - layer_id: int, - compressor: Compressor, - ) -> None: - assert is_overlap_compress(compressor.ratio) - token_to_kv_pool = self.token_to_kv_pool - if TYPE_CHECKING: - assert isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool) - - new_compressed_kv = compressor(x, forward_batch, attn_backend=self) - out_loc = self.forward_metadata.core_metadata.c4_out_loc - if out_loc.shape[0] > new_compressed_kv.shape[0]: - out_loc = out_loc[: new_compressed_kv.shape[0]] - if self.enable_deepseek_v4_fp4_indexer: - token_to_kv_pool.set_index_k_fp4( - layer_id=layer_id, - loc=out_loc, - cache_k=new_compressed_kv, - ) - else: - token_to_kv_pool.set_index_k_fused( - layer_id=layer_id, - loc=out_loc, - cache_k=new_compressed_kv, - ) - - -def is_overlap_compress(compress_ratio: int) -> bool: - return compress_ratio == 4 - - -def make_compressor_plan( - compress_ratio: Literal[4, 128], - forward_batch: ForwardBatch, -) -> Union[CompressorDecodePlan, CompressorPrefillPlan]: - if forward_batch.forward_mode.is_decode(): - seq_lens_32 = forward_batch.seq_lens.to(torch.int32) - return CompressorDecodePlan(compress_ratio, seq_lens_32) - if forward_batch.forward_mode.is_prefill(): - assert not forward_batch.forward_mode.is_target_verify() - extend_lens_list = forward_batch.extend_seq_lens_cpu - seq_lens_cpu = forward_batch.seq_lens_cpu - assert extend_lens_list is not None and seq_lens_cpu is not None - return CompressorPrefillPlan.generate( - compress_ratio=compress_ratio, - num_q_tokens=sum(extend_lens_list), - seq_lens=seq_lens_cpu, - extend_lens=torch.tensor(extend_lens_list), - device=forward_batch.seq_lens.device, - ) - elif forward_batch.forward_mode.is_target_verify(): - raise NotImplementedError("target verify mode to be implemented") - else: - raise NotImplementedError(f"unsupported mode {forward_batch.forward_mode=}") - - -def create_paged_compressor_data( - compress_ratio: Literal[4, 128], - *, - is_prefill: bool, - token_to_kv_pool: DeepSeekV4TokenToKVPool, - req_to_token: torch.Tensor, - req_pool_indices: torch.Tensor, - seq_lens: torch.Tensor, - extend_lens: Optional[torch.Tensor] = None, - seq_lens_cpu: Optional[List[int]] = None, - extend_lens_cpu: Optional[List[int]] = None, - use_prefill_cuda_graph: bool = False, - num_q_tokens: Optional[int] = None, -) -> FusedCompressMetadata: - swa_page_size = token_to_kv_pool.swa_page_size - ring_size = token_to_kv_pool.get_ring_size(compress_ratio=compress_ratio) - use_req_ring = compress_ratio == 4 and token_to_kv_pool._unified_kv - # assert ring_size % compress_ratio == 0 - - def clip_down(positions: torch.Tensor) -> torch.Tensor: - return positions // compress_ratio * compress_ratio - - def get_raw_loc(positions: torch.Tensor) -> torch.Tensor: - positions = positions.masked_fill(positions < 0, 0) - if compress_ratio == 128 or use_req_ring: - state_loc = req_pool_indices * ring_size + positions % ring_size - else: - loc = req_to_token[req_pool_indices, positions] - swa_loc = token_to_kv_pool.translate_loc_from_full_to_swa(loc) - swa_pages = swa_loc // swa_page_size - state_loc = swa_pages * ring_size + swa_loc % ring_size - return (state_loc // compress_ratio).to(torch.int32) - - is_overlap = is_overlap_compress(compress_ratio) - - if is_prefill: - assert extend_lens is not None - write_loc, extra_data = triton_create_paged_compress_data( - compress_ratio=compress_ratio, - is_overlap=is_overlap, - swa_page_size=swa_page_size, - ring_size=ring_size, - req_pool_indices=req_pool_indices, - seq_lens=seq_lens, - extend_seq_lens=extend_lens, - req_to_token=req_to_token, - full_to_swa_index_mapping=token_to_kv_pool.full_to_swa_index_mapping, - use_req_ring=use_req_ring, - ) - - plan_kwargs: dict - if seq_lens_cpu is None: - assert num_q_tokens is not None - plan_kwargs = dict( - num_q_tokens=num_q_tokens, - seq_lens=seq_lens, - extend_lens=extend_lens, - ) - else: - assert extend_lens_cpu is not None - plan_kwargs = dict( - num_q_tokens=sum(extend_lens_cpu), - seq_lens=torch.tensor(seq_lens_cpu), - extend_lens=torch.tensor(extend_lens_cpu), - ) - plan = CompressorPrefillPlan.generate( - compress_ratio=compress_ratio, - device=seq_lens.device, - use_cuda_graph=use_prefill_cuda_graph, - **plan_kwargs, - ) - else: - write_positions = clip_down(seq_lens - 1) - write_loc = get_raw_loc(write_positions) - if is_overlap: - write_overlap_loc = get_raw_loc(write_positions - compress_ratio) - extra_data = write_overlap_loc.view(-1, 1) - elif _is_hip: - extra_data = get_raw_loc(write_positions - compress_ratio) - else: - extra_data = None - plan = CompressorDecodePlan(compress_ratio, seq_lens.to(torch.int32)) - - return FusedCompressMetadata(write_loc=write_loc, extra_data=extra_data, plan=plan) - - class Compressor(BaseFusedOp): def __init__( self, diff --git a/python/sglang/srt/layers/attention/dsv4/indexer.py b/python/sglang/srt/layers/attention/dsv4/indexer.py index 32870c1ff6..571eff6b54 100644 --- a/python/sglang/srt/layers/attention/dsv4/indexer.py +++ b/python/sglang/srt/layers/attention/dsv4/indexer.py @@ -62,7 +62,7 @@ from sglang.srt.utils import add_prefix, is_cuda, is_hip, is_xpu if TYPE_CHECKING: from sglang.srt.layers.attention.base_attn_backend import AttentionBackend - from sglang.srt.layers.attention.dsv4.compressor import ( + from sglang.srt.layers.attention.dsv4.compressor_v2 import ( CompressorBackendMixin, ) from sglang.srt.layers.quantization import QuantizationConfig diff --git a/python/sglang/srt/layers/attention/linear/lightning_backend.py b/python/sglang/srt/layers/attention/linear/lightning_backend.py index 85760ea061..306953d1d3 100644 --- a/python/sglang/srt/layers/attention/linear/lightning_backend.py +++ b/python/sglang/srt/layers/attention/linear/lightning_backend.py @@ -297,7 +297,6 @@ class LightningAttentionBackend(MambaAttnBackendBase): track_lens=track_lens, track_state_indices=track_state_indices, softmax_scale=layer.scaling, - decouple=True, ) return hidden diff --git a/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py b/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py index 1a45727db3..d4f6c86331 100644 --- a/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py +++ b/python/sglang/srt/layers/deep_gemm_wrapper/compile_utils.py @@ -53,7 +53,7 @@ os.environ["DG_JIT_CACHE_DIR"] = envs.SGLANG_DG_CACHE_DIR.get() # Refer to https://github.com/deepseek-ai/DeepGEMM/commit/d75b218b7b8f4a5dd5406ac87905039ead3ae42f # NVRTC may have performance loss with some cases. # And NVCC JIT speed is also 9x faster in the ref commit -os.environ["DG_JIT_USE_NVRTC"] = os.getenv("SGL_DG_USE_NVRTC", "0") +os.environ["DG_JIT_USE_NVRTC"] = "1" if envs.SGLANG_DG_USE_NVRTC.get() else "0" def update_deep_gemm_config(gpu_id: int): diff --git a/python/sglang/srt/layers/engram.py b/python/sglang/srt/layers/engram.py index 8d401670a7..453d63811d 100644 --- a/python/sglang/srt/layers/engram.py +++ b/python/sglang/srt/layers/engram.py @@ -998,10 +998,11 @@ def engram_gate( ) -> torch.Tensor: """x [T, hc_mult, dim]; kv [T, (hc_mult + 1) * dim] holds one key per hc copy followed by the shared value. Adds the gated value to every copy.""" + # ``kv`` is the wkv projection of the same tokens, so a shape mismatch is a + # caller bug that ``fused_engram_gate`` asserts on, not a reason to fall back. if ( _cuda_kernels(x) and x.ndim == 3 - and kv.shape == (x.shape[0], (x.shape[1] + 1) * x.shape[2]) and x.dtype == kv.dtype and x.dtype in (torch.bfloat16, torch.float32) and q_weight.dtype in (torch.bfloat16, torch.float32) diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py b/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py index 98fc1b1a1b..3244e7e089 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/fused_marlin_moe.py @@ -22,6 +22,7 @@ if _is_cuda: from sglang.kernels.ops.moe.moe_wna16_marlin import moe_wna16_marlin_gemm +# Not the JIT situ_and_mul: tanh as 2 * sigmoid(2x) - 1 is not bit-exact with it. @triton.jit def _tl_tanh(x): return 2.0 * tl.sigmoid(2.0 * x) - 1.0 diff --git a/python/sglang/srt/layers/moe/moe_runner/aiter.py b/python/sglang/srt/layers/moe/moe_runner/aiter.py index 5548c8bd2e..f432629dfe 100644 --- a/python/sglang/srt/layers/moe/moe_runner/aiter.py +++ b/python/sglang/srt/layers/moe/moe_runner/aiter.py @@ -3,15 +3,12 @@ from __future__ import annotations import functools import inspect import logging -import os from dataclasses import dataclass from enum import Enum from typing import TYPE_CHECKING, Any, Optional, Union import torch -_SMALLM_MOE_ON = os.environ.get("SGLANG_ROCM_SMALLM_MOE", "1") != "0" - from sglang.srt.environ import envs from sglang.srt.layers.moe.moe_runner.base import ( MoeQuantInfo, @@ -25,6 +22,8 @@ from sglang.srt.layers.moe.moe_runner.base import ( from sglang.srt.layers.moe.utils import MoeRunnerBackend from sglang.srt.utils import get_bool_env_var, is_gfx95_supported +_SMALLM_MOE_ON = envs.SGLANG_ROCM_SMALLM_MOE.get() + if TYPE_CHECKING: from sglang.srt.layers.moe.token_dispatcher.base import CombineInput from sglang.srt.layers.moe.token_dispatcher.deepep import ( diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index a7130ec8d5..6d3a4183e3 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -1495,7 +1495,7 @@ class DummyModelLoader(BaseModelLoader): device_config: DeviceConfig, ) -> nn.Module: - if get_bool_env_var("SGL_CPU_QUANTIZATION"): + if envs.SGLANG_CPU_QUANTIZATION.get(): return load_model_with_cpu_quantization( self, model_config=model_config, device_config=device_config ) diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index ce8f877058..9b6e0973e4 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -12,6 +12,7 @@ from transformers import PretrainedConfig from sglang.kernels.ops.attention.fla.layernorm_gated import RMSNorm as RMSNormGated from sglang.kernels.ops.attention.fla.layernorm_gated import layernorm_fn from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz +from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul @@ -1159,7 +1160,7 @@ class BailingMoELinearForCausalLM(nn.Module): if ( deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM and not deep_gemm_wrapper.DEEPGEMM_BLACKWELL - and get_bool_env_var("SGL_USE_DEEPGEMM_BMM", "false") + and envs.SGLANG_USE_DEEPGEMM_BMM.get() ): block_scale = weight_scale use_deep_gemm_bmm = True diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/triton_qk_rmsnorm.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/triton_qk_rmsnorm.py index 86dab137a1..a809bd3d88 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/triton_qk_rmsnorm.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/triton_qk_rmsnorm.py @@ -17,6 +17,7 @@ import triton import triton.language as tl +# Not layernorm.rmsnorm_fp32: 1 / sqrt(var + eps) rounds differently from rsqrt. @triton.jit def _rmsnorm_kernel( x_ptr, diff --git a/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py b/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py index 774f325b7c..3a62ecc828 100644 --- a/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py +++ b/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py @@ -61,7 +61,7 @@ from sglang.srt.models.deepseek_common.utils import ( enable_nextn_moe_bf16_cast_to_fp8, is_wint4afp8_or_wint4a16_config, ) -from sglang.srt.utils import bind_or_assign, get_bool_env_var, log_info_on_rank0 +from sglang.srt.utils import bind_or_assign, log_info_on_rank0 if _use_aiter_gfx95: from sglang.srt.layers.quantization.quark.utils import quark_post_load_weights @@ -667,7 +667,7 @@ class DeepseekV2WeightLoaderMixin: if ( deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM and not deep_gemm_wrapper.DEEPGEMM_BLACKWELL - and get_bool_env_var("SGL_USE_DEEPGEMM_BMM", "false") + and envs.SGLANG_USE_DEEPGEMM_BMM.get() ): block_scale = weight_scale use_deep_gemm_bmm = True diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index b785dd7b21..db2cf3fe12 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -48,6 +48,7 @@ from sglang.srt.configs.dots3 import Dots3Config from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) +from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo @@ -126,7 +127,6 @@ from sglang.srt.utils import ( bind_or_assign, ceil_align, ceil_div, - get_bool_env_var, is_cuda, log_info_on_rank0, make_pp_layers, @@ -1791,7 +1791,7 @@ class Dots3LanguageModelForCausalLM(nn.Module): if not ( self_attn.qk_nope_head_dim % weight_block_size[0] == 0 and self_attn.v_head_dim % weight_block_size[0] == 0 - and get_bool_env_var("SGL_USE_DEEPGEMM_BMM", "false") + and envs.SGLANG_USE_DEEPGEMM_BMM.get() ): # NOTE(xiaozhi): At this point, we do not requant but change to use BF16. # The issue is that when multiplying w_kc the reduction dim can not be divided diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index 6c8a155a98..14d213dfb6 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -44,6 +44,7 @@ from sglang.kernels.ops.gemm.bf16_fp32 import ( from sglang.kernels.ops.moe.ep_moe_kernels import zero_experts_compute_triton from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz from sglang.srt.configs import LongcatFlashConfig +from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers import deep_gemm_wrapper @@ -753,7 +754,7 @@ class LongcatFlashForCausalLM(nn.Module): if ( deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM and not deep_gemm_wrapper.DEEPGEMM_BLACKWELL - and get_bool_env_var("SGL_USE_DEEPGEMM_BMM", "false") + and envs.SGLANG_USE_DEEPGEMM_BMM.get() ): block_scale = weight_scale use_deep_gemm_bmm = True diff --git a/python/sglang/srt/models/longcat_flash_nextn.py b/python/sglang/srt/models/longcat_flash_nextn.py index 4bc7ea86f9..70c4513318 100644 --- a/python/sglang/srt/models/longcat_flash_nextn.py +++ b/python/sglang/srt/models/longcat_flash_nextn.py @@ -39,6 +39,7 @@ from torch import nn from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz from sglang.srt.configs import LongcatFlashConfig +from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.dp_attention import ( @@ -360,7 +361,7 @@ class LongcatFlashForCausalLMNextN(LongcatFlashForCausalLM): if ( deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM and not deep_gemm_wrapper.DEEPGEMM_BLACKWELL - and get_bool_env_var("SGL_USE_DEEPGEMM_BMM", "false") + and envs.SGLANG_USE_DEEPGEMM_BMM.get() ): block_scale = weight_scale use_deep_gemm_bmm = True diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index dff0a590e6..2653380e9a 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -1260,9 +1260,9 @@ class MiniMaxM3Attention(nn.Module): positions, q, k, idx_q, idx_k ) and getattr(forward_batch, "out_cache_loc", None) is not None - and v.dim() == 2 + # V agreeing with K on rank and shape is the kernel's contract, which + # `sparse_qk_index_gemma_rmsnorm_rope_cache` asserts. and v.dtype == q.dtype - and v.shape == k.shape ) if can_use_cache_fusion and kv_pool is not None: layer_id = self.attn.layer_id diff --git a/python/sglang/test/kernels/dcp.py b/python/sglang/test/kernels/dcp.py new file mode 100644 index 0000000000..b9772d53db --- /dev/null +++ b/python/sglang/test/kernels/dcp.py @@ -0,0 +1,46 @@ +"""Pure-torch reference for the DCP LSE-weighted combine kernel.""" + +import torch + + +def lse_weighted_combine_cpu( + partial_outputs: torch.Tensor, + partial_lses: torch.Tensor, + is_lse_base_on_e: bool = True, +) -> torch.Tensor: + """CPU reference: combine N partial attention outputs using LSE weights. + + Args: + partial_outputs: [N, B, H_local, D] + partial_lses: [N, B, H_local] + is_lse_base_on_e: base-e (True) or base-2 (False) + + Returns: + [B, H_local, D] combined output + """ + N, B, H_local, D = partial_outputs.shape + partial_outputs = partial_outputs.float() + partial_lses = partial_lses.float() + + # Sanitize + partial_lses = torch.where( + torch.isnan(partial_lses) | torch.isinf(partial_lses), + torch.full_like(partial_lses, float("-inf")), + partial_lses, + ) + + # max LSE for numerical stability + lse_max, _ = partial_lses.max(dim=0) + lse_max = torch.where(lse_max == float("-inf"), torch.zeros_like(lse_max), lse_max) + + centered = partial_lses - lse_max.unsqueeze(0) + if is_lse_base_on_e: + weights = torch.exp(centered) + else: + weights = torch.pow(2.0, centered) + + weight_sum = weights.sum(dim=0, keepdim=True) + weights = weights / weight_sum.clamp_min(1.0) + + combined = (partial_outputs * weights.unsqueeze(-1)).sum(dim=0) + return combined diff --git a/python/sglang/test/kernels/deepseek_v4/common.py b/python/sglang/test/kernels/deepseek_v4/common.py index e16041ffc4..a890cf226f 100644 --- a/python/sglang/test/kernels/deepseek_v4/common.py +++ b/python/sglang/test/kernels/deepseek_v4/common.py @@ -8,6 +8,7 @@ import torch from sglang.kernels.ops.attention.dsv4 import ( CompressorDecodePlan, CompressorPrefillPlan, + compress_forward, ) from sglang.srt.utils import get_device @@ -197,3 +198,60 @@ def to_seq_extend( extend_lens = torch.tensor([e for _, e in seq_extend_pairs], dtype=torch.int64) num_q = int(extend_lens.sum().item()) return seq_lens, extend_lens, num_q + + +def run_compress_prefill( + ctx: LegacyContext | PagedContext, + pool: torch.Tensor, + kv_score_input: torch.Tensor, + ape: torch.Tensor, + seq_lens_cpu: torch.Tensor, + extend_lens_cpu: torch.Tensor, +) -> torch.Tensor: + num_q = int(extend_lens_cpu.sum().item()) + plan = ctx.make_prefill_plan(seq_lens_cpu, extend_lens_cpu, num_q) + return compress_forward( + pool, + kv_score_input, + ape, + plan, + head_dim=ctx.head_dim, + compress_ratio=ctx.compress_ratio, + ) + + +def run_compress_decode( + ctx: LegacyContext | PagedContext, + pool: torch.Tensor, + kv_score_input: torch.Tensor, + ape: torch.Tensor, + seq_lens_gpu: torch.Tensor, +) -> torch.Tensor: + plan = ctx.make_decode_plan(seq_lens_gpu) + return compress_forward( + pool, + kv_score_input, + ape, + plan, + head_dim=ctx.head_dim, + compress_ratio=ctx.compress_ratio, + ) + + +def ceil_ue8m0_exp_ref(x: torch.Tensor) -> torch.Tensor: + """UE8M0 exponent of ``x`` rounded up, as the FP4 indexer quantizers store it.""" + bits = x.to(torch.float32).contiguous().view(torch.int32) + exp = (bits >> 23) & 0xFF + mantissa = bits & 0x7FFFFF + exp = exp + (mantissa != 0).to(torch.int32) + return exp.clamp(1, 254) + + +def fp4_e2m1_code_ref(x: torch.Tensor) -> torch.Tensor: + """E2M1 nibble of ``x`` (sign in bit 3), saturating at 6.0.""" + ax = torch.minimum(x.abs(), torch.tensor(6.0, device=x.device)) + idx = torch.zeros_like(ax, dtype=torch.uint8) + for threshold in (0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0): + idx += (ax > threshold).to(torch.uint8) + sign = ((x < 0) & (idx != 0)).to(torch.uint8) * 8 + return idx | sign diff --git a/python/sglang/test/kernels/deepseek_v4/dsv41_kv_quant_reference.py b/python/sglang/test/kernels/deepseek_v4/dsv41_kv_quant_reference.py index be0d173d9b..92dc643cad 100644 --- a/python/sglang/test/kernels/deepseek_v4/dsv41_kv_quant_reference.py +++ b/python/sglang/test/kernels/deepseek_v4/dsv41_kv_quant_reference.py @@ -1,9 +1,11 @@ -"""Independent packed-FP4 (V41_FP4 layout) cache oracle for the V4.1 store and reader tests.""" +"""Independent V4.1 KV cache oracles (V41 fp8 and V41_FP4 layouts) for the store and reader tests.""" from typing import Optional import torch +from sglang.kernels.ops.attention.dsv4.torch_quant import ceil_pow2 + KV_DIM = 512 FP4_GROUP = 16 # values per e4m3 scale E2M1_BYTES_PER_TOKEN = KV_DIM // 2 @@ -89,3 +91,57 @@ def dequantize_k_cache_v41_fp4(pages: torch.Tensor, page_size: int) -> torch.Ten ) out = values * scale.view(torch.float8_e4m3fn).float().unsqueeze(-1) return out.view(num_pages, page_size, KV_DIM).to(torch.bfloat16) + + +# --------------------------------------------------------------------------- +# Pure-torch reference of the paged V4.1 fp8 KV cache format read by the sparse +# decode kernel (528 B/token, "V41"). +# --------------------------------------------------------------------------- + + +def ceil_pow2_scale(x: torch.Tensor) -> torch.Tensor: + """``2 ** ceil(log2(max(x, 1e-4)))`` as fp32, computed on the IEEE bits so + that it is exact at (and just above) powers of two.""" + x = x.float() + scale = ceil_pow2(torch.clamp_min(x, 1e-4)) + # ceil_pow2 works on the bits of a finite value; a NaN or inf amax passes + # through (both become the ue8m0 NaN byte, but only the NaN one turns the + # whole tile's payload into NaN). + return torch.where(torch.isfinite(x), scale, x) + + +def quantize_k_cache_v41( + k: torch.Tensor, page_bytes: Optional[int] = None +) -> torch.Tensor: + """``k`` ``[num_pages, page_size, 512]`` -> uint8 ``[num_pages, page_bytes]`` + pages of the V41 layout: 512 e4m3 per token, then 16 ue8m0 scales per token + (one per 32 values), ``scale = 2 ** ceil(log2(max(amax / 448, 1e-4)))``.""" + num_pages, page_size, d = k.shape + assert d == 512 + x = k.float().view(num_pages, page_size, 16, 32) + scale = ceil_pow2_scale(x.abs().amax(dim=-1) / 448.0) + data = (x / scale.unsqueeze(-1)).to(torch.float8_e4m3fn).view(torch.uint8) + scale_u8 = scale.to(torch.float8_e8m0fnu).view(torch.uint8) + raw = page_size * 528 + if page_bytes is None: + page_bytes = -(-raw // 512) * 512 + assert page_bytes >= raw + out = torch.zeros((num_pages, page_bytes), dtype=torch.uint8, device=k.device) + out[:, : page_size * 512] = data.reshape(num_pages, page_size * 512) + out[:, page_size * 512 : raw] = scale_u8.reshape(num_pages, page_size * 16) + return out + + +def dequantize_k_cache_v41(pages: torch.Tensor, page_size: int) -> torch.Tensor: + """Inverse of :func:`quantize_k_cache_v41`: ``[num_pages, page_size, 512]`` bf16.""" + num_pages = pages.shape[0] + pages = pages.view(torch.uint8) + data = pages[:, : page_size * 512].reshape(num_pages, page_size, 512) + scale = pages[:, page_size * 512 : page_size * 528].reshape( + num_pages, page_size, 16 + ) + values = data.view(torch.float8_e4m3fn).to(torch.bfloat16) + scale_bf16 = scale.view(torch.float8_e8m0fnu).to(torch.bfloat16) + return (values.view(num_pages, page_size, 16, 32) * scale_bf16.unsqueeze(-1)).view( + num_pages, page_size, 512 + ) diff --git a/python/sglang/test/kernels/deepseek_v4/k_cache_reference.py b/python/sglang/test/kernels/deepseek_v4/k_cache_reference.py new file mode 100644 index 0000000000..7a54a1e417 --- /dev/null +++ b/python/sglang/test/kernels/deepseek_v4/k_cache_reference.py @@ -0,0 +1,263 @@ +"""References for the DeepSeek-V4 FP8 K cache (V4 layout): quantize, store, dequantize.""" + +import torch +import triton +import triton.language as tl + +from sglang.kernels.ops.attention.dsv4.dequant_k_cache import ( + DIM_NOPE, + DIM_ROPE, + NOPE_ROPE_BYTES, + NUM_SCALE_TILES, + PADDED_SCALE_PER_TOKEN, + TILE_SIZE, +) +from sglang.kernels.ops.attention.dsv4.index_buf_accessor import NopeFp8RopeBf16Pack +from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz + +fp8_dtype = torch.float8_e4m3fnuz if is_fp8_fnuz() else torch.float8_e4m3fn + + +@triton.jit +def _quant_k_cache_fused_kernel( + k_bf16_ptr, + k_nope_fp8_ptr, + k_rope_bf16_ptr, + scale_k_nope_uint8_ptr, + k_bf16_stride_0, + k_nope_fp8_stride_0, + k_rope_bf16_stride_0, + scale_stride_0, + DIM_NOPE: tl.constexpr, + DIM_ROPE: tl.constexpr, + TILE_SIZE: tl.constexpr, + NUM_TILES: tl.constexpr, + FP8_MIN: tl.constexpr, + FP8_MAX: tl.constexpr, + EPS: tl.constexpr, +): + token_id = tl.program_id(0).to(tl.int64) + tile_id = tl.program_id(1) + + if tile_id == NUM_TILES: + rope_range = tl.arange(0, TILE_SIZE) + rope_mask = rope_range < DIM_ROPE + + in_rope_offsets = token_id * k_bf16_stride_0 + DIM_NOPE + rope_range + rope_data = tl.load(k_bf16_ptr + in_rope_offsets, mask=rope_mask, other=0.0) + + out_rope_offsets = token_id * k_rope_bf16_stride_0 + rope_range + tl.store(k_rope_bf16_ptr + out_rope_offsets, rope_data, mask=rope_mask) + else: + tile_range = tl.arange(0, TILE_SIZE) + + in_tile_offsets = token_id * k_bf16_stride_0 + tile_id * TILE_SIZE + tile_range + x_bf16 = tl.load(k_bf16_ptr + in_tile_offsets) + x_fp32 = x_bf16.to(tl.float32) + + abs_x = tl.abs(x_fp32) + max_abs = tl.max(abs_x) + max_abs_clamped = tl.maximum(max_abs, EPS) + scale = max_abs_clamped / FP8_MAX + + log2_scale = tl.log2(scale) + ceil_log2 = tl.math.ceil(log2_scale) + scale_pow2_fp32 = tl.exp2(ceil_log2) + scale_inv = 1.0 / scale_pow2_fp32 + x_scaled = x_fp32 * scale_inv + x_fp8 = tl.clamp(x_scaled, FP8_MIN, FP8_MAX).to(k_nope_fp8_ptr.dtype.element_ty) + + out_fp8_offsets = ( + token_id * k_nope_fp8_stride_0 + tile_id * TILE_SIZE + tile_range + ) + tl.store(k_nope_fp8_ptr + out_fp8_offsets, x_fp8) + + exponent = ceil_log2.to(tl.int32) + scale_uint8 = (exponent + 127).to(tl.uint8) + + out_scale_offset = token_id * scale_stride_0 + tile_id + tl.store(scale_k_nope_uint8_ptr + out_scale_offset, scale_uint8) + + +def quant_to_nope_fp8_rope_bf16_pack_triton( + k_bf16: torch.Tensor, +) -> NopeFp8RopeBf16Pack: + assert k_bf16.dtype == torch.bfloat16 + num_tokens, hidden_dim = k_bf16.shape + assert hidden_dim == 512 + dim_nope = 448 + dim_rope = 64 + tile_size = 64 + num_tiles = dim_nope // tile_size + + k_bf16 = k_bf16.contiguous() + + k_nope_fp8 = torch.empty( + (num_tokens, dim_nope), dtype=fp8_dtype, device=k_bf16.device + ) + k_rope_bf16 = torch.empty( + (num_tokens, dim_rope), dtype=torch.bfloat16, device=k_bf16.device + ) + scale_k_nope_ue8m0 = torch.empty( + (num_tokens, num_tiles), dtype=torch.uint8, device=k_bf16.device + ) + + fp8_dtype_info = torch.finfo(fp8_dtype) + + grid = (num_tokens, num_tiles + 1) + _quant_k_cache_fused_kernel[grid]( + k_bf16, + k_nope_fp8, + k_rope_bf16, + scale_k_nope_ue8m0, + k_bf16.stride(0), + k_nope_fp8.stride(0), + k_rope_bf16.stride(0), + scale_k_nope_ue8m0.stride(0), + DIM_NOPE=dim_nope, + DIM_ROPE=dim_rope, + TILE_SIZE=tile_size, + NUM_TILES=num_tiles, + FP8_MIN=fp8_dtype_info.min, + FP8_MAX=fp8_dtype_info.max, + EPS=1e-8, + ) + + return NopeFp8RopeBf16Pack( + k_nope_fp8=k_nope_fp8, + k_rope_bf16=k_rope_bf16, + scale_k_nope_ue8m0=scale_k_nope_ue8m0, + ) + + +def set_k_and_s_torch( + buf: torch.Tensor, + loc: torch.Tensor, + nope_fp8_rope_bf16_pack: NopeFp8RopeBf16Pack, + page_size: int, +): + num_pages, buf_numel_per_page = buf.shape + (num_tokens_to_write,) = loc.shape + + k_nope, k_rope, scale_k_nope = ( + nope_fp8_rope_bf16_pack.k_nope_fp8, + nope_fp8_rope_bf16_pack.k_rope_bf16, + nope_fp8_rope_bf16_pack.scale_k_nope_ue8m0, + ) + + num_tokens_to_write_nope, nope_dim = k_nope.shape + num_tokens_to_write_rope, rope_dim = k_rope.shape + num_tokens_to_write_scale, scale_dim = scale_k_nope.shape + + assert ( + num_tokens_to_write + == num_tokens_to_write_nope + == num_tokens_to_write_rope + == num_tokens_to_write_scale + ), ( + f"{num_tokens_to_write=} {num_tokens_to_write_nope=} {num_tokens_to_write_rope=} {num_tokens_to_write_scale=}" + ) + + assert buf.dtype == torch.uint8 + assert loc.dtype in [ + torch.int64, + torch.int32, + ], f"{loc.dtype=}" + + assert k_nope.dtype == fp8_dtype + assert k_rope.dtype == torch.bfloat16 + assert scale_k_nope.dtype == torch.uint8 + + assert buf.is_contiguous() + assert loc.is_contiguous() + assert k_nope.is_contiguous() + assert k_rope.is_contiguous() + assert scale_k_nope.is_contiguous() + + buf_fp8 = buf.view(fp8_dtype).flatten() + buf_bf16 = buf.view(torch.bfloat16).flatten() + buf_scale = buf.view(torch.uint8).flatten() + + loc_page_index = loc // page_size + loc_token_offset_in_page = loc % page_size + + s_offset_nbytes_in_page = page_size * (nope_dim + rope_dim * 2) + + nope_offset = loc_page_index * buf_numel_per_page + loc_token_offset_in_page * ( + nope_dim + rope_dim * 2 + ) + + rope_offset = ( + loc_page_index * buf_numel_per_page // 2 + + (loc_token_offset_in_page * (nope_dim + rope_dim * 2) + nope_dim) // 2 + ) + + s_offset = ( + loc_page_index * buf_numel_per_page + + s_offset_nbytes_in_page + + loc_token_offset_in_page * (scale_dim + 1) + ) + + for i in range(num_tokens_to_write): + buf_fp8[nope_offset[i] : nope_offset[i] + nope_dim] = k_nope[i] + buf_bf16[rope_offset[i] : rope_offset[i] + rope_dim] = k_rope[i] + buf_scale[s_offset[i] : s_offset[i] + scale_dim] = scale_k_nope[i] + + +def dequantize_k_cache_paged_ref( + quant_k_cache: torch.Tensor, + page_table_1_flattened: torch.Tensor, + page_size: int, +) -> torch.Tensor: + """Pure-torch reference for ``dsv4.dequant_k_cache.dequantize_k_cache_paged``. + + Decodes the same v4 paged layout with vectorized torch indexing instead of + a Triton kernel. + """ + assert page_table_1_flattened.dtype in (torch.int32, torch.int64) + u8 = quant_k_cache.view(torch.uint8) + bytes_per_page = u8.shape[-1] + s_offset_bytes = page_size * NOPE_ROPE_BYTES + + flat_u8 = u8.reshape(-1) + flat_fp8 = u8.view(fp8_dtype).reshape(-1) + flat_bf16 = u8.view(torch.bfloat16).reshape(-1) + + loc = page_table_1_flattened.to(torch.int64) + page_idx = loc // page_size + in_page = loc % page_size + page_byte_base = page_idx * bytes_per_page + token_data_base = page_byte_base + in_page * NOPE_ROPE_BYTES + token_scale_base = ( + page_byte_base + s_offset_bytes + in_page * PADDED_SCALE_PER_TOKEN + ) + + device = quant_k_cache.device + nope_byte = ( + token_data_base[:, None] + torch.arange(DIM_NOPE, device=device)[None, :] + ) + nope_fp8 = flat_fp8[nope_byte].to(torch.float32) + scale_byte = ( + token_scale_base[:, None] + + torch.arange(NUM_SCALE_TILES, device=device)[None, :] + ) + scale_u8 = flat_u8[scale_byte].to(torch.int32) + scale_pow2 = torch.exp2((scale_u8 - 127).to(torch.float32)) + scale_pow2 = torch.where( + scale_pow2 < (2.0**-126), torch.zeros_like(scale_pow2), scale_pow2 + ) + scale_full = scale_pow2.repeat_interleave(TILE_SIZE, dim=1) + nope = nope_fp8 * scale_full + + rope_bf16_base = (token_data_base + DIM_NOPE) // 2 + rope_idx = rope_bf16_base[:, None] + torch.arange(DIM_ROPE, device=device)[None, :] + rope = flat_bf16[rope_idx] + + out = torch.empty( + (loc.shape[0], 1, DIM_NOPE + DIM_ROPE), + dtype=torch.bfloat16, + device=device, + ) + out[:, 0, :DIM_NOPE] = nope.to(torch.bfloat16) + out[:, 0, DIM_NOPE:] = rope + return out diff --git a/python/sglang/test/kernels/dsa_transform_index.py b/python/sglang/test/kernels/dsa_transform_index.py new file mode 100644 index 0000000000..dbb9257028 --- /dev/null +++ b/python/sglang/test/kernels/dsa_transform_index.py @@ -0,0 +1,60 @@ +"""Pure-torch references for the DSA page-table index transform kernels.""" + +from typing import List, Optional + +import torch + +from sglang.kernels.ops.attention.dsa.transform_index import _allocate_prefill_result + + +def transform_index_page_table_decode_ref( + page_table: torch.Tensor, + topk_indices: torch.Tensor, + result: Optional[torch.Tensor] = None, + page_size: int = 1, +) -> torch.Tensor: + assert page_size == 1 + assert page_table.shape[0] == topk_indices.shape[0] + if result is None: + result = torch.empty_like(topk_indices, dtype=torch.int32) + assert result.shape == topk_indices.shape + torch.gather( + page_table.to(result.dtype), + dim=1, + index=topk_indices.clamp(min=0), + out=result, + ) + result[topk_indices < 0] = -1 + return result + + +def transform_index_page_table_prefill_ref( + page_table: torch.Tensor, + topk_indices: torch.Tensor, + extend_lens_cpu: List[int], + page_size: int = 1, + output_num_tokens: Optional[int] = None, + page_table_is_expanded: bool = False, +) -> torch.Tensor: + assert page_size == 1 + real_num_tokens = sum(extend_lens_cpu) + result = _allocate_prefill_result(topk_indices, real_num_tokens, output_num_tokens) + + if page_table_is_expanded: + if real_num_tokens > 0: + transform_index_page_table_decode_ref( + page_table[:real_num_tokens], + topk_indices[:real_num_tokens], + result=result[:real_num_tokens], + ) + return result + + offset = 0 + for i, l in enumerate(extend_lens_cpu): + transform_index_page_table_decode_ref( + page_table[i].unsqueeze(0).expand(l, -1), + topk_indices[offset : offset + l], + result=result[offset : offset + l], + ) + offset += l + return result diff --git a/python/sglang/test/kernels/dspark.py b/python/sglang/test/kernels/dspark.py new file mode 100644 index 0000000000..6b2d1efdd4 --- /dev/null +++ b/python/sglang/test/kernels/dspark.py @@ -0,0 +1,92 @@ +"""Torch/Triton dispatch pairs for DSpark kernels that only the parity tests call.""" + +from __future__ import annotations + +import torch + +from sglang.kernels.ops.speculative.dspark.dispatch import inputs_on_cuda +from sglang.kernels.ops.speculative.dspark.dspark_verify_window import ( + compact_row_index, + compact_row_index_triton, + compact_verify_ids, + compact_verify_ids_triton, +) +from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout + + +class CompactRowIndex: + @classmethod + def execute( + cls, *args, **kwargs + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + if inputs_on_cuda(*args, **kwargs): + return cls.triton(*args, **kwargs) + return cls.torch(*args, **kwargs) + + @classmethod + def torch( + cls, + *, + verify_lens: torch.Tensor, + padded_total: int, + device, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return compact_row_index( + verify_lens=verify_lens, + padded_total=padded_total, + device=device, + ) + + @classmethod + def triton( + cls, + *, + verify_lens: torch.Tensor, + padded_total: int, + device, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return compact_row_index_triton( + verify_lens=verify_lens, + padded_total=padded_total, + device=device, + ) + + +class CompactVerifyIds: + @classmethod + def execute(cls, *args, **kwargs) -> torch.Tensor: + if inputs_on_cuda(*args, **kwargs): + return cls.triton(*args, **kwargs) + return cls.torch(*args, **kwargs) + + @classmethod + def torch( + cls, + *, + draft_block_ids: torch.Tensor, + draft_tokens: torch.Tensor, + layout: RaggedVerifyLayout, + device: str, + ) -> torch.Tensor: + return compact_verify_ids( + draft_block_ids=draft_block_ids, + draft_tokens=draft_tokens, + layout=layout, + device=device, + ) + + @classmethod + def triton( + cls, + *, + draft_block_ids: torch.Tensor, + draft_tokens: torch.Tensor, + layout: RaggedVerifyLayout, + device: str, + ) -> torch.Tensor: + return compact_verify_ids_triton( + draft_block_ids=draft_block_ids, + draft_tokens=draft_tokens, + layout=layout, + device=device, + ) diff --git a/python/sglang/test/kernels/extend_attention.py b/python/sglang/test/kernels/extend_attention.py new file mode 100644 index 0000000000..3374e0e56b --- /dev/null +++ b/python/sglang/test/kernels/extend_attention.py @@ -0,0 +1,43 @@ +"""Reference for the Triton extend attention kernels: full prefill attention +over the whole sequence, keeping only the extend tokens.""" + +import torch + +from sglang.kernels.ops.attention.prefill_attention import context_attention_fwd + + +def redundant_attention( + q_extend, + o_extend, + k_buffer, + v_buffer, + b_req_idx, + b_start_loc, + b_seq_len, + b_seq_len_prefix, + max_len_in_batch, +): + total_token_num = k_buffer.shape[0] + B, H_Q, D = b_req_idx.shape[0], q_extend.shape[-2], q_extend.shape[-1] + q_buffer = torch.empty( + (total_token_num, H_Q, D), dtype=q_extend.dtype, device=q_extend.device + ) + + pt = 0 + for i in range(B): + cur_seq_len_extend = b_seq_len[i] - b_seq_len_prefix[i] + pl, pr = b_start_loc[i] + b_seq_len_prefix[i], b_start_loc[i] + b_seq_len[i] + q_buffer[pl:pr] = q_extend[pt : pt + cur_seq_len_extend] + pt += cur_seq_len_extend + + o_buffer = torch.empty_like(q_buffer) + context_attention_fwd( + q_buffer, k_buffer, v_buffer, o_buffer, b_start_loc, b_seq_len, max_len_in_batch + ) + + pt = 0 + for i in range(B): + cur_seq_len_extend = b_seq_len[i] - b_seq_len_prefix[i] + pl, pr = b_start_loc[i] + b_seq_len_prefix[i], b_start_loc[i] + b_seq_len[i] + o_extend[pt : pt + cur_seq_len_extend] = o_buffer[pl:pr] + pt += cur_seq_len_extend diff --git a/python/sglang/test/kernels/fp8.py b/python/sglang/test/kernels/fp8.py index 16fa06d69d..2a388e7020 100644 --- a/python/sglang/test/kernels/fp8.py +++ b/python/sglang/test/kernels/fp8.py @@ -1,4 +1,4 @@ -"""Shared input/reference construction for FP8 quantization and GEMM tests.""" +"""Shared input/reference construction for FP8 quantization tests.""" import torch @@ -12,12 +12,9 @@ class TestFP8Base(CustomTestCase): @classmethod def setUpClass(cls): cls.M = 256 - # test non-aligned - cls.N = 1024 + 64 cls.K = 512 cls.group_size = 128 cls.quant_type = torch.float8_e4m3fn - cls.output_type = torch.bfloat16 @staticmethod def _make_A(M, K, group_size, out_dtype): @@ -41,45 +38,3 @@ class TestFP8Base(CustomTestCase): A = A.reshape(M, K) quant_A = quant_A.reshape(M, K).to(out_dtype) return A, quant_A, scale - - @staticmethod - def _make_B(K, N, group_size, out_dtype): - def _aligned_size(a, b): - return (a + b - 1) // b * b - - K_aligned = _aligned_size(K, group_size) - N_aligned = _aligned_size(N, group_size) - - quant_B = torch.rand( - K_aligned // group_size, - group_size, - N_aligned // group_size, - group_size, - dtype=torch.float32, - device=device, - ) - quant_B = quant_B * 2 - 1 - - # scaling abs max to fmax - finfo = torch.finfo(out_dtype) - fmax = finfo.max - scaling = fmax / quant_B.abs().amax((1, 3), keepdim=True) - quant_B *= scaling - quant_B = quant_B.to(out_dtype).to(torch.float32) - - scale = torch.rand( - K_aligned // group_size, - 1, - N_aligned // group_size, - 1, - dtype=torch.float32, - device=device, - ) - scale /= fmax - - B = quant_B * scale - - B = B.reshape(K_aligned, N_aligned)[:K, :N] - quant_B = quant_B.reshape(K_aligned, N_aligned).to(out_dtype)[:K, :N] - scale = scale.reshape(K_aligned // group_size, N_aligned // group_size) - return B, quant_B, scale diff --git a/python/sglang/test/kernels/lplb.py b/python/sglang/test/kernels/lplb.py new file mode 100644 index 0000000000..8f64fd4793 --- /dev/null +++ b/python/sglang/test/kernels/lplb.py @@ -0,0 +1,125 @@ +"""Pure-torch references for the LPLB CUDA kernels (testing only).""" + +import torch + + +def solve_ipm_torch_reference( + A: torch.Tensor, + b: torch.Tensor, + c: torch.Tensor, + num_iters: int = 5, +) -> torch.Tensor: + """Pure-torch reference for the fused IPM kernel — testing only. + + Mirrors the barrier-method iteration in ``csrc/lplb/ipm.cuh`` + step-for-step so the two can be compared numerically: + + x <- 1 + for _ in range(num_iters): + ax2 = A * x^2 # (NC, NV) + ax2a = ax2 @ A^T # (NC, NC) KKT matrix + delta = solve(ax2a, ax2 @ c) + r = delta^T @ A # (NV,) + d = x * (c - r) + alpha = 0.999 / d_max (or 1.0 if d_max <= 1e-9) + x *= 1 - alpha * d + write 0.5 everywhere on non-convergence. + + NOT bit-equivalent to the kernel: the kernel factors the KKT system + with a hand-written block Cholesky while this uses + ``torch.linalg.solve`` (LU). The two agree to a small tolerance + (the numerical difference being the whole point of the comparison + test). This function is never on the production path — the fused + kernel is the only LP solver at runtime. + """ + nc, nv = A.shape + assert b.shape == (nc,), f"b shape mismatch: {b.shape} vs ({nc},)" + assert c.shape == (nv,), f"c shape mismatch: {c.shape} vs ({nv},)" + + x = torch.ones(nv, device=A.device, dtype=torch.float32) + d_max = torch.tensor(0.0, device=A.device, dtype=torch.float32) + for _ in range(num_iters): + ax2 = A * (x * x).unsqueeze(0) # (NC, NV) + ax2a = ax2 @ A.t() # (NC, NC) + ax2c = ax2 @ c # (NC,) + # Match the kernel's 1e-12 pivot clamp via a tiny diagonal jitter so + # a (near-)singular KKT system stays solvable instead of raising. + ax2a = ax2a + 1e-12 * torch.eye(nc, device=A.device, dtype=torch.float32) + delta = torch.linalg.solve(ax2a, ax2c) # (NC,) + r = A.t() @ delta # (NV,) + d = x * (c - r) # (NV,) + d_max = d.max() + alpha = 0.999 / d_max if d_max > 1e-9 else torch.tensor(1.0, device=A.device) + x = x * (1.0 - alpha * d) + + max_residual = (A @ x - b).abs().max() + converged = (d_max < 0.1) and (0 <= x[-1] < 1e-4) and (max_residual < 0.05) + if not converged: + return torch.full((nv,), 0.5, device=A.device, dtype=torch.float32) + return x + + +def dispatch_probability_torch_reference( + topk_ids: torch.Tensor, + log2phy_prob: torch.Tensor, + log2phy_map: torch.Tensor, + random_vals: torch.Tensor, +) -> torch.Tensor: + """Pure-torch reference of :func:`dispatch_probability`. + + Mirrors the CUDA kernel's algorithm exactly (inverse-CDF via cumsum + + threshold) so the two paths are bit-equivalent for identical + ``random_vals``, modulo floating-point rounding in the cumsum. Kept + for numerical comparison and testing — not on the production hot + path (it allocates and runs ~8 torch ops; the fused kernel collapses + them into one launch). + + Algorithm (matches ``csrc/lplb/dispatch_probability.cuh``): + + 1. Gather the per-row probability vector and physical-id map for + each logical id in ``topk_ids``. + 2. If the row sum is zero (LP gave no signal), fall back to + uniform over valid replicas (``log2phy_map != -1``). + 3. Sample: smallest ``c`` such that ``cumsum[0..c] > u * row_sum``, + where ``u = random_vals[i]``. Ties favor advancing ``c``, + matching the CUDA kernel. + 4. Return ``log2phy_map[logical_id, c]``. + + Args: + topk_ids: (num_tokens, topk) int, on CUDA or CPU. Logical expert ids. + log2phy_prob: (num_logical, max_copies) float32. LP solver output. + log2phy_map: (num_logical, max_copies) int. -1 = unused replica. + random_vals: (N,) float32, where N = ``topk_ids.numel()``. Uniform + samples in [0, 1) — same shape and semantics as the CUDA kernel. + + Returns: + Physical topk ids tensor with the same shape and dtype as ``topk_ids``. + """ + original_shape = topk_ids.shape + flat_ids = topk_ids.reshape(-1).long() + n = flat_ids.shape[0] + num_logical, max_copies = log2phy_prob.shape + assert log2phy_map.shape == (num_logical, max_copies) + assert random_vals.shape == (n,), ( + f"random_vals must be shape ({n},), got {tuple(random_vals.shape)}" + ) + + # Gather per-row probabilities and physical maps. + probs = log2phy_prob[flat_ids] # (N, max_copies), float32 + maps = log2phy_map[flat_ids] # (N, max_copies), same int dtype as input + + # Fallback when row_sum == 0: uniform over valid replicas. + row_sum = probs.sum(dim=-1, keepdim=True) # (N, 1) + fallback_probs = (maps >= 0).to(probs.dtype) # (N, max_copies) + probs = torch.where(row_sum > 0, probs, fallback_probs) + row_sum = probs.sum(dim=-1) # (N,) + + # Inverse-CDF sample: smallest c such that cumsum[..c] > u. + # ``(cum <= u).sum(dim=-1)`` counts how many slots are still below u, + # which equals the CUDA kernel's ``chosen`` after its for-loop. + u = (random_vals * row_sum).unsqueeze(-1) # (N, 1) + cum = probs.cumsum(dim=-1) # (N, max_copies) + chosen = (cum <= u).sum(dim=-1).clamp(max=max_copies - 1) # (N,) + + out = maps.gather(1, chosen.unsqueeze(-1)).squeeze(-1) + return out.view(original_shape).to(topk_ids.dtype) diff --git a/python/sglang/test/kernels/mhc.py b/python/sglang/test/kernels/mhc.py new file mode 100644 index 0000000000..45ee81d89a --- /dev/null +++ b/python/sglang/test/kernels/mhc.py @@ -0,0 +1,44 @@ +"""Pure-torch references for the mHC (hyper-connection) kernels.""" + +import torch + + +def hc_split_sinkhorn_torch( + mixes: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + hc_mult: int = 4, + sinkhorn_iters: int = 20, + eps: float = 1e-6, +): + """Pure-torch equivalent of ``mhc.hc_split_sinkhorn_kernel``. + + Layout mirrors the kernel exactly: the flattened ``mixes`` row holds ``pre`` + (hc), ``post`` (hc) and the ``comb`` matrix (hc * hc) consecutively. + """ + b, s, _ = mixes.size() + hc = hc_mult + flat = mixes.reshape(-1, (2 + hc) * hc).float() + scale = hc_scale.float() + base = hc_base.float() + + pre = torch.sigmoid(flat[:, :hc] * scale[0] + base[:hc]) + eps + post = 2 * torch.sigmoid(flat[:, hc : 2 * hc] * scale[1] + base[hc : 2 * hc]) + + comb = flat[:, 2 * hc :] * scale[2] + base[2 * hc :] + comb = comb.reshape(-1, hc, hc) + + # Initial row softmax (numerically stabilized) then column normalize. + row_max = comb.amax(dim=2, keepdim=True) + comb = torch.exp(comb - row_max) + comb = comb / comb.sum(dim=2, keepdim=True) + eps + comb = comb / (comb.sum(dim=1, keepdim=True) + eps) + + for _ in range(sinkhorn_iters - 1): + comb = comb / (comb.sum(dim=2, keepdim=True) + eps) + comb = comb / (comb.sum(dim=1, keepdim=True) + eps) + + pre = pre.reshape(b, s, hc).to(mixes.dtype) + post = post.reshape(b, s, hc).to(mixes.dtype) + comb = comb.reshape(b, s, hc, hc).to(mixes.dtype) + return pre, post, comb diff --git a/python/sglang/test/kernels/moe_align.py b/python/sglang/test/kernels/moe_align.py new file mode 100644 index 0000000000..f57cb073d5 --- /dev/null +++ b/python/sglang/test/kernels/moe_align.py @@ -0,0 +1,135 @@ +"""Triton reference for ``moe_align_block_size`` shared by the JIT and AOT tests.""" + +import torch +import triton +import triton.language as tl + + +def ceil_div(a, b): + return (a + b - 1) // b + + +@triton.jit +def moe_align_block_size_stage1( + topk_ids_ptr, + tokens_cnts_ptr, + num_experts: tl.constexpr, + numel: tl.constexpr, + tokens_per_thread: tl.constexpr, +): + pid = tl.program_id(0) + start_idx = pid * tokens_per_thread + off_c = (pid + 1) * num_experts + + for i in range(tokens_per_thread): + if start_idx + i < numel: + idx = tl.load(topk_ids_ptr + start_idx + i) + token_cnt = tl.load(tokens_cnts_ptr + off_c + idx) + tl.store(tokens_cnts_ptr + off_c + idx, token_cnt + 1) + + +@triton.jit +def moe_align_block_size_stage2( + tokens_cnts_ptr, + num_experts: tl.constexpr, +): + pid = tl.program_id(0) + last_cnt = 0 + for i in range(1, num_experts + 1): + token_cnt = tl.load(tokens_cnts_ptr + i * num_experts + pid) + last_cnt = last_cnt + token_cnt + tl.store(tokens_cnts_ptr + i * num_experts + pid, last_cnt) + + +@triton.jit +def moe_align_block_size_stage3( + total_tokens_post_pad_ptr, + tokens_cnts_ptr, + cumsum_ptr, + num_experts: tl.constexpr, + block_size: tl.constexpr, +): + last_cumsum = 0 + off_cnt = num_experts * num_experts + for i in range(1, num_experts + 1): + token_cnt = tl.load(tokens_cnts_ptr + off_cnt + i - 1) + last_cumsum = last_cumsum + tl.cdiv(token_cnt, block_size) * block_size + tl.store(cumsum_ptr + i, last_cumsum) + tl.store(total_tokens_post_pad_ptr, last_cumsum) + + +@triton.jit +def moe_align_block_size_stage4( + topk_ids_ptr, + sorted_token_ids_ptr, + expert_ids_ptr, + tokens_cnts_ptr, + cumsum_ptr, + num_experts: tl.constexpr, + block_size: tl.constexpr, + numel: tl.constexpr, + tokens_per_thread: tl.constexpr, +): + pid = tl.program_id(0) + start_idx = tl.load(cumsum_ptr + pid) + end_idx = tl.load(cumsum_ptr + pid + 1) + + for i in range(start_idx, end_idx, block_size): + tl.store(expert_ids_ptr + i // block_size, pid) + + start_idx = pid * tokens_per_thread + off_t = pid * num_experts + + for i in range(start_idx, tl.minimum(start_idx + tokens_per_thread, numel)): + expert_id = tl.load(topk_ids_ptr + i) + token_cnt = tl.load(tokens_cnts_ptr + off_t + expert_id) + rank_post_pad = token_cnt + tl.load(cumsum_ptr + expert_id) + tl.store(sorted_token_ids_ptr + rank_post_pad, i) + tl.store(tokens_cnts_ptr + off_t + expert_id, token_cnt + 1) + + +def moe_align_block_size_triton( + topk_ids: torch.Tensor, + num_experts: int, + block_size: int, + sorted_token_ids: torch.Tensor, + expert_ids: torch.Tensor, + num_tokens_post_pad: torch.Tensor, +) -> None: + numel = topk_ids.numel() + grid = (num_experts,) + tokens_cnts = torch.zeros( + (num_experts + 1, num_experts), dtype=torch.int32, device=topk_ids.device + ) + cumsum = torch.zeros((num_experts + 1,), dtype=torch.int32, device=topk_ids.device) + tokens_per_thread = ceil_div(numel, num_experts) + + moe_align_block_size_stage1[grid]( + topk_ids, + tokens_cnts, + num_experts, + numel, + tokens_per_thread, + ) + moe_align_block_size_stage2[grid]( + tokens_cnts, + num_experts, + ) + moe_align_block_size_stage3[(1,)]( + num_tokens_post_pad, + tokens_cnts, + cumsum, + num_experts, + block_size, + ) + moe_align_block_size_stage4[grid]( + topk_ids, + sorted_token_ids, + expert_ids, + tokens_cnts, + cumsum, + num_experts, + block_size, + numel, + tokens_per_thread, + ) diff --git a/python/sglang/test/kernels/multigpu.py b/python/sglang/test/kernels/multigpu.py new file mode 100644 index 0000000000..ab527efde8 --- /dev/null +++ b/python/sglang/test/kernels/multigpu.py @@ -0,0 +1,189 @@ +"""Multi-process / multi-GPU launching utilities (torchrun-based). + +Shared `multigpu_launch` helper that both `sglang.test.kernels.utils` and +`sglang.kernels.jit.benchmark.utils` build their domain-specific entry points on +top of (`multigpu_pytest_main`, `multigpu_bench_main`). + +When a script that calls one of those wrappers is run with plain `python`, the +launcher relaunches the same file under `torchrun` once for each `N` in +`num_gpus`. When the inner workers run (identified by an env_key being set), +the same launcher calls `inner()` on every rank, silences stdout on non-zero +ranks, and exits with its return code. +""" + +from __future__ import annotations + +import logging +import os +import signal +import subprocess +import sys +from typing import Callable, List, NoReturn, Optional, Sequence + +import psutil +import torch + +logger = logging.getLogger(__name__) + + +def _kill_pgroup(pgid: int) -> None: + try: + os.killpg(pgid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def _kill_descendants(pid: int) -> None: + """Snapshot every descendant of `pid` *now* and SIGKILL them all. + + Must be called BEFORE the direct child (torchrun) dies -- once it does, + its workers get reparented to init and we lose them via the process tree. + """ + try: + root = psutil.Process(pid) + except psutil.NoSuchProcess: + return + descendants = root.children(recursive=True) + for proc in descendants: + try: + proc.kill() + except psutil.Error: + # NoSuchProcess (already gone) or AccessDenied -- nothing to do. + pass + psutil.wait_procs(descendants, timeout=5) + + +def _extract_num_gpus_override( + argv: list[str], +) -> tuple[list[int] | None, list[str]]: + """Pop `--num-gpu(s)` flags out of `argv` and return them separately. + + Accepts `--num-gpu N`, `--num-gpu=N`, `--num-gpus ...`, and comma-separated + lists like `--num-gpu 2,4,8`. May be repeated. + """ + override: list[int] = [] + remaining: list[str] = [] + i = 0 + while i < len(argv): + a = argv[i] + if a in ("--num-gpu", "--num-gpus"): + if i + 1 >= len(argv): + raise ValueError(f"missing value for {a} (expected e.g. `{a} 2,4`)") + override.extend(int(x) for x in argv[i + 1].split(",")) + i += 2 + elif a.startswith("--num-gpu=") or a.startswith("--num-gpus="): + _, val = a.split("=", 1) + override.extend(int(x) for x in val.split(",")) + i += 1 + else: + remaining.append(a) + i += 1 + return (override if override else None), remaining + + +def multigpu_launch( + name: str, + file: str, + num_gpus: Sequence[int], + env_key: str, + inner: Callable[[], int], + kind: str, + pre_launch_fn: Optional[Callable[[List[int]], None]] = None, + timeout: Optional[int] = None, +) -> NoReturn | None: + """Shared torchrun-based launcher. + + See module docstring. `name` is the caller's `__name__`; `file` is its + `__file__`. `env_key` is a unique string per kind (test/benchmark) used to + detect the inside-torchrun state. `inner` returns an exit code. + + `pre_launch_fn`, if given, runs once in the outer process *before* any + torchrun child is spawned. It receives the list of world sizes that will + actually be launched (already filtered against the host's GPU count and + any ``--num-gpu`` override). Use it for parallel JIT precompilation so the + on-disk kernel cache is warm by the time the torchrun children import + their kernels. + + `timeout`, if given, bounds each per-world-size torchrun invocation (in + seconds). On expiry the child's whole process group is killed and the + launcher exits non-zero. `None` (the default) waits indefinitely. + """ + pid_key = env_key + "_PID" + if env_key in os.environ: + assert pid_key in os.environ + if name != "__main__": + return + rank = int(os.environ["LOCAL_RANK"]) + if rank != 0: + sys.stdout = open(os.devnull, "w") + torch.cuda.set_device(rank) + return sys.exit(inner()) + assert pid_key not in os.environ + if name != "__main__": + return logger.warning( + f"{file} can not directly run with `pytest`. " + "Use `python` to invoke it, which will internally relaunch it " + "under torchrun for each requested number of GPUs." + ) + num_devices = torch.cuda.device_count() + override, forwarded_args = _extract_num_gpus_override(sys.argv[1:]) + if override is not None: + logger.info(f"--num-gpu override: running only with {override}") + num_gpus = override + for N in num_gpus: + if N <= 1 or N > num_devices: + raise ValueError( + f"Invalid number of GPUs requested: {N} (available: {num_devices})" + ) + os.environ[env_key] = "1" + os.environ[pid_key] = str(os.getpid()) + os.environ.setdefault("OMP_NUM_THREADS", "1") + os.environ.setdefault("GLOO_SOCKET_IFNAME", "lo") # single-machine setup + # Unbuffered child stdout: when a worker is killed on timeout, pytest's + # block-buffered progress output is otherwise lost or flushed out of + # order into the CI log, making it impossible to tell which test hung. + os.environ.setdefault("PYTHONUNBUFFERED", "1") + signal.signal(signal.SIGINT, signal.default_int_handler) + runnable: List[int] = [] + for N in sorted(num_gpus): + assert N > 1 + if N > num_devices: + logger.warning(f"Skipping {kind} with {N} GPUs ({num_devices} available)") + continue + runnable.append(N) + if pre_launch_fn is not None and runnable: + logger.info(f"Running pre-launch hook for world sizes {runnable}") + pre_launch_fn(runnable) + for N in runnable: + logger.info(f"Running {kind} with {N} GPUs") + cmd = [ + "torchrun", + "--nproc_per_node", + str(N), + "--local-addr", + "127.0.0.1", + file, + ] + cmd += forwarded_args + proc = subprocess.Popen(cmd, start_new_session=True) + pgid = proc.pid + returncode = -1 + timed_out = False + try: + returncode = proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + finally: + _kill_descendants(os.getpid()) + _kill_pgroup(pgid) + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + if timed_out: + logger.error(f"{kind} (nproc={N}) timed out after {timeout} seconds") + sys.exit(1) + if returncode != 0: + logger.error(f"{kind} failed with {N} GPUs (exit {returncode})") + sys.exit(returncode) + logger.info(f"All {kind}s passed") diff --git a/python/sglang/test/kernels/per_token_group_quant.py b/python/sglang/test/kernels/per_token_group_quant.py new file mode 100644 index 0000000000..305c09e36d --- /dev/null +++ b/python/sglang/test/kernels/per_token_group_quant.py @@ -0,0 +1,146 @@ +"""Triton-reference vs production per-token-group 8-bit quantization entry points. + +Shared by the per-token-group quant test and benchmark: ``triton_*`` is the +Triton reference (``_per_token_group_quant_8bit_raw`` or the fused SiLU-and-mul +post-quant kernel), ``sglang_*`` the production int8 / fp8 path. +""" + +from typing import Optional, Tuple + +import torch + +from sglang.kernels.ops.quantization.fp8_kernel import ( + _per_token_group_quant_8bit_raw, + sglang_per_token_group_quant_fp8, +) +from sglang.kernels.ops.quantization.int8_kernel import ( + sglang_per_token_group_quant_int8, +) + + +def _triton_per_token_group_quant_8bit_fuse_silu_and_mul( + x: torch.Tensor, + group_size: int, + dst_dtype: torch.dtype, + column_major_scales: bool, + scale_tma_aligned: bool, + scale_ue8m0: bool, + masked_m: Optional[torch.Tensor], +) -> Tuple[torch.Tensor, torch.Tensor]: + from deep_gemm import transform_sf_into_required_layout + + from sglang.kernels.ops.moe.ep_moe_kernels import silu_and_mul_masked_post_quant_fwd + + assert column_major_scales + assert scale_tma_aligned + assert scale_ue8m0 + + needs_unsqueeze = x.dim() == 2 + if needs_unsqueeze: + num_tokens, _ = x.shape + x = x.unsqueeze(0) + assert masked_m is None + masked_m = torch.tensor([num_tokens], device=x.device, dtype=torch.int32) + + # Use `zeros` for easier testing + output = torch.zeros( + (*x.shape[:-1], x.shape[-1] // 2), + device=x.device, + dtype=dst_dtype, + ) + # Use `zeros` for easier testing + output_scale_for_kernel = torch.zeros( + (*x.shape[:-1], x.shape[-1] // 2 // group_size), + device=x.device, + dtype=torch.float32, + ) + silu_and_mul_masked_post_quant_fwd( + input=x, + output=output, + output_scale=output_scale_for_kernel, + quant_group_size=group_size, + masked_m=masked_m, + scale_ue8m0=scale_ue8m0, + ) + + output_scale = transform_sf_into_required_layout( + output_scale_for_kernel, + num_groups=output.shape[0], + mn=output.shape[-2], + k=output.shape[-1], + recipe=(1, group_size, group_size), + is_sfa=True, + ) + + if needs_unsqueeze: + output = output.squeeze(0) + output_scale = output_scale.squeeze(0) + + return output, output_scale + + +def triton_per_token_group_quant_8bit( + x: torch.Tensor, + group_size: int, + dst_dtype: torch.dtype, + eps: float = 1e-10, + column_major_scales: bool = False, + scale_tma_aligned: bool = False, + scale_ue8m0: bool = False, + fuse_silu_and_mul: bool = False, + masked_m: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + if fuse_silu_and_mul: + return _triton_per_token_group_quant_8bit_fuse_silu_and_mul( + x=x, + group_size=group_size, + dst_dtype=dst_dtype, + column_major_scales=column_major_scales, + scale_tma_aligned=scale_tma_aligned, + scale_ue8m0=scale_ue8m0, + masked_m=masked_m, + ) + return _per_token_group_quant_8bit_raw( + x=x, + group_size=group_size, + eps=eps, + column_major_scales=column_major_scales, + scale_tma_aligned=scale_tma_aligned, + scale_ue8m0=scale_ue8m0, + dtype=dst_dtype, + ) + + +def sglang_per_token_group_quant_8bit( + x: torch.Tensor, + group_size: int, + dst_dtype: torch.dtype, + eps: float = 1e-10, + column_major_scales: bool = False, + scale_tma_aligned: bool = False, + scale_ue8m0: bool = False, + fuse_silu_and_mul: bool = False, + masked_m: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + if dst_dtype == torch.int8: + assert not column_major_scales + assert not scale_tma_aligned + assert not fuse_silu_and_mul + assert masked_m is None + return sglang_per_token_group_quant_int8( + x=x, + group_size=group_size, + eps=eps, + dtype=dst_dtype, + ) + + return sglang_per_token_group_quant_fp8( + x=x, + group_size=group_size, + eps=eps, + column_major_scales=column_major_scales, + scale_tma_aligned=scale_tma_aligned, + scale_ue8m0=scale_ue8m0, + fuse_silu_and_mul=fuse_silu_and_mul, + masked_m=masked_m, + ) diff --git a/python/sglang/test/kernels/qknorm.py b/python/sglang/test/kernels/qknorm.py new file mode 100644 index 0000000000..ecfafb5dc5 --- /dev/null +++ b/python/sglang/test/kernels/qknorm.py @@ -0,0 +1,19 @@ +"""PyTorch reference for the in-place Q/K RMSNorm kernels.""" + +import torch + + +@torch.compile() +def torch_impl_qknorm( + q: torch.Tensor, + k: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + eps: float = 1e-6, +) -> None: + q_mean = q.float().pow(2).mean(dim=-1, keepdim=True) + k_mean = k.float().pow(2).mean(dim=-1, keepdim=True) + q_norm = (q_mean + eps).rsqrt() + k_norm = (k_mean + eps).rsqrt() + q.copy_(q.float() * q_norm * q_weight.float()) + k.copy_(k.float() * k_norm * k_weight.float()) diff --git a/python/sglang/test/kernels/utils.py b/python/sglang/test/kernels/utils.py index 6e6a1654fa..17fba14fb1 100644 --- a/python/sglang/test/kernels/utils.py +++ b/python/sglang/test/kernels/utils.py @@ -2,8 +2,9 @@ import sys from typing import Callable, List, Optional, Sequence import pytest +import torch -from sglang.kernels.ops.communication.mp import multigpu_launch +from sglang.test.kernels.multigpu import multigpu_launch def multigpu_pytest_main( @@ -62,3 +63,23 @@ def multigpu_pytest_main( pre_launch_fn=pre_launch_fn, timeout=timeout, ) + + +def cuda_event_us(fn, warmups: int, repeats: int, rounds: int) -> float: + """Median over ``rounds`` of CUDA-event microseconds per call of ``fn``.""" + for _ in range(warmups): + fn() + torch.cuda.synchronize() + + samples = [] + for _ in range(rounds): + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(repeats): + fn() + end.record() + end.synchronize() + samples.append(start.elapsed_time(end) * 1000.0 / repeats) + samples.sort() + return samples[len(samples) // 2] diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py index c23a40c898..eef4c6c16a 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/dsv4_attention.py @@ -19,9 +19,6 @@ from typing import Any import torch from torch import nn -from sglang.kernels.ops.attention.dsv4.quant_k_cache import ( - quant_to_nope_fp8_rope_bf16_pack_triton, -) from sglang.srt.environ import envs from sglang.srt.layers.attention.attention_registry import ATTENTION_BACKENDS from sglang.srt.layers.radix_attention import RadixAttention @@ -37,6 +34,9 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMo from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.runtime_context import get_context, get_parallel from sglang.srt.speculative.spec_info import SpeculativeAlgorithm +from sglang.test.kernels.deepseek_v4.k_cache_reference import ( + quant_to_nope_fp8_rope_bf16_pack_triton, +) # DSV4 backend pre-resolves attention TP at construction; pin to single-rank. _parallel_override = get_parallel().override( @@ -346,6 +346,8 @@ class MockDSV4ModelRunner: self.canary_manager = None self.page_size = case.page_size self.model_config = model_config + # The fixture uses unquantized projections, with no FP8 wo_a epilogue. + self.model = nn.Module() self._server_args_override = get_context().override_server_args( attention_backend=case.backend, chunked_prefill_size=-1, diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/lightning_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/lightning_attention.py index d3d8206a47..ae75f2e9d8 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/lightning_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/lightning_attention.py @@ -34,7 +34,7 @@ from sglang.srt.speculative.spec_info import SpeculativeAlgorithm _parallel_override = get_parallel().override(attn_tp_size=1, attn_tp_rank=0) _parallel_override.__enter__() -# seg_la kernel constraints (see seg_la.py:683-694): +# seg_la kernel constraints (see `seg_la_fwd` in seg_la.py): # - decode (`seg_la_d_kernel`): K_SPLIT_DIM = 128, so head_dim must be >= 128 # for `k_dim_block = head_dim // K_SPLIT_DIM` to be at least 1. # - prefill with bs > 2 (`seg_la_p_kernel`): V_SPLIT_DIM = 64, so head_dim must diff --git a/scripts/ci/cuda/warmup_deep_gemm.py b/scripts/ci/cuda/warmup_deep_gemm.py index f53b6eff9a..f7dde29cad 100644 --- a/scripts/ci/cuda/warmup_deep_gemm.py +++ b/scripts/ci/cuda/warmup_deep_gemm.py @@ -75,7 +75,7 @@ CRASH_MARKERS = ( # this warms the directory the server will actually compile into; duplicating # the default here is what let the two drift apart. os.environ["DG_JIT_CACHE_DIR"] = envs.SGLANG_DG_CACHE_DIR.get() -os.environ["DG_JIT_USE_NVRTC"] = os.getenv("SGL_DG_USE_NVRTC", "0") +os.environ["DG_JIT_USE_NVRTC"] = "1" if envs.SGLANG_DG_USE_NVRTC.get() else "0" BLOCK_SIZE = 128 diff --git a/scripts/ci/partition_rerun_tests.py b/scripts/ci/partition_rerun_tests.py index 83f57ed6c8..9f164d7d34 100644 --- a/scripts/ci/partition_rerun_tests.py +++ b/scripts/ci/partition_rerun_tests.py @@ -91,7 +91,6 @@ def _multimodal_estimates(root): try: suites = parser.collect_diffusion_suites( parser.resolve_case_config_path(root, run_suite), - run_suite, root / parser.BASELINE_REL_PATH, ) except Exception: diff --git a/scripts/ci/utils/diffusion/compute_diffusion_partitions.py b/scripts/ci/utils/diffusion/compute_diffusion_partitions.py index f5f920dffa..d2ba598533 100755 --- a/scripts/ci/utils/diffusion/compute_diffusion_partitions.py +++ b/scripts/ci/utils/diffusion/compute_diffusion_partitions.py @@ -283,11 +283,7 @@ def main(): print(f"Error: {exc}") sys.exit(1) - suites = collect_diffusion_suites( - case_config_path, - run_suite_path, - baseline_path, - ) + suites = collect_diffusion_suites(case_config_path, baseline_path) validate_suite_case_coverage(suites) requested_case_ids = set(args.case_ids or []) diff --git a/scripts/ci/utils/diffusion/diffusion_case_parser.py b/scripts/ci/utils/diffusion/diffusion_case_parser.py index fbbc69d8a6..2e00788f8d 100755 --- a/scripts/ci/utils/diffusion/diffusion_case_parser.py +++ b/scripts/ci/utils/diffusion/diffusion_case_parser.py @@ -2,8 +2,8 @@ """ AST-based parser for diffusion test cases. -This module parses the diffusion case source and run_suite.py using AST to -extract test case information without requiring sglang dependencies. The case +This module parses the diffusion case source using AST to extract test case +and standalone-file information without requiring sglang dependencies. The case source file is discovered from ONE_GPU_CASES/TWO_GPU_CASES imports in run_suite.py so CI keeps a single source of truth. @@ -11,7 +11,7 @@ Usage: # From sibling scripts in this directory: from diffusion_case_parser import collect_diffusion_suites, resolve_case_config_path case_config_path = resolve_case_config_path(repo_root, run_suite_path) - suites = collect_diffusion_suites(case_config_path, run_suite_path, baseline_path) + suites = collect_diffusion_suites(case_config_path, baseline_path) """ import ast @@ -285,9 +285,9 @@ def resolve_case_config_path(repo_root: Path, run_suite_path: Path) -> Path: return case_config_path -class RunSuiteVisitor(ast.NodeVisitor): +class StandaloneFilesVisitor(ast.NodeVisitor): """ - AST visitor to extract standalone metadata from run_suite.py. + AST visitor to extract standalone metadata from the case config. Parses: STANDALONE_FILES = { @@ -412,22 +412,22 @@ def parse_testcase_configs(config_path: Path) -> Dict[str, List[str]]: return visitor.cases -def parse_run_suite_standalone_data( - run_suite_path: Path, +def parse_standalone_data( + case_config_path: Path, ) -> tuple[Dict[str, List[str]], Dict[str, Dict[str, float]]]: """ - Parse run_suite.py to extract standalone file metadata. + Parse the case config to extract standalone file metadata. Returns: Tuple of: - suite -> standalone file list - suite -> standalone file -> estimated seconds """ - with open(run_suite_path, "r", encoding="utf-8") as f: + with open(case_config_path, "r", encoding="utf-8") as f: content = f.read() - tree = ast.parse(content, filename=str(run_suite_path)) - visitor = RunSuiteVisitor() + tree = ast.parse(content, filename=str(case_config_path)) + visitor = StandaloneFilesVisitor() visitor.visit(tree) return visitor.standalone_files, visitor.standalone_est_times @@ -452,7 +452,6 @@ def validate_standalone_est_times( def collect_diffusion_suites( case_config_path: Path, - run_suite_path: Path, baseline_path: Path, ) -> Dict[str, DiffusionSuiteInfo]: """ @@ -460,7 +459,6 @@ def collect_diffusion_suites( Args: case_config_path: Path to case config (resolved from run_suite.py) - run_suite_path: Path to run_suite.py baseline_path: Path to perf_baselines/ or a single baseline JSON file Returns: @@ -469,10 +467,8 @@ def collect_diffusion_suites( # Parse case IDs from the single source case config. case_lists = parse_testcase_configs(case_config_path) - # Parse standalone files from run_suite.py - standalone_files, standalone_est_times = parse_run_suite_standalone_data( - run_suite_path - ) + # STANDALONE_FILES is defined next to the cases. + standalone_files, standalone_est_times = parse_standalone_data(case_config_path) missing_standalone_estimates = validate_standalone_est_times( standalone_files, standalone_est_times ) diff --git a/scripts/ci/utils/diffusion/verify_diffusion_coverage.py b/scripts/ci/utils/diffusion/verify_diffusion_coverage.py index 2643391fca..cd77b7522a 100755 --- a/scripts/ci/utils/diffusion/verify_diffusion_coverage.py +++ b/scripts/ci/utils/diffusion/verify_diffusion_coverage.py @@ -49,11 +49,7 @@ def get_expected_cases(repo_root: Path) -> dict[str, set[str]]: run_suite_path = repo_root / RUN_SUITE_REL_PATH case_config_path = resolve_case_config_path(repo_root, run_suite_path) - suites = collect_diffusion_suites( - case_config_path, - run_suite_path, - baseline_path, - ) + suites = collect_diffusion_suites(case_config_path, baseline_path) expected = {} for suite_name, suite_info in suites.items(): diff --git a/test/manual/kernels/attention/check_dsa_transform_index.py b/test/manual/kernels/attention/check_dsa_transform_index.py index fd15f0c8eb..63471cab69 100644 --- a/test/manual/kernels/attention/check_dsa_transform_index.py +++ b/test/manual/kernels/attention/check_dsa_transform_index.py @@ -2,6 +2,8 @@ import torch from sglang.kernels.ops.attention.dsa.transform_index import ( transform_index_page_table_decode_fast, +) +from sglang.test.kernels.dsa_transform_index import ( transform_index_page_table_decode_ref, ) diff --git a/test/manual/kernels/attention/check_dsv4_dequant_k_cache.py b/test/manual/kernels/attention/check_dsv4_dequant_k_cache.py index 23c77f2093..021b052c79 100644 --- a/test/manual/kernels/attention/check_dsv4_dequant_k_cache.py +++ b/test/manual/kernels/attention/check_dsv4_dequant_k_cache.py @@ -4,6 +4,8 @@ from sglang.kernels.ops.attention.dsv4.dequant_k_cache import ( NOPE_ROPE_BYTES, PADDED_SCALE_PER_TOKEN, dequantize_k_cache_paged, +) +from sglang.test.kernels.deepseek_v4.k_cache_reference import ( dequantize_k_cache_paged_ref, ) diff --git a/test/manual/layers/attention/dsa/test_index_buf_accessor.py b/test/manual/layers/attention/dsa/test_index_buf_accessor.py index 6c9dcb3d0c..461cb56bb2 100644 --- a/test/manual/layers/attention/dsa/test_index_buf_accessor.py +++ b/test/manual/layers/attention/dsa/test_index_buf_accessor.py @@ -1,19 +1,14 @@ """ Correctness tests for DSA Indexer K/S Buffer Access with Fused Triton Kernels. -This test verifies that the optimized Triton implementations (GetK, GetS, GetKAndS) -produce identical results to the torch_fast baseline implementations. - -Test coverage: -- GetK.triton() vs GetK.torch_fast() -- GetS.triton() vs GetS.torch_fast() -- GetKAndS.triton() vs separate GetK.torch_fast() + GetS.torch_fast() +This test verifies that GetKAndS.triton() produces identical results to a +torch gather of the K and S regions of the paged buffer. """ import pytest import torch -from sglang.kernels.ops.attention.dsa.index_buf_accessor import GetK, GetKAndS, GetS +from sglang.kernels.ops.attention.dsa.index_buf_accessor import GetKAndS class MockDSATokenToKVPool: @@ -26,7 +21,7 @@ class MockDSATokenToKVPool: quant_block_size: int = 128, device: str = "cuda", ): - self.page_size = page_size + self.index_page_size = page_size self.index_head_dim = index_head_dim self.quant_block_size = quant_block_size self.device = device @@ -61,190 +56,30 @@ def create_test_buffer( return buf -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") -class TestGetK: - """Test cases for GetK.triton() correctness.""" - - @pytest.mark.parametrize("num_pages", [1, 2, 4, 8, 16]) - @pytest.mark.parametrize("seq_len", [64, 128, 256, 512, 1024]) - @pytest.mark.parametrize("page_size", [64]) - @pytest.mark.parametrize("index_head_dim", [128]) - def test_getk_correctness(self, num_pages, seq_len, page_size, index_head_dim): - """Test GetK.triton() produces same output as GetK.torch_fast().""" - device = torch.device("cuda") - - # Ensure seq_len doesn't exceed available pages - max_seq_len = num_pages * page_size - seq_len = min(seq_len, max_seq_len) - - # Create mock pool - pool = MockDSATokenToKVPool( - page_size=page_size, index_head_dim=index_head_dim, device=device - ) - - # Create test buffer - buf = create_test_buffer( - num_pages=num_pages, - page_size=page_size, - index_head_dim=index_head_dim, - device=device, - ) - - # Create page indices - num_pages_needed = (seq_len + page_size - 1) // page_size - page_indices = torch.randint( - 0, num_pages, (num_pages_needed,), dtype=torch.int32, device=device - ) - - # Run both implementations - output_torch = GetK.torch_fast(pool, buf, seq_len, page_indices) - output_triton = GetK.triton(pool, buf, seq_len, page_indices) - - # Verify shapes - assert output_torch.shape == (seq_len, index_head_dim) - assert output_triton.shape == (seq_len, index_head_dim) - assert output_torch.dtype == torch.uint8 - assert output_triton.dtype == torch.uint8 - - # Compare results (should be exact match) - torch.testing.assert_close( - output_triton, output_torch, rtol=0, atol=0, msg="GetK outputs differ" - ) - - def test_getk_sequential_pages(self): - """Test GetK with sequential page indices.""" - device = torch.device("cuda") - page_size = 64 - index_head_dim = 128 - num_pages = 10 - seq_len = 320 # 5 pages - - pool = MockDSATokenToKVPool( - page_size=page_size, index_head_dim=index_head_dim, device=device - ) - buf = create_test_buffer(num_pages, page_size, index_head_dim, device) - - # Sequential page indices [0, 1, 2, 3, 4] - page_indices = torch.arange(5, dtype=torch.int32, device=device) - - output_torch = GetK.torch_fast(pool, buf, seq_len, page_indices) - output_triton = GetK.triton(pool, buf, seq_len, page_indices) - - torch.testing.assert_close(output_triton, output_torch, rtol=0, atol=0) - - def test_getk_repeated_pages(self): - """Test GetK with repeated page indices.""" - device = torch.device("cuda") - page_size = 64 - index_head_dim = 128 - num_pages = 5 - seq_len = 192 # 3 pages - - pool = MockDSATokenToKVPool( - page_size=page_size, index_head_dim=index_head_dim, device=device - ) - buf = create_test_buffer(num_pages, page_size, index_head_dim, device) - - # Repeated page indices [2, 2, 2] - page_indices = torch.full((3,), 2, dtype=torch.int32, device=device) - - output_torch = GetK.torch_fast(pool, buf, seq_len, page_indices) - output_triton = GetK.triton(pool, buf, seq_len, page_indices) - - torch.testing.assert_close(output_triton, output_torch, rtol=0, atol=0) +def torch_get_k(pool, buf, seq_len: int, page_indices: torch.Tensor): + """Gather (seq_len, index_head_dim) uint8 K bytes from the paged buffer.""" + num_k_bytes_per_page = pool.index_page_size * pool.index_head_dim + flat_indices = (page_indices * buf.shape[1])[:, None] + torch.arange( + num_k_bytes_per_page, dtype=torch.int32, device=buf.device + )[None, :] + flat_indices = flat_indices.flatten()[: seq_len * pool.index_head_dim] + return buf.flatten()[flat_indices].view(-1, pool.index_head_dim) -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") -class TestGetS: - """Test cases for GetS.triton() correctness.""" - - @pytest.mark.parametrize("num_pages", [1, 2, 4, 8, 16]) - @pytest.mark.parametrize("seq_len", [64, 128, 256, 512, 1024]) - @pytest.mark.parametrize("page_size", [64]) - @pytest.mark.parametrize("index_head_dim", [128]) - def test_gets_correctness(self, num_pages, seq_len, page_size, index_head_dim): - """Test GetS.triton() produces same output as GetS.torch_fast().""" - device = torch.device("cuda") - - # Ensure seq_len doesn't exceed available pages - max_seq_len = num_pages * page_size - seq_len = min(seq_len, max_seq_len) - - # Create mock pool - pool = MockDSATokenToKVPool( - page_size=page_size, index_head_dim=index_head_dim, device=device - ) - - # Create test buffer - buf = create_test_buffer( - num_pages=num_pages, - page_size=page_size, - index_head_dim=index_head_dim, - device=device, - ) - - # Create page indices - num_pages_needed = (seq_len + page_size - 1) // page_size - page_indices = torch.randint( - 0, num_pages, (num_pages_needed,), dtype=torch.int32, device=device - ) - - # Run both implementations - output_torch = GetS.torch_fast(pool, buf, seq_len, page_indices) - output_triton = GetS.triton(pool, buf, seq_len, page_indices) - - # Verify shapes - assert output_torch.shape == (seq_len, 4) - assert output_triton.shape == (seq_len, 4) - assert output_torch.dtype == torch.uint8 - assert output_triton.dtype == torch.uint8 - - # Compare results (should be exact match) - torch.testing.assert_close( - output_triton, output_torch, rtol=0, atol=0, msg="GetS outputs differ" - ) - - def test_gets_sequential_pages(self): - """Test GetS with sequential page indices.""" - device = torch.device("cuda") - page_size = 64 - index_head_dim = 128 - num_pages = 10 - seq_len = 320 # 5 pages - - pool = MockDSATokenToKVPool( - page_size=page_size, index_head_dim=index_head_dim, device=device - ) - buf = create_test_buffer(num_pages, page_size, index_head_dim, device) - - # Sequential page indices [0, 1, 2, 3, 4] - page_indices = torch.arange(5, dtype=torch.int32, device=device) - - output_torch = GetS.torch_fast(pool, buf, seq_len, page_indices) - output_triton = GetS.triton(pool, buf, seq_len, page_indices) - - torch.testing.assert_close(output_triton, output_torch, rtol=0, atol=0) - - def test_gets_repeated_pages(self): - """Test GetS with repeated page indices.""" - device = torch.device("cuda") - page_size = 64 - index_head_dim = 128 - num_pages = 5 - seq_len = 192 # 3 pages - - pool = MockDSATokenToKVPool( - page_size=page_size, index_head_dim=index_head_dim, device=device - ) - buf = create_test_buffer(num_pages, page_size, index_head_dim, device) - - # Repeated page indices [2, 2, 2] - page_indices = torch.full((3,), 2, dtype=torch.int32, device=device) - - output_torch = GetS.torch_fast(pool, buf, seq_len, page_indices) - output_triton = GetS.triton(pool, buf, seq_len, page_indices) - - torch.testing.assert_close(output_triton, output_torch, rtol=0, atol=0) +def torch_get_s(pool, buf, seq_len: int, page_indices: torch.Tensor): + """Gather (seq_len, 4) uint8 fp32-scale bytes from the paged buffer.""" + s_offset_in_page = pool.index_page_size * pool.index_head_dim + num_s_bytes_per_page = buf.shape[1] - s_offset_in_page + num_s_bytes_per_token = pool.index_head_dim // pool.quant_block_size * 4 + flat_indices = ( + (page_indices * buf.shape[1])[:, None] + + torch.arange(num_s_bytes_per_page, dtype=torch.int32, device=buf.device)[ + None, : + ] + + s_offset_in_page + ) + flat_indices = flat_indices.flatten()[: seq_len * num_s_bytes_per_token] + return buf.flatten()[flat_indices].view(-1, 4) @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") @@ -258,7 +93,7 @@ class TestGetKAndS: def test_get_k_and_s_correctness( self, num_pages, seq_len, page_size, index_head_dim ): - """Test GetKAndS.triton() produces same output as separate torch_fast calls.""" + """Test GetKAndS.triton() produces same output as the torch gathers.""" device = torch.device("cuda") # Ensure seq_len doesn't exceed available pages @@ -286,9 +121,9 @@ class TestGetKAndS: ) page_indices_ = page_indices.unsqueeze(0) - # Run baseline: separate torch_fast calls - k_torch = GetK.torch_fast(pool, buf, seq_len, page_indices) - s_torch = GetS.torch_fast(pool, buf, seq_len, page_indices) + # Run baseline: separate torch gathers + k_torch = torch_get_k(pool, buf, seq_len, page_indices) + s_torch = torch_get_s(pool, buf, seq_len, page_indices) # Run fused Triton implementation k_triton, s_triton = GetKAndS.triton( @@ -336,8 +171,8 @@ class TestGetKAndS: page_indices_ = page_indices.unsqueeze(0) # Baseline - k_torch = GetK.torch_fast(pool, buf, seq_len, page_indices) - s_torch = GetS.torch_fast(pool, buf, seq_len, page_indices) + k_torch = torch_get_k(pool, buf, seq_len, page_indices) + s_torch = torch_get_s(pool, buf, seq_len, page_indices) # Fused k_triton, s_triton = GetKAndS.triton( @@ -366,8 +201,8 @@ class TestGetKAndS: page_indices_ = page_indices.unsqueeze(0) # Baseline - k_torch = GetK.torch_fast(pool, buf, seq_len, page_indices) - s_torch = GetS.torch_fast(pool, buf, seq_len, page_indices) + k_torch = torch_get_k(pool, buf, seq_len, page_indices) + s_torch = torch_get_s(pool, buf, seq_len, page_indices) # Fused k_triton, s_triton = GetKAndS.triton( @@ -396,8 +231,8 @@ class TestGetKAndS: page_indices_ = page_indices.unsqueeze(0) # Baseline - k_torch = GetK.torch_fast(pool, buf, seq_len, page_indices) - s_torch = GetS.torch_fast(pool, buf, seq_len, page_indices) + k_torch = torch_get_k(pool, buf, seq_len, page_indices) + s_torch = torch_get_s(pool, buf, seq_len, page_indices) # Fused k_triton, s_triton = GetKAndS.triton( @@ -429,22 +264,13 @@ class TestEdgeCases: page_indices = torch.tensor([0], dtype=torch.int32, device=device) page_indices_ = page_indices.unsqueeze(0) - # Test GetK - k_torch = GetK.torch_fast(pool, buf, seq_len, page_indices) - k_triton = GetK.triton(pool, buf, seq_len, page_indices) - torch.testing.assert_close(k_triton, k_torch, rtol=0, atol=0) - - # Test GetS - s_torch = GetS.torch_fast(pool, buf, seq_len, page_indices) - s_triton = GetS.triton(pool, buf, seq_len, page_indices) - torch.testing.assert_close(s_triton, s_torch, rtol=0, atol=0) - - # Test GetKAndS - k_triton2, s_triton2 = GetKAndS.triton( + k_torch = torch_get_k(pool, buf, seq_len, page_indices) + s_torch = torch_get_s(pool, buf, seq_len, page_indices) + k_triton, s_triton = GetKAndS.triton( pool, buf, page_indices_, seq_len_tensor, seq_len, seq_len ) - torch.testing.assert_close(k_triton2, k_torch, rtol=0, atol=0) - torch.testing.assert_close(s_triton2, s_torch, rtol=0, atol=0) + torch.testing.assert_close(k_triton, k_torch, rtol=0, atol=0) + torch.testing.assert_close(s_triton, s_torch, rtol=0, atol=0) def test_exact_page_boundary(self): """Test when seq_len exactly matches page boundaries.""" @@ -462,22 +288,13 @@ class TestEdgeCases: page_indices = torch.arange(3, dtype=torch.int32, device=device) page_indices_ = page_indices.unsqueeze(0) - # Test GetK - k_torch = GetK.torch_fast(pool, buf, seq_len, page_indices) - k_triton = GetK.triton(pool, buf, seq_len, page_indices) - torch.testing.assert_close(k_triton, k_torch, rtol=0, atol=0) - - # Test GetS - s_torch = GetS.torch_fast(pool, buf, seq_len, page_indices) - s_triton = GetS.triton(pool, buf, seq_len, page_indices) - torch.testing.assert_close(s_triton, s_torch, rtol=0, atol=0) - - # Test GetKAndS - k_triton2, s_triton2 = GetKAndS.triton( + k_torch = torch_get_k(pool, buf, seq_len, page_indices) + s_torch = torch_get_s(pool, buf, seq_len, page_indices) + k_triton, s_triton = GetKAndS.triton( pool, buf, page_indices_, seq_len_tensor, seq_len, seq_len ) - torch.testing.assert_close(k_triton2, k_torch, rtol=0, atol=0) - torch.testing.assert_close(s_triton2, s_torch, rtol=0, atol=0) + torch.testing.assert_close(k_triton, k_torch, rtol=0, atol=0) + torch.testing.assert_close(s_triton, s_torch, rtol=0, atol=0) def test_large_seq_len(self): """Test with large sequence length.""" @@ -499,22 +316,13 @@ class TestEdgeCases: ) page_indices_ = page_indices.unsqueeze(0) - # Test GetK - k_torch = GetK.torch_fast(pool, buf, seq_len, page_indices) - k_triton = GetK.triton(pool, buf, seq_len, page_indices) - torch.testing.assert_close(k_triton, k_torch, rtol=0, atol=0) - - # Test GetS - s_torch = GetS.torch_fast(pool, buf, seq_len, page_indices) - s_triton = GetS.triton(pool, buf, seq_len, page_indices) - torch.testing.assert_close(s_triton, s_torch, rtol=0, atol=0) - - # Test GetKAndS - k_triton2, s_triton2 = GetKAndS.triton( + k_torch = torch_get_k(pool, buf, seq_len, page_indices) + s_torch = torch_get_s(pool, buf, seq_len, page_indices) + k_triton, s_triton = GetKAndS.triton( pool, buf, page_indices_, seq_len_tensor, seq_len, seq_len ) - torch.testing.assert_close(k_triton2, k_torch, rtol=0, atol=0) - torch.testing.assert_close(s_triton2, s_torch, rtol=0, atol=0) + torch.testing.assert_close(k_triton, k_torch, rtol=0, atol=0) + torch.testing.assert_close(s_triton, s_torch, rtol=0, atol=0) def print_test_summary(): @@ -522,10 +330,7 @@ def print_test_summary(): print("\n" + "=" * 80) print("DSA Indexer K/S Buffer Accessor Correctness Tests") print("=" * 80) - print("Testing Triton implementations against torch_fast baseline:") - print(" - GetK.triton() vs GetK.torch_fast()") - print(" - GetS.triton() vs GetS.torch_fast()") - print(" - GetKAndS.triton() vs separate GetK/GetS torch_fast() calls") + print("Testing GetKAndS.triton() against separate torch K/S gathers") print("=" * 80) print() @@ -541,24 +346,6 @@ if __name__ == "__main__": # Run a few sample tests print("Running sample correctness tests...\n") - # Test GetK - print("Testing GetK...") - test_getk = TestGetK() - test_getk.test_getk_correctness( - num_pages=4, seq_len=256, page_size=64, index_head_dim=128 - ) - test_getk.test_getk_sequential_pages() - print("✓ GetK tests passed\n") - - # Test GetS - print("Testing GetS...") - test_gets = TestGetS() - test_gets.test_gets_correctness( - num_pages=4, seq_len=256, page_size=64, index_head_dim=128 - ) - test_gets.test_gets_sequential_pages() - print("✓ GetS tests passed\n") - # Test GetKAndS print("Testing GetKAndS SeqLen=256...") test_get_k_and_s = TestGetKAndS() diff --git a/test/manual/quant/test_block_fp8.py b/test/manual/quant/test_block_fp8.py index d96bb782e7..ba76b24fb7 100644 --- a/test/manual/quant/test_block_fp8.py +++ b/test/manual/quant/test_block_fp8.py @@ -4,13 +4,17 @@ from functools import lru_cache import torch -from sglang.kernels.ops.gemm.fp8_kernel import w8a8_block_fp8_matmul +from sglang.kernels.ops.gemm.fp8_kernel import ( + w8a8_block_fp8_matmul_deepgemm, + w8a8_block_fp8_matmul_triton, +) from sglang.kernels.ops.quantization.fp8_kernel import ( per_tensor_quant_mla_fp8, per_token_group_quant_fp8, per_token_group_quant_mla_deep_gemm_masked_fp8, static_quant_fp8, ) +from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import fused_moe from sglang.srt.layers.moe.topk import TopKConfig, select_experts @@ -342,6 +346,16 @@ def native_w8a8_block_fp8_matmul(A, B, As, Bs, block_size, output_dtype=torch.fl return C +def w8a8_block_fp8_matmul(A, B, As, Bs, block_size, output_dtype): + if output_dtype == torch.bfloat16 and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM: + return w8a8_block_fp8_matmul_deepgemm( + A, B, As, Bs, block_size, output_dtype=output_dtype + ) + return w8a8_block_fp8_matmul_triton( + A, B, As, Bs, block_size, output_dtype=output_dtype + ) + + class TestW8A8BlockFP8Matmul(CustomTestCase): if not _is_cuda: OUT_DTYPES = [torch.float32, torch.half, torch.bfloat16] diff --git a/test/registered/attention/test_triton_attention_kernels.py b/test/registered/attention/test_triton_attention_kernels.py index 0fae705d87..06cc49d61a 100644 --- a/test/registered/attention/test_triton_attention_kernels.py +++ b/test/registered/attention/test_triton_attention_kernels.py @@ -15,7 +15,6 @@ from sglang.kernels.ops.attention.extend_attention import ( build_unified_kv_indices, extend_attention_fwd, extend_attention_fwd_unified, - redundant_attention, ) from sglang.kernels.ops.attention.prefill_attention import ( context_attention_fwd, @@ -23,6 +22,7 @@ from sglang.kernels.ops.attention.prefill_attention import ( from sglang.srt.utils import get_device from sglang.srt.utils.common import temp_set_env from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.kernels.extend_attention import redundant_attention from sglang.test.test_utils import CustomTestCase, is_in_amd_ci # Triton attention kernel unit tests (decode, extend, prefill) diff --git a/test/registered/attention/test_wave_attention_kernels.py b/test/registered/attention/test_wave_attention_kernels.py index 0b92d57e4c..f9748631dd 100644 --- a/test/registered/attention/test_wave_attention_kernels.py +++ b/test/registered/attention/test_wave_attention_kernels.py @@ -8,7 +8,6 @@ from sglang.kernels.ops.attention.decode_attention import ( ) from sglang.kernels.ops.attention.extend_attention import ( extend_attention_fwd, - redundant_attention, ) from sglang.kernels.ops.attention.prefill_attention import ( context_attention_fwd, @@ -23,6 +22,7 @@ from sglang.srt.layers.attention.wave_ops.prefill_attention import ( ) from sglang.srt.utils import get_device from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.kernels.extend_attention import redundant_attention # Wave attention kernel unit tests (AMD only - requires wave_lang) register_amd_ci( diff --git a/test/registered/attention/unittests/lightning/test_triton.py b/test/registered/attention/unittests/lightning/test_triton.py index 961014bb13..adeba89009 100644 --- a/test/registered/attention/unittests/lightning/test_triton.py +++ b/test/registered/attention/unittests/lightning/test_triton.py @@ -200,7 +200,6 @@ class TestTritonLightningBackendCorrectness(CustomTestCase): track_state_indices=torch.tensor( [track_slot], dtype=torch.int64, device=device ), - decouple=True, ) expected = initial_state.float() diff --git a/test/registered/e2e/radix_cache/test_mamba_extra_buffer_spec_grammar.py b/test/registered/e2e/radix_cache/test_mamba_extra_buffer_spec_grammar.py index 72186c7887..0fb7735377 100644 --- a/test/registered/e2e/radix_cache/test_mamba_extra_buffer_spec_grammar.py +++ b/test/registered/e2e/radix_cache/test_mamba_extra_buffer_spec_grammar.py @@ -88,7 +88,10 @@ class TestMambaExtraBufferSpecGrammar(CustomTestCase): return response.json() def _flush_cache(self): - requests.post(self.base_url + "/flush_cache", timeout=60).raise_for_status() + # A generation response can arrive before the scheduler becomes idle. + requests.post( + self.base_url + "/flush_cache", params={"timeout": 30}, timeout=60 + ).raise_for_status() def _constrained(self, input_ids): result = self._generate( diff --git a/test/registered/eplb/test_lplb_distributed.py b/test/registered/eplb/test_lplb_distributed.py index 5d9220d619..fae507c3cb 100644 --- a/test/registered/eplb/test_lplb_distributed.py +++ b/test/registered/eplb/test_lplb_distributed.py @@ -83,10 +83,8 @@ def test_dispatch_probability_matches_torch_reference(): """The fused CUDA `dispatch_probability` and the pure-torch reference must produce identical outputs for the same ``random_vals``. Single-rank, runs on any CUDA GPU.""" - from sglang.kernels.ops.lplb.cuda_solver import ( - dispatch_probability, - dispatch_probability_torch_reference, - ) + from sglang.kernels.ops.lplb.cuda_solver import dispatch_probability + from sglang.test.kernels.lplb import dispatch_probability_torch_reference torch.manual_seed(0) device = torch.device("cuda:0") @@ -137,8 +135,8 @@ def test_solve_ipm_matches_torch_reference(): just return the 0.5 non-convergence sentinel — agreeing trivially without exercising the solve. Single-rank, any CUDA GPU.""" from sglang.kernels.ops.lplb.cuda_solver import solve_ipm as cuda_solve_ipm - from sglang.kernels.ops.lplb.torch_solver import solve_ipm_torch_reference from sglang.srt.eplb.lplb_solver import LPLBSolver + from sglang.test.kernels.lplb import solve_ipm_torch_reference torch.manual_seed(0) device = torch.device("cuda:0") diff --git a/test/registered/kernels/benchmark/communication/bench_custom_all_reduce.py b/test/registered/kernels/benchmark/communication/bench_custom_all_reduce.py index 9e8bcb7c46..0bdd388f25 100644 --- a/test/registered/kernels/benchmark/communication/bench_custom_all_reduce.py +++ b/test/registered/kernels/benchmark/communication/bench_custom_all_reduce.py @@ -15,7 +15,7 @@ from sglang.kernels.jit.benchmark.utils import get_benchmark_range, multigpu_ben from sglang.kernels.jit.utils import cache_once, is_arch_support_pdl from sglang.kernels.ops.communication.mp import register_comm_cleanup from sglang.srt.runtime_context import get_parallel -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci( est_time=120, @@ -23,7 +23,6 @@ register_cuda_ci( runner_config="1-gpu-large", disabled="requires multi-GPU, self-skips in CI", ) -register_amd_ci(est_time=120, stage="jit-kernel-benchmark", runner_config="amd") # --------------------------------------------------------------------------- diff --git a/test/registered/kernels/benchmark/communication/bench_symm_mem_all_gather.py b/test/registered/kernels/benchmark/communication/bench_symm_mem_all_gather.py index d683964444..f89595ff8a 100644 --- a/test/registered/kernels/benchmark/communication/bench_symm_mem_all_gather.py +++ b/test/registered/kernels/benchmark/communication/bench_symm_mem_all_gather.py @@ -34,7 +34,7 @@ from sglang.srt.distributed.device_communicators.triton_symm_mem_ag import ( create_state, ) from sglang.srt.runtime_context import get_parallel -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci( est_time=120, @@ -42,7 +42,6 @@ register_cuda_ci( runner_config="1-gpu-large", disabled="requires multi-GPU, self-skips in CI", ) -register_amd_ci(est_time=120, stage="jit-kernel-benchmark", runner_config="amd") # --------------------------------------------------------------------------- # Sweep parameters diff --git a/test/registered/kernels/benchmark/communication/bench_tp_qknorm.py b/test/registered/kernels/benchmark/communication/bench_tp_qknorm.py index 68bc890672..9ce8728718 100644 --- a/test/registered/kernels/benchmark/communication/bench_tp_qknorm.py +++ b/test/registered/kernels/benchmark/communication/bench_tp_qknorm.py @@ -37,7 +37,7 @@ from sglang.srt.distributed.device_communicators.custom_all_reduce_v2 import ( CustomAllReduceV2, ) from sglang.srt.runtime_context import get_parallel -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci( est_time=120, @@ -45,7 +45,6 @@ register_cuda_ci( runner_config="1-gpu-large", disabled="requires multi-GPU, self-skips in CI", ) -register_amd_ci(est_time=120, stage="jit-kernel-benchmark", runner_config="amd") # --------------------------------------------------------------------------- diff --git a/test/registered/kernels/benchmark/diffusion/bench_residual_gate_add.py b/test/registered/kernels/benchmark/diffusion/bench_residual_gate_add.py index 4e66961ce6..1c8ebf8317 100644 --- a/test/registered/kernels/benchmark/diffusion/bench_residual_gate_add.py +++ b/test/registered/kernels/benchmark/diffusion/bench_residual_gate_add.py @@ -6,6 +6,7 @@ import torch from sglang.kernels.ops.diffusion import fuse_scale_shift_kernel, residual_gate_add_cuda from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kernels.utils import cuda_event_us from sglang.utils import is_in_ci register_cuda_ci( @@ -48,25 +49,6 @@ CI_WORKLOADS = [ ] -def cuda_event_us(fn, warmups: int, repeats: int, rounds: int) -> float: - for _ in range(warmups): - fn() - torch.cuda.synchronize() - - samples = [] - for _ in range(rounds): - start = torch.cuda.Event(enable_timing=True) - end = torch.cuda.Event(enable_timing=True) - start.record() - for _ in range(repeats): - fn() - end.record() - end.synchronize() - samples.append(start.elapsed_time(end) * 1000.0 / repeats) - samples.sort() - return samples[len(samples) // 2] - - def benchmark() -> None: if not torch.cuda.is_available(): print("CUDA required") diff --git a/test/registered/kernels/benchmark/diffusion/bench_scale_shift_4d.py b/test/registered/kernels/benchmark/diffusion/bench_scale_shift_4d.py index dfc02a892f..bd88e290e8 100644 --- a/test/registered/kernels/benchmark/diffusion/bench_scale_shift_4d.py +++ b/test/registered/kernels/benchmark/diffusion/bench_scale_shift_4d.py @@ -7,6 +7,7 @@ import torch from sglang.kernels.ops.diffusion import fuse_scale_shift_kernel from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kernels.utils import cuda_event_us from sglang.utils import is_in_ci register_cuda_ci( @@ -33,25 +34,6 @@ CI_WORKLOADS = [ ] -def cuda_event_us(fn, warmups: int, repeats: int, rounds: int) -> float: - for _ in range(warmups): - fn() - torch.cuda.synchronize() - - samples = [] - for _ in range(rounds): - start = torch.cuda.Event(enable_timing=True) - end = torch.cuda.Event(enable_timing=True) - start.record() - for _ in range(repeats): - fn() - end.record() - end.synchronize() - samples.append(start.elapsed_time(end) * 1000.0 / repeats) - samples.sort() - return samples[len(samples) // 2] - - def benchmark() -> None: if not torch.cuda.is_available(): print("CUDA required") diff --git a/test/registered/kernels/benchmark/layernorm/bench_qknorm.py b/test/registered/kernels/benchmark/layernorm/bench_qknorm.py index 78f149a135..0104ddd462 100644 --- a/test/registered/kernels/benchmark/layernorm/bench_qknorm.py +++ b/test/registered/kernels/benchmark/layernorm/bench_qknorm.py @@ -5,6 +5,7 @@ from sglang.kernels.jit.benchmark.utils import create_random from sglang.kernels.ops.layernorm.norm import fused_inplace_qknorm from sglang.srt.utils import get_current_device_stream_fast from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kernels.qknorm import torch_impl_qknorm register_cuda_ci( est_time=10, stage="base-b-kernel-benchmark", runner_config="1-gpu-large" @@ -32,22 +33,6 @@ def sglang_aot_qknorm( current_stream.wait_stream(alt_stream) -@torch.compile() -def torch_impl_qknorm( - q: torch.Tensor, - k: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, - eps: float = 1e-6, -) -> None: - q_mean = q.float().pow(2).mean(dim=-1, keepdim=True) - k_mean = k.float().pow(2).mean(dim=-1, keepdim=True) - q_norm = (q_mean + eps).rsqrt() - k_norm = (k_mean + eps).rsqrt() - q.copy_(q.float() * q_norm * q_weight.float()) - k.copy_(k.float() * k_norm * k_weight.float()) - - FN_MAP = { "aot": sglang_aot_qknorm, "jit": fused_inplace_qknorm, diff --git a/test/registered/kernels/benchmark/layernorm/bench_qknorm_across_heads.py b/test/registered/kernels/benchmark/layernorm/bench_qknorm_across_heads.py deleted file mode 100644 index 298e3961de..0000000000 --- a/test/registered/kernels/benchmark/layernorm/bench_qknorm_across_heads.py +++ /dev/null @@ -1,125 +0,0 @@ -import itertools -from typing import Tuple - -import torch -import triton -import triton.testing -from sgl_kernel import rmsnorm - -from sglang.kernels.jit.benchmark.utils import run_benchmark -from sglang.kernels.ops.layernorm.norm import fused_inplace_qknorm_across_heads -from sglang.srt.utils import get_current_device_stream_fast -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.utils import is_in_ci - -register_cuda_ci( - est_time=12, stage="base-b-kernel-benchmark", runner_config="1-gpu-large" -) - -IS_CI = is_in_ci() - -alt_stream = torch.cuda.Stream() - - -def sglang_jit_qknorm_across_heads( - q: torch.Tensor, - k: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, -) -> None: - - fused_inplace_qknorm_across_heads(q, k, q_weight, k_weight) - - -def sglang_aot_qknorm_across_heads( - q: torch.Tensor, - k: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, -) -> None: - - current_stream = get_current_device_stream_fast() - alt_stream.wait_stream(current_stream) - rmsnorm(q, q_weight, out=q) - with torch.cuda.stream(alt_stream): - rmsnorm(k, k_weight, out=k) - current_stream.wait_stream(alt_stream) - - -def flashinfer_qknorm_across_heads( - q: torch.Tensor, - k: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, -) -> None: - from flashinfer import rmsnorm - - rmsnorm(q, q_weight, out=q) - rmsnorm(k, k_weight, out=k) - - -@torch.compile() -def torch_impl_qknorm_across_heads( - q: torch.Tensor, - k: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, - eps: float = 1e-6, -) -> None: - q_mean = q.float().pow(2).mean(dim=-1, keepdim=True) - k_mean = k.float().pow(2).mean(dim=-1, keepdim=True) - q_norm = (q_mean + eps).rsqrt() - k_norm = (k_mean + eps).rsqrt() - q.copy_(q.float() * q_norm * q_weight.float()) - k.copy_(k.float() * k_norm * k_weight.float()) - - -DTYPE = torch.bfloat16 -DEVICE = "cuda" - -if IS_CI: - BS_RANGE = [16] - HIDDEN_DIM_RANGE = [1024] -else: - BS_RANGE = [2**n for n in range(0, 14)] - HIDDEN_DIM_RANGE = [512, 1024, 2048, 4096, 8192] - -LINE_VALS = ["jit", "aot", "flashinfer", "torch"] -LINE_NAMES = ["SGL JIT Kernel", "SGL AOT Kernel", "FlashInfer", "PyTorch"] -STYLES = [("blue", "-"), ("orange", "--"), ("green", "-."), ("red", ":")] - -configs = list(itertools.product(BS_RANGE, HIDDEN_DIM_RANGE)) - - -@triton.testing.perf_report( - triton.testing.Benchmark( - x_names=["batch_size", "hidden_dim"], - x_vals=configs, - line_arg="provider", - line_vals=LINE_VALS, - line_names=LINE_NAMES, - styles=STYLES, - ylabel="us", - plot_name="qknorm-across-heads-performance", - args={}, - ) -) -def benchmark( - batch_size: int, hidden_dim: int, provider: str -) -> Tuple[float, float, float]: - q = torch.randn((batch_size, hidden_dim), dtype=DTYPE, device=DEVICE) - k = torch.randn((batch_size, hidden_dim), dtype=DTYPE, device=DEVICE) - q_weight = torch.randn(hidden_dim, dtype=DTYPE, device=DEVICE) - k_weight = torch.randn(hidden_dim, dtype=DTYPE, device=DEVICE) - FN_MAP = { - "jit": sglang_jit_qknorm_across_heads, - "aot": sglang_aot_qknorm_across_heads, - "flashinfer": flashinfer_qknorm_across_heads, - "torch": torch_impl_qknorm_across_heads, - } - fn = lambda: FN_MAP[provider](q, k, q_weight, k_weight) - return run_benchmark(fn) - - -if __name__ == "__main__": - benchmark.run(print_data=True) diff --git a/test/registered/kernels/benchmark/moe/bench_moe_fused_gate.py b/test/registered/kernels/benchmark/moe/bench_moe_fused_gate.py index e220cd490b..de05c61d2b 100644 --- a/test/registered/kernels/benchmark/moe/bench_moe_fused_gate.py +++ b/test/registered/kernels/benchmark/moe/bench_moe_fused_gate.py @@ -2,7 +2,7 @@ import torch from sglang.kernels.jit.benchmark import marker from sglang.kernels.jit.benchmark.utils import create_random -from sglang.kernels.ops.moe.moe_fused_gate import moe_fused_gate, moe_fused_gate_jit +from sglang.kernels.ops.moe.moe_fused_gate import moe_fused_gate from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci( @@ -31,7 +31,7 @@ def torch_router(scores, bias, topk, scoring_func): @marker.parametrize("scoring_func", ["sigmoid", "sqrtsoftplus"]) @marker.parametrize("num_experts", [128, 256, 384, 512], [256, 384]) @marker.parametrize("num_tokens", [1, 4, 16, 64, 512, 1024, 8192], [16, 1024]) -@marker.benchmark("provider", ["triton", "jit", "torch"]) +@marker.benchmark("provider", ["triton", "torch"]) def benchmark(num_tokens: int, num_experts: int, scoring_func: str, provider: str): torch.manual_seed(0) scores = create_random(num_tokens, num_experts, dtype=torch.float32) @@ -48,10 +48,6 @@ def benchmark(num_tokens: int, num_experts: int, scoring_func: str, provider: st return marker.do_bench( moe_fused_gate, input_args=(scores, bias), input_kwargs=common ) - if provider == "jit": - return marker.do_bench( - moe_fused_gate_jit, input_args=(scores, bias), input_kwargs=common - ) if provider == "torch": return marker.do_bench( torch_router, input_args=(scores, bias, TOPK, scoring_func) diff --git a/test/registered/kernels/benchmark/moe/bench_post_reorder_deepgemm.py b/test/registered/kernels/benchmark/moe/bench_post_reorder_deepgemm.py index 1c8a4d5a4f..234e2c23f2 100644 --- a/test/registered/kernels/benchmark/moe/bench_post_reorder_deepgemm.py +++ b/test/registered/kernels/benchmark/moe/bench_post_reorder_deepgemm.py @@ -1,10 +1,7 @@ import torch from sglang.kernels.jit.benchmark import marker -from sglang.kernels.ops.moe.ep_moe_kernels import ( - post_reorder_deepgemm, - post_reorder_triton_kernel, -) +from sglang.kernels.ops.moe.ep_moe_kernels import post_reorder_deepgemm from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci register_cuda_ci( @@ -51,25 +48,12 @@ def _new(down_output, src2dst, topk_ids, topk_weights): return out -def _old(down_output, src2dst, topk_ids, topk_weights): - num_tokens = topk_ids.shape[0] - out = torch.empty(num_tokens, HIDDEN, dtype=torch.bfloat16, device="cuda") - post_reorder_triton_kernel[(num_tokens,)]( - down_output, out, src2dst, topk_ids, topk_weights, TOP_K, HIDDEN, BLOCK_SIZE=512 - ) - out *= RSF - return out - - -FN_MAP = {"fused": _new, "legacy": _old} - - @marker.parametrize("num_tokens", [1, 8, 64, 256, 1024, 4096, 16384], [64, 4096]) -@marker.benchmark("impl", ["fused", "legacy"]) +@marker.benchmark("impl", ["fused"]) def benchmark(num_tokens: int, impl: str): down_output, src2dst, topk_ids, topk_weights = _build(num_tokens) return marker.do_bench( - FN_MAP[impl], + _new, input_args=(down_output, src2dst, topk_ids, topk_weights), graph_clone_args=(0,), memory_args=None, diff --git a/test/registered/kernels/benchmark/quantization/bench_per_token_group_quant.py b/test/registered/kernels/benchmark/quantization/bench_per_token_group_quant.py index 2d190f3a9d..8451976c82 100644 --- a/test/registered/kernels/benchmark/quantization/bench_per_token_group_quant.py +++ b/test/registered/kernels/benchmark/quantization/bench_per_token_group_quant.py @@ -7,8 +7,8 @@ from sglang.kernels.ops.quantization.fp8_kernel import ( fp8_min, ) -# per_token_group_quant_8bit_v2 is DEPRECATED (no production call sites); the -# kernel is kept only as the perf baseline for this benchmark. +# per_token_group_quant_8bit_v2 is the perf baseline; production only routes +# fp32 UE8M0 scales to it. from sglang.kernels.ops.quantization.per_token_group_quant import ( per_token_group_quant, ) diff --git a/test/registered/kernels/benchmark/quantization/bench_per_token_group_quant_masked.py b/test/registered/kernels/benchmark/quantization/bench_per_token_group_quant_masked.py index c9ca5d8241..d55a53f307 100644 --- a/test/registered/kernels/benchmark/quantization/bench_per_token_group_quant_masked.py +++ b/test/registered/kernels/benchmark/quantization/bench_per_token_group_quant_masked.py @@ -11,8 +11,8 @@ from sglang.kernels.ops.quantization.fp8_kernel import ( fp8_min, ) -# per_token_group_quant_8bit_v2 is DEPRECATED (no production call sites); the -# kernel is kept only as the perf baseline for this benchmark. +# per_token_group_quant_8bit_v2 is the perf baseline; production only routes +# fp32 UE8M0 scales to it. from sglang.kernels.ops.quantization.per_token_group_quant import ( per_token_group_quant, ) diff --git a/test/registered/kernels/ops/activation/test_fused_op_gpu_parity.py b/test/registered/kernels/ops/activation/test_fused_op_gpu_parity.py index 3a4dcbb804..6a709ddb18 100644 --- a/test/registered/kernels/ops/activation/test_fused_op_gpu_parity.py +++ b/test/registered/kernels/ops/activation/test_fused_op_gpu_parity.py @@ -33,16 +33,12 @@ def _close(got, ref, dtype, msg): torch.testing.assert_close(got, ref, **_TOL[dtype], msg=msg) -@pytest.mark.parametrize( - "op_attr", ["_SILU_AND_MUL", "_GELU_AND_MUL", "_GELU_TANH_AND_MUL"] -) @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) @pytest.mark.parametrize("shape", [(1, 8192), (128, 8192)]) -def test_gated_activation(op_attr, dtype, shape): - import sglang.kernels.ops.activation as act +def test_silu_and_mul(dtype, shape): + from sglang.kernels.ops.activation import _SILU_AND_MUL as op torch.manual_seed(0) - op = getattr(act, op_attr) x = torch.randn(shape, dtype=dtype, device="cuda") ref = op.forward_native(x) for b in _eligible(op): diff --git a/test/registered/kernels/ops/attention/dsv4/test_flydsl_v4nm_decode.py b/test/registered/kernels/ops/attention/dsv4/test_flydsl_v4nm_decode.py index 20fc92b62a..a93808398e 100644 --- a/test/registered/kernels/ops/attention/dsv4/test_flydsl_v4nm_decode.py +++ b/test/registered/kernels/ops/attention/dsv4/test_flydsl_v4nm_decode.py @@ -24,21 +24,26 @@ from sglang.test.ci.ci_register import register_amd_ci register_amd_ci(est_time=600, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") -if not (is_hip() and is_gfx95_supported()): - pytest.skip("FlyDSL v4 nm decode is gfx950-only", allow_module_level=True) +_skip_reason = "FlyDSL v4 nm decode is gfx950-only" +if is_hip() and is_gfx95_supported(): + try: + from sglang.kernels.ops.attention.dsv4 import flydsl_v4nm + from sglang.kernels.ops.attention.dsv4.flydsl_v4nm import ( + mla_v4_decode as fly_v4, + ) + except ImportError as e: # no flydsl, or an aiter before ROCm/aiter#6377 + _skip_reason = f"in-tree FlyDSL v4 nm decode unavailable: {e}" + else: + _skip_reason = None + flydsl_mla_decode_fwd_v4_nm = flydsl_v4nm.flydsl_mla_decode_fwd_v4_nm + flydsl_mla_v4_decode_fused = flydsl_v4nm.flydsl_mla_v4_decode_fused + flydsl_mla_v4_decode_supported = flydsl_v4nm.flydsl_mla_v4_decode_supported + flydsl_mla_v4_grouped_verify_meta = ( + flydsl_v4nm.flydsl_mla_v4_grouped_verify_meta + ) -try: - from sglang.kernels.ops.attention.dsv4 import flydsl_v4nm - from sglang.kernels.ops.attention.dsv4.flydsl_v4nm import mla_v4_decode as fly_v4 -except ImportError as e: # no flydsl, or an aiter before ROCm/aiter#6377 - pytest.skip( - f"in-tree FlyDSL v4 nm decode unavailable: {e}", allow_module_level=True - ) - -flydsl_mla_decode_fwd_v4_nm = flydsl_v4nm.flydsl_mla_decode_fwd_v4_nm -flydsl_mla_v4_decode_fused = flydsl_v4nm.flydsl_mla_v4_decode_fused -flydsl_mla_v4_decode_supported = flydsl_v4nm.flydsl_mla_v4_decode_supported -flydsl_mla_v4_grouped_verify_meta = flydsl_v4nm.flydsl_mla_v4_grouped_verify_meta +# CI runs this file as a script; module-level pytest.skip would escape __main__. +pytestmark = pytest.mark.skipif(_skip_reason is not None, reason=_skip_reason or "") FP8, BF16, I32 = torch.float8_e4m3fn, torch.bfloat16, torch.int32 NOPE, ROPE, DV, ROW, QB = 448, 64, 512, 512, 128 diff --git a/test/registered/kernels/ops/attention/dsv4/test_mega_attention.py b/test/registered/kernels/ops/attention/dsv4/test_mega_attention.py index 340e95a22e..2e023ad19f 100644 --- a/test/registered/kernels/ops/attention/dsv4/test_mega_attention.py +++ b/test/registered/kernels/ops/attention/dsv4/test_mega_attention.py @@ -5,7 +5,6 @@ from unittest.mock import patch import torch from sgl_kernel.flash_mla_mega import flash_mla_mega_decode -from sglang.kernels.ops.attention.dsv4.torch_quant import quantize_k_cache_v41 from sglang.srt.layers.attention.deepseek_v4_mega_backend import ( interleave_output_columns, interleave_query_rows, @@ -15,6 +14,7 @@ from sglang.srt.layers.attention.deepseek_v4_mega_backend import ( from sglang.srt.models.deepseek_v4 import _fp8_wo_a_einsum from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kernels.deepseek_v4.dsv41_kv_quant_reference import ( + quantize_k_cache_v41, quantize_k_cache_v41_fp4, ) from sglang.test.test_utils import CustomTestCase diff --git a/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py b/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py index f9ab0a9bb4..14b095ca31 100644 --- a/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py +++ b/test/registered/kernels/ops/attention/dsv4/test_swapab_attention.py @@ -9,7 +9,7 @@ from sglang.srt.utils import is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b200") # backend-specific: the HIP swap-AB Gluon kernel runs on gfx950. register_amd_ci(est_time=30, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") diff --git a/test/registered/kernels/ops/attention/dsv4/test_v41_kv_store.py b/test/registered/kernels/ops/attention/dsv4/test_v41_kv_store.py index 41d0ae2060..9ffba6ecd9 100644 --- a/test/registered/kernels/ops/attention/dsv4/test_v41_kv_store.py +++ b/test/registered/kernels/ops/attention/dsv4/test_v41_kv_store.py @@ -1,7 +1,7 @@ """Byte-exactness of the V4.1 (fp8 / fp4) FlashMLA KV cache store kernels. Every store kernel is compared byte for byte with the pure-torch quantizers of -the two formats (torch_quant.quantize_k_cache_v41 and +the two formats (dsv41_kv_quant_reference.quantize_k_cache_v41 and dsv41_kv_quant_reference.quantize_k_cache_v41_fp4), which follow the decode kernel's own reference quantizer. """ @@ -24,11 +24,7 @@ from sglang.kernels.ops.attention.dsv4.elementwise import ( fused_rope_inplace, ) from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout -from sglang.kernels.ops.attention.dsv4.torch_quant import ( - dequantize_k_cache_v41, - fake_quant_compressed_kv, - quantize_k_cache_v41, -) +from sglang.kernels.ops.attention.dsv4.torch_quant import fake_quant_compressed_kv from sglang.srt.utils import is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kernels.deepseek_v4 import dsv41_kv_quant_reference as tq @@ -42,11 +38,11 @@ register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="4-gpu-b register_amd_ci(est_time=60, suite="stage-b-test-1-gpu-small-amd-mi35x") REFERENCE = { - KVLayout.V41: quantize_k_cache_v41, + KVLayout.V41: tq.quantize_k_cache_v41, KVLayout.V41_FP4: tq.quantize_k_cache_v41_fp4, } DEQUANT = { - KVLayout.V41: dequantize_k_cache_v41, + KVLayout.V41: tq.dequantize_k_cache_v41, KVLayout.V41_FP4: tq.dequantize_k_cache_v41_fp4, } # One quantization step, relative: e4m3 has 3 mantissa bits, e2m1 one. @@ -476,7 +472,7 @@ class TestV41KVStore(CustomTestCase): CASES = { - KVLayout.V41: (quantize_k_cache_v41, dequantize_k_cache_v41), + KVLayout.V41: (tq.quantize_k_cache_v41, tq.dequantize_k_cache_v41), KVLayout.V41_FP4: (tq.quantize_k_cache_v41_fp4, tq.dequantize_k_cache_v41_fp4), } diff --git a/test/registered/kernels/ops/attention/kda_flydsl/test_kimi_k3_kda_decode.py b/test/registered/kernels/ops/attention/kda_flydsl/test_kimi_k3_kda_decode.py index ca1706b8ba..5ce8185029 100644 --- a/test/registered/kernels/ops/attention/kda_flydsl/test_kimi_k3_kda_decode.py +++ b/test/registered/kernels/ops/attention/kda_flydsl/test_kimi_k3_kda_decode.py @@ -17,7 +17,6 @@ pytest.importorskip("aiter") from sglang.kernels.ops.attention.kda_flydsl.kimi_k3_kda_decode import ( _fb_build_options, - flydsl_kimi_k3_kda_decode, flydsl_kimi_k3_kda_decode_with_f_b, is_flydsl_kimi_k3_kda_decode_supported, ) @@ -272,25 +271,6 @@ def _relative_rmse( ) -def _run(inputs: Inputs) -> torch.Tensor: - return flydsl_kimi_k3_kda_decode( - x=inputs.x, - conv_weight=inputs.conv_weight, - conv_bias=None, - conv_state=inputs.conv_state, - raw_g=inputs.raw_g, - raw_beta=inputs.raw_beta, - A_log=inputs.A_log, - dt_bias=inputs.dt_bias, - lower_bound=_LOWER_BOUND, - state=inputs.state, - state_indices=inputs.state_indices, - output_gate=inputs.output_gate, - norm_weight=inputs.norm_weight, - norm_eps=_NORM_EPS, - ) - - def _make_fb_inputs( batch: int, seed: int = 20260728, @@ -348,7 +328,6 @@ def _run_with_f_b( def test_public_api_and_support_predicate() -> None: import sglang.kernels.ops.attention.kda_flydsl as flydsl_ops - assert flydsl_ops.flydsl_kimi_k3_kda_decode is flydsl_kimi_k3_kda_decode assert ( flydsl_ops.is_flydsl_kimi_k3_kda_decode_supported is is_flydsl_kimi_k3_kda_decode_supported @@ -379,46 +358,6 @@ def test_f_b_batch_two_dispatch_is_guarded() -> None: } -@pytest.mark.parametrize("batch", [1, 8, 16]) -def test_kimi_k3_kda_decode_matches_reference(batch: int) -> None: - seed = _make_inputs(batch) - reference_inputs = _copy_inputs(seed) - actual_inputs = _copy_inputs(seed) - - reference = _reference(reference_inputs) - actual = _run(actual_inputs) - torch.cuda.synchronize() - - assert is_flydsl_kimi_k3_kda_decode_supported(_DEVICE) - assert not torch.isnan(actual).any() - assert _relative_rmse(reference, actual) < 1e-3 - assert ( - _relative_rmse( - reference_inputs.state, - actual_inputs.state, - ) - < 1e-3 - ) - assert torch.equal( - reference_inputs.conv_state, - actual_inputs.conv_state, - ) - - -def test_non_positive_slots_do_not_modify_caches() -> None: - inputs = _make_inputs(batch=2) - inputs.state_indices.copy_(torch.tensor([0, -1], dtype=torch.int32, device=_DEVICE)) - conv_before = inputs.conv_state.clone() - state_before = inputs.state.clone() - - actual = _run(inputs) - torch.cuda.synchronize() - - assert torch.count_nonzero(actual) == 0 - assert torch.equal(inputs.conv_state, conv_before) - assert torch.equal(inputs.state, state_before) - - @pytest.mark.parametrize("batch", [1, 8, 16]) def test_kimi_k3_kda_decode_with_f_b_matches_reference(batch: int) -> None: f_a, f_b_weight, seed = _make_fb_inputs(batch) @@ -505,14 +444,6 @@ def test_f_b_api_rejects_invalid_projection_inputs() -> None: _run_with_f_b(f_a, f_b_weight.transpose(1, 2), inputs) -def test_decode_api_rejects_invalid_input_rank() -> None: - inputs = _make_inputs(batch=1) - inputs.x = inputs.x.unsqueeze(0) - - with pytest.raises(ValueError, match="`x` must have rank 2"): - _run(inputs) - - def test_fused_kda_backend_is_opt_in(monkeypatch): from sglang.kernels.ops.attention import kda_fused_decode_aiter_hip diff --git a/test/registered/kernels/ops/attention/test_c128_v2.py b/test/registered/kernels/ops/attention/test_c128_v2.py index a5b0e74fab..13129576fe 100644 --- a/test/registered/kernels/ops/attention/test_c128_v2.py +++ b/test/registered/kernels/ops/attention/test_c128_v2.py @@ -7,7 +7,6 @@ import pytest import torch import triton -from sglang.kernels.ops.attention.dsv4 import compress_forward from sglang.srt.utils import get_device from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kernels.deepseek_v4.common import ( @@ -16,6 +15,8 @@ from sglang.test.kernels.deepseek_v4.common import ( make_legacy_context, make_paged_context, make_state_pool, + run_compress_decode, + run_compress_prefill, to_seq_extend, ) @@ -55,44 +56,6 @@ def _make_inputs( return kv_score_input_cpu, ape_cpu -def _run_prefill( - ctx: Context, - pool: torch.Tensor, - kv_score_input: torch.Tensor, - ape: torch.Tensor, - seq_lens_cpu: torch.Tensor, - extend_lens_cpu: torch.Tensor, -) -> torch.Tensor: - num_q = int(extend_lens_cpu.sum().item()) - plan = ctx.make_prefill_plan(seq_lens_cpu, extend_lens_cpu, num_q) - return compress_forward( - pool, - kv_score_input, - ape, - plan, - head_dim=ctx.head_dim, - compress_ratio=RATIO, - ) - - -def _run_decode( - ctx: Context, - pool: torch.Tensor, - kv_score_input: torch.Tensor, - ape: torch.Tensor, - seq_lens_gpu: torch.Tensor, -) -> torch.Tensor: - plan = ctx.make_decode_plan(seq_lens_gpu) - return compress_forward( - pool, - kv_score_input, - ape, - plan, - head_dim=ctx.head_dim, - compress_ratio=RATIO, - ) - - # ----------------------------------------------------------------------------- # Tests # ----------------------------------------------------------------------------- @@ -113,7 +76,7 @@ def test_prefill_no_context(mode: str, seq_len: int) -> None: kv_in_cpu, ape_cpu = _make_inputs(num_q, ctx.head_dim, seed=seq_len) pool = make_state_pool(ctx.num_pages, RATIO, ctx.head_dim) - out = _run_prefill( + out = run_compress_prefill( ctx, pool, kv_in_cpu.to(get_device()), @@ -148,7 +111,7 @@ def test_prefill_then_decode(mode: str, prefix_len: int) -> None: if prefix_len > 0: seq_lens_cpu, extend_lens_cpu, _ = to_seq_extend([(prefix_len, prefix_len)]) - _run_prefill( + run_compress_prefill( ctx, pool, kv_full_cpu[:prefix_len].to(get_device()), @@ -164,7 +127,9 @@ def test_prefill_then_decode(mode: str, prefix_len: int) -> None: [cur_seq_len], dtype=torch.int64, device=get_device() ) kv_step = kv_full_cpu[prefix_len + k : prefix_len + k + 1].to(get_device()) - out = _run_decode(ctx, pool, kv_step, ape_cpu.to(get_device()), seq_lens_gpu) + out = run_compress_decode( + ctx, pool, kv_step, ape_cpu.to(get_device()), seq_lens_gpu + ) if cur_seq_len % RATIO == 0: final_out = out @@ -198,7 +163,7 @@ def test_prefill_then_extend(mode: str, prefix_len: int, extend_len: int) -> Non pool = make_state_pool(ctx.num_pages, RATIO, ctx.head_dim) seq_lens_cpu, extend_lens_cpu, _ = to_seq_extend([(prefix_len, prefix_len)]) - _run_prefill( + run_compress_prefill( ctx, pool, kv_full_cpu[:prefix_len].to(get_device()), @@ -208,7 +173,7 @@ def test_prefill_then_extend(mode: str, prefix_len: int, extend_len: int) -> Non ) seq_lens_cpu, extend_lens_cpu, _ = to_seq_extend([(seq_len, extend_len)]) - out = _run_prefill( + out = run_compress_prefill( ctx, pool, kv_full_cpu[prefix_len:].to(get_device()), @@ -241,7 +206,7 @@ def test_prefill_multibatch(mode: str) -> None: seq_lens_cpu, extend_lens_cpu, num_q = to_seq_extend(seq_extend) kv_in_cpu, ape_cpu = _make_inputs(num_q, ctx.head_dim, seed=99) pool = make_state_pool(ctx.num_pages, RATIO, ctx.head_dim) - out = _run_prefill( + out = run_compress_prefill( ctx, pool, kv_in_cpu.to(get_device()), diff --git a/test/registered/kernels/ops/attention/test_c4_v2.py b/test/registered/kernels/ops/attention/test_c4_v2.py index cd419ef76d..e42d948a50 100644 --- a/test/registered/kernels/ops/attention/test_c4_v2.py +++ b/test/registered/kernels/ops/attention/test_c4_v2.py @@ -10,7 +10,6 @@ import triton from sglang.kernels.ops.attention.dsv4 import ( CompressorDecodePlan, CompressorPrefillPlan, - compress_forward, ) from sglang.srt.utils import get_device from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci @@ -20,6 +19,8 @@ from sglang.test.kernels.deepseek_v4.common import ( make_legacy_context, make_paged_context, make_state_pool, + run_compress_decode, + run_compress_prefill, to_seq_extend, ) @@ -72,44 +73,6 @@ def _gt_compress( # ----------------------------------------------------------------------------- -def _run_prefill( - ctx: Context, - pool: torch.Tensor, - kv_score_input: torch.Tensor, - ape: torch.Tensor, - seq_lens_cpu: torch.Tensor, - extend_lens_cpu: torch.Tensor, -) -> torch.Tensor: - num_q = int(extend_lens_cpu.sum().item()) - plan = ctx.make_prefill_plan(seq_lens_cpu, extend_lens_cpu, num_q) - return compress_forward( - pool, - kv_score_input, - ape, - plan, - head_dim=ctx.head_dim, - compress_ratio=RATIO, - ) - - -def _run_decode( - ctx: Context, - pool: torch.Tensor, - kv_score_input: torch.Tensor, - ape: torch.Tensor, - seq_lens_gpu: torch.Tensor, -) -> torch.Tensor: - plan = ctx.make_decode_plan(seq_lens_gpu) - return compress_forward( - pool, - kv_score_input, - ape, - plan, - head_dim=ctx.head_dim, - compress_ratio=RATIO, - ) - - def _make_inputs( num_q: int, head_dim: int, seed: int ) -> Tuple[torch.Tensor, torch.Tensor]: @@ -226,7 +189,7 @@ def test_prefill_no_context(mode: str, seq_len: int) -> None: kv_in_cpu, ape_cpu = _make_inputs(num_q, ctx.head_dim, seed=seq_len) pool = make_state_pool(ctx.num_pages, RATIO, ctx.head_dim) - out = _run_prefill( + out = run_compress_prefill( ctx, pool, kv_in_cpu.to(get_device()), @@ -269,7 +232,7 @@ def test_prefill_then_decode(mode: str, prefix_len: int) -> None: # Prefill the prefix. seq_lens_cpu, extend_lens_cpu, _ = to_seq_extend([(prefix_len, prefix_len)]) - _run_prefill( + run_compress_prefill( ctx, pool, kv_full_cpu[:prefix_len].to(get_device()), @@ -286,7 +249,9 @@ def test_prefill_then_decode(mode: str, prefix_len: int) -> None: [cur_seq_len], dtype=torch.int64, device=get_device() ) kv_step = kv_full_cpu[prefix_len + k : prefix_len + k + 1].to(get_device()) - out = _run_decode(ctx, pool, kv_step, ape_cpu.to(get_device()), seq_lens_gpu) + out = run_compress_decode( + ctx, pool, kv_step, ape_cpu.to(get_device()), seq_lens_gpu + ) if cur_seq_len % RATIO == 0: final_out = out @@ -322,7 +287,7 @@ def test_prefill_then_extend(mode: str, prefix_len: int, extend_len: int) -> Non # First prefill: seq=prefix, ext=prefix. seq_lens_cpu, extend_lens_cpu, _ = to_seq_extend([(prefix_len, prefix_len)]) - _run_prefill( + run_compress_prefill( ctx, pool, kv_full_cpu[:prefix_len].to(get_device()), @@ -333,7 +298,7 @@ def test_prefill_then_extend(mode: str, prefix_len: int, extend_len: int) -> Non # Second prefill: seq=prefix+extend, ext=extend, prefix=prefix_len. seq_lens_cpu, extend_lens_cpu, num_q = to_seq_extend([(seq_len, extend_len)]) - out = _run_prefill( + out = run_compress_prefill( ctx, pool, kv_full_cpu[prefix_len:].to(get_device()), @@ -371,7 +336,7 @@ def test_paged_buffer_intermediate() -> None: kv_in_cpu, ape_cpu = _make_inputs(num_q, ctx.head_dim, seed=42) pool = make_state_pool(ctx.num_pages, RATIO, ctx.head_dim) - _run_prefill( + run_compress_prefill( ctx, pool, kv_in_cpu.to(get_device()), @@ -417,7 +382,7 @@ def test_prefill_multibatch(mode: str) -> None: seq_lens_cpu, extend_lens_cpu, num_q = to_seq_extend(seq_extend) kv_in_cpu, ape_cpu = _make_inputs(num_q, ctx.head_dim, seed=99) pool = make_state_pool(ctx.num_pages, RATIO, ctx.head_dim) - out = _run_prefill( + out = run_compress_prefill( ctx, pool, kv_in_cpu.to(get_device()), diff --git a/test/registered/kernels/ops/attention/test_cutedsl_gdn.py b/test/registered/kernels/ops/attention/test_cutedsl_gdn.py index d7cae7dbe7..6328c59572 100644 --- a/test/registered/kernels/ops/attention/test_cutedsl_gdn.py +++ b/test/registered/kernels/ops/attention/test_cutedsl_gdn.py @@ -2,16 +2,13 @@ import sys -import numpy as np import pytest import torch from sglang.test.ci.ci_register import register_cuda_ci try: - import cuda.bindings.driver as cuda_driver import cutlass # noqa: F401 - from cutlass.cute.runtime import from_dlpack from sglang.kernels.ops.attention import cutedsl_gdn @@ -102,204 +99,5 @@ def test_cutedsl_gdn_precision(B: int, varlen: bool, pool_view: str): torch.testing.assert_close(state_cutedsl, state_triton, atol=1e-4, rtol=1e-4) -@pytest.mark.skipif( - True, - reason="Skip the performance test because the speedup ratio is highly unstable in the CI environment. ", -) -@pytest.mark.skipif(not CUTEDSL_AVAILABLE, reason="CuTe DSL not available") -@pytest.mark.skipif(not TRITON_AVAILABLE, reason="Triton kernel not available") -@pytest.mark.parametrize("B", [1, 128]) -def test_cutedsl_gdn_performance(B: int): - """Benchmark CuTe DSL GDN kernel against Triton reference.""" - torch.manual_seed(2025) - T, H, K, V, HV = 1, 16, 128, 128, 32 - N = B - scale = K**-0.5 - is_varlen = True - warmup, bench_iters, run_iters = 10, 100, 10 - - A_log = torch.randn(HV, dtype=torch.float32, device="cuda") - dt_bias = torch.randn(HV, dtype=torch.bfloat16, device="cuda") - indices = torch.arange(N, dtype=torch.int32, device="cuda") - state_cutedsl = torch.randn(N, HV, K, V, dtype=torch.float32, device="cuda") - state_triton = state_cutedsl.reshape(-1).contiguous() - cu_seqlens = torch.zeros(N + 1, dtype=torch.int32, device="cuda") - o_cutedsl = torch.zeros(1, N, HV, V, dtype=torch.bfloat16, device="cuda") - - # Prepare tensors for multiple runs - q_list, k_list, v_list, a_list, b_list = [], [], [], [], [] - q_tensor_list, k_tensor_list, v_tensor_list, a_tensor_list, b_tensor_list = ( - [], - [], - [], - [], - [], - ) - q_triton, k_triton, v_triton, a_triton, b_triton = [], [], [], [], [] - - for ri in range(run_iters): - torch.manual_seed(2025 + ri) - q_i = torch.randn(1, N, H, K, dtype=torch.bfloat16, device="cuda") - k_i = torch.randn(1, N, H, K, dtype=torch.bfloat16, device="cuda") - v_i = torch.randn(1, N, HV, V, dtype=torch.bfloat16, device="cuda") - a_i = torch.randn(N, HV, dtype=torch.bfloat16, device="cuda") - b_i = torch.randn(N, HV, dtype=torch.bfloat16, device="cuda") - - q_list.append(q_i) - k_list.append(k_i) - v_list.append(v_i) - a_list.append(a_i) - b_list.append(b_i) - q_tensor_list.append(from_dlpack(q_i, assumed_align=16)) - k_tensor_list.append(from_dlpack(k_i, assumed_align=16)) - v_tensor_list.append(from_dlpack(v_i, assumed_align=16)) - a_tensor_list.append(from_dlpack(a_i, assumed_align=16)) - b_tensor_list.append(from_dlpack(b_i, assumed_align=16)) - q_triton.append(q_i.transpose(0, 1).contiguous()) - k_triton.append(k_i.transpose(0, 1).contiguous()) - v_triton.append(v_i.transpose(0, 1).contiguous()) - a_triton.append(a_i.unsqueeze(1).contiguous()) - b_triton.append(b_i.unsqueeze(1).contiguous()) - - A_log_t = from_dlpack(A_log, assumed_align=16) - dt_bias_t = from_dlpack(dt_bias, assumed_align=16) - h0_t = from_dlpack(state_cutedsl.transpose(-1, -2), assumed_align=16) - idx_t = from_dlpack(indices, assumed_align=16) - o_t = from_dlpack(o_cutedsl, assumed_align=16) - cu_t = from_dlpack(cu_seqlens, assumed_align=16) - - torch_stream = torch.cuda.Stream() - stream = cuda_driver.CUstream(torch_stream.cuda_stream) - - # Compile kernels - compiled = cutedsl_gdn._get_compiled_kernel(N, H, HV, K, V, N, N < 32, is_varlen) - torch.cuda.synchronize() - - for ri in range(run_iters): - _ = run_triton_kernel( - A_log, - dt_bias, - q_triton[ri], - k_triton[ri], - v_triton[ri], - a_triton[ri], - b_triton[ri], - state_triton, - indices, - scale, - ) - torch.cuda.synchronize() - - def run_cutedsl(): - for ri in range(run_iters): - compiled( - cu_t, - q_tensor_list[ri], - k_tensor_list[ri], - v_tensor_list[ri], - a_tensor_list[ri], - b_tensor_list[ri], - A_log_t, - dt_bias_t, - h0_t, - idx_t, - o_t, - stream, - ) - - def run_triton(): - for ri in range(run_iters): - _ = run_triton_kernel( - A_log, - dt_bias, - q_triton[ri], - k_triton[ri], - v_triton[ri], - a_triton[ri], - b_triton[ri], - state_triton, - indices, - scale, - ) - - # Warmup - with torch.cuda.stream(torch_stream): - run_cutedsl() - torch.cuda.synchronize() - run_triton() - torch.cuda.synchronize() - - # Capture CUDA graphs - graph_triton = torch.cuda.CUDAGraph() - graph_cutedsl = torch.cuda.CUDAGraph() - try: - with torch.cuda.graph(graph_triton): - run_triton() - with torch.cuda.graph(graph_cutedsl, stream=torch_stream): - run_cutedsl() - torch.cuda.synchronize() - except Exception: - graph_triton = graph_cutedsl = None - - # Warmup with graphs - for _ in range(warmup): - if graph_cutedsl: - graph_cutedsl.replay() - else: - with torch.cuda.stream(torch_stream): - run_cutedsl() - torch.cuda.synchronize() - - if graph_triton: - graph_triton.replay() - else: - run_triton() - torch.cuda.synchronize() - - # Benchmark - triton_times, cutedsl_times = [], [] - for _ in range(bench_iters): - start, end = ( - torch.cuda.Event(enable_timing=True), - torch.cuda.Event(enable_timing=True), - ) - start.record() - if graph_triton: - graph_triton.replay() - else: - run_triton() - end.record() - torch.cuda.synchronize() - triton_times.append(start.elapsed_time(end)) - - start, end = ( - torch.cuda.Event(enable_timing=True), - torch.cuda.Event(enable_timing=True), - ) - with torch.cuda.stream(torch_stream): - start.record() - if graph_cutedsl: - graph_cutedsl.replay() - else: - run_cutedsl() - end.record() - torch.cuda.synchronize() - cutedsl_times.append(start.elapsed_time(end)) - - triton_mean = np.mean(triton_times) / run_iters * 1000 - triton_std = np.std(triton_times) / run_iters * 1000 - cutedsl_mean = np.mean(cutedsl_times) / run_iters * 1000 - cutedsl_std = np.std(cutedsl_times) / run_iters * 1000 - speedup = triton_mean / cutedsl_mean - - kernel_type = "SmallBatch" if B < 32 else "LargeBatch" - print( - f"\n B={B} ({kernel_type}): Triton={triton_mean:.2f}±{triton_std:.2f}μs, CuTeDSL={cutedsl_mean:.2f}±{cutedsl_std:.2f}μs, speedup={speedup:.2f}x" - ) - - min_speedup = 1.0 if B < 32 else 1.15 - assert speedup >= min_speedup, f"Speedup {speedup:.2f}x < {min_speedup}x for B={B}" - - if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/test/registered/kernels/ops/attention/test_dcp_lse_combine.py b/test/registered/kernels/ops/attention/test_dcp_lse_combine.py index bf2ccbfbea..23bc9a6de1 100644 --- a/test/registered/kernels/ops/attention/test_dcp_lse_combine.py +++ b/test/registered/kernels/ops/attention/test_dcp_lse_combine.py @@ -31,10 +31,8 @@ class TestLSECombineTritonVsCPU(CustomTestCase): def _run_combine_test( self, N, B, H_local, D, is_base_e, dtype=torch.bfloat16, atol=1e-2 ): - from sglang.kernels.ops.attention.dcp_kernels import ( - _lse_weighted_combine_cpu, - dcp_lse_combine_triton, - ) + from sglang.kernels.ops.attention.dcp_kernels import dcp_lse_combine_triton + from sglang.test.kernels.dcp import lse_weighted_combine_cpu torch.manual_seed(42) @@ -49,7 +47,7 @@ class TestLSECombineTritonVsCPU(CustomTestCase): * 5.0 ) - cpu_result = _lse_weighted_combine_cpu( + cpu_result = lse_weighted_combine_cpu( partial_outputs.cpu(), partial_lses.cpu(), is_lse_base_on_e=is_base_e, @@ -197,10 +195,8 @@ class TestLSECombineEdgeCases(CustomTestCase): def test_one_shard_dominant(self): """One shard has much larger LSE -- output should be close to that shard.""" - from sglang.kernels.ops.attention.dcp_kernels import ( - _lse_weighted_combine_cpu, - dcp_lse_combine_triton, - ) + from sglang.kernels.ops.attention.dcp_kernels import dcp_lse_combine_triton + from sglang.test.kernels.dcp import lse_weighted_combine_cpu N, B, H_local, D = 2, 1, 1, 64 partial_outputs = torch.randn( @@ -213,7 +209,7 @@ class TestLSECombineEdgeCases(CustomTestCase): triton_result, _ = dcp_lse_combine_triton( partial_outputs, partial_lses, is_lse_base_on_e=True ) - cpu_result = _lse_weighted_combine_cpu( + cpu_result = lse_weighted_combine_cpu( partial_outputs.cpu(), partial_lses.cpu(), is_lse_base_on_e=True ) diff --git a/test/registered/kernels/ops/attention/test_dsa_transform_index.py b/test/registered/kernels/ops/attention/test_dsa_transform_index.py index 2315405405..7791e7cdd6 100644 --- a/test/registered/kernels/ops/attention/test_dsa_transform_index.py +++ b/test/registered/kernels/ops/attention/test_dsa_transform_index.py @@ -7,9 +7,11 @@ import sglang.kernels.ops.attention.dsa.transform_index as transform_index_modul from sglang.kernels.ops.attention.dsa.transform_index import ( transform_index_page_table_decode_fast, transform_index_page_table_prefill_fast, - transform_index_page_table_prefill_ref, ) from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.kernels.dsa_transform_index import ( + transform_index_page_table_prefill_ref, +) from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=9, stage="base-b-kernel-unit", runner_config="1-gpu-large") diff --git a/test/registered/kernels/ops/attention/test_flash_attention_4.py b/test/registered/kernels/ops/attention/test_flash_attention_4.py index d344346b12..bfec86afe2 100644 --- a/test/registered/kernels/ops/attention/test_flash_attention_4.py +++ b/test/registered/kernels/ops/attention/test_flash_attention_4.py @@ -9,7 +9,6 @@ import unittest import pytest import torch -import torch.nn.functional as F from einops import rearrange, repeat from sglang.kernels.ops.attention.flash_attention import ( @@ -29,6 +28,13 @@ from sglang.kernels.ops.attention.flash_attn.cute.interface import ( _shear_bias_empty, _shear_bias_workspace, ) +from sglang.kernels.ops.attention.flash_attn.cute.testing import ( + construct_chunk_mask, + construct_local_mask, + generate_qkv, + pad_input, + unpad_input, +) from sglang.srt.utils import is_sm100_or_sm110_supported from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase @@ -141,58 +147,6 @@ def apply_rotary_emb( return torch.cat((x_rot, x_pass), dim=-1) -def unpad_input(hidden_states, attention_mask, unused_mask=None): - """ - Arguments: - hidden_states: (batch, seqlen, ...) - attention_mask: (batch, seqlen), bool / int, 1 means valid and 0 means not valid. - unused_mask: (batch, seqlen), bool / int, 1 means the element is allocated but unused. - Return: - hidden_states: (total_nnz, ...), where total_nnz = number of tokens selected in attention_mask + unused_mask. - indices: (total_nnz), the indices of masked tokens from the flattened input sequence. - cu_seqlens: (batch + 1), the cumulative sequence lengths, used to index into hidden_states. - max_seqlen_in_batch: int - seqused: (batch), returns the number of tokens selected in attention_mask + unused_mask. - """ - all_masks = ( - (attention_mask + unused_mask) if unused_mask is not None else attention_mask - ) - seqlens_in_batch = all_masks.sum(dim=-1, dtype=torch.int32) - used_seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) - indices = torch.nonzero(all_masks.flatten(), as_tuple=False).flatten() - max_seqlen_in_batch = seqlens_in_batch.max().item() - cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0)) - # TD [2022-03-04] We don't want to index with a bool mask, because Pytorch will expand the - # bool mask, then call nonzero to get the indices, then index with those. The indices is @dim - # times larger than it needs to be, wasting memory. It's faster and more memory-efficient to - # index with integer indices. - return ( - rearrange(hidden_states, "b s ... -> (b s) ...")[indices], - indices, - cu_seqlens, - max_seqlen_in_batch, - used_seqlens_in_batch, - ) - - -def pad_input(hidden_states, indices, batch, seqlen): - """ - Arguments: - hidden_states: (total_nnz, ...), where total_nnz = number of tokens in selected in attention_mask. - indices: (total_nnz), the indices that represent the non-masked tokens of the original padded input sequence. - batch: int, batch size for the padded sequence. - seqlen: int, maximum sequence length for the padded sequence. - Return: - hidden_states: (batch, seqlen, ...) - """ - dim = hidden_states.shape[1:] - output = torch.zeros( - (batch * seqlen), *dim, device=hidden_states.device, dtype=hidden_states.dtype - ) - output[indices] = hidden_states - return rearrange(output, "(b s) ... -> b s ...", b=batch) - - def generate_random_padding_mask( max_seqlen, batch_size, device, mode="random", zero_lengths=False ): @@ -231,234 +185,6 @@ def generate_random_padding_mask( return padding_mask -def generate_qkv( - q, - k, - v, - query_padding_mask=None, - key_padding_mask=None, - qv=None, - kvpacked=False, - qkvpacked=False, - query_unused_mask=None, - key_unused_mask=None, -): - """ - Arguments: - q: (batch_size, seqlen_q, nheads, d) - k: (batch_size, seqlen_k, nheads_k, d) - v: (batch_size, seqlen_k, nheads_k, d_v) - query_padding_mask: (batch_size, seqlen), bool - key_padding_mask: (batch_size, seqlen), bool - """ - assert not (kvpacked and qkvpacked) - batch_size, seqlen_q, nheads, d = q.shape - d_v = v.shape[-1] - _, seqlen_k, nheads_k, _ = k.shape - assert k.shape == (batch_size, seqlen_k, nheads_k, d) - assert v.shape == (batch_size, seqlen_k, nheads_k, d_v) - if query_unused_mask is not None or key_unused_mask is not None: - assert not kvpacked - assert not qkvpacked - - if query_padding_mask is not None: - q_unpad, indices_q, cu_seqlens_q, max_seqlen_q, seqused_q = unpad_input( - q, query_padding_mask, query_unused_mask - ) - output_pad_fn = lambda output_unpad: pad_input( - output_unpad, indices_q, batch_size, seqlen_q - ) - qv_unpad = ( - rearrange(qv, "b s ... -> (b s) ...")[indices_q] if qv is not None else None - ) - else: - q_unpad = rearrange(q, "b s h d -> (b s) h d") - cu_seqlens_q = torch.arange( - 0, - (batch_size + 1) * seqlen_q, - step=seqlen_q, - dtype=torch.int32, - device=q_unpad.device, - ) - seqused_q = None - max_seqlen_q = seqlen_q - output_pad_fn = lambda output_unpad: rearrange( - output_unpad, "(b s) h d -> b s h d", b=batch_size - ) - qv_unpad = rearrange(qv, "b s ... -> (b s) ...") if qv is not None else None - - if key_padding_mask is not None: - k_unpad, indices_k, cu_seqlens_k, max_seqlen_k, seqused_k = unpad_input( - k, key_padding_mask, key_unused_mask - ) - v_unpad, *rest = unpad_input(v, key_padding_mask, key_unused_mask) - else: - k_unpad = rearrange(k, "b s h d -> (b s) h d") - v_unpad = rearrange(v, "b s h d -> (b s) h d") - cu_seqlens_k = torch.arange( - 0, - (batch_size + 1) * seqlen_k, - step=seqlen_k, - dtype=torch.int32, - device=k_unpad.device, - ) - seqused_k = None - max_seqlen_k = seqlen_k - - if qkvpacked: - assert (query_padding_mask == key_padding_mask).all() - assert nheads == nheads_k - qkv_unpad = torch.stack([q_unpad, k_unpad, v_unpad], dim=1) - qkv = torch.stack([q, k, v], dim=2) - if query_padding_mask is not None: - dqkv_pad_fn = lambda dqkv_unpad: pad_input( - dqkv_unpad, indices_q, batch_size, seqlen_q - ) - else: - dqkv_pad_fn = lambda dqkv_unpad: rearrange( - dqkv_unpad, "(b s) t h d -> b s t h d", b=batch_size - ) - return ( - qkv_unpad.detach().requires_grad_(), - cu_seqlens_q, - max_seqlen_q, - qkv.detach().requires_grad_(), - output_pad_fn, - dqkv_pad_fn, - ) - elif kvpacked: - kv_unpad = torch.stack([k_unpad, v_unpad], dim=1) - kv = torch.stack([k, v], dim=2) - dq_pad_fn = output_pad_fn - if key_padding_mask is not None: - dkv_pad_fn = lambda dkv_unpad: pad_input( - dkv_unpad, indices_k, batch_size, seqlen_k - ) - else: - dkv_pad_fn = lambda dkv_unpad: rearrange( - dkv_unpad, "(b s) t h d -> b s t h d", b=batch_size - ) - return ( - q_unpad.detach().requires_grad_(), - kv_unpad.detach().requires_grad_(), - cu_seqlens_q, - cu_seqlens_k, - max_seqlen_q, - max_seqlen_k, - q.detach().requires_grad_(), - kv.detach().requires_grad_(), - output_pad_fn, - dq_pad_fn, - dkv_pad_fn, - ) - else: - dq_pad_fn = output_pad_fn - if key_padding_mask is not None: - dk_pad_fn = lambda dk_unpad: pad_input( - dk_unpad, indices_k, batch_size, seqlen_k - ) - else: - dk_pad_fn = lambda dk_unpad: rearrange( - dk_unpad, "(b s) h d -> b s h d", b=batch_size - ) - return ( - q_unpad.detach().requires_grad_(), - k_unpad.detach().requires_grad_(), - v_unpad.detach().requires_grad_(), - qv_unpad.detach() if qv is not None else None, - cu_seqlens_q, - cu_seqlens_k, - seqused_q, - seqused_k, - max_seqlen_q, - max_seqlen_k, - q.detach().requires_grad_(), - k.detach().requires_grad_(), - v.detach().requires_grad_(), - qv.detach() if qv is not None else None, - output_pad_fn, - dq_pad_fn, - dk_pad_fn, - ) - - -def construct_local_mask( - seqlen_q, - seqlen_k, - window_size=(None, None), - sink_token_length=0, - query_padding_mask=None, - key_padding_mask=None, - key_leftpad=None, - device=None, -): - row_idx = rearrange( - torch.arange(seqlen_q, device=device, dtype=torch.long), "s -> s 1" - ) - col_idx = torch.arange(seqlen_k, device=device, dtype=torch.long) - if key_leftpad is not None: - key_leftpad = rearrange(key_leftpad, "b -> b 1 1 1") - col_idx = repeat(col_idx, "s -> b 1 1 s", b=key_leftpad.shape[0]) - col_idx = torch.where(col_idx >= key_leftpad, col_idx - key_leftpad, 2**32) - sk = ( - seqlen_k - if key_padding_mask is None - else rearrange(key_padding_mask.sum(-1), "b -> b 1 1 1") - ) - sq = ( - seqlen_q - if query_padding_mask is None - else rearrange(query_padding_mask.sum(-1), "b -> b 1 1 1") - ) - if window_size[0] is None: - return col_idx > row_idx + sk - sq + window_size[1] - else: - sk = torch.full_like(col_idx, seqlen_k) if key_padding_mask is None else sk - return torch.logical_or( - col_idx > torch.minimum(row_idx + sk - sq + window_size[1], sk), - torch.logical_and( - col_idx < row_idx + sk - sq - window_size[0], - col_idx >= sink_token_length, - ), - ) - - -def construct_chunk_mask( - seqlen_q, - seqlen_k, - attention_chunk, - query_padding_mask=None, - key_padding_mask=None, - key_leftpad=None, - device=None, -): - row_idx = rearrange( - torch.arange(seqlen_q, device=device, dtype=torch.long), "s -> s 1" - ) - col_idx = torch.arange(seqlen_k, device=device, dtype=torch.long) - if key_leftpad is not None: - key_leftpad = rearrange(key_leftpad, "b -> b 1 1 1") - col_idx = repeat(col_idx, "s -> b 1 1 s", b=key_leftpad.shape[0]) - col_idx = torch.where(col_idx >= key_leftpad, col_idx - key_leftpad, 2**32) - sk = ( - seqlen_k - if key_padding_mask is None - else rearrange(key_padding_mask.sum(-1), "b -> b 1 1 1") - ) - sq = ( - seqlen_q - if query_padding_mask is None - else rearrange(query_padding_mask.sum(-1), "b -> b 1 1 1") - ) - sk = torch.full_like(col_idx, seqlen_k) if key_padding_mask is None else sk - # Subtract remainder instead of divide and then multiply to take care of negative values - col_limit_left_chunk = row_idx + sk - sq - (row_idx + sk - sq) % attention_chunk - return torch.logical_or( - col_idx < col_limit_left_chunk, - col_idx >= col_limit_left_chunk + attention_chunk, - ) - - def attention_ref( q, k, diff --git a/test/registered/kernels/ops/attention/test_fp4_indexer.py b/test/registered/kernels/ops/attention/test_fp4_indexer.py index 42154a7d48..83aced3ca5 100644 --- a/test/registered/kernels/ops/attention/test_fp4_indexer.py +++ b/test/registered/kernels/ops/attention/test_fp4_indexer.py @@ -25,6 +25,10 @@ from sglang.kernels.ops.attention.dsv4.fp4_indexer import ( ) from sglang.srt.utils import get_device, is_xpu from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kernels.deepseek_v4.common import ( + ceil_ue8m0_exp_ref, + fp4_e2m1_code_ref, +) register_cuda_ci(est_time=60, stage="base-b-kernel-unit", runner_config="1-gpu-large") @@ -43,32 +47,15 @@ PAGE_SIZE = 64 E2M1_MAX = 6.0 -def _ceil_ue8m0_exp_ref(x: torch.Tensor) -> torch.Tensor: - bits = x.to(torch.float32).contiguous().view(torch.int32) - exp = (bits >> 23) & 0xFF - mantissa = bits & 0x7FFFFF - exp = exp + (mantissa != 0).to(torch.int32) - return exp.clamp(1, 254) - - -def _fp4_e2m1_code_ref(x: torch.Tensor) -> torch.Tensor: - ax = torch.minimum(x.abs(), torch.tensor(E2M1_MAX, device=x.device)) - idx = torch.zeros_like(ax, dtype=torch.uint8) - for threshold in (0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0): - idx += (ax > threshold).to(torch.uint8) - sign = ((x < 0) & (idx != 0)).to(torch.uint8) * 8 - return idx | sign - - def _ref_quantize_fp4_indexer(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: x = x.contiguous().view(-1, HEAD_DIM).float() groups = x.view(-1, SCALE_GROUPS, GROUP_SIZE) scale_raw = (groups.abs().amax(dim=-1) / E2M1_MAX).clamp_min(1.0e-4) - scale_exp = _ceil_ue8m0_exp_ref(scale_raw) + scale_exp = ceil_ue8m0_exp_ref(scale_raw) scale = (scale_exp << 23).contiguous().view(torch.float32) scaled = (groups / scale.unsqueeze(-1)).view(-1, HEAD_DIM) - code = _fp4_e2m1_code_ref(scaled) + code = fp4_e2m1_code_ref(scaled) packed = (code[:, 0::2].to(torch.int16) | (code[:, 1::2].to(torch.int16) << 4)).to( torch.uint8 ) diff --git a/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py b/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py index 8a52a1b680..3192aea4da 100644 --- a/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py +++ b/test/registered/kernels/ops/attention/test_fp4_indexer_hip.py @@ -83,6 +83,10 @@ from sglang.kernels.ops.attention.dsv4.topk import ( from sglang.kernels.ops.gemm.router_gemv_hip import rocm_router_gemv_split_k from sglang.srt.utils import get_device, is_gfx95_supported, is_hip from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.kernels.deepseek_v4.common import ( + ceil_ue8m0_exp_ref, + fp4_e2m1_code_ref, +) register_amd_ci(est_time=60, suite="stage-b-test-1-gpu-small-amd-mi35x") @@ -112,23 +116,6 @@ SCALE_SHUFFLE_TILE = 16 _E2M1_GRID = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0] -def _ceil_ue8m0_exp_ref(x: torch.Tensor) -> torch.Tensor: - bits = x.to(torch.float32).contiguous().view(torch.int32) - exp = (bits >> 23) & 0xFF - mantissa = bits & 0x7FFFFF - exp = exp + (mantissa != 0).to(torch.int32) - return exp.clamp(1, 254) - - -def _fp4_e2m1_code_ref(x: torch.Tensor) -> torch.Tensor: - ax = torch.minimum(x.abs(), torch.tensor(E2M1_MAX, device=x.device)) - idx = torch.zeros_like(ax, dtype=torch.uint8) - for threshold in (0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0): - idx += (ax > threshold).to(torch.uint8) - sign = ((x < 0) & (idx != 0)).to(torch.uint8) * 8 - return idx | sign - - def _ref_quantize_fp4_indexer(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: """Nibble-packed E2M1 payload and per-group UE8M0 exponents. @@ -139,11 +126,11 @@ def _ref_quantize_fp4_indexer(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tens x = x.contiguous().view(-1, HEAD_DIM).float() groups = x.view(-1, SCALE_GROUPS, GROUP_SIZE) scale_raw = (groups.abs().amax(dim=-1) / E2M1_MAX).clamp_min(1.0e-4) - scale_exp = _ceil_ue8m0_exp_ref(scale_raw) + scale_exp = ceil_ue8m0_exp_ref(scale_raw) scale = (scale_exp << 23).contiguous().view(torch.float32) scaled = (groups / scale.unsqueeze(-1)).view(-1, HEAD_DIM) - code = _fp4_e2m1_code_ref(scaled) + code = fp4_e2m1_code_ref(scaled) packed = (code[:, 0::2].to(torch.int16) | (code[:, 1::2].to(torch.int16) << 4)).to( torch.uint8 ) diff --git a/test/registered/kernels/ops/attention/test_hy4_hpc_gated_mla.py b/test/registered/kernels/ops/attention/test_hy4_hpc_gated_mla.py index 15d4217611..d9ae4793e5 100644 --- a/test/registered/kernels/ops/attention/test_hy4_hpc_gated_mla.py +++ b/test/registered/kernels/ops/attention/test_hy4_hpc_gated_mla.py @@ -10,17 +10,9 @@ from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci( est_time=60, - nightly=False, - disabled=None, stage="base-b-kernel-unit", runner_config="4-gpu-b200", -) -register_cuda_ci( - est_time=120, - nightly=True, - disabled=None, - stage="nightly", - runner_config="4-gpu-b200", + disabled="needs HPC-Ops with hpc.gemm.gated_mla_gemm; not installed in CI", ) # Guards attention-TP shards from silently reverting to the global N=16384 shape. diff --git a/test/registered/kernels/ops/attention/test_kda_decode_mtp.py b/test/registered/kernels/ops/attention/test_kda_decode_mtp.py index d59955e7fe..cc94c19c70 100644 --- a/test/registered/kernels/ops/attention/test_kda_decode_mtp.py +++ b/test/registered/kernels/ops/attention/test_kda_decode_mtp.py @@ -4,9 +4,6 @@ from unittest import mock import torch from sglang.kernels.ops.attention import kda_decode_mtp -from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( - commit_kda_replayssm_spec, -) from sglang.kernels.ops.attention.kda_decode_mtp import fused_kda_decode_mtp_dspark from sglang.srt.utils import get_device_sm from sglang.test.ci.ci_register import register_cuda_ci @@ -112,93 +109,6 @@ class TestKdaDecodeMtp(CustomTestCase): if get_device_sm() < 100: raise unittest.SkipTest("Kimi K3 compute kernels require SM100a+") - def test_mtp_replayssm_ring(self): - num_requests, num_heads, num_spec, key_dim = 2, 2, 2, 128 - num_slots, ring_size = num_requests + 2, 16 - - def run(cache_ring): - kwargs = _make_inputs(num_requests, num_spec, key_dim, num_heads) - state = kwargs["recurrent_state"] - slots = kwargs["ssm_state_indices"] - scratch = kwargs["intermediate_state_indices"] - if not cache_ring: - intermediate = torch.zeros( - num_requests, - 1 + num_spec, - num_heads, - key_dim, - key_dim, - device="cuda", - ) - output = fused_kda_decode_mtp_dspark( - intermediate_ssm=intermediate, - **kwargs, - ) - return output, intermediate, slots, scratch - - raw_v = torch.zeros( - num_slots, - num_heads, - ring_size, - key_dim, - device="cuda", - dtype=torch.bfloat16, - ) - raw_k = torch.zeros_like(raw_v) - ring_gate = torch.zeros( - num_slots, - num_heads, - ring_size, - key_dim, - device="cuda", - ) - ring_beta = torch.zeros( - num_slots, - num_heads, - ring_size, - device="cuda", - ) - output = fused_kda_decode_mtp_dspark( - intermediate_ssm=None, - replayssm_rawv=raw_v, - replayssm_rawk=raw_k, - replayssm_g=ring_gate, - replayssm_beta=ring_beta, - **kwargs, - ) - return ( - output, - state, - slots, - (raw_v, raw_k, ring_gate, ring_beta), - ) - - baseline, intermediate, slots, scratch = run(cache_ring=False) - ring_output, checkpoint, ring_slots, rings = run(cache_ring=True) - self.assertTrue(torch.equal(ring_output, baseline)) - commit_kda_replayssm_spec( - checkpoint, - *rings, - ring_slots, - torch.full( - (num_requests,), - 1 + num_spec, - device="cuda", - dtype=torch.int32, - ), - max_cache_len=ring_size, - num_k_heads=num_heads, - use_qk_l2norm_in_kernel=True, - null_block_id=-1, - ) - for request in range(num_requests): - expected = intermediate[scratch[request], num_spec] - actual = checkpoint[slots[request]] - relative_error = ( - actual - expected - ).abs().max() / expected.abs().max().clamp_min(1e-6) - self.assertLess(relative_error.item(), 2e-2) - def _run(self, num_requests, num_spec, *, onorm, block_threads=None): kwargs = _make_inputs(num_requests, num_spec) _, num_tokens, num_heads, key_dim = kwargs["x_v"].shape diff --git a/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py b/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py index be694ad4e5..b3be0ef2d3 100644 --- a/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py +++ b/test/registered/kernels/ops/attention/test_q8kv8_sparse_prefill_backend.py @@ -16,11 +16,7 @@ from types import SimpleNamespace import pytest import torch -from sglang.kernels.ops.attention.dsv4.index_buf_accessor import SetKAndS from sglang.kernels.ops.attention.dsv4.kv_layout import KVLayout -from sglang.kernels.ops.attention.dsv4.quant_k_cache import ( - quant_to_nope_fp8_rope_bf16_pack_triton, -) from sglang.kernels.ops.attention.sparse_mla_q8kv8_prefill_sm90 import ( sparse_mla_q8kv8_prefill_fwd, ) @@ -34,6 +30,10 @@ from sglang.srt.layers.attention.dsv4.sparse_prefill_utils import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.utils import is_sm90_supported from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kernels.deepseek_v4.k_cache_reference import ( + quant_to_nope_fp8_rope_bf16_pack_triton, + set_k_and_s_torch, +) register_cuda_ci(est_time=120, stage="base-b-kernel-unit", runner_config="1-gpu-large") @@ -130,7 +130,7 @@ def _make_v4_paged_kv_cache( k_bf16 = (torch.randn(total_slots, 512, device=device) * 0.25).to(torch.bfloat16) pack = quant_to_nope_fp8_rope_bf16_pack_triton(k_bf16) loc = torch.arange(total_slots, dtype=torch.int32, device=device) - SetKAndS.torch(_Pool(page_size), quant_k_cache, loc, pack) + set_k_and_s_torch(quant_k_cache, loc, pack, page_size) return quant_k_cache diff --git a/test/registered/kernels/ops/attention/test_replayssm_ring_fold.py b/test/registered/kernels/ops/attention/test_replayssm_ring_fold.py deleted file mode 100644 index 716b3eb1c7..0000000000 --- a/test/registered/kernels/ops/attention/test_replayssm_ring_fold.py +++ /dev/null @@ -1,142 +0,0 @@ -"""ReplaySSM ring folding parity.""" - -import unittest - -import torch - -from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( - fused_sigmoid_gating_delta_rule_update, -) -from sglang.kernels.ops.attention.fla.kda_replayssm_spec_decode import ( - commit_kda_replayssm_spec, -) -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.test_utils import CustomTestCase - -register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") - - -class TestReplaySSMRingFold(CustomTestCase): - def test_replayssm_ring_fold(self): - batch_size, num_steps = 8, 4 - num_value_heads, num_key_heads = 8, 2 - key_dim = value_dim = 128 - ring_size = 16 - torch.manual_seed(6) - - q = torch.randn( - batch_size, - num_steps, - num_key_heads, - key_dim, - device="cuda", - ) - k = torch.randn_like(q) - v = torch.randn( - batch_size, - num_steps, - num_value_heads, - value_dim, - device="cuda", - ) - a = torch.randn( - batch_size, - num_steps, - num_value_heads, - key_dim, - device="cuda", - ) - b = torch.randn(batch_size, num_steps, num_value_heads, device="cuda") - a_log = torch.randn(num_value_heads, device="cuda") - dt_bias = torch.randn(num_value_heads, key_dim, device="cuda") - slots = torch.arange(1, batch_size + 1, device="cuda", dtype=torch.int32) - slots[-1] = -1 - num_slots = batch_size + 1 - state = torch.randn( - num_slots, - num_value_heads, - value_dim, - key_dim, - device="cuda", - ) - intermediate = torch.zeros( - num_slots, - num_steps, - num_value_heads, - value_dim, - key_dim, - device="cuda", - ) - raw_v = torch.zeros( - num_slots, - num_value_heads, - ring_size, - value_dim, - device="cuda", - ) - raw_k = torch.zeros( - num_slots, - num_key_heads, - ring_size, - key_dim, - device="cuda", - ) - gate = torch.zeros_like(raw_v) - beta = torch.zeros( - num_slots, - num_value_heads, - ring_size, - device="cuda", - ) - - fused_sigmoid_gating_delta_rule_update( - A_log=a_log, - a=a, - dt_bias=dt_bias, - softplus_beta=1.0, - softplus_threshold=20.0, - q=q, - k=k, - v=v, - b=b, - initial_state_source=state, - initial_state_indices=slots, - scale=key_dim**-0.5, - use_qk_l2norm_in_kernel=True, - is_kda=True, - lower_bound=-5.0, - disable_state_update=True, - intermediate_states_buffer=intermediate, - intermediate_state_indices=slots, - cache_steps=num_steps, - cache_ring=True, - replayssm_rawv=raw_v, - replayssm_rawk=raw_k, - replayssm_g=gate, - replayssm_beta=beta, - ) - checkpoint = state.clone() - commit_kda_replayssm_spec( - checkpoint, - raw_v, - raw_k, - gate, - beta, - slots, - torch.full((batch_size,), num_steps, device="cuda", dtype=torch.int32), - max_cache_len=ring_size, - num_k_heads=num_key_heads, - use_qk_l2norm_in_kernel=True, - null_block_id=-1, - ) - for slot in slots[:-1].tolist(): - expected = intermediate[slot, num_steps - 1] - actual = checkpoint[slot] - relative_error = ( - actual - expected - ).abs().max() / expected.abs().max().clamp_min(1e-6) - self.assertLess(relative_error.item(), 1e-3) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/kernels/ops/diffusion/test_fast_launch.py b/test/registered/kernels/ops/diffusion/test_fast_launch.py index 4262c83352..7e983c14c4 100644 --- a/test/registered/kernels/ops/diffusion/test_fast_launch.py +++ b/test/registered/kernels/ops/diffusion/test_fast_launch.py @@ -175,7 +175,10 @@ def _float(n=4096, offset=0): @torch.no_grad() -def test_toy_wrapper_replays(): +@pytest.mark.parametrize("has_launch_hooks", [True, False]) +def test_toy_wrapper_replays(monkeypatch, has_launch_hooks): + if not has_launch_hooks: + monkeypatch.setattr(fast_launch, "knobs", None) toy = _Counted(_copy_out) for _ in range(3): x = _float() diff --git a/test/registered/kernels/ops/diffusion/test_h3_vae_fused_norm.py b/test/registered/kernels/ops/diffusion/test_h3_vae_fused_norm.py index b7315f8a13..ab94b117f4 100644 --- a/test/registered/kernels/ops/diffusion/test_h3_vae_fused_norm.py +++ b/test/registered/kernels/ops/diffusion/test_h3_vae_fused_norm.py @@ -1,4 +1,4 @@ -"""MiniMax-H3 video VAE fused RMSNorm and Q/K RoPE.""" +"""MiniMax-H3 video VAE fused RMSNorm.""" import sys @@ -6,14 +6,7 @@ import pytest import torch import torch.nn.functional as F -from sglang.kernels.ops.diffusion import ( - h3_vae_qk_rmsnorm_rope, - h3_vae_rmsnorm, - h3_vae_scale_add_rmsnorm, -) -from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.vit_utils import ( - _apply_rotary_pos_emb_impl, -) +from sglang.kernels.ops.diffusion import h3_vae_rmsnorm, h3_vae_scale_add_rmsnorm from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large") @@ -52,28 +45,5 @@ def test_scale_add_residual_matches_eager_product(): assert fused_residual.data_ptr() == residual.data_ptr() -@torch.inference_mode() -def test_qk_rmsnorm_rope_matches_eager_neox(): - batch, seq_len, heads = 2, 17, 4 - packed = torch.randn(batch, seq_len, heads, 192, device="cuda", dtype=torch.float16) - query, key, _ = packed.chunk(3, dim=-1) - half_cos = torch.randn(seq_len, 24, device="cuda", dtype=torch.float16) - half_sin = torch.randn(seq_len, 24, device="cuda", dtype=torch.float16) - cos = torch.cat((half_cos, half_cos), dim=-1) - sin = torch.cat((half_sin, half_sin), dim=-1) - actual_q, actual_k = h3_vae_qk_rmsnorm_rope(query, key, 1e-5, cos, sin) - norm = torch.nn.RMSNorm(64, eps=1e-5, elementwise_affine=False).cuda() - rotary = ( - cos.view(1, seq_len, 1, 48).expand(batch, -1, -1, -1), - sin.view(1, seq_len, 1, 48).expand(batch, -1, -1, -1), - ) - with torch.autocast("cuda", enabled=False): - expected_q = _apply_rotary_pos_emb_impl(norm(query), rotary) - expected_k = _apply_rotary_pos_emb_impl(norm(key), rotary) - torch.testing.assert_close(actual_q, expected_q, rtol=1e-3, atol=1e-2) - torch.testing.assert_close(actual_k, expected_k, rtol=1e-3, atol=1e-2) - assert h3_vae_qk_rmsnorm_rope(query.float(), key.float(), 1e-5, cos, sin) is None - - if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/test/registered/kernels/ops/diffusion/test_longcat_image_norm_modulate.py b/test/registered/kernels/ops/diffusion/test_longcat_image_norm_modulate.py index fd22bbd90a..556378d49b 100644 --- a/test/registered/kernels/ops/diffusion/test_longcat_image_norm_modulate.py +++ b/test/registered/kernels/ops/diffusion/test_longcat_image_norm_modulate.py @@ -70,7 +70,9 @@ class TestLongCatNormModulation(CustomTestCase): self.assertTrue(longcat._LONGCAT_LN_MOD.verified) leaves = [t.detach().clone().requires_grad_() for t in (x, scale, shift)] refs = [t.detach().clone().requires_grad_() for t in leaves] - with patch.object(longcat.diffusion_ops, "fused_layernorm_modulate") as fused: + with patch.object( + longcat.diffusion_ops, "fused_layernorm_modulate_raw" + ) as fused: actual = longcat._longcat_norm_modulate(norm, *leaves) actual.float().sum().backward() fused.assert_not_called() @@ -100,7 +102,9 @@ class TestLongCatNormModulation(CustomTestCase): def test_unverified_capture_uses_eager_reference(self): norm, x, scale, shift = self.inputs(seq=17) expected = modulate_scale_shift(norm(x), scale, shift) - with patch.object(longcat.diffusion_ops, "fused_layernorm_modulate") as fused: + with patch.object( + longcat.diffusion_ops, "fused_layernorm_modulate_raw" + ) as fused: graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): actual = longcat._longcat_norm_modulate(norm, x, scale, shift) @@ -115,13 +119,15 @@ class TestLongCatNormModulation(CustomTestCase): expected = norm(x) * (1 + scale[:, None]) + shift[:, None] with patch.object( longcat.diffusion_ops, - "fused_layernorm_modulate", + "fused_layernorm_modulate_raw", return_value=torch.zeros_like(x), ): actual = longcat._longcat_norm_modulate(norm, x, scale, shift) self.assertTrue(longcat._LONGCAT_LN_MOD.disabled) self.assertTrue(torch.equal(actual, expected)) - with patch.object(longcat.diffusion_ops, "fused_layernorm_modulate") as fused: + with patch.object( + longcat.diffusion_ops, "fused_layernorm_modulate_raw" + ) as fused: actual = longcat._longcat_norm_modulate(norm, x, scale, shift) fused.assert_not_called() self.assertTrue(torch.equal(actual, expected)) diff --git a/test/registered/kernels/ops/diffusion/test_model_fast_paths.py b/test/registered/kernels/ops/diffusion/test_model_fast_paths.py index b85aecb3ce..b9da88d7c0 100644 --- a/test/registered/kernels/ops/diffusion/test_model_fast_paths.py +++ b/test/registered/kernels/ops/diffusion/test_model_fast_paths.py @@ -41,25 +41,16 @@ import sglang.multimodal_gen.runtime.models.dits.qwen_image as qwen_image import sglang.multimodal_gen.runtime.models.dits.qwen_image21 as qwen_image21 import sglang.multimodal_gen.runtime.models.dits.sana as sana from sglang.kernels.ops.diffusion import ( + FUSED_LN_MODULATE, + HUNYUAN_QKNORM, + LTX2_QKNORM_SPLIT_ROPE, + LTX2_RMS_NORM_MODULATE, + QWEN_IMAGE_ADDED_QKV, BitExactFusionGate, can_use_wan_rmsnorm_silu, fused_ltx2_rms_norm_modulate, hunyuan_qkv_rope_pack, - mark_fused_ln_modulate_site, - mark_hunyuan_qknorm_site, - mark_ltx2_qknorm_split_rope_site, - mark_ltx2_rms_norm_modulate_site, - mark_qwen_image_added_qkv_site, - mount_fused_ln_modulate, - mount_hunyuan_qknorm, - mount_ltx2_qknorm_split_rope, - mount_ltx2_rms_norm_modulate, - mount_qwen_image_added_qkv, try_flux2_token_cat_nvfp4, - unmount_hunyuan_qknorm, - unmount_ltx2_qknorm_split_rope, - unmount_ltx2_rms_norm_modulate, - unmount_qwen_image_added_qkv, wan_rmsnorm_silu, ) from sglang.kernels.ops.diffusion.common.platform import is_cuda @@ -379,8 +370,8 @@ def test_flux_norm_modulate_bitexact_supersedes_high_fold(): # With the quality="high" affine fold mounted, the bit-exact kernel # still takes priority, so the site output stays lossless. site = torch.nn.Module() - mark_fused_ln_modulate_site(site) - assert mount_fused_ln_modulate(site) + FUSED_LN_MODULATE.mark(site) + assert FUSED_LN_MODULATE.mount(site) norm, x, shift, scale = _flux_site_inputs((1, 128, 3072), 6, seed=1) out = _flux_norm_modulate(site, norm, x, scale, shift) assert torch.equal(out, _flux_eager(norm, x, scale, shift)) @@ -571,7 +562,7 @@ def _added_qkv_attention(packed): attention.use_fused_added_qkv = True attention._unquantized_added_qkv_is_packed = True attention.to_added_qkv = packed - mark_qwen_image_added_qkv_site(attention) + QWEN_IMAGE_ADDED_QKV.mark(attention) return attention @@ -606,7 +597,7 @@ def test_qwen_added_qkv_lossless_uses_three_reference_gemms(): for actual, expected in zip(actual_lossless, expected_lossless) ) - assert mount_qwen_image_added_qkv(attention) + assert QWEN_IMAGE_ADDED_QKV.mount(attention) actual_high = attention._get_added_qkv_projections(x) expected_high = tuple( tensor.contiguous() @@ -618,7 +609,7 @@ def test_qwen_added_qkv_lossless_uses_three_reference_gemms(): for actual, expected in zip(actual_high, expected_high) ) - unmount_qwen_image_added_qkv(attention) + QWEN_IMAGE_ADDED_QKV.unmount(attention) attention._get_added_qkv_projections(x) assert packed.calls == 1 @@ -996,12 +987,12 @@ def test_longcat_qknorm_rope_is_bit_exact(): def test_ltx2_qknorm_split_rope_hopper_site_is_quality_gated(): attention = nn.Module() - mark_ltx2_qknorm_split_rope_site(attention) - assert not ltx2_module.ltx2_qknorm_split_rope_active(attention) - assert mount_ltx2_qknorm_split_rope(attention) - assert ltx2_module.ltx2_qknorm_split_rope_active(attention) - unmount_ltx2_qknorm_split_rope(attention) - assert not ltx2_module.ltx2_qknorm_split_rope_active(attention) + LTX2_QKNORM_SPLIT_ROPE.mark(attention) + assert not LTX2_QKNORM_SPLIT_ROPE.is_enabled(attention) + assert LTX2_QKNORM_SPLIT_ROPE.mount(attention) + assert LTX2_QKNORM_SPLIT_ROPE.is_enabled(attention) + LTX2_QKNORM_SPLIT_ROPE.unmount(attention) + assert not LTX2_QKNORM_SPLIT_ROPE.is_enabled(attention) def _ltx2_eager(rms, x, scale, shift, eps): @@ -1022,7 +1013,7 @@ def test_ltx2_lossless_default_is_bitexact(hidden): # A marked-but-unmounted site uses only the self-verified bit-exact # modulate fast path after the reference aten RMSNorm. block = nn.Module() - mark_ltx2_rms_norm_modulate_site(block) + LTX2_RMS_NORM_MODULATE.mark(block) rms, x, scale, shift = _ltx2_inputs(hidden) out = _ltx2_rms_norm_modulate(block, rms, x, scale, shift, 1e-6) assert torch.equal(out, _ltx2_eager(rms, x, scale, shift, 1e-6)) @@ -1030,7 +1021,7 @@ def test_ltx2_lossless_default_is_bitexact(hidden): def test_ltx2_lossless_compile_keeps_expression_visible_to_inductor(monkeypatch): block = nn.Module() - mark_ltx2_rms_norm_modulate_site(block) + LTX2_RMS_NORM_MODULATE.mark(block) rms, x, scale, shift = _ltx2_inputs(2048, seq=126) monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True) monkeypatch.setattr( @@ -1047,8 +1038,8 @@ def test_ltx2_lossless_compile_keeps_expression_visible_to_inductor(monkeypatch) @pytest.mark.parametrize("hidden", [4096, 2048]) def test_ltx2_mounted_high_uses_fused_kernel(hidden): block = nn.Module() - mark_ltx2_rms_norm_modulate_site(block) - assert mount_ltx2_rms_norm_modulate(block) + LTX2_RMS_NORM_MODULATE.mark(block) + assert LTX2_RMS_NORM_MODULATE.mount(block) try: rms, x, scale, shift = _ltx2_inputs(hidden) out = _ltx2_rms_norm_modulate(block, rms, x, scale, shift, 1e-6) @@ -1058,7 +1049,7 @@ def test_ltx2_mounted_high_uses_fused_kernel(hidden): ref = _ltx2_eager(rms, x, scale, shift, 1e-6) assert torch.allclose(out.float(), ref.float(), atol=3e-2, rtol=1e-2) finally: - unmount_ltx2_rms_norm_modulate(block) + LTX2_RMS_NORM_MODULATE.unmount(block) # ------------------------------------------------------------------------- @@ -1139,7 +1130,7 @@ def test_hunyuan_qkv_rope_pack_uses_int64_row_offsets(): def test_hunyuan_quality_qknorm_matches_rmsnorm(): torch.manual_seed(1) site = torch.nn.Module() - mark_hunyuan_qknorm_site(site) + HUNYUAN_QKNORM.mark(site) q_norm = RMSNorm(128, eps=1e-6).to(device="cuda", dtype=torch.bfloat16) k_norm = RMSNorm(128, eps=1e-6).to(device="cuda", dtype=torch.bfloat16) packed = torch.randn(1, 257, 3, 24, 128, device="cuda", dtype=torch.bfloat16) @@ -1151,12 +1142,12 @@ def test_hunyuan_quality_qknorm_matches_rmsnorm(): assert torch.equal(q_unmounted, q_ref) assert torch.equal(k_unmounted, k_ref) - assert mount_hunyuan_qknorm(site) + assert HUNYUAN_QKNORM.mount(site) q_out, k_out = _hunyuan_qknorm(site, q, k, q_norm, k_norm) torch.testing.assert_close(q_out, q_ref, atol=2e-2, rtol=2e-2) torch.testing.assert_close(k_out, k_ref, atol=2e-2, rtol=2e-2) - unmount_hunyuan_qknorm(site) + HUNYUAN_QKNORM.unmount(site) q_unmounted, k_unmounted = _hunyuan_qknorm(site, q, k, q_norm, k_norm) assert torch.equal(q_unmounted, q_ref) assert torch.equal(k_unmounted, k_ref) @@ -1164,12 +1155,12 @@ def test_hunyuan_quality_qknorm_matches_rmsnorm(): def test_hunyuan_quality_qknorm_stays_unmounted_without_cute_kernel(): site = torch.nn.Module() - mark_hunyuan_qknorm_site(site) + HUNYUAN_QKNORM.mark(site) with patch.object(hunyuan_qknorm, "_get_qk_rmsnorm_cute", return_value=None): - assert not mount_hunyuan_qknorm(site) + assert not HUNYUAN_QKNORM.mount(site) - assert not hunyuan_qknorm._FUSION.is_enabled(site) + assert not HUNYUAN_QKNORM.is_enabled(site) # ------------------------------------------------------------------------- diff --git a/test/registered/kernels/ops/diffusion/test_sites.py b/test/registered/kernels/ops/diffusion/test_sites.py index 757e25ce10..0e14388787 100644 --- a/test/registered/kernels/ops/diffusion/test_sites.py +++ b/test/registered/kernels/ops/diffusion/test_sites.py @@ -31,20 +31,14 @@ import sglang.kernels.ops.diffusion.sites.lingbot_video_rmsnorm_site as lingbot_ import sglang.kernels.ops.diffusion.sites.qwen_image_added_qkv_site as qwen_image_added_qkv import sglang.kernels.ops.diffusion.sites.sana_video_linear_attention_site as sana_video_linear_attention from sglang.kernels.ops.diffusion import ( + FLUX2_NVFP4_SWIGLU_QUANT, + FUSED_LN_MODULATE, BitExactFusionGate, QualityGatedFusion, can_use_ln_modulate, flashinfer_rmsnorm_diagnostic_hint, - flux2_nvfp4_swiglu_quant_active, fused_ln_modulate, - fused_ln_modulate_active, - mark_flux2_nvfp4_swiglu_quant_site, - mark_fused_ln_modulate_site, - mount_flux2_nvfp4_swiglu_quant, - mount_fused_ln_modulate, tensors_equal, - unmount_flux2_nvfp4_swiglu_quant, - unmount_fused_ln_modulate, ) from sglang.test.ci.ci_register import register_cpu_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase @@ -84,14 +78,13 @@ def test_quality_gate_rejection_is_all_or_nothing(): name="test fusion", marker_attr="_test_fusion_site", enabled_attr="_test_fusion_enabled", + reject_reason=lambda site: "rejected" if fusion.metadata(site) else None, ) root = nn.ModuleList([nn.Module(), nn.Module()]) for index, site in enumerate(root): fusion.mark(site, index) - assert not fusion.mount( - root, reject_reason=lambda site: "rejected" if fusion.metadata(site) else None - ) + assert not fusion.mount(root) assert not any(fusion.is_enabled(site) for site in root) assert not fusion.mount(nn.Module()) @@ -99,13 +92,13 @@ def test_quality_gate_rejection_is_all_or_nothing(): def test_flux2_nvfp4_swiglu_quant_is_disabled_until_quality_gate_mounts(): site = nn.Module() root = nn.ModuleList([site]) - mark_flux2_nvfp4_swiglu_quant_site(site) + FLUX2_NVFP4_SWIGLU_QUANT.mark(site) - assert not flux2_nvfp4_swiglu_quant_active(site) - assert mount_flux2_nvfp4_swiglu_quant(root) - assert flux2_nvfp4_swiglu_quant_active(site) - unmount_flux2_nvfp4_swiglu_quant(root) - assert not flux2_nvfp4_swiglu_quant_active(site) + assert not FLUX2_NVFP4_SWIGLU_QUANT.is_enabled(site) + assert FLUX2_NVFP4_SWIGLU_QUANT.mount(root) + assert FLUX2_NVFP4_SWIGLU_QUANT.is_enabled(site) + FLUX2_NVFP4_SWIGLU_QUANT.unmount(root) + assert not FLUX2_NVFP4_SWIGLU_QUANT.is_enabled(site) def test_qwen_image_added_qkv_site_is_request_scoped(): @@ -113,13 +106,13 @@ def test_qwen_image_added_qkv_site_is_request_scoped(): site.to_added_qkv = nn.Module() site.to_added_qkv.quant_config = None site.to_added_qkv.output_partition_sizes = [8, 8, 8] - qwen_image_added_qkv.mark_qwen_image_added_qkv_site(site) + qwen_image_added_qkv.QWEN_IMAGE_ADDED_QKV.mark(site) - assert not qwen_image_added_qkv.qwen_image_added_qkv_active(site) - assert qwen_image_added_qkv.mount_qwen_image_added_qkv(site) - assert qwen_image_added_qkv.qwen_image_added_qkv_active(site) - qwen_image_added_qkv.unmount_qwen_image_added_qkv(site) - assert not qwen_image_added_qkv.qwen_image_added_qkv_active(site) + assert not qwen_image_added_qkv.QWEN_IMAGE_ADDED_QKV.is_enabled(site) + assert qwen_image_added_qkv.QWEN_IMAGE_ADDED_QKV.mount(site) + assert qwen_image_added_qkv.QWEN_IMAGE_ADDED_QKV.is_enabled(site) + qwen_image_added_qkv.QWEN_IMAGE_ADDED_QKV.unmount(site) + assert not qwen_image_added_qkv.QWEN_IMAGE_ADDED_QKV.is_enabled(site) # ------------------------------------------------------------------------- @@ -151,7 +144,6 @@ def test_bitexact_gate_mismatch_disables_permanently(): out = gate.accept_or_fallback( torch.tensor([1.0]), torch.tensor([2.0]), - mismatch_msg="mismatch", ) assert torch.equal(out, torch.tensor([2.0])) assert gate.disabled and not gate.verified @@ -180,6 +172,45 @@ def test_bitexact_gate_skips_first_sight_during_graph_capture(monkeypatch): assert gate.can_attempt_once() +def test_bitexact_gate_run_verifies_each_signature_outside_capture(monkeypatch): + # A new signature must not verify (eager reference + host sync) inside + # capture, while a verified one replays the fused kernel alone there. + gate = BitExactFusionGate("run", per_signature=True) + calls = [] + + def fused(): + calls.append("fused") + return torch.tensor([1.0]) + + def ref(): + calls.append("ref") + return torch.tensor([1.0]) + + def capturing(active): + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: active) + calls.clear() + + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + capturing(True) + gate.run(fused, ref, sig="a") + assert calls == ["ref"] and not gate.is_verified("a") + capturing(False) + gate.run(fused, ref, sig="a") + assert calls == ["fused", "ref"] and gate.is_verified("a") + capturing(True) + gate.run(fused, ref, sig="a") + assert calls == ["fused"] + gate.run(fused, ref, sig="b") + assert calls == ["fused", "ref"] and not gate.is_verified("b") + + def invalid(): + raise ValueError("bad operands") + + with pytest.raises(ValueError): + gate.run(invalid, ref, sig="a", reraise=(ValueError,)) + assert not gate.disabled + + def test_tensors_equal_supports_sequences(): assert tensors_equal( (torch.tensor([1.0]), torch.tensor([2.0])), @@ -299,13 +330,13 @@ def test_fused_ln_modulate_guards_and_mount_protocol(): root = nn.Module() root.child = nn.Module() - mark_fused_ln_modulate_site(root.child) - assert not fused_ln_modulate_active(root.child) - assert mount_fused_ln_modulate(root) - assert fused_ln_modulate_active(root.child) - unmount_fused_ln_modulate(root) - assert not fused_ln_modulate_active(root.child) - assert not mount_fused_ln_modulate(nn.Module()) # no marked sites + FUSED_LN_MODULATE.mark(root.child) + assert not FUSED_LN_MODULATE.is_enabled(root.child) + assert FUSED_LN_MODULATE.mount(root) + assert FUSED_LN_MODULATE.is_enabled(root.child) + FUSED_LN_MODULATE.unmount(root) + assert not FUSED_LN_MODULATE.is_enabled(root.child) + assert not FUSED_LN_MODULATE.mount(nn.Module()) # no marked sites @requires_cuda @@ -314,10 +345,12 @@ def test_mounted_ln_modulate_site_compiles_fullgraph(): class Site(nn.Module): def __init__(self): super().__init__() - mark_fused_ln_modulate_site(self) + FUSED_LN_MODULATE.mark(self) def forward(self, x, scale, shift): - if fused_ln_modulate_active(self) and can_use_ln_modulate(x, scale, shift): + if FUSED_LN_MODULATE.is_enabled(self) and can_use_ln_modulate( + x, scale, shift + ): return fused_ln_modulate(x, scale, shift, eps=1e-6) return ( F.layer_norm(x, (x.shape[-1],), eps=1e-6) * (1 + scale[:, None]) @@ -325,7 +358,7 @@ def test_mounted_ln_modulate_site_compiles_fullgraph(): ) site = Site() - assert mount_fused_ln_modulate(site) + assert FUSED_LN_MODULATE.mount(site) x = torch.randn(1, 64, 128, device="cuda", dtype=torch.bfloat16) scale = torch.randn(1, 128, device="cuda", dtype=torch.bfloat16) shift = torch.randn_like(scale) @@ -347,7 +380,7 @@ class _GateRMSNormSite(nn.Module): self.norm = nn.RMSNorm( GATE_RMSNORM_DIM, eps=GATE_RMSNORM_EPS, device="cuda", dtype=dtype ) - gate_rmsnorm.mark_fused_gate_rmsnorm_site(self, ("norm",)) + gate_rmsnorm.FUSED_GATE_RMSNORM.mark(self, ("norm",)) @requires_cuda @@ -362,7 +395,7 @@ def test_fused_gate_rmsnorm_matches_ideogram_reference(): mods = torch.randn(1, 1, 2 * dim, device="cuda", dtype=torch.bfloat16) scale, gate = mods.chunk(2, dim=-1) - assert gate_rmsnorm.mount_fused_gate_rmsnorm(site) + assert gate_rmsnorm.FUSED_GATE_RMSNORM.mount(site) got_scale = gate_rmsnorm.fused_rmsnorm_scale(x, w, 1.0 + scale, GATE_RMSNORM_EPS) got_gate = gate_rmsnorm.fused_rmsnorm_tanh_residual( x, gate, residual, w, GATE_RMSNORM_EPS @@ -379,23 +412,23 @@ def test_fused_gate_rmsnorm_matches_ideogram_reference(): def test_fused_gate_rmsnorm_mount_is_all_or_nothing(): good, bad = _GateRMSNormSite(), _GateRMSNormSite(torch.float32) # One fp32 norm anywhere in the tree keeps *every* site on the reference. - assert not gate_rmsnorm.mount_fused_gate_rmsnorm(nn.ModuleList([good, bad])) - assert not gate_rmsnorm.fused_gate_rmsnorm_active(good) - assert gate_rmsnorm.mount_fused_gate_rmsnorm(good) - gate_rmsnorm.unmount_fused_gate_rmsnorm(good) - assert not gate_rmsnorm.fused_gate_rmsnorm_active(good) + assert not gate_rmsnorm.FUSED_GATE_RMSNORM.mount(nn.ModuleList([good, bad])) + assert not gate_rmsnorm.FUSED_GATE_RMSNORM.is_enabled(good) + assert gate_rmsnorm.FUSED_GATE_RMSNORM.mount(good) + gate_rmsnorm.FUSED_GATE_RMSNORM.unmount(good) + assert not gate_rmsnorm.FUSED_GATE_RMSNORM.is_enabled(good) class _GeluSite(nn.Module): def __init__(self, dtype=torch.bfloat16, bias=True): super().__init__() self.proj = nn.Linear(64, 256, bias=bias, device="cuda", dtype=dtype) - linear_gelu.mark_fused_gelu_site(self, "proj") + linear_gelu.FUSED_LINEAR_GELU.mark(self, "proj") def forward(self, x): - if linear_gelu.fused_gelu_active(self) and linear_gelu.can_use_linear_gelu( - self.proj, x - ): + if linear_gelu.FUSED_LINEAR_GELU.is_enabled( + self + ) and linear_gelu.can_use_linear_gelu(self.proj, x): return linear_gelu.fused_linear_gelu_tanh( x, self.proj.weight, self.proj.bias ) @@ -409,7 +442,7 @@ def test_fused_linear_gelu_matches_reference(dtype): site = _GeluSite(dtype) x = torch.randn(512, 64, device="cuda", dtype=dtype) ref = site(x) # unmounted: the reference chain - assert linear_gelu.mount_fused_linear_gelu(site) + assert linear_gelu.FUSED_LINEAR_GELU.mount(site) atol = 2e-2 if dtype == torch.bfloat16 else 4e-3 torch.testing.assert_close(site(x), ref, atol=atol, rtol=2e-2) @@ -418,13 +451,13 @@ def test_fused_linear_gelu_matches_reference(dtype): def test_fused_linear_gelu_guards_and_lossless_path(): torch.manual_seed(0) good, bad = _GeluSite(), _GeluSite(torch.float32) - assert not linear_gelu.mount_fused_linear_gelu(nn.ModuleList([good, bad])) - assert not linear_gelu.fused_gelu_active(good) + assert not linear_gelu.FUSED_LINEAR_GELU.mount(nn.ModuleList([good, bad])) + assert not linear_gelu.FUSED_LINEAR_GELU.is_enabled(good) x = torch.randn(16, 64, device="cuda", dtype=torch.bfloat16) ref = good(x) - assert linear_gelu.mount_fused_linear_gelu(good) - linear_gelu.unmount_fused_linear_gelu(good) + assert linear_gelu.FUSED_LINEAR_GELU.mount(good) + linear_gelu.FUSED_LINEAR_GELU.unmount(good) # Unmounting must restore the reference path bit-for-bit. assert torch.equal(good(x), ref) @@ -441,7 +474,7 @@ def test_linear_gelu_verdict_is_reused_until_remount(monkeypatch): ) site = _GeluSite() x = torch.randn(16, 64, device="cuda", dtype=torch.bfloat16) - assert linear_gelu.mount_fused_linear_gelu(site) + assert linear_gelu.FUSED_LINEAR_GELU.mount(site) reads = [] static = linear_gelu.can_use_linear_gelu_static monkeypatch.setattr( @@ -452,8 +485,8 @@ def test_linear_gelu_verdict_is_reused_until_remount(monkeypatch): for _ in range(3): assert linear_gelu.can_use_linear_gelu(site.proj, x) assert reads == [site.proj] - linear_gelu.unmount_fused_linear_gelu(site) - assert linear_gelu.mount_fused_linear_gelu(site) + linear_gelu.FUSED_LINEAR_GELU.unmount(site) + assert linear_gelu.FUSED_LINEAR_GELU.mount(site) assert linear_gelu.can_use_linear_gelu(site.proj, x) assert reads == [site.proj, site.proj] @@ -463,7 +496,7 @@ def test_linear_gelu_verdict_is_reused_until_remount(monkeypatch): def test_mounted_gelu_site_compiles_fullgraph(): site = _GeluSite() x = torch.randn(16, 64, device="cuda", dtype=torch.bfloat16) - assert linear_gelu.mount_fused_linear_gelu(site) + assert linear_gelu.FUSED_LINEAR_GELU.mount(site) expected = site(x) torch.testing.assert_close( torch.compile(site, fullgraph=True)(x), expected, atol=2e-2, rtol=2e-2 @@ -475,7 +508,7 @@ def test_mounted_gelu_site_compiles_fullgraph(): def test_sana_video_linear_attention_quality_path_and_guards(): torch.manual_seed(0) site = nn.Module() - sana_video_linear_attention.mark_sana_video_linear_attention_site(site) + sana_video_linear_attention.SANA_VIDEO_LINEAR_ATTENTION.mark(site) shape = (1, 4, 16, 128) query = torch.randn(shape, device="cuda", dtype=torch.bfloat16) key = torch.randn_like(query) @@ -490,7 +523,7 @@ def test_sana_video_linear_attention_quality_path_and_guards(): ) is None ) - assert sana_video_linear_attention.mount_sana_video_linear_attention(site) + assert sana_video_linear_attention.SANA_VIDEO_LINEAR_ATTENTION.mount(site) output = sana_video_linear_attention.try_sana_video_linear_attention( site, query, key, value, normalizer ) @@ -509,8 +542,8 @@ def test_sana_video_linear_attention_quality_path_and_guards(): ) is None ) - sana_video_linear_attention.unmount_sana_video_linear_attention(site) - assert not sana_video_linear_attention.sana_video_linear_attention_active(site) + sana_video_linear_attention.SANA_VIDEO_LINEAR_ATTENTION.unmount(site) + assert not sana_video_linear_attention.SANA_VIDEO_LINEAR_ATTENTION.is_enabled(site) @requires_cuda @@ -519,7 +552,7 @@ def test_lingbot_video_rmsnorm_quality_path_and_guards(): torch.manual_seed(1) site = nn.Module() site.weight = nn.Parameter(torch.randn(2048, device="cuda", dtype=torch.float32)) - lingbot_video_rmsnorm.mark_lingbot_video_rmsnorm_site(site) + lingbot_video_rmsnorm.LINGBOT_VIDEO_RMSNORM.mark(site) hidden_states = torch.randn(1, 128, 2048, device="cuda", dtype=torch.bfloat16) assert ( @@ -528,7 +561,7 @@ def test_lingbot_video_rmsnorm_quality_path_and_guards(): ) is None ) - assert lingbot_video_rmsnorm.mount_lingbot_video_rmsnorm(site) + assert lingbot_video_rmsnorm.LINGBOT_VIDEO_RMSNORM.mount(site) output = lingbot_video_rmsnorm.try_lingbot_video_rmsnorm( site, hidden_states, site.weight, 1e-6 ) @@ -542,8 +575,8 @@ def test_lingbot_video_rmsnorm_quality_path_and_guards(): bf16_site.weight = nn.Parameter( torch.randn(2048, device="cuda", dtype=torch.bfloat16) ) - lingbot_video_rmsnorm.mark_lingbot_video_rmsnorm_site(bf16_site) - assert lingbot_video_rmsnorm.mount_lingbot_video_rmsnorm(bf16_site) + lingbot_video_rmsnorm.LINGBOT_VIDEO_RMSNORM.mark(bf16_site) + assert lingbot_video_rmsnorm.LINGBOT_VIDEO_RMSNORM.mount(bf16_site) bf16_output = lingbot_video_rmsnorm.try_lingbot_video_rmsnorm( bf16_site, hidden_states, bf16_site.weight, 1e-6 ) @@ -556,8 +589,8 @@ def test_lingbot_video_rmsnorm_quality_path_and_guards(): ) is None ) - lingbot_video_rmsnorm.unmount_lingbot_video_rmsnorm(site) - assert not lingbot_video_rmsnorm.lingbot_video_rmsnorm_active(site) + lingbot_video_rmsnorm.LINGBOT_VIDEO_RMSNORM.unmount(site) + assert not lingbot_video_rmsnorm.LINGBOT_VIDEO_RMSNORM.is_enabled(site) if __name__ == "__main__": diff --git a/test/registered/kernels/ops/gemm/test_fp8_kernel.py b/test/registered/kernels/ops/gemm/test_fp8_kernel.py deleted file mode 100644 index 3042fc7925..0000000000 --- a/test/registered/kernels/ops/gemm/test_fp8_kernel.py +++ /dev/null @@ -1,46 +0,0 @@ -import unittest - -import torch - -from sglang.kernels.ops.gemm.fp8_kernel import w8a8_block_fp8_matmul -from sglang.srt.utils import is_cuda, is_xpu -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kernels.fp8 import TestFP8Base - -register_cuda_ci(est_time=5, stage="base-b", runner_config="1-gpu-large") - - -_is_cuda = is_cuda() -_is_xpu = is_xpu() - - -class TestW8A8BlockFP8Matmul(TestFP8Base): - def test_w8a8_block_fp8_matmul(self): - if _is_cuda and torch.cuda.get_device_capability()[0] < 9: - return - elif _is_xpu: - # XPU doesn't provide traditional capability info like CUDA - pass - else: - return - - A, A_quant_gt, A_scale_gt = self._make_A( - M=self.M, K=self.K, group_size=self.group_size, out_dtype=self.quant_type - ) - B, B_quant_gt, B_scale_gt = self._make_B( - K=self.K, N=self.N, group_size=self.group_size, out_dtype=self.quant_type - ) - C_gt = A.to(self.output_type) @ B.to(self.output_type) - C = w8a8_block_fp8_matmul( - A=A_quant_gt, - B=B_quant_gt.T.contiguous(), - As=A_scale_gt, - Bs=B_scale_gt.T.contiguous(), - block_size=[128, 128], - output_dtype=self.output_type, - ) - torch.testing.assert_close(C, C_gt, atol=0.5, rtol=1e-4) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/kernels/ops/gemm/test_linear_bf16_fp32_hpc.py b/test/registered/kernels/ops/gemm/test_linear_bf16_fp32_hpc.py index 4143371c59..247d000fa6 100644 --- a/test/registered/kernels/ops/gemm/test_linear_bf16_fp32_hpc.py +++ b/test/registered/kernels/ops/gemm/test_linear_bf16_fp32_hpc.py @@ -20,7 +20,12 @@ from sglang.kernels.ops.gemm.bf16_fp32 import ( from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=8, stage="base-b", runner_config="1-gpu-large") +register_cuda_ci( + est_time=8, + stage="base-b", + runner_config="1-gpu-large", + disabled="HPC-Ops is only built in docker/Dockerfile; not installed in CI", +) # (hidden_size, n_routed_experts + zero experts) for LongCat-Flash Chat / Lite. _ROUTER_SHAPES = ((6144, 768), (3072, 384)) diff --git a/test/registered/kernels/ops/gemm/test_tiny_gemm.py b/test/registered/kernels/ops/gemm/test_tiny_gemm.py index dcfd43dc13..4d4f2730be 100644 --- a/test/registered/kernels/ops/gemm/test_tiny_gemm.py +++ b/test/registered/kernels/ops/gemm/test_tiny_gemm.py @@ -32,6 +32,8 @@ TINY_GEMM_CASES = get_ci_test_range( (384, 7168, 1, torch.float32), (384, 7168, 4, torch.float32), (896, 7168, 8, torch.float32), + (144, 7168, 2, torch.float32), + (1536, 128, 7, torch.bfloat16), (1536, 128, 16, torch.bfloat16), ], ) diff --git a/test/registered/kernels/ops/gemm/test_tiny_gemm_variants.py b/test/registered/kernels/ops/gemm/test_tiny_gemm_variants.py deleted file mode 100644 index 70bc7e7712..0000000000 --- a/test/registered/kernels/ops/gemm/test_tiny_gemm_variants.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Tiny GEMM variant parity.""" - -import unittest - -import torch - -from sglang.kernels.ops.gemm.tiny_gemm import tiny_gemm_bf16 -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.test_utils import CustomTestCase - -register_cuda_ci(est_time=5, stage="base-b-kernel-unit", runner_config="1-gpu-large") - - -class TestTinyGemmVariants(CustomTestCase): - def test_tiny_gemm_variants(self): - """Both K3 gate-projection shapes, one per kernel variant: N=144 is the - tiny dimension for the first, K=128 for the second.""" - torch.manual_seed(2) - x = torch.randn(2, 7168, device="cuda", dtype=torch.bfloat16) / 8 - weight = torch.randn(144, 7168, device="cuda", dtype=torch.bfloat16) / 8 - actual = tiny_gemm_bf16(x, weight, out_dtype=torch.float32) - torch.testing.assert_close( - actual.double(), x.double() @ weight.double().t(), rtol=1e-3, atol=1e-3 - ) - - x = torch.randn(7, 128, device="cuda", dtype=torch.bfloat16) / 4 - weight = torch.randn(1536, 128, device="cuda", dtype=torch.bfloat16) / 4 - actual = tiny_gemm_bf16(x, weight) - torch.testing.assert_close( - actual.double(), x.double() @ weight.double().t(), rtol=2e-2, atol=2e-2 - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/kernels/ops/kvcache/test_fused_fp8_qkv_kv_cache.py b/test/registered/kernels/ops/kvcache/test_fused_fp8_qkv_kv_cache.py index dbfcc8fa15..b67e96f73c 100644 --- a/test/registered/kernels/ops/kvcache/test_fused_fp8_qkv_kv_cache.py +++ b/test/registered/kernels/ops/kvcache/test_fused_fp8_qkv_kv_cache.py @@ -23,9 +23,9 @@ def _bytes(t: torch.Tensor) -> torch.Tensor: @pytest.mark.parametrize( "hq,hkv,head_dim", [(8, 1, 128), (8, 8, 128), (4, 2, 64), (64, 2, 128)] ) -@pytest.mark.parametrize( - "num_tokens", [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192] -) +# One block per token and no branch on the token count: 1 keeps q contiguous in +# the fused layout, 7 is odd, 4096 is a large grid. +@pytest.mark.parametrize("num_tokens", [1, 7, 4096]) @pytest.mark.parametrize("scale", [None, 0.5, 2.0]) @pytest.mark.parametrize("fused_qkv", [False, True]) @pytest.mark.parametrize("quantize_q", [True, False]) diff --git a/test/registered/kernels/ops/layernorm/test_fused_op_gpu_parity.py b/test/registered/kernels/ops/layernorm/test_fused_op_gpu_parity.py deleted file mode 100644 index 5481df2062..0000000000 --- a/test/registered/kernels/ops/layernorm/test_fused_op_gpu_parity.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Every-backend-vs-native parity for BaseFusedOp ops on real GPU (RFC #29630). - -For each reworked fused op, run every backend eligible on this platform and -assert it matches the pure-torch ``forward_native`` reference within dtype -tolerance. New backends are picked up automatically. -""" - -import pytest -import torch - -from sglang.kernels.spec import KernelBackend -from sglang.test.ci.ci_register import register_cuda_ci - -register_cuda_ci(est_time=8, stage="extra-a", runner_config="1-gpu-small") - -# torch_compile is native under the hood; skip it here (compile time dominates) -# -- it is exercised in the CPU lane. -_SKIP = {KernelBackend.TORCH, KernelBackend.TORCH_COMPILE} -_TOL = { - torch.float16: dict(atol=1e-2, rtol=1e-2), - torch.bfloat16: dict(atol=2e-2, rtol=2e-2), -} - -pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") - - -def _eligible(op): - return [ - b for b in op.available_backends() if b not in _SKIP and op.backend_eligible(b) - ] - - -def _close(got, ref, dtype, msg): - torch.testing.assert_close(got, ref, **_TOL[dtype], msg=msg) - - -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -@pytest.mark.parametrize("shape", [(1, 4096), (128, 4096), (7, 2048)]) -def test_rmsnorm(dtype, shape): - from sglang.kernels.ops.layernorm import _RMSNORM - - torch.manual_seed(0) - x = torch.randn(shape, dtype=dtype, device="cuda") - w = torch.randn(shape[-1], dtype=dtype, device="cuda") - ref = _RMSNORM.forward_native(x, w, 1e-6) - for b in _eligible(_RMSNORM): - _close( - _RMSNORM.forward(x, w, 1e-6, backend=b), ref, dtype, f"rmsnorm {b.value}" - ) - - -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -@pytest.mark.parametrize("shape", [(1, 4096), (128, 4096)]) -def test_fused_add_rmsnorm(dtype, shape): - from sglang.kernels.ops.layernorm import _FUSED_ADD_RMSNORM - - torch.manual_seed(0) - x0 = torch.randn(shape, dtype=dtype, device="cuda") - r0 = torch.randn(shape, dtype=dtype, device="cuda") - w = torch.randn(shape[-1], dtype=dtype, device="cuda") - x_ref, r_ref = x0.clone(), r0.clone() - _FUSED_ADD_RMSNORM.forward_native(x_ref, r_ref, w, 1e-6) - for b in _eligible(_FUSED_ADD_RMSNORM): - x, r = x0.clone(), r0.clone() - _FUSED_ADD_RMSNORM.forward(x, r, w, 1e-6, backend=b) - _close(x, x_ref, dtype, f"fused_add {b.value} (normed)") - _close(r, r_ref, dtype, f"fused_add {b.value} (residual)") - - -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_gemma_rmsnorm(dtype): - from sglang.kernels.ops.layernorm import _GEMMA_RMSNORM - - torch.manual_seed(0) - x = torch.randn(64, 2048, dtype=dtype, device="cuda") - w = torch.randn(2048, dtype=dtype, device="cuda") - ref = _GEMMA_RMSNORM.forward_native(x, w, 1e-6) - for b in _eligible(_GEMMA_RMSNORM): - _close( - _GEMMA_RMSNORM.forward(x, w, 1e-6, backend=b), - ref, - dtype, - f"gemma {b.value}", - ) - - -@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) -def test_gemma_fused_add_rmsnorm(dtype): - from sglang.kernels.ops.layernorm import _GEMMA_FUSED_ADD_RMSNORM - - torch.manual_seed(0) - x0 = torch.randn(64, 2048, dtype=dtype, device="cuda") - r0 = torch.randn(64, 2048, dtype=dtype, device="cuda") - w = torch.randn(2048, dtype=dtype, device="cuda") - x_ref, r_ref = x0.clone(), r0.clone() - _GEMMA_FUSED_ADD_RMSNORM.forward_native(x_ref, r_ref, w, 1e-6) - for b in _eligible(_GEMMA_FUSED_ADD_RMSNORM): - x, r = x0.clone(), r0.clone() - _GEMMA_FUSED_ADD_RMSNORM.forward(x, r, w, 1e-6, backend=b) - _close(x, x_ref, dtype, f"gemma_fused_add {b.value} (normed)") - _close(r, r_ref, dtype, f"gemma_fused_add {b.value} (residual)") - - -if __name__ == "__main__": - import sys - - sys.exit(pytest.main([__file__])) diff --git a/test/registered/kernels/ops/layernorm/test_hc_boundary_hip.py b/test/registered/kernels/ops/layernorm/test_hc_boundary_hip.py index 583e604db2..ba62c2b8f6 100644 --- a/test/registered/kernels/ops/layernorm/test_hc_boundary_hip.py +++ b/test/registered/kernels/ops/layernorm/test_hc_boundary_hip.py @@ -4,13 +4,10 @@ import unittest import torch -from sglang.kernels.ops.layernorm.mhc import ( - _hc_split_sinkhorn_torch, - hc_combine, - hc_mix_stats_sinkhorn, -) +from sglang.kernels.ops.layernorm.mhc import hc_combine, hc_mix_stats_sinkhorn from sglang.srt.utils import is_hip from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.kernels.mhc import hc_split_sinkhorn_torch from sglang.test.test_utils import CustomTestCase register_amd_ci(est_time=15, stage="stage-b", runner_config="1-gpu-small-amd-mi35x") @@ -35,7 +32,7 @@ def _ref_coefficients(residual, hc_fn, hc_scale, hc_base): x = residual.flatten(1).double() rsqrt = torch.rsqrt(x.square().mean(-1, keepdim=True) + RMS_EPS) mixes = ((x @ hc_fn.double().T) * rsqrt).float().unsqueeze(1) - pre, post, comb = _hc_split_sinkhorn_torch( + pre, post, comb = hc_split_sinkhorn_torch( mixes, hc_scale, hc_base, HC, ITERS, HC_EPS ) return pre.squeeze(1), post.squeeze(1), comb.squeeze(1) diff --git a/test/registered/kernels/ops/layernorm/test_hy4_hpc_ihc.py b/test/registered/kernels/ops/layernorm/test_hy4_hpc_ihc.py index 203455d95e..f41f77db8f 100644 --- a/test/registered/kernels/ops/layernorm/test_hy4_hpc_ihc.py +++ b/test/registered/kernels/ops/layernorm/test_hy4_hpc_ihc.py @@ -11,7 +11,12 @@ from sglang.kernels.ops.layernorm.hy4_ihc import ( ) from sglang.test.ci.ci_register import register_cuda_ci -register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_cuda_ci( + est_time=30, + stage="base-b-kernel-unit", + runner_config="1-gpu-large", + disabled="needs HPC-Ops with the iHC ops; not installed in CI", +) @pytest.mark.parametrize("hidden_size", [4096, 6144]) diff --git a/test/registered/kernels/ops/layernorm/test_qknorm.py b/test/registered/kernels/ops/layernorm/test_qknorm.py index 7b85ca6757..093172a998 100644 --- a/test/registered/kernels/ops/layernorm/test_qknorm.py +++ b/test/registered/kernels/ops/layernorm/test_qknorm.py @@ -51,22 +51,6 @@ def flashinfer_qknorm( rmsnorm(k, k_weight, out=k) -@torch.compile() -def torch_impl_qknorm( - q: torch.Tensor, - k: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, - eps: float = 1e-6, -) -> None: - q_mean = q.float().pow(2).mean(dim=-1, keepdim=True) - k_mean = k.float().pow(2).mean(dim=-1, keepdim=True) - q_norm = (q_mean + eps).rsqrt() - k_norm = (k_mean + eps).rsqrt() - q.copy_(q.float() * q_norm * q_weight.float()) - k.copy_(k.float() * k_norm * k_weight.float()) - - BS_LIST = [2**n for n in range(0, 14)] BS_LIST += [x + 1 + i for i, x in enumerate(BS_LIST)] BS_LIST = get_ci_test_range(BS_LIST, [1, 9, 256, 4109]) diff --git a/test/registered/kernels/ops/layernorm/test_qknorm_across_heads.py b/test/registered/kernels/ops/layernorm/test_qknorm_across_heads.py deleted file mode 100644 index 414d18abc0..0000000000 --- a/test/registered/kernels/ops/layernorm/test_qknorm_across_heads.py +++ /dev/null @@ -1,84 +0,0 @@ -import itertools -import sys - -import pytest -import torch -import triton - -from sglang.kernels.jit.utils import get_ci_test_range -from sglang.test.ci.ci_register import register_cuda_ci - -register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="1-gpu-large") -# Nightly is not redundant here: it sets SGLANG_JIT_KERNEL_RUN_FULL_TESTS=1 to expand get_ci_test_range sweeps. -register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large") - - -def sglang_jit_qknorm_across_heads( - q: torch.Tensor, - k: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, -) -> None: - from sglang.kernels.ops.layernorm.norm import fused_inplace_qknorm_across_heads - - fused_inplace_qknorm_across_heads(q, k, q_weight, k_weight) - - -def sglang_aot_qknorm_across_heads( - q: torch.Tensor, - k: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, -) -> None: - from sgl_kernel import rmsnorm - - rmsnorm(q, q_weight, out=q) - rmsnorm(k, k_weight, out=k) - - -@torch.compile() -def torch_impl_qknorm_across_heads( - q: torch.Tensor, - k: torch.Tensor, - q_weight: torch.Tensor, - k_weight: torch.Tensor, - eps: float = 1e-6, -) -> None: - q_mean = q.float().pow(2).mean(dim=-1, keepdim=True) - k_mean = k.float().pow(2).mean(dim=-1, keepdim=True) - q_norm = (q_mean + eps).rsqrt() - k_norm = (k_mean + eps).rsqrt() - q.copy_(q.float() * q_norm * q_weight.float()) - k.copy_(k.float() * k_norm * k_weight.float()) - - -BS_LIST = [2**n for n in range(0, 14)] -BS_LIST += [x + 1 + i for i, x in enumerate(BS_LIST)] -BS_LIST = get_ci_test_range(BS_LIST, [1, 9, 256, 4109]) -HIDDEN_DIM_LIST = get_ci_test_range([512, 1024, 2048, 4096], [512, 2048, 4096]) -DEVICE = "cuda" -DTYPE = torch.bfloat16 - - -@pytest.mark.parametrize( - "batch_size,hidden_dim", - list(itertools.product(BS_LIST, HIDDEN_DIM_LIST)), -) -def test_qknorm_across_heads(batch_size: int, hidden_dim: int) -> None: - q = torch.randn(batch_size, hidden_dim, device=DEVICE, dtype=DTYPE) - k = torch.randn(batch_size, hidden_dim, device=DEVICE, dtype=DTYPE) - q_weight = torch.randn(hidden_dim, device=DEVICE, dtype=DTYPE) - k_weight = torch.randn(hidden_dim, device=DEVICE, dtype=DTYPE) - - q_k_jit = (q.clone(), k.clone()) - q_k_aot = (q.clone(), k.clone()) - - sglang_jit_qknorm_across_heads(q_k_jit[0], q_k_jit[1], q_weight, k_weight) - sglang_aot_qknorm_across_heads(q_k_aot[0], q_k_aot[1], q_weight, k_weight) - - triton.testing.assert_close(q_k_jit[0], q_k_aot[0], atol=1e-2, rtol=1e-2) - triton.testing.assert_close(q_k_jit[1], q_k_aot[1], atol=1e-2, rtol=1e-2) - - -if __name__ == "__main__": - sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/test/registered/kernels/ops/memory/test_adler32_checksum.py b/test/registered/kernels/ops/memory/test_adler32_checksum.py index 7f39dd7fbc..c4d55a61ad 100644 --- a/test/registered/kernels/ops/memory/test_adler32_checksum.py +++ b/test/registered/kernels/ops/memory/test_adler32_checksum.py @@ -5,11 +5,7 @@ import zlib import torch -from sglang.kernels.ops.memory.adler32 import ( - adler32_checksum, - adler32_regions_checksum, - adler32_strided_checksum, -) +from sglang.kernels.ops.memory.adler32 import adler32_strided_checksum from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=15, stage="base-b-kernel-unit", runner_config="1-gpu-large") @@ -35,7 +31,10 @@ class TestAdler32(unittest.TestCase): torch.manual_seed(42) def _check_whole(self, tensor): - self.assertEqual(adler32_checksum(tensor), _ref_adler32(tensor)) + row_bytes = tensor[0].numel() * tensor.element_size() + rows = torch.arange(tensor.shape[0], dtype=torch.int64, device="cuda") + actual = adler32_strided_checksum([tensor.data_ptr()], [row_bytes], [rows]) + self.assertEqual(actual, _ref_adler32(tensor)) def test_whole_small(self): self._check_whole(torch.tensor([1.0, 2, 3, 4], device="cuda")) @@ -75,20 +74,15 @@ class TestAdler32(unittest.TestCase): idx = torch.arange(1000, dtype=torch.int64, device="cuda") self._check_strided([t], [idx], [128 * 2]) - def test_regions(self): + def test_strided_large_items(self): first = torch.randint( 0, 256, (5 * 1024 * 1024,), dtype=torch.uint8, device="cuda" ) second = torch.randint(0, 256, (12345,), dtype=torch.uint8, device="cuda") - actual = adler32_regions_checksum( - [first.data_ptr(), second.data_ptr()], - [first.numel(), second.numel()], - first.device, + idx = torch.zeros(1, dtype=torch.int64, device="cuda") + self._check_strided( + [first, second], [idx, idx], [first.numel(), second.numel()] ) - expected = zlib.adler32( - first.cpu().numpy().tobytes() + second.cpu().numpy().tobytes() - ) - self.assertEqual(actual, expected) if __name__ == "__main__": diff --git a/test/registered/kernels/ops/moe/test_minimax_quant_scatter.py b/test/registered/kernels/ops/moe/test_minimax_quant_scatter.py index 5b022e0a21..55515f5c76 100644 --- a/test/registered/kernels/ops/moe/test_minimax_quant_scatter.py +++ b/test/registered/kernels/ops/moe/test_minimax_quant_scatter.py @@ -11,8 +11,10 @@ from sglang.kernels.ops.moe.ep_moe_kernels import ( fill_gateup_input_triton_kernel, moe_ep_deepgemm_preprocess, ) +from sglang.kernels.ops.quantization.fp8_kernel import ( + sglang_per_token_group_quant_fp8, +) from sglang.kernels.ops.quantization.minimax_quant_ue8m0 import ( - per_token_quant_fp8_ue8m0, per_token_quant_fp8_ue8m0_scatter, ) from sglang.srt.environ import envs @@ -35,6 +37,12 @@ register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="4-gpu-b dev = "cuda" +def _quant_ue8m0(x, group): + return sglang_per_token_group_quant_fp8( + x, group, column_major_scales=True, scale_tma_aligned=True, scale_ue8m0=True + ) + + @pytest.fixture def stated_tp_group(): """Provide a TP-group placeholder for kernels with mocked symmetric memory.""" @@ -101,7 +109,7 @@ def test_quant_scatter_matches_quant_plus_fill(num_tokens, topk, hidden, group): tids.copy_(tids_cpu) s2d = torch.tensor(s2d, dtype=torch.int32, device=dev) - x_q, x_sf = per_token_quant_fp8_ue8m0(x, group) + x_q, x_sf = _quant_ue8m0(x, group) gi_ref = torch.zeros(E, m_max, hidden, device=dev, dtype=torch.float8_e4m3fn) gs_ref = torch.zeros(E, G4, m_max, device=dev, dtype=torch.int32) fill_gateup_input_triton_kernel[(num_tokens,)]( @@ -161,7 +169,7 @@ def test_standard_deepgemm_preprocess_quantizes_with_ue8m0_scale(): output_dtype=torch.float8_e4m3fn, use_mxfp8=False, ) - direct_x, direct_scale = per_token_quant_fp8_ue8m0(x, group) + direct_x, direct_scale = _quant_ue8m0(x, group) assert grouped_scale.dtype == torch.int32 for token in range(num_tokens): diff --git a/test/registered/kernels/ops/moe/test_moe_align_block_size.py b/test/registered/kernels/ops/moe/test_moe_align_block_size.py index f3acc05789..558916c709 100644 --- a/test/registered/kernels/ops/moe/test_moe_align_block_size.py +++ b/test/registered/kernels/ops/moe/test_moe_align_block_size.py @@ -3,148 +3,17 @@ import sys import pytest import torch -import triton -import triton.language as tl from sglang.kernels.jit.utils import get_ci_test_range from sglang.kernels.ops.moe.moe_align import moe_align_block_size from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kernels.moe_align import ceil_div, moe_align_block_size_triton register_cuda_ci(est_time=28, stage="base-b-kernel-unit", runner_config="1-gpu-large") # Nightly is not redundant here: it sets SGLANG_JIT_KERNEL_RUN_FULL_TESTS=1 to expand get_ci_test_range sweeps. register_cuda_ci(est_time=50, stage="nightly", runner_config="1-gpu-large") -def ceil_div(a, b): - return (a + b - 1) // b - - -@triton.jit -def moe_align_block_size_stage1( - topk_ids_ptr, - tokens_cnts_ptr, - num_experts: tl.constexpr, - numel: tl.constexpr, - tokens_per_thread: tl.constexpr, -): - pid = tl.program_id(0) - start_idx = pid * tokens_per_thread - off_c = (pid + 1) * num_experts - - for i in range(tokens_per_thread): - if start_idx + i < numel: - idx = tl.load(topk_ids_ptr + start_idx + i) - token_cnt = tl.load(tokens_cnts_ptr + off_c + idx) - tl.store(tokens_cnts_ptr + off_c + idx, token_cnt + 1) - - -@triton.jit -def moe_align_block_size_stage2( - tokens_cnts_ptr, - num_experts: tl.constexpr, -): - pid = tl.program_id(0) - last_cnt = 0 - for i in range(1, num_experts + 1): - token_cnt = tl.load(tokens_cnts_ptr + i * num_experts + pid) - last_cnt = last_cnt + token_cnt - tl.store(tokens_cnts_ptr + i * num_experts + pid, last_cnt) - - -@triton.jit -def moe_align_block_size_stage3( - total_tokens_post_pad_ptr, - tokens_cnts_ptr, - cumsum_ptr, - num_experts: tl.constexpr, - block_size: tl.constexpr, -): - last_cumsum = 0 - off_cnt = num_experts * num_experts - for i in range(1, num_experts + 1): - token_cnt = tl.load(tokens_cnts_ptr + off_cnt + i - 1) - last_cumsum = last_cumsum + tl.cdiv(token_cnt, block_size) * block_size - tl.store(cumsum_ptr + i, last_cumsum) - tl.store(total_tokens_post_pad_ptr, last_cumsum) - - -@triton.jit -def moe_align_block_size_stage4( - topk_ids_ptr, - sorted_token_ids_ptr, - expert_ids_ptr, - tokens_cnts_ptr, - cumsum_ptr, - num_experts: tl.constexpr, - block_size: tl.constexpr, - numel: tl.constexpr, - tokens_per_thread: tl.constexpr, -): - pid = tl.program_id(0) - start_idx = tl.load(cumsum_ptr + pid) - end_idx = tl.load(cumsum_ptr + pid + 1) - - for i in range(start_idx, end_idx, block_size): - tl.store(expert_ids_ptr + i // block_size, pid) - - start_idx = pid * tokens_per_thread - off_t = pid * num_experts - - for i in range(start_idx, tl.minimum(start_idx + tokens_per_thread, numel)): - expert_id = tl.load(topk_ids_ptr + i) - token_cnt = tl.load(tokens_cnts_ptr + off_t + expert_id) - rank_post_pad = token_cnt + tl.load(cumsum_ptr + expert_id) - tl.store(sorted_token_ids_ptr + rank_post_pad, i) - tl.store(tokens_cnts_ptr + off_t + expert_id, token_cnt + 1) - - -def moe_align_block_size_triton( - topk_ids: torch.Tensor, - num_experts: int, - block_size: int, - sorted_token_ids: torch.Tensor, - expert_ids: torch.Tensor, - num_tokens_post_pad: torch.Tensor, -) -> None: - numel = topk_ids.numel() - grid = (num_experts,) - tokens_cnts = torch.zeros( - (num_experts + 1, num_experts), dtype=torch.int32, device=topk_ids.device - ) - cumsum = torch.zeros((num_experts + 1,), dtype=torch.int32, device=topk_ids.device) - tokens_per_thread = ceil_div(numel, num_experts) - - moe_align_block_size_stage1[grid]( - topk_ids, - tokens_cnts, - num_experts, - numel, - tokens_per_thread, - ) - moe_align_block_size_stage2[grid]( - tokens_cnts, - num_experts, - ) - moe_align_block_size_stage3[(1,)]( - num_tokens_post_pad, - tokens_cnts, - cumsum, - num_experts, - block_size, - ) - moe_align_block_size_stage4[grid]( - topk_ids, - sorted_token_ids, - expert_ids, - tokens_cnts, - cumsum, - num_experts, - block_size, - numel, - tokens_per_thread, - ) - - MOE_ALIGN_CASES = get_ci_test_range( list( itertools.product( diff --git a/test/registered/kernels/ops/moe/test_moe_fused_gate.py b/test/registered/kernels/ops/moe/test_moe_fused_gate.py index 3e909fa958..e419acfe17 100644 --- a/test/registered/kernels/ops/moe/test_moe_fused_gate.py +++ b/test/registered/kernels/ops/moe/test_moe_fused_gate.py @@ -1,10 +1,9 @@ """Correctness tests for the Triton :func:`moe_fused_gate` router. -The Triton kernel is a drop-in reimplementation of the CUDA fused gate for the -ungrouped case (``num_expert_group == 1``). We validate it three ways: +For the ungrouped case (``num_expert_group == 1``) we validate it two ways: * against an explicit, definition-based torch reference (documents the math), -* against the CUDA JIT kernel it mirrors (:func:`moe_fused_gate_jit`), and + at small shapes and at model routing shapes, and * against the production ``biased_grouped_topk_impl`` for the sigmoid / no-shared path that the kernel actually replaces in ``topk.py``. @@ -22,7 +21,7 @@ import pytest import torch from sglang.kernels.jit.utils import get_ci_test_range -from sglang.kernels.ops.moe.moe_fused_gate import moe_fused_gate, moe_fused_gate_jit +from sglang.kernels.ops.moe.moe_fused_gate import moe_fused_gate from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=8, stage="base-b-kernel-unit", runner_config="1-gpu-large") @@ -55,7 +54,7 @@ def _reference_gate( routed_scaling_factor: float, apply_routed_scaling_factor_on_output: bool, ) -> Tuple[torch.Tensor, torch.Tensor]: - """Definition-based eager reference matching the CUDA fused-gate semantics.""" + """Definition-based eager reference for the fused-gate semantics.""" if scoring_func == "sigmoid": activated = scores.sigmoid() else: @@ -176,10 +175,9 @@ def test_moe_fused_gate_matches_reference( (8192, 512, 8, 1, "sigmoid"), ], ) -def test_moe_fused_gate_matches_cuda_jit( +def test_moe_fused_gate_matches_reference_at_model_shapes( M: int, num_experts: int, topk: int, num_shared: int, scoring_func: str ) -> None: - """Triton output must match the CUDA JIT kernel it reimplements.""" scores, bias = _make_inputs(M, num_experts, seed=123) kwargs = dict( @@ -191,13 +189,13 @@ def test_moe_fused_gate_matches_cuda_jit( apply_routed_scaling_factor_on_output=True, ) triton_w, triton_i = moe_fused_gate(scores, bias, **kwargs) - cuda_w, cuda_i = moe_fused_gate_jit(scores, bias, **kwargs) + ref_w, ref_i = _reference_gate(scores, bias, **kwargs) torch.cuda.synchronize() num_columns = num_experts + num_shared torch.testing.assert_close( _scatter_by_expert(triton_w, triton_i, num_columns), - _scatter_by_expert(cuda_w, cuda_i, num_columns), + _scatter_by_expert(ref_w, ref_i, num_columns), rtol=1e-4, atol=1e-5, ) diff --git a/test/registered/kernels/ops/moe/test_post_reorder_deepgemm.py b/test/registered/kernels/ops/moe/test_post_reorder_deepgemm.py index ef01918ce5..7e3ec9903f 100644 --- a/test/registered/kernels/ops/moe/test_post_reorder_deepgemm.py +++ b/test/registered/kernels/ops/moe/test_post_reorder_deepgemm.py @@ -1,10 +1,7 @@ import pytest import torch -from sglang.kernels.ops.moe.ep_moe_kernels import ( - post_reorder_deepgemm, - post_reorder_triton_kernel, -) +from sglang.kernels.ops.moe.ep_moe_kernels import post_reorder_deepgemm from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") @@ -74,14 +71,7 @@ def test_post_reorder_deepgemm(num_tokens, pad_frac, rsf): new = torch.empty(num_tokens, hidden, dtype=torch.bfloat16, device=dev) post_reorder_deepgemm(do, new, s2d, tids, tw, tk, num_tokens, hidden, rsf) - old = torch.empty(num_tokens, hidden, dtype=torch.bfloat16, device=dev) - post_reorder_triton_kernel[(num_tokens,)]( - do, old, s2d, tids, tw, tk, hidden, BLOCK_SIZE=512 - ) - old *= rsf - torch.testing.assert_close(new.float(), ref, rtol=2e-2, atol=2e-2) - torch.testing.assert_close(new.float(), old.float(), rtol=5e-2, atol=0.5) if __name__ == "__main__": diff --git a/test/registered/kernels/ops/moe/test_sigmoid_gate_mul_broadcast.py b/test/registered/kernels/ops/moe/test_sigmoid_gate_mul_broadcast.py index c9354491fe..8565aa721c 100644 --- a/test/registered/kernels/ops/moe/test_sigmoid_gate_mul_broadcast.py +++ b/test/registered/kernels/ops/moe/test_sigmoid_gate_mul_broadcast.py @@ -10,43 +10,6 @@ register_amd_ci(est_time=4, suite="jit-kernel-unit-test-amd") DEVICE = "cuda" -def reference_sigmoid_gate_mul(x, gate): - return x * torch.sigmoid(gate) - - -# ── element-wise variant ── - - -@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32]) -@pytest.mark.parametrize( - "shape", - [ - (1, 4096), - (4, 4096), - (8, 4096), - (32, 8192), - (1, 128), - (16, 16384), - ], -) -def test_sigmoid_gate_mul_correctness(shape, dtype): - from sglang.kernels.ops.moe.triton_sigmoid_gate_mul import sigmoid_gate_mul - - torch.manual_seed(42) - x = torch.randn(shape, dtype=dtype, device=DEVICE) - gate = torch.randn(shape, dtype=dtype, device=DEVICE) - - ref = reference_sigmoid_gate_mul(x, gate) - out = sigmoid_gate_mul(x, gate) - - rtol = 1e-2 if dtype == torch.bfloat16 else 1e-3 - atol = 2e-2 if dtype == torch.bfloat16 else 1e-3 - torch.testing.assert_close(out, ref, rtol=rtol, atol=atol) - - -# ── broadcast variant ── - - @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32]) @pytest.mark.parametrize( "shape", diff --git a/test/registered/kernels/ops/quantization/test_awq_dequant.py b/test/registered/kernels/ops/quantization/test_awq_dequant.py index 70fc205e85..0aca2709a8 100644 --- a/test/registered/kernels/ops/quantization/test_awq_dequant.py +++ b/test/registered/kernels/ops/quantization/test_awq_dequant.py @@ -15,7 +15,6 @@ import torch from sglang.kernels.ops.quantization.awq_triton import ( AWQ_TRITON_SUPPORTED_GROUP_SIZES, awq_dequantize_triton, - awq_gemm_triton, ) from sglang.srt.utils import get_device from sglang.test.ci.ci_register import register_amd_ci @@ -115,66 +114,6 @@ class TestAWQTriton(CustomTestCase): self.assertFalse(torch.any(torch.isinf(tri)) or torch.any(torch.isnan(tri))) torch.testing.assert_close(ref, tri) - # GEMM - def test_gemm(self): - N_list = [1, 2, 4, 8, 14, 17, 23, 32] - K_list = [128] - M_list = [16, 24, 32] - splitK_list = [1, 8] - - for N in N_list: - for K in K_list: - for M in M_list: - for group_size in AWQ_TRITON_SUPPORTED_GROUP_SIZES: - for splitK in splitK_list: - with self.subTest(N=N, K=K, M=M, g=group_size, sk=splitK): - self._run_gemm_case( - N=N, - K=K, - M=M, - group_size=group_size, - splitK=splitK, - ) - - def _run_gemm_case(self, N, K, M, group_size, splitK): - if group_size == -1: - group_size = K - - torch.manual_seed(0) - - x = torch.rand((N, K), dtype=torch.float32, device=device) - qweight = torch.randint( - 0, - torch.iinfo(torch.int32).max, - (K, M // 8), - dtype=torch.int32, - device=device, - ) - qzeros = torch.randint( - 0, - torch.iinfo(torch.int32).max, - (K // group_size, M // 8), - dtype=torch.int32, - device=device, - ) - scales = torch.rand((K // group_size, M), dtype=torch.float32, device=device) - - tri_out = awq_gemm_triton(x, qweight, scales, qzeros, splitK) - - self.assertFalse( - torch.any(torch.isinf(tri_out)) or torch.any(torch.isnan(tri_out)) - ) - - # dequantize & compare - w_deq = awq_dequantize_triton(qweight, scales, qzeros) - ref_out = torch.matmul(x, w_deq) - - self.assertFalse( - torch.any(torch.isinf(ref_out)) or torch.any(torch.isnan(ref_out)) - ) - - torch.testing.assert_close(tri_out.cpu(), ref_out.cpu(), atol=1e-1, rtol=1e-1) - if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/test/registered/kernels/ops/quantization/test_per_token_quant_fp8.py b/test/registered/kernels/ops/quantization/test_per_token_quant_fp8.py index 386c1d388d..07201d2416 100644 --- a/test/registered/kernels/ops/quantization/test_per_token_quant_fp8.py +++ b/test/registered/kernels/ops/quantization/test_per_token_quant_fp8.py @@ -10,7 +10,6 @@ from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=16, stage="base-b-kernel-unit", runner_config="1-gpu-large") register_cuda_ci(est_time=16, stage="base-b-kernel-unit", runner_config="4-gpu-b200") -register_cuda_ci(est_time=30, stage="nightly", runner_config="1-gpu-large") def _run_impl(input: torch.Tensor, *, use_jit: bool): diff --git a/test/registered/kernels/ops/sampling/test_top_p_renorm.py b/test/registered/kernels/ops/sampling/test_top_p_renorm.py deleted file mode 100644 index 49a9538a5f..0000000000 --- a/test/registered/kernels/ops/sampling/test_top_p_renorm.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Top-p probability renormalization parity.""" - -import unittest - -import torch - -from sglang.kernels.ops.sampling.top_p_renorm_triton import ( - top_p_renorm_probs_triton, -) -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.test_utils import CustomTestCase - -register_cuda_ci(est_time=5, stage="base-b-kernel-unit", runner_config="1-gpu-large") - - -class TestTopPRenorm(CustomTestCase): - def test_top_p_renorm(self): - torch.manual_seed(3) - probs = torch.randn(3, 1024, device="cuda").softmax(-1) - top_p = torch.tensor([0.5, 0.8, 0.95], device="cuda") - sorted_probs = probs.sort(-1).values - cutoff = torch.searchsorted( - sorted_probs.cumsum(-1), (1 - top_p).unsqueeze(1) - ).squeeze(1) - cutoff.clamp_(max=probs.shape[1] - 1) - pivot = sorted_probs.gather(1, cutoff[:, None]) - expected = torch.where(probs >= pivot, probs, 0) - expected /= expected.sum(-1, keepdim=True) - torch.testing.assert_close( - top_p_renorm_probs_triton(probs, top_p), - expected, - rtol=2e-6, - atol=1e-8, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/spec/dspark/test_dspark_kernel_parity.py b/test/registered/spec/dspark/test_dspark_kernel_parity.py index 6d25f7abe9..9422859059 100644 --- a/test/registered/spec/dspark/test_dspark_kernel_parity.py +++ b/test/registered/spec/dspark/test_dspark_kernel_parity.py @@ -28,6 +28,7 @@ from sglang.srt.speculative.dspark_components.dspark_planner import ( ) from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kernels import dspark as dspark_kernels from sglang.test.test_utils import CustomTestCase, get_device register_cuda_ci(est_time=11, stage="base-b", runner_config="1-gpu-small") @@ -250,13 +251,13 @@ def _case_compact_layout(tc): total = int(verify_lens.sum().item()) for padded_total in (total, bs * t): # exact and bucket padding tc._parity( - dspark_verify_window.CompactRowIndex, + dspark_kernels.CompactRowIndex, verify_lens=verify_lens, padded_total=padded_total, device=DEVICE, ) tc._parity( - dspark_verify_window.CompactVerifyIds, + dspark_kernels.CompactVerifyIds, draft_block_ids=_ri(0, VOCAB, (bs, gamma)), draft_tokens=_ri(0, VOCAB, (bs, gamma)), layout=_layout(verify_lens, padded_total), diff --git a/test/registered/unit/hardware_backend/mlx/test_attention_patching.py b/test/registered/unit/hardware_backend/mlx/test_attention_patching.py index acecaed853..2fba5de275 100644 --- a/test/registered/unit/hardware_backend/mlx/test_attention_patching.py +++ b/test/registered/unit/hardware_backend/mlx/test_attention_patching.py @@ -1247,7 +1247,7 @@ class TestMlxOverlapScheduler(unittest.TestCase): mamba_track_mask_next_cpu=None, mamba_decode_batch_idx_cpu=None, ) - result = SimpleNamespace() + result = GenerationBatchResult() i = 0 logits_output = SimpleNamespace(customized_info=None) original_release = batch_result_processor_module.release_kv_cache diff --git a/test/registered/unit/kernels/test_fused_op.py b/test/registered/unit/kernels/test_fused_op.py index 9d37da4afd..4da4441f35 100644 --- a/test/registered/unit/kernels/test_fused_op.py +++ b/test/registered/unit/kernels/test_fused_op.py @@ -83,6 +83,22 @@ def test_forced_backend_env_var(): m._forced_backend = m._UNRESOLVED +_TOY_ADD_BACKENDS = ( + KernelBackend.TORCH, + KernelBackend.TORCH_COMPILE, + KernelBackend.TRITON, +) + + +@pytest.mark.parametrize( + "backend", [b for b in KernelBackend if b not in _TOY_ADD_BACKENDS] +) +def test_forced_unimplemented_backend_falls_back(backend): + # Every backend needs a raising stub, else forcing it is an AttributeError. + K.set_fused_op_backend(backend) + assert _ToyAdd()(_t(1.0), _t(2.0)).item() == 1003.0 + + def test_trace_records_op_backend_and_shapes(): K.enable_fused_op_trace() _ToyAdd()(torch.zeros(2, 3), torch.zeros(2, 3)) @@ -93,14 +109,13 @@ def test_trace_records_op_backend_and_shapes(): def test_fused_op_registers_all_backends(): - backends = {s.backend for s in K.registry.get("layernorm.rmsnorm")} + backends = {s.backend for s in K.registry.get("activation.silu_and_mul")} assert backends == { KernelBackend.TORCH, KernelBackend.TORCH_COMPILE, KernelBackend.JIT, KernelBackend.AOT, KernelBackend.AITER, - KernelBackend.TORCH_NPU, } @@ -148,7 +163,7 @@ def test_registry_get_backend_missing_raises(): def test_spec_load_simple_and_dotted(): assert _spec(target="math:sqrt").load() is math.sqrt - dotted = _spec(target="sglang.kernels.ops.layernorm:_RMSNORM.forward_native") + dotted = _spec(target="sglang.kernels.ops.activation:_SILU_AND_MUL.forward_native") assert callable(dotted.load()) @@ -160,54 +175,17 @@ def test_spec_load_bad_target_raises(): # --- native reference math of the reworked ops (CPU) --- -def _ref_rmsnorm(x, w, eps): - xf = x.float() - return (xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps) * w).to(x.dtype) - - -def test_rmsnorm_native(): - from sglang.kernels.ops.layernorm import _RMSNORM - - x, w = torch.randn(8, 128), torch.randn(128) - out = _RMSNORM.forward_native(x, w, 1e-6) - assert torch.allclose(out, _ref_rmsnorm(x, w, 1e-6)) - buf = torch.empty_like(x) - assert _RMSNORM.forward_native(x, w, 1e-6, out=buf) is buf # out= is in place - assert torch.allclose(buf, out) - - -def test_fused_add_rmsnorm_native(): - from sglang.kernels.ops.layernorm import _FUSED_ADD_RMSNORM - - x, residual, w = torch.randn(8, 128), torch.randn(8, 128), torch.randn(128) - x0, r0 = x.clone(), residual.clone() - assert _FUSED_ADD_RMSNORM.forward_native(x, residual, w, 1e-6) is None - acc = x0.float() + r0.float() - assert torch.allclose(residual, acc) - assert torch.allclose( - x, acc * torch.rsqrt(acc.pow(2).mean(-1, keepdim=True) + 1e-6) * w - ) - - -@pytest.mark.parametrize( - "op_attr, approximate", - [ - ("_SILU_AND_MUL", None), - ("_GELU_AND_MUL", "none"), - ("_GELU_TANH_AND_MUL", "tanh"), - ], -) -def test_gated_activation_native(op_attr, approximate): +def test_silu_and_mul_native(): import torch.nn.functional as F - import sglang.kernels.ops.activation as act + from sglang.kernels.ops.activation import _SILU_AND_MUL x = torch.randn(8, 256) - gate, up = x[..., :128], x[..., 128:] - ref = ( - F.silu(gate) if approximate is None else F.gelu(gate, approximate=approximate) - ) * up - assert torch.allclose(getattr(act, op_attr).forward_native(x), ref) + ref = F.silu(x[..., :128]) * x[..., 128:] + assert torch.allclose(_SILU_AND_MUL.forward_native(x), ref) + buf = torch.empty_like(ref) + assert _SILU_AND_MUL.forward_native(x, buf) is buf # out= is in place + assert torch.allclose(buf, ref) if __name__ == "__main__": diff --git a/test/registered/unit/kernels/test_kernels_namespace.py b/test/registered/unit/kernels/test_kernels_namespace.py index 41f47e9a19..c12429fa18 100644 --- a/test/registered/unit/kernels/test_kernels_namespace.py +++ b/test/registered/unit/kernels/test_kernels_namespace.py @@ -1,7 +1,6 @@ """GPU-free import / registry / selector tests for ``sglang.kernels`` (RFC #29630).""" import ast -import importlib import subprocess import sys from pathlib import Path @@ -68,10 +67,10 @@ def test_multi_backend_requires_explicit_backend(monkeypatch): # the current device means selection must name one. monkeypatch.setattr(sel, "_platform", lambda: _SM90) with pytest.raises(ValueError): - K.select_kernel("layernorm.rmsnorm") - spec = K.select_kernel("layernorm.rmsnorm", backend=KernelBackend.JIT) + K.select_kernel("activation.silu_and_mul") + spec = K.select_kernel("activation.silu_and_mul", backend=KernelBackend.JIT) assert spec.backend is KernelBackend.JIT - assert spec.target == "sglang.kernels.ops.layernorm:_RMSNORM.forward_jit" + assert spec.target == "sglang.kernels.ops.activation:_SILU_AND_MUL.forward_jit" @pytest.mark.parametrize("device, expect", [("cuda", "jit"), ("hip", "aot")]) @@ -83,25 +82,6 @@ def test_activation_default_backend(monkeypatch, device, expect): assert _SILU_AND_MUL.auto_selected_backend().value == expect -@pytest.mark.parametrize( - "op_attr, device, expect", - [ - ("_RMSNORM", "cuda", "aot"), - ("_RMSNORM", "hip", "aiter"), - ("_RMSNORM", "npu", "torch_npu"), - ("_GEMMA_RMSNORM", "cuda", "aot"), - ("_GEMMA_RMSNORM", "hip", "jit"), # rocm-triton JIT pinned to HIP - ("_GEMMA_RMSNORM", "npu", "torch_npu"), - ], -) -def test_layernorm_default_backend(monkeypatch, op_attr, device, expect): - # Same AOT provenance, different device coverage per op: rmsnorm's AOT is - # CUDA-only, so HIP falls to aiter and NPU to torch_npu. - ln = importlib.import_module("sglang.kernels.ops.layernorm") - monkeypatch.setattr(fo, "_platform", lambda: PlatformInfo(device_type=device)) - assert getattr(ln, op_attr).auto_selected_backend().value == expect - - @pytest.mark.parametrize( "req, plat, ok", [ @@ -195,7 +175,6 @@ def test_import_stays_metadata_only(): ("attention.attn_res_hip", KernelBackend.TRITON, "hip"), ("attention.kimi_k3_mla_output_gate", KernelBackend.JIT, "cuda"), ("attention.fused_kda_decode_mtp_dspark", KernelBackend.CUTE_DSL, "cuda"), - ("attention.flydsl_kimi_k3_kda_decode", KernelBackend.FLYDSL, "hip"), ("attention.flydsl_kimi_k3_kda_decode_with_f_b", KernelBackend.FLYDSL, "hip"), ("communication.all_reduce_push_res", KernelBackend.JIT, "cuda"), ("communication.gemm_ag_up_proj", KernelBackend.JIT, "cuda"), diff --git a/test/registered/unit/multimodal_gen/test_minimax_h3_step_contract.py b/test/registered/unit/multimodal_gen/test_minimax_h3_step_contract.py index 1b15b99d15..9607953894 100644 --- a/test/registered/unit/multimodal_gen/test_minimax_h3_step_contract.py +++ b/test/registered/unit/multimodal_gen/test_minimax_h3_step_contract.py @@ -33,9 +33,6 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.m from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.resolved_plan import ( MINIMAX_H3_CANONICAL_REQUEST_EXTRA_KEY, ) -from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import ( - MiniMaxH3DenoisingStage, -) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.timestep_preparation import ( MiniMaxH3TimestepPreparationStage, ) @@ -98,8 +95,6 @@ def test_admission_schedule_loop_and_workload_agree(admission, steps): for schedule in sigmas.values(): assert len(schedule) == steps + 1 assert schedule[0] == 1.0 and schedule[-1] == 0.0 - stage = MiniMaxH3DenoisingStage.__new__(MiniMaxH3DenoisingStage) - assert stage.default_workload_iterations(request, steps) == steps packed = minimax_h3_packed_sequence( text_len=3, latent_t=2,