From 33ecf4bcd82364f3e19f457963d053db94226c22 Mon Sep 17 00:00:00 2001 From: Sugar920 <121632458+Sugar920@users.noreply.github.com> Date: Sat, 1 Aug 2026 15:03:21 +0800 Subject: [PATCH] Add pr tests (#31952) Co-authored-by: Claude Co-authored-by: sglang-npu-bot Co-authored-by: Cherry_ming <136634645@qq.com> --- .github/CODEOWNERS | 2 +- .github/workflows/nightly-test-npu.yml | 135 +++++++++---- .github/workflows/pr-test-npu.yml | 75 ++++++- .../development/contribution_guide.mdx | 8 +- .../development/support_new_models.mdx | 2 +- .../sglang/test/ascend/test_ascend_utils.py | 6 +- test/registered/README.md | 2 +- .../test_npu_deepseek_v3_2_8p_aime25.py | 0 .../test_npu_glm4_6v_flash_1p_mmmu.py | 0 .../test_npu_glm4_7_flash_1p_aime25.py | 0 ...1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py | 26 +-- ..._k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py | 23 +-- ..._w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py | 109 ++++++++++ ...max_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py | 107 ++++++++++ ...qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py | 25 +-- ..._qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py | 21 +- ...n3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py | 25 +-- .../test_npu_qwen3_32b_bf16_8p_gpqa.py | 0 ...wen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py | 27 +-- ...3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py | 108 ++++++++++ .../test_npu_qwen3_6_27b_1p_gpqa.py | 0 ...n3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py | 106 ++++++++++ .../test_npu_qwen3_6_35b_a3b_1p_aime26.py | 0 ...a3b_1p_in64k_out1k_prefix90_50ms_aime26.py | 107 ++++++++++ ...ext_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py | 20 +- ...npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py | 0 ...t_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py | 0 .../test_npu_qwen3_vl_8b_thinking_1p_mmmu.py | 6 +- .../HiCache/test_npu_hicache_mha.py | 0 .../HiCache/test_npu_hicache_mla.py | 0 .../HiCache/test_npu_hierarchical_cache.py | 0 .../test_npu_hierarchical_cache_mla.py | 0 ...u_hierarchical_cache_mutually_exclusive.py | 0 .../test_npu_hierarchical_cache_ttft_mha.py | 0 .../HiCache/test_npu_radix_cache.py | 0 .../backends/test_npu_sampling_backend.py | 0 .../dllm/test_npu_llada2_mini.py | 0 .../test_npu_no_chunked_prefill.py | 0 .../offloading/test_npu_offload_modes.py | 0 .../test_npu_compile_graph_tp1_bf16.py | 0 .../test_npu_graph_tp1_bf16.py | 3 +- .../test_npu_graph_tp2_bf16.py | 3 +- .../test_npu_piecewise_graph_prefill.py | 0 .../expert_parallelism/test_npu_deepep.py | 0 ...test_npu_deepep_auto_deepseek_v3_2_w8a8.py | 0 .../test_npu_deepep_auto_qwen3_480b.py | 0 .../test_npu_deepep_auto_qwen3_next.py | 0 ...u_deepep_low_latency_deepseek_v3_2_w8a8.py | 0 .../test_npu_deepep_low_latency_qwen3_480b.py | 0 .../test_npu_deepep_low_latency_qwen3_next.py | 0 .../parameter/deepseek_coder.json | 0 .../parameter/test_npu_fim_completion.py | 0 .../parameter/test_npu_log_level.py | 0 .../test_npu_no_overlap_scheduler.py | 0 .../parameter/test_npu_original_logprobs.py | 0 .../parameter/test_npu_warmups.py | 0 .../quant/test_npu_autoround_dense.py | 0 .../quant/test_npu_autoround_moe.py | 3 +- .../basic_function/quant/test_npu_gguf.py | 0 .../basic_function/quant/test_npu_gguf_moe.py | 0 .../basic_function/quant/test_npu_gptq_moe.py | 0 .../quant/test_npu_w4a4_quantization.py | 0 .../runtime_opts/test_npu_mla_fia_w8a8int8.py | 0 .../runtime_opts/test_npu_mla_w8a8int8.py | 0 .../runtime_opts/test_npu_tp1_bf16.py | 0 .../runtime_opts/test_npu_tp2_bf16.py | 0 .../runtime_opts/test_npu_tp2_fia_bf16.py | 0 .../runtime_opts/test_npu_tp4_bf16.py | 0 .../speculative_inference/test_npu_eagle3.py | 0 .../test_npu_bge_large_en_v1_5.py | 0 .../{ascend => npu}/interface/test_npu_api.py | 0 .../interface/test_npu_api_abort_request.py | 0 .../interface/test_npu_api_encode.py | 0 .../interface/test_npu_enable_thinking.py | 0 .../interface/test_npu_matched_stop.py | 0 .../test_npu_openai_function_calling.py | 0 .../test_npu_openai_server_ignore_eos.py | 0 .../interface/test_npu_penalty.py | 0 .../llm_models/test_ascend_dbrx_instruct.py | 0 .../llm_models/test_ascend_minimax_m2.py | 0 .../llm_models/test_ascend_trinity_mini.py | 0 .../llm_models/test_npu_afm_4_5b.py | 0 .../llm_models/test_npu_baichuan2_13b_chat.py | 0 .../llm_models/test_npu_c4ai_command_r_v01.py | 0 .../llm_models/test_npu_chatglm2_6b.py | 0 .../test_npu_deepseek_v3_2_exp_w8a8.py | 0 .../llm_models/test_npu_exaone_3.py | 0 .../llm_models/test_npu_gemma_3_4b_it_llm.py | 0 .../llm_models/test_npu_glm4_9b_chat.py | 0 .../test_npu_granite_3_0_3b_a800m.py | 0 .../llm_models/test_npu_granite_3_1_8b.py | 0 .../llm_models/test_npu_grok_2.py | 0 .../llm_models/test_npu_internlm2_7b.py | 0 .../llm_models/test_npu_ling_lite.py | 0 .../test_npu_llama4_scount_17b_16e.py | 0 .../llm_models/test_npu_llama_2_7b.py | 0 ...u_llama_2_7b_communications_compression.py | 0 .../llm_models/test_npu_mimo_7b_rl.py | 0 .../llm_models/test_npu_minicpm3_4b.py | 0 .../llm_models/test_npu_mistral_7b.py | 0 .../llm_models/test_npu_persimmon_8b_chat.py | 0 .../test_npu_phi_4_multimodal_llm.py | 0 .../llm_models/test_npu_qwen3_0_6b.py | 0 .../test_npu_qwen3_1_7b_gptq_int8.py | 0 .../test_npu_qwen3_235b_a22b_w8a8.py | 0 .../llm_models/test_npu_qwen3_30b.py | 0 .../llm_models/test_npu_qwen3_30b_attn_cp.py | 0 .../llm_models/test_npu_qwen3_30b_fuseep.py | 0 .../llm_models/test_npu_qwen3_30b_w4a4.py | 0 .../llm_models/test_npu_qwen3_32b.py | 0 ...pu_qwen3_8b_communications_quantization.py | 0 .../test_npu_qwen3_coder_480b_a35b.py | 0 .../llm_models/test_npu_qwq_32b_w8a8.py | 0 .../llm_models/test_npu_smollm_1_7b.py | 0 .../llm_models/test_npu_stablelm_2_1_6b.py | 0 ...ool_chat_template_c4ai_command_r_v01.jinja | 0 ...epseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py | 0 ...u_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py | 189 ++++++++++++++++++ ...pu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py | 113 +++++++++++ ...npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py | 128 ++++++++++++ ...mo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py | 0 ...v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py | 0 ...m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py} | 18 +- ...minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py} | 17 -- ..._npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py | 109 ++++++++++ ...t_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py | 94 +++++++++ ...pu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py | 125 ++++++++++++ ...npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py | 114 +++++++++++ ...t_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py | 0 ...npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py | 111 ++++++++++ ..._qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py | 20 -- ..._qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py | 140 +++++++++++++ ...n3_6_27b_1p_in1024x1024_30_out1024_50ms.py | 0 ...u_qwen3_6_27b_1p_in1080p_30_out256_50ms.py | 0 ...wen3_6_27b_2p_in64k_out1k_prefix90_50ms.py | 0 ..._qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py} | 23 --- ...pu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py | 0 ...u_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py | 0 ...pu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py | 0 ...pu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py | 0 ...pu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py | 0 ...6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py | 0 ...pu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py | 0 ...npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py | 0 ...6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py} | 32 --- ...qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py | 115 +++++++++++ .../test_npu_bge_reranker_v2_m3.py | 0 .../test_npu_gemma_2_27b_v0_2.py | 0 .../test_npu_internlm2_7b_reward.py | 0 .../test_npu_llama_3_1_8b_v0_2.py | 0 .../test_npu_memory_consumption.py | 0 .../{ascend => npu}/vlm_models/mmmu-val.yaml | 0 .../vlm_models/test_ascend_glm_4_5v.py | 0 .../vlm_models/test_npu_deepseek_vl2.py | 0 .../vlm_models/test_npu_gemma_3_4b_it.py | 0 .../vlm_models/test_npu_janus_pro_1b.py | 0 .../vlm_models/test_npu_janus_pro_7b.py | 0 .../test_npu_kimi_vl_a3b_instruct.py | 0 .../test_npu_llama_3_2_11b_vision_instruct.py | 0 .../vlm_models/test_npu_mimo_vl_7b_rl.py | 0 .../vlm_models/test_npu_minicpm_o_2_6.py | 0 .../vlm_models/test_npu_minicpm_v_2_6.py | 0 ...npu_mistral_small_3_1_24b_instruct_2503.py | 0 .../test_npu_phi4_multimodal_instruct.py | 0 .../test_npu_qwen2_5_vl_3b_instruct.py | 0 .../test_npu_qwen2_5_vl_72b_instruct.py | 0 .../test_npu_qwen3_vl_235b_a22b_instruct.py | 0 .../test_npu_qwen3_vl_30b_a3b_instruct.py | 0 .../test_npu_qwen3_vl_4b_instruct.py | 0 .../test_npu_qwen3_vl_8b_instruct.py | 0 170 files changed, 1968 insertions(+), 329 deletions(-) rename test/registered/{ascend => npu}/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py (100%) rename test/registered/{ascend => npu}/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py (100%) rename test/registered/{ascend => npu}/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py (100%) rename test/registered/{ascend/performance => npu/accuracy}/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py (84%) rename test/registered/{ascend/performance => npu/accuracy}/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py (77%) create mode 100644 test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py create mode 100644 test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py rename test/registered/{ascend/performance => npu/accuracy}/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py (76%) rename test/registered/{ascend/performance => npu/accuracy}/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py (79%) rename test/registered/{ascend/performance => npu/accuracy}/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py (76%) rename test/registered/{ascend => npu}/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py (100%) rename test/registered/{ascend/performance => npu/accuracy}/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py (75%) create mode 100644 test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py rename test/registered/{ascend => npu}/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py (100%) create mode 100644 test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py rename test/registered/{ascend => npu}/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py (100%) create mode 100644 test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py rename test/registered/{ascend/performance => npu/accuracy}/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py (85%) rename test/registered/{ascend => npu}/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py (100%) rename test/registered/{ascend => npu}/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py (100%) rename test/registered/{ascend => npu}/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py (89%) rename test/registered/{ascend => npu}/basic_function/HiCache/test_npu_hicache_mha.py (100%) rename test/registered/{ascend => npu}/basic_function/HiCache/test_npu_hicache_mla.py (100%) rename test/registered/{ascend => npu}/basic_function/HiCache/test_npu_hierarchical_cache.py (100%) rename test/registered/{ascend => npu}/basic_function/HiCache/test_npu_hierarchical_cache_mla.py (100%) rename test/registered/{ascend => npu}/basic_function/HiCache/test_npu_hierarchical_cache_mutually_exclusive.py (100%) rename test/registered/{ascend => npu}/basic_function/HiCache/test_npu_hierarchical_cache_ttft_mha.py (100%) rename test/registered/{ascend => npu}/basic_function/HiCache/test_npu_radix_cache.py (100%) rename test/registered/{ascend => npu}/basic_function/backends/test_npu_sampling_backend.py (100%) rename test/registered/{ascend => npu}/basic_function/dllm/test_npu_llada2_mini.py (100%) rename test/registered/{ascend => npu}/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py (100%) rename test/registered/{ascend => npu}/basic_function/offloading/test_npu_offload_modes.py (100%) rename test/registered/{ascend => npu}/basic_function/optimization_debug/test_npu_compile_graph_tp1_bf16.py (100%) rename test/registered/{ascend => npu}/basic_function/optimization_debug/test_npu_graph_tp1_bf16.py (93%) rename test/registered/{ascend => npu}/basic_function/optimization_debug/test_npu_graph_tp2_bf16.py (93%) rename test/registered/{ascend => npu}/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py (100%) rename test/registered/{ascend => npu}/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py (100%) rename test/registered/{ascend => npu}/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_deepseek_v3_2_w8a8.py (100%) rename test/registered/{ascend => npu}/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py (100%) rename test/registered/{ascend => npu}/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py (100%) rename test/registered/{ascend => npu}/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py (100%) rename test/registered/{ascend => npu}/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py (100%) rename test/registered/{ascend => npu}/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py (100%) rename test/registered/{ascend => npu}/basic_function/parameter/deepseek_coder.json (100%) rename test/registered/{ascend => npu}/basic_function/parameter/test_npu_fim_completion.py (100%) rename test/registered/{ascend => npu}/basic_function/parameter/test_npu_log_level.py (100%) rename test/registered/{ascend => npu}/basic_function/parameter/test_npu_no_overlap_scheduler.py (100%) rename test/registered/{ascend => npu}/basic_function/parameter/test_npu_original_logprobs.py (100%) rename test/registered/{ascend => npu}/basic_function/parameter/test_npu_warmups.py (100%) rename test/registered/{ascend => npu}/basic_function/quant/test_npu_autoround_dense.py (100%) rename test/registered/{ascend => npu}/basic_function/quant/test_npu_autoround_moe.py (95%) rename test/registered/{ascend => npu}/basic_function/quant/test_npu_gguf.py (100%) rename test/registered/{ascend => npu}/basic_function/quant/test_npu_gguf_moe.py (100%) rename test/registered/{ascend => npu}/basic_function/quant/test_npu_gptq_moe.py (100%) rename test/registered/{ascend => npu}/basic_function/quant/test_npu_w4a4_quantization.py (100%) rename test/registered/{ascend => npu}/basic_function/runtime_opts/test_npu_mla_fia_w8a8int8.py (100%) rename test/registered/{ascend => npu}/basic_function/runtime_opts/test_npu_mla_w8a8int8.py (100%) rename test/registered/{ascend => npu}/basic_function/runtime_opts/test_npu_tp1_bf16.py (100%) rename test/registered/{ascend => npu}/basic_function/runtime_opts/test_npu_tp2_bf16.py (100%) rename test/registered/{ascend => npu}/basic_function/runtime_opts/test_npu_tp2_fia_bf16.py (100%) rename test/registered/{ascend => npu}/basic_function/runtime_opts/test_npu_tp4_bf16.py (100%) rename test/registered/{ascend => npu}/basic_function/speculative_inference/test_npu_eagle3.py (100%) rename test/registered/{ascend => npu}/embedding_models/test_npu_bge_large_en_v1_5.py (100%) rename test/registered/{ascend => npu}/interface/test_npu_api.py (100%) rename test/registered/{ascend => npu}/interface/test_npu_api_abort_request.py (100%) rename test/registered/{ascend => npu}/interface/test_npu_api_encode.py (100%) rename test/registered/{ascend => npu}/interface/test_npu_enable_thinking.py (100%) rename test/registered/{ascend => npu}/interface/test_npu_matched_stop.py (100%) rename test/registered/{ascend => npu}/interface/test_npu_openai_function_calling.py (100%) rename test/registered/{ascend => npu}/interface/test_npu_openai_server_ignore_eos.py (100%) rename test/registered/{ascend => npu}/interface/test_npu_penalty.py (100%) rename test/registered/{ascend => npu}/llm_models/test_ascend_dbrx_instruct.py (100%) rename test/registered/{ascend => npu}/llm_models/test_ascend_minimax_m2.py (100%) rename test/registered/{ascend => npu}/llm_models/test_ascend_trinity_mini.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_afm_4_5b.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_baichuan2_13b_chat.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_c4ai_command_r_v01.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_chatglm2_6b.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_deepseek_v3_2_exp_w8a8.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_exaone_3.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_gemma_3_4b_it_llm.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_glm4_9b_chat.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_granite_3_0_3b_a800m.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_granite_3_1_8b.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_grok_2.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_internlm2_7b.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_ling_lite.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_llama4_scount_17b_16e.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_llama_2_7b.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_llama_2_7b_communications_compression.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_mimo_7b_rl.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_minicpm3_4b.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_mistral_7b.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_persimmon_8b_chat.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_phi_4_multimodal_llm.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_qwen3_0_6b.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_qwen3_1_7b_gptq_int8.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_qwen3_235b_a22b_w8a8.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_qwen3_30b.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_qwen3_30b_attn_cp.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_qwen3_30b_fuseep.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_qwen3_30b_w4a4.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_qwen3_32b.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_qwen3_8b_communications_quantization.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_qwen3_coder_480b_a35b.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_qwq_32b_w8a8.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_smollm_1_7b.py (100%) rename test/registered/{ascend => npu}/llm_models/test_npu_stablelm_2_1_6b.py (100%) rename test/registered/{ascend => npu}/llm_models/tool_chat_template_c4ai_command_r_v01.jinja (100%) rename test/registered/{ascend => npu}/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py (100%) create mode 100644 test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py create mode 100644 test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py create mode 100644 test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py rename test/registered/{ascend => npu}/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py (100%) rename test/registered/{ascend => npu}/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py (100%) rename test/registered/{ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py => npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py} (84%) rename test/registered/{ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py => npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py} (84%) create mode 100644 test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py create mode 100644 test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py create mode 100644 test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py create mode 100644 test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py rename test/registered/{ascend => npu}/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py (100%) create mode 100644 test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py rename test/registered/{ascend => npu}/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py (79%) create mode 100644 test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py rename test/registered/{ascend => npu}/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py (100%) rename test/registered/{ascend => npu}/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py (100%) rename test/registered/{ascend => npu}/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py (100%) rename test/registered/{ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py => npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py} (79%) rename test/registered/{ascend => npu}/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py (100%) rename test/registered/{ascend => npu}/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py (100%) rename test/registered/{ascend => npu}/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py (100%) rename test/registered/{ascend => npu}/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py (100%) rename test/registered/{ascend => npu}/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py (100%) rename test/registered/{ascend => npu}/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py (100%) rename test/registered/{ascend => npu}/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py (100%) rename test/registered/{ascend => npu}/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py (100%) rename test/registered/{ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py => npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py} (75%) create mode 100644 test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py rename test/registered/{ascend => npu}/rerank_models/test_npu_bge_reranker_v2_m3.py (100%) rename test/registered/{ascend => npu}/reward_models/test_npu_gemma_2_27b_v0_2.py (100%) rename test/registered/{ascend => npu}/reward_models/test_npu_internlm2_7b_reward.py (100%) rename test/registered/{ascend => npu}/reward_models/test_npu_llama_3_1_8b_v0_2.py (100%) rename test/registered/{ascend => npu}/test_npu_memory_consumption.py (100%) rename test/registered/{ascend => npu}/vlm_models/mmmu-val.yaml (100%) rename test/registered/{ascend => npu}/vlm_models/test_ascend_glm_4_5v.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_deepseek_vl2.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_gemma_3_4b_it.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_janus_pro_1b.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_janus_pro_7b.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_kimi_vl_a3b_instruct.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_llama_3_2_11b_vision_instruct.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_mimo_vl_7b_rl.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_minicpm_o_2_6.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_minicpm_v_2_6.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_mistral_small_3_1_24b_instruct_2503.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_phi4_multimodal_instruct.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_qwen2_5_vl_3b_instruct.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_qwen2_5_vl_72b_instruct.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_qwen3_vl_235b_a22b_instruct.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_qwen3_vl_30b_a3b_instruct.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_qwen3_vl_4b_instruct.py (100%) rename test/registered/{ascend => npu}/vlm_models/test_npu_qwen3_vl_8b_instruct.py (100%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9730ceb3d..1ae931b65 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -90,7 +90,7 @@ /sgl-model-gateway/examples/wasm @slin1237 /test/registered/prefill_only @sundar24295s @chanh @fortunecookiee /benchmark/prefill_only/bench_score.py @sundar24295s @chanh @fortunecookiee -/test/registered/ascend @ping1jing2 @ssshinigami @e-martirosian +/test/registered/npu @ping1jing2 @ssshinigami @e-martirosian /test/srt/test_modelopt* @Edwardf0t1 /python/sglang/srt/layers/gemma4_fused_ops.py @merrymercy @Ying1123 @Fridge003 @ispobock @HaiShaw @ch-wan @BBuf @Edwardf0t1 @kpham-sgl @pyc96 /python/sglang/srt/function_call/gemma4_detector.py @CatherineSue @JustinTong0323 @kpham-sgl @pyc96 diff --git a/.github/workflows/nightly-test-npu.yml b/.github/workflows/nightly-test-npu.yml index c261e66fc..7b6216230 100644 --- a/.github/workflows/nightly-test-npu.yml +++ b/.github/workflows/nightly-test-npu.yml @@ -114,8 +114,11 @@ jobs: test_config: - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2 runner: linux-aarch64-a2-4 - test_case: test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py + test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py test_type: 'perf' + - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2 + runner: linux-aarch64-a2-4 + test_case: test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml with: runner: ${{ matrix.test_config.runner }} @@ -138,119 +141,158 @@ jobs: # qwen3_6_35b_a3b performance tests - name: qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py + test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py test_type: 'perf' - name: qwen3_6_35b_a3b_1p_aime26 runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py + test_case: test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py - name: qwen3_6_35b_a3b_1p_in64k_out1k_50ms runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py + test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py test_type: 'perf' - name: qwen3_6_35b_a3b_1p_in128k_out1k_50ms runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py + test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py + test_type: 'perf' + - name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms + runner: linux-aarch64-a3-2 + test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py test_type: 'perf' - name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26 runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py - test_type: 'perf' + test_case: test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py - name: qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py + test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py test_type: 'perf' # qwen3_6_27b performance tests + - name: qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms + runner: linux-aarch64-a3-2 + test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py + test_type: 'perf' - name: qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py - test_type: 'perf' + test_case: test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py - name: qwen3_6_27b_w8a8_2p_in16k_out1k_50ms runner: linux-aarch64-a3-4 - test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py + test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py test_type: 'perf' - name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms runner: linux-aarch64-a3-4 - test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py + test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py test_type: 'perf' - name: qwen3_6_27b_w8a8_2p_in128k_out1k_50ms runner: linux-aarch64-a3-4 - test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py + test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py test_type: 'perf' - name: qwen3_6_27b_2p_in64k_out1k_prefix90_50ms runner: linux-aarch64-a3-4 - test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py + test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py test_type: 'perf' - name: qwen3_6_27b_1p_in1024x1024_30_out1024_50ms runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py + test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py test_type: 'perf' - name: qwen3_6_27b_1p_in1080p_30_out256_50ms runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py + test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py test_type: 'perf' - name: qwen3_6_27b_1p_gpqa runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py + test_case: test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py # qwen3_32b performance tests + - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms + runner: linux-aarch64-a3-4 + test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py + test_type: 'perf' - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa runner: linux-aarch64-a3-4 - test_case: test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py - test_type: 'perf' + test_case: test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py - name: qwen3_32b_bf16_8p_in18k_out4k_6ms runner: linux-aarch64-a3-16 - test_case: test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py + test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py test_type: 'perf' - name: qwen3_32b_bf16_8p_gpqa runner: linux-aarch64-a3-16 - test_case: test/registered/ascend/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py + test_case: test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py # qwen3_30b_a3b performance tests + - name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms + runner: linux-aarch64-a3-2 + test_case: test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py + test_type: 'perf' - name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25 runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py - test_type: 'perf' + test_case: test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py # qwen3-8b performance tests + - name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms + runner: linux-aarch64-a3-2 + test_case: test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py + test_type: 'perf' - name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py + test_case: test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py + - name: qwen3_8b_w8a8_1p_in6k_out1k5_bs16 + runner: linux-aarch64-a3-2 + test_case: test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py test_type: 'perf' - name: qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py - test_type: 'perf' + test_case: test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py # qwen3_next_80b_a3b_instruct performance tests + - name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16 + runner: linux-aarch64-a3-4 + test_case: test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py + test_type: 'perf' - name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25 runner: linux-aarch64-a3-4 - test_case: test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py - test_type: 'perf' + test_case: test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py # minimax_m2_5 performance tests + - name: minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms + runner: linux-aarch64-a3-16 + test_case: test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py + test_type: 'perf' - name: minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa runner: linux-aarch64-a3-16 - test_case: test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py + test_case: test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py + - name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms + runner: linux-aarch64-a3-16 + test_case: test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py test_type: 'perf' - name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa runner: linux-aarch64-a3-16 - test_case: test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py - test_type: 'perf' + test_case: test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py # deepseek_v3_2 accuracy tests - name: deepseek_v3_2_8p_aime25 runner: linux-aarch64-a3-16 - test_case: test/registered/ascend/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py + test_case: test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py # glm4_7_flash accuracy tests - name: glm4_7_flash_1p_aime25 runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py + test_case: test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py # qwen3_vl_8b_thinking accuracy tests - name: qwen3_vl_8b_thinking_1p_mmmu runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py + test_case: test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py # qwen3_vl_30b_a3b_thinking accuracy tests - name: qwen3_vl_30b_a3b_thinking_1p_mmmu runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py + test_case: test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py + - name: qwen3_235b_w8a8_8p_in3k5_out1k5_50ms + runner: linux-aarch64-a3-16 + test_case: test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py + test_type: 'perf' + - name: qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms + runner: linux-aarch64-a3-16 + test_case: test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py + test_type: 'perf' + - name: kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms + runner: linux-aarch64-a3-16 + test_case: test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py + test_type: 'perf' # deepseek_v4_flash performance tests - name: deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms runner: linux-aarch64-a3-16 - test_case: test/registered/ascend/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py + test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py test_type: 'perf' uses: ./.github/workflows/nightly-test-npu-e2e-single-node.yml with: @@ -272,26 +314,32 @@ jobs: matrix: test_config: # glm5_1 performance tests + - name: glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms + prefill_size: 2 + decode_size: 2 + router_size: 1 + test_case: test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py + test_type: 'perf' + prefill_decode_deployment: 'separation' - name: glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26 prefill_size: 2 decode_size: 2 router_size: 1 - test_case: test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py - test_type: 'perf' + test_case: test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py prefill_decode_deployment: 'separation' # mimo_v2_flash performance tests - name: test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s prefill_size: 1 decode_size: 1 router_size: 1 - test_case: test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py + test_case: test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py test_type: 'perf' prefill_decode_deployment: 'separation' - name: test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms prefill_size: 1 decode_size: 1 router_size: 1 - test_case: test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py + test_case: test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py test_type: 'perf' prefill_decode_deployment: 'separation' uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml @@ -318,10 +366,13 @@ jobs: matrix: test_config: # kimi_k2_6 performance tests + - name: kimi_k2_6_w4a8_16p_in64k_out1k_100ms + node_size: 2 + test_case: test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py + test_type: 'perf' - name: kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25 node_size: 2 - test_case: test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py - test_type: 'perf' + test_case: test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml with: runner: linux-amd64-cpu-8 diff --git a/.github/workflows/pr-test-npu.yml b/.github/workflows/pr-test-npu.yml index 600d52ebb..3df37dab5 100644 --- a/.github/workflows/pr-test-npu.yml +++ b/.github/workflows/pr-test-npu.yml @@ -59,7 +59,7 @@ jobs: - "python/sglang/!(multimodal_gen)/**/!(*.md)" - "python/pyproject_npu.toml" - "scripts/ci/npu/npu_ci_install_dependency.sh" - - "test/registered/ascend/**" + - "test/registered/npu/**" - "test/registered/unit/npu/**" - ".github/workflows/pr-test-npu.yml" multimodal_gen: @@ -461,7 +461,78 @@ jobs: # qwen3_6_27b performance tests - name: qwen3_6_27b_w8a8_1p_in64k_out1k_50ms runner: linux-aarch64-a3-2 - test_case: test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py + test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py + test_type: 'perf' + + - name: qwen3_6_27b_1p_in1024x1024_30_out1024_50ms + runner: linux-aarch64-a3-2 + test_case: test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py + test_type: 'perf' + + # qwen3_8b performance tests + - name: qwen3_8b_w8a8_1p_in3k5_out1k5_50ms + runner: linux-aarch64-a3-2 + test_case: test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py + test_type: 'perf' + + # qwen3_30b_a3b performance tests + - name: qwen3_30b_w8a8_1p_in3k5_out1k5_50ms + runner: linux-aarch64-a3-2 + test_case: test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py + test_type: 'perf' + + # qwen3_6_35b_a3b performance tests + - name: qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms + runner: linux-aarch64-a3-2 + test_case: test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py + test_type: 'perf' + + # qwen3_vl_8b_thinking accuracy tests + - name: qwen3_vl_8b_thinking_1p_mmmu + runner: linux-aarch64-a3-2 + test_case: test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py + test_type: 'accuracy' + + # qwen3_32b performance tests + - name: qwen3_32b_w8a8_2p_in3k5_out1k5_50ms + runner: linux-aarch64-a3-4 + test_case: test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py + test_type: 'perf' + + # qwen3_next_80b_a3b performance tests + - name: qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16 + runner: linux-aarch64-a3-4 + test_case: test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py + test_type: 'perf' + + # minimax_m2_5 performance tests + - name: minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms + runner: linux-aarch64-a3-8 + test_case: test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py + test_type: 'perf' + + # deepseek_v4_flash performance tests + - name: deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms + runner: linux-aarch64-a3-16 + test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py + test_type: 'perf' + + # kimi_k2_6 performance tests + - name: kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms + runner: linux-aarch64-a3-16 + test_case: test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py + test_type: 'perf' + + # qwen3_235b performance tests + - name: qwen3_235b_w8a8_8p_in3k5_out1k5_50ms + runner: linux-aarch64-a3-16 + test_case: test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py + test_type: 'perf' + + # qwen3_5_397b performance tests + - name: qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms + runner: linux-aarch64-a3-16 + test_case: test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py test_type: 'perf' # NPU accuracy tests - name: glm4_7_flash_1p_gsm8k diff --git a/docs_new/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx b/docs_new/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx index d0bd47100..25bb50ff1 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx @@ -42,7 +42,7 @@ pre-commit run --all-files ## Run and add tests All NPU tests are end-to-end (E2E) and require launching a server with real model weights. -Tests live under [`test/registered/ascend/`](https://github.com/sgl-project/sglang/tree/main/test/registered/ascend), organized by model type and functionality: +Tests live under [`test/registered/npu/`](https://github.com/sgl-project/sglang/tree/main/test/registered/npu), organized by model type and functionality: ``` ascend/ @@ -58,9 +58,9 @@ ascend/ ### Adding a test -See [`test_npu_sampling_backend.py`](https://github.com/sgl-project/sglang/tree/main/test/registered/ascend/basic_function/backends/test_npu_sampling_backend.py) for a complete example. Key steps: +See [`test_npu_sampling_backend.py`](https://github.com/sgl-project/sglang/tree/main/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py) for a complete example. Key steps: -1. Place your test file in the appropriate directory under `test/registered/ascend/`. +1. Place your test file in the appropriate directory under `test/registered/npu/`. 2. Extend `CustomTestCase` (from `sglang.test.test_utils`) for CI retry support. 3. Launch server with `popen_launch_server()` in `setUpClass` and clean up with `kill_process_tree()` in `tearDownClass`. 4. Register your test with `register_npu_ci()`: @@ -73,7 +73,7 @@ See [`test_npu_sampling_backend.py`](https://github.com/sgl-project/sglang/tree/ ### Running tests locally ```bash -pytest test/registered/ascend/llm_models/test_npu_qwen3_0_6b.py -v +pytest test/registered/npu/llm_models/test_npu_qwen3_0_6b.py -v ``` For detailed instructions, refer to [test/README.md](https://github.com/sgl-project/sglang/tree/main/test/README.md). diff --git a/docs_new/docs/hardware-platforms/ascend-npus/development/support_new_models.mdx b/docs_new/docs/hardware-platforms/ascend-npus/development/support_new_models.mdx index 348f1f60d..0982a85f8 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/development/support_new_models.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/development/support_new_models.mdx @@ -71,7 +71,7 @@ Please note all your testing and benchmarking results in PR description. - **(Optional) Other evals**: If you ran other evals, please note the results in PR description. -For NPU-adapted models: add the corresponding test under `test/registered/ascend/` and verify correctness on Ascend NPU +For NPU-adapted models: add the corresponding test under `test/registered/npu/` and verify correctness on Ascend NPU hardware; run benchmarks on the NPU device and report performance metrics (TTFT, throughput), comparing against SGLang GPU results as the primary baseline. Fall back to HF Transformer comparison when no GPU adaptation is available. diff --git a/python/sglang/test/ascend/test_ascend_utils.py b/python/sglang/test/ascend/test_ascend_utils.py index 3c63bbc50..1e1ac3196 100644 --- a/python/sglang/test/ascend/test_ascend_utils.py +++ b/python/sglang/test/ascend/test_ascend_utils.py @@ -53,7 +53,7 @@ BAICHUAN2_13B_CHAT_WEIGHTS_PATH = os.path.join( C4AI_COMMAND_R_V01_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "CohereForAI/c4ai-command-r-v01" ) -C4AI_COMMAND_R_V01_CHAT_TEMPLATE_PATH = "/__w/sglang/sglang/test/registered/ascend/llm_models/tool_chat_template_c4ai_command_r_v01.jinja" +C4AI_COMMAND_R_V01_CHAT_TEMPLATE_PATH = "/__w/sglang/sglang/test/registered/npu/llm_models/tool_chat_template_c4ai_command_r_v01.jinja" CHATGLM2_6B_WEIGHTS_PATH = os.path.join(MODEL_WEIGHTS_DIR, "ZhipuAI/chatglm2-6b") DBRX_INSTRUCT_WEIGHTS_PATH = os.path.join( MODEL_WEIGHTS_DIR, "AI-ModelScope/dbrx-instruct" @@ -378,10 +378,10 @@ INVOICE_WITH_BARCODE_LOGO_IMAGES_PATH = os.path.join( # fmt: on # Other -DEEPSEEK_CODER_JSON_PATH = "/__w/sglang/sglang/test/registered/ascend/basic_function/parameter/deepseek_coder.json" +DEEPSEEK_CODER_JSON_PATH = "/__w/sglang/sglang/test/registered/npu/basic_function/parameter/deepseek_coder.json" FR_SPEC_TOKEN_MAP_PATH = "/root/.cache/sglang/FR-Spec/freq_32768.pt" CONFIG_YAML_PATH = ( - "/__w/sglang/sglang/test/registered/ascend/basic_function/config/config.yaml" + "/__w/sglang/sglang/test/registered/npu/basic_function/config/config.yaml" ) diff --git a/test/registered/README.md b/test/registered/README.md index 433c9ffb3..34f0c00f6 100644 --- a/test/registered/README.md +++ b/test/registered/README.md @@ -18,6 +18,6 @@ Tests under this directory are auto-discovered by `run_suite.py` via CI registra | Model inference correctness | `models/`, `4-gpu-models/`, `8-gpu-models/` | GPU | | Feature-specific (OpenAI API, LoRA, speculative, distributed, VLM, etc.) | `openai_server/`, `lora/`, `spec/`, `distributed/`, ... | GPU | | Benchmarks (performance, accuracy, stress) | `benchmark/` | GPU | -| Platform-specific | `amd/`, `ascend/` | Vendor GPU | +| Platform-specific | `amd/`, `npu/` | Vendor GPU | See [`unit/README.md`](unit/README.md) for unit test conventions. diff --git a/test/registered/ascend/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py b/test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py similarity index 100% rename from test/registered/ascend/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py rename to test/registered/npu/accuracy/deepseek_v3_2/test_npu_deepseek_v3_2_8p_aime25.py diff --git a/test/registered/ascend/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py b/test/registered/npu/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py similarity index 100% rename from test/registered/ascend/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py rename to test/registered/npu/accuracy/glm4_6v_flash/test_npu_glm4_6v_flash_1p_mmmu.py diff --git a/test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py b/test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py similarity index 100% rename from test/registered/ascend/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py rename to test/registered/npu/accuracy/glm4_7_flash/test_npu_glm4_7_flash_1p_aime25.py diff --git a/test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py b/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py similarity index 84% rename from test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py rename to test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py index 708a9f9b0..bd977dfb6 100644 --- a/test/registered/ascend/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py +++ b/test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py @@ -5,10 +5,7 @@ from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( ) from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, GLM_5_1_W4A8_MODEL_PATH, - TestNpuPerfMultiNodePdSepTestCaseBase, ) from sglang.test.ci.ci_register import register_npu_ci @@ -16,7 +13,7 @@ register_npu_ci( est_time=3600, suite="", nightly=True, - disabled="performance testcase", + disabled="accuracy testcase", ) GLM_5_1_PD_SEP_PREFILL_ENVS = { @@ -181,26 +178,5 @@ class TestNPUGLM5_1_W4A8_PD_SEP_AIME2026(TestNpuAccuracyMultiNodePdSepTestCaseBa self.run_accuracy() -class TestNPUGLM5_1_W4A8_PD_SEP_In3k5_Out1k5(TestNpuPerfMultiNodePdSepTestCaseBase): - """Test NPU performance for GLM-5.1-w4a8 PD separation 4 nodes in3k5 out1k5""" - - model_config = GLM_5_1_PD_SEP_MODEL_CONFIG - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - dataset_name = "random" - max_concurrency = 1 - num_prompts = 1 - input_len = 65536 - output_len = 1024 - random_range_ratio = 1 - seed = 1 - tpot = 50 - output_token_throughput = 160 - - def test_npu_glm5_1_w4a8_pd_sep_in3k5_out1k5(self): - """Run NPU performance test for GLM-5.1-w4a8 PD separation""" - self.run_throughput() - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py b/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py similarity index 77% rename from test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py rename to test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py index acf865171..24cd79560 100644 --- a/test/registered/ascend/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py +++ b/test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py @@ -5,11 +5,8 @@ from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( ) from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, KIMI_K2_6_EAGLE3_MODEL_PATH, KIMI_K2_6_W4A8_MODEL_PATH, - TestNpuPerfMultiNodePdMixTestCaseBase, ) from sglang.test.ci.ci_register import register_npu_ci @@ -17,7 +14,7 @@ register_npu_ci( est_time=1800, suite="nightly-8-npu-a3", nightly=True, - disabled="Currently it is executed by the npu performance workflow.", + disabled="accuracy testcase", ) ENVS = { @@ -107,23 +104,5 @@ class TestNPUKimiK2_6_W4A8_16P_AIME2025(TestNpuAccuracyMultiNodePdMixTestCaseBas self.run_accuracy() -class TestNPUKimiK2_6_W4A8_16P_In64k_Out1k_100ms(TestNpuPerfMultiNodePdMixTestCaseBase): - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - model_config = MODEL_CONFIG - dataset_name = "random" - max_concurrency = 32 - num_prompts = 32 - input_len = 64000 - output_len = 1000 - random_range_ratio = 1 - seed = 1 - tpot = 100 - output_token_throughput = 160 - - def test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms(self): - self.run_throughput() - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py new file mode 100644 index 000000000..250e3c002 --- /dev/null +++ b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py @@ -0,0 +1,109 @@ +import os +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + MINIMAX_M2_5_EAGLE3_MODEL_PATH, + MINIMAX_M2_5_W8A8_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="accuracy testcase", +) + +MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "TASK_QUEUE_ENABLE": "1", + "ASCEND_USE_FIA": "1", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_NPU_FUSED_MOE_MODE": "2", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "140000", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "DEEPEP_HCCL_BUFFSIZE": "1024", + "SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3", + "PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}", +} + +MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS = [ + "--tp-size", + 8, + "--mem-fraction-static", + 0.63, + "--max-running-requests", + 26, + "--reasoning-parser", + "minimax-append-think", + "--tool-call-parser", + "minimax-m2", + "--enable-prefill-delayer", + "--prefill-max-requests", + 10, + "--chunked-prefill-size", + 67072, + "--max-prefill-token", + 67000, + "--cuda-graph-bs", + 2, + 4, + 8, + 12, + 16, + 18, + 20, + 22, + 24, + 26, + "--moe-a2a-backend", + "ascend_fuseep", + "--deepep-mode", + "auto", + "--quantization", + "modelslim", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + MINIMAX_M2_5_EAGLE3_MODEL_PATH, + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--speculative-draft-model-quantization", + "unquant", + "--dtype", + "bfloat16", + "--trust-remote-code", + "--reasoning-parser", + "minimax-append-think", + "--tool-call-parser", + "minimax-m2", +] + + +class TestNPUMiniMaxM2_5_W8A8_4P_Gpqa(TestNpuAccuracyTestCaseBase): + model = MINIMAX_M2_5_W8A8_MODEL_PATH + other_args = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS + envs = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS + accuracy = 0.852 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + generation_config = {"max_tokens": 65536, "temperature": 1.0} + eval_batch_size = 64 + + def test_accuracy(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py new file mode 100644 index 000000000..a92462eea --- /dev/null +++ b/test/registered/npu/accuracy/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py @@ -0,0 +1,107 @@ +import os +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + MINIMAX_M2_5_EAGLE3_MODEL_PATH, + MINIMAX_M2_5_W8A8_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="full-16-npu-a3", + nightly=True, + disabled="accuracy testcase", +) + +MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "TASK_QUEUE_ENABLE": "1", + "DEEPEP_HCCL_BUFFSIZE": "1024", + "ASCEND_USE_FIA": "1", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "204800", + "PYTHONPATH": f"{MINIMAX_M2_5_EAGLE3_MODEL_PATH}:{os.environ.get('PYTHONPATH', '')}", + "SGLANG_EXTERNAL_MODEL_PACKAGE": "custom_eagle3", +} + +MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS = [ + "--tp-size", + 16, + "--enable-dp-attention", + "--dp-size", + 16, + "--mem-fraction-static", + 0.75, + "--max-running-requests", + 320, + "--disable-radix-cache", + "--reasoning-parser", + "minimax-append-think", + "--tool-call-parser", + "minimax-m2", + "--prefill-delayer-max-delay-passes", + 500, + "--enable-prefill-delayer", + "--chunked-prefill-size", + 196608, + "--max-prefill-token", + 8192, + "--cuda-graph-bs", + 1, + 2, + 4, + 8, + 12, + 16, + 20, + "--moe-a2a-backend", + "ascend_fuseep", + "--fuseep-mode", + 2, + "--quantization", + "modelslim", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + MINIMAX_M2_5_EAGLE3_MODEL_PATH, + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--speculative-draft-model-quantization", + "unquant", + "--dtype", + "bfloat16", + "--reasoning-parser", + "minimax-append-think", + "--tool-call-parser", + "minimax-m2", +] + + +class TestNPUMiniMaxM2_5_W8A8_8P_In3k5_Out1k5_GPQA(TestNpuAccuracyTestCaseBase): + model = MINIMAX_M2_5_W8A8_MODEL_PATH + envs = MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS + other_args = MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS + accuracy = 0.852 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + generation_config = {"max_tokens": 65536, "temperature": 1.0} + eval_batch_size = 64 + + def test_accuracy(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py similarity index 76% rename from test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py rename to test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py index 207475191..6eb55f49c 100644 --- a/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py @@ -4,11 +4,8 @@ from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( TestNpuAccuracyTestCaseBase, ) from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, QWEN3_8B_EAGLE_MODEL_PATH, QWEN3_8B_W8A8_MODEL_PATH, - TestNpuPerformanceTestCaseBase, ) from sglang.test.ci.ci_register import register_npu_ci @@ -16,7 +13,7 @@ register_npu_ci( est_time=3600, suite="", nightly=True, - disabled="performance testcase", + disabled="accuracy testcase", ) QWEN3_8B_ENVS = { @@ -88,26 +85,6 @@ QWEN3_8B_OTHER_ARGS = [ ] -class TestQwen8B(TestNpuPerformanceTestCaseBase): - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - model = QWEN3_8B_W8A8_MODEL_PATH - other_args = QWEN3_8B_OTHER_ARGS - envs = QWEN3_8B_ENVS - dataset_name = "random" - max_concurrency = 64 - num_prompts = 256 - input_len = 3500 - output_len = 1500 - random_range_ratio = 1 - seed = 1 - tpot = 37 - output_token_throughput = 1586 - - def test_qwen3_8b(self): - self.run_throughput() - - class TestQwen8B_gpqa(TestNpuAccuracyTestCaseBase): model = QWEN3_8B_W8A8_MODEL_PATH envs = QWEN3_8B_ENVS diff --git a/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py similarity index 79% rename from test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py rename to test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py index 0d16ebea4..b336a5087 100644 --- a/test/registered/ascend/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py +++ b/test/registered/npu/accuracy/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16_gpqa.py @@ -6,7 +6,6 @@ from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( from sglang.test.ascend.e2e.test_npu_performance_utils import ( QWEN3_8B_EAGLE_MODEL_PATH, QWEN3_8B_W8A8_MODEL_PATH, - TestNpuPerformanceTestCaseBase, ) from sglang.test.ci.ci_register import register_npu_ci @@ -14,7 +13,7 @@ register_npu_ci( est_time=3600, suite="", nightly=True, - disabled="performance testcase", + disabled="accuracy testcase", ) QWEN3_8B_ENVS = { @@ -75,24 +74,6 @@ QWEN3_8B_OTHER_ARGS = [ ] -class TestQwen8B(TestNpuPerformanceTestCaseBase): - max_attempts = 5 - model = QWEN3_8B_W8A8_MODEL_PATH - other_args = QWEN3_8B_OTHER_ARGS - envs = QWEN3_8B_ENVS - dataset_name = "random" - max_concurrency = 16 - num_prompts = 16 - input_len = 6144 - output_len = 1500 - random_range_ratio = 1 - tpot = 11.79 - output_token_throughput = 1040.96 - - def test_qwen3_8b(self): - self.run_throughput() - - class TestQwen8B_gpqa(TestNpuAccuracyTestCaseBase): model = QWEN3_8B_W8A8_MODEL_PATH envs = QWEN3_8B_ENVS diff --git a/test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py b/test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py similarity index 76% rename from test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py rename to test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py index a4eaa6303..802441071 100644 --- a/test/registered/ascend/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py +++ b/test/registered/npu/accuracy/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms_aime25.py @@ -4,11 +4,8 @@ from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( TestNpuAccuracyTestCaseBase, ) from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH, QWEN3_A3B_EAGLE_MODEL_PATH, - TestNpuPerformanceTestCaseBase, ) from sglang.test.ci.ci_register import register_npu_ci @@ -16,7 +13,7 @@ register_npu_ci( est_time=3600, suite="", nightly=True, - disabled="performance testcase", + disabled="accuracy testcase", ) QWEN3_30B_A3B_ENVS = { @@ -93,26 +90,6 @@ QWEN3_30B_A3B_OTHER_ARGS = [ ] -class TestQwen30B(TestNpuPerformanceTestCaseBase): - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - model = QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH - other_args = QWEN3_30B_A3B_OTHER_ARGS - envs = QWEN3_30B_A3B_ENVS - dataset_name = "random" - max_concurrency = 160 - num_prompts = int(max_concurrency) * 4 - input_len = 3500 - output_len = 1500 - random_range_ratio = 1 - seed = 1 - tpot = 50 - output_token_throughput = 3200 - - def test_qwen3_30b(self): - self.run_throughput() - - class TestQwen30B_A3B_aime25(TestNpuAccuracyTestCaseBase): model = QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH envs = QWEN3_30B_A3B_ENVS diff --git a/test/registered/ascend/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py similarity index 100% rename from test/registered/ascend/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py rename to test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_bf16_8p_gpqa.py diff --git a/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py similarity index 75% rename from test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py rename to test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py index c47d8b3d8..6fb1dad81 100644 --- a/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa.py @@ -4,11 +4,8 @@ from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( TestNpuAccuracyTestCaseBase, ) from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, QWEN3_32B_EAGLE_MODEL_PATH, QWEN3_32B_W8A8_MODEL_PATH, - TestNpuPerformanceTestCaseBase, ) from sglang.test.ci.ci_register import register_npu_ci @@ -16,7 +13,7 @@ register_npu_ci( est_time=3600, suite="", nightly=True, - disabled="performance testcase", + disabled="accuracy testcase", ) QWEN3_32B_ENVS = { @@ -90,27 +87,7 @@ QWEN3_32B_OTHER_ARGS = [ ] -class TestQwen32B(TestNpuPerformanceTestCaseBase): - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - model = QWEN3_32B_W8A8_MODEL_PATH - other_args = QWEN3_32B_OTHER_ARGS - envs = QWEN3_32B_ENVS - dataset_name = "random" - max_concurrency = 100 - num_prompts = 400 - input_len = 3584 - output_len = 1536 - random_range_ratio = 1 - seed = 1 - tpot = 50 - output_token_throughput = 1600 - - def test_qwen3_32b(self): - self.run_throughput() - - -class TestQwen32B_mmlupro(TestNpuAccuracyTestCaseBase): +class TestQwen32B_gpqa(TestNpuAccuracyTestCaseBase): model = QWEN3_32B_W8A8_MODEL_PATH envs = QWEN3_32B_ENVS other_args = QWEN3_32B_OTHER_ARGS diff --git a/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py new file mode 100644 index 000000000..eacac5ec0 --- /dev/null +++ b/test/registered/npu/accuracy/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_gpqa_a2.py @@ -0,0 +1,108 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_32B_EAGLE_MODEL_PATH, + QWEN3_32B_W8A8_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="accuracy testcase", +) + +QWEN3_32B_ENVS = { + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", + "SGLANG_NPU_USE_DEEPGEMM": "1", +} + +QWEN3_32B_OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--max-running-requests", + 101, + "--disable-radix-cache", + "--speculative-draft-model-quantization", + "unquant", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 35000, + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + QWEN3_32B_EAGLE_MODEL_PATH, + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--tp-size", + 4, + "--mem-fraction-static", + 0.845, + "--cuda-graph-bs", + 16, + 32, + 64, + 72, + 88, + 90, + 92, + 94, + 96, + 97, + 98, + 99, + 100, + 101, + "--dtype", + "bfloat16", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen", +] + + +class TestQwen32B_GPQA(TestNpuAccuracyTestCaseBase): + """Test NPU accuracy for Qwen3-32B-W8A8 on qpqa""" + + model = QWEN3_32B_W8A8_MODEL_PATH + other_args = QWEN3_32B_OTHER_ARGS + envs = QWEN3_32B_ENVS + accuracy = 0.4949 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + eval_batch_size = 64 + generation_config = {"max_tokens": 40000, "temperature": 1.0} + + def test_qwen3_32b_qpqa(self): + """Run NPU accuracy test for Qwen3-32B-W8A8 on qpqa""" + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py b/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py similarity index 100% rename from test/registered/ascend/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py rename to test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_1p_gpqa.py diff --git a/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py new file mode 100644 index 000000000..30bbc0d75 --- /dev/null +++ b/test/registered/npu/accuracy/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py @@ -0,0 +1,106 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_6_27B_W8A8_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="full-2-npu-a3", + nightly=True, + disabled="accuracy testcase", +) + +QWEN3_6_27B_3K5_1K5_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "130", + "ASCEND_USE_FIA": "1", +} + +QWEN3_6_27B_3K5_1K5_OTHER_ARGS = [ + "--tp-size", + 2, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 60000, + "--disable-radix-cache", + "--trust-remote-code", + "--max-running-requests", + 64, + "--max-mamba-cache-size", + 74, + "--mem-fraction-static", + 0.7, + "--cuda-graph-bs", + 2, + 8, + 16, + 32, + 40, + 45, + 50, + 54, + "--enable-multimodal", + "--quantization", + "modelslim", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", +] + + +class TestNPUQwen3_6_27B_1P_In3k5_Out1k5_gpqa(TestNpuAccuracyTestCaseBase): + model = QWEN3_6_27B_W8A8_MODEL_PATH + envs = QWEN3_6_27B_3K5_1K5_ENVS + other_args = QWEN3_6_27B_3K5_1K5_OTHER_ARGS + accuracy = 0.878 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + eval_batch_size = 8 + generation_config = { + "max_tokens": 81920, + "temperature": 1.0, + "extra_body": { + "chat_template_kwargs": {"enable_thinking": True}, + }, + } + + def test_accuracy(self): + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py b/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py similarity index 100% rename from test/registered/ascend/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py rename to test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_aime26.py diff --git a/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py b/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py new file mode 100644 index 000000000..66befc0f3 --- /dev/null +++ b/test/registered/npu/accuracy/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py @@ -0,0 +1,107 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_6_35B_A3B_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="accuracy testcase", +) + +QWEN3_6_35B_A3B_64K_PREFIX_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "DEEPEP_HCCL_BUFFSIZE": "300", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_SET_CPU_AFFINITY": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "0", + "ASCEND_USE_FIA": "1", + "GDN_ATTN_BACKEND_TRITON": "1", +} + +QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [ + "--tp-size", + 2, + "--nnodes", + 1, + "--attention-backend", + "ascend", + "--device", + "npu", + "--chunked-prefill-size", + -1, + "--max-total-tokens", + 470784, + "--max-prefill-tokens", + 65536, + "--trust-remote-code", + "--mamba-scheduler-strategy", + "extra_buffer", + "--max-running-requests", + 40, + "--max-mamba-cache-size", + 200, + "--mem-fraction-static", + 0.9, + "--cuda-graph-bs", + 2, + 8, + 16, + 24, + 32, + 36, + 40, + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", +] + + +class TestNPUQwen3_6_35BA3B_1P_AIME2026(TestNpuAccuracyTestCaseBase): + """Test NPU accuracy for Qwen3.6-35B-A3B 1p on AIME2026""" + + model = QWEN3_6_35B_A3B_MODEL_PATH + other_args = QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS + envs = QWEN3_6_35B_A3B_64K_PREFIX_ENVS + accuracy = 0.927 + datasets = ["aime26"] + few_shot_num = 0 + eval_batch_size = 64 + generation_config = { + "max_tokens": 65536, + "temperature": 0.2, + "repetition_penalty": 1.08, + } + + def test_npu_qwen3_6_35b_a3b_1p_aime2026(self): + """Run NPU accuracy test for Qwen3.6-35B-A3B on AIME2026""" + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py b/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py similarity index 85% rename from test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py rename to test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py index 2d3d74f1a..c6e3aaea3 100644 --- a/test/registered/ascend/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py +++ b/test/registered/npu/accuracy/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16_aime25.py @@ -6,7 +6,6 @@ from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( from sglang.test.ascend.e2e.test_npu_performance_utils import ( QWEN3_NEXT_80B_A3B_MODEL_PATH, QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH, - TestNpuPerformanceTestCaseBase, ) from sglang.test.ci.ci_register import register_npu_ci @@ -14,7 +13,7 @@ register_npu_ci( est_time=3600, suite="", nightly=True, - disabled="performance testcase", + disabled="accuracy testcase", ) QWEN3_NEXT_80B_A3B_ENVS = { @@ -97,23 +96,6 @@ QWEN3_NEXT_80B_A3B_OTHER_ARGS = [ ] -class TestQwen3Next80BA3B(TestNpuPerformanceTestCaseBase): - max_attempts = 5 - model = QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH - other_args = QWEN3_NEXT_80B_A3B_OTHER_ARGS - envs = QWEN3_NEXT_80B_A3B_ENVS - dataset_name = "random" - max_concurrency = 16 - num_prompts = 16 - input_len = 6144 - output_len = 1500 - random_range_ratio = 1 - tpot = 15.62 - - def test_qwen3_next_80b_a3b(self): - self.run_throughput() - - class TestQwen3Next80BA3B_aime25(TestNpuAccuracyTestCaseBase): model = QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH envs = QWEN3_NEXT_80B_A3B_ENVS diff --git a/test/registered/ascend/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py b/test/registered/npu/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py similarity index 100% rename from test/registered/ascend/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py rename to test/registered/npu/accuracy/qwen3_omni_30b_a3b_thinking/test_npu_qwen3_omni_30b_a3b_thinking_1p_mmmu.py diff --git a/test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py b/test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py similarity index 100% rename from test/registered/ascend/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py rename to test/registered/npu/accuracy/qwen3_vl_30b_a3b_thinking/test_npu_qwen3_vl_30b_a3b_thinking_1p_mmmu.py diff --git a/test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py b/test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py similarity index 89% rename from test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py rename to test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py index 774b78a3e..2f9ce2f8d 100644 --- a/test/registered/ascend/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py +++ b/test/registered/npu/accuracy/qwen3_vl_8b_thinking/test_npu_qwen3_vl_8b_thinking_1p_mmmu.py @@ -1,3 +1,4 @@ +import os import unittest from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( @@ -15,6 +16,8 @@ register_npu_ci( disabled="performance testcase", ) +_is_pr_pipeline = os.environ.get("GITHUB_EVENT_NAME") == "pull_request" + ENVS = { "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", @@ -57,9 +60,10 @@ class TestQwen3(TestNpuAccuracyTestCaseBase): model = QWEN3_VL_8B_THINKING_MODEL_PATH envs = ENVS other_args = OTHER_ARGS - accuracy = 0.7011 + accuracy = 0.64 if _is_pr_pipeline else 0.7011 datasets = ["mmmu"] few_shot_num = 0 + limit = 5 if _is_pr_pipeline else 100000 generation_config = {"max_tokens": 65536} eval_batch_size = 64 diff --git a/test/registered/ascend/basic_function/HiCache/test_npu_hicache_mha.py b/test/registered/npu/basic_function/HiCache/test_npu_hicache_mha.py similarity index 100% rename from test/registered/ascend/basic_function/HiCache/test_npu_hicache_mha.py rename to test/registered/npu/basic_function/HiCache/test_npu_hicache_mha.py diff --git a/test/registered/ascend/basic_function/HiCache/test_npu_hicache_mla.py b/test/registered/npu/basic_function/HiCache/test_npu_hicache_mla.py similarity index 100% rename from test/registered/ascend/basic_function/HiCache/test_npu_hicache_mla.py rename to test/registered/npu/basic_function/HiCache/test_npu_hicache_mla.py diff --git a/test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache.py b/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache.py similarity index 100% rename from test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache.py rename to test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache.py diff --git a/test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache_mla.py b/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_mla.py similarity index 100% rename from test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache_mla.py rename to test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_mla.py diff --git a/test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache_mutually_exclusive.py b/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_mutually_exclusive.py similarity index 100% rename from test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache_mutually_exclusive.py rename to test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_mutually_exclusive.py diff --git a/test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache_ttft_mha.py b/test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_ttft_mha.py similarity index 100% rename from test/registered/ascend/basic_function/HiCache/test_npu_hierarchical_cache_ttft_mha.py rename to test/registered/npu/basic_function/HiCache/test_npu_hierarchical_cache_ttft_mha.py diff --git a/test/registered/ascend/basic_function/HiCache/test_npu_radix_cache.py b/test/registered/npu/basic_function/HiCache/test_npu_radix_cache.py similarity index 100% rename from test/registered/ascend/basic_function/HiCache/test_npu_radix_cache.py rename to test/registered/npu/basic_function/HiCache/test_npu_radix_cache.py diff --git a/test/registered/ascend/basic_function/backends/test_npu_sampling_backend.py b/test/registered/npu/basic_function/backends/test_npu_sampling_backend.py similarity index 100% rename from test/registered/ascend/basic_function/backends/test_npu_sampling_backend.py rename to test/registered/npu/basic_function/backends/test_npu_sampling_backend.py diff --git a/test/registered/ascend/basic_function/dllm/test_npu_llada2_mini.py b/test/registered/npu/basic_function/dllm/test_npu_llada2_mini.py similarity index 100% rename from test/registered/ascend/basic_function/dllm/test_npu_llada2_mini.py rename to test/registered/npu/basic_function/dllm/test_npu_llada2_mini.py diff --git a/test/registered/ascend/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py b/test/registered/npu/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py similarity index 100% rename from test/registered/ascend/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py rename to test/registered/npu/basic_function/memory_and_scheduling/test_npu_no_chunked_prefill.py diff --git a/test/registered/ascend/basic_function/offloading/test_npu_offload_modes.py b/test/registered/npu/basic_function/offloading/test_npu_offload_modes.py similarity index 100% rename from test/registered/ascend/basic_function/offloading/test_npu_offload_modes.py rename to test/registered/npu/basic_function/offloading/test_npu_offload_modes.py diff --git a/test/registered/ascend/basic_function/optimization_debug/test_npu_compile_graph_tp1_bf16.py b/test/registered/npu/basic_function/optimization_debug/test_npu_compile_graph_tp1_bf16.py similarity index 100% rename from test/registered/ascend/basic_function/optimization_debug/test_npu_compile_graph_tp1_bf16.py rename to test/registered/npu/basic_function/optimization_debug/test_npu_compile_graph_tp1_bf16.py diff --git a/test/registered/ascend/basic_function/optimization_debug/test_npu_graph_tp1_bf16.py b/test/registered/npu/basic_function/optimization_debug/test_npu_graph_tp1_bf16.py similarity index 93% rename from test/registered/ascend/basic_function/optimization_debug/test_npu_graph_tp1_bf16.py rename to test/registered/npu/basic_function/optimization_debug/test_npu_graph_tp1_bf16.py index 916e3965d..f5cccafde 100644 --- a/test/registered/ascend/basic_function/optimization_debug/test_npu_graph_tp1_bf16.py +++ b/test/registered/npu/basic_function/optimization_debug/test_npu_graph_tp1_bf16.py @@ -12,8 +12,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-1-npu-a2", nightly=False) -register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-1-npu-a3", nightly=True) TEST_MODEL_MATRIX = { "/root/.cache/modelscope/hub/models/Qwen/Qwen2.5-7B-Instruct": { diff --git a/test/registered/ascend/basic_function/optimization_debug/test_npu_graph_tp2_bf16.py b/test/registered/npu/basic_function/optimization_debug/test_npu_graph_tp2_bf16.py similarity index 93% rename from test/registered/ascend/basic_function/optimization_debug/test_npu_graph_tp2_bf16.py rename to test/registered/npu/basic_function/optimization_debug/test_npu_graph_tp2_bf16.py index 37bb7fd22..7a333b9a1 100644 --- a/test/registered/ascend/basic_function/optimization_debug/test_npu_graph_tp2_bf16.py +++ b/test/registered/npu/basic_function/optimization_debug/test_npu_graph_tp2_bf16.py @@ -12,8 +12,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_npu_ci(est_time=400, suite="stage-b-test-2-npu-a2", nightly=False) -register_npu_ci(est_time=400, suite="nightly-2-npu-a3", nightly=True) +register_npu_ci(est_time=400, suite="full-2-npu-a3", nightly=True) TEST_MODEL_MATRIX = { "/root/.cache/modelscope/hub/models/Qwen/Qwen2.5-7B-Instruct": { diff --git a/test/registered/ascend/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py b/test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py similarity index 100% rename from test/registered/ascend/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py rename to test/registered/npu/basic_function/optimization_debug/test_npu_piecewise_graph_prefill.py diff --git a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py similarity index 100% rename from test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py rename to test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep.py diff --git a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_deepseek_v3_2_w8a8.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_deepseek_v3_2_w8a8.py similarity index 100% rename from test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_deepseek_v3_2_w8a8.py rename to test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_deepseek_v3_2_w8a8.py diff --git a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py similarity index 100% rename from test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py rename to test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_480b.py diff --git a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py similarity index 100% rename from test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py rename to test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_auto_qwen3_next.py diff --git a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py similarity index 100% rename from test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py rename to test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_deepseek_v3_2_w8a8.py diff --git a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py similarity index 100% rename from test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py rename to test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_480b.py diff --git a/test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py b/test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py similarity index 100% rename from test/registered/ascend/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py rename to test/registered/npu/basic_function/parallel_strategy/expert_parallelism/test_npu_deepep_low_latency_qwen3_next.py diff --git a/test/registered/ascend/basic_function/parameter/deepseek_coder.json b/test/registered/npu/basic_function/parameter/deepseek_coder.json similarity index 100% rename from test/registered/ascend/basic_function/parameter/deepseek_coder.json rename to test/registered/npu/basic_function/parameter/deepseek_coder.json diff --git a/test/registered/ascend/basic_function/parameter/test_npu_fim_completion.py b/test/registered/npu/basic_function/parameter/test_npu_fim_completion.py similarity index 100% rename from test/registered/ascend/basic_function/parameter/test_npu_fim_completion.py rename to test/registered/npu/basic_function/parameter/test_npu_fim_completion.py diff --git a/test/registered/ascend/basic_function/parameter/test_npu_log_level.py b/test/registered/npu/basic_function/parameter/test_npu_log_level.py similarity index 100% rename from test/registered/ascend/basic_function/parameter/test_npu_log_level.py rename to test/registered/npu/basic_function/parameter/test_npu_log_level.py diff --git a/test/registered/ascend/basic_function/parameter/test_npu_no_overlap_scheduler.py b/test/registered/npu/basic_function/parameter/test_npu_no_overlap_scheduler.py similarity index 100% rename from test/registered/ascend/basic_function/parameter/test_npu_no_overlap_scheduler.py rename to test/registered/npu/basic_function/parameter/test_npu_no_overlap_scheduler.py diff --git a/test/registered/ascend/basic_function/parameter/test_npu_original_logprobs.py b/test/registered/npu/basic_function/parameter/test_npu_original_logprobs.py similarity index 100% rename from test/registered/ascend/basic_function/parameter/test_npu_original_logprobs.py rename to test/registered/npu/basic_function/parameter/test_npu_original_logprobs.py diff --git a/test/registered/ascend/basic_function/parameter/test_npu_warmups.py b/test/registered/npu/basic_function/parameter/test_npu_warmups.py similarity index 100% rename from test/registered/ascend/basic_function/parameter/test_npu_warmups.py rename to test/registered/npu/basic_function/parameter/test_npu_warmups.py diff --git a/test/registered/ascend/basic_function/quant/test_npu_autoround_dense.py b/test/registered/npu/basic_function/quant/test_npu_autoround_dense.py similarity index 100% rename from test/registered/ascend/basic_function/quant/test_npu_autoround_dense.py rename to test/registered/npu/basic_function/quant/test_npu_autoround_dense.py diff --git a/test/registered/ascend/basic_function/quant/test_npu_autoround_moe.py b/test/registered/npu/basic_function/quant/test_npu_autoround_moe.py similarity index 95% rename from test/registered/ascend/basic_function/quant/test_npu_autoround_moe.py rename to test/registered/npu/basic_function/quant/test_npu_autoround_moe.py index 1864ec6ee..f771c67c6 100644 --- a/test/registered/ascend/basic_function/quant/test_npu_autoround_moe.py +++ b/test/registered/npu/basic_function/quant/test_npu_autoround_moe.py @@ -10,7 +10,6 @@ from sglang.test.ascend.test_ascend_utils import ( from sglang.test.ci.ci_register import register_npu_ci from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, popen_launch_server, @@ -53,7 +52,7 @@ class TestAscendAutoRoundMoE(CustomTestCase): process = popen_launch_server( model, self.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + timeout=3600, other_args=[ *self.common_args, ], diff --git a/test/registered/ascend/basic_function/quant/test_npu_gguf.py b/test/registered/npu/basic_function/quant/test_npu_gguf.py similarity index 100% rename from test/registered/ascend/basic_function/quant/test_npu_gguf.py rename to test/registered/npu/basic_function/quant/test_npu_gguf.py diff --git a/test/registered/ascend/basic_function/quant/test_npu_gguf_moe.py b/test/registered/npu/basic_function/quant/test_npu_gguf_moe.py similarity index 100% rename from test/registered/ascend/basic_function/quant/test_npu_gguf_moe.py rename to test/registered/npu/basic_function/quant/test_npu_gguf_moe.py diff --git a/test/registered/ascend/basic_function/quant/test_npu_gptq_moe.py b/test/registered/npu/basic_function/quant/test_npu_gptq_moe.py similarity index 100% rename from test/registered/ascend/basic_function/quant/test_npu_gptq_moe.py rename to test/registered/npu/basic_function/quant/test_npu_gptq_moe.py diff --git a/test/registered/ascend/basic_function/quant/test_npu_w4a4_quantization.py b/test/registered/npu/basic_function/quant/test_npu_w4a4_quantization.py similarity index 100% rename from test/registered/ascend/basic_function/quant/test_npu_w4a4_quantization.py rename to test/registered/npu/basic_function/quant/test_npu_w4a4_quantization.py diff --git a/test/registered/ascend/basic_function/runtime_opts/test_npu_mla_fia_w8a8int8.py b/test/registered/npu/basic_function/runtime_opts/test_npu_mla_fia_w8a8int8.py similarity index 100% rename from test/registered/ascend/basic_function/runtime_opts/test_npu_mla_fia_w8a8int8.py rename to test/registered/npu/basic_function/runtime_opts/test_npu_mla_fia_w8a8int8.py diff --git a/test/registered/ascend/basic_function/runtime_opts/test_npu_mla_w8a8int8.py b/test/registered/npu/basic_function/runtime_opts/test_npu_mla_w8a8int8.py similarity index 100% rename from test/registered/ascend/basic_function/runtime_opts/test_npu_mla_w8a8int8.py rename to test/registered/npu/basic_function/runtime_opts/test_npu_mla_w8a8int8.py diff --git a/test/registered/ascend/basic_function/runtime_opts/test_npu_tp1_bf16.py b/test/registered/npu/basic_function/runtime_opts/test_npu_tp1_bf16.py similarity index 100% rename from test/registered/ascend/basic_function/runtime_opts/test_npu_tp1_bf16.py rename to test/registered/npu/basic_function/runtime_opts/test_npu_tp1_bf16.py diff --git a/test/registered/ascend/basic_function/runtime_opts/test_npu_tp2_bf16.py b/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_bf16.py similarity index 100% rename from test/registered/ascend/basic_function/runtime_opts/test_npu_tp2_bf16.py rename to test/registered/npu/basic_function/runtime_opts/test_npu_tp2_bf16.py diff --git a/test/registered/ascend/basic_function/runtime_opts/test_npu_tp2_fia_bf16.py b/test/registered/npu/basic_function/runtime_opts/test_npu_tp2_fia_bf16.py similarity index 100% rename from test/registered/ascend/basic_function/runtime_opts/test_npu_tp2_fia_bf16.py rename to test/registered/npu/basic_function/runtime_opts/test_npu_tp2_fia_bf16.py diff --git a/test/registered/ascend/basic_function/runtime_opts/test_npu_tp4_bf16.py b/test/registered/npu/basic_function/runtime_opts/test_npu_tp4_bf16.py similarity index 100% rename from test/registered/ascend/basic_function/runtime_opts/test_npu_tp4_bf16.py rename to test/registered/npu/basic_function/runtime_opts/test_npu_tp4_bf16.py diff --git a/test/registered/ascend/basic_function/speculative_inference/test_npu_eagle3.py b/test/registered/npu/basic_function/speculative_inference/test_npu_eagle3.py similarity index 100% rename from test/registered/ascend/basic_function/speculative_inference/test_npu_eagle3.py rename to test/registered/npu/basic_function/speculative_inference/test_npu_eagle3.py diff --git a/test/registered/ascend/embedding_models/test_npu_bge_large_en_v1_5.py b/test/registered/npu/embedding_models/test_npu_bge_large_en_v1_5.py similarity index 100% rename from test/registered/ascend/embedding_models/test_npu_bge_large_en_v1_5.py rename to test/registered/npu/embedding_models/test_npu_bge_large_en_v1_5.py diff --git a/test/registered/ascend/interface/test_npu_api.py b/test/registered/npu/interface/test_npu_api.py similarity index 100% rename from test/registered/ascend/interface/test_npu_api.py rename to test/registered/npu/interface/test_npu_api.py diff --git a/test/registered/ascend/interface/test_npu_api_abort_request.py b/test/registered/npu/interface/test_npu_api_abort_request.py similarity index 100% rename from test/registered/ascend/interface/test_npu_api_abort_request.py rename to test/registered/npu/interface/test_npu_api_abort_request.py diff --git a/test/registered/ascend/interface/test_npu_api_encode.py b/test/registered/npu/interface/test_npu_api_encode.py similarity index 100% rename from test/registered/ascend/interface/test_npu_api_encode.py rename to test/registered/npu/interface/test_npu_api_encode.py diff --git a/test/registered/ascend/interface/test_npu_enable_thinking.py b/test/registered/npu/interface/test_npu_enable_thinking.py similarity index 100% rename from test/registered/ascend/interface/test_npu_enable_thinking.py rename to test/registered/npu/interface/test_npu_enable_thinking.py diff --git a/test/registered/ascend/interface/test_npu_matched_stop.py b/test/registered/npu/interface/test_npu_matched_stop.py similarity index 100% rename from test/registered/ascend/interface/test_npu_matched_stop.py rename to test/registered/npu/interface/test_npu_matched_stop.py diff --git a/test/registered/ascend/interface/test_npu_openai_function_calling.py b/test/registered/npu/interface/test_npu_openai_function_calling.py similarity index 100% rename from test/registered/ascend/interface/test_npu_openai_function_calling.py rename to test/registered/npu/interface/test_npu_openai_function_calling.py diff --git a/test/registered/ascend/interface/test_npu_openai_server_ignore_eos.py b/test/registered/npu/interface/test_npu_openai_server_ignore_eos.py similarity index 100% rename from test/registered/ascend/interface/test_npu_openai_server_ignore_eos.py rename to test/registered/npu/interface/test_npu_openai_server_ignore_eos.py diff --git a/test/registered/ascend/interface/test_npu_penalty.py b/test/registered/npu/interface/test_npu_penalty.py similarity index 100% rename from test/registered/ascend/interface/test_npu_penalty.py rename to test/registered/npu/interface/test_npu_penalty.py diff --git a/test/registered/ascend/llm_models/test_ascend_dbrx_instruct.py b/test/registered/npu/llm_models/test_ascend_dbrx_instruct.py similarity index 100% rename from test/registered/ascend/llm_models/test_ascend_dbrx_instruct.py rename to test/registered/npu/llm_models/test_ascend_dbrx_instruct.py diff --git a/test/registered/ascend/llm_models/test_ascend_minimax_m2.py b/test/registered/npu/llm_models/test_ascend_minimax_m2.py similarity index 100% rename from test/registered/ascend/llm_models/test_ascend_minimax_m2.py rename to test/registered/npu/llm_models/test_ascend_minimax_m2.py diff --git a/test/registered/ascend/llm_models/test_ascend_trinity_mini.py b/test/registered/npu/llm_models/test_ascend_trinity_mini.py similarity index 100% rename from test/registered/ascend/llm_models/test_ascend_trinity_mini.py rename to test/registered/npu/llm_models/test_ascend_trinity_mini.py diff --git a/test/registered/ascend/llm_models/test_npu_afm_4_5b.py b/test/registered/npu/llm_models/test_npu_afm_4_5b.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_afm_4_5b.py rename to test/registered/npu/llm_models/test_npu_afm_4_5b.py diff --git a/test/registered/ascend/llm_models/test_npu_baichuan2_13b_chat.py b/test/registered/npu/llm_models/test_npu_baichuan2_13b_chat.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_baichuan2_13b_chat.py rename to test/registered/npu/llm_models/test_npu_baichuan2_13b_chat.py diff --git a/test/registered/ascend/llm_models/test_npu_c4ai_command_r_v01.py b/test/registered/npu/llm_models/test_npu_c4ai_command_r_v01.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_c4ai_command_r_v01.py rename to test/registered/npu/llm_models/test_npu_c4ai_command_r_v01.py diff --git a/test/registered/ascend/llm_models/test_npu_chatglm2_6b.py b/test/registered/npu/llm_models/test_npu_chatglm2_6b.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_chatglm2_6b.py rename to test/registered/npu/llm_models/test_npu_chatglm2_6b.py diff --git a/test/registered/ascend/llm_models/test_npu_deepseek_v3_2_exp_w8a8.py b/test/registered/npu/llm_models/test_npu_deepseek_v3_2_exp_w8a8.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_deepseek_v3_2_exp_w8a8.py rename to test/registered/npu/llm_models/test_npu_deepseek_v3_2_exp_w8a8.py diff --git a/test/registered/ascend/llm_models/test_npu_exaone_3.py b/test/registered/npu/llm_models/test_npu_exaone_3.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_exaone_3.py rename to test/registered/npu/llm_models/test_npu_exaone_3.py diff --git a/test/registered/ascend/llm_models/test_npu_gemma_3_4b_it_llm.py b/test/registered/npu/llm_models/test_npu_gemma_3_4b_it_llm.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_gemma_3_4b_it_llm.py rename to test/registered/npu/llm_models/test_npu_gemma_3_4b_it_llm.py diff --git a/test/registered/ascend/llm_models/test_npu_glm4_9b_chat.py b/test/registered/npu/llm_models/test_npu_glm4_9b_chat.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_glm4_9b_chat.py rename to test/registered/npu/llm_models/test_npu_glm4_9b_chat.py diff --git a/test/registered/ascend/llm_models/test_npu_granite_3_0_3b_a800m.py b/test/registered/npu/llm_models/test_npu_granite_3_0_3b_a800m.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_granite_3_0_3b_a800m.py rename to test/registered/npu/llm_models/test_npu_granite_3_0_3b_a800m.py diff --git a/test/registered/ascend/llm_models/test_npu_granite_3_1_8b.py b/test/registered/npu/llm_models/test_npu_granite_3_1_8b.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_granite_3_1_8b.py rename to test/registered/npu/llm_models/test_npu_granite_3_1_8b.py diff --git a/test/registered/ascend/llm_models/test_npu_grok_2.py b/test/registered/npu/llm_models/test_npu_grok_2.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_grok_2.py rename to test/registered/npu/llm_models/test_npu_grok_2.py diff --git a/test/registered/ascend/llm_models/test_npu_internlm2_7b.py b/test/registered/npu/llm_models/test_npu_internlm2_7b.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_internlm2_7b.py rename to test/registered/npu/llm_models/test_npu_internlm2_7b.py diff --git a/test/registered/ascend/llm_models/test_npu_ling_lite.py b/test/registered/npu/llm_models/test_npu_ling_lite.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_ling_lite.py rename to test/registered/npu/llm_models/test_npu_ling_lite.py diff --git a/test/registered/ascend/llm_models/test_npu_llama4_scount_17b_16e.py b/test/registered/npu/llm_models/test_npu_llama4_scount_17b_16e.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_llama4_scount_17b_16e.py rename to test/registered/npu/llm_models/test_npu_llama4_scount_17b_16e.py diff --git a/test/registered/ascend/llm_models/test_npu_llama_2_7b.py b/test/registered/npu/llm_models/test_npu_llama_2_7b.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_llama_2_7b.py rename to test/registered/npu/llm_models/test_npu_llama_2_7b.py diff --git a/test/registered/ascend/llm_models/test_npu_llama_2_7b_communications_compression.py b/test/registered/npu/llm_models/test_npu_llama_2_7b_communications_compression.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_llama_2_7b_communications_compression.py rename to test/registered/npu/llm_models/test_npu_llama_2_7b_communications_compression.py diff --git a/test/registered/ascend/llm_models/test_npu_mimo_7b_rl.py b/test/registered/npu/llm_models/test_npu_mimo_7b_rl.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_mimo_7b_rl.py rename to test/registered/npu/llm_models/test_npu_mimo_7b_rl.py diff --git a/test/registered/ascend/llm_models/test_npu_minicpm3_4b.py b/test/registered/npu/llm_models/test_npu_minicpm3_4b.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_minicpm3_4b.py rename to test/registered/npu/llm_models/test_npu_minicpm3_4b.py diff --git a/test/registered/ascend/llm_models/test_npu_mistral_7b.py b/test/registered/npu/llm_models/test_npu_mistral_7b.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_mistral_7b.py rename to test/registered/npu/llm_models/test_npu_mistral_7b.py diff --git a/test/registered/ascend/llm_models/test_npu_persimmon_8b_chat.py b/test/registered/npu/llm_models/test_npu_persimmon_8b_chat.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_persimmon_8b_chat.py rename to test/registered/npu/llm_models/test_npu_persimmon_8b_chat.py diff --git a/test/registered/ascend/llm_models/test_npu_phi_4_multimodal_llm.py b/test/registered/npu/llm_models/test_npu_phi_4_multimodal_llm.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_phi_4_multimodal_llm.py rename to test/registered/npu/llm_models/test_npu_phi_4_multimodal_llm.py diff --git a/test/registered/ascend/llm_models/test_npu_qwen3_0_6b.py b/test/registered/npu/llm_models/test_npu_qwen3_0_6b.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_qwen3_0_6b.py rename to test/registered/npu/llm_models/test_npu_qwen3_0_6b.py diff --git a/test/registered/ascend/llm_models/test_npu_qwen3_1_7b_gptq_int8.py b/test/registered/npu/llm_models/test_npu_qwen3_1_7b_gptq_int8.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_qwen3_1_7b_gptq_int8.py rename to test/registered/npu/llm_models/test_npu_qwen3_1_7b_gptq_int8.py diff --git a/test/registered/ascend/llm_models/test_npu_qwen3_235b_a22b_w8a8.py b/test/registered/npu/llm_models/test_npu_qwen3_235b_a22b_w8a8.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_qwen3_235b_a22b_w8a8.py rename to test/registered/npu/llm_models/test_npu_qwen3_235b_a22b_w8a8.py diff --git a/test/registered/ascend/llm_models/test_npu_qwen3_30b.py b/test/registered/npu/llm_models/test_npu_qwen3_30b.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_qwen3_30b.py rename to test/registered/npu/llm_models/test_npu_qwen3_30b.py diff --git a/test/registered/ascend/llm_models/test_npu_qwen3_30b_attn_cp.py b/test/registered/npu/llm_models/test_npu_qwen3_30b_attn_cp.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_qwen3_30b_attn_cp.py rename to test/registered/npu/llm_models/test_npu_qwen3_30b_attn_cp.py diff --git a/test/registered/ascend/llm_models/test_npu_qwen3_30b_fuseep.py b/test/registered/npu/llm_models/test_npu_qwen3_30b_fuseep.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_qwen3_30b_fuseep.py rename to test/registered/npu/llm_models/test_npu_qwen3_30b_fuseep.py diff --git a/test/registered/ascend/llm_models/test_npu_qwen3_30b_w4a4.py b/test/registered/npu/llm_models/test_npu_qwen3_30b_w4a4.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_qwen3_30b_w4a4.py rename to test/registered/npu/llm_models/test_npu_qwen3_30b_w4a4.py diff --git a/test/registered/ascend/llm_models/test_npu_qwen3_32b.py b/test/registered/npu/llm_models/test_npu_qwen3_32b.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_qwen3_32b.py rename to test/registered/npu/llm_models/test_npu_qwen3_32b.py diff --git a/test/registered/ascend/llm_models/test_npu_qwen3_8b_communications_quantization.py b/test/registered/npu/llm_models/test_npu_qwen3_8b_communications_quantization.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_qwen3_8b_communications_quantization.py rename to test/registered/npu/llm_models/test_npu_qwen3_8b_communications_quantization.py diff --git a/test/registered/ascend/llm_models/test_npu_qwen3_coder_480b_a35b.py b/test/registered/npu/llm_models/test_npu_qwen3_coder_480b_a35b.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_qwen3_coder_480b_a35b.py rename to test/registered/npu/llm_models/test_npu_qwen3_coder_480b_a35b.py diff --git a/test/registered/ascend/llm_models/test_npu_qwq_32b_w8a8.py b/test/registered/npu/llm_models/test_npu_qwq_32b_w8a8.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_qwq_32b_w8a8.py rename to test/registered/npu/llm_models/test_npu_qwq_32b_w8a8.py diff --git a/test/registered/ascend/llm_models/test_npu_smollm_1_7b.py b/test/registered/npu/llm_models/test_npu_smollm_1_7b.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_smollm_1_7b.py rename to test/registered/npu/llm_models/test_npu_smollm_1_7b.py diff --git a/test/registered/ascend/llm_models/test_npu_stablelm_2_1_6b.py b/test/registered/npu/llm_models/test_npu_stablelm_2_1_6b.py similarity index 100% rename from test/registered/ascend/llm_models/test_npu_stablelm_2_1_6b.py rename to test/registered/npu/llm_models/test_npu_stablelm_2_1_6b.py diff --git a/test/registered/ascend/llm_models/tool_chat_template_c4ai_command_r_v01.jinja b/test/registered/npu/llm_models/tool_chat_template_c4ai_command_r_v01.jinja similarity index 100% rename from test/registered/ascend/llm_models/tool_chat_template_c4ai_command_r_v01.jinja rename to test/registered/npu/llm_models/tool_chat_template_c4ai_command_r_v01.jinja diff --git a/test/registered/ascend/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py similarity index 100% rename from test/registered/ascend/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py rename to test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py diff --git a/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py b/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py new file mode 100644 index 000000000..59bf7ec5d --- /dev/null +++ b/test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py @@ -0,0 +1,189 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + GLM_5_1_W4A8_MODEL_PATH, + TestNpuPerfMultiNodePdSepTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +GLM_5_1_PD_SEP_PREFILL_ENVS = { + "SGLANG_SET_CPU_AFFINITY": "1", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "1200", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "1200", + "DEEPEP_HCCL_BUFFSIZE": "1200", + "DEEPEP_NORMAL_LONG_SEQ_ROUND": "72", + "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "1024", + "DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ": "1", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "TASK_QUEUE_ENABLE": "2", + "ENABLE_PROFILING": "0", + "HCCL_SOCKET_IFNAME": NIC_NAME, + "GLOO_SOCKET_IFNAME": NIC_NAME, +} + +GLM_5_1_PD_SEP_DECODE_ENVS = { + "SGLANG_SET_CPU_AFFINITY": "1", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "1200", + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT": "1200", + "SGLANG_SPEC_ENABLE_OVERLAP_REFLOW": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "HCCL_BUFFSIZE": "200", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "16", + "TASK_QUEUE_ENABLE": "0", + "HCCL_SOCKET_IFNAME": NIC_NAME, + "GLOO_SOCKET_IFNAME": NIC_NAME, +} + +GLM_5_1_PD_SEP_PREFILL_ARGS = [ + "--disaggregation-mode", + "prefill", + "--tp-size", + 4, + "--nnodes", + 2, + "--mem-fraction-static", + 0.72, + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--disaggregation-transfer-backend", + "ascend", + "--max-running-requests", + 16, + "--served-model-name", + "glm-5", + "--chunked-prefill-size", + 16384, + "--max-prefill-tokens", + 180000, + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "normal", + "--disable-shared-experts-fusion", + "--disable-cuda-graph", + "--dtype", + "bfloat16", + "--speculative-draft-model-quantization", + "unquant", + "--enable-nsa-prefill-context-parallel", + "--nsa-prefill-cp-mode", + "in-seq-split", + "--attn-cp-size", + 4, + "--enable-dp-lm-head", + "--moe-dense-tp", + 1, + "--pp-size", + 8, + "--reasoning-parser", + "glm45", + "--tool-call-parser", + "glm47", +] + +GLM_5_1_PD_SEP_DECODE_ARGS = [ + "--disaggregation-mode", + "decode", + "--tp-size", + 32, + "--nnodes", + 2, + "--dp-size", + 32, + "--enable-dp-attention", + "--ep-size", + 32, + "--mem-fraction-static", + 0.85, + "--max-running-requests", + 32, + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--served-model-name", + "glm-5", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "low_latency", + "--cuda-graph-bs", + 1, + 2, + 3, + "--disaggregation-transfer-backend", + "ascend", + "--watchdog-timeout", + 9000, + "--context-length", + 180000, + "--tokenizer-worker-num", + 16, + "--prefill-round-robin-balance", + "--disable-shared-experts-fusion", + "--dtype", + "bfloat16", + "--load-balance-method", + "round_robin", + "--speculative-draft-model-quantization", + "unquant", + "--reasoning-parser", + "glm45", + "--tool-call-parser", + "glm47", +] + +GLM_5_1_PD_SEP_MODEL_CONFIG = { + "model_path": GLM_5_1_W4A8_MODEL_PATH, + "prefill_args": GLM_5_1_PD_SEP_PREFILL_ARGS, + "decode_args": GLM_5_1_PD_SEP_DECODE_ARGS, + "prefill_envs": GLM_5_1_PD_SEP_PREFILL_ENVS, + "decode_envs": GLM_5_1_PD_SEP_DECODE_ENVS, + "router_args": ["--policy", "round_robin"], + "router_envs": {}, +} + + +class TestNPUGLM5_1_W4A8_PD_SEP_In3k5_Out1k5(TestNpuPerfMultiNodePdSepTestCaseBase): + """Test NPU performance for GLM-5.1-w4a8 PD separation 4 nodes in3k5 out1k5""" + + model_config = GLM_5_1_PD_SEP_MODEL_CONFIG + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + dataset_name = "random" + max_concurrency = 1 + num_prompts = 1 + input_len = 65536 + output_len = 1024 + random_range_ratio = 1 + seed = 1 + tpot = 50 + output_token_throughput = 160 + + def test_npu_glm5_1_w4a8_pd_sep_in3k5_out1k5(self): + """Run NPU performance test for GLM-5.1-w4a8 PD separation""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py new file mode 100644 index 000000000..18e2d0504 --- /dev/null +++ b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms.py @@ -0,0 +1,113 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + KIMI_K2_6_EAGLE3_MODEL_PATH, + KIMI_K2_6_W4A8_MODEL_PATH, + TestNpuPerfMultiNodePdMixTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=1800, + suite="nightly-8-npu-a3", + nightly=True, + disabled="Currently it is executed by the npu performance workflow.", +) + +ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "SGLANG_SET_CPU_AFFINITY": "1", + "STREAMS_PER_DEVICE": "32", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "64", + "DEEPEP_HCCL_BUFFSIZE": "4400", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "HCCL_SOCKET_IFNAME": NIC_NAME, + "GLOO_SOCKET_IFNAME": NIC_NAME, +} + +OTHER_ARGS = [ + "--trust-remote-code", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--dtype", + "bfloat16", + "--tp-size", + 32, + "--nnodes", + 2, + "--mem-fraction-static", + 0.662, + "--max-running-requests", + 32, + "--chunked-prefill-size", + 262144, + "--context-length", + 75000, + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", + "--sampling-backend", + "ascend", + "--enable-dp-attention", + "--dp-size", + 32, + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", + "--cuda-graph-bs", + 1, + "--disable-radix-cache", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + KIMI_K2_6_EAGLE3_MODEL_PATH, + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--speculative-draft-model-quantization", + "unquant", + "--reasoning-parser", + "kimi_k2", + "--tool-call-parser", + "kimi_k2", +] + +MODEL_CONFIG = { + "model_path": KIMI_K2_6_W4A8_MODEL_PATH, + "other_args": OTHER_ARGS, + "node_envs": ENVS, +} + + +class TestNPUKimiK2_6_W4A8_16P_In64k_Out1k_100ms(TestNpuPerfMultiNodePdMixTestCaseBase): + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + model_config = MODEL_CONFIG + dataset_name = "random" + max_concurrency = 32 + num_prompts = 32 + input_len = 64000 + output_len = 1000 + random_range_ratio = 1 + seed = 1 + tpot = 100 + output_token_throughput = 160 + + def test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms(self): + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py new file mode 100644 index 000000000..43c7cc761 --- /dev/null +++ b/test/registered/npu/performance/kimi_k2_6/test_npu_kimi_k2_6_w4a8_8p_in3k5_out1k5_20ms.py @@ -0,0 +1,128 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_multi_node_utils import NIC_NAME +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + KIMI_K2_6_EAGLE3_MODEL_PATH, + KIMI_K2_6_W4A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=1800, + suite="full-16-npu-a3", + nightly=True, + disabled="Currently it is executed by the npu performance workflow.", +) + +KIMI_K2_6_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "SGLANG_SET_CPU_AFFINITY": "1", + "HCCL_SOCKET_IFNAME": NIC_NAME, + "GLOO_SOCKET_IFNAME": NIC_NAME, + "STREAMS_PER_DEVICE": "32", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "96", + "DEEPEP_HCCL_BUFFSIZE": "1200", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_NPU_USE_MLAPO": "1", + "SGLANG_NPU_USE_MULTI_STREAM": "1", +} + +KIMI_K2_6_OTHER_ARGS = [ + "--trust-remote-code", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--dtype", + "bfloat16", + "--tp-size", + 16, + "--mem-fraction-static", + 0.865, + "--max-running-requests", + 80, + "--chunked-prefill-size", + 32768, + "--context-length", + 6144, + "--max-prefill-tokens", + 65536, + "--enable-multimodal", + "--mm-attention-backend", + "ascend_attn", + "--sampling-backend", + "ascend", + "--enable-dp-attention", + "--dp-size", + 16, + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", + "--cuda-graph-bs-decode", + 1, + 2, + 3, + 4, + 5, + "--disable-radix-cache", + "--model-loader-extra-config", + '{"enable_multithread_load": true}', + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + KIMI_K2_6_EAGLE3_MODEL_PATH, + "--speculative-num-steps", + 4, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 5, + "--speculative-draft-model-quantization", + "unquant", + "--prefill-delayer-max-delay-passes", + 200, + "--enable-prefill-delayer", + "--reasoning-parser", + "kimi_k2", + "--tool-call-parser", + "kimi_k2", +] + + +class TestKimiK25W4A8( + TestNpuPerformanceTestCaseBase, +): + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + max_attempts = 5 + model = KIMI_K2_6_W4A8_MODEL_PATH + other_args = KIMI_K2_6_OTHER_ARGS + envs = KIMI_K2_6_ENVS + backend = "sglang" + dataset_name = "random" + max_concurrency = 64 + num_prompts = 256 + input_len = 3500 + output_len = 1500 + random_range_ratio = 1 + warmup_requests = 0 + seed = 1 + tpot = 20 + output_token_throughput = 1900 + + def test_kimi_k2_6_w4a8(self): + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py b/test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py similarity index 100% rename from test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py rename to test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1_ttft_5s.py diff --git a/test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py b/test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py similarity index 100% rename from test/registered/ascend/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py rename to test/registered/npu/performance/mimo_v2_flash/test_npu_mimo_v2_flash_1p1d_12p_in16k_out1k_tpot_20ms.py diff --git a/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py similarity index 84% rename from test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py rename to test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py index 6248c435a..a1d35c737 100644 --- a/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms_gpqa.py +++ b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_4p_in64k_out1k_prefix90_50ms.py @@ -1,9 +1,6 @@ import os import unittest -from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( - TestNpuAccuracyTestCaseBase, -) from sglang.test.ascend.e2e.test_npu_performance_utils import ( BENCHMARK_TOOL_DEFAULT, MINIMAX_M2_5_EAGLE3_MODEL_PATH, @@ -16,6 +13,7 @@ register_npu_ci( est_time=3600, suite="npu-performance", nightly=True, + disabled="performance testcase", ) MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS = { @@ -118,19 +116,5 @@ class TestNPUMiniMaxM2_5W8A8_4P_In64k_Out1k_Prefix90_50ms( self.run_throughput() -class TestNPUMiniMaxM2_5_W8A8_4P_In3k5_Out1k5_GPQA(TestNpuAccuracyTestCaseBase): - model = MINIMAX_M2_5_W8A8_MODEL_PATH - other_args = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_OTHER_ARGS - envs = MINIMAX_M2_5_W8A8_4P_IN64K_OUT1K_PREFIX90_ENVS - accuracy = 0.852 - datasets = ["gpqa_diamond"] - few_shot_num = 0 - generation_config = {"max_tokens": 65536, "temperature": 1.0} - eval_batch_size = 64 - - def test_accuracy(self): - self.run_accuracy() - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py similarity index 84% rename from test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py rename to test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py index 5a82dbf79..fa395166b 100644 --- a/test/registered/ascend/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/performance/minimax_m2_5/test_npu_minimax_m2_5_w8a8_8p_in3k5_out1k5_50ms.py @@ -1,9 +1,6 @@ import os import unittest -from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( - TestNpuAccuracyTestCaseBase, -) from sglang.test.ascend.e2e.test_npu_performance_utils import ( AISBENCHMARK_DATASET_DEFAULT, BENCHMARK_TOOL_DEFAULT, @@ -117,19 +114,5 @@ class TestNPUMiniMaxM2_5_W8A8_8P_In3k5_Out1k5_HighThroughput( self.run_throughput() -class TestNPUMiniMaxM2_5_W8A8_8P_In3k5_Out1k5_GPQA(TestNpuAccuracyTestCaseBase): - model = MINIMAX_M2_5_W8A8_MODEL_PATH - envs = MINIMAX_M2_5_HIGH_THROUGHPUT_ENVS - other_args = MINIMAX_M2_5_HIGH_THROUGHPUT_OTHER_ARGS - accuracy = 0.852 - datasets = ["gpqa_diamond"] - few_shot_num = 0 - generation_config = {"max_tokens": 65536, "temperature": 1.0} - eval_batch_size = 64 - - def test_accuracy(self): - self.run_accuracy() - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py new file mode 100644 index 000000000..4509463c4 --- /dev/null +++ b/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in3k5_out1k5_50ms.py @@ -0,0 +1,109 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_8B_EAGLE_MODEL_PATH, + QWEN3_8B_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_8B_ENVS = { + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "50", +} + +QWEN3_8B_OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--max-running-requests", + 70, + "--max-prefill-tokens", + 16384, + "--disable-radix-cache", + "--chunked-prefill-size", + 16384, + "--tp-size", + 1, + "--mem-fraction-static", + 0.85, + "--cuda-graph-bs", + 8, + 12, + 24, + 36, + 48, + 51, + 55, + 60, + 63, + 64, + 66, + 68, + 70, + "--dtype", + "bfloat16", + "--speculative-draft-model-quantization", + "unquant", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + QWEN3_8B_EAGLE_MODEL_PATH, + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen", +] + + +class TestQwen8B(TestNpuPerformanceTestCaseBase): + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_8B_W8A8_MODEL_PATH + other_args = QWEN3_8B_OTHER_ARGS + envs = QWEN3_8B_ENVS + dataset_name = "random" + max_concurrency = 64 + num_prompts = 256 + input_len = 3500 + output_len = 1500 + random_range_ratio = 1 + seed = 1 + tpot = 37 + output_token_throughput = 1586 + + def test_qwen3_8b(self): + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py b/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py new file mode 100644 index 000000000..592b806f3 --- /dev/null +++ b/test/registered/npu/performance/qwen3-8b/test_npu_qwen3_8b_w8a8_1p_in6k_out1k5_bs16.py @@ -0,0 +1,94 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_8B_EAGLE_MODEL_PATH, + QWEN3_8B_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_8B_ENVS = { + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", +} + +QWEN3_8B_OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--max-running-requests", + 16, + "--max-prefill-tokens", + 16384, + "--disable-radix-cache", + "--chunked-prefill-size", + -1, + "--tp-size", + 2, + "--mem-fraction-static", + 0.894, + "--cuda-graph-bs", + 1, + 5, + 15, + 16, + "--dtype", + "bfloat16", + "--speculative-draft-model-quantization", + "unquant", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + QWEN3_8B_EAGLE_MODEL_PATH, + "--speculative-num-steps", + 4, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 5, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen", +] + + +class TestQwen8B(TestNpuPerformanceTestCaseBase): + max_attempts = 5 + model = QWEN3_8B_W8A8_MODEL_PATH + other_args = QWEN3_8B_OTHER_ARGS + envs = QWEN3_8B_ENVS + dataset_name = "random" + max_concurrency = 16 + num_prompts = 16 + input_len = 6144 + output_len = 1500 + random_range_ratio = 1 + tpot = 11.79 + output_token_throughput = 1040.96 + + def test_qwen3_8b(self): + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py new file mode 100644 index 000000000..74335bafc --- /dev/null +++ b/test/registered/npu/performance/qwen3_235b_a22b/test_npu_qwen3_235b_w8a8_8p_in3k5_out1k5_50ms.py @@ -0,0 +1,125 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_235B_A22B_EAGLE_MODEL_PATH, + QWEN3_235B_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_235B_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "DEEPEP_HCCL_BUFFSIZE": "570", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", + "SGLANG_NPU_PROFILING": "0", + "SGLANG_NPU_PROFILING_BS": "27", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "188416", + "SGLANG_NPU_FUSED_MOE_MODE": "2", +} + +QWEN3_235B_OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--max-running-requests", + 432, + "--context-length", + 8192, + "--dtype", + "bfloat16", + "--chunked-prefill-size", + 94208, + "--max-prefill-tokens", + 458880, + "--sampling-backend", + "ascend", + "--ep-dispatch-algorithm", + "static", + "--init-expert-location", + "/root/.cache/modelscope/hub/models/hot_map/235B_3_5k_decode.pt", + "--disable-radix-cache", + "--moe-a2a-backend", + "ascend_fuseep", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + QWEN3_235B_A22B_EAGLE_MODEL_PATH, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--speculative-draft-model-quantization", + "unquant", + "--tp", + "16", + "--dp-size", + "16", + "--enable-dp-attention", + "--enable-dp-lm-head", + "--mem-fraction-static", + "0.8", + "--cuda-graph-bs", + "1", + "2", + "4", + "8", + "16", + "20", + "24", + "26", + "27", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen25", +] + + +class TestQwen235B(TestNpuPerformanceTestCaseBase): + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_235B_W8A8_MODEL_PATH + other_args = QWEN3_235B_OTHER_ARGS + envs = QWEN3_235B_ENVS + dataset_name = "random" + max_concurrency = 432 + num_prompts = int(max_concurrency) * 4 + input_len = 3500 + output_len = 1500 + random_range_ratio = 1 + seed = 1 + tpot = 50.1 + output_token_throughput = 6189 + + def test_qwen3_235b(self): + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py new file mode 100644 index 000000000..3bf536ac5 --- /dev/null +++ b/test/registered/npu/performance/qwen3_30b_a3b/test_npu_qwen3_30b_w8a8_1p_in3k5_out1k5_50ms.py @@ -0,0 +1,114 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH, + QWEN3_A3B_EAGLE_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_30B_A3B_ENVS = { + "ASCEND_LAUNCH_BLOCKING": "0", + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "200", + "DEEPEP_HCCL_BUFFSIZE": "400", +} + +QWEN3_30B_A3B_OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--max-running-requests", + 162, + "--disable-radix-cache", + "--speculative-draft-model-quantization", + "unquant", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 35000, + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + QWEN3_A3B_EAGLE_MODEL_PATH, + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--tp-size", + 2, + "--mem-fraction-static", + 0.87, + "--cuda-graph-bs", + 1, + 5, + 15, + 40, + 70, + 100, + 120, + 130, + 140, + 146, + 150, + 154, + 156, + 158, + 160, + 162, + "--dtype", + "bfloat16", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen", +] + + +class TestQwen30B(TestNpuPerformanceTestCaseBase): + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_30B_A3B_W8A8_VLLM_MODEL_PATH + other_args = QWEN3_30B_A3B_OTHER_ARGS + envs = QWEN3_30B_A3B_ENVS + dataset_name = "random" + max_concurrency = 160 + num_prompts = int(max_concurrency) * 4 + input_len = 3500 + output_len = 1500 + random_range_ratio = 1 + seed = 1 + tpot = 50 + output_token_throughput = 3200 + + def test_qwen3_30b(self): + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py similarity index 100% rename from test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py rename to test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_bf16_8p_in18k_out4k_6ms.py diff --git a/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py new file mode 100644 index 000000000..89cf2a304 --- /dev/null +++ b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms.py @@ -0,0 +1,111 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_32B_EAGLE_MODEL_PATH, + QWEN3_32B_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_32B_ENVS = { + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT": "600", + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "HCCL_OP_EXPANSION_MODE": "AIV", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE": "1", + "SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES": "100", + "SGLANG_NPU_USE_DEEPGEMM": "1", +} + +QWEN3_32B_OTHER_ARGS = [ + "--trust-remote-code", + "--nnodes", + "1", + "--node-rank", + "0", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--max-running-requests", + 101, + "--disable-radix-cache", + "--speculative-draft-model-quantization", + "unquant", + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 35000, + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + QWEN3_32B_EAGLE_MODEL_PATH, + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--tp-size", + 4, + "--mem-fraction-static", + 0.845, + "--cuda-graph-bs", + 16, + 32, + 64, + 72, + 88, + 90, + 92, + 94, + 96, + 97, + 98, + 99, + 100, + 101, + "--dtype", + "bfloat16", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen", +] + + +class TestQwen32B(TestNpuPerformanceTestCaseBase): + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_32B_W8A8_MODEL_PATH + other_args = QWEN3_32B_OTHER_ARGS + envs = QWEN3_32B_ENVS + dataset_name = "random" + max_concurrency = 100 + num_prompts = 400 + input_len = 3584 + output_len = 1536 + random_range_ratio = 1 + seed = 1 + tpot = 50 + output_token_throughput = 1600 + + def test_qwen3_32b(self): + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py similarity index 79% rename from test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py rename to test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py index 070a0bb1b..dd012b889 100644 --- a/test/registered/ascend/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py +++ b/test/registered/npu/performance/qwen3_32b/test_npu_qwen3_32b_w8a8_2p_in3k5_out1k5_50ms_a2.py @@ -1,8 +1,5 @@ import unittest -from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( - TestNpuAccuracyTestCaseBase, -) from sglang.test.ascend.e2e.test_npu_performance_utils import ( AISBENCHMARK_DATASET_DEFAULT, DEFAULT_URL_FOR_TEST, @@ -90,23 +87,6 @@ QWEN3_32B_OTHER_ARGS = [ ] -class TestQwen32B_GPQA(TestNpuAccuracyTestCaseBase): - """Test NPU accuracy for Qwen3-32B-W8A8 on qpqa""" - - model = QWEN3_32B_W8A8_MODEL_PATH - other_args = QWEN3_32B_OTHER_ARGS - envs = QWEN3_32B_ENVS - accuracy = 0.4949 - datasets = ["gpqa_diamond"] - few_shot_num = 0 - eval_batch_size = 64 - generation_config = {"max_tokens": 40000, "temperature": 1.0} - - def test_qwen3_32b_qpqa(self): - """Run NPU accuracy test for Qwen3-32B-W8A8 on qpqa""" - self.run_accuracy() - - class TestQwen32B(TestNpuPerformanceTestCaseBase): base_url = DEFAULT_URL_FOR_TEST dataset_type = AISBENCHMARK_DATASET_DEFAULT diff --git a/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py new file mode 100644 index 000000000..68d5d75ef --- /dev/null +++ b/test/registered/npu/performance/qwen3_5_397b/test_npu_qwen3_5_397b_w4a8_8p_in3k5_out1k5_50ms.py @@ -0,0 +1,140 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + AISBENCHMARK_DATASET_DEFAULT, + BENCHMARK_TOOL_DEFAULT, + QWEN3_5_397B_W4A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="nightly-16-npu-a3", + nightly=True, + disabled="performance testcase", +) + +QWEN3_5_397B_A17B_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "SGLANG_SET_CPU_AFFINITY": "1", + "STREAMS_PER_DEVICE": "32", + "ASCEND_USE_FIA": "1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "128", + "HCCL_BUFFSIZE": "0", + "DEEPEP_NORMAL_LONG_SEQ_ROUND": "6", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "GDN_ATTN_BACKEND_TRITON": "1", + "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "3584", + "HCCL_OP_EXPANSION_MODE": "AIV", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "SGLANG_ENABLE_SPEC_V2": "1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_ZBAL_LOCAL_MEM_SIZE": "59648", + "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", + "SGLANG_ZBAL_BOOTSTRAP_URL": "tcp://127.0.0.1:24669", + "ZBAL_NPU_ALLOC_CONF": "use_vmm_for_static_memory:True", + "ZBAL_ENABLE_GRAPH": "1", +} + +QWEN3_5_397B_A17B_3K5_1K5_OTHER_ARGS = [ + "--attention-backend", + "ascend", + "--device", + "npu", + "--tp-size", + 16, + "--chunked-prefill-size", + -1, + "--max-prefill-tokens", + 17500, + "--max-total-tokens", + 280000, + "--disable-radix-cache", + "--trust-remote-code", + "--max-running-requests", + 432, + "--mem-fraction-static", + 0.8, + "--cuda-graph-bs", + 2, + 4, + 6, + 8, + 12, + 16, + 20, + 24, + 28, + 32, + 36, + 40, + 44, + 48, + 50, + 52, + 54, + "--quantization", + "modelslim", + "--enable-multimodal", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", + "--mm-attention-backend", + "ascend_attn", + "--dtype", + "bfloat16", + "--mamba-ssm-dtype", + "bfloat16", + "--dp-size", + 8, + "--enable-dp-attention", + "--enable-dp-lm-head", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--speculative-draft-model-quantization", + "unquant", + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", +] + + +class TestNPUQwen3_5_397B_A17B_3K5_1K5_50ms(TestNpuPerformanceTestCaseBase): + """Test NPU performance for Qwen3.5-397B-A17B 16p in3k5 out1k5 50ms""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + dataset_type = AISBENCHMARK_DATASET_DEFAULT + model = QWEN3_5_397B_W4A8_MODEL_PATH + other_args = QWEN3_5_397B_A17B_3K5_1K5_OTHER_ARGS + envs = QWEN3_5_397B_A17B_ENVS + dataset_name = "random" + warmup_requests = 16 + max_concurrency = 432 + num_prompts = 432 + input_len = 3500 + output_len = 1500 + random_range_ratio = 1 + seed = 1 + tpot = 50 + output_token_throughput = 5415 + request_rate = float("inf") + temperature = 0.6 + top_p = 0.95 + + def test_npu_qwen3_5_397b_a17b_3k5_1k5(self): + """Run NPU performance test for Qwen3.5-397B-A17B in3k5 out1k5""" + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py similarity index 100% rename from test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py rename to test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1024x1024_30_out1024_50ms.py diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py similarity index 100% rename from test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py rename to test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_1p_in1080p_30_out256_50ms.py diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py similarity index 100% rename from test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py rename to test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_2p_in64k_out1k_prefix90_50ms.py diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py similarity index 79% rename from test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py rename to test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py index 3ea1c80bf..92764ae9f 100644 --- a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms_gpqa.py +++ b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in3k5_out1k5_50ms.py @@ -1,8 +1,5 @@ import unittest -from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( - TestNpuAccuracyTestCaseBase, -) from sglang.test.ascend.e2e.test_npu_performance_utils import ( AISBENCHMARK_DATASET_DEFAULT, BENCHMARK_TOOL_DEFAULT, @@ -108,25 +105,5 @@ class TestNPUQwen3_6_27B_1P_In3k5_Out1k5_50ms(TestNpuPerformanceTestCaseBase): self.run_throughput() -class TestNPUQwen3_6_27B_1P_In3k5_Out1k5_gpqa(TestNpuAccuracyTestCaseBase): - model = QWEN3_6_27B_W8A8_MODEL_PATH - envs = QWEN3_6_27B_3K5_1K5_ENVS - other_args = QWEN3_6_27B_3K5_1K5_OTHER_ARGS - accuracy = 0.878 - datasets = ["gpqa_diamond"] - few_shot_num = 0 - eval_batch_size = 8 - generation_config = { - "max_tokens": 81920, - "temperature": 1.0, - "extra_body": { - "chat_template_kwargs": {"enable_thinking": True}, - }, - } - - def test_accuracy(self): - self.run_accuracy() - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py similarity index 100% rename from test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py rename to test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_1p_in64k_out1k_50ms.py diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py similarity index 100% rename from test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py rename to test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in128k_out1k_50ms.py diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py similarity index 100% rename from test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py rename to test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in16k_out1k_50ms.py diff --git a/test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py similarity index 100% rename from test/registered/ascend/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py rename to test/registered/npu/performance/qwen3_6_27b/test_npu_qwen3_6_27b_w8a8_2p_in64k_out1k_50ms.py diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py similarity index 100% rename from test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py rename to test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_50ms.py diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py similarity index 100% rename from test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py rename to test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in128k_out1k_prefix90_50ms.py diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py similarity index 100% rename from test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py rename to test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in3k5_out1k5_50ms.py diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py similarity index 100% rename from test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py rename to test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_50ms.py diff --git a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py similarity index 75% rename from test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py rename to test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py index de3cbab02..371a9cf43 100644 --- a/test/registered/ascend/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms_aime26.py +++ b/test/registered/npu/performance/qwen3_6_35b_a3b/test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms.py @@ -1,8 +1,5 @@ import unittest -from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( - TestNpuAccuracyTestCaseBase, -) from sglang.test.ascend.e2e.test_npu_performance_utils import ( BENCHMARK_TOOL_DEFAULT, DEFAULT_URL_FOR_TEST, @@ -85,31 +82,6 @@ QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS = [ ] -class TestNPUQwen3_6_35BA3B_1P_AIME2026(TestNpuAccuracyTestCaseBase): - """Test NPU accuracy for Qwen3.6-35B-A3B 1p on AIME2026""" - - model = QWEN3_6_35B_A3B_MODEL_PATH - other_args = QWEN3_6_35B_A3B_64K_PREFIX_OTHER_ARGS - envs = QWEN3_6_35B_A3B_64K_PREFIX_ENVS - accuracy = 0.927 - datasets = ["aime26"] - few_shot_num = 0 - eval_batch_size = 64 - generation_config = { - "max_tokens": 65536, - "temperature": 0.2, - "repetition_penalty": 1.08, - } - - @classmethod - def tearDownClass(cls): - pass - - def test_npu_qwen3_6_35b_a3b_1p_aime2026(self): - """Run NPU accuracy test for Qwen3.6-35B-A3B on AIME2026""" - self.run_accuracy() - - class TestNPUQwen3_6_35BA3B_1P_In64k_Out1k_Prefix90_50ms( TestNpuPerformanceTestCaseBase ): @@ -133,10 +105,6 @@ class TestNPUQwen3_6_35BA3B_1P_In64k_Out1k_Prefix90_50ms( output_token_throughput = 660 pop_sglang_is_in_ci_for_gsp = True - @classmethod - def setUpClass(cls): - pass - def test_npu_qwen3_6_35b_a3b_1p_in64k_out1k_prefix90_50ms(self): """Run NPU performance test for Qwen3.6-35B-A3B in64k out1k prefix90 50ms""" self.run_throughput() diff --git a/test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py b/test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py new file mode 100644 index 000000000..1952c1ef8 --- /dev/null +++ b/test/registered/npu/performance/qwen3_next_80b_a3b_instruct/test_npu_qwen3_next_80b_w8a8_2p_in6k_out1k5_bs16.py @@ -0,0 +1,115 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + QWEN3_NEXT_80B_A3B_MODEL_PATH, + QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH, + TestNpuPerformanceTestCaseBase, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=3600, + suite="", + nightly=True, + disabled="performance testcase", +) + +QWEN3_NEXT_80B_A3B_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "400", + "DEEPEP_NORMAL_LONG_SEQ_ROUND": "10", + "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "2048", + "HCCL_OP_EXPANSION_MODE": "AIV", + "TASK_QUEUE_ENABLE": "1", + "ASCEND_USE_FIA": "1", + "SGLANG_NPU_USE_MULTI_STREAM": "0", + "SGLANG_WARMUP_TIMEOUT": "3600", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "FORCE_DRAFT_MODEL_NON_QUANT": "1", + "DEEPEP_HCCL_BUFFSIZE": "2000", + "ZBCCL_LOCAL_MEM_SIZE": "60416", + "SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK": "0", + "ZBCCL_BOOTSTRAP_URL": "tcp://127.0.0.1:24669", + "ZBCCL_NPU_ALLOC_CONF": "use_vmm_for_static_memory:True", + "ZBCCL_ENABLE_GRAPH": "1", +} + +QWEN3_NEXT_80B_A3B_OTHER_ARGS = [ + "--trust-remote-code", + "--attention-backend", + "ascend", + "--device", + "npu", + "--quantization", + "modelslim", + "--page-size", + 128, + "--tp-size", + 4, + "--watchdog-timeout", + 9000, + "--mem-fraction-static", + 0.85, + "--disable-radix-cache", + "--max-prefill-tokens", + 28672, + "--context-length", + 81920, + "--max-total-tokens", + 122304, + "--dp-size", + 2, + "--enable-dp-attention", + "--enable-dp-lm-head", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + 3, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 4, + "--speculative-draft-model-quantization", + "unquant", + "--chunked-prefill-size", + -1, + "--max-running-requests", + 16, + "--cuda-graph-bs", + 2, + 4, + 8, + "--mamba-ssm-dtype", + "bfloat16", + "--speculative-draft-model-path", + QWEN3_NEXT_80B_A3B_MODEL_PATH, + "--reasoning-parser", + "qwen3", + "--tool-call-parser", + "qwen3_coder", +] + + +class TestQwen3Next80BA3B(TestNpuPerformanceTestCaseBase): + max_attempts = 5 + model = QWEN3_NEXT_80B_A3B_W8A8_MODEL_PATH + other_args = QWEN3_NEXT_80B_A3B_OTHER_ARGS + envs = QWEN3_NEXT_80B_A3B_ENVS + dataset_name = "random" + max_concurrency = 16 + num_prompts = 16 + input_len = 6144 + output_len = 1500 + random_range_ratio = 1 + tpot = 15.62 + + def test_qwen3_next_80b_a3b(self): + self.run_throughput() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/ascend/rerank_models/test_npu_bge_reranker_v2_m3.py b/test/registered/npu/rerank_models/test_npu_bge_reranker_v2_m3.py similarity index 100% rename from test/registered/ascend/rerank_models/test_npu_bge_reranker_v2_m3.py rename to test/registered/npu/rerank_models/test_npu_bge_reranker_v2_m3.py diff --git a/test/registered/ascend/reward_models/test_npu_gemma_2_27b_v0_2.py b/test/registered/npu/reward_models/test_npu_gemma_2_27b_v0_2.py similarity index 100% rename from test/registered/ascend/reward_models/test_npu_gemma_2_27b_v0_2.py rename to test/registered/npu/reward_models/test_npu_gemma_2_27b_v0_2.py diff --git a/test/registered/ascend/reward_models/test_npu_internlm2_7b_reward.py b/test/registered/npu/reward_models/test_npu_internlm2_7b_reward.py similarity index 100% rename from test/registered/ascend/reward_models/test_npu_internlm2_7b_reward.py rename to test/registered/npu/reward_models/test_npu_internlm2_7b_reward.py diff --git a/test/registered/ascend/reward_models/test_npu_llama_3_1_8b_v0_2.py b/test/registered/npu/reward_models/test_npu_llama_3_1_8b_v0_2.py similarity index 100% rename from test/registered/ascend/reward_models/test_npu_llama_3_1_8b_v0_2.py rename to test/registered/npu/reward_models/test_npu_llama_3_1_8b_v0_2.py diff --git a/test/registered/ascend/test_npu_memory_consumption.py b/test/registered/npu/test_npu_memory_consumption.py similarity index 100% rename from test/registered/ascend/test_npu_memory_consumption.py rename to test/registered/npu/test_npu_memory_consumption.py diff --git a/test/registered/ascend/vlm_models/mmmu-val.yaml b/test/registered/npu/vlm_models/mmmu-val.yaml similarity index 100% rename from test/registered/ascend/vlm_models/mmmu-val.yaml rename to test/registered/npu/vlm_models/mmmu-val.yaml diff --git a/test/registered/ascend/vlm_models/test_ascend_glm_4_5v.py b/test/registered/npu/vlm_models/test_ascend_glm_4_5v.py similarity index 100% rename from test/registered/ascend/vlm_models/test_ascend_glm_4_5v.py rename to test/registered/npu/vlm_models/test_ascend_glm_4_5v.py diff --git a/test/registered/ascend/vlm_models/test_npu_deepseek_vl2.py b/test/registered/npu/vlm_models/test_npu_deepseek_vl2.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_deepseek_vl2.py rename to test/registered/npu/vlm_models/test_npu_deepseek_vl2.py diff --git a/test/registered/ascend/vlm_models/test_npu_gemma_3_4b_it.py b/test/registered/npu/vlm_models/test_npu_gemma_3_4b_it.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_gemma_3_4b_it.py rename to test/registered/npu/vlm_models/test_npu_gemma_3_4b_it.py diff --git a/test/registered/ascend/vlm_models/test_npu_janus_pro_1b.py b/test/registered/npu/vlm_models/test_npu_janus_pro_1b.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_janus_pro_1b.py rename to test/registered/npu/vlm_models/test_npu_janus_pro_1b.py diff --git a/test/registered/ascend/vlm_models/test_npu_janus_pro_7b.py b/test/registered/npu/vlm_models/test_npu_janus_pro_7b.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_janus_pro_7b.py rename to test/registered/npu/vlm_models/test_npu_janus_pro_7b.py diff --git a/test/registered/ascend/vlm_models/test_npu_kimi_vl_a3b_instruct.py b/test/registered/npu/vlm_models/test_npu_kimi_vl_a3b_instruct.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_kimi_vl_a3b_instruct.py rename to test/registered/npu/vlm_models/test_npu_kimi_vl_a3b_instruct.py diff --git a/test/registered/ascend/vlm_models/test_npu_llama_3_2_11b_vision_instruct.py b/test/registered/npu/vlm_models/test_npu_llama_3_2_11b_vision_instruct.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_llama_3_2_11b_vision_instruct.py rename to test/registered/npu/vlm_models/test_npu_llama_3_2_11b_vision_instruct.py diff --git a/test/registered/ascend/vlm_models/test_npu_mimo_vl_7b_rl.py b/test/registered/npu/vlm_models/test_npu_mimo_vl_7b_rl.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_mimo_vl_7b_rl.py rename to test/registered/npu/vlm_models/test_npu_mimo_vl_7b_rl.py diff --git a/test/registered/ascend/vlm_models/test_npu_minicpm_o_2_6.py b/test/registered/npu/vlm_models/test_npu_minicpm_o_2_6.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_minicpm_o_2_6.py rename to test/registered/npu/vlm_models/test_npu_minicpm_o_2_6.py diff --git a/test/registered/ascend/vlm_models/test_npu_minicpm_v_2_6.py b/test/registered/npu/vlm_models/test_npu_minicpm_v_2_6.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_minicpm_v_2_6.py rename to test/registered/npu/vlm_models/test_npu_minicpm_v_2_6.py diff --git a/test/registered/ascend/vlm_models/test_npu_mistral_small_3_1_24b_instruct_2503.py b/test/registered/npu/vlm_models/test_npu_mistral_small_3_1_24b_instruct_2503.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_mistral_small_3_1_24b_instruct_2503.py rename to test/registered/npu/vlm_models/test_npu_mistral_small_3_1_24b_instruct_2503.py diff --git a/test/registered/ascend/vlm_models/test_npu_phi4_multimodal_instruct.py b/test/registered/npu/vlm_models/test_npu_phi4_multimodal_instruct.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_phi4_multimodal_instruct.py rename to test/registered/npu/vlm_models/test_npu_phi4_multimodal_instruct.py diff --git a/test/registered/ascend/vlm_models/test_npu_qwen2_5_vl_3b_instruct.py b/test/registered/npu/vlm_models/test_npu_qwen2_5_vl_3b_instruct.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_qwen2_5_vl_3b_instruct.py rename to test/registered/npu/vlm_models/test_npu_qwen2_5_vl_3b_instruct.py diff --git a/test/registered/ascend/vlm_models/test_npu_qwen2_5_vl_72b_instruct.py b/test/registered/npu/vlm_models/test_npu_qwen2_5_vl_72b_instruct.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_qwen2_5_vl_72b_instruct.py rename to test/registered/npu/vlm_models/test_npu_qwen2_5_vl_72b_instruct.py diff --git a/test/registered/ascend/vlm_models/test_npu_qwen3_vl_235b_a22b_instruct.py b/test/registered/npu/vlm_models/test_npu_qwen3_vl_235b_a22b_instruct.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_qwen3_vl_235b_a22b_instruct.py rename to test/registered/npu/vlm_models/test_npu_qwen3_vl_235b_a22b_instruct.py diff --git a/test/registered/ascend/vlm_models/test_npu_qwen3_vl_30b_a3b_instruct.py b/test/registered/npu/vlm_models/test_npu_qwen3_vl_30b_a3b_instruct.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_qwen3_vl_30b_a3b_instruct.py rename to test/registered/npu/vlm_models/test_npu_qwen3_vl_30b_a3b_instruct.py diff --git a/test/registered/ascend/vlm_models/test_npu_qwen3_vl_4b_instruct.py b/test/registered/npu/vlm_models/test_npu_qwen3_vl_4b_instruct.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_qwen3_vl_4b_instruct.py rename to test/registered/npu/vlm_models/test_npu_qwen3_vl_4b_instruct.py diff --git a/test/registered/ascend/vlm_models/test_npu_qwen3_vl_8b_instruct.py b/test/registered/npu/vlm_models/test_npu_qwen3_vl_8b_instruct.py similarity index 100% rename from test/registered/ascend/vlm_models/test_npu_qwen3_vl_8b_instruct.py rename to test/registered/npu/vlm_models/test_npu_qwen3_vl_8b_instruct.py