diff --git a/.github/workflows/_npu-single-node-test-stage.yml b/.github/workflows/_npu-single-node-test-stage.yml index e65a7ee66..a8e56b0a5 100644 --- a/.github/workflows/_npu-single-node-test-stage.yml +++ b/.github/workflows/_npu-single-node-test-stage.yml @@ -345,6 +345,7 @@ jobs: source /usr/local/Ascend/nnal/atb/set_env.sh || true source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/customize/bin/set_env.bash || true source /usr/local/Ascend/ascend-toolkit/latest/opp/vendors/custom_transformer/bin/set_env.bash || true + source /usr/local/Ascend/ascend-toolkit/latest/set_env.sh || true set -u # Nightly log path mirrors the output layout; PR keeps the original {date}/{testcase}. diff --git a/.github/workflows/nightly-test-npu.yml b/.github/workflows/nightly-test-npu.yml index cf0c094d4..28d1261ca 100644 --- a/.github/workflows/nightly-test-npu.yml +++ b/.github/workflows/nightly-test-npu.yml @@ -29,6 +29,16 @@ on: required: false type: string default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3' + image_a3_cann910: + description: 'The a3 running docker image of the test task with cann 9.1.0.' + required: false + type: string + default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-a3-nightly' + image_a5: + description: 'The a5 running docker image of the test task.' + required: false + type: string + default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-950-nightly' skip_install_flag: description: 'Indicates whether to skip the installation of sglang, defaulting to false.' required: false @@ -56,6 +66,16 @@ on: required: false type: string default: '' + image_a3_cann910: + description: 'The a3 running docker image of the test task with cann 9.1.0.' + required: false + type: string + default: '' + image_a5: + description: 'The a5 running docker image of the test task.' + required: false + type: string + default: '' skip_install_flag: description: 'Indicates whether to skip the installation of sglang, defaulting to false.' required: false @@ -72,6 +92,8 @@ jobs: job_filter: ${{ steps.set-vars.outputs.job_filter }} image_a2: ${{ steps.set-vars.outputs.image_a2 }} image_a3: ${{ steps.set-vars.outputs.image_a3 }} + image_a3_cann910: ${{ steps.set-vars.outputs.image_a3_cann910 }} + image_a5: ${{ steps.set-vars.outputs.image_a5 }} skip_install_flag: ${{ steps.set-vars.outputs.skip_install_flag }} run_start_metadata: ${{ steps.set-vars.outputs.run_start_metadata }} steps: @@ -103,6 +125,16 @@ jobs: else echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT fi + if [ -z "${{ inputs.image_a3_cann910 }}" ]; then + echo "image_a3_cann910=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-a3-nightly" >> $GITHUB_OUTPUT + else + echo "image_a3_cann910=${{ inputs.image_a3_cann910 }}" >> $GITHUB_OUTPUT + fi + if [ -z "${{ inputs.image_a5 }}" ]; then + echo "image_a5=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:cann9.1.0-950-nightly" >> $GITHUB_OUTPUT + else + echo "image_a5=${{ inputs.image_a5 }}" >> $GITHUB_OUTPUT + fi if [ -z "${{ inputs.skip_install_flag }}" ]; then echo "skip_install_flag=false" >> $GITHUB_OUTPUT else @@ -282,6 +314,56 @@ jobs: device_type_for_deps: 'a3' run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} + nightly-acc-4-npu-a5: + name: nightly-acc-4-npu-a5 + if: ${{ !cancelled() }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-amd64-a5-4 + test_type: 'accuracy' + test_suite: nightly-acc-4-npu-a5 + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + image: ${{ needs.set-image-config.outputs.image_a5 }} + install_sglang_deps: false + device_type_for_deps: 'a5' + run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} + + # Suites suffixed with `-cann910` run against the CANN 9.1.0 image. + nightly-perf-16-npu-a3-cann910: + name: nightly-perf-16-npu-a3-cann910 + if: ${{ !cancelled() }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-800t-16 + test_type: 'perf' + test_suite: nightly-perf-16-npu-a3-cann910 + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + image: ${{ needs.set-image-config.outputs.image_a3_cann910 }} + install_sglang_deps: false + device_type_for_deps: 'a3' + run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} + test_timeout_minutes: '180' + + nightly-acc-16-npu-a3-cann910: + name: nightly-acc-16-npu-a3-cann910 + if: ${{ !cancelled() }} + needs: [set-image-config] + uses: ./.github/workflows/_npu-single-node-test-stage.yml + with: + runner: linux-aarch64-a3-16- + test_type: 'accuracy' + test_suite: nightly-acc-16-npu-a3-cann910 + is_nightly_pipeline_job: true + skip_pr_test_health_check: 'true' + image: ${{ needs.set-image-config.outputs.image_a3_cann910 }} + install_sglang_deps: false + device_type_for_deps: 'a3' + run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} + nightly-poc-multi-node-tests: name: multi-node-poc if: ${{ !cancelled() }} @@ -362,6 +444,72 @@ jobs: transformers_version: '' run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} + nightly-poc-multi-node-tests-cann910: + name: multi-node-poc-cann910 + if: ${{ !cancelled() }} + # Multi-node jobs run strictly one at a time to avoid contending for the + # same A3 multi-node resources: 9.0.0 multi -> 9.0.0 mix -> cann910 multi -> cann910 mix. + needs: [set-image-config, nightly-perf-16-npu-a3-cann910, nightly-acc-16-npu-a3-cann910, nightly-poc-multi-node-tests, nightly-poc-multi-node-mix-tests] + strategy: + fail-fast: false + max-parallel: 1 + matrix: + test_config: + # deepseek_v4_flash performance tests + - name: deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms + prefill_size: 1 + decode_size: 1 + router_size: 1 + test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py + test_type: 'perf' + prefill_decode_deployment: 'separation' + uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml + with: + runner: linux-amd64-cpu-4 + test_type: ${{ matrix.test_config.test_type || 'perf' }} + test_config_name: ${{ matrix.test_config.name }} + prefill_size: ${{ matrix.test_config.prefill_size }} + decode_size: ${{ matrix.test_config.decode_size }} + router_size: ${{ matrix.test_config.router_size }} + test_case: ${{ matrix.test_config.test_case }} + image: ${{ needs.set-image-config.outputs.image_a3_cann910 }} + install_sglang_from_source: false + prefill_decode_deployment: ${{ matrix.test_config.prefill_decode_deployment }} + transformers_version: '' + run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} + + nightly-poc-multi-node-mix-tests-cann910: + name: multi-node-mix-poc-cann910 + if: ${{ !cancelled() }} + needs: [set-image-config, nightly-poc-multi-node-tests-cann910] + strategy: + fail-fast: false + max-parallel: 1 + matrix: + test_config: + # glm_5_2 accuracy tests + - name: glm_5_2_w4a8_16p_gpqa + node_size: 2 + test_case: test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py + test_type: 'accuracy' + # kimi_k3 accuracy tests + - name: kimi_k3_w4a8_32p_gpqa + node_size: 4 + test_case: test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py + test_type: 'accuracy' + uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml + with: + runner: linux-amd64-cpu-4 + test_type: ${{ matrix.test_config.test_type || 'perf' }} + test_config_name: ${{ matrix.test_config.name }} + node_size: ${{ matrix.test_config.node_size }} + test_case: ${{ matrix.test_config.test_case }} + image: ${{ needs.set-image-config.outputs.image_a3_cann910 }} + install_sglang_from_source: false + prefill_decode_deployment: 'mix' + transformers_version: '' + run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }} + check-all-jobs: if: ${{ !cancelled() }} needs: @@ -375,14 +523,21 @@ jobs: - nightly-perf-16-npu-a3 - nightly-acc-2-npu-a3 - nightly-acc-16-npu-a3 + - nightly-perf-16-npu-a3-cann910 + - nightly-acc-16-npu-a3-cann910 + - nightly-acc-4-npu-a5 - nightly-poc-multi-node-tests - nightly-poc-multi-node-mix-tests + - nightly-poc-multi-node-tests-cann910 + - nightly-poc-multi-node-mix-tests-cann910 runs-on: ubuntu-latest steps: - name: Generate results table run: | multi_result="${{ needs.nightly-poc-multi-node-tests.result }}" mix_result="${{ needs.nightly-poc-multi-node-mix-tests.result }}" + multi_cann910_result="${{ needs.nightly-poc-multi-node-tests-cann910.result }}" + mix_cann910_result="${{ needs.nightly-poc-multi-node-mix-tests-cann910.result }}" # single-node suites are reported per suite; overall status is success if none failed single_result="success" @@ -396,7 +551,10 @@ jobs: "${{ needs.nightly-perf-4-npu-a3.result }}" \ "${{ needs.nightly-perf-16-npu-a3.result }}" \ "${{ needs.nightly-acc-2-npu-a3.result }}" \ - "${{ needs.nightly-acc-16-npu-a3.result }}"; do + "${{ needs.nightly-acc-16-npu-a3.result }}" \ + "${{ needs.nightly-perf-16-npu-a3-cann910.result }}" \ + "${{ needs.nightly-acc-16-npu-a3-cann910.result }}" \ + "${{ needs.nightly-acc-4-npu-a5.result }}"; do if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then single_result="failure" fi @@ -426,17 +584,22 @@ jobs: "nightly-perf-4-npu-a3:${{ needs.nightly-perf-4-npu-a3.result }}" \ "nightly-perf-16-npu-a3:${{ needs.nightly-perf-16-npu-a3.result }}" \ "nightly-acc-2-npu-a3:${{ needs.nightly-acc-2-npu-a3.result }}" \ - "nightly-acc-16-npu-a3:${{ needs.nightly-acc-16-npu-a3.result }}"; do + "nightly-acc-16-npu-a3:${{ needs.nightly-acc-16-npu-a3.result }}" \ + "nightly-perf-16-npu-a3-cann910:${{ needs.nightly-perf-16-npu-a3-cann910.result }}" \ + "nightly-acc-16-npu-a3-cann910:${{ needs.nightly-acc-16-npu-a3-cann910.result }}" \ + "nightly-acc-4-npu-a5:${{ needs.nightly-acc-4-npu-a5.result }}"; do suite="${entry%%:*}" r="${entry##*:}" echo "| ${suite} | $(group_icon ${r}) ${r} |" >> $GITHUB_STEP_SUMMARY done echo "| multi-node-poc | $(group_icon ${multi_result}) ${multi_result} |" >> $GITHUB_STEP_SUMMARY echo "| multi-node-mix-poc | $(group_icon ${mix_result}) ${mix_result} |" >> $GITHUB_STEP_SUMMARY + echo "| multi-node-poc-cann910 | $(group_icon ${multi_cann910_result}) ${multi_cann910_result} |" >> $GITHUB_STEP_SUMMARY + echo "| multi-node-mix-poc-cann910 | $(group_icon ${mix_cann910_result}) ${mix_cann910_result} |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY FAIL=0 - for r in "${single_result}" "${multi_result}" "${mix_result}"; do + for r in "${single_result}" "${multi_result}" "${mix_result}" "${multi_cann910_result}" "${mix_cann910_result}"; do if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then FAIL=1; fi done exit $FAIL diff --git a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py index e74d96972..eb598edf5 100644 --- a/python/sglang/test/ascend/e2e/test_npu_performance_utils.py +++ b/python/sglang/test/ascend/e2e/test_npu_performance_utils.py @@ -183,6 +183,9 @@ QWEN3_5_397B_W8A8_MODEL_PATH = ( DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH = ( "/root/.cache/modelscope/hub/models/Eco-Tech/DeepSeek-V4-Flash-0731-w8a8" ) +DEEPSEEK_V4_FLASH_DEFAULT_MODEL_PATH = ( + "/root/.cache/modelscope/hub/models/deepseek-ai/DeepSeek-V4-Flash" +) QWEN3_5_397B_W4A8_MODEL_PATH = ( "/root/.cache/modelscope/hub/models/Eco-Tech/Qwen3.5-397B-A17B-w4a8-mtp" ) diff --git a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_fp8_4p_gpqa_a5.py b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_fp8_4p_gpqa_a5.py new file mode 100644 index 000000000..5549d6a5a --- /dev/null +++ b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_fp8_4p_gpqa_a5.py @@ -0,0 +1,143 @@ +import unittest + +from sglang.test.ascend.e2e.test_npu_accuracy_utils import ( + BENCHMARK_TOOL_DEFAULT, + TestNpuAccuracyTestCaseBase, +) +from sglang.test.ascend.e2e.test_npu_performance_utils import ( + DEEPSEEK_V4_FLASH_DEFAULT_MODEL_PATH, +) +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci( + est_time=7200, + suite="nightly-acc-4-npu-a5", + nightly=True, +) + +DEEPSEEK_V4_FLASH_FP8_4P_ENVS = { + "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", + "STREAMS_PER_DEVICE": "32", + "SGLANG_SET_CPU_AFFINITY": "1", + "TASK_QUEUE_ENABLE": "1", + "INF_NAN_MODE_FORCE_DISABLE": "1", + "SGLANG_DEFAULT_THINKING": "1", + "SGLANG_DSV4_REASONING_EFFORT": "max", + # HCCL deepep + "HCCL_BUFFSIZE": "1024", + "HCCL_SOCKET_IFNAME": "lo", + "GLOO_SOCKET_IFNAME": "lo", + # DeepEP long-sequence normal combine (ROUND * TOKENS >= chunked_prefill_size / tp * dp) + "DEEPEP_NORMAL_COMBINE_ENABLE_LONG_SEQ": "1", + "DEEPEP_NORMAL_LONG_SEQ_ROUND": "16", + "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS": "2048", + # "DEEPEP_HYBRID_DEPLOYMENT": "1", + # dsv4 + "IS_DEEPSEEK_V4": "1", + "USE_FUSED_HC_PRE_ASCENDC": "1", + "SGLANG_DSV4_NPU_FUSED_COMPRESSOR": "1", + "SGLANG_DSV4_NPU_FUSED_COMPRESSOR_PREFILL": "1", + # skip gpu branch + "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "False", + "FORCE_DRAFT_MODEL_NON_QUANT": "1", + "SGLANG_DSV4_FP4_EXPERTS": "True", + "SGLANG_OPT_FUSE_WQA_WKV": "0", + "SGLANG_OPT_BF16_FP32_GEMM_ALGO": "torch", + "SGLANG_OPT_USE_FUSED_HASH_TOPK": "False", + "SGLANG_OPT_USE_TILELANG_MHC_PRE": "False", + "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "False", + "SGLANG_OPT_USE_TILELANG_MHC_POST": "False", + "SGLANG_OPT_FP8_WO_A_GEMM": "False", + # performance + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", + "SGLANG_NPU_USE_MULTI_STREAM": "1", + "USE_NPU_MOE_GATING_TOP_K": "1", + "SGLANG_PROFILE_WITH_STACK": "False", +} + +DEEPSEEK_V4_FLASH_FP8_4P_OTHER_ARGS = [ + "--page-size", + 128, + "--tp-size", + 4, + "--trust-remote-code", + "--attention-backend", + "dsv4", + "--device", + "npu", + "--watchdog-timeout", + 9000, + "--mem-fraction-static", + 0.72, + "--max-running-requests", + 64, + "--chunked-prefill-size", + 131072, + "--max-prefill-tokens", + 131072, + "--cuda-graph-bs-decode", + 1, + 2, + 4, + 8, + 10, + 16, + "--kv-cache-dtype", + "auto", + "--enable-dp-lm-head", + "--disable-radix-cache", + "--enable-dp-attention", + "--dp-size", + 4, + "--reasoning-parser", + "deepseek-v4", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + 2, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 3, + "--quantization", + "fp8", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", +] + +DEEPSEEK_V4_FLASH_FP8_4P_GENERATION_CONFIG_MAX = { + "max_tokens": 125000, + "top_p": 1, + "temperature": 1, + "n": 1, + "extra_body": { + "chat_template_kwargs": {"thinking": True, "reasoning_effort": "max"} + }, +} + + +class TestNPUDeepSeekV4FlashFP84PGPQA(TestNpuAccuracyTestCaseBase): + """Test NPU accuracy for DeepSeek-V4-Flash FP8 4p EAGLE GPQA.""" + + benchmark_tool = BENCHMARK_TOOL_DEFAULT + model = DEEPSEEK_V4_FLASH_DEFAULT_MODEL_PATH + other_args = DEEPSEEK_V4_FLASH_FP8_4P_OTHER_ARGS + envs = DEEPSEEK_V4_FLASH_FP8_4P_ENVS + accuracy = 0.874 + datasets = ["gpqa_diamond"] + few_shot_num = 0 + generation_config = DEEPSEEK_V4_FLASH_FP8_4P_GENERATION_CONFIG_MAX + eval_batch_size = 128 + stream = True + timeout = 6000 + seed = 1 + + def test_npu_deepseek_v4_flash_fp8_4p_gpqa(self): + """Run NPU accuracy test for DeepSeek-V4-Flash FP8 4p EAGLE GPQA.""" + self.run_accuracy() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py index d1c8e5317..eabddae0a 100644 --- a/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py +++ b/test/registered/npu/accuracy/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_gpqa.py @@ -15,6 +15,12 @@ register_npu_ci( nightly=True, ) +register_npu_ci( + est_time=7200, + suite="nightly-acc-16-npu-a3-cann910", + nightly=True, +) + DEEPSEEK_V4_FLASH_W8A8_DSPARK_8P_ENVS = { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "STREAMS_PER_DEVICE": "32", diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py deleted file mode 100644 index ddcc68d59..000000000 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms.py +++ /dev/null @@ -1,132 +0,0 @@ -import unittest - -from sglang.test.ascend.e2e.test_npu_performance_utils import ( - AISBENCHMARK_DATASET_DEFAULT, - BENCHMARK_TOOL_DEFAULT, - DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH, - TestNpuPerformanceTestCaseBase, -) -from sglang.test.ci.ci_register import register_npu_ci - -register_npu_ci(est_time=1800, suite="nightly-perf-16-npu-a3", nightly=True) - -# Environment variables for DSV4-Flash single-node PD-mix deployment. -DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = { - "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", - "STREAMS_PER_DEVICE": "32", - "INF_NAN_MODE_FORCE_DISABLE": "1", - "SGLANG_SET_CPU_AFFINITY": "1", - "HCCL_SOCKET_IFNAME": "lo", - "GLOO_SOCKET_IFNAME": "lo", - "HCCL_OP_EXPANSION_MODE": "AIV", - "USE_NPU_MOE_GATING_TOP_K": "1", - "SGLANG_NPU_USE_MULTI_STREAM": "1", - # deepep - "DEEP_NORMAL_MODE_USE_INT8_QUANT": "1", - "DEEPEP_HCCL_BUFFSIZE": "2048", - "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "35", - "DEEPEP_HYBRID_DEPLOYMENT": "1", - # war barrier - "SGLANG_ENABLE_WAR_BARRIER": "1", - "SGLANG_FORCE_COARSE_WAR_BARRIER": "1", - # skip gpu branch - "SGLANG_OPT_FP8_WO_A_GEMM": "0", - "SGLANG_OPT_USE_OVERLAP_STORE_CACHE": "False", - "FORCE_DRAFT_MODEL_NON_QUANT": "1", - "SGLANG_DSV4_FP4_EXPERTS": "False", - "SGLANG_OPT_FUSE_WQA_WKV": "0", - "SGLANG_OPT_BF16_FP32_GEMM_ALGO": "torch", - "SGLANG_OPT_USE_FUSED_HASH_TOPK": "False", - "SGLANG_OPT_USE_TILELANG_MHC_PRE": "False", - "SGLANG_OPT_DEEPGEMM_HC_PRENORM": "False", - "SGLANG_OPT_USE_TILELANG_MHC_POST": "False", - # mtp - "SGLANG_ENABLE_SPEC_V2": "1", - "SGLANG_ENABLE_OVERLAP_PLAN_STREAM": "1", -} - -# Server launch arguments for DSV4-Flash W8A8 single-node 8p PD-mix. -DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS = [ - "--page-size", - 128, - "--tp-size", - 16, - "--trust-remote-code", - "--device", - "npu", - "--prefill-max-requests", - 32, - "--max-prefill-tokens", - 80000, - "--attention-backend", - "dsv4", - "--watchdog-timeout", - 9000, - "--mem-fraction-static", - 0.68, - "--chunked-prefill-size", - 131072, - "--max-running-requests", - 64, - "--dp-size", - 16, - "--enable-dp-attention", - "--moe-a2a-backend", - "deepep", - "--deepep-mode", - "auto", - "--quantization", - "modelslim", - "--enable-dp-lm-head", - "--kv-cache-dtype", - "bfloat16", - "--skip-server-warmup", - "--cuda-graph-bs-decode", - 1, - 2, - 4, - 8, - # MTP (EAGLE) configuration. - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - 2, - "--speculative-eagle-topk", - 1, - "--speculative-num-draft-tokens", - 3, - "--ep-size", - 16, - "--disable-radix-cache", -] - - -class TestNPUDeepSeekV4FlashW8A88PIn32kOut1k50ms(TestNpuPerformanceTestCaseBase): - """Test NPU performance for DeepSeek-V4-Flash W8A8 8p in32k out1k.""" - - benchmark_tool = BENCHMARK_TOOL_DEFAULT - dataset_type = AISBENCHMARK_DATASET_DEFAULT - model = DEEPSEEK_V4_FLASH_0731_W8A8_MODEL_PATH - other_args = DEEPSEEK_V4_FLASH_W8A8_8P_OTHER_ARGS - envs = DEEPSEEK_V4_FLASH_W8A8_8P_ENVS - dataset_name = "random" - dataset_path = "/root/.cache/modelscope/hub/datasets/gsm8k_deepseekv4/cache0_32000/formal_run1_64_32000_cache0.json" - input_len = 32000 - output_len = 1000 - num_prompts = 64 - max_concurrency = 64 - random_range_ratio = 1 - warmup_requests = 16 - request_rate = float("inf") - seed = 1 - tpot = 50 - max_attempts = 3 - output_token_throughput = 927 - - def test_npu_deepseek_v4_flash_w8a8_8p_in32k_out1k_50ms(self): - """Run NPU performance test for DeepSeek-V4-Flash W8A8 8p in32k out1k.""" - self.run_throughput() - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py index 93486464b..2de42b09b 100644 --- a/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py +++ b/test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_8p_in8k_out1k_50ms.py @@ -9,6 +9,7 @@ from sglang.test.ascend.e2e.test_npu_performance_utils import ( from sglang.test.ci.ci_register import register_npu_ci register_npu_ci(est_time=1800, suite="nightly-perf-16-npu-a3", nightly=True) +register_npu_ci(est_time=1800, suite="nightly-perf-16-npu-a3-cann910", nightly=True) # Environment variables for DSV4-Flash single-node PD-mix deployment. DEEPSEEK_V4_FLASH_W8A8_8P_ENVS = { diff --git a/test/run_suite.py b/test/run_suite.py index 3edbff392..3eca4c441 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -170,9 +170,12 @@ NIGHTLY_SUITES = { "nightly-acc-2-npu-a3", "nightly-acc-8-npu-a3", "nightly-acc-16-npu-a3", + "nightly-acc-4-npu-a5", "nightly-perf-2-npu-a3", "nightly-perf-4-npu-a3", "nightly-perf-16-npu-a3", + "nightly-perf-16-npu-a3-cann910", + "nightly-acc-16-npu-a3-cann910", "full-4-npu-a2", "full-1-npu-a3", "full-2-npu-a3",