sglang/.github/workflows/nightly-test-npu.yml

name: Nightly Test (NPU)
on:
  schedule:
    - cron: '0 16 * * *'  # Execute at 0:00 a.m. Beijing Time every day
  pull_request:
    branches:
      - main
    paths:
      - ".github/workflows/nightly-test-npu.yml"
  workflow_dispatch:
    inputs:
      ref:
        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
        required: false
        type: string
        default: ''
      job_filter:
        description: 'Select which job to run (leave empty or "all" to run all jobs)'
        required: false
        type: string
        default: 'all'
      image_a2:
        description: 'The a2 running docker image of the test task.'
        required: false
        type: string
        default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b'
      image_a3:
        description: 'The a3 running docker image of the test task.'
        required: false
        type: string
        default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3'
      skip_install_flag:
        description: 'Indicates whether to skip the installation of sglang, defaulting to false.'
        required: false
        type: string
        default: 'true'
  workflow_call:
    inputs:
      ref:
        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
        required: false
        type: string
        default: ''
      job_filter:
        description: 'Select which job to run (leave empty or "all" to run all jobs)'
        required: false
        type: string
        default: 'all'
      image_a2:
        description: 'The a2 running docker image of the test task.'
        required: false
        type: string
        default: ''
      image_a3:
        description: 'The a3 running docker image of the test task.'
        required: false
        type: string
        default: ''
      skip_install_flag:
        description: 'Indicates whether to skip the installation of sglang, defaulting to false.'
        required: false
        type: string
        default: ''
concurrency:
  group: nightly-test-npu-${{ inputs.ref || github.ref }}
  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
jobs:
  set-image-config:
    runs-on: ubuntu-latest
    outputs:
      ref: ${{ steps.set-vars.outputs.ref }}
      job_filter: ${{ steps.set-vars.outputs.job_filter }}
      image_a2: ${{ steps.set-vars.outputs.image_a2 }}
      image_a3: ${{ steps.set-vars.outputs.image_a3 }}
      skip_install_flag: ${{ steps.set-vars.outputs.skip_install_flag }}
      run_start_metadata: ${{ steps.set-vars.outputs.run_start_metadata }}
    steps:
      # When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
      - name: Set image config
        id: set-vars
        env:
          # Pass untrusted PR metadata through environment variables to avoid shell‑script interpolation,
          # mitigating script injection risks from fork branch names.
          HEAD_LABEL: ${{ github.event.pull_request.head.label }}
        run: |
          if [ -z "${{ inputs.ref }}" ]; then
            echo "ref=" >> $GITHUB_OUTPUT
          else
            echo "ref=${{ inputs.ref }}" >> $GITHUB_OUTPUT
          fi
          if [ -z "${{ inputs.job_filter }}" ]; then
            echo "job_filter=all" >> $GITHUB_OUTPUT
          else
            echo "job_filter=${{ inputs.job_filter }}" >> $GITHUB_OUTPUT
          fi
          if [ -z "${{ inputs.image_a2 }}" ]; then
            echo "image_a2=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b" >> $GITHUB_OUTPUT
          else
            echo "image_a2=${{ inputs.image_a2 }}" >> $GITHUB_OUTPUT
          fi
          if [ -z "${{ inputs.image_a3 }}" ]; then
            echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3" >> $GITHUB_OUTPUT
          else
            echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT
          fi
          if [ -z "${{ inputs.skip_install_flag }}" ]; then
            echo "skip_install_flag=false" >> $GITHUB_OUTPUT
          else
            echo "skip_install_flag=${{ inputs.skip_install_flag }}" >> $GITHUB_OUTPUT
          fi
          # If the nightly pipeline is triggered by a PR, this variable takes the source branch label of the PR;
          # otherwise, it uses the branch or tag name of the current ref.
          BRANCH_LABEL="$HEAD_LABEL"
          if [ -z "${BRANCH_LABEL}" ]; then
            BRANCH_LABEL="${{ github.ref_name }}"
          fi
          BRANCH_LABEL="$(echo "${BRANCH_LABEL}" | tr -cd 'a-zA-Z0-9._/:' | tr '/:' '--')"
          # Package run metadata (branch/workflow/create_time) as a single JSON for reuse by all
          # test jobs; create_time is UTC+8 date + time (minute precision) recorded once at run
          # start, shared even across midnight.
          RUN_START_METADATA="$(python3 -c "import json,sys; print(json.dumps({'branch_label':sys.argv[1],'workflow_name':sys.argv[2],'create_time':sys.argv[3]}))" "$BRANCH_LABEL" "${{ github.workflow }}" "$(date -u -d '+8 hours' +%Y%m%d-%H%M)")"
          # Write to GITHUB_OUTPUT and print to the log in one command.
          echo "run_start_metadata=${RUN_START_METADATA}" | tee -a $GITHUB_OUTPUT

  nightly-1-npu-a3:
    name: nightly-1-npu-a3
    if: ${{ !cancelled() }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-pr-test-stage.yml
    with:
      self_name: nightly-1-npu-a3
      runner_config: linux-aarch64-a3-2-
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      run_timeout_minutes: '60'
      timeout_per_file: '3600'
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
    secrets: inherit

  nightly-2-npu-a3:
    name: nightly-2-npu-a3
    if: ${{ !cancelled() }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-pr-test-stage.yml
    with:
      self_name: nightly-2-npu-a3
      runner_config: linux-aarch64-a3-2-
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      run_timeout_minutes: '60'
      timeout_per_file: '3600'
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
    secrets: inherit

  # The nightly-4-npu-a3 suite is split across 2 parallel partition jobs, mirroring base-b-test-4-npu-a3 in the PR pipeline.
  nightly-4-npu-a3:
    name: nightly-4-npu-a3
    if: ${{ !cancelled() }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-pr-test-stage.yml
    with:
      self_name: nightly-4-npu-a3
      runner_config: linux-aarch64-a3-4-
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      run_timeout_minutes: '120'
      timeout_per_file: '3600'
      partitions: '{"size":2,"arr":[0,1]}'
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
    secrets: inherit

  nightly-8-npu-a3:
    name: nightly-8-npu-a3
    if: ${{ !cancelled() }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-pr-test-stage.yml
    with:
      self_name: nightly-8-npu-a3
      runner_config: linux-aarch64-a3-8-
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      run_timeout_minutes: '60'
      timeout_per_file: '3600'
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
    secrets: inherit

  nightly-16-npu-a3:
    name: nightly-16-npu-a3
    if: ${{ !cancelled() }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-pr-test-stage.yml
    with:
      self_name: nightly-16-npu-a3
      runner_config: linux-aarch64-a3-16-
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      run_timeout_minutes: '120'
      timeout_per_file: '3600'
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
    secrets: inherit

  nightly-perf-2-npu-a3:
    name: nightly-perf-2-npu-a3
    if: ${{ !cancelled() }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-single-node-test-stage.yml
    with:
      runner: linux-aarch64-a3-800t-2
      test_type: 'perf'
      test_suite: nightly-perf-2-npu-a3
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      install_sglang_deps: false
      device_type_for_deps: 'a3'
      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}

  nightly-perf-4-npu-a3:
    name: nightly-perf-4-npu-a3
    if: ${{ !cancelled() }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-single-node-test-stage.yml
    with:
      runner: linux-aarch64-a3-800t-4
      test_type: 'perf'
      test_suite: nightly-perf-4-npu-a3
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      install_sglang_deps: false
      device_type_for_deps: 'a3'
      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}

  nightly-perf-16-npu-a3:
    name: nightly-perf-16-npu-a3
    if: ${{ !cancelled() }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-single-node-test-stage.yml
    with:
      runner: linux-aarch64-a3-800t-16
      test_type: 'perf'
      test_suite: nightly-perf-16-npu-a3
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      install_sglang_deps: false
      device_type_for_deps: 'a3'
      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}
      test_timeout_minutes: '180'

  # The acc-2 suite is split across 4 parallel partition jobs via the single `partitions` input.
  nightly-acc-2-npu-a3:
    name: nightly-acc-2-npu-a3
    if: ${{ !cancelled() }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-single-node-test-stage.yml
    with:
      runner: linux-aarch64-a3-2-
      test_type: 'accuracy'
      test_suite: nightly-acc-2-npu-a3
      is_nightly_pipeline_job: true
      partitions: '{"size":4,"arr":[0,1,2,3]}'
      skip_pr_test_health_check: 'true'
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      install_sglang_deps: false
      device_type_for_deps: 'a3'
      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}

  nightly-acc-16-npu-a3:
    name: nightly-acc-16-npu-a3
    if: ${{ !cancelled() }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-single-node-test-stage.yml
    with:
      runner: linux-aarch64-a3-16-
      test_type: 'accuracy'
      test_suite: nightly-acc-16-npu-a3
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      install_sglang_deps: false
      device_type_for_deps: 'a3'
      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}

  nightly-poc-multi-node-tests:
    name: multi-node-poc
    if: ${{ !cancelled() }}
    needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-16-npu-a3]
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        test_config:
          # glm5_1 performance tests
          - name: glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms
            prefill_size: 2
            decode_size: 2
            router_size: 1
            test_case: test/registered/npu/performance/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms.py
            test_type: 'perf'
            prefill_decode_deployment: 'separation'
          - name: glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26
            prefill_size: 2
            decode_size: 2
            router_size: 1
            test_case: test/registered/npu/accuracy/glm5_1/test_npu_glm5_1_w4a8_1p1d_32p_in64k_out1k_50ms_aime26.py
            test_type: 'accuracy'
            prefill_decode_deployment: 'separation'
          # deepseek_v4_flash performance tests
          - name: deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms
            prefill_size: 1
            decode_size: 1
            router_size: 1
            test_case: test/registered/npu/performance/deepseek_v4_flash/test_npu_deepseek_v4_flash_w8a8_1p1d_16p_in8k_out1k_50ms.py
            test_type: 'perf'
            prefill_decode_deployment: 'separation'
    uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
    with:
      runner: linux-amd64-cpu-4
      test_type: ${{ matrix.test_config.test_type || 'perf' }}
      test_config_name: ${{ matrix.test_config.name }}
      prefill_size: ${{ matrix.test_config.prefill_size }}
      decode_size: ${{ matrix.test_config.decode_size }}
      router_size: ${{ matrix.test_config.router_size }}
      test_case: ${{ matrix.test_config.test_case }}
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      install_sglang_from_source: false
      prefill_decode_deployment: ${{ matrix.test_config.prefill_decode_deployment }}
      transformers_version: ''
      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}

  nightly-poc-multi-node-mix-tests:
    name: multi-node-mix-poc
    if: ${{ !cancelled() }}
    needs: [set-image-config, nightly-perf-2-npu-a3, nightly-perf-4-npu-a3, nightly-perf-16-npu-a3, nightly-acc-2-npu-a3, nightly-acc-16-npu-a3, nightly-poc-multi-node-tests]
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        test_config:
          # glm_5_2 accuracy tests
          - name: glm_5_2_w4a8_16p_gpqa
            node_size: 2
            test_case: test/registered/npu/accuracy/glm5_2/test_npu_glm_5_2_w4a8_16p_gpqa.py
            test_type: 'accuracy'
          # kimi_k3 accuracy tests
          - name: kimi_k3_w4a8_32p_gpqa
            node_size: 4
            test_case: test/registered/npu/accuracy/kimi_k3/test_npu_kimi_k3_w4a8_32p_gpqa.py
            test_type: 'accuracy'
    uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
    with:
      runner: linux-amd64-cpu-4
      test_type: ${{ matrix.test_config.test_type || 'perf' }}
      test_config_name: ${{ matrix.test_config.name }}
      node_size: ${{ matrix.test_config.node_size }}
      test_case: ${{ matrix.test_config.test_case }}
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      install_sglang_from_source: false
      prefill_decode_deployment: 'mix'
      transformers_version: ''
      run_start_metadata: ${{ needs.set-image-config.outputs.run_start_metadata }}

  check-all-jobs:
    if: ${{ !cancelled() }}
    needs:
      - nightly-1-npu-a3
      - nightly-2-npu-a3
      - nightly-4-npu-a3
      - nightly-8-npu-a3
      - nightly-16-npu-a3
      - nightly-perf-2-npu-a3
      - nightly-perf-4-npu-a3
      - nightly-perf-16-npu-a3
      - nightly-acc-2-npu-a3
      - nightly-acc-16-npu-a3
      - nightly-poc-multi-node-tests
      - nightly-poc-multi-node-mix-tests
    runs-on: ubuntu-latest
    steps:
      - name: Generate results table
        run: |
          multi_result="${{ needs.nightly-poc-multi-node-tests.result }}"
          mix_result="${{ needs.nightly-poc-multi-node-mix-tests.result }}"

          # single-node suites are reported per suite; overall status is success if none failed
          single_result="success"
          for r in \
            "${{ needs.nightly-1-npu-a3.result }}" \
            "${{ needs.nightly-2-npu-a3.result }}" \
            "${{ needs.nightly-4-npu-a3.result }}" \
            "${{ needs.nightly-8-npu-a3.result }}" \
            "${{ needs.nightly-16-npu-a3.result }}" \
            "${{ needs.nightly-perf-2-npu-a3.result }}" \
            "${{ needs.nightly-perf-4-npu-a3.result }}" \
            "${{ needs.nightly-perf-16-npu-a3.result }}" \
            "${{ needs.nightly-acc-2-npu-a3.result }}" \
            "${{ needs.nightly-acc-16-npu-a3.result }}"; do
            if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then
              single_result="failure"
            fi
          done

          group_icon() {
            case "$1" in
              success)   echo "✅" ;;
              failure)   echo "❌" ;;
              cancelled) echo "⏭️" ;;
              skipped)   echo "⏭️" ;;
              *)         echo "❓" ;;
            esac
          }

          echo "## Nightly Test Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Group | Status |" >> $GITHUB_STEP_SUMMARY
          echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY
          for entry in \
            "nightly-1-npu-a3:${{ needs.nightly-1-npu-a3.result }}" \
            "nightly-2-npu-a3:${{ needs.nightly-2-npu-a3.result }}" \
            "nightly-4-npu-a3:${{ needs.nightly-4-npu-a3.result }}" \
            "nightly-8-npu-a3:${{ needs.nightly-8-npu-a3.result }}" \
            "nightly-16-npu-a3:${{ needs.nightly-16-npu-a3.result }}" \
            "nightly-perf-2-npu-a3:${{ needs.nightly-perf-2-npu-a3.result }}" \
            "nightly-perf-4-npu-a3:${{ needs.nightly-perf-4-npu-a3.result }}" \
            "nightly-perf-16-npu-a3:${{ needs.nightly-perf-16-npu-a3.result }}" \
            "nightly-acc-2-npu-a3:${{ needs.nightly-acc-2-npu-a3.result }}" \
            "nightly-acc-16-npu-a3:${{ needs.nightly-acc-16-npu-a3.result }}"; do
            suite="${entry%%:*}"
            r="${entry##*:}"
            echo "| ${suite} | $(group_icon ${r}) ${r} |" >> $GITHUB_STEP_SUMMARY
          done
          echo "| multi-node-poc | $(group_icon ${multi_result}) ${multi_result} |" >> $GITHUB_STEP_SUMMARY
          echo "| multi-node-mix-poc | $(group_icon ${mix_result}) ${mix_result} |" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          FAIL=0
          for r in "${single_result}" "${multi_result}" "${mix_result}"; do
            if [ "${r}" != "success" ] && [ "${r}" != "skipped" ]; then FAIL=1; fi
          done
          exit $FAIL