name: Full Test (NPU)

on:
#  pull_request:
#    branches:
#      - main
#    paths:
#      - ".github/workflows/full-test-npu.yml"
  workflow_dispatch:
    inputs:
      ref:
        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
        required: false
        type: string
        default: ''
      job_filter:
        description: 'Select which job to run (leave empty or "all" to run all jobs)'
        required: false
        type: string
        default: 'all'
      image_a2:
        description: 'The a2 running docker image of the test task.'
        required: false
        type: string
        default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b'
      image_a3:
        description: 'The a3 running docker image of the test task. Must be an image with sglang preinstalled (e.g. lmsysorg/sglang:*a3*).'
        required: false
        type: string
        default: 'swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3'

concurrency:
  group: full-test-npu-${{ inputs.ref || github.ref }}
  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}

jobs:
  set-image-config:
    runs-on: ubuntu-latest
    outputs:
      ref: ${{ steps.set-vars.outputs.ref }}
      job_filter: ${{ steps.set-vars.outputs.job_filter }}
      image_a2: ${{ steps.set-vars.outputs.image_a2 }}
      image_a3: ${{ steps.set-vars.outputs.image_a3 }}
    steps:
      # When triggered by PR, no inputs parameters are used. The latest community code is tested by default.
      - name: Set image config
        id: set-vars
        run: |
          if [ -z "${{ inputs.ref }}" ]; then
            echo "ref=" >> $GITHUB_OUTPUT
          else
            echo "ref=${{ inputs.ref }}" >> $GITHUB_OUTPUT
          fi

          if [ -z "${{ inputs.job_filter }}" ]; then
            echo "job_filter=all" >> $GITHUB_OUTPUT
          else
            echo "job_filter=${{ inputs.job_filter }}" >> $GITHUB_OUTPUT
          fi

          if [ -z "${{ inputs.image_a2 }}" ]; then
            echo "image_a2=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-910b" >> $GITHUB_OUTPUT
          else
            echo "image_a2=${{ inputs.image_a2 }}" >> $GITHUB_OUTPUT
          fi

          if [ -z "${{ inputs.image_a3 }}" ]; then
            echo "image_a3=swr.cn-southwest-2.myhuaweicloud.com/base_image/dockerhub/lmsysorg/sglang:main-cann9.0.0-a3" >> $GITHUB_OUTPUT
          else
            echo "image_a3=${{ inputs.image_a3 }}" >> $GITHUB_OUTPUT
          fi

  nighly-test-npu:
    needs: [set-image-config]
    name: nightly-test-npu
    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
    uses: ./.github/workflows/nightly-test-npu.yml
    with:
      ref:  ${{ needs.set-image-config.outputs.ref }}
      job_filter: ${{ needs.set-image-config.outputs.job_filter }}
      image_a3: ${{ needs.set-image-config.outputs.image_a3 }}
    secrets: inherit

  full-4-npu-a2:
    name: full-4-npu-a2
    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-single-node-test-stage.yml
    with:
      runner: linux-aarch64-a2-4
      test_type: 'perf'
      test_suite: full-4-npu-a2
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
      image: ${{ needs.set-image-config.outputs.image_a2 }}
      device_type_for_deps: '910b'

  full-1-npu-a3:
    name: full-1-npu-a3
    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-pr-test-stage.yml
    with:
      self_name: full-1-npu-a3
      runner_config: linux-aarch64-a3-2
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      run_timeout_minutes: '240'
      timeout_per_file: '3600'
      ref: ${{ needs.set-image-config.outputs.ref }}
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
    secrets: inherit

  full-2-npu-a3:
    name: full-2-npu-a3
    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-pr-test-stage.yml
    with:
      self_name: full-2-npu-a3
      runner_config: linux-aarch64-a3-2
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      run_timeout_minutes: '240'
      timeout_per_file: '3600'
      ref: ${{ needs.set-image-config.outputs.ref }}
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
    secrets: inherit

  full-4-npu-a3:
    name: full-4-npu-a3
    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-pr-test-stage.yml
    with:
      self_name: full-4-npu-a3
      runner_config: linux-aarch64-a3-4
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      run_timeout_minutes: '240'
      timeout_per_file: '3600'
      ref: ${{ needs.set-image-config.outputs.ref }}
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
    secrets: inherit

  full-8-npu-a3:
    name: full-8-npu-a3
    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-pr-test-stage.yml
    with:
      self_name: full-8-npu-a3
      runner_config: linux-aarch64-a3-8
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      run_timeout_minutes: '240'
      timeout_per_file: '3600'
      ref: ${{ needs.set-image-config.outputs.ref }}
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
    secrets: inherit

  full-16-npu-a3:
    name: full-16-npu-a3
    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-pr-test-stage.yml
    with:
      self_name: full-16-npu-a3
      runner_config: linux-aarch64-a3-16
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      run_timeout_minutes: '240'
      timeout_per_file: '3600'
      ref: ${{ needs.set-image-config.outputs.ref }}
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
    secrets: inherit

  full-acc-2-npu-a3:
    name: full-acc-2-npu-a3
    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-single-node-test-stage.yml
    with:
      runner: linux-aarch64-a3-2-
      test_type: 'accuracy'
      test_suite: full-acc-2-npu-a3
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      is_nightly_pipeline_job: true
      install_sglang_deps: true
      device_type_for_deps: 'a3'
      skip_pr_test_health_check: 'true'
    secrets: inherit

  full-acc-4-npu-a3:
    name: full-acc-4-npu-a3
    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-single-node-test-stage.yml
    with:
      runner: linux-aarch64-a3-4-
      test_type: 'accuracy'
      test_suite: full-acc-4-npu-a3
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      is_nightly_pipeline_job: true
      install_sglang_deps: true
      device_type_for_deps: 'a3'
      skip_pr_test_health_check: 'true'
    secrets: inherit

  full-acc-16-npu-a3:
    name: full-acc-16-npu-a3
    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-single-node-test-stage.yml
    with:
      runner: linux-aarch64-a3-16-
      test_type: 'accuracy'
      test_suite: full-acc-16-npu-a3
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      is_nightly_pipeline_job: true
      install_sglang_deps: true
      device_type_for_deps: 'a3'
      skip_pr_test_health_check: 'true'
    secrets: inherit

  full-perf-2-npu-a3:
    name: full-perf-2-npu-a3
    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-pr-test-stage.yml
    with:
      self_name: full-perf-2-npu-a3
      runner_config: linux-aarch64-a3-800t-2
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      run_timeout_minutes: '240'
      timeout_per_file: '3600'
      ref: ${{ needs.set-image-config.outputs.ref }}
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
    secrets: inherit

  full-perf-4-npu-a3:
    name: full-perf-4-npu-a3
    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-pr-test-stage.yml
    with:
      self_name: full-perf-4-npu-a3
      runner_config: linux-aarch64-a3-800t-4
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      run_timeout_minutes: '240'
      timeout_per_file: '3600'
      ref: ${{ needs.set-image-config.outputs.ref }}
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
    secrets: inherit

  full-perf-8-npu-a3:
    name: full-perf-8-npu-a3
    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-pr-test-stage.yml
    with:
      self_name: full-perf-8-npu-a3
      runner_config: linux-aarch64-a3-800t-8
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      run_timeout_minutes: '240'
      timeout_per_file: '3600'
      ref: ${{ needs.set-image-config.outputs.ref }}
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
    secrets: inherit

  full-perf-16-npu-a3:
    name: full-perf-16-npu-a3
    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
    needs: [set-image-config]
    uses: ./.github/workflows/_npu-pr-test-stage.yml
    with:
      self_name: full-perf-16-npu-a3
      runner_config: linux-aarch64-a3-800t-16
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      run_timeout_minutes: '240'
      timeout_per_file: '3600'
      ref: ${{ needs.set-image-config.outputs.ref }}
      is_nightly_pipeline_job: true
      skip_pr_test_health_check: 'true'
    secrets: inherit

  full-poc-multi-node-mix-tests:
    name: multi-node-mix-poc
    if: ${{ (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') }}
    needs: [set-image-config]
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        test_config:
          # kimi_k2_6 performance tests
          - name: kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25
            node_size: 2
            test_case: test/registered/npu/accuracy/kimi_k2_6/test_npu_kimi_k2_6_w4a8_16p_in64k_out1k_100ms_aime25.py
            test_type: 'accuracy'
    uses: ./.github/workflows/nightly-test-npu-e2e-multi-node.yml
    with:
      runner: linux-amd64-cpu-4
      test_type: ${{ matrix.test_config.test_type || 'perf' }}
      test_config_name: ${{ matrix.test_config.name }}
      node_size: ${{ matrix.test_config.node_size }}
      test_case: ${{ matrix.test_config.test_case }}
      image: ${{ needs.set-image-config.outputs.image_a3 }}
      install_sglang_from_source: false
      prefill_decode_deployment: 'mix'
      transformers_version: ''

  check-all-jobs:
    if: github.repository == 'sgl-project/sglang' && always()
    needs:
      - nighly-test-npu
      - full-4-npu-a2
      - full-1-npu-a3
      - full-2-npu-a3
      - full-4-npu-a3
      - full-8-npu-a3
      - full-16-npu-a3
      - full-acc-2-npu-a3
      - full-acc-4-npu-a3
      - full-acc-16-npu-a3
      - full-perf-2-npu-a3
      - full-perf-4-npu-a3
      - full-perf-8-npu-a3
      - full-perf-16-npu-a3
      - full-poc-multi-node-mix-tests
    runs-on: ubuntu-latest
    container:
      image: docker.m.daocloud.io/ubuntu:22.04
    steps:
      - name: Check if any job failed
        run: |
          if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
            echo "One or more nightly test jobs failed"
            exit 1
          fi
          if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
            echo "One or more nightly test jobs were cancelled"
            exit 1
          fi
          echo "All nightly test jobs passed"