name: Nightly Test (MUSA)

on:
  schedule:
    - cron: '0 16 * * *'
  workflow_dispatch:
    inputs:
      job_filter:
        description: "Select which job to run (empty/all to run all jobs)"
        required: false
        type: choice
        default: 'all'
        options:
          - 'all'
          - 'nightly-test-musa-general-kernel'
          - 'nightly-test-musa-general-multimodal-layer'
          - 'nightly-test-llm-server-1-gpu-musa'
          - 'nightly-test-multimodal-server-1-gpu-musa'
          - 'nightly-test-multimodal-server-2-gpu-musa'
  workflow_call:
    inputs:
      ref:
        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
        required: false
        type: string
        default: ''
      job_filter:
        description: 'Select which job to run (empty or "all" to run all jobs)'
        required: false
        type: string
        default: 'all'

concurrency:
  group: nightly-test-musa-${{ inputs.ref || github.ref }}
  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}

env:
  SGLANG_IS_IN_CI: true

jobs:
  # ==================== General: kernel ====================
  nightly-test-musa-general-kernel:
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
    runs-on: s5000-1-gpu-runner
    timeout-minutes: 240
    env:
      TORCHADA_ENABLE_CPP_OPS: 1
      HF_HUB_CACHE: /hf-cache/hub
    steps:
      - name: Gate by job_filter
        id: gate
        run: |
          filter="${{ inputs.job_filter || 'all' }}"
          if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-musa-general-kernel" ]]; then
            echo "run_job=true" >> "$GITHUB_OUTPUT"
          else
            echo "run_job=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Checkout code
        if: steps.gate.outputs.run_job == 'true'
        uses: actions/checkout@v4
        timeout-minutes: 10
        with:
          ref: ${{ inputs.ref || github.ref }}

      - name: Install dependencies
        if: steps.gate.outputs.run_job == 'true'
        timeout-minutes: 30
        run: |
          bash scripts/ci/musa/musa_install_dependency.sh

      - name: Run sgl-kernel unit tests (MUSA)
        if: steps.gate.outputs.run_job == 'true'
        timeout-minutes: 30
        run: |
          pytest test/registered/kernels/ops/speculative/test_eagle_utils.py
          pytest test/registered/kernels/ops/speculative/test_ngram_utils.py
          pytest test/registered/kernels/ops/speculative/test_speculative_sampling.py
          pytest python/sglang/kernels/aot/tests/test_torch_defaults_reset.py

  # ==================== General: multimodal layer ====================
  nightly-test-musa-general-multimodal-layer:
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
    runs-on: s5000-1-gpu-runner
    timeout-minutes: 240
    env:
      SGLANG_USE_MODELSCOPE: false
      SGLANG_IS_IN_CI: true
      TORCHADA_ENABLE_CPP_OPS: 1
      HF_HOME: /hf-cache
      HF_HUB_CACHE: /hf-cache/hub
      HF_HUB_OFFLINE: 1
    steps:
      - name: Gate by job_filter
        id: gate
        run: |
          filter="${{ inputs.job_filter || 'all' }}"
          if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-musa-general-multimodal-layer" ]]; then
            echo "run_job=true" >> "$GITHUB_OUTPUT"
          else
            echo "run_job=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Checkout code
        timeout-minutes: 10
        if: steps.gate.outputs.run_job == 'true'
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.ref }}

      - name: Install dependencies
        timeout-minutes: 30
        if: steps.gate.outputs.run_job == 'true'
        run: |
          bash scripts/ci/musa/musa_install_dependency.sh

      - name: Run multimodal MUSA layer unit tests
        if: steps.gate.outputs.run_job == 'true'
        timeout-minutes: 30
        run: |
          pytest python/sglang/multimodal_gen/test/unit/musa/layers/test_musa_rmsnorm.py
          pytest python/sglang/multimodal_gen/test/unit/musa/layers/test_musa_silu_and_mul.py

  # ==================== LLM server: 1-GPU ====================
  nightly-test-llm-server-1-gpu-musa:
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
    runs-on: s5000-1-gpu-runner
    timeout-minutes: 240
    env:
      SGLANG_USE_MODELSCOPE: false
      SGLANG_IS_IN_CI: true
      TORCHADA_ENABLE_CPP_OPS: 1
      HF_HOME: /hf-cache
      HF_HUB_CACHE: /hf-cache/hub
      HF_HUB_OFFLINE: 1
    steps:
      - name: Gate by job_filter
        id: gate
        run: |
          filter="${{ inputs.job_filter || 'all' }}"
          if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-llm-server-1-gpu-musa" ]]; then
            echo "run_job=true" >> "$GITHUB_OUTPUT"
          else
            echo "run_job=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Checkout code
        timeout-minutes: 10
        if: steps.gate.outputs.run_job == 'true'
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.ref }}

      - name: Install dependencies
        timeout-minutes: 30
        if: steps.gate.outputs.run_job == 'true'
        run: |
          bash scripts/ci/musa/musa_install_dependency.sh

      - name: Run DeepSeek-V2-Lite-Chat LLM server smoke test
        if: steps.gate.outputs.run_job == 'true'
        timeout-minutes: 120
        run: |
          cd test
          python3 run_suite.py \
            --hw musa \
            --suite nightly-musa-1-gpu \
            --nightly \
            --timeout-per-file 7200

  # ==================== Multimodal: 1-GPU (split) ====================
  nightly-test-multimodal-server-1-gpu-musa:
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
    runs-on: s5000-1-gpu-runner
    strategy:
      fail-fast: false
      max-parallel: 2
      matrix:
        part: [0, 1]
    timeout-minutes: 240
    env:
      SGLANG_USE_MODELSCOPE: false
      SGLANG_IS_IN_CI: true
      TORCHADA_ENABLE_CPP_OPS: 1
      HF_HOME: /hf-cache
      HF_HUB_CACHE: /hf-cache/hub
      HF_HUB_OFFLINE: 1
    steps:
      - name: Gate by job_filter
        id: gate
        run: |
          filter="${{ inputs.job_filter || 'all' }}"
          if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-multimodal-server-1-gpu-musa" ]]; then
            echo "run_job=true" >> "$GITHUB_OUTPUT"
          else
            echo "run_job=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Checkout code
        timeout-minutes: 10
        if: steps.gate.outputs.run_job == 'true'
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.ref }}

      - name: Install dependencies
        timeout-minutes: 30
        if: steps.gate.outputs.run_job == 'true'
        run: |
          bash scripts/ci/musa/musa_install_dependency.sh

      - name: Run diffusion server tests (1-GPU)
        if: steps.gate.outputs.run_job == 'true'
        timeout-minutes: 60
        env:
          RUNAI_STREAMER_MEMORY_LIMIT: 0
        run: |
          cd python
          python3 sglang/multimodal_gen/test/server/musa/run_suite.py \
            --suite 1-gpu-musa-nightly \
            --partition-id ${{ matrix.part }} \
            --total-partitions 2 \
            --continue-on-error

  # ==================== Multimodal: 2-GPU ====================
  nightly-test-multimodal-server-2-gpu-musa:
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
    runs-on: s5000-2-gpu-runner
    timeout-minutes: 240
    env:
      SGLANG_USE_MODELSCOPE: false
      SGLANG_IS_IN_CI: true
      TORCHADA_ENABLE_CPP_OPS: 1
      HF_HOME: /hf-cache
      HF_HUB_CACHE: /hf-cache/hub
      HF_HUB_OFFLINE: 1
    steps:
      - name: Gate by job_filter
        id: gate
        run: |
          filter="${{ inputs.job_filter || 'all' }}"
          if [[ -z "$filter" || "$filter" == "all" || "$filter" == "nightly-test-multimodal-server-2-gpu-musa" ]]; then
            echo "run_job=true" >> "$GITHUB_OUTPUT"
          else
            echo "run_job=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Checkout code
        timeout-minutes: 10
        if: steps.gate.outputs.run_job == 'true'
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref || github.ref }}

      - name: Install dependencies
        timeout-minutes: 30
        if: steps.gate.outputs.run_job == 'true'
        run: |
          bash scripts/ci/musa/musa_install_dependency.sh

      - name: Run diffusion server tests (2-GPU)
        if: steps.gate.outputs.run_job == 'true'
        timeout-minutes: 60
        env:
          RUNAI_STREAMER_MEMORY_LIMIT: 0
        run: |
          cd python
          python3 sglang/multimodal_gen/test/server/musa/run_suite.py \
            --suite 2-gpu-musa \
            --continue-on-error

  # ==================== finish ====================
  nightly-test-musa-finish:
    needs:
      [
        nightly-test-musa-general-kernel,
        nightly-test-musa-general-multimodal-layer,
        nightly-test-llm-server-1-gpu-musa,
        nightly-test-multimodal-server-1-gpu-musa,
        nightly-test-multimodal-server-2-gpu-musa,
      ]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check all dependent job statuses
        run: |
          json_needs='${{ toJson(needs) }}'
          job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]')

          for job in $job_names; do
            result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')
            echo "$job: $result"
            if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
              echo "Nightly failed."
              exit 1
            fi
          done
          echo "All jobs completed successfully."