name: PR Test Base
run-name: >-
  ${{
    github.event_name == 'schedule' && 'Scheduled Full Run'
    || github.event_name == 'workflow_dispatch' && format('Manual run by {0}', github.actor)
    || github.event_name == 'pull_request' && format('PR #{0} - {1}', github.event.pull_request.number, github.event.pull_request.title)
    || 'PR Test Base'
  }}

on:
  schedule:
    - cron: '0 11,23 * * *'  # Run 2x daily, 12h apart, off PR-push peaks
  pull_request:
  workflow_dispatch:
    inputs:
      force_continue_on_error:
        description: "Force continue-on-error (test scheduled CI behavior)"
        required: false
        type: boolean
        default: false
      test_parallel_dispatch:
        description: "Test parallel dispatch behavior (simulates scheduled run)"
        required: false
        type: boolean
        default: false
      run_all_tests:
        description: "Run all tests (mirrors the scheduled cron's behavior — combine with test_parallel_dispatch for full sim)"
        required: false
        type: boolean
        default: false
  workflow_call:
    inputs:
      git_ref:
        description: 'Git ref (branch, tag, or SHA) to test. If not provided, uses the default branch.'
        required: false
        type: string
        default: ''
      run_all_tests:
        description: "Run all tests (for releasing or testing purpose)"
        required: false
        type: boolean
        default: false
      skip_pr_test_health_check:
        description: "Skip PR test health check fail-fast (e.g. for release branch cuts)"
        required: false
        type: boolean
        default: false

concurrency:
  # - event_name prevents scheduled runs from colliding with fork PRs whose branch is named 'main'
  #   (without it, both resolve the branch segment to 'main' and block each other)
  # - a PR keys on its number: github.head_ref is a bare branch name with no owner,
  #   so two forks using the same name would share one group and cancel each other
  group: pr-test-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }}
  cancel-in-progress: ${{ github.event_name != 'workflow_call' }}

env:
  SGLANG_IS_IN_CI: true
  SGLANG_CUDA_COREDUMP: "1"
  SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true
  SKIP_PR_TEST_HEALTH_CHECK: ${{ (inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true) && 'true' || 'false' }}
  # Schedule / main-branch dispatch / workflow_call from main use refs/heads/main; PR events use refs/pull/*/merge
  PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }}
  USE_VENV: false

permissions:
  actions: write
  contents: read
  issues: read
  pull-requests: read

jobs:
  # =============================================== check changes ====================================================
  check-changes:
    uses: ./.github/workflows/_pr-test-check-changes.yml
    with:
      git_ref: ${{ inputs.git_ref || '' }}
      run_all_tests: ${{ inputs.run_all_tests == true }}
      force_continue_on_error: ${{ inputs.force_continue_on_error == true }}
    secrets: inherit

  call-pr-test-extra:
    if: github.event_name == 'schedule' || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true
    uses: ./.github/workflows/pr-test-extra.yml
    with:
      git_ref: ${{ inputs.git_ref || '' }}
      run_all_tests: true
    secrets: inherit

  # =============================================== Wait Jobs for Sequential PR Execution ====================================================

  wait-for-base-a:
    needs: [check-changes, call-gate]
    if: |
      always() &&
      !cancelled() &&
      needs.check-changes.result == 'success' &&
      github.event_name == 'pull_request' &&
      inputs.test_parallel_dispatch != true &&
      (needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') &&
      (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped')
    runs-on: ubuntu-latest
    outputs:
      base_a_result: ${{ steps.wait.outputs.result }}
    steps:
      - uses: actions/checkout@v4

      - uses: ./.github/actions/check-maintenance

      - uses: ./.github/actions/wait-for-jobs
        id: wait
        with:
          stage-name: base-a
          jobs: |
            [
              {"prefix": "base-a-test-1-gpu-small", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-a-test-1-gpu-small'].size }}},
              {"prefix": "base-a-test-cpu", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-a-test-cpu'].size }}}
            ]
          max-wait-minutes: '240'

  wait-for-base-b:
    needs: [check-changes, call-gate, wait-for-base-a]
    if: |
      always() &&
      !cancelled() &&
      needs.check-changes.result == 'success' &&
      github.event_name == 'pull_request' &&
      inputs.test_parallel_dispatch != true &&
      (needs.check-changes.outputs.main_package == 'true' || needs.check-changes.outputs.sgl_kernel == 'true') &&
      (needs.wait-for-base-a.result == 'success' || needs.wait-for-base-a.result == 'skipped') &&
      (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped')
    runs-on: ubuntu-latest
    outputs:
      base_b_result: ${{ steps.wait.outputs.result }}
    steps:
      - uses: actions/checkout@v4

      - uses: ./.github/actions/check-maintenance

      - uses: ./.github/actions/wait-for-jobs
        id: wait
        with:
          stage-name: base-b
          jobs: |
            [
              {"prefix": "base-b-test-1-gpu-small", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-1-gpu-small'].size }}},
              {"prefix": "base-b-test-1-gpu-large", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-1-gpu-large'].size }}},
              {"prefix": "base-b-test-2-gpu-large", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-2-gpu-large'].size }}},
              {"prefix": "base-b-test-4-gpu-b200", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-4-gpu-b200'].size }}}
            ]
          max-wait-minutes: '480'

  # =============================================== PR Gate ====================================================
  call-gate:
    needs: check-changes
    # Skip for scheduled runs (they run all tests)
    if: |
      github.event_name != 'schedule' &&
      inputs.test_parallel_dispatch != true &&
      (
        needs.check-changes.outputs.main_package == 'true' ||
        needs.check-changes.outputs.sgl_kernel == 'true' ||
        needs.check-changes.outputs.jit_kernel == 'true' ||
        needs.check-changes.outputs.multimodal_gen == 'true'
      )
    uses: ./.github/workflows/pr-gate.yml
    secrets: inherit

  # =============================================== sgl-kernel ====================================================

  sgl-kernel-build-wheels:
    needs: [check-changes, call-gate]
    # Skip for scheduled runs (they run stages independently).
    if: |
      github.event_name != 'schedule' &&
      inputs.test_parallel_dispatch != true &&
      needs.check-changes.result == 'success' &&
      needs.check-changes.outputs.sgl_kernel == 'true' &&
      needs.call-gate.result == 'success'
    uses: ./.github/workflows/_pr-test-sgl-kernel-build.yml
    with:
      runs_on: x64-kernel-build-node
      job_display_name: Build Wheel
      git_ref: ${{ inputs.git_ref || '' }}
      skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true }}
    secrets: inherit

  sgl-kernel-build-wheels-arm:
    needs: [check-changes, call-gate]
    # Skip for scheduled runs (they run stages independently).
    if: |
      github.event_name != 'schedule' &&
      inputs.test_parallel_dispatch != true &&
      needs.check-changes.result == 'success' &&
      needs.check-changes.outputs.sgl_kernel == 'true' &&
      needs.call-gate.result == 'success'
    uses: ./.github/workflows/_pr-test-sgl-kernel-build.yml
    with:
      runs_on: arm-kernel-build-node
      job_display_name: Build Wheel Arm
      arch_suffix: '-aarch64'
      git_ref: ${{ inputs.git_ref || '' }}
      skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true }}
    secrets: inherit

  # =============================================== Rust extensions ====================================================

  # diffusion-only PRs do not consume SRT Rust extensions; other runs seed or
  # reuse the source-hash cache
  rust-ext-build:
    needs: [check-changes, call-gate]
    # Ungated on the event, unlike the jobs above: a cache entry is visible to
    # every PR only if it was written from the default branch's ref, so every
    # main-ref run - schedule, dispatch, workflow_call - has to seed it.
    #
    # !cancelled() so a skipped call-gate does not take this job with it, since a
    # docs-only PR still installs sglang in every stage.
    if: |
      !cancelled() &&
      needs.check-changes.result == 'success' &&
      (github.event_name != 'pull_request' ||
       needs.check-changes.outputs.multimodal_gen != 'true' ||
       needs.check-changes.outputs.main_package == 'true' ||
       needs.check-changes.outputs.jit_kernel == 'true' ||
       needs.check-changes.outputs.sgl_kernel == 'true') &&
      (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped')
    uses: ./.github/workflows/_pr-test-rust-ext-build.yml
    with:
      runs_on: ubuntu-22.04
      git_ref: ${{ inputs.git_ref || '' }}
      skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true }}
    secrets: inherit

  # gb300 is Grace, and PyO3 modules are portable across neither arch nor
  # interpreter, so it can use nothing the job above builds.
  rust-ext-build-aarch64:
    needs: [check-changes, call-gate]
    # Same gate as rust-ext-build: every main-ref run has to seed the cache, and a
    # docs-only PR still installs sglang on gb300.
    if: |
      !cancelled() &&
      needs.check-changes.result == 'success' &&
      (github.event_name != 'pull_request' ||
       needs.check-changes.outputs.multimodal_gen != 'true' ||
       needs.check-changes.outputs.main_package == 'true' ||
       needs.check-changes.outputs.jit_kernel == 'true' ||
       needs.check-changes.outputs.sgl_kernel == 'true') &&
      (needs.call-gate.result == 'success' || needs.call-gate.result == 'skipped')
    uses: ./.github/workflows/_pr-test-rust-ext-build.yml
    with:
      runs_on: ubuntu-22.04-arm
      collect_runs_on: ubuntu-22.04-arm
      artifact_name: rust-ext-aarch64
      cache_key_prefix: rust-ext-aarch64-cp310-cp312
      git_ref: ${{ inputs.git_ref || '' }}
      skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true }}
    secrets: inherit

  call-sgl-kernel-tests:
    needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
    if: |
      github.event_name != 'schedule' &&
      inputs.test_parallel_dispatch != true &&
      needs.check-changes.outputs.sgl_kernel == 'true'
    uses: ./.github/workflows/pr-test-sgl-kernel.yml
    with:
      runner_config: 4-gpu-b200
      runs_on_map: ${{ needs.check-changes.outputs.runs_on_map }}
      sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }}
      git_ref: ${{ inputs.git_ref || '' }}
      skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true }}
      rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
    secrets: inherit

  # =============================================== jit-kernel ====================================================

  call-jit-kernel-tests:
    needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
    # Run on scheduled/parallel-dispatch runs (same pattern as the base-* stages) so the
    # jit_kernel suite is exercised on main 3x daily, not only on PRs that touch kernels/**.
    # check-changes already forces jit_kernel='true' on scheduled runs (run_all_tests).
    if: |
      always() &&
      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
      needs.check-changes.outputs.jit_kernel == 'true'
    uses: ./.github/workflows/pr-test-jit-kernel.yml
    with:
      runs_on_map: ${{ needs.check-changes.outputs.runs_on_map }}
      jit_kernel: ${{ needs.check-changes.outputs.jit_kernel }}
      # On scheduled/parallel-dispatch runs sgl-kernel-build-wheels is skipped, so the wheel
      # artifact does not exist. Force sgl_kernel='false' there so the jit-kernel jobs skip the
      # wheel download and run against the released/pinned sgl-kernel instead of a fresh build.
      sgl_kernel: ${{ (github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) && 'false' || needs.check-changes.outputs.sgl_kernel }}
      git_ref: ${{ inputs.git_ref || '' }}
      test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }}
      skip_pr_test_health_check: ${{ inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true }}
      rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
    secrets: inherit

  # =============================================== primary ====================================================

  # `name:` is load-bearing: wait-for-jobs gates a stage by job-name prefix,
  # and check-pr-test-health carves h20 out by the same name.
  base-a-test:
    name: base-a-test-${{ matrix.runner_config }}
    needs: [check-changes, call-gate, sgl-kernel-build-wheels, rust-ext-build]
    if: ${{ !failure() && !cancelled() }}
    strategy:
      fail-fast: false
      matrix:
        include:
          # 5090 (32GB, SM120)
          - { runner_config: 1-gpu-small, run_timeout: '10', timeout_per_file: '' }
    uses: ./.github/workflows/_pr-test-stage.yml
    with:
      self_name: base-a-test-${{ matrix.runner_config }}
      runner_config: ${{ matrix.runner_config }}
      check_changes: ${{ toJson(needs.check-changes.outputs) }}
      caller_inputs: ${{ toJson(inputs) }}
      partitions: ${{ needs.check-changes.outputs.partitions }}
      run_timeout_minutes: ${{ matrix.run_timeout }}
      timeout_per_file: ${{ matrix.timeout_per_file }}
      rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
    secrets: inherit

  base-a-test-cpu:
    needs: [check-changes, call-gate, rust-ext-build]
    if: |
      always() &&
      needs.check-changes.result == 'success' &&
      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled()))
    uses: ./.github/workflows/_pr-test-stage-cpu.yml
    with:
      self_name: base-a-test-cpu
      check_changes: ${{ toJson(needs.check-changes.outputs) }}
      caller_inputs: ${{ toJson(inputs) }}
      partitions: ${{ needs.check-changes.outputs.partitions }}
      run_timeout_minutes: '15'
      rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
    # No `secrets: inherit`: this stage has no secret consumer, unlike the GPU
    # stages' coredump upload. GITHUB_TOKEN and permissions inherit regardless.

  base-b-test:
    name: base-b-test-${{ matrix.runner_config }}
    needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build]
    if: ${{ !failure() && !cancelled() }}
    strategy:
      fail-fast: false
      matrix:
        include:
          # 5090 (32GB, SM120)
          - { runner_config: 1-gpu-small, run_timeout: '30', timeout_per_file: '' }
          - { runner_config: 1-gpu-large, run_timeout: '60', timeout_per_file: '1800' }
          - { runner_config: 2-gpu-large, run_timeout: '30', timeout_per_file: '' }
          - { runner_config: 4-gpu-b200, run_timeout: '40', timeout_per_file: '' }
    uses: ./.github/workflows/_pr-test-stage.yml
    with:
      self_name: base-b-test-${{ matrix.runner_config }}
      runner_config: ${{ matrix.runner_config }}
      check_changes: ${{ toJson(needs.check-changes.outputs) }}
      caller_inputs: ${{ toJson(inputs) }}
      partitions: ${{ needs.check-changes.outputs.partitions }}
      run_timeout_minutes: ${{ matrix.run_timeout }}
      timeout_per_file: ${{ matrix.timeout_per_file }}
      rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
    secrets: inherit

  call-multimodal-gen-tests:
    needs: [check-changes, call-gate, sgl-kernel-build-wheels]
    if: |
      always() &&
      !cancelled() &&
      ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
      needs.check-changes.outputs.multimodal_gen == 'true'
    uses: ./.github/workflows/pr-test-multimodal-gen.yml
    with:
      runner_config: 4-gpu-b200
      runs_on_map: ${{ needs.check-changes.outputs.runs_on_map }}
      multimodal_gen: ${{ needs.check-changes.outputs.multimodal_gen }}
      sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }}
      continue_on_error: ${{ needs.check-changes.outputs.continue_on_error }}
      git_ref: ${{ inputs.git_ref || '' }}
      test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }}
      caller_needs_failure: ${{ (needs.call-gate.result == 'failure' || needs.sgl-kernel-build-wheels.result == 'failure' || needs.check-changes.result == 'failure') && 'true' || 'false' }}
      skip_pr_test_health_check: ${{ (inputs.skip_pr_test_health_check == true || inputs.test_parallel_dispatch == true || inputs.run_all_tests == true) && 'true' || 'false' }}
    secrets: inherit

  base-c-test:
    name: base-c-test-${{ matrix.runner_config }}
    needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build]
    if: ${{ !failure() && !cancelled() }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - {
              runner_config: 4-gpu-h100,
              run_timeout: '30',
              timeout_per_file: '',
              warmup_deep_gemm_models: 'lmsys/sglang-ci-dsv3-test:4',
              warmup_server_models: 'lmsys/sglang-ci-dsv3-test:4',
            }
          - {
              runner_config: 8-gpu-h200,
              run_timeout: '30',
              timeout_per_file: '1800',
              warmup_deep_gemm_models: 'deepseek-ai/DeepSeek-V3-0324:8 deepseek-ai/DeepSeek-V3.2:8 zai-org/GLM-5-FP8:8 XiaomiMiMo/MiMo-V2-Flash:4 XiaomiMiMo/MiMo-V2.5:8',
              warmup_timeout: '60',
            }
          - { runner_config: 8-gpu-h20, run_timeout: '30', timeout_per_file: '' }
          - { runner_config: 4-gpu-b200, run_timeout: '30', timeout_per_file: '1800' }
          - { runner_config: 8-gpu-b300, run_timeout: '60', timeout_per_file: '3600' }
    uses: ./.github/workflows/_pr-test-stage.yml
    with:
      self_name: base-c-test-${{ matrix.runner_config }}
      runner_config: ${{ matrix.runner_config }}
      check_changes: ${{ toJson(needs.check-changes.outputs) }}
      caller_inputs: ${{ toJson(inputs) }}
      partitions: ${{ needs.check-changes.outputs.partitions }}
      run_timeout_minutes: ${{ matrix.run_timeout }}
      timeout_per_file: ${{ matrix.timeout_per_file }}
      warmup_deep_gemm_models: ${{ matrix.warmup_deep_gemm_models }}
      warmup_server_models: ${{ matrix.warmup_server_models }}
      # Rows that do not set it keep _pr-test-stage.yml's own default.
      warmup_timeout_minutes: ${{ matrix.warmup_timeout || '25' }}
      rust_ext_artifact: ${{ needs.rust-ext-build.outputs.artifact_name }}
    secrets: inherit

  # Separate only because `needs` cannot vary per matrix row: this stage takes
  # the aarch64 build, and neither table should wait on the other's.
  base-c-test-aarch64:
    name: base-c-test-${{ matrix.runner_config }}
    needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build-aarch64]
    if: ${{ !failure() && !cancelled() }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - { runner_config: 4-gpu-gb300, run_timeout: '30', timeout_per_file: '1800' }
    uses: ./.github/workflows/_pr-test-stage.yml
    with:
      self_name: base-c-test-${{ matrix.runner_config }}
      runner_config: ${{ matrix.runner_config }}
      check_changes: ${{ toJson(needs.check-changes.outputs) }}
      caller_inputs: ${{ toJson(inputs) }}
      partitions: ${{ needs.check-changes.outputs.partitions }}
      run_timeout_minutes: ${{ matrix.run_timeout }}
      timeout_per_file: ${{ matrix.timeout_per_file }}
      rust_ext_artifact: ${{ needs.rust-ext-build-aarch64.outputs.artifact_name }}
    secrets: inherit

  # List every build and test job: `skipped` passes here, so an omission turns that
  # job's failure into a green run with no tests.
  pr-test-finish:
    needs:
      [
        call-gate,
        check-changes,

        sgl-kernel-build-wheels,
        sgl-kernel-build-wheels-arm,
        call-sgl-kernel-tests,

        rust-ext-build,
        rust-ext-build-aarch64,

        wait-for-base-a,
        wait-for-base-b,

        call-jit-kernel-tests,

        call-multimodal-gen-tests,

        base-a-test,
        base-a-test-cpu,
        base-b-test,
        base-c-test,
        base-c-test-aarch64,
      ]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check all dependent job statuses
        run: |
          # Convert the 'needs' context to a JSON string
          json_needs='${{ toJson(needs) }}'

          # Get a list of all job names from the JSON keys
          job_names=$(echo "$json_needs" | jq -r 'keys_unsorted[]')

          for job in $job_names; do
            # For each job, extract its result
            result=$(echo "$json_needs" | jq -r --arg j "$job" '.[$j].result')

            # Print the job name and its result
            echo "$job: $result"

            # Check for failure or cancellation and exit if found
            if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
              echo "The above jobs failed."
              exit 1
            fi
          done
          # If the loop completes, all jobs were successful
          echo "All jobs completed successfully"
          exit 0

  # =============================================== notify pr-states ====================================================
  # Dispatches pr-states.yml after every attempt (initial + every rerun).
  # workflow_run.completed only fires on the initial completion (one
  # workflow_run record across attempts), so we explicitly dispatch —
  # workflow_dispatch is the only event GITHUB_TOKEN can cascade-create.
  #
  # Fork PRs are excluded: their pull_request GITHUB_TOKEN is forced
  # read-only regardless of declared `actions: write`, so dispatch 403s.
  # Fork PRs get pr-states updates via the workflow_run subscription
  # there (initial completion) and pull_request_target (push / label).
  notify-pr-states:
    needs: [pr-test-finish]
    if: |
      always() &&
      github.event_name == 'pull_request' &&
      github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    steps:
      - name: Dispatch pr-states refresh
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
          gh workflow run pr-states.yml \
            --repo ${{ github.repository }} \
            --ref main \
            -f pr_number="$PR_NUMBER"