From 85d9c77c57b9872b0dcef1ec6dec49ed6587541d Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Wed, 13 May 2026 21:16:57 -0700 Subject: [PATCH] ci: extract cuda stage actions + runner_config mapping (#25138) Co-authored-by: Alison Shao <54658187+alisonshao@users.noreply.github.com> --- .github/workflows/_pr-test-stage.yml | 198 +++++ .github/workflows/pr-test.yml | 1053 ++++++-------------------- scripts/ci/runner_configs.py | 27 + scripts/ci/runner_configs.yml | 26 + 4 files changed, 478 insertions(+), 826 deletions(-) create mode 100644 .github/workflows/_pr-test-stage.yml create mode 100644 scripts/ci/runner_configs.py create mode 100644 scripts/ci/runner_configs.yml diff --git a/.github/workflows/_pr-test-stage.yml b/.github/workflows/_pr-test-stage.yml new file mode 100644 index 000000000..d25a22906 --- /dev/null +++ b/.github/workflows/_pr-test-stage.yml @@ -0,0 +1,198 @@ +name: PR Test Stage +# Reusable workflow that owns one CUDA test stage. Caller (pr-test.yml) passes +# (self_name, runner_config, runs_on, partitions, ...) and the +# per-commit/target_stage gating + matrix fanout + setup/run/teardown all live +# here. Only stage-a-test-cpu still lives inline in pr-test.yml (bespoke +# uv pip / protoc / rust-cache install path). + +on: + workflow_call: + inputs: + self_name: + description: 'Caller job key, used for $GITHUB_JOB-style gating + partitions[suite] lookup.' + type: string + required: true + runner_config: + description: 'Looked up in scripts/ci/runner_configs.yml for install script / artifact version / install timeout.' + type: string + required: true + runs_on: + description: 'Physical GHA runner label, e.g. "1-gpu-5090" or the b200_runner output from check-changes for B200 stages.' + type: string + required: true + target_stage: + description: 'Forwarded from pr-test.yml inputs.target_stage (used by /rerun-stage to skip everything except the targeted stage).' + type: string + default: '' + test_parallel_dispatch: + description: 'Forwarded from pr-test.yml inputs.test_parallel_dispatch.' + type: string + default: 'false' + partitions: + description: 'JSON from check-changes (size, arr, max_parallel per suite).' + type: string + required: true + main_package: + description: 'check-changes.outputs.main_package — gates the default per-commit run.' + type: string + required: true + sgl_kernel: + description: 'check-changes.outputs.sgl_kernel — both gates the run and forwarded to the install/download steps.' + type: string + required: true + continue_on_error_flag: + description: 'Empty or `--continue-on-error`; forwarded to run_suite.py.' + type: string + default: '' + run_timeout_minutes: + description: 'Per-suite wall-clock cap (minutes), enforced via bash `timeout` around run_suite.py.' + type: string + default: '30' + timeout_per_file: + description: 'Optional run_suite.py --timeout-per-file value.' + type: string + default: '' + warmup_deep_gemm_models: + description: 'Space-separated `model:gpus` list. Empty = skip DeepGEMM warmup.' + type: string + default: '' + warmup_server_models: + description: 'Space-separated `model:gpus` list. Empty = skip server-CUDA-graph warmup.' + type: string + default: '' + warmup_timeout_minutes: + description: 'Wall-clock cap for each warmup step (DeepGEMM + server). Default 25min; bump for stages with large cold-cache warmup model lists.' + type: string + default: '25' + extra_pytest_path: + description: 'Optional pytest path to run after the suite (stage-b-test-4-gpu-b200 uses this for FA4 jit_kernel tests). Empty = skip.' + type: string + default: '' + pr_head_sha: + description: 'Forwarded from pr-test.yml inputs.pr_head_sha (for /rerun-stage on fork PRs).' + type: string + default: '' + git_ref: + description: 'Forwarded from pr-test.yml inputs.git_ref (for workflow_call from main).' + type: string + default: '' + skip_stage_health_check: + description: 'Forwarded from pr-test.yml inputs.skip_stage_health_check (release branch cut bypass).' + type: boolean + default: false + +# Mirror pr-test.yml top-level env. Reusable workflows do NOT inherit caller's +# workflow-level env across the workflow_call boundary, so anything pr-test.yml +# defines must be redeclared here for the called job to see the same context. +env: + SGLANG_IS_IN_CI: true + SGLANG_CUDA_COREDUMP: "1" + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: true + SKIP_STAGE_HEALTH_CHECK: ${{ inputs.skip_stage_health_check && 'true' || 'false' }} + FORCE_REBUILD_DEEPEP: '1' + PR_TEST_BYPASS_MAINTENANCE_ON_MAIN: ${{ github.ref == 'refs/heads/main' && 'true' || 'false' }} + USE_VENV: false + +jobs: + run: + # Mirror the inline gating that used to live in pr-test.yml on every CUDA + # stage job. target_stage takes precedence; otherwise default per-commit + # gating runs the stage on schedule / parallel-dispatch / non-failed PR + # with main_package or sgl_kernel changes. + if: | + always() && + ( + (inputs.target_stage == inputs.self_name) || + ( + !inputs.target_stage && + ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (!failure() && !cancelled())) && + (inputs.main_package == 'true' || inputs.sgl_kernel == 'true') + ) + ) + runs-on: ${{ inputs.runs_on }} + timeout-minutes: 240 + env: + # Only stage-c-test-8-gpu-h20 needs the RDMA device list. Empty for + # everyone else (env var unset, harmless). + SGLANG_CI_RDMA_ALL_DEVICES: ${{ inputs.runner_config == '8-gpu-h20' && 'mlx5_1,mlx5_2,mlx5_3,mlx5_4' || '' }} + strategy: + fail-fast: false + max-parallel: ${{ fromJson(inputs.partitions)[inputs.self_name].max_parallel }} + matrix: + partition: ${{ fromJson(inputs.partitions)[inputs.self_name].arr }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + + - name: Resolve runner_config + id: rc + run: python3 scripts/ci/runner_configs.py '${{ inputs.runner_config }}' >> "$GITHUB_OUTPUT" + + - uses: ./.github/actions/check-stage-health + + - uses: ./.github/actions/check-maintenance + + - name: Download artifacts (v4) + if: ${{ inputs.sgl_kernel == 'true' && steps.rc.outputs.artifact_version == 'v4' }} + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda* + + - name: Download artifacts (v6) + if: ${{ inputs.sgl_kernel == 'true' && steps.rc.outputs.artifact_version == 'v6' }} + uses: actions/download-artifact@v6 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda* + + - name: Install dependencies + timeout-minutes: ${{ fromJson(steps.rc.outputs.install_timeout) }} + run: | + CUSTOM_BUILD_SGL_KERNEL=${{ inputs.sgl_kernel }} bash ${{ steps.rc.outputs.install }} + + - name: Warmup DeepGEMM JIT Compilation + if: inputs.warmup_deep_gemm_models != '' + timeout-minutes: ${{ fromJson(inputs.warmup_timeout_minutes) }} + run: | + # Activate venv if available (GITHUB_ENV may have failed to propagate) + [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate" + [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh" + python3 scripts/ci/cuda/warmup_deep_gemm.py ${{ inputs.warmup_deep_gemm_models }} + + - name: Warmup Server CUDA Graphs + if: inputs.warmup_server_models != '' + timeout-minutes: ${{ fromJson(inputs.warmup_timeout_minutes) }} + run: | + [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate" + [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh" + python3 scripts/ci/cuda/warmup_server.py ${{ inputs.warmup_server_models }} + + - name: Run test + timeout-minutes: ${{ fromJson(inputs.run_timeout_minutes) }} + env: + CONTINUE_ON_ERROR_FLAG: ${{ inputs.continue_on_error_flag }} + run: | + cd test + python3 run_suite.py --hw cuda --suite ${{ inputs.self_name }} \ + --auto-partition-id ${{ matrix.partition }} \ + --auto-partition-size ${{ fromJson(inputs.partitions)[inputs.self_name].size }} \ + ${{ inputs.timeout_per_file && format('--timeout-per-file {0}', inputs.timeout_per_file) || '' }} \ + $CONTINUE_ON_ERROR_FLAG + + - name: Run extra pytest + if: inputs.extra_pytest_path != '' + timeout-minutes: 10 + run: python3 -m pytest -q ${{ inputs.extra_pytest_path }} + + - uses: ./.github/actions/upload-cuda-coredumps + if: failure() + with: + artifact-suffix: ${{ matrix.partition }} + + - name: Cleanup venv + if: always() + run: bash scripts/ci/cuda/ci_cleanup_venv.sh diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index b647d8af4..0abbe971e 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -277,60 +277,23 @@ jobs: # Runs on 5090 (32GB, SM120) stage-a-test-1-gpu-small: needs: [check-changes, call-gate, sgl-kernel-build-wheels] - if: | - always() && - ( - (inputs.target_stage == 'stage-a-test-1-gpu-small') || - ( - !inputs.target_stage && - ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: 1-gpu-5090 - timeout-minutes: 240 - strategy: - fail-fast: false - max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-a-test-1-gpu-small'].max_parallel }} - matrix: - partition: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-a-test-1-gpu-small'].arr }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-stage-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda* - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh - - - name: Run test - timeout-minutes: 10 - env: - CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - run: | - cd test/ - python3 run_suite.py --hw cuda --suite stage-a-test-1-gpu-small --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['stage-a-test-1-gpu-small'].size }} $CONTINUE_ON_ERROR_FLAG - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - - - name: Cleanup venv - if: always() - run: bash scripts/ci/cuda/ci_cleanup_venv.sh + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: stage-a-test-1-gpu-small + runner_config: 1-gpu-small + runs_on: 1-gpu-5090 + target_stage: ${{ inputs.target_stage || '' }} + test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} + partitions: ${{ needs.check-changes.outputs.partitions }} + main_package: ${{ needs.check-changes.outputs.main_package }} + sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} + continue_on_error_flag: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + run_timeout_minutes: '10' + pr_head_sha: ${{ inputs.pr_head_sha || '' }} + git_ref: ${{ inputs.git_ref || '' }} + skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} + secrets: inherit stage-a-test-cpu: needs: [check-changes, call-gate] @@ -406,244 +369,83 @@ jobs: # Runs on 5090 (32GB, SM120) stage-b-test-1-gpu-small: needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels] - if: | - always() && - ( - (inputs.target_stage == 'stage-b-test-1-gpu-small') || - ( - !inputs.target_stage && - ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: 1-gpu-5090 - timeout-minutes: 240 - strategy: - fail-fast: false - max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-1-gpu-small'].max_parallel }} - matrix: - partition: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-1-gpu-small'].arr }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-stage-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda* - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - env: - CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - run: | - cd test/ - python3 run_suite.py --hw cuda --suite stage-b-test-1-gpu-small --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-1-gpu-small'].size }} $CONTINUE_ON_ERROR_FLAG - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - with: - artifact-suffix: ${{ matrix.partition }} - - - name: Cleanup venv - if: always() - run: bash scripts/ci/cuda/ci_cleanup_venv.sh + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: stage-b-test-1-gpu-small + runner_config: 1-gpu-small + runs_on: 1-gpu-5090 + target_stage: ${{ inputs.target_stage || '' }} + test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} + partitions: ${{ needs.check-changes.outputs.partitions }} + main_package: ${{ needs.check-changes.outputs.main_package }} + sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} + continue_on_error_flag: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + pr_head_sha: ${{ inputs.pr_head_sha || '' }} + git_ref: ${{ inputs.git_ref || '' }} + skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} + secrets: inherit # Runs on H100 (80GB, SM90) - tests that don't pass on 5090 (FA3, FP8, high VRAM, etc.) stage-b-test-1-gpu-large: needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels] - if: | - always() && - ( - (inputs.target_stage == 'stage-b-test-1-gpu-large') || - ( - !inputs.target_stage && - ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: 1-gpu-h100 - timeout-minutes: 240 - strategy: - fail-fast: false - max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-1-gpu-large'].max_parallel }} - matrix: - partition: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-1-gpu-large'].arr }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-stage-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda* - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - env: - CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - run: | - cd test/ - python3 run_suite.py --hw cuda --suite stage-b-test-1-gpu-large --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-1-gpu-large'].size }} --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - with: - artifact-suffix: ${{ matrix.partition }} - - - name: Cleanup venv - if: always() - run: bash scripts/ci/cuda/ci_cleanup_venv.sh + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: stage-b-test-1-gpu-large + runner_config: 1-gpu-large + runs_on: 1-gpu-h100 + target_stage: ${{ inputs.target_stage || '' }} + test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} + partitions: ${{ needs.check-changes.outputs.partitions }} + main_package: ${{ needs.check-changes.outputs.main_package }} + sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} + continue_on_error_flag: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + timeout_per_file: '1800' + pr_head_sha: ${{ inputs.pr_head_sha || '' }} + git_ref: ${{ inputs.git_ref || '' }} + skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} + secrets: inherit stage-b-test-2-gpu-large: needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels] - if: | - always() && - ( - (inputs.target_stage == 'stage-b-test-2-gpu-large') || - ( - !inputs.target_stage && - ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: 2-gpu-h100 - timeout-minutes: 240 - strategy: - fail-fast: false - max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-2-gpu-large'].max_parallel }} - matrix: - partition: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-2-gpu-large'].arr }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-stage-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda* - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - env: - CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - run: | - cd test/ - python3 run_suite.py --hw cuda --suite stage-b-test-2-gpu-large --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-2-gpu-large'].size }} $CONTINUE_ON_ERROR_FLAG - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - with: - artifact-suffix: ${{ matrix.partition }} - - - name: Cleanup venv - if: always() - run: bash scripts/ci/cuda/ci_cleanup_venv.sh + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: stage-b-test-2-gpu-large + runner_config: 2-gpu-large + runs_on: 2-gpu-h100 + target_stage: ${{ inputs.target_stage || '' }} + test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} + partitions: ${{ needs.check-changes.outputs.partitions }} + main_package: ${{ needs.check-changes.outputs.main_package }} + sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} + continue_on_error_flag: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + pr_head_sha: ${{ inputs.pr_head_sha || '' }} + git_ref: ${{ inputs.git_ref || '' }} + skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} + secrets: inherit stage-b-test-4-gpu-b200: needs: [check-changes, call-gate, wait-for-stage-a, sgl-kernel-build-wheels] - if: | - always() && - ( - (inputs.target_stage == 'stage-b-test-4-gpu-b200') || - ( - !inputs.target_stage && - ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: ${{ needs.check-changes.outputs.b200_runner }} - timeout-minutes: 240 - strategy: - fail-fast: false - max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-4-gpu-b200'].max_parallel }} - matrix: - partition: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-4-gpu-b200'].arr }} - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-stage-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v6 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda* - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh - - - name: Run test - timeout-minutes: 40 - env: - CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - run: | - cd test - python3 run_suite.py --hw cuda --suite stage-b-test-4-gpu-b200 --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['stage-b-test-4-gpu-b200'].size }} $CONTINUE_ON_ERROR_FLAG - - - name: Run FA4 jit_kernel tests (SM100+) - timeout-minutes: 10 - run: | - python3 -m pytest -q python/sglang/jit_kernel/tests/test_flash_attention_4.py - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - - - name: Cleanup venv - if: always() - run: bash scripts/ci/cuda/ci_cleanup_venv.sh + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: stage-b-test-4-gpu-b200 + runner_config: 4-gpu-b200 + runs_on: ${{ needs.check-changes.outputs.b200_runner }} + target_stage: ${{ inputs.target_stage || '' }} + test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} + partitions: ${{ needs.check-changes.outputs.partitions }} + main_package: ${{ needs.check-changes.outputs.main_package }} + sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} + continue_on_error_flag: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + run_timeout_minutes: '40' + extra_pytest_path: 'python/sglang/jit_kernel/tests/test_flash_attention_4.py' + pr_head_sha: ${{ inputs.pr_head_sha || '' }} + git_ref: ${{ inputs.git_ref || '' }} + skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} + secrets: inherit call-multimodal-gen-tests: needs: [check-changes, call-gate, sgl-kernel-build-wheels] @@ -680,571 +482,170 @@ jobs: stage-c-test-4-gpu-h100: needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels] - if: | - always() && - ( - (inputs.target_stage == 'stage-c-test-4-gpu-h100') || - ( - !inputs.target_stage && - ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: 4-gpu-h100 - timeout-minutes: 240 - strategy: - fail-fast: false - max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-4-gpu-h100'].max_parallel }} - matrix: - partition: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-4-gpu-h100'].arr }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-stage-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda* - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - env: - CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - run: | - cd test - python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-h100 --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-4-gpu-h100'].size }} $CONTINUE_ON_ERROR_FLAG - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - with: - artifact-suffix: ${{ matrix.partition }} - - - name: Cleanup venv - if: always() - run: bash scripts/ci/cuda/ci_cleanup_venv.sh + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: stage-c-test-4-gpu-h100 + runner_config: 4-gpu-h100 + runs_on: 4-gpu-h100 + target_stage: ${{ inputs.target_stage || '' }} + test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} + partitions: ${{ needs.check-changes.outputs.partitions }} + main_package: ${{ needs.check-changes.outputs.main_package }} + sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} + continue_on_error_flag: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + pr_head_sha: ${{ inputs.pr_head_sha || '' }} + git_ref: ${{ inputs.git_ref || '' }} + skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} + secrets: inherit stage-c-test-8-gpu-h200: needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels] - if: | - always() && - ( - (inputs.target_stage == 'stage-c-test-8-gpu-h200') || - ( - !inputs.target_stage && - ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: 8-gpu-h200 - timeout-minutes: 240 - strategy: - fail-fast: false - max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-8-gpu-h200'].max_parallel }} - matrix: - partition: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-8-gpu-h200'].arr }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-stage-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda* - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh - - - name: Warmup DeepGEMM JIT Compilation - # Per-model TP must match the test's launch in test/registered/ — see - # FALLBACK_ARGS in scripts/ci/cuda/warmup_deep_gemm.py for extra dp/ep - # flags. Only models that actually invoke DeepGEMM kernels at runtime - # are listed. Cold-cache ~13 min; warm-cache ≤30 s via marker file. - timeout-minutes: 60 - run: | - [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate" - [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh" - python3 scripts/ci/cuda/warmup_deep_gemm.py \ - deepseek-ai/DeepSeek-V3-0324:8 \ - deepseek-ai/DeepSeek-V3.2:8 \ - zai-org/GLM-5-FP8:8 \ - XiaomiMiMo/MiMo-V2-Flash:4 \ - XiaomiMiMo/MiMo-V2.5:8 - - - name: Run test - timeout-minutes: 30 - env: - CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - run: | - cd test - python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h200 --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-8-gpu-h200'].size }} $CONTINUE_ON_ERROR_FLAG - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - with: - artifact-suffix: ${{ matrix.partition }} - - - name: Cleanup venv - if: always() - run: bash scripts/ci/cuda/ci_cleanup_venv.sh + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: stage-c-test-8-gpu-h200 + runner_config: 8-gpu-h200 + runs_on: 8-gpu-h200 + target_stage: ${{ inputs.target_stage || '' }} + test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} + partitions: ${{ needs.check-changes.outputs.partitions }} + main_package: ${{ needs.check-changes.outputs.main_package }} + sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} + continue_on_error_flag: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + # Per-model TP must match the test's launch in test/registered/ -- see + # FALLBACK_ARGS in scripts/ci/cuda/warmup_deep_gemm.py for extra dp/ep + # flags. Only models that actually invoke DeepGEMM kernels at runtime + # are listed. Cold-cache ~13 min; warm-cache <=30 s via marker file. + # Server CUDA Graph warmup is combined into this step (warmup_server_models unset). + warmup_deep_gemm_models: 'deepseek-ai/DeepSeek-V3-0324:8 deepseek-ai/DeepSeek-V3.2:8 zai-org/GLM-5-FP8:8 XiaomiMiMo/MiMo-V2-Flash:4 XiaomiMiMo/MiMo-V2.5:8' + warmup_timeout_minutes: '60' + pr_head_sha: ${{ inputs.pr_head_sha || '' }} + git_ref: ${{ inputs.git_ref || '' }} + skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} + secrets: inherit stage-c-test-8-gpu-h20: needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels] - if: | - always() && - ( - (inputs.target_stage == 'stage-c-test-8-gpu-h20') || - ( - !inputs.target_stage && - ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: 8-gpu-h20 - timeout-minutes: 240 - env: - SGLANG_CI_RDMA_ALL_DEVICES: "mlx5_1,mlx5_2,mlx5_3,mlx5_4" - strategy: - fail-fast: false - max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-8-gpu-h20'].max_parallel }} - matrix: - partition: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-8-gpu-h20'].arr }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-stage-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda* - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_deepep.sh - - - name: Run test - timeout-minutes: 30 - env: - CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - run: | - cd test - python3 run_suite.py --hw cuda --suite stage-c-test-8-gpu-h20 --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-8-gpu-h20'].size }} $CONTINUE_ON_ERROR_FLAG - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - - - name: Cleanup venv - if: always() - run: bash scripts/ci/cuda/ci_cleanup_venv.sh + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: stage-c-test-8-gpu-h20 + runner_config: 8-gpu-h20 + runs_on: 8-gpu-h20 + target_stage: ${{ inputs.target_stage || '' }} + test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} + partitions: ${{ needs.check-changes.outputs.partitions }} + main_package: ${{ needs.check-changes.outputs.main_package }} + sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} + continue_on_error_flag: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + pr_head_sha: ${{ inputs.pr_head_sha || '' }} + git_ref: ${{ inputs.git_ref || '' }} + skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} + secrets: inherit stage-c-test-deepep-4-gpu-h100: needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels] - if: | - always() && - ( - (inputs.target_stage == 'stage-c-test-deepep-4-gpu-h100') || - ( - !inputs.target_stage && - ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: 4-gpu-h100 - timeout-minutes: 240 - strategy: - fail-fast: false - max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-deepep-4-gpu-h100'].max_parallel }} - matrix: - partition: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-deepep-4-gpu-h100'].arr }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-stage-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda* - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_deepep.sh - - - name: Warmup DeepGEMM JIT Compilation - timeout-minutes: 25 - run: | - # Activate venv if available (GITHUB_ENV may have failed to propagate) - [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate" - [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh" - python3 scripts/ci/cuda/warmup_deep_gemm.py \ - lmsys/sglang-ci-dsv3-test:4 - - - name: Warmup Server CUDA Graphs - timeout-minutes: 25 - run: | - [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate" - [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh" - python3 scripts/ci/cuda/warmup_server.py \ - lmsys/sglang-ci-dsv3-test:4 - - - name: Run test - timeout-minutes: 30 - env: - CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - run: | - cd test - python3 run_suite.py --hw cuda --suite stage-c-test-deepep-4-gpu-h100 --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-deepep-4-gpu-h100'].size }} $CONTINUE_ON_ERROR_FLAG - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - - - name: Cleanup venv - if: always() - run: bash scripts/ci/cuda/ci_cleanup_venv.sh + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: stage-c-test-deepep-4-gpu-h100 + runner_config: deepep-4-gpu-h100 + runs_on: 4-gpu-h100 + target_stage: ${{ inputs.target_stage || '' }} + test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} + partitions: ${{ needs.check-changes.outputs.partitions }} + main_package: ${{ needs.check-changes.outputs.main_package }} + sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} + continue_on_error_flag: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + warmup_deep_gemm_models: 'lmsys/sglang-ci-dsv3-test:4' + warmup_server_models: 'lmsys/sglang-ci-dsv3-test:4' + pr_head_sha: ${{ inputs.pr_head_sha || '' }} + git_ref: ${{ inputs.git_ref || '' }} + skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} + secrets: inherit stage-c-test-deepep-8-gpu-h200: needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels] - if: | - always() && - ( - (inputs.target_stage == 'stage-c-test-deepep-8-gpu-h200') || - ( - !inputs.target_stage && - ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: 8-gpu-h200-deepep - timeout-minutes: 240 - strategy: - fail-fast: false - max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-deepep-8-gpu-h200'].max_parallel }} - matrix: - partition: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-deepep-8-gpu-h200'].arr }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-stage-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda* - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_deepep.sh - - - name: Warmup DeepGEMM JIT Compilation - timeout-minutes: 25 - run: | - # Activate venv if available (GITHUB_ENV may have failed to propagate) - [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate" - [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh" - python3 scripts/ci/cuda/warmup_deep_gemm.py \ - deepseek-ai/DeepSeek-V3-0324:8 \ - deepseek-ai/DeepSeek-V3.2:8 - - - name: Warmup Server CUDA Graphs - timeout-minutes: 25 - run: | - [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/bin/activate" ] && source "${SGLANG_CI_VENV_PATH}/bin/activate" - [ -f "${SGLANG_CI_VENV_PATH:-/dev/null}/env.sh" ] && source "${SGLANG_CI_VENV_PATH}/env.sh" - python3 scripts/ci/cuda/warmup_server.py \ - deepseek-ai/DeepSeek-V3-0324:8 - - - name: Run test - timeout-minutes: 45 - env: - CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - run: | - cd test - python3 run_suite.py --hw cuda --suite stage-c-test-deepep-8-gpu-h200 --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-deepep-8-gpu-h200'].size }} $CONTINUE_ON_ERROR_FLAG - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - - - name: Cleanup venv - if: always() - run: bash scripts/ci/cuda/ci_cleanup_venv.sh + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: stage-c-test-deepep-8-gpu-h200 + runner_config: deepep-8-gpu-h200 + runs_on: 8-gpu-h200-deepep + target_stage: ${{ inputs.target_stage || '' }} + test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} + partitions: ${{ needs.check-changes.outputs.partitions }} + main_package: ${{ needs.check-changes.outputs.main_package }} + sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} + continue_on_error_flag: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + run_timeout_minutes: '45' + warmup_deep_gemm_models: 'deepseek-ai/DeepSeek-V3-0324:8 deepseek-ai/DeepSeek-V3.2:8' + warmup_server_models: 'deepseek-ai/DeepSeek-V3-0324:8' + pr_head_sha: ${{ inputs.pr_head_sha || '' }} + git_ref: ${{ inputs.git_ref || '' }} + skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} + secrets: inherit stage-c-test-4-gpu-b200: needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels] - if: | - always() && - ( - (inputs.target_stage == 'stage-c-test-4-gpu-b200') || - ( - !inputs.target_stage && - ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: ${{ needs.check-changes.outputs.b200_runner }} - timeout-minutes: 240 - strategy: - fail-fast: false - max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-4-gpu-b200'].max_parallel }} - matrix: - partition: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-4-gpu-b200'].arr }} - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-stage-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v6 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda* - - - name: Install dependencies - timeout-minutes: 20 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh - - - name: Run test - timeout-minutes: 30 - env: - CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - run: | - cd test - python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-b200 --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-4-gpu-b200'].size }} --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - with: - artifact-suffix: ${{ matrix.partition }} - - - name: Cleanup venv - if: always() - run: bash scripts/ci/cuda/ci_cleanup_venv.sh + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: stage-c-test-4-gpu-b200 + runner_config: 4-gpu-b200 + runs_on: ${{ needs.check-changes.outputs.b200_runner }} + target_stage: ${{ inputs.target_stage || '' }} + test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} + partitions: ${{ needs.check-changes.outputs.partitions }} + main_package: ${{ needs.check-changes.outputs.main_package }} + sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} + continue_on_error_flag: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + timeout_per_file: '1800' + pr_head_sha: ${{ inputs.pr_head_sha || '' }} + git_ref: ${{ inputs.git_ref || '' }} + skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} + secrets: inherit stage-c-test-dsv4-4-gpu-b200: needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels] - if: | - always() && - ( - (inputs.target_stage == 'stage-c-test-dsv4-4-gpu-b200') || - ( - !inputs.target_stage && - ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: ${{ needs.check-changes.outputs.b200_runner }} - timeout-minutes: 240 - strategy: - fail-fast: false - max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-dsv4-4-gpu-b200'].max_parallel }} - matrix: - partition: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-dsv4-4-gpu-b200'].arr }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-stage-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v6 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda* - - - name: Install dependencies - timeout-minutes: 30 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dsv4_dep.sh - - - name: Run test - timeout-minutes: 30 - env: - CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - run: | - cd test - python3 run_suite.py --hw cuda --suite stage-c-test-dsv4-4-gpu-b200 --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-dsv4-4-gpu-b200'].size }} --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - - - name: Cleanup venv - if: always() - run: bash scripts/ci/cuda/ci_cleanup_venv.sh + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: stage-c-test-dsv4-4-gpu-b200 + runner_config: dsv4-4-gpu-b200 + runs_on: ${{ needs.check-changes.outputs.b200_runner }} + target_stage: ${{ inputs.target_stage || '' }} + test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} + partitions: ${{ needs.check-changes.outputs.partitions }} + main_package: ${{ needs.check-changes.outputs.main_package }} + sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} + continue_on_error_flag: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + timeout_per_file: '1800' + pr_head_sha: ${{ inputs.pr_head_sha || '' }} + git_ref: ${{ inputs.git_ref || '' }} + skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} + secrets: inherit stage-c-test-dsv4-8-gpu-h200: needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels] - if: | - always() && - ( - (inputs.target_stage == 'stage-c-test-dsv4-8-gpu-h200') || - ( - !inputs.target_stage && - ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - ) - ) - runs-on: 8-gpu-h200 - timeout-minutes: 240 - strategy: - fail-fast: false - max-parallel: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-dsv4-8-gpu-h200'].max_parallel }} - matrix: - partition: ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-dsv4-8-gpu-h200'].arr }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - - - uses: ./.github/actions/check-stage-health - - - uses: ./.github/actions/check-maintenance - - - name: Download artifacts - if: needs.check-changes.outputs.sgl_kernel == 'true' - uses: actions/download-artifact@v4 - with: - path: sgl-kernel/dist/ - merge-multiple: true - pattern: wheel-python3.10-cuda* - - - name: Install dependencies - timeout-minutes: 30 - run: | - CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dsv4_dep.sh - - - name: Run test - timeout-minutes: 30 - env: - CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - run: | - cd test - python3 run_suite.py --hw cuda --suite stage-c-test-dsv4-8-gpu-h200 --auto-partition-id ${{ matrix.partition }} --auto-partition-size ${{ fromJson(needs.check-changes.outputs.partitions)['stage-c-test-dsv4-8-gpu-h200'].size }} --timeout-per-file 1800 $CONTINUE_ON_ERROR_FLAG - - - uses: ./.github/actions/upload-cuda-coredumps - if: failure() - - - name: Cleanup venv - if: always() - run: bash scripts/ci/cuda/ci_cleanup_venv.sh - - # NOTE: GB200 stage temporarily disabled — no company-owned GB200 runner available yet. - # Re-enable when a 4-gpu-gb200 runner is provisioned. - # stage-c-test-4-gpu-gb200: - # needs: [check-changes, call-gate, wait-for-stage-b, sgl-kernel-build-wheels-arm] - # if: | - # always() && - # ( - # (inputs.target_stage == 'stage-c-test-4-gpu-gb200') || - # ( - # !inputs.target_stage && - # ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == true) || (!failure() && !cancelled())) && - # ((needs.check-changes.outputs.main_package == 'true') || (needs.check-changes.outputs.sgl_kernel == 'true')) - # ) - # ) - # runs-on: 4-gpu-gb200 - # timeout-minutes: 240 - # strategy: - # fail-fast: false - # steps: - # - uses: ./.github/actions/check-maintenance - # with: - # github-token: ${{ github.token }} - # - # - name: Checkout code - # uses: actions/checkout@v4 - # with: - # ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} - # - # - name: Download artifacts - # if: needs.check-changes.outputs.sgl_kernel == 'true' - # uses: actions/download-artifact@v4 - # with: - # path: sgl-kernel/dist/ - # merge-multiple: true - # pattern: wheel-python3.10-cuda13.0-aarch64 - # - # - name: Install dependencies - # timeout-minutes: 20 - # run: | - # CUSTOM_BUILD_SGL_KERNEL=${{needs.check-changes.outputs.sgl_kernel}} GRACE_BLACKWELL=1 bash scripts/ci/cuda/ci_install_deepep.sh - # - # - name: Run test - # timeout-minutes: 45 - # env: - # CONTINUE_ON_ERROR_FLAG: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} - # run: | - # cd test - # python3 run_suite.py --hw cuda --suite stage-c-test-4-gpu-gb200 --timeout-per-file 3600 $CONTINUE_ON_ERROR_FLAG - # - # - uses: ./.github/actions/upload-cuda-coredumps - # if: failure() + if: ${{ !failure() && !cancelled() }} + uses: ./.github/workflows/_pr-test-stage.yml + with: + self_name: stage-c-test-dsv4-8-gpu-h200 + runner_config: dsv4-8-gpu-h200 + runs_on: 8-gpu-h200 + target_stage: ${{ inputs.target_stage || '' }} + test_parallel_dispatch: ${{ inputs.test_parallel_dispatch == true && 'true' || 'false' }} + partitions: ${{ needs.check-changes.outputs.partitions }} + main_package: ${{ needs.check-changes.outputs.main_package }} + sgl_kernel: ${{ needs.check-changes.outputs.sgl_kernel }} + continue_on_error_flag: ${{ needs.check-changes.outputs.continue_on_error == 'true' && '--continue-on-error' || '' }} + timeout_per_file: '1800' + pr_head_sha: ${{ inputs.pr_head_sha || '' }} + git_ref: ${{ inputs.git_ref || '' }} + skip_stage_health_check: ${{ inputs.skip_stage_health_check == true }} + secrets: inherit pr-test-finish: needs: diff --git a/scripts/ci/runner_configs.py b/scripts/ci/runner_configs.py new file mode 100644 index 000000000..227dcf73b --- /dev/null +++ b/scripts/ci/runner_configs.py @@ -0,0 +1,27 @@ +"""Emit a runner_config's setup details (install / artifact_version / +install_timeout) in $GITHUB_OUTPUT format. Reads scripts/ci/runner_configs.yml. +Called by .github/workflows/_pr-test-stage.yml. +""" + +import os +import sys + +import yaml + +_YAML_PATH = os.path.join(os.path.dirname(__file__), "runner_configs.yml") + + +def load() -> dict: + with open(_YAML_PATH) as f: + return yaml.safe_load(f)["runner_configs"] + + +if __name__ == "__main__": + if len(sys.argv) != 2: + sys.exit("usage: runner_configs.py ") + rc = sys.argv[1] + config = load().get(rc) + if config is None: + sys.exit(f"unknown runner_config: {rc!r}") + for key, value in config.items(): + print(f"{key}={value}") diff --git a/scripts/ci/runner_configs.yml b/scripts/ci/runner_configs.yml new file mode 100644 index 000000000..fde798da3 --- /dev/null +++ b/scripts/ci/runner_configs.yml @@ -0,0 +1,26 @@ +# Per-runner-config CUDA setup details. Single source of truth for the +# `runner_config` field on `register_cuda_ci(...)` calls. Consumed by +# scripts/ci/runner_configs.py (CLI wrapper), which is in turn called by +# .github/workflows/_pr-test-stage.yml. +# +# Each runner_config carries install script, actions/download-artifact major +# version, and install-step wall-clock cap (minutes, enforced via +# `timeout-minutes:` on the install step in _pr-test-stage.yml). + +_anchors: + default_install: &default scripts/ci/cuda/ci_install_dependency.sh + deepep_install: &deepep scripts/ci/cuda/ci_install_deepep.sh + dsv4_install: &dsv4 scripts/ci/cuda/ci_install_dsv4_dep.sh + +runner_configs: + 1-gpu-small: { install: *default, artifact_version: v4, install_timeout: "20" } + 1-gpu-large: { install: *default, artifact_version: v4, install_timeout: "20" } + 2-gpu-large: { install: *default, artifact_version: v4, install_timeout: "20" } + 4-gpu-b200: { install: *default, artifact_version: v6, install_timeout: "20" } + 4-gpu-h100: { install: *default, artifact_version: v4, install_timeout: "20" } + 8-gpu-h200: { install: *default, artifact_version: v4, install_timeout: "20" } + 8-gpu-h20: { install: *deepep, artifact_version: v4, install_timeout: "20" } + deepep-4-gpu-h100: { install: *deepep, artifact_version: v4, install_timeout: "20" } + deepep-8-gpu-h200: { install: *deepep, artifact_version: v4, install_timeout: "20" } + dsv4-4-gpu-b200: { install: *dsv4, artifact_version: v6, install_timeout: "30" } + dsv4-8-gpu-h200: { install: *dsv4, artifact_version: v4, install_timeout: "30" }