minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

ci: limit nightly test parallelism to 1 job per hardware type (#23314)

Browse Source
This commit is contained in:
Alison Shao
2026-04-30 21:49:18 -07:00
committed by GitHub
parent 9d95783603
commit 357e3789ea
1 changed files with 63 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-test-nvidia.yml
+63 -13
View File
@@ -57,6 +57,9 @@ jobs:
nightly-test-general-1-gpu-h100: nightly-test-general-1-gpu-h100:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-1-gpu-h100') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-1-gpu-h100')
runs-on: 1-gpu-h100 runs-on: 1-gpu-h100
concurrency:
group: nightly-hw-h100
cancel-in-progress: false
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
@@ -84,7 +87,10 @@ jobs:
nightly-test-kernel-1-gpu-h100: nightly-test-kernel-1-gpu-h100:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-kernel-1-gpu-h100') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-kernel-1-gpu-h100')
runs-on: 1-gpu-h100 runs-on: 1-gpu-h100
timeout-minutes: 240 concurrency:
group: nightly-hw-h100
cancel-in-progress: false
timeout-minutes: 60
env: env:
# Full jit_kernel test grids (see sglang.jit_kernel.utils.should_run_full_tests) # Full jit_kernel test grids (see sglang.jit_kernel.utils.should_run_full_tests)
SGLANG_JIT_KERNEL_RUN_FULL_TESTS: "1" SGLANG_JIT_KERNEL_RUN_FULL_TESTS: "1"
@@ -117,6 +123,9 @@ jobs:
nightly-test-kernel-8-gpu-h200: nightly-test-kernel-8-gpu-h200:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-kernel-8-gpu-h200') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-kernel-8-gpu-h200')
runs-on: 8-gpu-h200 runs-on: 8-gpu-h200
concurrency:
group: nightly-hw-h200
cancel-in-progress: false
timeout-minutes: 240 timeout-minutes: 240
env: env:
SGLANG_JIT_KERNEL_RUN_FULL_TESTS: "1" SGLANG_JIT_KERNEL_RUN_FULL_TESTS: "1"
@@ -148,6 +157,9 @@ jobs:
nightly-test-general-4-gpu-h100: nightly-test-general-4-gpu-h100:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-4-gpu-h100') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-4-gpu-h100')
runs-on: 4-gpu-h100 runs-on: 4-gpu-h100
concurrency:
group: nightly-hw-h100
cancel-in-progress: false
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
@@ -161,7 +173,7 @@ jobs:
bash scripts/ci/cuda/ci_install_dependency.sh bash scripts/ci/cuda/ci_install_dependency.sh
- name: Run test - name: Run test
timeout-minutes: 30 timeout-minutes: 60
run: | run: |
cd test cd test
python3 run_suite.py --hw cuda --suite nightly-4-gpu --nightly --continue-on-error python3 run_suite.py --hw cuda --suite nightly-4-gpu --nightly --continue-on-error
@@ -173,8 +185,12 @@ jobs:
nightly-test-general-8-gpu-h200: nightly-test-general-8-gpu-h200:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h200') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h200')
runs-on: 8-gpu-h200 runs-on: 8-gpu-h200
concurrency:
group: nightly-hw-h200
cancel-in-progress: false
strategy: strategy:
fail-fast: false fail-fast: false
max-parallel: 2
matrix: matrix:
partition: [0, 1, 2, 3] partition: [0, 1, 2, 3]
env: env:
@@ -259,6 +275,9 @@ jobs:
nightly-test-general-8-gpu-h20: nightly-test-general-8-gpu-h20:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-h20')
runs-on: 8-gpu-h20 runs-on: 8-gpu-h20
concurrency:
group: nightly-hw-h20
cancel-in-progress: false
env: env:
SGLANG_CI_RDMA_ALL_DEVICES: "mlx5_1,mlx5_2,mlx5_3,mlx5_4" SGLANG_CI_RDMA_ALL_DEVICES: "mlx5_1,mlx5_2,mlx5_3,mlx5_4"
steps: steps:
@@ -288,8 +307,12 @@ jobs:
nightly-test-general-8-gpu-b200: nightly-test-general-8-gpu-b200:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-b200') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-general-8-gpu-b200')
runs-on: 8-gpu-b200 runs-on: 8-gpu-b200
concurrency:
group: nightly-hw-b200
cancel-in-progress: false
strategy: strategy:
fail-fast: false fail-fast: false
max-parallel: 2
matrix: matrix:
partition: [0, 1, 2, 3] partition: [0, 1, 2, 3]
steps: steps:
@@ -306,7 +329,7 @@ jobs:
- name: Run common 8-GPU model tests - name: Run common 8-GPU model tests
if: always() if: always()
timeout-minutes: 300 timeout-minutes: 200
env: env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
@@ -363,6 +386,9 @@ jobs:
nightly-test-text-accuracy-2-gpu-h100: nightly-test-text-accuracy-2-gpu-h100:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-accuracy-2-gpu-h100') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-accuracy-2-gpu-h100')
runs-on: 2-gpu-h100 runs-on: 2-gpu-h100
concurrency:
group: nightly-hw-h100
cancel-in-progress: false
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
@@ -388,6 +414,9 @@ jobs:
nightly-test-text-perf-2-gpu-h100: nightly-test-text-perf-2-gpu-h100:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-perf-2-gpu-h100') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-text-perf-2-gpu-h100')
runs-on: 2-gpu-h100 runs-on: 2-gpu-h100
concurrency:
group: nightly-hw-h100
cancel-in-progress: false
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
@@ -401,7 +430,7 @@ jobs:
bash scripts/ci/cuda/ci_install_dependency.sh bash scripts/ci/cuda/ci_install_dependency.sh
- name: Run performance test for text models - name: Run performance test for text models
timeout-minutes: 180 timeout-minutes: 30
env: env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
@@ -426,6 +455,9 @@ jobs:
nightly-test-vlm-accuracy-2-gpu-h100: nightly-test-vlm-accuracy-2-gpu-h100:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-accuracy-2-gpu-h100') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-accuracy-2-gpu-h100')
runs-on: 2-gpu-h100 runs-on: 2-gpu-h100
concurrency:
group: nightly-hw-h100
cancel-in-progress: false
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
@@ -439,7 +471,7 @@ jobs:
bash scripts/ci/cuda/ci_install_dependency.sh bash scripts/ci/cuda/ci_install_dependency.sh
- name: Run eval test for VLM models (fixed MMMU-100) - name: Run eval test for VLM models (fixed MMMU-100)
timeout-minutes: 240 timeout-minutes: 120
run: | run: |
cd test cd test
python3 run_suite.py --hw cuda --suite nightly-eval-vlm-2-gpu --nightly --continue-on-error --timeout-per-file 9000 python3 run_suite.py --hw cuda --suite nightly-eval-vlm-2-gpu --nightly --continue-on-error --timeout-per-file 9000
@@ -451,6 +483,9 @@ jobs:
nightly-test-vlm-perf-2-gpu-h100: nightly-test-vlm-perf-2-gpu-h100:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-perf-2-gpu-h100') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-vlm-perf-2-gpu-h100')
runs-on: 2-gpu-h100 runs-on: 2-gpu-h100
concurrency:
group: nightly-hw-h100
cancel-in-progress: false
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
@@ -464,7 +499,7 @@ jobs:
bash scripts/ci/cuda/ci_install_dependency.sh bash scripts/ci/cuda/ci_install_dependency.sh
- name: Run perf test for VLM models (MMMU) - name: Run perf test for VLM models (MMMU)
timeout-minutes: 240 timeout-minutes: 30
env: env:
TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }} TRACE_BASE_URL: https://raw.githubusercontent.com/sglang-bot/sglang-ci-data/main/traces/${{ github.run_id }}
PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }} PERFETTO_RELAY_URL: ${{ vars.PERFETTO_RELAY_URL }}
@@ -489,9 +524,12 @@ jobs:
nightly-test-multimodal-server-1-gpu: nightly-test-multimodal-server-1-gpu:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-multimodal-server-1-gpu') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-multimodal-server-1-gpu')
runs-on: 1-gpu-h100 runs-on: 1-gpu-h100
concurrency:
group: nightly-hw-h100
cancel-in-progress: false
strategy: strategy:
fail-fast: false fail-fast: false
max-parallel: 5 max-parallel: 2
matrix: matrix:
part: [0, 1] part: [0, 1]
steps: steps:
@@ -513,7 +551,7 @@ jobs:
GITHUB_RUN_ID: ${{ github.run_id }} GITHUB_RUN_ID: ${{ github.run_id }}
GPU_CONFIG: "1-gpu-h100" GPU_CONFIG: "1-gpu-h100"
timeout-minutes: 90 timeout-minutes: 60
run: | run: |
cd python cd python
python3 sglang/multimodal_gen/test/run_suite.py \ python3 sglang/multimodal_gen/test/run_suite.py \
@@ -547,9 +585,12 @@ jobs:
nightly-test-multimodal-server-2-gpu: nightly-test-multimodal-server-2-gpu:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-multimodal-server-2-gpu') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-multimodal-server-2-gpu')
runs-on: 2-gpu-h100 runs-on: 2-gpu-h100
concurrency:
group: nightly-hw-h100
cancel-in-progress: false
strategy: strategy:
fail-fast: false fail-fast: false
max-parallel: 5 max-parallel: 2
matrix: matrix:
part: [0, 1] part: [0, 1]
steps: steps:
@@ -571,7 +612,7 @@ jobs:
GITHUB_RUN_ID: ${{ github.run_id }} GITHUB_RUN_ID: ${{ github.run_id }}
GPU_CONFIG: "2-gpu-h100" GPU_CONFIG: "2-gpu-h100"
timeout-minutes: 90 timeout-minutes: 210
run: | run: |
cd python cd python
python3 sglang/multimodal_gen/test/run_suite.py \ python3 sglang/multimodal_gen/test/run_suite.py \
@@ -606,6 +647,9 @@ jobs:
nightly-test-perf-4-gpu-b200: nightly-test-perf-4-gpu-b200:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-b200') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-4-gpu-b200')
runs-on: 4-gpu-b200 runs-on: 4-gpu-b200
concurrency:
group: nightly-hw-b200
cancel-in-progress: false
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4
@@ -619,7 +663,7 @@ jobs:
bash scripts/ci/cuda/ci_install_dependency.sh bash scripts/ci/cuda/ci_install_dependency.sh
- name: Run test - name: Run test
timeout-minutes: 300 timeout-minutes: 200
run: | run: |
cd test cd test
python3 run_suite.py --hw cuda --suite nightly-4-gpu-b200 --nightly --continue-on-error --timeout-per-file 12000 python3 run_suite.py --hw cuda --suite nightly-4-gpu-b200 --nightly --continue-on-error --timeout-per-file 12000
@@ -631,6 +675,9 @@ jobs:
nightly-test-specialized-8-gpu-b200: nightly-test-specialized-8-gpu-b200:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200' || inputs.job_filter == 'nightly-test-specialized-8-gpu-b200') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-perf-8-gpu-b200' || inputs.job_filter == 'nightly-test-specialized-8-gpu-b200')
runs-on: 8-gpu-b200 runs-on: 8-gpu-b200
concurrency:
group: nightly-hw-b200
cancel-in-progress: false
env: env:
RUNNER_LABELS: 8-gpu-b200 RUNNER_LABELS: 8-gpu-b200
steps: steps:
@@ -646,7 +693,7 @@ jobs:
bash scripts/ci/cuda/ci_install_dependency.sh bash scripts/ci/cuda/ci_install_dependency.sh
- name: Run test - name: Run test
timeout-minutes: 120 timeout-minutes: 60
env: env:
GPU_CONFIG: "8-gpu-b200" GPU_CONFIG: "8-gpu-b200"
run: | run: |
@@ -660,7 +707,10 @@ jobs:
nightly-test-diffusion-comparison: nightly-test-diffusion-comparison:
if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-diffusion-comparison') if: github.repository == 'sgl-project/sglang' && (inputs.job_filter == '' || inputs.job_filter == 'all' || inputs.job_filter == 'nightly-test-diffusion-comparison')
runs-on: 4-gpu-h100 runs-on: 4-gpu-h100
timeout-minutes: 240 concurrency:
group: nightly-hw-h100
cancel-in-progress: false
timeout-minutes: 300
steps: steps:
- name: Checkout code - name: Checkout code
uses: actions/checkout@v4 uses: actions/checkout@v4