[CI] Add GB200 nightly perf regression pipeline (#22461)
This commit is contained in:
Sahithi Chigurupati
@@ -0,0 +1,178 @@
|
||||
name: Nightly Perf Regression (GB200)
|
||||
|
||||
# NOTE: This workflow is intentionally cron-only.
|
||||
# It must NOT be triggered manually (no workflow_dispatch) to prevent
|
||||
# individuals from queuing arbitrary jobs on the shared GB200 cluster.
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 2 * * *' # 2 AM UTC daily (offset from other nightly runs)
|
||||
|
||||
concurrency:
|
||||
group: nightly-test-gb200
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
SRT_SLURM_BRANCH: sglang-nightly-regression
|
||||
SLURM_PARTITION: batch
|
||||
SLURM_ACCOUNT: sglang
|
||||
|
||||
jobs:
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reads scripts/ci/slurm/nightly-configs.yaml and generates one matrix entry
|
||||
# per recipe YAML. Each job runs the full concurrency sweep defined in the
|
||||
# recipe as a single Slurm job.
|
||||
# To add/remove configs, edit nightly-configs.yaml only.
|
||||
# ---------------------------------------------------------------------------
|
||||
setup:
|
||||
if: github.repository == 'sgl-project/sglang'
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
matrix: ${{ steps.generate.outputs.matrix }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Generate benchmark matrix
|
||||
id: generate
|
||||
run: |
|
||||
pip install pyyaml -q
|
||||
MATRIX=$(python3 scripts/ci/slurm/generate_matrix.py scripts/ci/slurm/nightly-configs.yaml --runner gb200)
|
||||
echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Import Docker images to Lustre squash files once before all benchmark jobs.
|
||||
# This avoids parallel jobs racing to enroot import the same image.
|
||||
# ---------------------------------------------------------------------------
|
||||
prepare-image:
|
||||
needs: setup
|
||||
if: github.repository == 'sgl-project/sglang'
|
||||
runs-on: gb200
|
||||
outputs:
|
||||
squash_file: ${{ steps.import.outputs.squash_file }}
|
||||
nginx_squash_file: ${{ steps.import.outputs.nginx_squash_file }}
|
||||
env:
|
||||
IMAGE: lmsysorg/sglang:dev-cu13
|
||||
NGINX_IMAGE: nginx:1.27.4
|
||||
steps:
|
||||
- name: Import Docker images to Lustre
|
||||
id: import
|
||||
run: |
|
||||
SQUASH_FILE="/mnt/lustre01/users-public/sglang-ci/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g')_$(date +%Y%m%d).sqsh"
|
||||
NGINX_SQUASH_FILE="/mnt/lustre01/users-public/sglang-ci/$(echo "$NGINX_IMAGE" | sed 's/[\/:@#]/_/g').sqsh"
|
||||
|
||||
if [ -f "$SQUASH_FILE" ]; then
|
||||
echo "Squash file already exists, skipping import: $SQUASH_FILE"
|
||||
else
|
||||
enroot import -o "$SQUASH_FILE" "docker://$IMAGE"
|
||||
fi
|
||||
|
||||
if [ -f "$NGINX_SQUASH_FILE" ]; then
|
||||
echo "Nginx squash file already exists, skipping import: $NGINX_SQUASH_FILE"
|
||||
else
|
||||
enroot import -o "$NGINX_SQUASH_FILE" "docker://$NGINX_IMAGE"
|
||||
fi
|
||||
|
||||
echo "squash_file=$SQUASH_FILE" >> $GITHUB_OUTPUT
|
||||
echo "nginx_squash_file=$NGINX_SQUASH_FILE" >> $GITHUB_OUTPUT
|
||||
|
||||
nightly-gb200-benchmark:
|
||||
needs: [setup, prepare-image]
|
||||
if: github.repository == 'sgl-project/sglang'
|
||||
runs-on: gb200
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
config: ${{ fromJson(needs.setup.outputs.matrix) }}
|
||||
env:
|
||||
FRAMEWORK: dynamo-sglang
|
||||
MODEL: ${{ matrix.config.model }}
|
||||
MODEL_PREFIX: ${{ matrix.config.model_prefix }}
|
||||
PRECISION: ${{ matrix.config.precision }}
|
||||
ISL: ${{ matrix.config.isl }}
|
||||
OSL: ${{ matrix.config.osl }}
|
||||
CONFIG_FILE: ${{ matrix.config.config_file }}
|
||||
RESULT_FILENAME: gb200-${{ matrix.config.name }}
|
||||
SQUASH_FILE: ${{ needs.prepare-image.outputs.squash_file }}
|
||||
NGINX_SQUASH_FILE: ${{ needs.prepare-image.outputs.nginx_squash_file }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Clean up prior Slurm jobs from this runner
|
||||
continue-on-error: true
|
||||
env:
|
||||
RUNNER_NAME: ${{ runner.name }}
|
||||
run: |
|
||||
STALE_JOBS=$(squeue --noheader --format="%i %j" | grep "${RUNNER_NAME}" | awk '{print $1}')
|
||||
if [ -n "$STALE_JOBS" ]; then
|
||||
echo "Cancelling stale jobs: $STALE_JOBS"
|
||||
scancel $STALE_JOBS
|
||||
fi
|
||||
|
||||
- name: Launch GB200 benchmark via srt-slurm
|
||||
timeout-minutes: 360
|
||||
env:
|
||||
RUNNER_NAME: ${{ runner.name }}
|
||||
run: bash scripts/ci/slurm/launch_gb200.sh
|
||||
|
||||
- name: Process results
|
||||
if: always()
|
||||
env:
|
||||
RUNNER_NAME: ${{ runner.name }}
|
||||
run: |
|
||||
pip install tabulate pyyaml -q
|
||||
SRT_REPO_DIR="/mnt/lustre01/users-public/sglang-ci/workspace/${RUNNER_NAME}/srt-slurm"
|
||||
for result_file in ${{ github.workspace }}/${RESULT_FILENAME}_*.json; do
|
||||
[ -f "$result_file" ] || continue
|
||||
basename_file=$(basename "$result_file")
|
||||
ctx=$(echo "$basename_file" | sed -n 's/.*_ctx_\([0-9]*\)_gen.*/\1/p')
|
||||
gen=$(echo "$basename_file" | sed -n 's/.*_gen_\([0-9]*\)\.json/\1/p')
|
||||
[ -n "$ctx" ] && [ -n "$gen" ] || continue
|
||||
RESULT_FILENAME="${result_file%.json}" PREFILL_GPUS="$ctx" DECODE_GPUS="$gen" \
|
||||
RECIPE_FILE="$SRT_REPO_DIR/$CONFIG_FILE" \
|
||||
python3 scripts/ci/slurm/process_result.py
|
||||
done
|
||||
|
||||
- name: Upload results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: gb200-${{ matrix.config.name }}-${{ github.run_id }}
|
||||
path: |
|
||||
${{ github.workspace }}/*.json
|
||||
${{ github.workspace }}/multinode_server_logs.tar.gz
|
||||
retention-days: 30
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Clean up Slurm jobs on failure/cancel
|
||||
if: failure() || cancelled()
|
||||
continue-on-error: true
|
||||
env:
|
||||
RUNNER_NAME: ${{ runner.name }}
|
||||
run: |
|
||||
ACTIVE_JOBS=$(squeue --noheader --format="%i %j" | grep "${RUNNER_NAME}" | awk '{print $1}')
|
||||
if [ -n "$ACTIVE_JOBS" ]; then
|
||||
echo "Cancelling jobs: $ACTIVE_JOBS"
|
||||
scancel $ACTIVE_JOBS
|
||||
fi
|
||||
|
||||
collect-results:
|
||||
needs: nightly-gb200-benchmark
|
||||
if: github.repository == 'sgl-project/sglang' && always()
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: results/
|
||||
pattern: gb200-*
|
||||
|
||||
- name: Print summary
|
||||
run: |
|
||||
pip install tabulate -q
|
||||
python3 scripts/ci/slurm/summarize.py results/ >> $GITHUB_STEP_SUMMARY
|
||||
Reference in New Issue
Block a user