diff --git a/.github/workflows/nightly-perf-regression-gb200.yml b/.github/workflows/nightly-perf-regression-gb200.yml new file mode 100644 index 000000000..f053b5e42 --- /dev/null +++ b/.github/workflows/nightly-perf-regression-gb200.yml @@ -0,0 +1,178 @@ +name: Nightly Perf Regression (GB200) + +# NOTE: This workflow is intentionally cron-only. +# It must NOT be triggered manually (no workflow_dispatch) to prevent +# individuals from queuing arbitrary jobs on the shared GB200 cluster. +on: + schedule: + - cron: '0 2 * * *' # 2 AM UTC daily (offset from other nightly runs) + +concurrency: + group: nightly-test-gb200 + cancel-in-progress: false + +env: + SGLANG_IS_IN_CI: true + SRT_SLURM_BRANCH: sglang-nightly-regression + SLURM_PARTITION: batch + SLURM_ACCOUNT: sglang + +jobs: + # --------------------------------------------------------------------------- + # Reads scripts/ci/slurm/nightly-configs.yaml and generates one matrix entry + # per recipe YAML. Each job runs the full concurrency sweep defined in the + # recipe as a single Slurm job. + # To add/remove configs, edit nightly-configs.yaml only. + # --------------------------------------------------------------------------- + setup: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.generate.outputs.matrix }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Generate benchmark matrix + id: generate + run: | + pip install pyyaml -q + MATRIX=$(python3 scripts/ci/slurm/generate_matrix.py scripts/ci/slurm/nightly-configs.yaml --runner gb200) + echo "matrix=$MATRIX" >> $GITHUB_OUTPUT + + # --------------------------------------------------------------------------- + # Import Docker images to Lustre squash files once before all benchmark jobs. + # This avoids parallel jobs racing to enroot import the same image. + # --------------------------------------------------------------------------- + prepare-image: + needs: setup + if: github.repository == 'sgl-project/sglang' + runs-on: gb200 + outputs: + squash_file: ${{ steps.import.outputs.squash_file }} + nginx_squash_file: ${{ steps.import.outputs.nginx_squash_file }} + env: + IMAGE: lmsysorg/sglang:dev-cu13 + NGINX_IMAGE: nginx:1.27.4 + steps: + - name: Import Docker images to Lustre + id: import + run: | + SQUASH_FILE="/mnt/lustre01/users-public/sglang-ci/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g')_$(date +%Y%m%d).sqsh" + NGINX_SQUASH_FILE="/mnt/lustre01/users-public/sglang-ci/$(echo "$NGINX_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" + + if [ -f "$SQUASH_FILE" ]; then + echo "Squash file already exists, skipping import: $SQUASH_FILE" + else + enroot import -o "$SQUASH_FILE" "docker://$IMAGE" + fi + + if [ -f "$NGINX_SQUASH_FILE" ]; then + echo "Nginx squash file already exists, skipping import: $NGINX_SQUASH_FILE" + else + enroot import -o "$NGINX_SQUASH_FILE" "docker://$NGINX_IMAGE" + fi + + echo "squash_file=$SQUASH_FILE" >> $GITHUB_OUTPUT + echo "nginx_squash_file=$NGINX_SQUASH_FILE" >> $GITHUB_OUTPUT + + nightly-gb200-benchmark: + needs: [setup, prepare-image] + if: github.repository == 'sgl-project/sglang' + runs-on: gb200 + strategy: + fail-fast: false + matrix: + config: ${{ fromJson(needs.setup.outputs.matrix) }} + env: + FRAMEWORK: dynamo-sglang + MODEL: ${{ matrix.config.model }} + MODEL_PREFIX: ${{ matrix.config.model_prefix }} + PRECISION: ${{ matrix.config.precision }} + ISL: ${{ matrix.config.isl }} + OSL: ${{ matrix.config.osl }} + CONFIG_FILE: ${{ matrix.config.config_file }} + RESULT_FILENAME: gb200-${{ matrix.config.name }} + SQUASH_FILE: ${{ needs.prepare-image.outputs.squash_file }} + NGINX_SQUASH_FILE: ${{ needs.prepare-image.outputs.nginx_squash_file }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Clean up prior Slurm jobs from this runner + continue-on-error: true + env: + RUNNER_NAME: ${{ runner.name }} + run: | + STALE_JOBS=$(squeue --noheader --format="%i %j" | grep "${RUNNER_NAME}" | awk '{print $1}') + if [ -n "$STALE_JOBS" ]; then + echo "Cancelling stale jobs: $STALE_JOBS" + scancel $STALE_JOBS + fi + + - name: Launch GB200 benchmark via srt-slurm + timeout-minutes: 360 + env: + RUNNER_NAME: ${{ runner.name }} + run: bash scripts/ci/slurm/launch_gb200.sh + + - name: Process results + if: always() + env: + RUNNER_NAME: ${{ runner.name }} + run: | + pip install tabulate pyyaml -q + SRT_REPO_DIR="/mnt/lustre01/users-public/sglang-ci/workspace/${RUNNER_NAME}/srt-slurm" + for result_file in ${{ github.workspace }}/${RESULT_FILENAME}_*.json; do + [ -f "$result_file" ] || continue + basename_file=$(basename "$result_file") + ctx=$(echo "$basename_file" | sed -n 's/.*_ctx_\([0-9]*\)_gen.*/\1/p') + gen=$(echo "$basename_file" | sed -n 's/.*_gen_\([0-9]*\)\.json/\1/p') + [ -n "$ctx" ] && [ -n "$gen" ] || continue + RESULT_FILENAME="${result_file%.json}" PREFILL_GPUS="$ctx" DECODE_GPUS="$gen" \ + RECIPE_FILE="$SRT_REPO_DIR/$CONFIG_FILE" \ + python3 scripts/ci/slurm/process_result.py + done + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: gb200-${{ matrix.config.name }}-${{ github.run_id }} + path: | + ${{ github.workspace }}/*.json + ${{ github.workspace }}/multinode_server_logs.tar.gz + retention-days: 30 + if-no-files-found: warn + + - name: Clean up Slurm jobs on failure/cancel + if: failure() || cancelled() + continue-on-error: true + env: + RUNNER_NAME: ${{ runner.name }} + run: | + ACTIVE_JOBS=$(squeue --noheader --format="%i %j" | grep "${RUNNER_NAME}" | awk '{print $1}') + if [ -n "$ACTIVE_JOBS" ]; then + echo "Cancelling jobs: $ACTIVE_JOBS" + scancel $ACTIVE_JOBS + fi + + collect-results: + needs: nightly-gb200-benchmark + if: github.repository == 'sgl-project/sglang' && always() + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: results/ + pattern: gb200-* + + - name: Print summary + run: | + pip install tabulate -q + python3 scripts/ci/slurm/summarize.py results/ >> $GITHUB_STEP_SUMMARY diff --git a/scripts/ci/slurm/generate_matrix.py b/scripts/ci/slurm/generate_matrix.py new file mode 100644 index 000000000..974147ff7 --- /dev/null +++ b/scripts/ci/slurm/generate_matrix.py @@ -0,0 +1,73 @@ +""" +Reads nightly-configs.yaml and generates one matrix entry per recipe YAML, +where each srt-slurm recipe runs its full concurrency sweep as a single Slurm job. + +conc-list in the config is documentation only and is not used to split jobs. + +Output: JSON array written to stdout, consumed by the workflow setup job as +a dynamic matrix via fromJson(needs.setup.outputs.matrix). + +Usage: + python3 generate_matrix.py --runner