[AMD] Register MI355X 2N 1P1D disagg nightly workflow for testing (#29091)
This commit is contained in:
Michael
@@ -0,0 +1,159 @@
|
||||
name: Nightly Test (AMD MI355X 2N 1P1D Disagg)
|
||||
|
||||
# A self-hosted runner on the amd-sglang login node salloc's two MI355X nodes
|
||||
# and runs the Docker harness (prefill + decode over MORI, a standalone load
|
||||
# balancer, then an sglang.bench_serving concurrency sweep) via
|
||||
# scripts/ci/slurm/launch_mi355x.sh.
|
||||
on:
|
||||
schedule:
|
||||
- cron: '17 3 * * *' # 03:17 UTC
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
image:
|
||||
description: 'Optional. ROCm SGLang Docker image to benchmark. Leave empty for the recipe default.'
|
||||
required: false
|
||||
default: ''
|
||||
configs:
|
||||
description: 'Optional. Comma-separated names to run only a subset. Leave empty to run all. Available names are listed in the setup job log.'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
group: nightly-test-mi355x
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
SGLANG_IS_IN_CI: true
|
||||
|
||||
jobs:
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reads scripts/ci/slurm/nightly-configs.yaml and generates one matrix entry
|
||||
# per recipe YAML for runner=mi355x. To add/remove configs, edit
|
||||
# nightly-configs.yaml only.
|
||||
# ---------------------------------------------------------------------------
|
||||
setup:
|
||||
if: github.repository == 'sgl-project/sglang'
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
matrix: ${{ steps.generate.outputs.matrix }}
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Generate benchmark matrix
|
||||
id: generate
|
||||
env:
|
||||
CONFIGS_FILTER: ${{ inputs.configs }}
|
||||
run: |
|
||||
pip install pyyaml -q
|
||||
|
||||
ALL_MATRIX=$(python3 scripts/ci/slurm/generate_matrix.py \
|
||||
scripts/ci/slurm/nightly-configs.yaml --runner mi355x)
|
||||
echo "Available config names for runner mi355x:"
|
||||
echo "$ALL_MATRIX" | python3 -c "import json,sys; [print(f' - {e[\"name\"]}') for e in json.load(sys.stdin)]"
|
||||
|
||||
FILTER_ARG=()
|
||||
if [ -n "$CONFIGS_FILTER" ]; then
|
||||
echo ""
|
||||
echo "Filtering to: $CONFIGS_FILTER"
|
||||
FILTER_ARG=(--filter "$CONFIGS_FILTER")
|
||||
fi
|
||||
MATRIX=$(python3 scripts/ci/slurm/generate_matrix.py \
|
||||
scripts/ci/slurm/nightly-configs.yaml --runner mi355x "${FILTER_ARG[@]}")
|
||||
echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
|
||||
|
||||
nightly-mi355x-benchmark:
|
||||
needs: setup
|
||||
if: |
|
||||
always() && github.repository == 'sgl-project/sglang'
|
||||
&& needs.setup.result == 'success'
|
||||
runs-on: linux-sglang-mi35x-di
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
config: ${{ fromJson(needs.setup.outputs.matrix) }}
|
||||
env:
|
||||
FRAMEWORK: sglang
|
||||
HW: mi355x
|
||||
MODEL: ${{ matrix.config.model }}
|
||||
MODEL_PREFIX: ${{ matrix.config.model_prefix }}
|
||||
PRECISION: ${{ matrix.config.precision }}
|
||||
ISL: ${{ matrix.config.isl }}
|
||||
OSL: ${{ matrix.config.osl }}
|
||||
CONFIG_FILE: ${{ matrix.config.config_file }}
|
||||
RESULT_FILENAME: mi355x-${{ matrix.config.name }}
|
||||
MATRIX_CONFIG_NAME: ${{ matrix.config.name }}
|
||||
# Local snapshot on the cluster's shared NFS (preferred over downloading).
|
||||
MODEL_PATH: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Flash-FP8/snapshots/ae01d80c06cdfe30581edfd0e1c5449dc7ed7f17
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Clean up prior Slurm jobs from this runner
|
||||
continue-on-error: true
|
||||
run: |
|
||||
STALE_JOBS=$(squeue --me --noheader --format="%i" || true)
|
||||
if [ -n "$STALE_JOBS" ]; then
|
||||
echo "Cancelling stale jobs: $STALE_JOBS"
|
||||
scancel $STALE_JOBS
|
||||
fi
|
||||
|
||||
- name: Launch MI355X 2N 1P1D benchmark
|
||||
timeout-minutes: 180
|
||||
env:
|
||||
IMAGE_OVERRIDE: ${{ inputs.image }}
|
||||
run: bash scripts/ci/slurm/launch_mi355x.sh
|
||||
|
||||
- name: Process results
|
||||
if: always()
|
||||
run: |
|
||||
pip install tabulate pyyaml -q
|
||||
for result_file in ${{ github.workspace }}/${RESULT_FILENAME}_*.json; do
|
||||
[ -f "$result_file" ] || continue
|
||||
basename_file=$(basename "$result_file")
|
||||
ctx=$(echo "$basename_file" | sed -n 's/.*_ctx_\([0-9]*\)_gen.*/\1/p')
|
||||
gen=$(echo "$basename_file" | sed -n 's/.*_gen_\([0-9]*\)\.json/\1/p')
|
||||
[ -n "$ctx" ] && [ -n "$gen" ] || continue
|
||||
RESULT_FILENAME="${result_file%.json}" PREFILL_GPUS="$ctx" DECODE_GPUS="$gen" \
|
||||
RECIPE_FILE="${{ github.workspace }}/$CONFIG_FILE" \
|
||||
python3 scripts/ci/slurm/process_result.py
|
||||
done
|
||||
|
||||
- name: Upload results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: mi355x-${{ matrix.config.name }}-${{ github.run_id }}
|
||||
path: ${{ github.workspace }}/*.json
|
||||
retention-days: 30
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Clean up Slurm jobs on failure/cancel
|
||||
if: failure() || cancelled()
|
||||
continue-on-error: true
|
||||
run: |
|
||||
ACTIVE_JOBS=$(squeue --me --noheader --format="%i" || true)
|
||||
if [ -n "$ACTIVE_JOBS" ]; then
|
||||
echo "Cancelling jobs: $ACTIVE_JOBS"
|
||||
scancel $ACTIVE_JOBS
|
||||
fi
|
||||
|
||||
collect-results:
|
||||
needs: nightly-mi355x-benchmark
|
||||
if: github.repository == 'sgl-project/sglang' && always()
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: results/
|
||||
pattern: mi355x-*
|
||||
|
||||
- name: Print summary
|
||||
run: |
|
||||
pip install tabulate -q
|
||||
python3 scripts/ci/slurm/summarize.py results/ >> $GITHUB_STEP_SUMMARY
|
||||
Reference in New Issue
Block a user