name: Nightly Test (AMD MI355X 2N 1P1D Disagg) # Self-hosted runners on the amd-sglang login node salloc two or four MI355X # nodes and run the Docker harness (prefill + decode over MORI, a standalone # load balancer, then an sglang.bench_serving concurrency sweep) via # scripts/ci/slurm/launch_mi355x.sh. on: # Temporarily paused while the DI CI nodes are unavailable. Uncomment the # schedule below when the nodes return. # schedule: # - cron: '0 7 * * *' workflow_dispatch: inputs: image: description: 'Optional. ROCm SGLang Docker image to benchmark. Leave empty for the recipe default.' required: false default: '' configs: description: 'Optional. Comma-separated names to run only a subset. Leave empty to run all. Available names are listed in the setup job log.' required: false default: '' concurrency: group: nightly-test-mi355x cancel-in-progress: false env: SGLANG_IS_IN_CI: true jobs: # --------------------------------------------------------------------------- # Reads scripts/ci/slurm/nightly-configs.yaml and generates one matrix entry # per recipe YAML for runner=mi355x. To add/remove configs, edit # nightly-configs.yaml only. # --------------------------------------------------------------------------- setup: if: github.repository == 'sgl-project/sglang' runs-on: ubuntu-latest outputs: matrix-2n: ${{ steps.generate.outputs.matrix-2n }} matrix-4n: ${{ steps.generate.outputs.matrix-4n }} image: ${{ steps.image.outputs.image }} steps: - name: Checkout code uses: actions/checkout@v4 - name: Resolve latest MI35x nightly image id: image # Track the newest published image so the nightly actually catches # regressions instead of re-testing one pinned build. We resolve on the # GitHub-hosted runner (clean Docker Hub access, no rate limits) and pass # the tag to the self-hosted runner. Scope is deliberately tight: only # the clean main MI35x ROCm 7.2 line # `lmsysorg/sglang-rocm:v*-rocm720-mi35x-YYYYMMDD` (excludes -test-, # miles-, mori-, vllm-, rocm7_14 variants). On any failure we emit an # empty string and the launcher falls back to the recipe's pinned # `runtime.image`. run: | set -o pipefail RESOLVED=$(curl -fsSL "https://hub.docker.com/v2/repositories/lmsysorg/sglang-rocm/tags?page_size=100&ordering=last_updated" \ | python3 -c "import json,re,sys;p=re.compile(r'^v[0-9][0-9A-Za-z.]*-rocm720-mi35x-([0-9]{8})\$');c=sorted((m.group(1),t['name']) for t in json.load(sys.stdin).get('results',[]) for m in [p.match(t.get('name',''))] if m);print(c[-1][1] if c else '')" || true) if [ -n "$RESOLVED" ]; then echo "Resolved latest MI35x image: lmsysorg/sglang-rocm:$RESOLVED" echo "image=lmsysorg/sglang-rocm:$RESOLVED" >> $GITHUB_OUTPUT else echo "Could not resolve a latest image; launcher will use recipe default." echo "image=" >> $GITHUB_OUTPUT fi - name: Generate benchmark matrix id: generate env: CONFIGS_FILTER: ${{ inputs.configs }} run: | pip install pyyaml -q ALL_MATRIX=$(python3 scripts/ci/slurm/generate_matrix.py \ scripts/ci/slurm/nightly-configs.yaml --runner mi355x) echo "Available config names for runner mi355x:" echo "$ALL_MATRIX" | python3 -c "import json,sys; [print(f' - {e[\"name\"]}') for e in json.load(sys.stdin)]" FILTER_ARG=() if [ -n "$CONFIGS_FILTER" ]; then echo "" echo "Filtering to: $CONFIGS_FILTER" FILTER_ARG=(--filter "$CONFIGS_FILTER") fi MATRIX=$(python3 scripts/ci/slurm/generate_matrix.py \ scripts/ci/slurm/nightly-configs.yaml --runner mi355x "${FILTER_ARG[@]}") MATRIX="$MATRIX" python3 - <<'PY' import json import os matrix = json.loads(os.environ["MATRIX"]) groups = { "matrix-2n": [entry for entry in matrix if "-1p1d" in entry["name"]], "matrix-4n": [ entry for entry in matrix if "-2p1d-ep16" in entry["name"] ], } matched = {entry["name"] for entries in groups.values() for entry in entries} unmatched = [entry["name"] for entry in matrix if entry["name"] not in matched] if unmatched: raise SystemExit( "Unrecognized MI355X topology for config(s): " + ", ".join(unmatched) ) with open(os.environ["GITHUB_OUTPUT"], "a") as output: for name, entries in groups.items(): value = {"include": [{"config": entry} for entry in entries]} print(f"{name}={json.dumps(value, separators=(',', ':'))}", file=output) PY nightly-mi355x-2n: needs: setup if: | always() && github.repository == 'sgl-project/sglang' && needs.setup.result == 'success' && needs.setup.outputs.matrix-2n != '{"include":[]}' runs-on: linux-sglang-mi35x-di strategy: fail-fast: false max-parallel: 2 matrix: ${{ fromJson(needs.setup.outputs.matrix-2n) }} env: &mi355x-env FRAMEWORK: sglang HW: mi355x MODEL: ${{ matrix.config.model }} MODEL_PREFIX: ${{ matrix.config.model_prefix }} PRECISION: ${{ matrix.config.precision }} ISL: ${{ matrix.config.isl }} OSL: ${{ matrix.config.osl }} CONFIG_FILE: ${{ matrix.config.config_file }} RESULT_FILENAME: mi355x-${{ matrix.config.name }} MATRIX_CONFIG_NAME: ${{ matrix.config.name }} SLURM_EXCLUDE: mia1-p01-g20,mia1-p02-g09 # NOTE: RUNNER_NAME is a built-in default env var on every runner, so the # launch + cleanup steps read it directly. Do NOT set it here from # ${{ runner.name }} -- the `runner` context is not available in job-level # env and makes the whole workflow file invalid. # Shared-NFS HuggingFace cache dir; the launcher resolves the live snapshot # via refs/main. Per-model value comes from nightly-configs.yaml model_path. MODEL_PATH: ${{ matrix.config.model_path }} steps: &mi355x-steps - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 persist-credentials: false - name: Clean up prior Slurm jobs from this runner continue-on-error: true run: | # launch_mi355x.sh names the allocation # mi355x-ci--- # Here we clear THIS runner's leftovers from a crashed previous run, so # we match the RUNNER_NAME prefix (older runs have a different run id). # Never a blanket match on all jobs, which would kill a concurrent leg. # %200j: squeue truncates the job name (%j) by default -- widen it so # the grep sees the full name. # No --me: Spur's squeue does not accept it. The RUNNER_NAME prefix is # what makes this selective, and that is unique per runner, so the # filter is equally precise without it. if [ -z "${RUNNER_NAME:-}" ]; then echo "RUNNER_NAME unset; skipping"; exit 0; fi STALE_JOBS=$(squeue --noheader --format="%i %200j" | grep -F "mi355x-ci-${RUNNER_NAME}-" | awk '{print $1}' || true) if [ -n "$STALE_JOBS" ]; then echo "Cancelling stale jobs for ${RUNNER_NAME}: $STALE_JOBS" scancel $STALE_JOBS fi - name: Launch MI355X benchmark timeout-minutes: 180 env: # Manual dispatch input wins; otherwise use the latest image resolved # by the setup job; otherwise the launcher falls back to the recipe default. IMAGE_OVERRIDE: ${{ inputs.image != '' && inputs.image || needs.setup.outputs.image }} run: bash scripts/ci/slurm/launch_mi355x.sh - name: Pack logs if: always() run: | LOG_DIR="$HOME/.mi355x_ci/${MATRIX_CONFIG_NAME}" LOG_BUNDLE="${GITHUB_WORKSPACE}/${RESULT_FILENAME}_logs.tar.gz" if [ ! -d "$LOG_DIR" ]; then echo "WARN: log directory not found: $LOG_DIR" exit 0 fi mapfile -d '' LOG_FILES < <( find "$LOG_DIR" -maxdepth 1 -type f \ \( -name '*.log' -o -name 'server_exit_*' -o -name 'bench_exit' \) \ -printf '%P\0' | sort -z ) if [ "${#LOG_FILES[@]}" -eq 0 ]; then echo "WARN: no log files found in $LOG_DIR" exit 0 fi STAGING_DIR="$(mktemp -d)" trap 'rm -rf "$STAGING_DIR"' EXIT for log_file in "${LOG_FILES[@]}"; do cp -a "$LOG_DIR/$log_file" "$STAGING_DIR/$log_file" || true done tar czf "$LOG_BUNDLE" -C "$STAGING_DIR" . echo "Packed ${#LOG_FILES[@]} log file(s) -> $LOG_BUNDLE" - name: Process results if: always() run: | pip install tabulate pyyaml -q for result_file in ${{ github.workspace }}/${RESULT_FILENAME}_*.json; do [ -f "$result_file" ] || continue basename_file=$(basename "$result_file") ctx=$(echo "$basename_file" | sed -n 's/.*_ctx_\([0-9]*\)_gen.*/\1/p') gen=$(echo "$basename_file" | sed -n 's/.*_gen_\([0-9]*\)\.json/\1/p') [ -n "$ctx" ] && [ -n "$gen" ] || continue RESULT_FILENAME="${result_file%.json}" PREFILL_GPUS="$ctx" DECODE_GPUS="$gen" \ RECIPE_FILE="${{ github.workspace }}/$CONFIG_FILE" \ python3 scripts/ci/slurm/process_result.py done - name: Upload results if: always() uses: actions/upload-artifact@v4 with: name: mi355x-${{ matrix.config.name }}-${{ github.run_id }} path: | ${{ github.workspace }}/*.json ${{ github.workspace }}/*_logs.tar.gz retention-days: 30 if-no-files-found: warn - name: Clean up Slurm jobs on failure/cancel if: failure() || cancelled() continue-on-error: true run: | # Cancel only THIS leg's allocation -- match the full job name # mi355x-ci---. GITHUB_RUN_ID + # config are unique per matrix leg, so a concurrent leg is never hit. if [ -z "${RUNNER_NAME:-}" ]; then echo "RUNNER_NAME unset; skipping"; exit 0; fi JOB_TAG="mi355x-ci-${RUNNER_NAME}-${GITHUB_RUN_ID}-${MATRIX_CONFIG_NAME}" ACTIVE_JOBS=$(squeue --noheader --format="%i %200j" | grep -F "$JOB_TAG" | awk '{print $1}' || true) if [ -n "$ACTIVE_JOBS" ]; then echo "Cancelling jobs for ${JOB_TAG}: $ACTIVE_JOBS" scancel $ACTIVE_JOBS fi - name: Clean up MI355X scratch if: always() continue-on-error: true run: | LOG_DIR="$HOME/.mi355x_ci/${MATRIX_CONFIG_NAME}" if [ -d "$LOG_DIR" ]; then echo "Removing MI355X scratch directory: $LOG_DIR" rm -rf "$LOG_DIR" fi nightly-mi355x-4n: needs: - setup - nightly-mi355x-2n if: | false && !cancelled() && github.repository == 'sgl-project/sglang' && needs.setup.result == 'success' && needs.setup.outputs.matrix-4n != '{"include":[]}' runs-on: linux-sglang-mi35x-di strategy: fail-fast: false max-parallel: 1 matrix: ${{ fromJson(needs.setup.outputs.matrix-4n) }} env: *mi355x-env steps: *mi355x-steps collect-results: needs: - nightly-mi355x-2n - nightly-mi355x-4n if: github.repository == 'sgl-project/sglang' && always() runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Download artifacts uses: actions/download-artifact@v4 with: path: results/ pattern: mi355x-* - name: Print summary run: | pip install tabulate -q python3 scripts/ci/slurm/summarize.py results/ >> $GITHUB_STEP_SUMMARY