[AMD][DI][CI] 1/N: MI355X disaggregation nightly benchmark (#29084)

Co-authored-by: Michael <13900043+michaelzhang-ai@users.noreply.github.com>
Co-authored-by: bingxche <bingxche@amd.com>
Co-authored-by: yctseng0211 <yctseng@amd.com>

.github/workflows/nightly-amd-mi355x-disagg.yml

@@ -36,10 +36,34 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.generate.outputs.matrix }}
+      image: ${{ steps.image.outputs.image }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Resolve latest MI35x nightly image
+        id: image
+        # Track the newest published image so the nightly actually catches
+        # regressions instead of re-testing one pinned build. We resolve on the
+        # GitHub-hosted runner (clean Docker Hub access, no rate limits) and pass
+        # the tag to the self-hosted runner. Scope is deliberately tight: only
+        # the clean main MI35x ROCm 7.2 line
+        # `lmsysorg/sglang-rocm:v*-rocm720-mi35x-YYYYMMDD` (excludes -test-,
+        # miles-, mori-, vllm-, rocm7_14 variants). On any failure we emit an
+        # empty string and the launcher falls back to the recipe's pinned
+        # `runtime.image`.
+        run: |
+          set -o pipefail
+          RESOLVED=$(curl -fsSL "https://hub.docker.com/v2/repositories/lmsysorg/sglang-rocm/tags?page_size=100&ordering=last_updated" \
+            | python3 -c "import json,re,sys;p=re.compile(r'^v[0-9][0-9A-Za-z.]*-rocm720-mi35x-([0-9]{8})\$');c=sorted((m.group(1),t['name']) for t in json.load(sys.stdin).get('results',[]) for m in [p.match(t.get('name',''))] if m);print(c[-1][1] if c else '')" || true)
+          if [ -n "$RESOLVED" ]; then
+            echo "Resolved latest MI35x image: lmsysorg/sglang-rocm:$RESOLVED"
+            echo "image=lmsysorg/sglang-rocm:$RESOLVED" >> $GITHUB_OUTPUT
+          else
+            echo "Could not resolve a latest image; launcher will use recipe default."
+            echo "image=" >> $GITHUB_OUTPUT
+          fi
+
       - name: Generate benchmark matrix
         id: generate
         env:
@@ -83,8 +107,13 @@ jobs:
       CONFIG_FILE: ${{ matrix.config.config_file }}
       RESULT_FILENAME: mi355x-${{ matrix.config.name }}
       MATRIX_CONFIG_NAME: ${{ matrix.config.name }}
-      # Local snapshot on the cluster's shared NFS (preferred over downloading).
+      # NOTE: RUNNER_NAME is a built-in default env var on every runner, so the
-      MODEL_PATH: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Flash-FP8/snapshots/ae01d80c06cdfe30581edfd0e1c5449dc7ed7f17
+      # launch + cleanup steps read it directly. Do NOT set it here from
+      # ${{ runner.name }} -- the `runner` context is not available in job-level
+      # env and makes the whole workflow file invalid.
+      # Shared-NFS HuggingFace cache dir; the launcher resolves the live snapshot
+      # via refs/main. Per-model value comes from nightly-configs.yaml model_path.
+      MODEL_PATH: ${{ matrix.config.model_path }}
 
     steps:
       - name: Checkout code
@@ -93,16 +122,26 @@ jobs:
       - name: Clean up prior Slurm jobs from this runner
         continue-on-error: true
         run: |
-          STALE_JOBS=$(squeue --me --noheader --format="%i" || true)
+          # launch_mi355x.sh names the allocation
+          #   mi355x-ci-<RUNNER_NAME>-<GITHUB_RUN_ID>-<config>
+          # Here we clear THIS runner's leftovers from a crashed previous run, so
+          # we match the RUNNER_NAME prefix (older runs have a different run id).
+          # Never a blanket `squeue --me`, which would kill a concurrent leg.
+          # %200j: squeue truncates the job name (%j) by default -- widen it so
+          # the grep sees the full name.
+          if [ -z "${RUNNER_NAME:-}" ]; then echo "RUNNER_NAME unset; skipping"; exit 0; fi
+          STALE_JOBS=$(squeue --me --noheader --format="%i %200j" | grep -F "mi355x-ci-${RUNNER_NAME}-" | awk '{print $1}' || true)
           if [ -n "$STALE_JOBS" ]; then
-            echo "Cancelling stale jobs: $STALE_JOBS"
+            echo "Cancelling stale jobs for ${RUNNER_NAME}: $STALE_JOBS"
             scancel $STALE_JOBS
           fi
 
       - name: Launch MI355X 2N 1P1D benchmark
         timeout-minutes: 180
         env:
-          IMAGE_OVERRIDE: ${{ inputs.image }}
+          # Manual dispatch input wins; otherwise use the latest image resolved
+          # by the setup job; otherwise the launcher falls back to the recipe default.
+          IMAGE_OVERRIDE: ${{ inputs.image != '' && inputs.image || needs.setup.outputs.image }}
         run: bash scripts/ci/slurm/launch_mi355x.sh
 
       - name: Process results
@@ -133,9 +172,14 @@ jobs:
         if: failure() || cancelled()
         continue-on-error: true
         run: |
-          ACTIVE_JOBS=$(squeue --me --noheader --format="%i" || true)
+          # Cancel only THIS leg's allocation -- match the full job name
+          # mi355x-ci-<RUNNER_NAME>-<GITHUB_RUN_ID>-<config>. GITHUB_RUN_ID +
+          # config are unique per matrix leg, so a concurrent leg is never hit.
+          if [ -z "${RUNNER_NAME:-}" ]; then echo "RUNNER_NAME unset; skipping"; exit 0; fi
+          JOB_TAG="mi355x-ci-${RUNNER_NAME}-${GITHUB_RUN_ID}-${MATRIX_CONFIG_NAME}"
+          ACTIVE_JOBS=$(squeue --me --noheader --format="%i %200j" | grep -F "$JOB_TAG" | awk '{print $1}' || true)
           if [ -n "$ACTIVE_JOBS" ]; then
-            echo "Cancelling jobs: $ACTIVE_JOBS"
+            echo "Cancelling jobs for ${JOB_TAG}: $ACTIVE_JOBS"
             scancel $ACTIVE_JOBS
           fi
 

scripts/ci/slurm/generate_matrix.py

@@ -70,6 +70,7 @@ def main():
                         "exp_name": exp_name,
                         "model": exp["model"],
                         "model_prefix": exp["model-prefix"],
+                        "model_path": exp.get("model_path", ""),
                         "precision": exp["precision"],
                         "isl": str(isl),
                         "osl": str(osl),

scripts/ci/slurm/launch_mi355x.sh

@@ -0,0 +1,465 @@
+#!/usr/bin/env bash
+# Launch a 2-node 1P1D disaggregation benchmark on the AMD MI355X `amd-sglang`
+# Slurm cluster, then emit per-concurrency result JSONs that
+# scripts/ci/slurm/process_result.py aggregates.
+#
+# salloc's (prefill_workers + decode_workers) nodes -- one server per node --
+# and runs the Docker harness: prefill server(s) on the first nodes, decode
+# server(s) on the rest, a standalone load balancer on the prefill node, then an
+# sglang.bench_serving concurrency sweep over MORI. Default recipe is 1P1D (2
+# nodes); see the drive.sh note on reserving 2P2D / 1P3D / 3P1D.
+#
+# Required environment variables (set by the GitHub Actions workflow):
+#   MODEL              - HuggingFace model id (table label / served model)
+#   MODEL_PREFIX       - short prefix, e.g. dsv4flash
+#   PRECISION          - fp8 / fp4
+#   ISL, OSL           - input / output sequence lengths for the sweep
+#   CONFIG_FILE        - path to the recipe YAML (relative to repo root)
+#   RESULT_FILENAME    - prefix for the emitted result JSONs
+#   MATRIX_CONFIG_NAME - matrix entry name (used in filenames/tags)
+#   GITHUB_WORKSPACE   - set by GitHub Actions; where result JSONs are written
+# Optional:
+#   MODEL_PATH         - local snapshot dir (preferred over downloading MODEL)
+#   SLURM_PARTITION    - default: amd-sglang
+#   SLURM_NODELIST     - optional explicit node pin (else scheduler chooses)
+#   RUNNER_NAME        - GitHub runner name (a built-in default env var)
+#   GITHUB_RUN_ID      - GitHub Actions run id (a built-in default env var)
+#                        The allocation is named
+#                          mi355x-ci-<RUNNER_NAME>-<GITHUB_RUN_ID>-<config>
+#                        so workflow cleanup can scancel exactly this leg's job
+#                        (full name) or this runner's stale jobs (RUNNER_NAME
+#                        prefix) -- never a blanket `squeue --me`. The run id +
+#                        config make the name unique per matrix leg even if two
+#                        runners happen to share a name.
+#   SLURM_EXCLUSIVE    - request whole nodes (default 1); set 0 to disable
+#   TIME_LIMIT         - salloc time limit, default 02:30:00 (covers server
+#                        load + perf sweep + full GSM8K, under the 180m step cap)
+
+set -euo pipefail
+set -x
+
+: "${MODEL_PREFIX:?}"
+: "${PRECISION:?}"
+: "${ISL:?}"
+: "${OSL:?}"
+: "${CONFIG_FILE:?}"
+: "${RESULT_FILENAME:?}"
+: "${MATRIX_CONFIG_NAME:?}"
+: "${GITHUB_WORKSPACE:?}"
+
+SLURM_PARTITION="${SLURM_PARTITION:-amd-sglang}"
+TIME_LIMIT="${TIME_LIMIT:-02:30:00}"
+MODEL_PATH="${MODEL_PATH:-${MODEL:-}}"
+
+if [[ -z "$MODEL_PATH" ]]; then
+    echo "ERROR: set MODEL_PATH (local snapshot) or MODEL" >&2
+    exit 1
+fi
+
+# Resolve a HuggingFace cache dir (models--org--name) to its live snapshot dir.
+# Lets nightly-configs point at the shared cache without hardcoding a snapshot
+# hash; if MODEL_PATH is already a concrete snapshot (or plain dir), use as-is.
+if [[ -f "$MODEL_PATH/refs/main" && -d "$MODEL_PATH/snapshots" ]]; then
+    SNAP_HASH="$(cat "$MODEL_PATH/refs/main")"
+    RESOLVED="$MODEL_PATH/snapshots/$SNAP_HASH"
+    if [[ -d "$RESOLVED" ]]; then
+        echo "resolved snapshot: $MODEL_PATH -> $RESOLVED"
+        MODEL_PATH="$RESOLVED"
+    else
+        echo "ERROR: refs/main=$SNAP_HASH but $RESOLVED missing" >&2
+        exit 1
+    fi
+fi
+
+# ---------------------------------------------------------------------------
+# Parse the recipe (runtime + bench + topology) into shell vars.
+# ---------------------------------------------------------------------------
+# Ensure PyYAML is available to the host python used for parsing.
+python3 -c 'import yaml' 2>/dev/null || pip install pyyaml -q 2>/dev/null \
+    || pip install --user pyyaml -q 2>/dev/null || true
+
+# Emit KEY=value lines and eval them (robust single-level command substitution;
+# avoids a nested read<<EOF/$(<<PY) heredoc that misparses on some shells).
+RECIPE_VARS="$(python3 - "$CONFIG_FILE" <<'PY'
+import sys, yaml
+r = yaml.safe_load(open(sys.argv[1]))
+rt = r["runtime"]; b = r["backend"]["sglang_config"]; bn = r["bench"]
+res = r.get("resources", {})
+def emit(k, v): print(f"{k}={v}")
+emit("IMAGE", rt["image"])
+emit("ATTN", rt["attention_backend"])
+emit("IB", rt["ib_devices"])
+emit("PPORT", rt["prefill_port"])
+emit("DPORT", rt["decode_port"])
+emit("PBOOT", rt["prefill_bootstrap_port"])
+emit("DBOOT", rt["decode_bootstrap_port"])
+emit("LBPORT", rt["lb_port"])
+emit("MEMFRAC", rt["mem_fraction_static"])
+emit("PAGE", rt["page_size"])
+emit("MAXREQ", rt["max_running_requests"])
+emit("CHUNK", rt["chunked_prefill_size"])
+emit("SWA", rt["swa_full_tokens_ratio"])
+emit("PTP", b["prefill"]["tensor-parallel-size"])
+emit("DTP", b["decode"]["tensor-parallel-size"])
+# Worker counts double as node counts here: one server per node (TP == GPUs/node).
+# 1P1D today; bumping these reserves 2P2D / 1P3D / 3P1D. Multi-node-per-worker
+# (TP > GPUs/node, needs --dist-init-addr/--nnodes/--node-rank) is out of scope.
+emit("PW", res.get("prefill_workers", 1))
+emit("DW", res.get("decode_workers", 1))
+emit("CONCS", ",".join(str(c) for c in bn["concurrencies"]))
+emit("NPF", bn["num_prompts_factor"])
+emit("RRR", bn["random_range_ratio"])
+acc = bn.get("accuracy", {}) or {}
+emit("ACC_ENABLED", 1 if acc.get("enabled") else 0)
+emit("ACC_SHOTS", acc.get("num_shots", 8))
+emit("ACC_NQ", acc.get("num_questions", 1319))
+emit("ACC_THR", acc.get("threshold", 0.91))
+PY
+)"
+if [[ -z "$RECIPE_VARS" ]]; then
+    echo "ERROR: failed to parse recipe $CONFIG_FILE (empty output from python3/yaml)" >&2
+    exit 1
+fi
+eval "$RECIPE_VARS"
+# Optional image override from workflow_dispatch input.
+if [[ -n "${IMAGE_OVERRIDE:-}" ]]; then
+    IMAGE="$IMAGE_OVERRIDE"
+fi
+echo "recipe: image=$IMAGE attn=$ATTN ib=$IB ptp=$PTP dtp=$DTP concs=$CONCS isl=$ISL osl=$OSL"
+
+# ---------------------------------------------------------------------------
+# Shared NFS scratch (visible to login node + compute nodes). Raw bench output
+# lands here; the launcher normalizes it into GITHUB_WORKSPACE afterwards.
+# ---------------------------------------------------------------------------
+WORKDIR="$HOME/.mi355x_ci/${MATRIX_CONFIG_NAME}"
+rm -rf "$WORKDIR"; mkdir -p "$WORKDIR"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Accuracy-gate helpers (written when enabled). Pre-stage the GSM8K test set on
+# shared NFS from the login node (which has internet) so the in-container eval
+# doesn't depend on compute-node connectivity; fall back to in-container
+# download if the pre-fetch fails.
+if [[ "$ACC_ENABLED" == "1" ]]; then
+    GSM8K_URL="https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
+    curl -fsSL "$GSM8K_URL" -o "$WORKDIR/gsm8k_test.jsonl" 2>/dev/null \
+        && echo "gsm8k dataset staged at $WORKDIR/gsm8k_test.jsonl" \
+        || echo "WARN: gsm8k pre-stage failed; in-container download will be attempted"
+    cat > "$WORKDIR/check_acc.py" <<'PY'
+import sys
+acc, thr = float(sys.argv[1]), float(sys.argv[2])
+print(f"[gsm8k] accuracy={acc:.3f} threshold={thr}")
+sys.exit(0 if acc > thr else 1)
+PY
+fi
+
+# DSV4 load-bearing env (see test/registered/amd/test_deepseek_v4_flash_fp8.py).
+# SGLANG_DSV4_FP4_EXPERTS is precision-driven: true for fp4 weights, false for fp8.
+if [[ "$PRECISION" == "fp4" ]]; then
+    FP4_EXPERTS=true
+else
+    FP4_EXPERTS=false
+fi
+DSV4_ENV=(
+  -e SGLANG_DEFAULT_THINKING=1 -e SGLANG_DSV4_REASONING_EFFORT=max
+  -e SGLANG_OPT_DEEPGEMM_HC_PRENORM=false -e SGLANG_USE_AITER=1
+  -e SGLANG_USE_ROCM700A=1 -e SGLANG_OPT_USE_FUSED_COMPRESS=true
+  -e SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true
+  -e SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton
+  -e SGLANG_OPT_FP8_WO_A_GEMM=false -e SGLANG_OPT_USE_JIT_INDEXER_METADATA=false
+  -e SGLANG_OPT_USE_TOPK_V2=false -e SGLANG_OPT_USE_AITER_INDEXER=true
+  -e SGLANG_OPT_USE_TILELANG_INDEXER=false -e SGLANG_OPT_USE_TILELANG_MHC_PRE=false
+  -e SGLANG_OPT_USE_TILELANG_MHC_POST=false -e SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1
+  -e SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false -e SGLANG_ROCM_USE_MULTI_STREAM=false
+  -e AITER_BF16_FP8_MOE_BOUND=0 -e SGLANG_DSV4_FP4_EXPERTS=$FP4_EXPERTS
+)
+DSV4_ENV_STR="${DSV4_ENV[*]}"
+MORI_ENV="-e MORI_DISABLE_AUTO_XGMI=1 -e NCCL_IB_HCA=ionic -e NCCL_IB_GID_INDEX=1 -e NCCL_CROSS_NIC=1"
+
+COMMON_FLAGS="--trust-remote-code --tp $PTP --disable-radix-cache \
+--attention-backend $ATTN --max-running-requests $MAXREQ --page-size $PAGE \
+--mem-fraction-static $MEMFRAC --swa-full-tokens-ratio $SWA \
+--chunked-prefill-size $CHUNK --disable-shared-experts-fusion \
+--tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 \
+--disaggregation-transfer-backend mori --disaggregation-ib-device $IB"
+
+DOCKER_COMMON="--rm --network host --ipc host --shm-size 32g --privileged \
+--security-opt seccomp=unconfined \
+--device /dev/kfd --device /dev/dri --device /dev/infiniband \
+-v /it-share:/it-share:ro -v $HOME:/host_home"
+
+# ---------------------------------------------------------------------------
+# Write per-role scripts that srun dispatches to each compute node.
+# ---------------------------------------------------------------------------
+cat > "$WORKDIR/prefill.sh" <<EOF
+#!/bin/bash
+docker rm -f mi355x_prefill 2>/dev/null || true
+docker run $DOCKER_COMMON --name mi355x_prefill \
+  -e HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 $MORI_ENV $DSV4_ENV_STR \
+  $IMAGE python3 -m sglang.launch_server \
+  --model-path $MODEL_PATH --host 0.0.0.0 --port $PPORT \
+  $COMMON_FLAGS --disaggregation-mode prefill --disaggregation-bootstrap-port $PBOOT
+EOF
+
+cat > "$WORKDIR/decode.sh" <<EOF
+#!/bin/bash
+docker rm -f mi355x_decode 2>/dev/null || true
+docker run $DOCKER_COMMON --name mi355x_decode \
+  -e HIP_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 $MORI_ENV $DSV4_ENV_STR \
+  $IMAGE python3 -m sglang.launch_server \
+  --model-path $MODEL_PATH --host 0.0.0.0 --port $DPORT \
+  $COMMON_FLAGS --disaggregation-mode decode --disaggregation-bootstrap-port $DBOOT
+EOF
+
+# Smoke-test payload + validator (separate files to avoid quoting inside the
+# bench.sh `bash -lc '...'` block). One real request exercises the full
+# prefill->decode KV handoff before we commit to the whole sweep.
+cat > "$WORKDIR/smoke.json" <<'JSON'
+{"text": "The capital of France is", "sampling_params": {"max_new_tokens": 16, "temperature": 0.0}}
+JSON
+cat > "$WORKDIR/assert_nonempty.py" <<'PY'
+import sys, json
+d = json.load(sys.stdin)
+t = d.get("text", "") if isinstance(d, dict) else ""
+if not (t and t.strip()):
+    print("[smoke] empty/invalid output:", str(d)[:200])
+    sys.exit(1)
+print("[smoke] ok:", t[:80].replace("\n", " "))
+PY
+
+# Bench script runs on the prefill node; \$PIP/\$DIP injected at srun time.
+cat > "$WORKDIR/bench.sh" <<EOF
+#!/bin/bash
+set -e
+PIP=\$1; DIP=\$2
+docker rm -f mi355x_bench 2>/dev/null || true
+docker run $DOCKER_COMMON --name mi355x_bench \
+  -e PIP=\$PIP -e DIP=\$DIP \
+  $IMAGE bash -lc '
+    export PYTHONPATH=/sgl-workspace/sglang/python:\$PYTHONPATH
+    echo "[wait] prefill"; for i in \$(seq 1 600); do curl -sf http://\$PIP:$PPORT/health >/dev/null && break; sleep 5; done
+    echo "[wait] decode";  for i in \$(seq 1 600); do curl -sf http://\$DIP:$DPORT/health >/dev/null && break; sleep 5; done
+    python3 -m sglang_router.launch_router \
+      --pd-disaggregation \
+      --prefill http://\$PIP:$PPORT $PBOOT \
+      --decode http://\$DIP:$DPORT \
+      --host 0.0.0.0 --port $LBPORT \
+      --disable-circuit-breaker &
+    for i in \$(seq 1 30); do curl -sf http://127.0.0.1:$LBPORT/health >/dev/null && break; sleep 2; done
+    CIDIR=/host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}
+    echo "[smoke] PD end-to-end check via LB"
+    curl -sf -X POST http://127.0.0.1:$LBPORT/generate \
+      -H "content-type: application/json" -d @\$CIDIR/smoke.json > \$CIDIR/smoke_out.json \
+      || { echo "[smoke] request failed -- PD path not serving; aborting before sweep"; exit 1; }
+    python3 \$CIDIR/assert_nonempty.py < \$CIDIR/smoke_out.json \
+      || { echo "[smoke] empty/invalid generation; aborting before sweep"; exit 1; }
+    # Correctness gate runs BEFORE the perf sweep: if the model is wrong there
+    # is no point spending ~15min measuring how fast it is wrong, so a failure
+    # here exits immediately and the sweep never runs.
+    if [ "$ACC_ENABLED" = "1" ]; then
+      echo "=== GSM8K accuracy gate (num_questions=$ACC_NQ shots=$ACC_SHOTS) ==="
+      DP_ARG=""
+      [ -s \$CIDIR/gsm8k_test.jsonl ] && DP_ARG="--data-path \$CIDIR/gsm8k_test.jsonl"
+      python3 -m sglang.test.few_shot_gsm8k \
+        --num-shots $ACC_SHOTS --num-questions $ACC_NQ --parallel $MAXREQ \
+        --max-new-tokens 512 --host http://127.0.0.1 --port $LBPORT \
+        \$DP_ARG 2>&1 | tee \$CIDIR/gsm8k.log
+      ACC=\$(grep -oE "Accuracy: [0-9.]+" \$CIDIR/gsm8k.log | tail -1 | cut -d" " -f2)
+      [ -n "\$ACC" ] || { echo "[gsm8k] could not parse accuracy from harness output"; exit 1; }
+      python3 \$CIDIR/check_acc.py "\$ACC" "$ACC_THR" || { echo "[gsm8k] accuracy below threshold -- failing before sweep"; exit 1; }
+    fi
+    for C in ${CONCS//,/ }; do
+      echo "=== concurrency=\$C ==="
+      OUT=/host_home/.mi355x_ci/${MATRIX_CONFIG_NAME}/raw_conc\${C}.json
+      rm -f \$OUT
+      python3 -m sglang.bench_serving --backend sglang \
+        --host 127.0.0.1 --port $LBPORT --model $MODEL_PATH \
+        --dataset-name random --random-input-len $ISL --random-output-len $OSL \
+        --random-range-ratio $RRR --max-concurrency \$C \
+        --num-prompts \$((C*$NPF)) --warmup-requests \$C \
+        --output-file \$OUT || true
+    done
+  '
+EOF
+chmod +x "$WORKDIR"/*.sh
+
+# ---------------------------------------------------------------------------
+# Orchestration drive (runs inside the salloc allocation on the login node).
+# ---------------------------------------------------------------------------
+# drive.sh splits the allocation into the first PW nodes (prefill) and the next
+# DW nodes (decode), launches one server per node, then benches. For 1P1D
+# (PW=DW=1) this is exactly prefill-on-node-A / decode-on-node-B. Larger PW/DW
+# reserve 2P2D / 1P3D / 3P1D: all servers come up, but the load balancer and
+# bench still target the first prefill + first decode (multi-P/D fan-out is the
+# remaining LB piece), so a >1 topology logs an explicit NOTE rather than
+# silently producing partial-coverage numbers.
+cat > "$WORKDIR/drive.sh" <<'DRIVE'
+#!/bin/bash
+set -x
+WORKDIR="$1"; PW="${2:-1}"; DW="${3:-1}"
+mapfile -t NODES < <(scontrol show hostnames "$SLURM_JOB_NODELIST")
+PNODES=("${NODES[@]:0:PW}")
+DNODES=("${NODES[@]:PW:DW}")
+PNODE="${PNODES[0]}"; DNODE="${DNODES[0]}"
+PIP=$(getent ahostsv4 "$PNODE" | head -1 | awk '{print $1}')
+DIP=$(getent ahostsv4 "$DNODE" | head -1 | awk '{print $1}')
+echo "[drive] prefill nodes: ${PNODES[*]} ; decode nodes: ${DNODES[*]}"
+echo "[drive] bench targets prefill=$PNODE($PIP) decode=$DNODE($DIP)"
+if (( PW > 1 || DW > 1 )); then
+  echo "[drive] NOTE: router + bench use the first prefill and first decode only;"
+  echo "[drive]       multi-prefill/multi-decode fan-out is not wired yet (LB work)."
+fi
+# Each server's srun runs here on the login node and returns exactly when its
+# compute-node container exits. Wrap it so the return code lands in a marker
+# file on shared NFS. The monitor then watches for markers instead of polling
+# PIDs -- unambiguous (no zombie/kill -0 guesswork) and it records which role
+# died and with what code. (A hung-but-alive server is NOT caught here; that is
+# bounded by bench.sh's health-wait timeout.)
+rm -f "$WORKDIR"/server_exit_* "$WORKDIR/bench_exit"
+for n in "${PNODES[@]}"; do
+  ( srun --overlap -N1 --nodelist="$n" bash "$WORKDIR/prefill.sh" > "$WORKDIR/prefill_$n.log" 2>&1
+    echo "prefill@$n rc=$?" > "$WORKDIR/server_exit_prefill_$n" ) &
+done
+for n in "${DNODES[@]}"; do
+  ( srun --overlap -N1 --nodelist="$n" bash "$WORKDIR/decode.sh" > "$WORKDIR/decode_$n.log" 2>&1
+    echo "decode@$n rc=$?" > "$WORKDIR/server_exit_decode_$n" ) &
+done
+sleep 5
+# Bench in the background with its own marker, so the wait loop is purely file
+# based: finish when bench writes its marker, abort if any server marker shows up
+# first (a server died before the sweep completed).
+( srun --overlap -N1 --nodelist="$PNODE" bash "$WORKDIR/bench.sh" "$PIP" "$DIP" > "$WORKDIR/bench.log" 2>&1
+  echo $? > "$WORKDIR/bench_exit" ) &
+BENCH_BG=$!
+# Stream bench output live and poll the markers with xtrace OFF, so the console
+# shows clean benchmark/accuracy output instead of a compgen/sleep trace every
+# 10s. (Mirrors NVIDIA's launch_gb200.sh, which set +x around its log stream.)
+touch "$WORKDIR/bench.log"
+tail -n +1 -F "$WORKDIR/bench.log" 2>/dev/null &
+TAIL_PID=$!
+set +x
+RC=0
+while [[ ! -f "$WORKDIR/bench_exit" ]]; do
+  if compgen -G "$WORKDIR/server_exit_*" > /dev/null; then
+    echo "[drive] ERROR: a server exited early before bench finished:"
+    cat "$WORKDIR"/server_exit_* || true
+    kill "$BENCH_BG" 2>/dev/null || true
+    RC=1
+    break
+  fi
+  sleep 10
+done
+set -x
+kill "$TAIL_PID" 2>/dev/null || true
+[[ "$RC" -eq 0 ]] && RC=$(cat "$WORKDIR/bench_exit" 2>/dev/null || echo 1)
+echo "[drive] bench finished (rc=$RC), tearing down"
+for n in "${PNODES[@]}"; do srun --overlap -N1 --nodelist="$n" docker kill mi355x_prefill >/dev/null 2>&1 || true; done
+for n in "${DNODES[@]}"; do srun --overlap -N1 --nodelist="$n" docker kill mi355x_decode  >/dev/null 2>&1 || true; done
+exit "$RC"
+DRIVE
+chmod +x "$WORKDIR/drive.sh"
+
+NODELIST_ARG=()
+[[ -n "${SLURM_NODELIST:-}" ]] && NODELIST_ARG=(--nodelist="$SLURM_NODELIST")
+
+# Request whole nodes so a co-scheduled job can't share a node and skew the
+# benchmark numbers. Toggle off with SLURM_EXCLUSIVE=0 on partitions that
+# disallow --exclusive.
+EXCLUSIVE_ARG=()
+[[ "${SLURM_EXCLUSIVE:-1}" == "1" ]] && EXCLUSIVE_ARG=(--exclusive)
+
+# One node per prefill/decode worker (TP == GPUs/node). 1P1D -> 2 nodes.
+TOTAL_NODES=$((PW + DW))
+
+# Name the allocation <RUNNER_NAME>-<GITHUB_RUN_ID>-<config> so the workflow's
+# cleanup steps can scancel precisely instead of a blanket `squeue --me` that
+# would kill a concurrent matrix leg. RUNNER_NAME alone is not assumed unique;
+# GITHUB_RUN_ID + config make the name unique per matrix leg regardless.
+JOB_NAME="mi355x-ci-${RUNNER_NAME:-norunner}-${GITHUB_RUN_ID:-0}-${MATRIX_CONFIG_NAME}"
+
+set +e
+salloc -p "$SLURM_PARTITION" -N"$TOTAL_NODES" "${NODELIST_ARG[@]}" "${EXCLUSIVE_ARG[@]}" \
+    --job-name "$JOB_NAME" -t "$TIME_LIMIT" \
+    bash "$WORKDIR/drive.sh" "$WORKDIR" "$PW" "$DW"
+SALLOC_RC=$?
+set -e
+
+# bench output already streamed live from drive.sh (tail -F). drive.sh exits
+# non-zero when a server died or bench failed; on failure dump bench.log + the
+# server logs (the actual root cause). We still fall through to normalize
+# whatever raw results the completed concurrencies produced -- partial perf data
+# is worth uploading -- and propagate the failure via the exit code at the end.
+if [[ "$SALLOC_RC" -ne 0 ]]; then
+    echo "ERROR: allocation/bench failed (rc=$SALLOC_RC); bench + server logs:" >&2
+    echo "--- bench.log (tail) ---"; tail -40 "$WORKDIR/bench.log" 2>/dev/null || true
+    for f in "$WORKDIR"/prefill_*.log "$WORKDIR"/decode_*.log; do
+        [[ -f "$f" ]] && { echo "--- $f (tail) ---"; tail -30 "$f"; }
+    done
+fi
+
+# Surface the GSM8K accuracy in the job summary -- it scrolls past in the live
+# log, and the perf table (collect-results/summarize.py) doesn't include it.
+if [[ "$ACC_ENABLED" == "1" && -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+    ACC_LINE=$(grep -aoE "Accuracy: [0-9.]+" "$WORKDIR/bench.log" 2>/dev/null | tail -1 || true)
+    {
+        echo "### GSM8K accuracy gate — ${MATRIX_CONFIG_NAME}"
+        echo '```'
+        echo "${ACC_LINE:-Accuracy: <not found in bench.log>}   (threshold > ${ACC_THR})"
+        echo '```'
+    } >> "$GITHUB_STEP_SUMMARY"
+fi
+
+# ---------------------------------------------------------------------------
+# Normalize raw bench_serving output -> process_result.py schema.
+#
+# bench_serving and process_result.py disagree on field names, so we remap the
+# last JSON line of each raw file. If bench_serving ever renames an output
+# field, the KeyError raised here (rather than a silently wrong table) is the
+# signal to update this mapping. Field-by-field:
+#
+#   bench_serving key          ->  process_result.py key       (purpose)
+#   --------------------------     ------------------------     -------------------------
+#   max_concurrency            ->  max_concurrency             (sweep point; falls back to $C)
+#   total_throughput           ->  total_token_throughput      (in+out tok/s, tput_per_gpu)
+#   output_throughput          ->  output_throughput           (out tok/s, output_tput_per_gpu)
+#   median_ttft_ms             ->  median_ttft_ms              (TTFT; /1000 -> s)
+#   median_tpot_ms             ->  median_tpot_ms              (TPOT; -> interactivity)
+#   median_e2e_latency_ms      ->  median_e2el_ms              (E2E latency; /1000 -> s)
+#   (none; injected here)      ->  model_id                    (served model, from $MODEL_PATH)
+# ---------------------------------------------------------------------------
+TOTAL_GPUS=$((PTP + DTP))
+PROCESSED=0
+for C in ${CONCS//,/ }; do
+    RAW="$WORKDIR/raw_conc${C}.json"
+    [[ -f "$RAW" ]] || { echo "WARN: missing $RAW"; continue; }
+    DEST="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${MATRIX_CONFIG_NAME}_conc${C}_gpus_${TOTAL_GPUS}_ctx_${PTP}_gen_${DTP}.json"
+    MODEL_ID="$MODEL_PATH" python3 - "$RAW" "$DEST" "$C" <<'PY'
+import json, os, sys
+raw_path, dest, conc = sys.argv[1], sys.argv[2], int(sys.argv[3])
+line = [l for l in open(raw_path).read().splitlines() if l.strip()][-1]
+r = json.loads(line)
+norm = {
+    "max_concurrency": r.get("max_concurrency") or conc,
+    "model_id": os.environ["MODEL_ID"],
+    "total_token_throughput": r["total_throughput"],
+    "output_throughput": r["output_throughput"],
+    "median_ttft_ms": r["median_ttft_ms"],
+    "median_tpot_ms": r["median_tpot_ms"],
+    "median_e2el_ms": r["median_e2e_latency_ms"],
+}
+json.dump(norm, open(dest, "w"), indent=2)
+print("normalized ->", dest)
+PY
+    PROCESSED=$((PROCESSED + 1))
+done
+
+# Propagate a benchmark/allocation failure even though we emitted partial
+# results above (the workflow uploads them with `always()`).
+if [[ "$SALLOC_RC" -ne 0 ]]; then
+    echo "ERROR: benchmark failed (rc=$SALLOC_RC); emitted $PROCESSED partial result file(s)." >&2
+    exit "$SALLOC_RC"
+fi
+if [[ "$PROCESSED" -eq 0 ]]; then
+    echo "ERROR: no result files produced" >&2
+    exit 1
+fi
+echo "Done. $PROCESSED result file(s) in $GITHUB_WORKSPACE."

scripts/ci/slurm/nightly-configs.yaml

@@ -44,3 +44,76 @@ dsr1-fp4-gb200-dynamo-sglang:
         - conc-list: [512, 2048, 4096, 8192]
           # https://github.com/NVIDIA/srt-slurm/blob/sglang-nightly-regression/recipes/gb200-fp4/1k1k/mid-curve.yaml
           config_file: recipes/gb200-fp4/1k1k/mid-curve.yaml
+
+# AMD MI355X 2-node 1P1D disaggregation. Driven by
+# scripts/ci/slurm/launch_mi355x.sh, which reads each recipe's `runtime`,
+# `bench`, and `bench.accuracy` sections. Every nightly runs ALL four
+# DeepSeek-V4 model x precision combos below (full matrix; GitHub
+# strategy.matrix.config, fail-fast: false). Each runs a GSM8K accuracy
+# hard-gate before the perf sweep.
+#
+# model_path points at the shared NFS HuggingFace cache dir (models--org--name);
+# the launcher resolves the live snapshot via refs/main, so no hash is hardcoded.
+dsv4flash-fp8-mi355x-sglang:
+  model: sgl-project/DeepSeek-V4-Flash-FP8
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Flash-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d.yaml
+
+dsv4pro-fp8-mi355x-sglang:
+  model: sgl-project/DeepSeek-V4-Pro-FP8
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--sgl-project--DeepSeek-V4-Pro-FP8
+  runner: mi355x
+  precision: fp8
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d.yaml
+
+dsv4flash-fp4-mi355x-sglang:
+  model: deepseek-ai/DeepSeek-V4-Flash
+  model-prefix: dsv4flash
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Flash
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d.yaml
+
+dsv4pro-fp4-mi355x-sglang:
+  model: deepseek-ai/DeepSeek-V4-Pro
+  model-prefix: dsv4pro
+  model_path: /it-share/model_coverage/models--deepseek-ai--DeepSeek-V4-Pro
+  runner: mi355x
+  precision: fp4
+  framework: sglang
+  multinode: true
+  disagg: true
+  seq-len-configs:
+    - isl: 1024
+      osl: 1024
+      search-space:
+        - conc-list: [1, 8, 16, 32, 64, 128, 256]
+          config_file: scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d.yaml

scripts/ci/slurm/process_result.py

@@ -76,7 +76,7 @@ if recipe_file and Path(recipe_file).exists():
 total_gpus = prefill_gpus + decode_gpus
 
 data = {
-    "hw": "gb200",
+    "hw": os.environ.get("HW", "gb200"),
     "conc": int(raw["max_concurrency"]),
     "model": raw["model_id"],
     "infmax_model_prefix": model_prefix,

scripts/ci/slurm/recipes/mi355x-fp4/dsv4flash/1k1k/1p1d.yaml

@@ -0,0 +1,54 @@
+# MI355X DeepSeek-V4-Flash (FP4) 2-node 1P1D disaggregation recipe.
+#
+# FP4 enables SGLANG_DSV4_FP4_EXPERTS in launch_mi355x.sh (driven by PRECISION).
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep
+  # (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
+  # even when throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp4/dsv4pro/1k1k/1p1d.yaml

@@ -0,0 +1,56 @@
+# MI355X DeepSeek-V4-Pro (FP4) 2-node 1P1D disaggregation recipe.
+#
+# FP4 enables SGLANG_DSV4_FP4_EXPERTS in launch_mi355x.sh (driven by PRECISION).
+# Pro mirrors the Flash topology (TP8 1P1D) as a starting point; Pro weights are
+# larger, so mem_fraction_static / max_running_requests may need tuning.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep
+  # (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
+  # even when throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4flash/1k1k/1p1d.yaml

@@ -0,0 +1,53 @@
+# MI355X DeepSeek-V4-Flash-FP8 2-node 1P1D disaggregation recipe.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path after the perf sweep. Mirrors the
+  # single-node registered test test/registered/amd/test_deepseek_v4_flash_fp8.py
+  # (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
+  # even when throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/1p1d.yaml

@@ -0,0 +1,55 @@
+# MI355X DeepSeek-V4-Pro-FP8 2-node 1P1D disaggregation recipe.
+#
+# Pro mirrors the Flash topology (TP8 1P1D) as a starting point; Pro weights are
+# larger, so mem_fraction_static / max_running_requests may need tuning.
+#
+# Consumed by:
+#   * scripts/ci/slurm/process_result.py reads `resources` and
+#     `backend.sglang_config` (TP/EP/DP + worker counts) for the summary table.
+#   * scripts/ci/slurm/launch_mi355x.sh reads `runtime` and `bench`.
+
+resources:
+  prefill_workers: 1
+  decode_workers: 1
+
+backend:
+  sglang_config:
+    prefill:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+    decode:
+      tensor-parallel-size: 8
+      expert-parallel-size: 1
+      data-parallel-size: 1
+
+runtime:
+  image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
+  attention_backend: dsv4
+  # RoCE HCAs MORI uses for cross-node KV transfer.
+  ib_devices: rdma0,rdma1,rdma2,rdma3
+  prefill_port: 30025
+  decode_port: 30026
+  prefill_bootstrap_port: 8998
+  decode_bootstrap_port: 9001
+  lb_port: 8000
+  mem_fraction_static: 0.90
+  page_size: 256
+  max_running_requests: 256
+  chunked_prefill_size: 8192
+  swa_full_tokens_ratio: 0.1
+
+bench:
+  # bench_serving --max-concurrency sweep; one result JSON per concurrency.
+  concurrencies: [1, 8, 16, 32, 64, 128, 256]
+  num_prompts_factor: 4      # num-prompts = concurrency * factor
+  random_range_ratio: 1.0
+
+  # Correctness gate run through the PD path before the perf sweep
+  # (full GSM8K, 8-shot, accuracy > 0.91). A regression here fails the nightly
+  # even when throughput looks fine ("fast but wrong").
+  accuracy:
+    enabled: true
+    num_shots: 8
+    num_questions: 1319    # full GSM8K test set
+    threshold: 0.91

scripts/ci/slurm/summarize.py

@@ -113,7 +113,13 @@ def main():
         for r in results
     ]
 
+    hw_label = "/".join(sorted({r["hw"].upper() for r in results}))
+    if hw_label == "GB200":
+        # NVIDIA GB200 nightly: keep the original hardcoded title untouched.
         print("## GB200 Nightly Benchmark Results\n")
+    else:
+        # AMD (e.g. MI355X) nightly: derive the title from the result hardware.
+        print(f"## {hw_label} Nightly Benchmark Results\n")
     print(tabulate(rows, headers=HEADERS, tablefmt="github"))
     print()