minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[AMD][DI][CI] Add a SPUR cluster profile to AMD DI CI (#40113)

Browse Source
This commit is contained in:
Zhaoyi Li
2026-09-21 01:56:46 -07:00
committed by GitHub
parent 8faa2d6731
commit b86a30afba
2 changed files with 687 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/nightly-amd-mi355x-disagg.yml
+6 -3
View File
@@ -156,11 +156,14 @@ jobs:
# mi355x-ci-<RUNNER_NAME>-<GITHUB_RUN_ID>-<config>
# Here we clear THIS runner's leftovers from a crashed previous run, so
# we match the RUNNER_NAME prefix (older runs have a different run id).
# Never a blanket `squeue --me`, which would kill a concurrent leg.
# Never a blanket match on all jobs, which would kill a concurrent leg.
# %200j: squeue truncates the job name (%j) by default -- widen it so
# the grep sees the full name.
# No --me: Spur's squeue does not accept it. The RUNNER_NAME prefix is
# what makes this selective, and that is unique per runner, so the
# filter is equally precise without it.
if [ -z "${RUNNER_NAME:-}" ]; then echo "RUNNER_NAME unset; skipping"; exit 0; fi
STALE_JOBS=$(squeue --me --noheader --format="%i %200j" | grep -F "mi355x-ci-${RUNNER_NAME}-" | awk '{print $1}' || true)
STALE_JOBS=$(squeue --noheader --format="%i %200j" | grep -F "mi355x-ci-${RUNNER_NAME}-" | awk '{print $1}' || true)
if [ -n "$STALE_JOBS" ]; then
echo "Cancelling stale jobs for ${RUNNER_NAME}: $STALE_JOBS"
scancel $STALE_JOBS
@@ -237,7 +240,7 @@ jobs:
# config are unique per matrix leg, so a concurrent leg is never hit.
if [ -z "${RUNNER_NAME:-}" ]; then echo "RUNNER_NAME unset; skipping"; exit 0; fi
JOB_TAG="mi355x-ci-${RUNNER_NAME}-${GITHUB_RUN_ID}-${MATRIX_CONFIG_NAME}"
ACTIVE_JOBS=$(squeue --me --noheader --format="%i %200j" | grep -F "$JOB_TAG" | awk '{print $1}' || true)
ACTIVE_JOBS=$(squeue --noheader --format="%i %200j" | grep -F "$JOB_TAG" | awk '{print $1}' || true)
if [ -n "$ACTIVE_JOBS" ]; then
echo "Cancelling jobs for ${JOB_TAG}: $ACTIVE_JOBS"
scancel $ACTIVE_JOBS
scripts/ci/slurm/launch_mi355x.sh
+681 -31
View File
@@ -58,6 +58,51 @@ set -x
SLURM_PARTITION="${SLURM_PARTITION:-amd-sglang}"
TIME_LIMIT="${TIME_LIMIT:-02:30:00}"
MODEL_PATH="${MODEL_PATH:-${MODEL:-}}"
# Scheduler profile. `slurm` (default) is the amd-sglang MI355X cluster and is
# the original code path, unchanged. `spur` is the TW PIT cluster, which runs
# Spur -- a Slurm-compatible scheduler that differs in three ways that matter
# here: salloc takes no trailing command, `scontrol show hostnames` does not
# exist (its nodelist env var is already comma-expanded), and `srun --overlap`
# requires an explicit --jobid instead of inheriting one from the environment.
CLUSTER="${CLUSTER:-slurm}"
case "$CLUSTER" in
slurm|spur) ;;
*) echo "ERROR: CLUSTER must be 'slurm' or 'spur', got '$CLUSTER'" >&2; exit 1 ;;
esac
# Spur reads a two-field time limit as HH:MM where Slurm reads MM:SS, so "30:00"
# silently becomes 30 HOURS. Require the unambiguous HH:MM:SS / D-HH:MM:SS form
# there rather than let a typo book a node for a day.
if [[ "$CLUSTER" == "spur" && ! "$TIME_LIMIT" =~ ^([0-9]+-)?[0-9]+:[0-9]{2}:[0-9]{2}$ ]]; then
echo "ERROR: on spur TIME_LIMIT must be HH:MM:SS or D-HH:MM:SS (got '$TIME_LIMIT');" >&2
echo " a two-field value like '30:00' is parsed as 30 hours, not 30 minutes." >&2
exit 1
fi
# Optional account / QoS. Empty on the mi355x cluster (which gates on partition
# alone); required on spur, where every job must name an account and its QoS.
# Both flags are spelled the same in real Slurm, so passing them is portable.
SLURM_ACCOUNT="${SLURM_ACCOUNT:-}"
SLURM_QOS="${SLURM_QOS:-}"
# Relocate the shared HuggingFace cache root. nightly-configs.yaml pins
# model_path under /it-share/model_coverage (the mi355x cluster's NFS); a
# cluster that mirrors the same models elsewhere sets MODEL_ROOT and every
# recipe/config path is rewritten, so no recipe or config file needs editing.
MODEL_ROOT="${MODEL_ROOT:-}"
# Root used to RESOLVE the snapshot hash; defaults to MODEL_ROOT. Set it when
# MODEL_ROOT is node-local and therefore unreadable from the driver node.
MODEL_RESOLVE_ROOT="${MODEL_RESOLVE_ROOT:-$MODEL_ROOT}"
MODEL_ROOT_FROM="${MODEL_ROOT_FROM:-/it-share/model_coverage}"
relocate_model_root() {
local p="$1"
if [[ -n "$MODEL_ROOT" && "$p" == "$MODEL_ROOT_FROM"/* ]]; then
printf '%s\n' "$MODEL_ROOT/${p#"$MODEL_ROOT_FROM"/}"
else
printf '%s\n' "$p"
fi
}
SGLANG_USE_CHECKOUT_RUNTIME="${SGLANG_USE_CHECKOUT_RUNTIME:-1}"
case "${SGLANG_USE_CHECKOUT_RUNTIME,,}" in
0|false|no|off) SGLANG_USE_CHECKOUT_RUNTIME=0 ;;
@@ -89,7 +134,15 @@ resolve_snapshot() {
fi
echo "$p"
}
# Resolve the snapshot against the shared cache first, then relocate. MODEL_ROOT
# may name a node-local copy that does not exist on the node running this
# script; resolve_snapshot would then find no refs/main and pass the bare cache
# dir to the container. Three steps, since the resolve root and the destination
# root can differ: rewrite onto a readable root, resolve refs/main, then rewrite
# onto the root the container will see.
MODEL_PATH="$(MODEL_ROOT="$MODEL_RESOLVE_ROOT" relocate_model_root "$MODEL_PATH")"
MODEL_PATH="$(resolve_snapshot "$MODEL_PATH")" || exit 1
MODEL_PATH="$(MODEL_ROOT_FROM="$MODEL_RESOLVE_ROOT" relocate_model_root "$MODEL_PATH")"
# ---------------------------------------------------------------------------
# Parse the recipe (runtime + bench + topology) into shell vars.
@@ -233,7 +286,33 @@ echo "recipe: image=$IMAGE attn=${ATTN:-$PATTN/$DATTN} ib=$IB ptp=$PTP dtp=$DTP
# lands here; the launcher normalizes it into GITHUB_WORKSPACE afterwards.
# ---------------------------------------------------------------------------
WORKDIR="$HOME/.mi355x_ci/${MATRIX_CONFIG_NAME}"
if [[ "$CLUSTER" == "spur" ]]; then
# `rm -rf "$WORKDIR"` is not safe to gate a leg on over NFS: if anything still
# holds a descriptor inside (an orphaned `tail -F` on bench.log), the server
# renames the file to .nfsXXXX rather than unlinking it, so rm sees a
# non-empty directory and exits non-zero. Clear the contents instead, retry,
# and fall back to moving the directory aside. Housekeeping must not fail a
# leg.
for _try in 1 2 3; do
rm -rf "$WORKDIR"/* "$WORKDIR"/.[!.]* 2>/dev/null || true
# Anything left is a silent .nfsXXXX handle; give the server a moment.
[[ -z "$(ls -A "$WORKDIR" 2>/dev/null)" ]] && break
sleep 5
done
mkdir -p "$WORKDIR"
if [[ -n "$(ls -A "$WORKDIR" 2>/dev/null)" ]]; then
_stale="$WORKDIR.stale.$$"
if mv "$WORKDIR" "$_stale" 2>/dev/null; then
echo "WARN: $WORKDIR would not clear (open handles); moved to $_stale" >&2
else
WORKDIR="$WORKDIR.$$"
echo "WARN: could not clear or move workdir; using fresh $WORKDIR" >&2
fi
mkdir -p "$WORKDIR"
fi
else
rm -rf "$WORKDIR"; mkdir -p "$WORKDIR"
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# Stage the workflow checkout on shared NFS so Slurm compute-node containers can
@@ -246,8 +325,26 @@ if [[ "$SGLANG_USE_CHECKOUT_RUNTIME" == "1" ]]; then
echo "Staging checkout runtime: sha=$CHECKOUT_SHA -> $CHECKOUT_STAGE"
rm -rf "$CHECKOUT_STAGE"
mkdir -p "$CHECKOUT_STAGE"
# See install_checkout_sglang.sh: tar exit 1 is a warning ("file changed as
# we read it", raised spuriously by NFS attribute caching), 2+ is an error.
set +e
tar --exclude='__pycache__' --exclude='*.pyc' --exclude='.git/config' \
-C "$GITHUB_WORKSPACE" -cf - . | tar -C "$CHECKOUT_STAGE" -xf -
STAGE_TAR_RC=("${PIPESTATUS[@]}")
set -e
if (( STAGE_TAR_RC[0] > 1 || STAGE_TAR_RC[1] != 0 )); then
echo "ERROR: staging tar failed (create=${STAGE_TAR_RC[0]} extract=${STAGE_TAR_RC[1]})" >&2
exit 1
fi
# Tolerating create-rc=1 above means a warning cannot abort the copy -- but a
# TRUNCATED copy must still not reach a container, where it only surfaces
# ~15 minutes later as "invalid checkout mount". Assert the same file the
# in-container installer checks, right where the copy was made.
if [[ ! -f "$CHECKOUT_STAGE/python/sglang/version.py" ]]; then
echo "ERROR: staged checkout is incomplete (no python/sglang/version.py in $CHECKOUT_STAGE);" >&2
echo " tar rc were create=${STAGE_TAR_RC[0]} extract=${STAGE_TAR_RC[1]}" >&2
exit 1
fi
CHECKOUT_DOCKER_ARGS="$CHECKOUT_DOCKER_ARGS -e SGLANG_CHECKOUT_SHA=$CHECKOUT_SHA -v $CHECKOUT_STAGE:/sglang-checkout:ro"
else
echo "SGLANG_USE_CHECKOUT_RUNTIME=0; using sglang package baked into image."
@@ -293,7 +390,37 @@ DSV4_ENV_STR="${DSV4_ENV[*]}"
# A recipe carrying a `model:` block supplies its OWN docker env (below), so the
# DSV4 env must not leak into it; the DSV4 recipes keep the string above.
[[ "$HAS_MODEL" == "1" ]] && DSV4_ENV_STR=""
MORI_ENV="-e MORI_DISABLE_AUTO_XGMI=1 -e NCCL_IB_HCA=ionic -e NCCL_IB_GID_INDEX=1 -e NCCL_CROSS_NIC=1"
# NCCL_IB_HCA must name real devices. "ionic" is the mi355x cluster's spelling;
# on pit2 the HCAs are rdma0..7 (the recipe's $IB) and "ionic" matches nothing,
# which fails TP group init outright with
# RuntimeError: NCCL error: remote process exited or there was a network error
# from ncclCommInitRank. The wide-EP block below already overrides this for
# EP>8; EP<=8 recipes need it too.
MORI_NCCL_HCA=ionic
[[ "$CLUSTER" == "spur" ]] && MORI_NCCL_HCA="$IB"
MORI_ENV="-e MORI_DISABLE_AUTO_XGMI=1 -e NCCL_IB_HCA=$MORI_NCCL_HCA -e NCCL_IB_GID_INDEX=1 -e NCCL_CROSS_NIC=1"
# Whether NCCL may use IB depends on the role, not on the recipe. A single-node
# engine does all its NCCL traffic intra-node (cross-node KV goes over mori),
# and on pit2 letting it reach for IB hangs ncclCommInitRank. Gating on the
# whole recipe is wrong for 2P1D-EP16 (PN_PER=1, DN_PER=2), whose prefill
# engines are single-node TP8 but whose recipe-wide test left IB on for both
# roles.
#
# Both cases still need a socket interface: NCCL uses one for its bootstrap
# handshake even on an intra-node data path, and pit2 exposes eight tw-eth* /31
# links that auto-selection picks and hangs on. Wide recipes set it in the block
# below; pin it here for any role that block does not cover.
SPUR_PNCCL=""
SPUR_DNCCL=""
if [[ "$CLUSTER" == "spur" ]]; then
SPUR_SOCK="${DIST_SOCK:-${ADDR_NIC_OVERRIDE:-eno0}}"
if (( PN_PER == 1 )); then
SPUR_PNCCL=" -e NCCL_IB_DISABLE=1 -e NCCL_SOCKET_IFNAME=$SPUR_SOCK -e GLOO_SOCKET_IFNAME=$SPUR_SOCK"
fi
if (( DN_PER == 1 )); then
SPUR_DNCCL=" -e NCCL_IB_DISABLE=1 -e NCCL_SOCKET_IFNAME=$SPUR_SOCK -e GLOO_SOCKET_IFNAME=$SPUR_SOCK"
fi
fi
# Wide-EP (engine spans >1 node) adds mori all-to-all MoE tuning + the cross-node
# torch-dist socket NIC. Gated on nodes-per-engine>1 so EP<=8 recipes are untouched.
if (( PN_PER > 1 || DN_PER > 1 )); then
@@ -411,7 +538,9 @@ if [[ "$MTP_ENABLED" == "1" ]]; then
# EAGLE3 (and other draft-model algos) need an external draft checkpoint;
# built-in EAGLE (DSV4) omits draft_model_path and this stays unset.
if [[ -n "$MTP_DRAFT_PATH" ]]; then
DRAFT_RESOLVED="$(resolve_snapshot "$MTP_DRAFT_PATH")" || exit 1
_d="$(MODEL_ROOT="$MODEL_RESOLVE_ROOT" relocate_model_root "$MTP_DRAFT_PATH")"
_d="$(resolve_snapshot "$_d")" || exit 1
DRAFT_RESOLVED="$(MODEL_ROOT_FROM="$MODEL_RESOLVE_ROOT" relocate_model_root "$_d")"
EXTRA_COMMON="$EXTRA_COMMON --speculative-draft-model-path $DRAFT_RESOLVED"
fi
fi
@@ -466,20 +595,87 @@ else
--disaggregation-transfer-backend $XFER --disaggregation-ib-device $IB$KV_FLAG$DECODE_TAIL"
fi
DOCKER_COMMON="--rm --network host --ipc host --shm-size 32g --privileged \
# /it-share is the mi355x cluster's model NFS. Mount it only where it exists --
# docker would otherwise silently create a root-owned empty /it-share on every
# node of a cluster that keeps its models elsewhere (see MODEL_ROOT).
IT_SHARE_MOUNT="-v /it-share:/it-share:ro "
[[ -d /it-share ]] || IT_SHARE_MOUNT=""
# A cluster that relocates the model cache must bind that root in too, at the
# same path: --model-path is resolved inside the container, and a missing
# directory does not fail as "no such file" -- transformers falls back to
# treating it as a HuggingFace repo id and dies with a confusing
# "Repo id must be in the form 'repo_name' or 'namespace/repo_name'".
if [[ -n "$MODEL_ROOT" && "$MODEL_ROOT" != /it-share/* ]]; then
IT_SHARE_MOUNT="$IT_SHARE_MOUNT-v $MODEL_ROOT:$MODEL_ROOT:ro "
fi
# Slurm hosts each server as a long-lived job step and reaps the container when
# the step ends, so --rm is right there. Spur has no equivalent step (its
# --overlap path is an interactive exec-into-job), so the container runs
# DETACHED under dockerd and drive.sh polls it -- which means it must NOT be
# --rm, or the exit code is gone before the poller can read it. Teardown removes
# it explicitly instead.
DOCKER_LIFECYCLE="--rm"
[[ "$CLUSTER" == "spur" ]] && DOCKER_LIFECYCLE="-d"
# pit2 runs rootless docker, whose spur-authz plugin refuses --privileged
# outright ("denied [P1]"). Spell out what --privileged was actually buying us:
# the device nodes (already listed below), the groups that own kfd/dri, and two
# capabilities -- IPC_LOCK, which RDMA memory registration needs, and SYS_PTRACE
# for profiling. Verified on g11: ibv_devinfo sees all 8 HCAs PORT_ACTIVE and
# --ipc/--network host and --shm-size are all permitted under this set.
PRIV_ARGS="--privileged"
if [[ "$CLUSTER" == "spur" ]]; then
PRIV_ARGS="--group-add video --group-add render --cap-add IPC_LOCK --cap-add SYS_PTRACE"
fi
DOCKER_COMMON="$DOCKER_LIFECYCLE --network host --ipc host --shm-size 32g $PRIV_ARGS \
--security-opt seccomp=unconfined \
--device /dev/kfd --device /dev/dri --device /dev/infiniband \
-v /it-share:/it-share:ro -v $HOME:/host_home $CHECKOUT_DOCKER_ARGS"
${IT_SHARE_MOUNT}-v $HOME:/host_home $CHECKOUT_DOCKER_ARGS"
# Optional extra docker args (e.g. bind-mounting a locally-built lib for
# validation). Empty by default so the docker argv is byte-identical otherwise.
[[ -n "${EXTRA_DOCKER_ARGS:-}" ]] && DOCKER_COMMON="$DOCKER_COMMON ${EXTRA_DOCKER_ARGS}"
# Spur compute nodes run prolog/epilog hooks that reap any container not tagged
# with the owning job, so an untagged server is SIGKILLed (rc=137) seconds after
# it starts. Single-quoted so the literal $SPUR_JOB_ID survives into the
# generated per-role script and expands on the compute node, where the job
# environment actually defines it.
if [[ "$CLUSTER" == "spur" ]]; then
DOCKER_COMMON="$DOCKER_COMMON "'--label spur_job_id=$SPUR_JOB_ID'
fi
# Per-role wide-EP docker env (MORI dispatch-token tuning etc.). Empty for EP<=8
# recipes; carries its own leading space so an empty value leaves the docker argv
# byte-identical (no stray double space).
PENV_ARG=""; [[ -n "$PENV" ]] && PENV_ARG=" $PENV"
DENV_ARG=""; [[ -n "$DENV" ]] && DENV_ARG=" $DENV"
# Pin the torch-dist rendezvous port for single-node engines. Without
# --nccl-port, get_free_port() binds port 0, closes the socket, and rank 0
# re-binds the same number later for the TCPStore; anything handed that port in
# between makes rank 0 die with EADDRINUSE. It reproduces rather than being a
# lottery, since the kernel walks its ephemeral range in order.
#
# Only single-node engines need this -- a multi-node engine gets
# --dist-init-addr, which _resolve_dist_init_method() prefers over the port.
# The value is the recipe's own per-role port +1000, which clears the server
# port and stays under the 32768 ephemeral floor. These recipes place at most
# one single-node engine per node, so two engines cannot collide.
#
# Pinning is safe here because --dp-size is only emitted together with
# --enable-dp-attention, and that path gives every DP rank the same nccl port.
# launch_dp_schedulers(), which needs a distinct port per worker, is never
# reached by these recipes.
PNCCL_ARG=""
DNCCL_ARG=""
if [[ "$CLUSTER" == "spur" ]]; then
PNCCL_ARG=" --nccl-port $((PPORT + 1000))"
DNCCL_ARG=" --nccl-port $((DPORT + 1000))"
fi
# Per-role NCCL IB/socket policy (spur only; empty under slurm, so the argv
# there is unchanged). Appended after $PENV/$DENV so docker's last-wins applies.
PENV_ARG="$PENV_ARG$SPUR_PNCCL"
DENV_ARG="$DENV_ARG$SPUR_DNCCL"
# ---------------------------------------------------------------------------
# Write per-role scripts that srun dispatches to each compute node.
# ---------------------------------------------------------------------------
@@ -511,8 +707,20 @@ fi
echo "[checkout-sglang] reinstalling sglang from $CHECKOUT_SRC"
rm -rf "$RUNTIME_CHECKOUT"
mkdir -p "$RUNTIME_CHECKOUT"
# tar exits 1 for warnings -- notably "file changed as we read it", which NFS
# attribute caching raises spuriously on a tree nothing is writing (seen on the
# pit2 shared-NFS checkout). Only 2+ is a real error, and the extracting tar
# must still succeed outright. Without this, set -o pipefail aborts the install
# on a warning and the server container exits before it ever starts.
set +e
tar --exclude='__pycache__' --exclude='*.pyc' \
-C "$CHECKOUT_SRC" -cf - . | tar -C "$RUNTIME_CHECKOUT" -xf -
_tar_rc=("${PIPESTATUS[@]}")
set -e
if (( _tar_rc[0] > 1 || _tar_rc[1] != 0 )); then
echo "[checkout-sglang] ERROR: staging tar failed (create=${_tar_rc[0]} extract=${_tar_rc[1]})" >&2
exit 1
fi
git config --global --add safe.directory "$RUNTIME_CHECKOUT" || true
@@ -646,6 +854,10 @@ fi
DIST_ARGS=""
if [[ "\${NNODES:-1}" != "1" ]]; then
DIST_ARGS="--nnodes \$NNODES --node-rank \$NODE_RANK --dist-init-addr \$DIST_ADDR:\${DIST_PORT:-29500}"
else
# Single-node engine: no --dist-init-addr, so the rendezvous port is the one
# sglang would otherwise pick with get_free_port(). Pin it. Empty on slurm.
DIST_ARGS="$PNCCL_ARG"
fi
exec python3 -m sglang.launch_server \
--model-path $MODEL_PATH --host 0.0.0.0 --port $PPORT \
@@ -665,6 +877,10 @@ fi
DIST_ARGS=""
if [[ "\${NNODES:-1}" != "1" ]]; then
DIST_ARGS="--nnodes \$NNODES --node-rank \$NODE_RANK --dist-init-addr \$DIST_ADDR:\${DIST_PORT:-29500}"
else
# Single-node engine: no --dist-init-addr, so the rendezvous port is the one
# sglang would otherwise pick with get_free_port(). Pin it. Empty on slurm.
DIST_ARGS="$DNCCL_ARG"
fi
exec python3 -m sglang.launch_server \
--model-path $MODEL_PATH --host 0.0.0.0 --port $DPORT \
@@ -715,7 +931,7 @@ fi
exec python3 -m sglang.launch_server \
--model-path $MODEL_PATH --host 0.0.0.0 --port $PPORT \
$PREFILL_COMMON_FLAGS "\${MODEL_SERVER_ARGS[@]}" \
--disaggregation-mode prefill --disaggregation-bootstrap-port $PBOOT
--disaggregation-mode prefill --disaggregation-bootstrap-port $PBOOT$PNCCL_ARG
EOF
cat > "$WORKDIR/decode_entry.sh" <<EOF
@@ -730,7 +946,7 @@ fi
exec python3 -m sglang.launch_server \
--model-path $MODEL_PATH --host 0.0.0.0 --port $DPORT \
$DECODE_COMMON_FLAGS "\${MODEL_SERVER_ARGS[@]}" \
--disaggregation-mode decode --disaggregation-bootstrap-port $DBOOT
--disaggregation-mode decode --disaggregation-bootstrap-port $DBOOT$DNCCL_ARG
EOF
cat > "$WORKDIR/prefill.sh" <<EOF
@@ -864,7 +1080,167 @@ cat > "$WORKDIR/drive.sh" <<'DRIVE'
#!/bin/bash
set -x
WORKDIR="$1"; PW="${2:-1}"; DW="${3:-1}"; PN_PER="${4:-1}"; DN_PER="${5:-1}"; DIST_NIC="${6:-}"
mapfile -t NODES < <(scontrol show hostnames "$SLURM_JOB_NODELIST")
# Interface resolve_ip() falls back to when the recipe names no dist NIC (every
# EP<=8 recipe). Empty on Slurm, where drive.sh runs on the login node and never
# resolves a node it is running on; set on Spur, where it runs on an allocated
# node and that node's own name resolves to 127.0.1.1 via Ubuntu's stock
# /etc/hosts -- useless as an address to advertise to the other engine.
ADDR_NIC="${7:-}"
# Scheduler seam (see the CLUSTER block at the top of launch_mi355x.sh). Slurm
# hands out a bracket-compressed hostlist that only `scontrol show hostnames`
# can expand, and srun inherits the allocation from the environment. Spur's
# nodelist env var is already comma-expanded, has no `show hostnames`, and its
# `srun --overlap` requires an explicit --jobid.
CLUSTER="${CLUSTER:-slurm}"
if [[ "$CLUSTER" == "spur" ]]; then
SRUN=(srun --jobid="$SPUR_JOB_ID" --overlap)
sched_hostnames() { tr ',' '\n' <<< "$SPUR_JOB_NODELIST" | sed '/^$/d'; }
else
SRUN=(srun --overlap)
sched_hostnames() { scontrol show hostnames "$SLURM_JOB_NODELIST"; }
fi
mapfile -t NODES < <(sched_hostnames)
# Dispatch one long-running, log-producing step to a node.
#
# Spur allocates a pty for every --overlap step. drive.sh runs the servers and
# the bench as BACKGROUND subshells, and a background process group writing
# through that pty is stopped by SIGTTOU -- the step then dies (marker rc=137)
# and everything it printed after the first few bytes is lost, which is why a
# failing leg used to leave a log containing nothing but the srun banner. Having
# srun write the file itself keeps the step's output off the pty entirely.
# Slurm has no pty here, so it keeps the original shell redirect.
srun_log() { # srun_log <logfile> <node> <cmd...>
local log="$1" node="$2"; shift 2
# On spur drive.sh runs on one of the allocated nodes, so a dispatch to that
# node would be srun exec-ing into its own job -- the one spur case that has
# repeatedly misbehaved. Run it directly instead; the result is identical and
# it removes a whole class of scheduler interaction.
if [[ "$CLUSTER" == "spur" && "$node" == "$(hostname)" ]]; then
"$@" > "$log" 2>&1
return
fi
"${SRUN[@]}" -N1 --nodelist="$node" "$@" > "$log" 2>&1
}
srun_local_or_step() { # same rule, for short commands whose output we capture
local node="$1"; shift
if [[ "$CLUSTER" == "spur" && "$node" == "$(hostname)" ]]; then
"$@"
return
fi
"${SRUN[@]}" -N1 --nodelist="$node" "$@"
}
# --- spur: containers are detached, liveness comes from dockerd ---------------
# Spur cannot host a long-lived process in a job step, so on spur the servers
# and the bench run as DETACHED containers and these helpers derive the same
# server_exit_*/bench_exit markers the monitor loop already consumes -- from
# real container state rather than from srun's status. Every srun issued here is
# a sub-second foreground command, which is the part of spur that behaves.
declare -A SPUR_SEEN=() # container -> 1 once observed running, so a container
# that has not started yet is not mistaken for a crash
declare -A SPUR_ABSENT=() # container -> consecutive polls that found it gone
SPUR_SERVERS=() # "role:node:container"
SPUR_POLLS=0
SPUR_ABSENT_LIMIT=3 # polls (10s apart) before "gone" is believed
container_state() { # <node> <name> -> "running" | "<exit code>" | absent | unknown
# The obvious `docker inspect | tail -1` cannot be trusted on spur. drive.sh
# runs on one allocated node, so every poll of a container on the OTHER node
# is an `srun --overlap` step, and those intermittently fail to spawn
# ("step command failed to spawn: Permission denied (os error 13)"). A failed
# step and a genuinely absent container are the same observation -- nonzero
# exit, empty stdout -- so one flaky poll was enough to declare a healthy
# server dead. Observed on dsv4flash-fp8-1k1k-1p1d: prefill marked
# "rc=missing" while `docker ps` on that node still showed it Up.
#
# The __ok__ sentinel is emitted only if the remote command actually ran, so a
# dispatch failure is reported as "unknown" and the caller ignores it rather
# than counting it as evidence.
local out
out="$(srun_local_or_step "$1" bash -c \
"docker inspect -f '{{if .State.Running}}running{{else}}{{.State.ExitCode}}{{end}}' $2 2>/dev/null || echo absent; echo __ok__" \
2>/dev/null | tr -d '\r')"
[[ "$out" == *__ok__* ]] || { echo unknown; return; }
grep -vxE '__ok__|' <<< "$out" | tail -1
}
container_log() { # <node> <name> <logfile>
# Fetch to a temp file and never let a failed fetch shrink the log: the same
# flaky step that produced the false "missing" also truncated the 5-line
# prefill log to 0 bytes, destroying the only evidence of what happened.
local tmp="$3.fetch"
srun_local_or_step "$1" docker logs "$2" > "$tmp" 2>/dev/null || true
# Publish in place (truncate + rewrite the SAME inode), never by renaming over
# the target. drive.sh keeps a `tail -F` on bench.log, and swapping the inode
# every 10s made tail reopen and re-emit the whole file each poll -- the
# driver log reached 177 MB and legs took 2 h instead of 36 min.
if [[ -s "$tmp" || ! -f "$3" ]]; then cat "$tmp" > "$3"; fi
rm -f "$tmp"
}
spur_rm() { # <node> <container...> -- remove, retrying past a flaky dispatch
local n="$1"; shift
local i
for i in 1 2 3; do
srun_local_or_step "$n" bash -c "docker rm -f $* >/dev/null 2>&1; echo __ok__" \
2>/dev/null | grep -q __ok__ && return 0
sleep 3
done
return 1
}
# Refresh one container's log and write its marker once it is no longer running.
spur_track() { # <node> <name> <logfile> <markerfile> <marker prefix>
local node="$1" name="$2" log="$3" marker="$4" label="$5" st
container_log "$node" "$name" "$log"
st="$(container_state "$node" "$name")"
if [[ "$st" == "running" ]]; then
SPUR_SEEN["$name"]=1
SPUR_ABSENT["$name"]=0
return
fi
# A step that never ran says nothing about the container.
[[ "$st" == "unknown" ]] && return
if [[ "$st" == "absent" || -z "$st" ]]; then
# Not created yet, or one poll that came back empty. Only a container that
# is absent on SPUR_ABSENT_LIMIT polls in a row is treated as a crash, so
# neither a slow `docker run -d` nor a single flaky poll ends the leg.
SPUR_ABSENT["$name"]=$(( ${SPUR_ABSENT[$name]:-0} + 1 ))
if [[ -z "${SPUR_SEEN[$name]:-}" && "$SPUR_POLLS" -lt 6 ]]; then return; fi
[[ "${SPUR_ABSENT[$name]}" -lt "$SPUR_ABSENT_LIMIT" ]] && return
st="missing"
fi
echo "$label rc=$st" > "$marker"
}
spur_refresh_markers() {
local entry role node cname st
SPUR_POLLS=$((SPUR_POLLS + 1))
for entry in "${SPUR_SERVERS[@]}"; do
IFS=: read -r role node cname <<< "$entry"
[[ -f "$WORKDIR/server_exit_${role}_${node}" ]] && continue
spur_track "$node" "$cname" "$WORKDIR/${role}_${node}.log" \
"$WORKDIR/server_exit_${role}_${node}" "$role@$node"
done
# The bench marker holds a bare exit code, not the "role@node rc=" form.
container_log "$PNODE" mi355x_bench "$WORKDIR/bench.log"
st="$(container_state "$PNODE" mi355x_bench)"
if [[ "$st" == "running" ]]; then
SPUR_SEEN[mi355x_bench]=1
SPUR_ABSENT[mi355x_bench]=0
elif [[ "$st" == "unknown" ]]; then
:
elif [[ "$st" == "absent" || -z "$st" ]]; then
SPUR_ABSENT[mi355x_bench]=$(( ${SPUR_ABSENT[mi355x_bench]:-0} + 1 ))
if [[ -n "${SPUR_SEEN[mi355x_bench]:-}" \
&& "${SPUR_ABSENT[mi355x_bench]}" -ge "$SPUR_ABSENT_LIMIT" ]]; then
echo 1 > "$WORKDIR/bench_exit"
fi
else
echo "$st" > "$WORKDIR/bench_exit"
fi
}
# Resolve a node's IP. When a cross-node dist NIC is named (wide engines set
# $DIST_NIC to the recipe's dist_socket_ifname), read the address off that
# interface on the node itself -- the multi-node dist-init-addr must be a
@@ -873,20 +1249,54 @@ mapfile -t NODES < <(scontrol show hostnames "$SLURM_JOB_NODELIST")
# root whose hostname resolved to an unpingable IP, ZMQ bind => "Cannot assign
# requested address"). Fall back to Slurm's NodeAddr, then DNS.
resolve_ip() {
local n="$1" ip="" addr="" field
if [[ -n "$DIST_NIC" ]]; then
ip=$(srun --overlap -N1 --nodelist="$n" ip -4 -o addr show "$DIST_NIC" 2>/dev/null \
| awk '{print $4}' | cut -d/ -f1 | head -1)
local n="$1" ip="" addr="" field nic="${DIST_NIC:-$ADDR_NIC}"
if [[ -n "$nic" ]]; then
# Go through srun_local_or_step, not a bare srun: on spur drive.sh runs on an
# allocated node, and resolving its OWN node through `srun --overlap` is the
# dispatch that has repeatedly misbehaved. Here it did not fail, it HUNG --
# a leg sat 30 min on `srun ... ip -4 -o addr show eno0` for the node it was
# already running on, before any container was started. The helper runs it
# directly in that case. The timeout bounds the genuinely remote call so a
# wedged step costs 60s and a DNS fallback instead of the whole leg.
if [[ "$CLUSTER" == "spur" ]]; then
if [[ "$n" == "$(hostname)" ]]; then
ip=$(command ip -4 -o addr show "$nic" 2>/dev/null \
| awk '{print $4}' | cut -d/ -f1 | head -1)
else
ip=$(timeout 60 "${SRUN[@]}" -N1 --nodelist="$n" ip -4 -o addr show "$nic" 2>/dev/null \
| awk '{print $4}' | cut -d/ -f1 | head -1)
fi
else
ip=$("${SRUN[@]}" -N1 --nodelist="$n" ip -4 -o addr show "$nic" 2>/dev/null \
| awk '{print $4}' | cut -d/ -f1 | head -1)
fi
fi
if [[ -z "$ip" ]]; then
for field in $(scontrol show node "$n" -o); do
case "$field" in
NodeAddr=*) addr="${field#NodeAddr=}"; break ;;
esac
done
# Spur's `scontrol show node` does not emit Slurm's NodeAddr= field, so the
# loop below would just fall through; skip it there and go straight to DNS.
if [[ "$CLUSTER" != "spur" ]]; then
for field in $(scontrol show node "$n" -o); do
case "$field" in
NodeAddr=*) addr="${field#NodeAddr=}"; break ;;
esac
done
fi
[[ -n "$addr" ]] || addr="$n"
read -r ip _ < <(getent ahostsv4 "$addr")
fi
# A loopback or empty answer is never a usable advertised address. Fail here
# rather than let a server bind 127.x (or nothing) and have the peer engine
# time out connecting to it with no obvious cause. Callers must use
# `x=$(resolve_ip n) || exit 1` -- the status is visible at the assignment,
# but an `exit` inside the substitution would only leave the subshell.
if [[ -z "$ip" ]]; then
echo "[drive] ERROR: no address found for $n" >&2
return 1
fi
if [[ "$ip" == 127.* ]]; then
echo "[drive] ERROR: $n resolved to loopback $ip; set ADDR_NIC to a real interface" >&2
return 1
fi
printf '%s\n' "$ip"
}
# SLURM canonicalizes (sorts) SLURM_JOB_NODELIST, so the requested --nodelist
@@ -913,8 +1323,8 @@ PN_TOTAL=$((PW * PN_PER)); DN_TOTAL=$((DW * DN_PER))
PNODES=("${NODES[@]:0:PN_TOTAL}")
DNODES=("${NODES[@]:PN_TOTAL:DN_TOTAL}")
PNODE="${PNODES[0]}"; DNODE="${DNODES[0]}"
PIP=$(resolve_ip "$PNODE")
DIP=$(resolve_ip "$DNODE")
PIP=$(resolve_ip "$PNODE") || exit 1
DIP=$(resolve_ip "$DNODE") || exit 1
echo "[drive] prefill nodes: ${PNODES[*]} ; decode nodes: ${DNODES[*]}"
echo "[drive] bench targets prefill=$PNODE($PIP) decode=$DNODE($DIP)"
if (( DW > 1 )); then
@@ -928,6 +1338,17 @@ fi
# died and with what code. (A hung-but-alive server is NOT caught here; that is
# bounded by bench.sh's health-wait timeout.)
rm -f "$WORKDIR"/server_exit_* "$WORKDIR/bench_exit"
if [[ "$CLUSTER" == "spur" ]]; then
# Detached containers are not --rm, and a container whose docker client was
# killed can survive as an exited husk. Clear all three names on every
# allocated node first: otherwise `docker run` fails with a name conflict and,
# worse, the poller reads the STALE container's exit code and reports a crash
# that never happened.
for n in "${NODES[@]}"; do
srun_local_or_step "$n" \
docker rm -f mi355x_prefill mi355x_decode mi355x_bench >/dev/null 2>&1 || true
done
fi
# Launch PW prefill engines; each spans PN_PER nodes as its own torch-dist group
# (engine node0 = dist-init addr; node_rank = position WITHIN the engine, so an
# engine's ranks are 0..PN_PER-1, not a global index). PN_PER=1 => NNODES=1 in
@@ -936,21 +1357,34 @@ rm -f "$WORKDIR"/server_exit_* "$WORKDIR/bench_exit"
PCSV=""
for ((k=0; k<PW; k++)); do
e0="${PNODES[k*PN_PER]}"
eip=$(resolve_ip "$e0")
eip=$(resolve_ip "$e0") || exit 1
PCSV="${PCSV:+$PCSV,}$eip"
for ((j=0; j<PN_PER; j++)); do
n="${PNODES[k*PN_PER + j]}"
( srun --overlap -N1 --nodelist="$n" bash "$WORKDIR/prefill.sh" "$j" "$PN_PER" "$eip" > "$WORKDIR/prefill_$n.log" 2>&1
if [[ "$CLUSTER" == "spur" ]]; then
# Returns as soon as the container is detached; liveness is polled later.
srun_log "$WORKDIR/prefill_start_$n.log" "$n" \
bash "$WORKDIR/prefill.sh" "$j" "$PN_PER" "$eip" || true
SPUR_SERVERS+=("prefill:$n:mi355x_prefill")
else
( srun_log "$WORKDIR/prefill_$n.log" "$n" bash "$WORKDIR/prefill.sh" "$j" "$PN_PER" "$eip"
echo "prefill@$n rc=$?" > "$WORKDIR/server_exit_prefill_$n" ) &
fi
done
done
for ((k=0; k<DW; k++)); do
e0="${DNODES[k*DN_PER]}"
eip=$(resolve_ip "$e0")
eip=$(resolve_ip "$e0") || exit 1
for ((j=0; j<DN_PER; j++)); do
n="${DNODES[k*DN_PER + j]}"
( srun --overlap -N1 --nodelist="$n" bash "$WORKDIR/decode.sh" "$j" "$DN_PER" "$eip" > "$WORKDIR/decode_$n.log" 2>&1
if [[ "$CLUSTER" == "spur" ]]; then
srun_log "$WORKDIR/decode_start_$n.log" "$n" \
bash "$WORKDIR/decode.sh" "$j" "$DN_PER" "$eip" || true
SPUR_SERVERS+=("decode:$n:mi355x_decode")
else
( srun_log "$WORKDIR/decode_$n.log" "$n" bash "$WORKDIR/decode.sh" "$j" "$DN_PER" "$eip"
echo "decode@$n rc=$?" > "$WORKDIR/server_exit_decode_$n" ) &
fi
done
done
echo "[drive] prefill engine endpoints (fan-out): $PCSV"
@@ -958,9 +1392,15 @@ sleep 5
# Bench in the background with its own marker, so the wait loop is purely file
# based: finish when bench writes its marker, abort if any server marker shows up
# first (a server died before the sweep completed).
( srun --overlap -N1 --nodelist="$PNODE" bash "$WORKDIR/bench.sh" "$PIP" "$DIP" "$PCSV" > "$WORKDIR/bench.log" 2>&1
BENCH_BG=""
if [[ "$CLUSTER" == "spur" ]]; then
srun_log "$WORKDIR/bench_start.log" "$PNODE" \
bash "$WORKDIR/bench.sh" "$PIP" "$DIP" "$PCSV" || true
else
( srun_log "$WORKDIR/bench.log" "$PNODE" bash "$WORKDIR/bench.sh" "$PIP" "$DIP" "$PCSV"
echo $? > "$WORKDIR/bench_exit" ) &
BENCH_BG=$!
fi
# Stream bench output live and poll the markers with xtrace OFF, so the console
# shows clean benchmark/accuracy output instead of a compgen/sleep trace every
# 10s. (Mirrors NVIDIA's launch_gb200.sh, which set +x around its log stream.)
@@ -970,10 +1410,13 @@ TAIL_PID=$!
set +x
RC=0
while [[ ! -f "$WORKDIR/bench_exit" ]]; do
# On spur the markers are produced here, from container state, instead of by a
# backgrounded srun. Everything below is unchanged.
[[ "$CLUSTER" == "spur" ]] && spur_refresh_markers
if compgen -G "$WORKDIR/server_exit_*" > /dev/null; then
echo "[drive] ERROR: a server exited early before bench finished:"
cat "$WORKDIR"/server_exit_* || true
kill "$BENCH_BG" 2>/dev/null || true
[[ -n "$BENCH_BG" ]] && kill "$BENCH_BG" 2>/dev/null || true
RC=1
break
fi
@@ -983,8 +1426,19 @@ set -x
kill "$TAIL_PID" 2>/dev/null || true
[[ "$RC" -eq 0 ]] && RC=$(cat "$WORKDIR/bench_exit" 2>/dev/null || echo 1)
echo "[drive] bench finished (rc=$RC), tearing down"
for n in "${PNODES[@]}"; do srun --overlap -N1 --nodelist="$n" docker kill mi355x_prefill >/dev/null 2>&1 || true; done
for n in "${DNODES[@]}"; do srun --overlap -N1 --nodelist="$n" docker kill mi355x_decode >/dev/null 2>&1 || true; done
if [[ "$CLUSTER" == "spur" ]]; then
# Detached containers are not --rm, so kill is not enough -- remove them, and
# the bench container too, or the next leg on this node inherits them.
# Retry the removals: a step that fails to spawn used to leave the containers
# running, and the next leg on that node then inherited engines holding all
# eight GPUs.
for n in "${PNODES[@]}"; do spur_rm "$n" mi355x_prefill || true; done
for n in "${DNODES[@]}"; do spur_rm "$n" mi355x_decode || true; done
spur_rm "$PNODE" mi355x_bench || true
else
for n in "${PNODES[@]}"; do "${SRUN[@]}" -N1 --nodelist="$n" docker kill mi355x_prefill >/dev/null 2>&1 || true; done
for n in "${DNODES[@]}"; do "${SRUN[@]}" -N1 --nodelist="$n" docker kill mi355x_decode >/dev/null 2>&1 || true; done
fi
exit "$RC"
DRIVE
chmod +x "$WORKDIR/drive.sh"
@@ -1002,6 +1456,11 @@ EXCLUSIVE_ARG=()
EXCLUDE_ARG=()
[[ -n "${SLURM_EXCLUDE:-}" ]] && EXCLUDE_ARG=(--exclude="$SLURM_EXCLUDE")
# Optional account / QoS (spur requires both; mi355x sets neither).
ACCT_ARG=()
[[ -n "$SLURM_ACCOUNT" ]] && ACCT_ARG+=(-A "$SLURM_ACCOUNT")
[[ -n "$SLURM_QOS" ]] && ACCT_ARG+=(-q "$SLURM_QOS")
# Nodes = sum over engines of nodes-per-engine. EP<=8 (PN_PER=DN_PER=1) gives the
# original PW+DW (1P1D -> 2 nodes); wide EP16 1P1D gives 2+2 = 4 nodes.
TOTAL_NODES=$(( PW * PN_PER + DW * DN_PER ))
@@ -1018,11 +1477,185 @@ fi
# GITHUB_RUN_ID + config make the name unique per matrix leg regardless.
JOB_NAME="mi355x-ci-${RUNNER_NAME:-norunner}-${GITHUB_RUN_ID:-0}-${MATRIX_CONFIG_NAME}"
# Address-resolution fallback NIC for drive.sh (see ADDR_NIC there). Only spur
# needs one, because only there does drive.sh run on an allocated node.
ADDR_NIC=""
[[ "$CLUSTER" == "spur" ]] && ADDR_NIC="${ADDR_NIC_OVERRIDE:-eno0}"
set +e
salloc -p "$SLURM_PARTITION" -N"$TOTAL_NODES" "${NODELIST_ARG[@]}" "${EXCLUDE_ARG[@]}" "${EXCLUSIVE_ARG[@]}" \
--job-name "$JOB_NAME" -t "$TIME_LIMIT" \
bash "$WORKDIR/drive.sh" "$WORKDIR" "$PW" "$DW" "$PN_PER" "$DN_PER" "$DIST_SOCK"
SALLOC_RC=$?
if [[ "$CLUSTER" == "spur" ]]; then
# Spur's salloc takes no trailing command, so drive.sh is submitted as a
# batch job instead. That runs it on the first allocated node rather than
# the login node, which is fine: $WORKDIR and $GITHUB_WORKSPACE are on
# shared NFS, and drive.sh only ever reaches the other nodes through srun.
# We stream the batch output back live and take the exit code from a file
# drive_batch.sh writes, since sbatch returns as soon as the job is queued.
cat > "$WORKDIR/drive_batch.sh" <<EOF
#!/bin/bash
# Set explicitly rather than relying on sbatch to propagate the submit
# environment; drive.sh needs it to pick the spur scheduler seam.
export CLUSTER=spur
export PATH="$PATH"
# Slurm runs a batch script once, on the first allocated node. Spur runs it as
# one task PER NODE (scontrol reports NumTasks=NumNodes), so without this guard
# every node gets its own drive.sh: the copies race, and each one's pre-launch
# container cleanup kills the servers the other just started. Only rank 0
# drives; the rest exit immediately and simply hold their node in the
# allocation, which is all drive.sh needs them for.
# Elect the driver with an atomic mkdir on shared NFS rather than by rank:
# spur does not consistently place the batch script on rank 0 (it has been
# observed running only on SPUR_NODEID=1), so a rank test would sometimes elect
# nobody and the leg would never start.
if ! mkdir "$WORKDIR/driver.lock" 2>/dev/null; then
# Must BLOCK, not exit: spur starts tearing the job down as soon as one of its
# tasks finishes, and a job in Completing rejects further steps
# ("CreateJobStep failed: job N is not running") -- which killed the decode
# launch. Holding here keeps the node in the allocation until rank 0 is done.
echo "[drive_batch] \$(hostname) standing by; another task drives" >> "$WORKDIR/standby.log"
while [[ ! -f "$WORKDIR/drive_exit" ]]; do sleep 10; done
exit 0
fi
echo "[drive_batch] \$(hostname) rank \${SPUR_NODEID:-?} elected driver"
# Send the driver's output to a private per-node file, not to the shared
# --output. All tasks of the job write sbatch.out concurrently, so it loses
# records: a leg that failed in the monitor loop left an sbatch.out that simply
# stopped mid-run, with the teardown lines never visible.
#
# A plain redirect, deliberately NOT `| tee`. With a pipeline, drive_batch waits
# for the whole pipeline, and that does not finish when drive.sh does -- the
# backgrounded `tail -F` on bench.log inherits drive.sh's stdout and holds the
# pipe's write end open. drive.sh exited 0 and drive_exit was still unwritten
# 11 minutes later, so the standby tasks kept the allocation alive and the leg
# looked hung after it had actually passed.
bash "$WORKDIR/drive.sh" "$WORKDIR" "$PW" "$DW" "$PN_PER" "$DN_PER" "$DIST_SOCK" "$ADDR_NIC" \
> "$WORKDIR/drive_\$(hostname).log" 2>&1
echo \$? > "$WORKDIR/drive_exit"
EOF
chmod +x "$WORKDIR/drive_batch.sh"
rm -f "$WORKDIR/drive_exit"
SBATCH_OUT="$WORKDIR/sbatch.out"
: > "$SBATCH_OUT"
# Node count is what matters here; do NOT pass -n1 to make the task count
# match. Spur sizes the allocation from the task count, so -n1 collapses a
# 2-node request to a single node and the decode engine has nowhere to run.
# Uniqueness of the driver is handled in drive_batch.sh by an atomic lock,
# not by the task count.
# Spur's controller is a Raft cluster, so submission can fail for reasons
# that have nothing to do with the request: during a leader election it
# answers `The service is currently unavailable ... "not the Raft leader"`.
# That is transient and clears in seconds, but an unretried submit turns it
# into a failed leg. Worse, the message is multi-line, so the ${VAR##* }
# job-id parse below extracted `leader"` from it and the leg was reported as
# a benchmark failure with an empty bench.log -- indistinguishable, at a
# glance, from a model bug. Observed on glm52-fp4-1k1k-2p1d-ep16 2026-09-17.
# Retry only on that signature; a genuinely bad sbatch request must still
# fail immediately rather than being retried six times.
SPUR_LEG_JOB_ID=""
for _attempt in 1 2 3 4 5 6; do
SBATCH_MSG=$(sbatch -p "$SLURM_PARTITION" -N"$TOTAL_NODES" "${NODELIST_ARG[@]}" \
"${EXCLUDE_ARG[@]}" "${EXCLUSIVE_ARG[@]}" "${ACCT_ARG[@]}" \
--job-name "$JOB_NAME" -t "$TIME_LIMIT" \
--output "$SBATCH_OUT" --error "$SBATCH_OUT" \
"$WORKDIR/drive_batch.sh" 2>&1)
echo "$SBATCH_MSG"
SPUR_LEG_JOB_ID="${SBATCH_MSG##* }"
[[ "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]] && break
if [[ "$SBATCH_MSG" == *"not the Raft leader"* \
|| "$SBATCH_MSG" == *"service is currently unavailable"* ]]; then
echo "[launch] spur controller unavailable (attempt $_attempt/6); retrying in 20s" >&2
SPUR_LEG_JOB_ID=""
sleep 20
continue
fi
break
done
if [[ ! "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]]; then
echo "ERROR: could not parse a job id out of: $SBATCH_MSG" >&2
SALLOC_RC=1
else
echo "[launch] spur job $SPUR_LEG_JOB_ID submitted; streaming $SBATCH_OUT"
tail -F "$SBATCH_OUT" 2>/dev/null &
SPUR_TAIL_PID=$!
# drive_batch.sh writes drive_exit last, so that file is the primary
# completion signal; the squeue check catches a job that died without
# ever running it (node failure, scheduler kill, time limit).
while :; do
[[ -f "$WORKDIR/drive_exit" ]] && break
if ! squeue -h -o "%i" 2>/dev/null | grep -qx "$SPUR_LEG_JOB_ID"; then
# The job has left the queue. drive_exit is written on a compute
# node and read here on the login node, so NFS close-to-open
# visibility can delay it well past a single short sleep. A five
# second grace produced three false reds on 2026-09-16, each on a
# leg that had finished every concurrency point and passed the
# accuracy gate; one of them was the whole GLM EP16 row.
for _ in $(seq 1 30); do
[[ -f "$WORKDIR/drive_exit" ]] && break
sleep 3
done
if [[ ! -f "$WORKDIR/drive_exit" ]]; then
# Still not visible. Do not invent a failure we did not
# observe -- decide from what the run actually produced.
# bench_exit is written earlier and is usually visible by now;
# when even that is missing (seen on glm52-fp4-1k1k-2p1d-ep16,
# which had all seven results at 0.945), a passed accuracy
# gate with results on disk is the stronger evidence.
#
# "Some results plus a passed gate" is not enough. A leg
# whose decode ran at ~9 tok/s got through conc=1, 8 and 16
# of seven points before the job ended, and this branch
# called it green: the gate had passed and raw files
# existed. Require every expected concurrency point, so a
# sweep that stopped early is red rather than a green row
# standing on someone else's numbers.
WANT=0; HAVE=0
for _c in ${CONCS//,/ }; do
WANT=$((WANT + 1))
[[ -f "$WORKDIR/raw_conc${_c}.json" ]] && HAVE=$((HAVE + 1))
done
if [[ "$(cat "$WORKDIR/bench_exit" 2>/dev/null)" == "0" ]]; then
echo 0 > "$WORKDIR/drive_exit"
elif [[ "$HAVE" -eq "$WANT" ]] \
&& awk '/\[gsm8k\] accuracy=/{
match($0,/accuracy=([0-9.]+)/,a)
match($0,/threshold=([0-9.]+)/,t)
if (a[1]+0 >= t[1]+0) ok=1
} END{ exit ok?0:1 }' \
"$WORKDIR/bench.log" 2>/dev/null; then
echo 0 > "$WORKDIR/drive_exit"
else
echo "[launch] incomplete sweep: $HAVE/$WANT concurrency points" >&2
echo 1 > "$WORKDIR/drive_exit"
fi
fi
break
fi
sleep 10
done
kill "$SPUR_TAIL_PID" 2>/dev/null || true
SALLOC_RC=$(cat "$WORKDIR/drive_exit" 2>/dev/null || echo 1)
# Release the allocation. On slurm the job ends when salloc's command
# returns; on spur drive.sh is an sbatch job, and the loop above exits as
# soon as drive_exit appears -- while the batch job, and every container
# it started, is still running. Nothing else ever cancels it, so the
# allocation is held until the time limit. The next leg then cannot get
# nodes and dies with `JobLaunchFailure (dispatch confirmation failed
# (2/4 confirmed))`, which reads as an unrelated infrastructure fault and
# cascades through every leg after it. Observed 2026-09-17: jobs
# 1774/1775 held all four nodes long after their legs had been recorded
# rc=1. Unconditional, because reaching here means this leg is done with
# its nodes either way.
if squeue -h -o "%i" 2>/dev/null | grep -qx "$SPUR_LEG_JOB_ID"; then
echo "[launch] releasing spur job $SPUR_LEG_JOB_ID"
scancel "$SPUR_LEG_JOB_ID" 2>/dev/null || true
fi
fi
else
salloc -p "$SLURM_PARTITION" -N"$TOTAL_NODES" "${NODELIST_ARG[@]}" "${EXCLUDE_ARG[@]}" "${EXCLUSIVE_ARG[@]}" \
"${ACCT_ARG[@]}" \
--job-name "$JOB_NAME" -t "$TIME_LIMIT" \
bash "$WORKDIR/drive.sh" "$WORKDIR" "$PW" "$DW" "$PN_PER" "$DN_PER" "$DIST_SOCK"
SALLOC_RC=$?
fi
set -e
# bench output already streamed live from drive.sh (tail -F). drive.sh exits
@@ -1075,10 +1708,27 @@ fi
# so they must be the role totals (Oren EP16: ctx=2*8=16, gen=1*16=16, gpus=32).
PREFILL_GPUS_TOTAL=$((PW * PTP)); DECODE_GPUS_TOTAL=$((DW * DTP))
TOTAL_GPUS=$((PREFILL_GPUS_TOTAL + DECODE_GPUS_TOTAL))
# Clear this leg's results from any previous run before writing new ones.
# The filenames are a pure function of the config, so a run that produces
# fewer concurrency points than the last one leaves the missing slots filled
# by the old run's files and the published table silently mixes two runs.
# Seen on kimik26-mxfp4-1k1k-2p1d-ep16-mxfp4: 2 points measured, 7 published,
# 5 of them a day old. Deleting first means a short run looks short.
rm -f "$GITHUB_WORKSPACE/${RESULT_FILENAME}_${MATRIX_CONFIG_NAME}_conc"*"_gpus_"*".json"
PROCESSED=0
for C in ${CONCS//,/ }; do
RAW="$WORKDIR/raw_conc${C}.json"
[[ -f "$RAW" ]] || { echo "WARN: missing $RAW"; continue; }
# raw_conc*.json is written by the bench container on a compute node; this
# loop runs on the driver. NFS close-to-open means the last file written is
# routinely not visible here yet, and skipping it outright silently dropped
# one or two concurrency points from four legs of a 34-leg sweep -- the runs
# were fine, only the published results were short. Wait a bounded 60s.
for _ in $(seq 1 20); do
[[ -f "$RAW" ]] && break
sleep 3
done
[[ -f "$RAW" ]] || { echo "WARN: missing $RAW (not visible after 60s)"; continue; }
DEST="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${MATRIX_CONFIG_NAME}_conc${C}_gpus_${TOTAL_GPUS}_ctx_${PREFILL_GPUS_TOTAL}_gen_${DECODE_GPUS_TOTAL}.json"
MODEL_ID="$MODEL_PATH" python3 - "$RAW" "$DEST" "$C" <<'PY'
import json, os, sys