From b86a30afbae389eec8763f4bcf38f14904d2d00c Mon Sep 17 00:00:00 2001 From: Zhaoyi Li <36555117+Lzy17@users.noreply.github.com> Date: Mon, 21 Sep 2026 03:56:46 -0500 Subject: [PATCH] [AMD][DI][CI] Add a SPUR cluster profile to AMD DI CI (#40113) --- .../workflows/nightly-amd-mi355x-disagg.yml | 9 +- scripts/ci/slurm/launch_mi355x.sh | 712 +++++++++++++++++- 2 files changed, 687 insertions(+), 34 deletions(-) diff --git a/.github/workflows/nightly-amd-mi355x-disagg.yml b/.github/workflows/nightly-amd-mi355x-disagg.yml index a19189269..3ea8c5bb3 100644 --- a/.github/workflows/nightly-amd-mi355x-disagg.yml +++ b/.github/workflows/nightly-amd-mi355x-disagg.yml @@ -156,11 +156,14 @@ jobs: # mi355x-ci--- # Here we clear THIS runner's leftovers from a crashed previous run, so # we match the RUNNER_NAME prefix (older runs have a different run id). - # Never a blanket `squeue --me`, which would kill a concurrent leg. + # Never a blanket match on all jobs, which would kill a concurrent leg. # %200j: squeue truncates the job name (%j) by default -- widen it so # the grep sees the full name. + # No --me: Spur's squeue does not accept it. The RUNNER_NAME prefix is + # what makes this selective, and that is unique per runner, so the + # filter is equally precise without it. if [ -z "${RUNNER_NAME:-}" ]; then echo "RUNNER_NAME unset; skipping"; exit 0; fi - STALE_JOBS=$(squeue --me --noheader --format="%i %200j" | grep -F "mi355x-ci-${RUNNER_NAME}-" | awk '{print $1}' || true) + STALE_JOBS=$(squeue --noheader --format="%i %200j" | grep -F "mi355x-ci-${RUNNER_NAME}-" | awk '{print $1}' || true) if [ -n "$STALE_JOBS" ]; then echo "Cancelling stale jobs for ${RUNNER_NAME}: $STALE_JOBS" scancel $STALE_JOBS @@ -237,7 +240,7 @@ jobs: # config are unique per matrix leg, so a concurrent leg is never hit. if [ -z "${RUNNER_NAME:-}" ]; then echo "RUNNER_NAME unset; skipping"; exit 0; fi JOB_TAG="mi355x-ci-${RUNNER_NAME}-${GITHUB_RUN_ID}-${MATRIX_CONFIG_NAME}" - ACTIVE_JOBS=$(squeue --me --noheader --format="%i %200j" | grep -F "$JOB_TAG" | awk '{print $1}' || true) + ACTIVE_JOBS=$(squeue --noheader --format="%i %200j" | grep -F "$JOB_TAG" | awk '{print $1}' || true) if [ -n "$ACTIVE_JOBS" ]; then echo "Cancelling jobs for ${JOB_TAG}: $ACTIVE_JOBS" scancel $ACTIVE_JOBS diff --git a/scripts/ci/slurm/launch_mi355x.sh b/scripts/ci/slurm/launch_mi355x.sh index 54a2ad7dc..a20439e49 100755 --- a/scripts/ci/slurm/launch_mi355x.sh +++ b/scripts/ci/slurm/launch_mi355x.sh @@ -58,6 +58,51 @@ set -x SLURM_PARTITION="${SLURM_PARTITION:-amd-sglang}" TIME_LIMIT="${TIME_LIMIT:-02:30:00}" MODEL_PATH="${MODEL_PATH:-${MODEL:-}}" + +# Scheduler profile. `slurm` (default) is the amd-sglang MI355X cluster and is +# the original code path, unchanged. `spur` is the TW PIT cluster, which runs +# Spur -- a Slurm-compatible scheduler that differs in three ways that matter +# here: salloc takes no trailing command, `scontrol show hostnames` does not +# exist (its nodelist env var is already comma-expanded), and `srun --overlap` +# requires an explicit --jobid instead of inheriting one from the environment. +CLUSTER="${CLUSTER:-slurm}" +case "$CLUSTER" in + slurm|spur) ;; + *) echo "ERROR: CLUSTER must be 'slurm' or 'spur', got '$CLUSTER'" >&2; exit 1 ;; +esac + +# Spur reads a two-field time limit as HH:MM where Slurm reads MM:SS, so "30:00" +# silently becomes 30 HOURS. Require the unambiguous HH:MM:SS / D-HH:MM:SS form +# there rather than let a typo book a node for a day. +if [[ "$CLUSTER" == "spur" && ! "$TIME_LIMIT" =~ ^([0-9]+-)?[0-9]+:[0-9]{2}:[0-9]{2}$ ]]; then + echo "ERROR: on spur TIME_LIMIT must be HH:MM:SS or D-HH:MM:SS (got '$TIME_LIMIT');" >&2 + echo " a two-field value like '30:00' is parsed as 30 hours, not 30 minutes." >&2 + exit 1 +fi + +# Optional account / QoS. Empty on the mi355x cluster (which gates on partition +# alone); required on spur, where every job must name an account and its QoS. +# Both flags are spelled the same in real Slurm, so passing them is portable. +SLURM_ACCOUNT="${SLURM_ACCOUNT:-}" +SLURM_QOS="${SLURM_QOS:-}" + +# Relocate the shared HuggingFace cache root. nightly-configs.yaml pins +# model_path under /it-share/model_coverage (the mi355x cluster's NFS); a +# cluster that mirrors the same models elsewhere sets MODEL_ROOT and every +# recipe/config path is rewritten, so no recipe or config file needs editing. +MODEL_ROOT="${MODEL_ROOT:-}" +# Root used to RESOLVE the snapshot hash; defaults to MODEL_ROOT. Set it when +# MODEL_ROOT is node-local and therefore unreadable from the driver node. +MODEL_RESOLVE_ROOT="${MODEL_RESOLVE_ROOT:-$MODEL_ROOT}" +MODEL_ROOT_FROM="${MODEL_ROOT_FROM:-/it-share/model_coverage}" +relocate_model_root() { + local p="$1" + if [[ -n "$MODEL_ROOT" && "$p" == "$MODEL_ROOT_FROM"/* ]]; then + printf '%s\n' "$MODEL_ROOT/${p#"$MODEL_ROOT_FROM"/}" + else + printf '%s\n' "$p" + fi +} SGLANG_USE_CHECKOUT_RUNTIME="${SGLANG_USE_CHECKOUT_RUNTIME:-1}" case "${SGLANG_USE_CHECKOUT_RUNTIME,,}" in 0|false|no|off) SGLANG_USE_CHECKOUT_RUNTIME=0 ;; @@ -89,7 +134,15 @@ resolve_snapshot() { fi echo "$p" } +# Resolve the snapshot against the shared cache first, then relocate. MODEL_ROOT +# may name a node-local copy that does not exist on the node running this +# script; resolve_snapshot would then find no refs/main and pass the bare cache +# dir to the container. Three steps, since the resolve root and the destination +# root can differ: rewrite onto a readable root, resolve refs/main, then rewrite +# onto the root the container will see. +MODEL_PATH="$(MODEL_ROOT="$MODEL_RESOLVE_ROOT" relocate_model_root "$MODEL_PATH")" MODEL_PATH="$(resolve_snapshot "$MODEL_PATH")" || exit 1 +MODEL_PATH="$(MODEL_ROOT_FROM="$MODEL_RESOLVE_ROOT" relocate_model_root "$MODEL_PATH")" # --------------------------------------------------------------------------- # Parse the recipe (runtime + bench + topology) into shell vars. @@ -233,7 +286,33 @@ echo "recipe: image=$IMAGE attn=${ATTN:-$PATTN/$DATTN} ib=$IB ptp=$PTP dtp=$DTP # lands here; the launcher normalizes it into GITHUB_WORKSPACE afterwards. # --------------------------------------------------------------------------- WORKDIR="$HOME/.mi355x_ci/${MATRIX_CONFIG_NAME}" +if [[ "$CLUSTER" == "spur" ]]; then + # `rm -rf "$WORKDIR"` is not safe to gate a leg on over NFS: if anything still + # holds a descriptor inside (an orphaned `tail -F` on bench.log), the server + # renames the file to .nfsXXXX rather than unlinking it, so rm sees a + # non-empty directory and exits non-zero. Clear the contents instead, retry, + # and fall back to moving the directory aside. Housekeeping must not fail a + # leg. + for _try in 1 2 3; do + rm -rf "$WORKDIR"/* "$WORKDIR"/.[!.]* 2>/dev/null || true + # Anything left is a silent .nfsXXXX handle; give the server a moment. + [[ -z "$(ls -A "$WORKDIR" 2>/dev/null)" ]] && break + sleep 5 + done + mkdir -p "$WORKDIR" + if [[ -n "$(ls -A "$WORKDIR" 2>/dev/null)" ]]; then + _stale="$WORKDIR.stale.$$" + if mv "$WORKDIR" "$_stale" 2>/dev/null; then + echo "WARN: $WORKDIR would not clear (open handles); moved to $_stale" >&2 + else + WORKDIR="$WORKDIR.$$" + echo "WARN: could not clear or move workdir; using fresh $WORKDIR" >&2 + fi + mkdir -p "$WORKDIR" + fi +else rm -rf "$WORKDIR"; mkdir -p "$WORKDIR" +fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # Stage the workflow checkout on shared NFS so Slurm compute-node containers can @@ -246,8 +325,26 @@ if [[ "$SGLANG_USE_CHECKOUT_RUNTIME" == "1" ]]; then echo "Staging checkout runtime: sha=$CHECKOUT_SHA -> $CHECKOUT_STAGE" rm -rf "$CHECKOUT_STAGE" mkdir -p "$CHECKOUT_STAGE" + # See install_checkout_sglang.sh: tar exit 1 is a warning ("file changed as + # we read it", raised spuriously by NFS attribute caching), 2+ is an error. + set +e tar --exclude='__pycache__' --exclude='*.pyc' --exclude='.git/config' \ -C "$GITHUB_WORKSPACE" -cf - . | tar -C "$CHECKOUT_STAGE" -xf - + STAGE_TAR_RC=("${PIPESTATUS[@]}") + set -e + if (( STAGE_TAR_RC[0] > 1 || STAGE_TAR_RC[1] != 0 )); then + echo "ERROR: staging tar failed (create=${STAGE_TAR_RC[0]} extract=${STAGE_TAR_RC[1]})" >&2 + exit 1 + fi + # Tolerating create-rc=1 above means a warning cannot abort the copy -- but a + # TRUNCATED copy must still not reach a container, where it only surfaces + # ~15 minutes later as "invalid checkout mount". Assert the same file the + # in-container installer checks, right where the copy was made. + if [[ ! -f "$CHECKOUT_STAGE/python/sglang/version.py" ]]; then + echo "ERROR: staged checkout is incomplete (no python/sglang/version.py in $CHECKOUT_STAGE);" >&2 + echo " tar rc were create=${STAGE_TAR_RC[0]} extract=${STAGE_TAR_RC[1]}" >&2 + exit 1 + fi CHECKOUT_DOCKER_ARGS="$CHECKOUT_DOCKER_ARGS -e SGLANG_CHECKOUT_SHA=$CHECKOUT_SHA -v $CHECKOUT_STAGE:/sglang-checkout:ro" else echo "SGLANG_USE_CHECKOUT_RUNTIME=0; using sglang package baked into image." @@ -293,7 +390,37 @@ DSV4_ENV_STR="${DSV4_ENV[*]}" # A recipe carrying a `model:` block supplies its OWN docker env (below), so the # DSV4 env must not leak into it; the DSV4 recipes keep the string above. [[ "$HAS_MODEL" == "1" ]] && DSV4_ENV_STR="" -MORI_ENV="-e MORI_DISABLE_AUTO_XGMI=1 -e NCCL_IB_HCA=ionic -e NCCL_IB_GID_INDEX=1 -e NCCL_CROSS_NIC=1" +# NCCL_IB_HCA must name real devices. "ionic" is the mi355x cluster's spelling; +# on pit2 the HCAs are rdma0..7 (the recipe's $IB) and "ionic" matches nothing, +# which fails TP group init outright with +# RuntimeError: NCCL error: remote process exited or there was a network error +# from ncclCommInitRank. The wide-EP block below already overrides this for +# EP>8; EP<=8 recipes need it too. +MORI_NCCL_HCA=ionic +[[ "$CLUSTER" == "spur" ]] && MORI_NCCL_HCA="$IB" +MORI_ENV="-e MORI_DISABLE_AUTO_XGMI=1 -e NCCL_IB_HCA=$MORI_NCCL_HCA -e NCCL_IB_GID_INDEX=1 -e NCCL_CROSS_NIC=1" +# Whether NCCL may use IB depends on the role, not on the recipe. A single-node +# engine does all its NCCL traffic intra-node (cross-node KV goes over mori), +# and on pit2 letting it reach for IB hangs ncclCommInitRank. Gating on the +# whole recipe is wrong for 2P1D-EP16 (PN_PER=1, DN_PER=2), whose prefill +# engines are single-node TP8 but whose recipe-wide test left IB on for both +# roles. +# +# Both cases still need a socket interface: NCCL uses one for its bootstrap +# handshake even on an intra-node data path, and pit2 exposes eight tw-eth* /31 +# links that auto-selection picks and hangs on. Wide recipes set it in the block +# below; pin it here for any role that block does not cover. +SPUR_PNCCL="" +SPUR_DNCCL="" +if [[ "$CLUSTER" == "spur" ]]; then + SPUR_SOCK="${DIST_SOCK:-${ADDR_NIC_OVERRIDE:-eno0}}" + if (( PN_PER == 1 )); then + SPUR_PNCCL=" -e NCCL_IB_DISABLE=1 -e NCCL_SOCKET_IFNAME=$SPUR_SOCK -e GLOO_SOCKET_IFNAME=$SPUR_SOCK" + fi + if (( DN_PER == 1 )); then + SPUR_DNCCL=" -e NCCL_IB_DISABLE=1 -e NCCL_SOCKET_IFNAME=$SPUR_SOCK -e GLOO_SOCKET_IFNAME=$SPUR_SOCK" + fi +fi # Wide-EP (engine spans >1 node) adds mori all-to-all MoE tuning + the cross-node # torch-dist socket NIC. Gated on nodes-per-engine>1 so EP<=8 recipes are untouched. if (( PN_PER > 1 || DN_PER > 1 )); then @@ -411,7 +538,9 @@ if [[ "$MTP_ENABLED" == "1" ]]; then # EAGLE3 (and other draft-model algos) need an external draft checkpoint; # built-in EAGLE (DSV4) omits draft_model_path and this stays unset. if [[ -n "$MTP_DRAFT_PATH" ]]; then - DRAFT_RESOLVED="$(resolve_snapshot "$MTP_DRAFT_PATH")" || exit 1 + _d="$(MODEL_ROOT="$MODEL_RESOLVE_ROOT" relocate_model_root "$MTP_DRAFT_PATH")" + _d="$(resolve_snapshot "$_d")" || exit 1 + DRAFT_RESOLVED="$(MODEL_ROOT_FROM="$MODEL_RESOLVE_ROOT" relocate_model_root "$_d")" EXTRA_COMMON="$EXTRA_COMMON --speculative-draft-model-path $DRAFT_RESOLVED" fi fi @@ -466,20 +595,87 @@ else --disaggregation-transfer-backend $XFER --disaggregation-ib-device $IB$KV_FLAG$DECODE_TAIL" fi -DOCKER_COMMON="--rm --network host --ipc host --shm-size 32g --privileged \ +# /it-share is the mi355x cluster's model NFS. Mount it only where it exists -- +# docker would otherwise silently create a root-owned empty /it-share on every +# node of a cluster that keeps its models elsewhere (see MODEL_ROOT). +IT_SHARE_MOUNT="-v /it-share:/it-share:ro " +[[ -d /it-share ]] || IT_SHARE_MOUNT="" +# A cluster that relocates the model cache must bind that root in too, at the +# same path: --model-path is resolved inside the container, and a missing +# directory does not fail as "no such file" -- transformers falls back to +# treating it as a HuggingFace repo id and dies with a confusing +# "Repo id must be in the form 'repo_name' or 'namespace/repo_name'". +if [[ -n "$MODEL_ROOT" && "$MODEL_ROOT" != /it-share/* ]]; then + IT_SHARE_MOUNT="$IT_SHARE_MOUNT-v $MODEL_ROOT:$MODEL_ROOT:ro " +fi +# Slurm hosts each server as a long-lived job step and reaps the container when +# the step ends, so --rm is right there. Spur has no equivalent step (its +# --overlap path is an interactive exec-into-job), so the container runs +# DETACHED under dockerd and drive.sh polls it -- which means it must NOT be +# --rm, or the exit code is gone before the poller can read it. Teardown removes +# it explicitly instead. +DOCKER_LIFECYCLE="--rm" +[[ "$CLUSTER" == "spur" ]] && DOCKER_LIFECYCLE="-d" +# pit2 runs rootless docker, whose spur-authz plugin refuses --privileged +# outright ("denied [P1]"). Spell out what --privileged was actually buying us: +# the device nodes (already listed below), the groups that own kfd/dri, and two +# capabilities -- IPC_LOCK, which RDMA memory registration needs, and SYS_PTRACE +# for profiling. Verified on g11: ibv_devinfo sees all 8 HCAs PORT_ACTIVE and +# --ipc/--network host and --shm-size are all permitted under this set. +PRIV_ARGS="--privileged" +if [[ "$CLUSTER" == "spur" ]]; then + PRIV_ARGS="--group-add video --group-add render --cap-add IPC_LOCK --cap-add SYS_PTRACE" +fi +DOCKER_COMMON="$DOCKER_LIFECYCLE --network host --ipc host --shm-size 32g $PRIV_ARGS \ --security-opt seccomp=unconfined \ --device /dev/kfd --device /dev/dri --device /dev/infiniband \ --v /it-share:/it-share:ro -v $HOME:/host_home $CHECKOUT_DOCKER_ARGS" +${IT_SHARE_MOUNT}-v $HOME:/host_home $CHECKOUT_DOCKER_ARGS" # Optional extra docker args (e.g. bind-mounting a locally-built lib for # validation). Empty by default so the docker argv is byte-identical otherwise. [[ -n "${EXTRA_DOCKER_ARGS:-}" ]] && DOCKER_COMMON="$DOCKER_COMMON ${EXTRA_DOCKER_ARGS}" +# Spur compute nodes run prolog/epilog hooks that reap any container not tagged +# with the owning job, so an untagged server is SIGKILLed (rc=137) seconds after +# it starts. Single-quoted so the literal $SPUR_JOB_ID survives into the +# generated per-role script and expands on the compute node, where the job +# environment actually defines it. +if [[ "$CLUSTER" == "spur" ]]; then + DOCKER_COMMON="$DOCKER_COMMON "'--label spur_job_id=$SPUR_JOB_ID' +fi + # Per-role wide-EP docker env (MORI dispatch-token tuning etc.). Empty for EP<=8 # recipes; carries its own leading space so an empty value leaves the docker argv # byte-identical (no stray double space). PENV_ARG=""; [[ -n "$PENV" ]] && PENV_ARG=" $PENV" DENV_ARG=""; [[ -n "$DENV" ]] && DENV_ARG=" $DENV" +# Pin the torch-dist rendezvous port for single-node engines. Without +# --nccl-port, get_free_port() binds port 0, closes the socket, and rank 0 +# re-binds the same number later for the TCPStore; anything handed that port in +# between makes rank 0 die with EADDRINUSE. It reproduces rather than being a +# lottery, since the kernel walks its ephemeral range in order. +# +# Only single-node engines need this -- a multi-node engine gets +# --dist-init-addr, which _resolve_dist_init_method() prefers over the port. +# The value is the recipe's own per-role port +1000, which clears the server +# port and stays under the 32768 ephemeral floor. These recipes place at most +# one single-node engine per node, so two engines cannot collide. +# +# Pinning is safe here because --dp-size is only emitted together with +# --enable-dp-attention, and that path gives every DP rank the same nccl port. +# launch_dp_schedulers(), which needs a distinct port per worker, is never +# reached by these recipes. +PNCCL_ARG="" +DNCCL_ARG="" +if [[ "$CLUSTER" == "spur" ]]; then + PNCCL_ARG=" --nccl-port $((PPORT + 1000))" + DNCCL_ARG=" --nccl-port $((DPORT + 1000))" +fi +# Per-role NCCL IB/socket policy (spur only; empty under slurm, so the argv +# there is unchanged). Appended after $PENV/$DENV so docker's last-wins applies. +PENV_ARG="$PENV_ARG$SPUR_PNCCL" +DENV_ARG="$DENV_ARG$SPUR_DNCCL" + # --------------------------------------------------------------------------- # Write per-role scripts that srun dispatches to each compute node. # --------------------------------------------------------------------------- @@ -511,8 +707,20 @@ fi echo "[checkout-sglang] reinstalling sglang from $CHECKOUT_SRC" rm -rf "$RUNTIME_CHECKOUT" mkdir -p "$RUNTIME_CHECKOUT" +# tar exits 1 for warnings -- notably "file changed as we read it", which NFS +# attribute caching raises spuriously on a tree nothing is writing (seen on the +# pit2 shared-NFS checkout). Only 2+ is a real error, and the extracting tar +# must still succeed outright. Without this, set -o pipefail aborts the install +# on a warning and the server container exits before it ever starts. +set +e tar --exclude='__pycache__' --exclude='*.pyc' \ -C "$CHECKOUT_SRC" -cf - . | tar -C "$RUNTIME_CHECKOUT" -xf - +_tar_rc=("${PIPESTATUS[@]}") +set -e +if (( _tar_rc[0] > 1 || _tar_rc[1] != 0 )); then + echo "[checkout-sglang] ERROR: staging tar failed (create=${_tar_rc[0]} extract=${_tar_rc[1]})" >&2 + exit 1 +fi git config --global --add safe.directory "$RUNTIME_CHECKOUT" || true @@ -646,6 +854,10 @@ fi DIST_ARGS="" if [[ "\${NNODES:-1}" != "1" ]]; then DIST_ARGS="--nnodes \$NNODES --node-rank \$NODE_RANK --dist-init-addr \$DIST_ADDR:\${DIST_PORT:-29500}" +else + # Single-node engine: no --dist-init-addr, so the rendezvous port is the one + # sglang would otherwise pick with get_free_port(). Pin it. Empty on slurm. + DIST_ARGS="$PNCCL_ARG" fi exec python3 -m sglang.launch_server \ --model-path $MODEL_PATH --host 0.0.0.0 --port $PPORT \ @@ -665,6 +877,10 @@ fi DIST_ARGS="" if [[ "\${NNODES:-1}" != "1" ]]; then DIST_ARGS="--nnodes \$NNODES --node-rank \$NODE_RANK --dist-init-addr \$DIST_ADDR:\${DIST_PORT:-29500}" +else + # Single-node engine: no --dist-init-addr, so the rendezvous port is the one + # sglang would otherwise pick with get_free_port(). Pin it. Empty on slurm. + DIST_ARGS="$DNCCL_ARG" fi exec python3 -m sglang.launch_server \ --model-path $MODEL_PATH --host 0.0.0.0 --port $DPORT \ @@ -715,7 +931,7 @@ fi exec python3 -m sglang.launch_server \ --model-path $MODEL_PATH --host 0.0.0.0 --port $PPORT \ $PREFILL_COMMON_FLAGS "\${MODEL_SERVER_ARGS[@]}" \ - --disaggregation-mode prefill --disaggregation-bootstrap-port $PBOOT + --disaggregation-mode prefill --disaggregation-bootstrap-port $PBOOT$PNCCL_ARG EOF cat > "$WORKDIR/decode_entry.sh" < "$WORKDIR/prefill.sh" < "$WORKDIR/drive.sh" <<'DRIVE' #!/bin/bash set -x WORKDIR="$1"; PW="${2:-1}"; DW="${3:-1}"; PN_PER="${4:-1}"; DN_PER="${5:-1}"; DIST_NIC="${6:-}" -mapfile -t NODES < <(scontrol show hostnames "$SLURM_JOB_NODELIST") +# Interface resolve_ip() falls back to when the recipe names no dist NIC (every +# EP<=8 recipe). Empty on Slurm, where drive.sh runs on the login node and never +# resolves a node it is running on; set on Spur, where it runs on an allocated +# node and that node's own name resolves to 127.0.1.1 via Ubuntu's stock +# /etc/hosts -- useless as an address to advertise to the other engine. +ADDR_NIC="${7:-}" +# Scheduler seam (see the CLUSTER block at the top of launch_mi355x.sh). Slurm +# hands out a bracket-compressed hostlist that only `scontrol show hostnames` +# can expand, and srun inherits the allocation from the environment. Spur's +# nodelist env var is already comma-expanded, has no `show hostnames`, and its +# `srun --overlap` requires an explicit --jobid. +CLUSTER="${CLUSTER:-slurm}" +if [[ "$CLUSTER" == "spur" ]]; then + SRUN=(srun --jobid="$SPUR_JOB_ID" --overlap) + sched_hostnames() { tr ',' '\n' <<< "$SPUR_JOB_NODELIST" | sed '/^$/d'; } +else + SRUN=(srun --overlap) + sched_hostnames() { scontrol show hostnames "$SLURM_JOB_NODELIST"; } +fi +mapfile -t NODES < <(sched_hostnames) +# Dispatch one long-running, log-producing step to a node. +# +# Spur allocates a pty for every --overlap step. drive.sh runs the servers and +# the bench as BACKGROUND subshells, and a background process group writing +# through that pty is stopped by SIGTTOU -- the step then dies (marker rc=137) +# and everything it printed after the first few bytes is lost, which is why a +# failing leg used to leave a log containing nothing but the srun banner. Having +# srun write the file itself keeps the step's output off the pty entirely. +# Slurm has no pty here, so it keeps the original shell redirect. +srun_log() { # srun_log + local log="$1" node="$2"; shift 2 + # On spur drive.sh runs on one of the allocated nodes, so a dispatch to that + # node would be srun exec-ing into its own job -- the one spur case that has + # repeatedly misbehaved. Run it directly instead; the result is identical and + # it removes a whole class of scheduler interaction. + if [[ "$CLUSTER" == "spur" && "$node" == "$(hostname)" ]]; then + "$@" > "$log" 2>&1 + return + fi + "${SRUN[@]}" -N1 --nodelist="$node" "$@" > "$log" 2>&1 +} + +srun_local_or_step() { # same rule, for short commands whose output we capture + local node="$1"; shift + if [[ "$CLUSTER" == "spur" && "$node" == "$(hostname)" ]]; then + "$@" + return + fi + "${SRUN[@]}" -N1 --nodelist="$node" "$@" +} + +# --- spur: containers are detached, liveness comes from dockerd --------------- +# Spur cannot host a long-lived process in a job step, so on spur the servers +# and the bench run as DETACHED containers and these helpers derive the same +# server_exit_*/bench_exit markers the monitor loop already consumes -- from +# real container state rather than from srun's status. Every srun issued here is +# a sub-second foreground command, which is the part of spur that behaves. +declare -A SPUR_SEEN=() # container -> 1 once observed running, so a container + # that has not started yet is not mistaken for a crash +declare -A SPUR_ABSENT=() # container -> consecutive polls that found it gone +SPUR_SERVERS=() # "role:node:container" +SPUR_POLLS=0 +SPUR_ABSENT_LIMIT=3 # polls (10s apart) before "gone" is believed + +container_state() { # -> "running" | "" | absent | unknown + # The obvious `docker inspect | tail -1` cannot be trusted on spur. drive.sh + # runs on one allocated node, so every poll of a container on the OTHER node + # is an `srun --overlap` step, and those intermittently fail to spawn + # ("step command failed to spawn: Permission denied (os error 13)"). A failed + # step and a genuinely absent container are the same observation -- nonzero + # exit, empty stdout -- so one flaky poll was enough to declare a healthy + # server dead. Observed on dsv4flash-fp8-1k1k-1p1d: prefill marked + # "rc=missing" while `docker ps` on that node still showed it Up. + # + # The __ok__ sentinel is emitted only if the remote command actually ran, so a + # dispatch failure is reported as "unknown" and the caller ignores it rather + # than counting it as evidence. + local out + out="$(srun_local_or_step "$1" bash -c \ + "docker inspect -f '{{if .State.Running}}running{{else}}{{.State.ExitCode}}{{end}}' $2 2>/dev/null || echo absent; echo __ok__" \ + 2>/dev/null | tr -d '\r')" + [[ "$out" == *__ok__* ]] || { echo unknown; return; } + grep -vxE '__ok__|' <<< "$out" | tail -1 +} + +container_log() { # + # Fetch to a temp file and never let a failed fetch shrink the log: the same + # flaky step that produced the false "missing" also truncated the 5-line + # prefill log to 0 bytes, destroying the only evidence of what happened. + local tmp="$3.fetch" + srun_local_or_step "$1" docker logs "$2" > "$tmp" 2>/dev/null || true + # Publish in place (truncate + rewrite the SAME inode), never by renaming over + # the target. drive.sh keeps a `tail -F` on bench.log, and swapping the inode + # every 10s made tail reopen and re-emit the whole file each poll -- the + # driver log reached 177 MB and legs took 2 h instead of 36 min. + if [[ -s "$tmp" || ! -f "$3" ]]; then cat "$tmp" > "$3"; fi + rm -f "$tmp" +} + +spur_rm() { # -- remove, retrying past a flaky dispatch + local n="$1"; shift + local i + for i in 1 2 3; do + srun_local_or_step "$n" bash -c "docker rm -f $* >/dev/null 2>&1; echo __ok__" \ + 2>/dev/null | grep -q __ok__ && return 0 + sleep 3 + done + return 1 +} + +# Refresh one container's log and write its marker once it is no longer running. +spur_track() { # + local node="$1" name="$2" log="$3" marker="$4" label="$5" st + container_log "$node" "$name" "$log" + st="$(container_state "$node" "$name")" + if [[ "$st" == "running" ]]; then + SPUR_SEEN["$name"]=1 + SPUR_ABSENT["$name"]=0 + return + fi + # A step that never ran says nothing about the container. + [[ "$st" == "unknown" ]] && return + if [[ "$st" == "absent" || -z "$st" ]]; then + # Not created yet, or one poll that came back empty. Only a container that + # is absent on SPUR_ABSENT_LIMIT polls in a row is treated as a crash, so + # neither a slow `docker run -d` nor a single flaky poll ends the leg. + SPUR_ABSENT["$name"]=$(( ${SPUR_ABSENT[$name]:-0} + 1 )) + if [[ -z "${SPUR_SEEN[$name]:-}" && "$SPUR_POLLS" -lt 6 ]]; then return; fi + [[ "${SPUR_ABSENT[$name]}" -lt "$SPUR_ABSENT_LIMIT" ]] && return + st="missing" + fi + echo "$label rc=$st" > "$marker" +} + +spur_refresh_markers() { + local entry role node cname st + SPUR_POLLS=$((SPUR_POLLS + 1)) + for entry in "${SPUR_SERVERS[@]}"; do + IFS=: read -r role node cname <<< "$entry" + [[ -f "$WORKDIR/server_exit_${role}_${node}" ]] && continue + spur_track "$node" "$cname" "$WORKDIR/${role}_${node}.log" \ + "$WORKDIR/server_exit_${role}_${node}" "$role@$node" + done + # The bench marker holds a bare exit code, not the "role@node rc=" form. + container_log "$PNODE" mi355x_bench "$WORKDIR/bench.log" + st="$(container_state "$PNODE" mi355x_bench)" + if [[ "$st" == "running" ]]; then + SPUR_SEEN[mi355x_bench]=1 + SPUR_ABSENT[mi355x_bench]=0 + elif [[ "$st" == "unknown" ]]; then + : + elif [[ "$st" == "absent" || -z "$st" ]]; then + SPUR_ABSENT[mi355x_bench]=$(( ${SPUR_ABSENT[mi355x_bench]:-0} + 1 )) + if [[ -n "${SPUR_SEEN[mi355x_bench]:-}" \ + && "${SPUR_ABSENT[mi355x_bench]}" -ge "$SPUR_ABSENT_LIMIT" ]]; then + echo 1 > "$WORKDIR/bench_exit" + fi + else + echo "$st" > "$WORKDIR/bench_exit" + fi +} # Resolve a node's IP. When a cross-node dist NIC is named (wide engines set # $DIST_NIC to the recipe's dist_socket_ifname), read the address off that # interface on the node itself -- the multi-node dist-init-addr must be a @@ -873,20 +1249,54 @@ mapfile -t NODES < <(scontrol show hostnames "$SLURM_JOB_NODELIST") # root whose hostname resolved to an unpingable IP, ZMQ bind => "Cannot assign # requested address"). Fall back to Slurm's NodeAddr, then DNS. resolve_ip() { - local n="$1" ip="" addr="" field - if [[ -n "$DIST_NIC" ]]; then - ip=$(srun --overlap -N1 --nodelist="$n" ip -4 -o addr show "$DIST_NIC" 2>/dev/null \ - | awk '{print $4}' | cut -d/ -f1 | head -1) + local n="$1" ip="" addr="" field nic="${DIST_NIC:-$ADDR_NIC}" + if [[ -n "$nic" ]]; then + # Go through srun_local_or_step, not a bare srun: on spur drive.sh runs on an + # allocated node, and resolving its OWN node through `srun --overlap` is the + # dispatch that has repeatedly misbehaved. Here it did not fail, it HUNG -- + # a leg sat 30 min on `srun ... ip -4 -o addr show eno0` for the node it was + # already running on, before any container was started. The helper runs it + # directly in that case. The timeout bounds the genuinely remote call so a + # wedged step costs 60s and a DNS fallback instead of the whole leg. + if [[ "$CLUSTER" == "spur" ]]; then + if [[ "$n" == "$(hostname)" ]]; then + ip=$(command ip -4 -o addr show "$nic" 2>/dev/null \ + | awk '{print $4}' | cut -d/ -f1 | head -1) + else + ip=$(timeout 60 "${SRUN[@]}" -N1 --nodelist="$n" ip -4 -o addr show "$nic" 2>/dev/null \ + | awk '{print $4}' | cut -d/ -f1 | head -1) + fi + else + ip=$("${SRUN[@]}" -N1 --nodelist="$n" ip -4 -o addr show "$nic" 2>/dev/null \ + | awk '{print $4}' | cut -d/ -f1 | head -1) + fi fi if [[ -z "$ip" ]]; then - for field in $(scontrol show node "$n" -o); do - case "$field" in - NodeAddr=*) addr="${field#NodeAddr=}"; break ;; - esac - done + # Spur's `scontrol show node` does not emit Slurm's NodeAddr= field, so the + # loop below would just fall through; skip it there and go straight to DNS. + if [[ "$CLUSTER" != "spur" ]]; then + for field in $(scontrol show node "$n" -o); do + case "$field" in + NodeAddr=*) addr="${field#NodeAddr=}"; break ;; + esac + done + fi [[ -n "$addr" ]] || addr="$n" read -r ip _ < <(getent ahostsv4 "$addr") fi + # A loopback or empty answer is never a usable advertised address. Fail here + # rather than let a server bind 127.x (or nothing) and have the peer engine + # time out connecting to it with no obvious cause. Callers must use + # `x=$(resolve_ip n) || exit 1` -- the status is visible at the assignment, + # but an `exit` inside the substitution would only leave the subshell. + if [[ -z "$ip" ]]; then + echo "[drive] ERROR: no address found for $n" >&2 + return 1 + fi + if [[ "$ip" == 127.* ]]; then + echo "[drive] ERROR: $n resolved to loopback $ip; set ADDR_NIC to a real interface" >&2 + return 1 + fi printf '%s\n' "$ip" } # SLURM canonicalizes (sorts) SLURM_JOB_NODELIST, so the requested --nodelist @@ -913,8 +1323,8 @@ PN_TOTAL=$((PW * PN_PER)); DN_TOTAL=$((DW * DN_PER)) PNODES=("${NODES[@]:0:PN_TOTAL}") DNODES=("${NODES[@]:PN_TOTAL:DN_TOTAL}") PNODE="${PNODES[0]}"; DNODE="${DNODES[0]}" -PIP=$(resolve_ip "$PNODE") -DIP=$(resolve_ip "$DNODE") +PIP=$(resolve_ip "$PNODE") || exit 1 +DIP=$(resolve_ip "$DNODE") || exit 1 echo "[drive] prefill nodes: ${PNODES[*]} ; decode nodes: ${DNODES[*]}" echo "[drive] bench targets prefill=$PNODE($PIP) decode=$DNODE($DIP)" if (( DW > 1 )); then @@ -928,6 +1338,17 @@ fi # died and with what code. (A hung-but-alive server is NOT caught here; that is # bounded by bench.sh's health-wait timeout.) rm -f "$WORKDIR"/server_exit_* "$WORKDIR/bench_exit" +if [[ "$CLUSTER" == "spur" ]]; then + # Detached containers are not --rm, and a container whose docker client was + # killed can survive as an exited husk. Clear all three names on every + # allocated node first: otherwise `docker run` fails with a name conflict and, + # worse, the poller reads the STALE container's exit code and reports a crash + # that never happened. + for n in "${NODES[@]}"; do + srun_local_or_step "$n" \ + docker rm -f mi355x_prefill mi355x_decode mi355x_bench >/dev/null 2>&1 || true + done +fi # Launch PW prefill engines; each spans PN_PER nodes as its own torch-dist group # (engine node0 = dist-init addr; node_rank = position WITHIN the engine, so an # engine's ranks are 0..PN_PER-1, not a global index). PN_PER=1 => NNODES=1 in @@ -936,21 +1357,34 @@ rm -f "$WORKDIR"/server_exit_* "$WORKDIR/bench_exit" PCSV="" for ((k=0; k "$WORKDIR/prefill_$n.log" 2>&1 + if [[ "$CLUSTER" == "spur" ]]; then + # Returns as soon as the container is detached; liveness is polled later. + srun_log "$WORKDIR/prefill_start_$n.log" "$n" \ + bash "$WORKDIR/prefill.sh" "$j" "$PN_PER" "$eip" || true + SPUR_SERVERS+=("prefill:$n:mi355x_prefill") + else + ( srun_log "$WORKDIR/prefill_$n.log" "$n" bash "$WORKDIR/prefill.sh" "$j" "$PN_PER" "$eip" echo "prefill@$n rc=$?" > "$WORKDIR/server_exit_prefill_$n" ) & + fi done done for ((k=0; k "$WORKDIR/decode_$n.log" 2>&1 + if [[ "$CLUSTER" == "spur" ]]; then + srun_log "$WORKDIR/decode_start_$n.log" "$n" \ + bash "$WORKDIR/decode.sh" "$j" "$DN_PER" "$eip" || true + SPUR_SERVERS+=("decode:$n:mi355x_decode") + else + ( srun_log "$WORKDIR/decode_$n.log" "$n" bash "$WORKDIR/decode.sh" "$j" "$DN_PER" "$eip" echo "decode@$n rc=$?" > "$WORKDIR/server_exit_decode_$n" ) & + fi done done echo "[drive] prefill engine endpoints (fan-out): $PCSV" @@ -958,9 +1392,15 @@ sleep 5 # Bench in the background with its own marker, so the wait loop is purely file # based: finish when bench writes its marker, abort if any server marker shows up # first (a server died before the sweep completed). -( srun --overlap -N1 --nodelist="$PNODE" bash "$WORKDIR/bench.sh" "$PIP" "$DIP" "$PCSV" > "$WORKDIR/bench.log" 2>&1 +BENCH_BG="" +if [[ "$CLUSTER" == "spur" ]]; then + srun_log "$WORKDIR/bench_start.log" "$PNODE" \ + bash "$WORKDIR/bench.sh" "$PIP" "$DIP" "$PCSV" || true +else +( srun_log "$WORKDIR/bench.log" "$PNODE" bash "$WORKDIR/bench.sh" "$PIP" "$DIP" "$PCSV" echo $? > "$WORKDIR/bench_exit" ) & BENCH_BG=$! +fi # Stream bench output live and poll the markers with xtrace OFF, so the console # shows clean benchmark/accuracy output instead of a compgen/sleep trace every # 10s. (Mirrors NVIDIA's launch_gb200.sh, which set +x around its log stream.) @@ -970,10 +1410,13 @@ TAIL_PID=$! set +x RC=0 while [[ ! -f "$WORKDIR/bench_exit" ]]; do + # On spur the markers are produced here, from container state, instead of by a + # backgrounded srun. Everything below is unchanged. + [[ "$CLUSTER" == "spur" ]] && spur_refresh_markers if compgen -G "$WORKDIR/server_exit_*" > /dev/null; then echo "[drive] ERROR: a server exited early before bench finished:" cat "$WORKDIR"/server_exit_* || true - kill "$BENCH_BG" 2>/dev/null || true + [[ -n "$BENCH_BG" ]] && kill "$BENCH_BG" 2>/dev/null || true RC=1 break fi @@ -983,8 +1426,19 @@ set -x kill "$TAIL_PID" 2>/dev/null || true [[ "$RC" -eq 0 ]] && RC=$(cat "$WORKDIR/bench_exit" 2>/dev/null || echo 1) echo "[drive] bench finished (rc=$RC), tearing down" -for n in "${PNODES[@]}"; do srun --overlap -N1 --nodelist="$n" docker kill mi355x_prefill >/dev/null 2>&1 || true; done -for n in "${DNODES[@]}"; do srun --overlap -N1 --nodelist="$n" docker kill mi355x_decode >/dev/null 2>&1 || true; done +if [[ "$CLUSTER" == "spur" ]]; then + # Detached containers are not --rm, so kill is not enough -- remove them, and + # the bench container too, or the next leg on this node inherits them. + # Retry the removals: a step that fails to spawn used to leave the containers + # running, and the next leg on that node then inherited engines holding all + # eight GPUs. + for n in "${PNODES[@]}"; do spur_rm "$n" mi355x_prefill || true; done + for n in "${DNODES[@]}"; do spur_rm "$n" mi355x_decode || true; done + spur_rm "$PNODE" mi355x_bench || true +else +for n in "${PNODES[@]}"; do "${SRUN[@]}" -N1 --nodelist="$n" docker kill mi355x_prefill >/dev/null 2>&1 || true; done +for n in "${DNODES[@]}"; do "${SRUN[@]}" -N1 --nodelist="$n" docker kill mi355x_decode >/dev/null 2>&1 || true; done +fi exit "$RC" DRIVE chmod +x "$WORKDIR/drive.sh" @@ -1002,6 +1456,11 @@ EXCLUSIVE_ARG=() EXCLUDE_ARG=() [[ -n "${SLURM_EXCLUDE:-}" ]] && EXCLUDE_ARG=(--exclude="$SLURM_EXCLUDE") +# Optional account / QoS (spur requires both; mi355x sets neither). +ACCT_ARG=() +[[ -n "$SLURM_ACCOUNT" ]] && ACCT_ARG+=(-A "$SLURM_ACCOUNT") +[[ -n "$SLURM_QOS" ]] && ACCT_ARG+=(-q "$SLURM_QOS") + # Nodes = sum over engines of nodes-per-engine. EP<=8 (PN_PER=DN_PER=1) gives the # original PW+DW (1P1D -> 2 nodes); wide EP16 1P1D gives 2+2 = 4 nodes. TOTAL_NODES=$(( PW * PN_PER + DW * DN_PER )) @@ -1018,11 +1477,185 @@ fi # GITHUB_RUN_ID + config make the name unique per matrix leg regardless. JOB_NAME="mi355x-ci-${RUNNER_NAME:-norunner}-${GITHUB_RUN_ID:-0}-${MATRIX_CONFIG_NAME}" +# Address-resolution fallback NIC for drive.sh (see ADDR_NIC there). Only spur +# needs one, because only there does drive.sh run on an allocated node. +ADDR_NIC="" +[[ "$CLUSTER" == "spur" ]] && ADDR_NIC="${ADDR_NIC_OVERRIDE:-eno0}" + set +e -salloc -p "$SLURM_PARTITION" -N"$TOTAL_NODES" "${NODELIST_ARG[@]}" "${EXCLUDE_ARG[@]}" "${EXCLUSIVE_ARG[@]}" \ - --job-name "$JOB_NAME" -t "$TIME_LIMIT" \ - bash "$WORKDIR/drive.sh" "$WORKDIR" "$PW" "$DW" "$PN_PER" "$DN_PER" "$DIST_SOCK" -SALLOC_RC=$? +if [[ "$CLUSTER" == "spur" ]]; then + # Spur's salloc takes no trailing command, so drive.sh is submitted as a + # batch job instead. That runs it on the first allocated node rather than + # the login node, which is fine: $WORKDIR and $GITHUB_WORKSPACE are on + # shared NFS, and drive.sh only ever reaches the other nodes through srun. + # We stream the batch output back live and take the exit code from a file + # drive_batch.sh writes, since sbatch returns as soon as the job is queued. + cat > "$WORKDIR/drive_batch.sh" </dev/null; then + # Must BLOCK, not exit: spur starts tearing the job down as soon as one of its + # tasks finishes, and a job in Completing rejects further steps + # ("CreateJobStep failed: job N is not running") -- which killed the decode + # launch. Holding here keeps the node in the allocation until rank 0 is done. + echo "[drive_batch] \$(hostname) standing by; another task drives" >> "$WORKDIR/standby.log" + while [[ ! -f "$WORKDIR/drive_exit" ]]; do sleep 10; done + exit 0 +fi +echo "[drive_batch] \$(hostname) rank \${SPUR_NODEID:-?} elected driver" +# Send the driver's output to a private per-node file, not to the shared +# --output. All tasks of the job write sbatch.out concurrently, so it loses +# records: a leg that failed in the monitor loop left an sbatch.out that simply +# stopped mid-run, with the teardown lines never visible. +# +# A plain redirect, deliberately NOT `| tee`. With a pipeline, drive_batch waits +# for the whole pipeline, and that does not finish when drive.sh does -- the +# backgrounded `tail -F` on bench.log inherits drive.sh's stdout and holds the +# pipe's write end open. drive.sh exited 0 and drive_exit was still unwritten +# 11 minutes later, so the standby tasks kept the allocation alive and the leg +# looked hung after it had actually passed. +bash "$WORKDIR/drive.sh" "$WORKDIR" "$PW" "$DW" "$PN_PER" "$DN_PER" "$DIST_SOCK" "$ADDR_NIC" \ + > "$WORKDIR/drive_\$(hostname).log" 2>&1 +echo \$? > "$WORKDIR/drive_exit" +EOF + chmod +x "$WORKDIR/drive_batch.sh" + rm -f "$WORKDIR/drive_exit" + SBATCH_OUT="$WORKDIR/sbatch.out" + : > "$SBATCH_OUT" + # Node count is what matters here; do NOT pass -n1 to make the task count + # match. Spur sizes the allocation from the task count, so -n1 collapses a + # 2-node request to a single node and the decode engine has nowhere to run. + # Uniqueness of the driver is handled in drive_batch.sh by an atomic lock, + # not by the task count. + # Spur's controller is a Raft cluster, so submission can fail for reasons + # that have nothing to do with the request: during a leader election it + # answers `The service is currently unavailable ... "not the Raft leader"`. + # That is transient and clears in seconds, but an unretried submit turns it + # into a failed leg. Worse, the message is multi-line, so the ${VAR##* } + # job-id parse below extracted `leader"` from it and the leg was reported as + # a benchmark failure with an empty bench.log -- indistinguishable, at a + # glance, from a model bug. Observed on glm52-fp4-1k1k-2p1d-ep16 2026-09-17. + # Retry only on that signature; a genuinely bad sbatch request must still + # fail immediately rather than being retried six times. + SPUR_LEG_JOB_ID="" + for _attempt in 1 2 3 4 5 6; do + SBATCH_MSG=$(sbatch -p "$SLURM_PARTITION" -N"$TOTAL_NODES" "${NODELIST_ARG[@]}" \ + "${EXCLUDE_ARG[@]}" "${EXCLUSIVE_ARG[@]}" "${ACCT_ARG[@]}" \ + --job-name "$JOB_NAME" -t "$TIME_LIMIT" \ + --output "$SBATCH_OUT" --error "$SBATCH_OUT" \ + "$WORKDIR/drive_batch.sh" 2>&1) + echo "$SBATCH_MSG" + SPUR_LEG_JOB_ID="${SBATCH_MSG##* }" + [[ "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]] && break + if [[ "$SBATCH_MSG" == *"not the Raft leader"* \ + || "$SBATCH_MSG" == *"service is currently unavailable"* ]]; then + echo "[launch] spur controller unavailable (attempt $_attempt/6); retrying in 20s" >&2 + SPUR_LEG_JOB_ID="" + sleep 20 + continue + fi + break + done + if [[ ! "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]]; then + echo "ERROR: could not parse a job id out of: $SBATCH_MSG" >&2 + SALLOC_RC=1 + else + echo "[launch] spur job $SPUR_LEG_JOB_ID submitted; streaming $SBATCH_OUT" + tail -F "$SBATCH_OUT" 2>/dev/null & + SPUR_TAIL_PID=$! + # drive_batch.sh writes drive_exit last, so that file is the primary + # completion signal; the squeue check catches a job that died without + # ever running it (node failure, scheduler kill, time limit). + while :; do + [[ -f "$WORKDIR/drive_exit" ]] && break + if ! squeue -h -o "%i" 2>/dev/null | grep -qx "$SPUR_LEG_JOB_ID"; then + # The job has left the queue. drive_exit is written on a compute + # node and read here on the login node, so NFS close-to-open + # visibility can delay it well past a single short sleep. A five + # second grace produced three false reds on 2026-09-16, each on a + # leg that had finished every concurrency point and passed the + # accuracy gate; one of them was the whole GLM EP16 row. + for _ in $(seq 1 30); do + [[ -f "$WORKDIR/drive_exit" ]] && break + sleep 3 + done + if [[ ! -f "$WORKDIR/drive_exit" ]]; then + # Still not visible. Do not invent a failure we did not + # observe -- decide from what the run actually produced. + # bench_exit is written earlier and is usually visible by now; + # when even that is missing (seen on glm52-fp4-1k1k-2p1d-ep16, + # which had all seven results at 0.945), a passed accuracy + # gate with results on disk is the stronger evidence. + # + # "Some results plus a passed gate" is not enough. A leg + # whose decode ran at ~9 tok/s got through conc=1, 8 and 16 + # of seven points before the job ended, and this branch + # called it green: the gate had passed and raw files + # existed. Require every expected concurrency point, so a + # sweep that stopped early is red rather than a green row + # standing on someone else's numbers. + WANT=0; HAVE=0 + for _c in ${CONCS//,/ }; do + WANT=$((WANT + 1)) + [[ -f "$WORKDIR/raw_conc${_c}.json" ]] && HAVE=$((HAVE + 1)) + done + if [[ "$(cat "$WORKDIR/bench_exit" 2>/dev/null)" == "0" ]]; then + echo 0 > "$WORKDIR/drive_exit" + elif [[ "$HAVE" -eq "$WANT" ]] \ + && awk '/\[gsm8k\] accuracy=/{ + match($0,/accuracy=([0-9.]+)/,a) + match($0,/threshold=([0-9.]+)/,t) + if (a[1]+0 >= t[1]+0) ok=1 + } END{ exit ok?0:1 }' \ + "$WORKDIR/bench.log" 2>/dev/null; then + echo 0 > "$WORKDIR/drive_exit" + else + echo "[launch] incomplete sweep: $HAVE/$WANT concurrency points" >&2 + echo 1 > "$WORKDIR/drive_exit" + fi + fi + break + fi + sleep 10 + done + kill "$SPUR_TAIL_PID" 2>/dev/null || true + SALLOC_RC=$(cat "$WORKDIR/drive_exit" 2>/dev/null || echo 1) + # Release the allocation. On slurm the job ends when salloc's command + # returns; on spur drive.sh is an sbatch job, and the loop above exits as + # soon as drive_exit appears -- while the batch job, and every container + # it started, is still running. Nothing else ever cancels it, so the + # allocation is held until the time limit. The next leg then cannot get + # nodes and dies with `JobLaunchFailure (dispatch confirmation failed + # (2/4 confirmed))`, which reads as an unrelated infrastructure fault and + # cascades through every leg after it. Observed 2026-09-17: jobs + # 1774/1775 held all four nodes long after their legs had been recorded + # rc=1. Unconditional, because reaching here means this leg is done with + # its nodes either way. + if squeue -h -o "%i" 2>/dev/null | grep -qx "$SPUR_LEG_JOB_ID"; then + echo "[launch] releasing spur job $SPUR_LEG_JOB_ID" + scancel "$SPUR_LEG_JOB_ID" 2>/dev/null || true + fi + fi +else + salloc -p "$SLURM_PARTITION" -N"$TOTAL_NODES" "${NODELIST_ARG[@]}" "${EXCLUDE_ARG[@]}" "${EXCLUSIVE_ARG[@]}" \ + "${ACCT_ARG[@]}" \ + --job-name "$JOB_NAME" -t "$TIME_LIMIT" \ + bash "$WORKDIR/drive.sh" "$WORKDIR" "$PW" "$DW" "$PN_PER" "$DN_PER" "$DIST_SOCK" + SALLOC_RC=$? +fi set -e # bench output already streamed live from drive.sh (tail -F). drive.sh exits @@ -1075,10 +1708,27 @@ fi # so they must be the role totals (Oren EP16: ctx=2*8=16, gen=1*16=16, gpus=32). PREFILL_GPUS_TOTAL=$((PW * PTP)); DECODE_GPUS_TOTAL=$((DW * DTP)) TOTAL_GPUS=$((PREFILL_GPUS_TOTAL + DECODE_GPUS_TOTAL)) +# Clear this leg's results from any previous run before writing new ones. +# The filenames are a pure function of the config, so a run that produces +# fewer concurrency points than the last one leaves the missing slots filled +# by the old run's files and the published table silently mixes two runs. +# Seen on kimik26-mxfp4-1k1k-2p1d-ep16-mxfp4: 2 points measured, 7 published, +# 5 of them a day old. Deleting first means a short run looks short. +rm -f "$GITHUB_WORKSPACE/${RESULT_FILENAME}_${MATRIX_CONFIG_NAME}_conc"*"_gpus_"*".json" + PROCESSED=0 for C in ${CONCS//,/ }; do RAW="$WORKDIR/raw_conc${C}.json" - [[ -f "$RAW" ]] || { echo "WARN: missing $RAW"; continue; } + # raw_conc*.json is written by the bench container on a compute node; this + # loop runs on the driver. NFS close-to-open means the last file written is + # routinely not visible here yet, and skipping it outright silently dropped + # one or two concurrency points from four legs of a 34-leg sweep -- the runs + # were fine, only the published results were short. Wait a bounded 60s. + for _ in $(seq 1 20); do + [[ -f "$RAW" ]] && break + sleep 3 + done + [[ -f "$RAW" ]] || { echo "WARN: missing $RAW (not visible after 60s)"; continue; } DEST="$GITHUB_WORKSPACE/${RESULT_FILENAME}_${MATRIX_CONFIG_NAME}_conc${C}_gpus_${TOTAL_GPUS}_ctx_${PREFILL_GPUS_TOTAL}_gen_${DECODE_GPUS_TOTAL}.json" MODEL_ID="$MODEL_PATH" python3 - "$RAW" "$DEST" "$C" <<'PY' import json, os, sys