[AMD][DI][CI] Fix four failures in the MI355X disaggregation nightly (#41025)

This commit is contained in:
Zhaoyi Li
2026-09-29 23:54:40 -07:00
committed by GitHub
parent 24ee67d2fa
commit f85ca901e2
3 changed files with 233 additions and 29 deletions
+216 -28
View File
@@ -654,6 +654,13 @@ ${IT_SHARE_MOUNT}-v $HOME:/host_home $CHECKOUT_DOCKER_ARGS"
# validation). Empty by default so the docker argv is byte-identical otherwise.
[[ -n "${EXTRA_DOCKER_ARGS:-}" ]] && DOCKER_COMMON="$DOCKER_COMMON ${EXTRA_DOCKER_ARGS}"
# Checkpoint paths whose trust_remote_code modules are pre-loaded in the entry
# script (see warm_remote_code.sh). Colon-separated, draft appended only when a
# recipe uses an external draft model.
WARM_MODEL_PATHS="$MODEL_PATH"
[[ -n "${DRAFT_RESOLVED:-}" ]] && WARM_MODEL_PATHS="$WARM_MODEL_PATHS:$DRAFT_RESOLVED"
DOCKER_COMMON="$DOCKER_COMMON -e WARM_MODEL_PATHS=$WARM_MODEL_PATHS"
# Spur compute nodes run prolog/epilog hooks that reap any container not tagged
# with the owning job, so an untagged server is SIGKILLed (rc=137) seconds after
# it starts. Single-quoted so the literal $SPUR_JOB_ID survives into the
@@ -783,6 +790,101 @@ if not actual.startswith(expected):
PY
EOF
# Pre-populate the HuggingFace dynamic-module cache.
#
# transformers copies a trust_remote_code checkpoint's .py files into
# HF_MODULES_CACHE and then imports them. The copy is a plain shutil.copy,
# which truncates the destination before writing, and every scheduler on the
# node does it concurrently against one cache. A scheduler that imports while
# another is mid-copy sees a partial module and dies on a class that is
# plainly there:
#
# AttributeError: module 'transformers_modules...tokenization_kimi'
# has no attribute 'TikTokenTokenizer'
#
# On Kimi-K2.6 EP16 that took down two ranks of sixteen during init. Doing it
# once here, before launch_server starts, leaves the schedulers nothing to
# copy: transformers guards the copy with filecmp, so a cache that is already
# correct turns every later call into a read. Measured on the Kimi checkpoint
# with 16 concurrent loaders over 5 trials: 10 copies from a cold cache, 0
# after this script has run.
#
# Best effort. A failure here is not fatal -- the schedulers still do their own
# load and will report a real problem with their own traceback.
cat > "$WORKDIR/warm_remote_code.sh" <<'EOF'
#!/bin/bash
# Never abort the server on a warm-up failure.
set -uo pipefail
if [[ -z "${WARM_MODEL_PATHS:-}" ]]; then
exit 0
fi
python3 - <<'PY' || echo "[warm-remote-code] skipped (non-fatal)"
import json
import os
paths = [p for p in os.environ.get("WARM_MODEL_PATHS", "").split(":") if p]
try:
# The function that does the copying, so it is the one to pre-run. Warming
# the class instead would also import it, which can fail on a checkpoint
# that ships modules the installed transformers cannot import -- a copy
# that succeeded would still be reported as an error.
from transformers.dynamic_module_utils import get_cached_module_file
except Exception as e:
print(f"[warm-remote-code] transformers unavailable: {e}")
raise SystemExit(0)
# auto_map lives in more than one file: the model classes are in config.json,
# while the tokenizer and processor declare their own in tokenizer_config.json
# and preprocessor_config.json. Kimi-K2.6 raced on the tokenizer, reached
# through the processor's entry.
CONFIGS = (
"config.json",
"tokenizer_config.json",
"preprocessor_config.json",
"processor_config.json",
)
warmed = missing = 0
for path in paths:
done = set()
for name in CONFIGS:
try:
with open(os.path.join(path, name)) as f:
auto_map = json.load(f).get("auto_map") or {}
except (OSError, ValueError):
continue
for ref in auto_map.values():
for one in ref if isinstance(ref, (list, tuple)) else [ref]:
# "repo--module.Class" resolves against a different repo; leave
# those to the normal path rather than guessing where it lives.
if not one or "--" in one:
continue
module_file = one.rsplit(".", 1)[0] + ".py"
if not os.path.exists(os.path.join(path, module_file)):
# A checkpoint can list auto_map entries whose source is
# not shipped -- Kimi-K2.6 still points at K2.5 files.
# Nothing to warm, and not a problem.
missing += 1
continue
if module_file in done:
continue
done.add(module_file)
try:
get_cached_module_file(path, module_file)
warmed += 1
except Exception as e:
print(f"[warm-remote-code] could not pre-load {module_file}: {e}")
print(
f"[warm-remote-code] warmed {warmed} module(s) from {len(paths)} path(s)"
+ (f"; {missing} auto_map entr(ies) had no source file" if missing else "")
)
PY
EOF
cat > "$WORKDIR/install_checkout_router.sh" <<'EOF'
#!/bin/bash
set -euo pipefail
@@ -871,6 +973,7 @@ bash "\$CIDIR/install_checkout_sglang.sh"
if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then
export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-}
fi
bash "\$CIDIR/warm_remote_code.sh"
DIST_ARGS=""
if [[ "\${NNODES:-1}" != "1" ]]; then
DIST_ARGS="--nnodes \$NNODES --node-rank \$NODE_RANK --dist-init-addr \$DIST_ADDR:\${DIST_PORT:-29500}"
@@ -894,6 +997,7 @@ bash "\$CIDIR/install_checkout_sglang.sh"
if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then
export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-}
fi
bash "\$CIDIR/warm_remote_code.sh"
DIST_ARGS=""
if [[ "\${NNODES:-1}" != "1" ]]; then
DIST_ARGS="--nnodes \$NNODES --node-rank \$NODE_RANK --dist-init-addr \$DIST_ADDR:\${DIST_PORT:-29500}"
@@ -948,6 +1052,7 @@ bash "\$CIDIR/install_checkout_sglang.sh"
if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then
export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-}
fi
bash "\$CIDIR/warm_remote_code.sh"
exec python3 -m sglang.launch_server \
--model-path $MODEL_PATH --host 0.0.0.0 --port $PPORT \
$PREFILL_COMMON_FLAGS "\${MODEL_SERVER_ARGS[@]}" \
@@ -963,6 +1068,7 @@ bash "\$CIDIR/install_checkout_sglang.sh"
if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then
export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-}
fi
bash "\$CIDIR/warm_remote_code.sh"
exec python3 -m sglang.launch_server \
--model-path $MODEL_PATH --host 0.0.0.0 --port $DPORT \
$DECODE_COMMON_FLAGS "\${MODEL_SERVER_ARGS[@]}" \
@@ -1190,7 +1296,18 @@ container_log() { # <node> <name> <logfile>
# flaky step that produced the false "missing" also truncated the 5-line
# prefill log to 0 bytes, destroying the only evidence of what happened.
local tmp="$3.fetch"
srun_local_or_step "$1" docker logs "$2" > "$tmp" 2>/dev/null || true
# `docker logs` writes the container's stdout and stderr to its own stdout and
# stderr, so merge them on the target host -- hence bash -c rather than passing
# `docker logs` straight through. Without it only the remote branch of
# srun_local_or_step kept stderr; the local branch runs docker directly and the
# 2>/dev/null below swallowed it. Every sglang scheduler line goes to stderr and
# only uvicorn's access log goes to stdout, so the engine sitting on the node
# that runs drive.sh -- always decode node 0 -- logged nothing but
# "POST /generate 200 OK" for the whole run, with no trace of the crash that
# ended it. Visible in 2p1d-ep16, where decode node 1 is fetched remotely and
# has the full log while node 0 has none of it. The outer 2>/dev/null still
# drops srun's own dispatch noise.
srun_local_or_step "$1" bash -c "docker logs $2 2>&1" > "$tmp" 2>/dev/null || true
# Publish in place (truncate + rewrite the SAME inode), never by renaming over
# the target. drive.sh keeps a `tail -F` on bench.log, and swapping the inode
# every 10s made tail reopen and re-emit the whole file each poll -- the
@@ -1560,12 +1677,15 @@ echo "[drive_batch] \$(hostname) rank \${SPUR_NODEID:-?} elected driver"
# records: a leg that failed in the monitor loop left an sbatch.out that simply
# stopped mid-run, with the teardown lines never visible.
#
# A plain redirect, deliberately NOT `| tee`. With a pipeline, drive_batch waits
# for the whole pipeline, and that does not finish when drive.sh does -- the
# backgrounded `tail -F` on bench.log inherits drive.sh's stdout and holds the
# pipe's write end open. drive.sh exited 0 and drive_exit was still unwritten
# 11 minutes later, so the standby tasks kept the allocation alive and the leg
# looked hung after it had actually passed.
# A plain redirect, deliberately not a tee pipeline. With a pipeline,
# drive_batch waits for the whole pipeline, and that does not finish when
# drive.sh does -- the backgrounded tail on bench.log inherits drive.sh's
# stdout and holds the pipe's write end open. drive.sh exited 0 and drive_exit
# was still unwritten 11 minutes later, so the standby tasks kept the
# allocation alive and the leg looked hung after it had actually passed.
# No backticks anywhere in this heredoc: it is unquoted, so bash runs them at
# generation time. The pair that used to be here printed a syntax error on
# every launch and ran a stray tail on the submitting host.
bash "$WORKDIR/drive.sh" "$WORKDIR" "$PW" "$DW" "$PN_PER" "$DN_PER" "$DIST_SOCK" "$ADDR_NIC" \
> "$WORKDIR/drive_\$(hostname).log" 2>&1
echo \$? > "$WORKDIR/drive_exit"
@@ -1589,25 +1709,75 @@ EOF
# glance, from a model bug. Observed on glm52-fp4-1k1k-2p1d-ep16 2026-09-17.
# Retry only on that signature; a genuinely bad sbatch request must still
# fail immediately rather than being retried six times.
SPUR_LEG_JOB_ID=""
for _attempt in 1 2 3 4 5 6; do
SBATCH_MSG=$(sbatch -p "$SLURM_PARTITION" -N"$TOTAL_NODES" "${NODELIST_ARG[@]}" \
"${EXCLUDE_ARG[@]}" "${EXCLUSIVE_ARG[@]}" "${ACCT_ARG[@]}" \
--job-name "$JOB_NAME" -t "$TIME_LIMIT" \
--output "$SBATCH_OUT" --error "$SBATCH_OUT" \
"$WORKDIR/drive_batch.sh" 2>&1)
echo "$SBATCH_MSG"
SPUR_LEG_JOB_ID="${SBATCH_MSG##* }"
[[ "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]] && break
if [[ "$SBATCH_MSG" == *"not the Raft leader"* \
|| "$SBATCH_MSG" == *"service is currently unavailable"* ]]; then
echo "[launch] spur controller unavailable (attempt $_attempt/6); retrying in 20s" >&2
SPUR_LEG_JOB_ID=""
sleep 20
continue
fi
break
done
spur_submit_leg() {
SPUR_LEG_JOB_ID=""
: > "$SBATCH_OUT"
for _attempt in 1 2 3 4 5 6; do
SBATCH_MSG=$(sbatch -p "$SLURM_PARTITION" -N"$TOTAL_NODES" "${NODELIST_ARG[@]}" \
"${EXCLUDE_ARG[@]}" "${EXCLUSIVE_ARG[@]}" "${ACCT_ARG[@]}" \
--job-name "$JOB_NAME" -t "$TIME_LIMIT" \
--output "$SBATCH_OUT" --error "$SBATCH_OUT" \
"$WORKDIR/drive_batch.sh" 2>&1)
echo "$SBATCH_MSG"
SPUR_LEG_JOB_ID="${SBATCH_MSG##* }"
[[ "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]] && break
if [[ "$SBATCH_MSG" == *"not the Raft leader"* \
|| "$SBATCH_MSG" == *"service is currently unavailable"* ]]; then
echo "[launch] spur controller unavailable (attempt $_attempt/6); retrying in 20s" >&2
SPUR_LEG_JOB_ID=""
sleep 20
continue
fi
break
done
}
# Spur does not hold a job it cannot place. When the partition is full it
# cancels the submission within seconds -- JobState=CANCELLED,
# Reason=QOSGrpNodeLimit, StartTime=N/A -- and the monitor below then sees the
# job leave the queue having produced nothing, which it reports as an
# incomplete sweep. That is a red result for a test that never ran. Observed
# 2026-09-23 on jobs 4124 and 4125, both cancelled inside 40s while the four
# amd-sglang nodes were held by another run.
#
# A job can also be cancelled just after it is placed: job 4147 on 2026-09-24
# had StartTime set and RunTime=00:00:05, long enough to start the containers
# and nothing else. Keying on StartTime=N/A alone misses that, so also accept
# a cancelled job that produced no bench results -- a real sweep writes
# raw_conc*.json and bench_exit, and neither can appear in five seconds.
spur_never_started() {
local _info
_info=$(scontrol show job "$1" 2>/dev/null) || return 1
[[ "$_info" == *"JobState=CANCELLED"* ]] || return 1
[[ "$_info" == *"StartTime=N/A"* ]] && return 0
[[ -f "$WORKDIR/bench_exit" ]] && return 1
! compgen -G "$WORKDIR/raw_conc*.json" > /dev/null
}
# squeue is not authoritative for "has this job left the queue". It asks the
# spur controller for the whole queue, and a controller hiccup -- the same
# Raft leader election spur_submit_leg already retries around -- answers with
# an empty list, the error swallowed by 2>/dev/null. A single miss is enough
# to end the leg.
#
# That is what happened to job 4177 on 2026-09-24. The monitor decided the
# job was gone 21 minutes into a healthy sweep, waited 90s for a drive_exit
# that could not arrive because the job was still benchmarking, reported
# "incomplete sweep: 3/7", and then scancelled the run it had just given up
# on -- the teardown below found the job in squeue, because it had never
# left. scontrol, which answers per job, still had it RUNNING throughout.
# Ask scontrol before believing the queue listing.
spur_job_alive() {
squeue -h -o "%i" 2>/dev/null | grep -qx "$1" && return 0
local _info
_info=$(scontrol show job "$1" 2>/dev/null) || return 1
[[ "$_info" == *"JobState=RUNNING"* \
|| "$_info" == *"JobState=PENDING"* \
|| "$_info" == *"JobState=CONFIGURING"* \
|| "$_info" == *"JobState=COMPLETING"* ]]
}
SPUR_RESUBMITS_LEFT=${SPUR_RESUBMITS_LEFT:-20}
spur_submit_leg
if [[ ! "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]]; then
echo "ERROR: could not parse a job id out of: $SBATCH_MSG" >&2
SALLOC_RC=1
@@ -1620,7 +1790,25 @@ EOF
# ever running it (node failure, scheduler kill, time limit).
while :; do
[[ -f "$WORKDIR/drive_exit" ]] && break
if ! squeue -h -o "%i" 2>/dev/null | grep -qx "$SPUR_LEG_JOB_ID"; then
if ! spur_job_alive "$SPUR_LEG_JOB_ID"; then
# Cancelled before it ever ran: wait for the nodes and submit
# again rather than reporting a result we never measured.
if spur_never_started "$SPUR_LEG_JOB_ID" && (( SPUR_RESUBMITS_LEFT > 0 )); then
SPUR_RESUBMITS_LEFT=$((SPUR_RESUBMITS_LEFT - 1))
echo "[launch] spur job $SPUR_LEG_JOB_ID cancelled before start;" \
"resubmitting in 5m ($SPUR_RESUBMITS_LEFT left)" >&2
kill "$SPUR_TAIL_PID" 2>/dev/null || true
sleep 300
spur_submit_leg
if [[ ! "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]]; then
echo 1 > "$WORKDIR/drive_exit"
break
fi
echo "[launch] spur job $SPUR_LEG_JOB_ID submitted; streaming $SBATCH_OUT"
tail -F "$SBATCH_OUT" 2>/dev/null &
SPUR_TAIL_PID=$!
continue
fi
# The job has left the queue. drive_exit is written on a compute
# node and read here on the login node, so NFS close-to-open
# visibility can delay it well past a single short sleep. A five
@@ -1683,7 +1871,7 @@ EOF
# 1774/1775 held all four nodes long after their legs had been recorded
# rc=1. Unconditional, because reaching here means this leg is done with
# its nodes either way.
if squeue -h -o "%i" 2>/dev/null | grep -qx "$SPUR_LEG_JOB_ID"; then
if spur_job_alive "$SPUR_LEG_JOB_ID"; then
echo "[launch] releasing spur job $SPUR_LEG_JOB_ID"
scancel "$SPUR_LEG_JOB_ID" 2>/dev/null || true
fi
@@ -58,7 +58,13 @@ runtime:
# differ, so these split what the single-node EP<=8 recipes leave symmetric.
wide_ep:
kv_cache_dtype: fp8_e4m3
prefill_mem_fraction_static: 0.8
# 0.85, not the 0.8 the other ep16 recipes use. FP8 Pro weights are 218.92 GB
# per rank; at 0.8 of a 288 GB card the static budget is 230.4 GB, leaving
# ~11 GB for activations and the KV pool. max_total_num_tokens resolves to 0
# and the prefill engine aborts with "Not enough memory. Please try to
# increase --mem-fraction-static." The FP4 recipes fit at 0.8 because their
# weights are half the size, and the FP8 MTP recipe already runs at 0.85.
prefill_mem_fraction_static: 0.85
decode_mem_fraction_static: 0.85
prefill_chunked_prefill_size: 131072
prefill_max_running_requests: 1024
@@ -66,6 +66,16 @@ runtime:
decode_bootstrap_port: 9001
lb_port: 8000
mem_fraction_static: 0.90
# Decode-side headroom for MTP. With the draft head the decode server ends
# startup with 12.9 GB free per GPU (draft weights + a 4.6 GB draft-decode
# graph + a 9.8 GB target-verify graph, against 2.8 GB for the non-MTP target
# graph); the non-MTP sibling leaves 24.2 GB at the identical 0.90. At
# concurrency 256 the draft-extend path overruns that margin and ROCr aborts
# the queues with HSA_STATUS_ERROR_OUT_OF_RESOURCES ("Available Free mem :
# 0 MB"), killing a scheduler process. 0.85 buys back ~19 GB on decode only;
# prefill and every other emitted value are unchanged.
wide_ep:
decode_mem_fraction_static: 0.85
page_size: 256
max_running_requests: 256
chunked_prefill_size: 8192