[AMD][DI][CI] Fix four failures in the MI355X disaggregation nightly (#41025)
This commit is contained in:
@@ -654,6 +654,13 @@ ${IT_SHARE_MOUNT}-v $HOME:/host_home $CHECKOUT_DOCKER_ARGS"
|
||||
# validation). Empty by default so the docker argv is byte-identical otherwise.
|
||||
[[ -n "${EXTRA_DOCKER_ARGS:-}" ]] && DOCKER_COMMON="$DOCKER_COMMON ${EXTRA_DOCKER_ARGS}"
|
||||
|
||||
# Checkpoint paths whose trust_remote_code modules are pre-loaded in the entry
|
||||
# script (see warm_remote_code.sh). Colon-separated, draft appended only when a
|
||||
# recipe uses an external draft model.
|
||||
WARM_MODEL_PATHS="$MODEL_PATH"
|
||||
[[ -n "${DRAFT_RESOLVED:-}" ]] && WARM_MODEL_PATHS="$WARM_MODEL_PATHS:$DRAFT_RESOLVED"
|
||||
DOCKER_COMMON="$DOCKER_COMMON -e WARM_MODEL_PATHS=$WARM_MODEL_PATHS"
|
||||
|
||||
# Spur compute nodes run prolog/epilog hooks that reap any container not tagged
|
||||
# with the owning job, so an untagged server is SIGKILLed (rc=137) seconds after
|
||||
# it starts. Single-quoted so the literal $SPUR_JOB_ID survives into the
|
||||
@@ -783,6 +790,101 @@ if not actual.startswith(expected):
|
||||
PY
|
||||
EOF
|
||||
|
||||
# Pre-populate the HuggingFace dynamic-module cache.
|
||||
#
|
||||
# transformers copies a trust_remote_code checkpoint's .py files into
|
||||
# HF_MODULES_CACHE and then imports them. The copy is a plain shutil.copy,
|
||||
# which truncates the destination before writing, and every scheduler on the
|
||||
# node does it concurrently against one cache. A scheduler that imports while
|
||||
# another is mid-copy sees a partial module and dies on a class that is
|
||||
# plainly there:
|
||||
#
|
||||
# AttributeError: module 'transformers_modules...tokenization_kimi'
|
||||
# has no attribute 'TikTokenTokenizer'
|
||||
#
|
||||
# On Kimi-K2.6 EP16 that took down two ranks of sixteen during init. Doing it
|
||||
# once here, before launch_server starts, leaves the schedulers nothing to
|
||||
# copy: transformers guards the copy with filecmp, so a cache that is already
|
||||
# correct turns every later call into a read. Measured on the Kimi checkpoint
|
||||
# with 16 concurrent loaders over 5 trials: 10 copies from a cold cache, 0
|
||||
# after this script has run.
|
||||
#
|
||||
# Best effort. A failure here is not fatal -- the schedulers still do their own
|
||||
# load and will report a real problem with their own traceback.
|
||||
cat > "$WORKDIR/warm_remote_code.sh" <<'EOF'
|
||||
#!/bin/bash
|
||||
# Never abort the server on a warm-up failure.
|
||||
set -uo pipefail
|
||||
|
||||
if [[ -z "${WARM_MODEL_PATHS:-}" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
python3 - <<'PY' || echo "[warm-remote-code] skipped (non-fatal)"
|
||||
import json
|
||||
import os
|
||||
|
||||
paths = [p for p in os.environ.get("WARM_MODEL_PATHS", "").split(":") if p]
|
||||
|
||||
try:
|
||||
# The function that does the copying, so it is the one to pre-run. Warming
|
||||
# the class instead would also import it, which can fail on a checkpoint
|
||||
# that ships modules the installed transformers cannot import -- a copy
|
||||
# that succeeded would still be reported as an error.
|
||||
from transformers.dynamic_module_utils import get_cached_module_file
|
||||
except Exception as e:
|
||||
print(f"[warm-remote-code] transformers unavailable: {e}")
|
||||
raise SystemExit(0)
|
||||
|
||||
# auto_map lives in more than one file: the model classes are in config.json,
|
||||
# while the tokenizer and processor declare their own in tokenizer_config.json
|
||||
# and preprocessor_config.json. Kimi-K2.6 raced on the tokenizer, reached
|
||||
# through the processor's entry.
|
||||
CONFIGS = (
|
||||
"config.json",
|
||||
"tokenizer_config.json",
|
||||
"preprocessor_config.json",
|
||||
"processor_config.json",
|
||||
)
|
||||
|
||||
warmed = missing = 0
|
||||
for path in paths:
|
||||
done = set()
|
||||
for name in CONFIGS:
|
||||
try:
|
||||
with open(os.path.join(path, name)) as f:
|
||||
auto_map = json.load(f).get("auto_map") or {}
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
for ref in auto_map.values():
|
||||
for one in ref if isinstance(ref, (list, tuple)) else [ref]:
|
||||
# "repo--module.Class" resolves against a different repo; leave
|
||||
# those to the normal path rather than guessing where it lives.
|
||||
if not one or "--" in one:
|
||||
continue
|
||||
module_file = one.rsplit(".", 1)[0] + ".py"
|
||||
if not os.path.exists(os.path.join(path, module_file)):
|
||||
# A checkpoint can list auto_map entries whose source is
|
||||
# not shipped -- Kimi-K2.6 still points at K2.5 files.
|
||||
# Nothing to warm, and not a problem.
|
||||
missing += 1
|
||||
continue
|
||||
if module_file in done:
|
||||
continue
|
||||
done.add(module_file)
|
||||
try:
|
||||
get_cached_module_file(path, module_file)
|
||||
warmed += 1
|
||||
except Exception as e:
|
||||
print(f"[warm-remote-code] could not pre-load {module_file}: {e}")
|
||||
|
||||
print(
|
||||
f"[warm-remote-code] warmed {warmed} module(s) from {len(paths)} path(s)"
|
||||
+ (f"; {missing} auto_map entr(ies) had no source file" if missing else "")
|
||||
)
|
||||
PY
|
||||
EOF
|
||||
|
||||
cat > "$WORKDIR/install_checkout_router.sh" <<'EOF'
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
@@ -871,6 +973,7 @@ bash "\$CIDIR/install_checkout_sglang.sh"
|
||||
if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then
|
||||
export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-}
|
||||
fi
|
||||
bash "\$CIDIR/warm_remote_code.sh"
|
||||
DIST_ARGS=""
|
||||
if [[ "\${NNODES:-1}" != "1" ]]; then
|
||||
DIST_ARGS="--nnodes \$NNODES --node-rank \$NODE_RANK --dist-init-addr \$DIST_ADDR:\${DIST_PORT:-29500}"
|
||||
@@ -894,6 +997,7 @@ bash "\$CIDIR/install_checkout_sglang.sh"
|
||||
if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then
|
||||
export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-}
|
||||
fi
|
||||
bash "\$CIDIR/warm_remote_code.sh"
|
||||
DIST_ARGS=""
|
||||
if [[ "\${NNODES:-1}" != "1" ]]; then
|
||||
DIST_ARGS="--nnodes \$NNODES --node-rank \$NODE_RANK --dist-init-addr \$DIST_ADDR:\${DIST_PORT:-29500}"
|
||||
@@ -948,6 +1052,7 @@ bash "\$CIDIR/install_checkout_sglang.sh"
|
||||
if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then
|
||||
export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-}
|
||||
fi
|
||||
bash "\$CIDIR/warm_remote_code.sh"
|
||||
exec python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH --host 0.0.0.0 --port $PPORT \
|
||||
$PREFILL_COMMON_FLAGS "\${MODEL_SERVER_ARGS[@]}" \
|
||||
@@ -963,6 +1068,7 @@ bash "\$CIDIR/install_checkout_sglang.sh"
|
||||
if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then
|
||||
export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-}
|
||||
fi
|
||||
bash "\$CIDIR/warm_remote_code.sh"
|
||||
exec python3 -m sglang.launch_server \
|
||||
--model-path $MODEL_PATH --host 0.0.0.0 --port $DPORT \
|
||||
$DECODE_COMMON_FLAGS "\${MODEL_SERVER_ARGS[@]}" \
|
||||
@@ -1190,7 +1296,18 @@ container_log() { # <node> <name> <logfile>
|
||||
# flaky step that produced the false "missing" also truncated the 5-line
|
||||
# prefill log to 0 bytes, destroying the only evidence of what happened.
|
||||
local tmp="$3.fetch"
|
||||
srun_local_or_step "$1" docker logs "$2" > "$tmp" 2>/dev/null || true
|
||||
# `docker logs` writes the container's stdout and stderr to its own stdout and
|
||||
# stderr, so merge them on the target host -- hence bash -c rather than passing
|
||||
# `docker logs` straight through. Without it only the remote branch of
|
||||
# srun_local_or_step kept stderr; the local branch runs docker directly and the
|
||||
# 2>/dev/null below swallowed it. Every sglang scheduler line goes to stderr and
|
||||
# only uvicorn's access log goes to stdout, so the engine sitting on the node
|
||||
# that runs drive.sh -- always decode node 0 -- logged nothing but
|
||||
# "POST /generate 200 OK" for the whole run, with no trace of the crash that
|
||||
# ended it. Visible in 2p1d-ep16, where decode node 1 is fetched remotely and
|
||||
# has the full log while node 0 has none of it. The outer 2>/dev/null still
|
||||
# drops srun's own dispatch noise.
|
||||
srun_local_or_step "$1" bash -c "docker logs $2 2>&1" > "$tmp" 2>/dev/null || true
|
||||
# Publish in place (truncate + rewrite the SAME inode), never by renaming over
|
||||
# the target. drive.sh keeps a `tail -F` on bench.log, and swapping the inode
|
||||
# every 10s made tail reopen and re-emit the whole file each poll -- the
|
||||
@@ -1560,12 +1677,15 @@ echo "[drive_batch] \$(hostname) rank \${SPUR_NODEID:-?} elected driver"
|
||||
# records: a leg that failed in the monitor loop left an sbatch.out that simply
|
||||
# stopped mid-run, with the teardown lines never visible.
|
||||
#
|
||||
# A plain redirect, deliberately NOT `| tee`. With a pipeline, drive_batch waits
|
||||
# for the whole pipeline, and that does not finish when drive.sh does -- the
|
||||
# backgrounded `tail -F` on bench.log inherits drive.sh's stdout and holds the
|
||||
# pipe's write end open. drive.sh exited 0 and drive_exit was still unwritten
|
||||
# 11 minutes later, so the standby tasks kept the allocation alive and the leg
|
||||
# looked hung after it had actually passed.
|
||||
# A plain redirect, deliberately not a tee pipeline. With a pipeline,
|
||||
# drive_batch waits for the whole pipeline, and that does not finish when
|
||||
# drive.sh does -- the backgrounded tail on bench.log inherits drive.sh's
|
||||
# stdout and holds the pipe's write end open. drive.sh exited 0 and drive_exit
|
||||
# was still unwritten 11 minutes later, so the standby tasks kept the
|
||||
# allocation alive and the leg looked hung after it had actually passed.
|
||||
# No backticks anywhere in this heredoc: it is unquoted, so bash runs them at
|
||||
# generation time. The pair that used to be here printed a syntax error on
|
||||
# every launch and ran a stray tail on the submitting host.
|
||||
bash "$WORKDIR/drive.sh" "$WORKDIR" "$PW" "$DW" "$PN_PER" "$DN_PER" "$DIST_SOCK" "$ADDR_NIC" \
|
||||
> "$WORKDIR/drive_\$(hostname).log" 2>&1
|
||||
echo \$? > "$WORKDIR/drive_exit"
|
||||
@@ -1589,25 +1709,75 @@ EOF
|
||||
# glance, from a model bug. Observed on glm52-fp4-1k1k-2p1d-ep16 2026-09-17.
|
||||
# Retry only on that signature; a genuinely bad sbatch request must still
|
||||
# fail immediately rather than being retried six times.
|
||||
SPUR_LEG_JOB_ID=""
|
||||
for _attempt in 1 2 3 4 5 6; do
|
||||
SBATCH_MSG=$(sbatch -p "$SLURM_PARTITION" -N"$TOTAL_NODES" "${NODELIST_ARG[@]}" \
|
||||
"${EXCLUDE_ARG[@]}" "${EXCLUSIVE_ARG[@]}" "${ACCT_ARG[@]}" \
|
||||
--job-name "$JOB_NAME" -t "$TIME_LIMIT" \
|
||||
--output "$SBATCH_OUT" --error "$SBATCH_OUT" \
|
||||
"$WORKDIR/drive_batch.sh" 2>&1)
|
||||
echo "$SBATCH_MSG"
|
||||
SPUR_LEG_JOB_ID="${SBATCH_MSG##* }"
|
||||
[[ "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]] && break
|
||||
if [[ "$SBATCH_MSG" == *"not the Raft leader"* \
|
||||
|| "$SBATCH_MSG" == *"service is currently unavailable"* ]]; then
|
||||
echo "[launch] spur controller unavailable (attempt $_attempt/6); retrying in 20s" >&2
|
||||
SPUR_LEG_JOB_ID=""
|
||||
sleep 20
|
||||
continue
|
||||
fi
|
||||
break
|
||||
done
|
||||
spur_submit_leg() {
|
||||
SPUR_LEG_JOB_ID=""
|
||||
: > "$SBATCH_OUT"
|
||||
for _attempt in 1 2 3 4 5 6; do
|
||||
SBATCH_MSG=$(sbatch -p "$SLURM_PARTITION" -N"$TOTAL_NODES" "${NODELIST_ARG[@]}" \
|
||||
"${EXCLUDE_ARG[@]}" "${EXCLUSIVE_ARG[@]}" "${ACCT_ARG[@]}" \
|
||||
--job-name "$JOB_NAME" -t "$TIME_LIMIT" \
|
||||
--output "$SBATCH_OUT" --error "$SBATCH_OUT" \
|
||||
"$WORKDIR/drive_batch.sh" 2>&1)
|
||||
echo "$SBATCH_MSG"
|
||||
SPUR_LEG_JOB_ID="${SBATCH_MSG##* }"
|
||||
[[ "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]] && break
|
||||
if [[ "$SBATCH_MSG" == *"not the Raft leader"* \
|
||||
|| "$SBATCH_MSG" == *"service is currently unavailable"* ]]; then
|
||||
echo "[launch] spur controller unavailable (attempt $_attempt/6); retrying in 20s" >&2
|
||||
SPUR_LEG_JOB_ID=""
|
||||
sleep 20
|
||||
continue
|
||||
fi
|
||||
break
|
||||
done
|
||||
}
|
||||
|
||||
# Spur does not hold a job it cannot place. When the partition is full it
|
||||
# cancels the submission within seconds -- JobState=CANCELLED,
|
||||
# Reason=QOSGrpNodeLimit, StartTime=N/A -- and the monitor below then sees the
|
||||
# job leave the queue having produced nothing, which it reports as an
|
||||
# incomplete sweep. That is a red result for a test that never ran. Observed
|
||||
# 2026-09-23 on jobs 4124 and 4125, both cancelled inside 40s while the four
|
||||
# amd-sglang nodes were held by another run.
|
||||
#
|
||||
# A job can also be cancelled just after it is placed: job 4147 on 2026-09-24
|
||||
# had StartTime set and RunTime=00:00:05, long enough to start the containers
|
||||
# and nothing else. Keying on StartTime=N/A alone misses that, so also accept
|
||||
# a cancelled job that produced no bench results -- a real sweep writes
|
||||
# raw_conc*.json and bench_exit, and neither can appear in five seconds.
|
||||
spur_never_started() {
|
||||
local _info
|
||||
_info=$(scontrol show job "$1" 2>/dev/null) || return 1
|
||||
[[ "$_info" == *"JobState=CANCELLED"* ]] || return 1
|
||||
[[ "$_info" == *"StartTime=N/A"* ]] && return 0
|
||||
[[ -f "$WORKDIR/bench_exit" ]] && return 1
|
||||
! compgen -G "$WORKDIR/raw_conc*.json" > /dev/null
|
||||
}
|
||||
# squeue is not authoritative for "has this job left the queue". It asks the
|
||||
# spur controller for the whole queue, and a controller hiccup -- the same
|
||||
# Raft leader election spur_submit_leg already retries around -- answers with
|
||||
# an empty list, the error swallowed by 2>/dev/null. A single miss is enough
|
||||
# to end the leg.
|
||||
#
|
||||
# That is what happened to job 4177 on 2026-09-24. The monitor decided the
|
||||
# job was gone 21 minutes into a healthy sweep, waited 90s for a drive_exit
|
||||
# that could not arrive because the job was still benchmarking, reported
|
||||
# "incomplete sweep: 3/7", and then scancelled the run it had just given up
|
||||
# on -- the teardown below found the job in squeue, because it had never
|
||||
# left. scontrol, which answers per job, still had it RUNNING throughout.
|
||||
# Ask scontrol before believing the queue listing.
|
||||
spur_job_alive() {
|
||||
squeue -h -o "%i" 2>/dev/null | grep -qx "$1" && return 0
|
||||
local _info
|
||||
_info=$(scontrol show job "$1" 2>/dev/null) || return 1
|
||||
[[ "$_info" == *"JobState=RUNNING"* \
|
||||
|| "$_info" == *"JobState=PENDING"* \
|
||||
|| "$_info" == *"JobState=CONFIGURING"* \
|
||||
|| "$_info" == *"JobState=COMPLETING"* ]]
|
||||
}
|
||||
SPUR_RESUBMITS_LEFT=${SPUR_RESUBMITS_LEFT:-20}
|
||||
|
||||
spur_submit_leg
|
||||
if [[ ! "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]]; then
|
||||
echo "ERROR: could not parse a job id out of: $SBATCH_MSG" >&2
|
||||
SALLOC_RC=1
|
||||
@@ -1620,7 +1790,25 @@ EOF
|
||||
# ever running it (node failure, scheduler kill, time limit).
|
||||
while :; do
|
||||
[[ -f "$WORKDIR/drive_exit" ]] && break
|
||||
if ! squeue -h -o "%i" 2>/dev/null | grep -qx "$SPUR_LEG_JOB_ID"; then
|
||||
if ! spur_job_alive "$SPUR_LEG_JOB_ID"; then
|
||||
# Cancelled before it ever ran: wait for the nodes and submit
|
||||
# again rather than reporting a result we never measured.
|
||||
if spur_never_started "$SPUR_LEG_JOB_ID" && (( SPUR_RESUBMITS_LEFT > 0 )); then
|
||||
SPUR_RESUBMITS_LEFT=$((SPUR_RESUBMITS_LEFT - 1))
|
||||
echo "[launch] spur job $SPUR_LEG_JOB_ID cancelled before start;" \
|
||||
"resubmitting in 5m ($SPUR_RESUBMITS_LEFT left)" >&2
|
||||
kill "$SPUR_TAIL_PID" 2>/dev/null || true
|
||||
sleep 300
|
||||
spur_submit_leg
|
||||
if [[ ! "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]]; then
|
||||
echo 1 > "$WORKDIR/drive_exit"
|
||||
break
|
||||
fi
|
||||
echo "[launch] spur job $SPUR_LEG_JOB_ID submitted; streaming $SBATCH_OUT"
|
||||
tail -F "$SBATCH_OUT" 2>/dev/null &
|
||||
SPUR_TAIL_PID=$!
|
||||
continue
|
||||
fi
|
||||
# The job has left the queue. drive_exit is written on a compute
|
||||
# node and read here on the login node, so NFS close-to-open
|
||||
# visibility can delay it well past a single short sleep. A five
|
||||
@@ -1683,7 +1871,7 @@ EOF
|
||||
# 1774/1775 held all four nodes long after their legs had been recorded
|
||||
# rc=1. Unconditional, because reaching here means this leg is done with
|
||||
# its nodes either way.
|
||||
if squeue -h -o "%i" 2>/dev/null | grep -qx "$SPUR_LEG_JOB_ID"; then
|
||||
if spur_job_alive "$SPUR_LEG_JOB_ID"; then
|
||||
echo "[launch] releasing spur job $SPUR_LEG_JOB_ID"
|
||||
scancel "$SPUR_LEG_JOB_ID" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
@@ -58,7 +58,13 @@ runtime:
|
||||
# differ, so these split what the single-node EP<=8 recipes leave symmetric.
|
||||
wide_ep:
|
||||
kv_cache_dtype: fp8_e4m3
|
||||
prefill_mem_fraction_static: 0.8
|
||||
# 0.85, not the 0.8 the other ep16 recipes use. FP8 Pro weights are 218.92 GB
|
||||
# per rank; at 0.8 of a 288 GB card the static budget is 230.4 GB, leaving
|
||||
# ~11 GB for activations and the KV pool. max_total_num_tokens resolves to 0
|
||||
# and the prefill engine aborts with "Not enough memory. Please try to
|
||||
# increase --mem-fraction-static." The FP4 recipes fit at 0.8 because their
|
||||
# weights are half the size, and the FP8 MTP recipe already runs at 0.85.
|
||||
prefill_mem_fraction_static: 0.85
|
||||
decode_mem_fraction_static: 0.85
|
||||
prefill_chunked_prefill_size: 131072
|
||||
prefill_max_running_requests: 1024
|
||||
|
||||
@@ -66,6 +66,16 @@ runtime:
|
||||
decode_bootstrap_port: 9001
|
||||
lb_port: 8000
|
||||
mem_fraction_static: 0.90
|
||||
# Decode-side headroom for MTP. With the draft head the decode server ends
|
||||
# startup with 12.9 GB free per GPU (draft weights + a 4.6 GB draft-decode
|
||||
# graph + a 9.8 GB target-verify graph, against 2.8 GB for the non-MTP target
|
||||
# graph); the non-MTP sibling leaves 24.2 GB at the identical 0.90. At
|
||||
# concurrency 256 the draft-extend path overruns that margin and ROCr aborts
|
||||
# the queues with HSA_STATUS_ERROR_OUT_OF_RESOURCES ("Available Free mem :
|
||||
# 0 MB"), killing a scheduler process. 0.85 buys back ~19 GB on decode only;
|
||||
# prefill and every other emitted value are unchanged.
|
||||
wide_ep:
|
||||
decode_mem_fraction_static: 0.85
|
||||
page_size: 256
|
||||
max_running_requests: 256
|
||||
chunked_prefill_size: 8192
|
||||
|
||||
Reference in New Issue
Block a user