diff --git a/scripts/ci/slurm/launch_mi355x.sh b/scripts/ci/slurm/launch_mi355x.sh index 10a980e939..9a6488be60 100755 --- a/scripts/ci/slurm/launch_mi355x.sh +++ b/scripts/ci/slurm/launch_mi355x.sh @@ -654,6 +654,13 @@ ${IT_SHARE_MOUNT}-v $HOME:/host_home $CHECKOUT_DOCKER_ARGS" # validation). Empty by default so the docker argv is byte-identical otherwise. [[ -n "${EXTRA_DOCKER_ARGS:-}" ]] && DOCKER_COMMON="$DOCKER_COMMON ${EXTRA_DOCKER_ARGS}" +# Checkpoint paths whose trust_remote_code modules are pre-loaded in the entry +# script (see warm_remote_code.sh). Colon-separated, draft appended only when a +# recipe uses an external draft model. +WARM_MODEL_PATHS="$MODEL_PATH" +[[ -n "${DRAFT_RESOLVED:-}" ]] && WARM_MODEL_PATHS="$WARM_MODEL_PATHS:$DRAFT_RESOLVED" +DOCKER_COMMON="$DOCKER_COMMON -e WARM_MODEL_PATHS=$WARM_MODEL_PATHS" + # Spur compute nodes run prolog/epilog hooks that reap any container not tagged # with the owning job, so an untagged server is SIGKILLed (rc=137) seconds after # it starts. Single-quoted so the literal $SPUR_JOB_ID survives into the @@ -783,6 +790,101 @@ if not actual.startswith(expected): PY EOF +# Pre-populate the HuggingFace dynamic-module cache. +# +# transformers copies a trust_remote_code checkpoint's .py files into +# HF_MODULES_CACHE and then imports them. The copy is a plain shutil.copy, +# which truncates the destination before writing, and every scheduler on the +# node does it concurrently against one cache. A scheduler that imports while +# another is mid-copy sees a partial module and dies on a class that is +# plainly there: +# +# AttributeError: module 'transformers_modules...tokenization_kimi' +# has no attribute 'TikTokenTokenizer' +# +# On Kimi-K2.6 EP16 that took down two ranks of sixteen during init. Doing it +# once here, before launch_server starts, leaves the schedulers nothing to +# copy: transformers guards the copy with filecmp, so a cache that is already +# correct turns every later call into a read. Measured on the Kimi checkpoint +# with 16 concurrent loaders over 5 trials: 10 copies from a cold cache, 0 +# after this script has run. +# +# Best effort. A failure here is not fatal -- the schedulers still do their own +# load and will report a real problem with their own traceback. +cat > "$WORKDIR/warm_remote_code.sh" <<'EOF' +#!/bin/bash +# Never abort the server on a warm-up failure. +set -uo pipefail + +if [[ -z "${WARM_MODEL_PATHS:-}" ]]; then + exit 0 +fi + +python3 - <<'PY' || echo "[warm-remote-code] skipped (non-fatal)" +import json +import os + +paths = [p for p in os.environ.get("WARM_MODEL_PATHS", "").split(":") if p] + +try: + # The function that does the copying, so it is the one to pre-run. Warming + # the class instead would also import it, which can fail on a checkpoint + # that ships modules the installed transformers cannot import -- a copy + # that succeeded would still be reported as an error. + from transformers.dynamic_module_utils import get_cached_module_file +except Exception as e: + print(f"[warm-remote-code] transformers unavailable: {e}") + raise SystemExit(0) + +# auto_map lives in more than one file: the model classes are in config.json, +# while the tokenizer and processor declare their own in tokenizer_config.json +# and preprocessor_config.json. Kimi-K2.6 raced on the tokenizer, reached +# through the processor's entry. +CONFIGS = ( + "config.json", + "tokenizer_config.json", + "preprocessor_config.json", + "processor_config.json", +) + +warmed = missing = 0 +for path in paths: + done = set() + for name in CONFIGS: + try: + with open(os.path.join(path, name)) as f: + auto_map = json.load(f).get("auto_map") or {} + except (OSError, ValueError): + continue + for ref in auto_map.values(): + for one in ref if isinstance(ref, (list, tuple)) else [ref]: + # "repo--module.Class" resolves against a different repo; leave + # those to the normal path rather than guessing where it lives. + if not one or "--" in one: + continue + module_file = one.rsplit(".", 1)[0] + ".py" + if not os.path.exists(os.path.join(path, module_file)): + # A checkpoint can list auto_map entries whose source is + # not shipped -- Kimi-K2.6 still points at K2.5 files. + # Nothing to warm, and not a problem. + missing += 1 + continue + if module_file in done: + continue + done.add(module_file) + try: + get_cached_module_file(path, module_file) + warmed += 1 + except Exception as e: + print(f"[warm-remote-code] could not pre-load {module_file}: {e}") + +print( + f"[warm-remote-code] warmed {warmed} module(s) from {len(paths)} path(s)" + + (f"; {missing} auto_map entr(ies) had no source file" if missing else "") +) +PY +EOF + cat > "$WORKDIR/install_checkout_router.sh" <<'EOF' #!/bin/bash set -euo pipefail @@ -871,6 +973,7 @@ bash "\$CIDIR/install_checkout_sglang.sh" if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-} fi +bash "\$CIDIR/warm_remote_code.sh" DIST_ARGS="" if [[ "\${NNODES:-1}" != "1" ]]; then DIST_ARGS="--nnodes \$NNODES --node-rank \$NODE_RANK --dist-init-addr \$DIST_ADDR:\${DIST_PORT:-29500}" @@ -894,6 +997,7 @@ bash "\$CIDIR/install_checkout_sglang.sh" if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-} fi +bash "\$CIDIR/warm_remote_code.sh" DIST_ARGS="" if [[ "\${NNODES:-1}" != "1" ]]; then DIST_ARGS="--nnodes \$NNODES --node-rank \$NODE_RANK --dist-init-addr \$DIST_ADDR:\${DIST_PORT:-29500}" @@ -948,6 +1052,7 @@ bash "\$CIDIR/install_checkout_sglang.sh" if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-} fi +bash "\$CIDIR/warm_remote_code.sh" exec python3 -m sglang.launch_server \ --model-path $MODEL_PATH --host 0.0.0.0 --port $PPORT \ $PREFILL_COMMON_FLAGS "\${MODEL_SERVER_ARGS[@]}" \ @@ -963,6 +1068,7 @@ bash "\$CIDIR/install_checkout_sglang.sh" if [[ "\${SGLANG_USE_CHECKOUT_RUNTIME:-1}" != "0" ]]; then export PYTHONPATH=/tmp/sglang-checkout-runtime/python:\${PYTHONPATH:-} fi +bash "\$CIDIR/warm_remote_code.sh" exec python3 -m sglang.launch_server \ --model-path $MODEL_PATH --host 0.0.0.0 --port $DPORT \ $DECODE_COMMON_FLAGS "\${MODEL_SERVER_ARGS[@]}" \ @@ -1190,7 +1296,18 @@ container_log() { # # flaky step that produced the false "missing" also truncated the 5-line # prefill log to 0 bytes, destroying the only evidence of what happened. local tmp="$3.fetch" - srun_local_or_step "$1" docker logs "$2" > "$tmp" 2>/dev/null || true + # `docker logs` writes the container's stdout and stderr to its own stdout and + # stderr, so merge them on the target host -- hence bash -c rather than passing + # `docker logs` straight through. Without it only the remote branch of + # srun_local_or_step kept stderr; the local branch runs docker directly and the + # 2>/dev/null below swallowed it. Every sglang scheduler line goes to stderr and + # only uvicorn's access log goes to stdout, so the engine sitting on the node + # that runs drive.sh -- always decode node 0 -- logged nothing but + # "POST /generate 200 OK" for the whole run, with no trace of the crash that + # ended it. Visible in 2p1d-ep16, where decode node 1 is fetched remotely and + # has the full log while node 0 has none of it. The outer 2>/dev/null still + # drops srun's own dispatch noise. + srun_local_or_step "$1" bash -c "docker logs $2 2>&1" > "$tmp" 2>/dev/null || true # Publish in place (truncate + rewrite the SAME inode), never by renaming over # the target. drive.sh keeps a `tail -F` on bench.log, and swapping the inode # every 10s made tail reopen and re-emit the whole file each poll -- the @@ -1560,12 +1677,15 @@ echo "[drive_batch] \$(hostname) rank \${SPUR_NODEID:-?} elected driver" # records: a leg that failed in the monitor loop left an sbatch.out that simply # stopped mid-run, with the teardown lines never visible. # -# A plain redirect, deliberately NOT `| tee`. With a pipeline, drive_batch waits -# for the whole pipeline, and that does not finish when drive.sh does -- the -# backgrounded `tail -F` on bench.log inherits drive.sh's stdout and holds the -# pipe's write end open. drive.sh exited 0 and drive_exit was still unwritten -# 11 minutes later, so the standby tasks kept the allocation alive and the leg -# looked hung after it had actually passed. +# A plain redirect, deliberately not a tee pipeline. With a pipeline, +# drive_batch waits for the whole pipeline, and that does not finish when +# drive.sh does -- the backgrounded tail on bench.log inherits drive.sh's +# stdout and holds the pipe's write end open. drive.sh exited 0 and drive_exit +# was still unwritten 11 minutes later, so the standby tasks kept the +# allocation alive and the leg looked hung after it had actually passed. +# No backticks anywhere in this heredoc: it is unquoted, so bash runs them at +# generation time. The pair that used to be here printed a syntax error on +# every launch and ran a stray tail on the submitting host. bash "$WORKDIR/drive.sh" "$WORKDIR" "$PW" "$DW" "$PN_PER" "$DN_PER" "$DIST_SOCK" "$ADDR_NIC" \ > "$WORKDIR/drive_\$(hostname).log" 2>&1 echo \$? > "$WORKDIR/drive_exit" @@ -1589,25 +1709,75 @@ EOF # glance, from a model bug. Observed on glm52-fp4-1k1k-2p1d-ep16 2026-09-17. # Retry only on that signature; a genuinely bad sbatch request must still # fail immediately rather than being retried six times. - SPUR_LEG_JOB_ID="" - for _attempt in 1 2 3 4 5 6; do - SBATCH_MSG=$(sbatch -p "$SLURM_PARTITION" -N"$TOTAL_NODES" "${NODELIST_ARG[@]}" \ - "${EXCLUDE_ARG[@]}" "${EXCLUSIVE_ARG[@]}" "${ACCT_ARG[@]}" \ - --job-name "$JOB_NAME" -t "$TIME_LIMIT" \ - --output "$SBATCH_OUT" --error "$SBATCH_OUT" \ - "$WORKDIR/drive_batch.sh" 2>&1) - echo "$SBATCH_MSG" - SPUR_LEG_JOB_ID="${SBATCH_MSG##* }" - [[ "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]] && break - if [[ "$SBATCH_MSG" == *"not the Raft leader"* \ - || "$SBATCH_MSG" == *"service is currently unavailable"* ]]; then - echo "[launch] spur controller unavailable (attempt $_attempt/6); retrying in 20s" >&2 - SPUR_LEG_JOB_ID="" - sleep 20 - continue - fi - break - done + spur_submit_leg() { + SPUR_LEG_JOB_ID="" + : > "$SBATCH_OUT" + for _attempt in 1 2 3 4 5 6; do + SBATCH_MSG=$(sbatch -p "$SLURM_PARTITION" -N"$TOTAL_NODES" "${NODELIST_ARG[@]}" \ + "${EXCLUDE_ARG[@]}" "${EXCLUSIVE_ARG[@]}" "${ACCT_ARG[@]}" \ + --job-name "$JOB_NAME" -t "$TIME_LIMIT" \ + --output "$SBATCH_OUT" --error "$SBATCH_OUT" \ + "$WORKDIR/drive_batch.sh" 2>&1) + echo "$SBATCH_MSG" + SPUR_LEG_JOB_ID="${SBATCH_MSG##* }" + [[ "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]] && break + if [[ "$SBATCH_MSG" == *"not the Raft leader"* \ + || "$SBATCH_MSG" == *"service is currently unavailable"* ]]; then + echo "[launch] spur controller unavailable (attempt $_attempt/6); retrying in 20s" >&2 + SPUR_LEG_JOB_ID="" + sleep 20 + continue + fi + break + done + } + + # Spur does not hold a job it cannot place. When the partition is full it + # cancels the submission within seconds -- JobState=CANCELLED, + # Reason=QOSGrpNodeLimit, StartTime=N/A -- and the monitor below then sees the + # job leave the queue having produced nothing, which it reports as an + # incomplete sweep. That is a red result for a test that never ran. Observed + # 2026-09-23 on jobs 4124 and 4125, both cancelled inside 40s while the four + # amd-sglang nodes were held by another run. + # + # A job can also be cancelled just after it is placed: job 4147 on 2026-09-24 + # had StartTime set and RunTime=00:00:05, long enough to start the containers + # and nothing else. Keying on StartTime=N/A alone misses that, so also accept + # a cancelled job that produced no bench results -- a real sweep writes + # raw_conc*.json and bench_exit, and neither can appear in five seconds. + spur_never_started() { + local _info + _info=$(scontrol show job "$1" 2>/dev/null) || return 1 + [[ "$_info" == *"JobState=CANCELLED"* ]] || return 1 + [[ "$_info" == *"StartTime=N/A"* ]] && return 0 + [[ -f "$WORKDIR/bench_exit" ]] && return 1 + ! compgen -G "$WORKDIR/raw_conc*.json" > /dev/null + } + # squeue is not authoritative for "has this job left the queue". It asks the + # spur controller for the whole queue, and a controller hiccup -- the same + # Raft leader election spur_submit_leg already retries around -- answers with + # an empty list, the error swallowed by 2>/dev/null. A single miss is enough + # to end the leg. + # + # That is what happened to job 4177 on 2026-09-24. The monitor decided the + # job was gone 21 minutes into a healthy sweep, waited 90s for a drive_exit + # that could not arrive because the job was still benchmarking, reported + # "incomplete sweep: 3/7", and then scancelled the run it had just given up + # on -- the teardown below found the job in squeue, because it had never + # left. scontrol, which answers per job, still had it RUNNING throughout. + # Ask scontrol before believing the queue listing. + spur_job_alive() { + squeue -h -o "%i" 2>/dev/null | grep -qx "$1" && return 0 + local _info + _info=$(scontrol show job "$1" 2>/dev/null) || return 1 + [[ "$_info" == *"JobState=RUNNING"* \ + || "$_info" == *"JobState=PENDING"* \ + || "$_info" == *"JobState=CONFIGURING"* \ + || "$_info" == *"JobState=COMPLETING"* ]] + } + SPUR_RESUBMITS_LEFT=${SPUR_RESUBMITS_LEFT:-20} + + spur_submit_leg if [[ ! "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]]; then echo "ERROR: could not parse a job id out of: $SBATCH_MSG" >&2 SALLOC_RC=1 @@ -1620,7 +1790,25 @@ EOF # ever running it (node failure, scheduler kill, time limit). while :; do [[ -f "$WORKDIR/drive_exit" ]] && break - if ! squeue -h -o "%i" 2>/dev/null | grep -qx "$SPUR_LEG_JOB_ID"; then + if ! spur_job_alive "$SPUR_LEG_JOB_ID"; then + # Cancelled before it ever ran: wait for the nodes and submit + # again rather than reporting a result we never measured. + if spur_never_started "$SPUR_LEG_JOB_ID" && (( SPUR_RESUBMITS_LEFT > 0 )); then + SPUR_RESUBMITS_LEFT=$((SPUR_RESUBMITS_LEFT - 1)) + echo "[launch] spur job $SPUR_LEG_JOB_ID cancelled before start;" \ + "resubmitting in 5m ($SPUR_RESUBMITS_LEFT left)" >&2 + kill "$SPUR_TAIL_PID" 2>/dev/null || true + sleep 300 + spur_submit_leg + if [[ ! "$SPUR_LEG_JOB_ID" =~ ^[0-9]+$ ]]; then + echo 1 > "$WORKDIR/drive_exit" + break + fi + echo "[launch] spur job $SPUR_LEG_JOB_ID submitted; streaming $SBATCH_OUT" + tail -F "$SBATCH_OUT" 2>/dev/null & + SPUR_TAIL_PID=$! + continue + fi # The job has left the queue. drive_exit is written on a compute # node and read here on the login node, so NFS close-to-open # visibility can delay it well past a single short sleep. A five @@ -1683,7 +1871,7 @@ EOF # 1774/1775 held all four nodes long after their legs had been recorded # rc=1. Unconditional, because reaching here means this leg is done with # its nodes either way. - if squeue -h -o "%i" 2>/dev/null | grep -qx "$SPUR_LEG_JOB_ID"; then + if spur_job_alive "$SPUR_LEG_JOB_ID"; then echo "[launch] releasing spur job $SPUR_LEG_JOB_ID" scancel "$SPUR_LEG_JOB_ID" 2>/dev/null || true fi diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16.yaml index f881654316..f2a747d405 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp8/dsv4pro/1k1k/2p1d-ep16.yaml @@ -58,7 +58,13 @@ runtime: # differ, so these split what the single-node EP<=8 recipes leave symmetric. wide_ep: kv_cache_dtype: fp8_e4m3 - prefill_mem_fraction_static: 0.8 + # 0.85, not the 0.8 the other ep16 recipes use. FP8 Pro weights are 218.92 GB + # per rank; at 0.8 of a 288 GB card the static budget is 230.4 GB, leaving + # ~11 GB for activations and the KV pool. max_total_num_tokens resolves to 0 + # and the prefill engine aborts with "Not enough memory. Please try to + # increase --mem-fraction-static." The FP4 recipes fit at 0.8 because their + # weights are half the size, and the FP8 MTP recipe already runs at 0.85. + prefill_mem_fraction_static: 0.85 decode_mem_fraction_static: 0.85 prefill_chunked_prefill_size: 131072 prefill_max_running_requests: 1024 diff --git a/scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/1p1d-mtp.yaml b/scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/1p1d-mtp.yaml index 4e6ef7b5f2..b0fac0fca1 100644 --- a/scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/1p1d-mtp.yaml +++ b/scripts/ci/slurm/recipes/mi355x-fp8/kimik26/1k1k/1p1d-mtp.yaml @@ -66,6 +66,16 @@ runtime: decode_bootstrap_port: 9001 lb_port: 8000 mem_fraction_static: 0.90 + # Decode-side headroom for MTP. With the draft head the decode server ends + # startup with 12.9 GB free per GPU (draft weights + a 4.6 GB draft-decode + # graph + a 9.8 GB target-verify graph, against 2.8 GB for the non-MTP target + # graph); the non-MTP sibling leaves 24.2 GB at the identical 0.90. At + # concurrency 256 the draft-extend path overruns that margin and ROCr aborts + # the queues with HSA_STATUS_ERROR_OUT_OF_RESOURCES ("Available Free mem : + # 0 MB"), killing a scheduler process. 0.85 buys back ~19 GB on decode only; + # prefill and every other emitted value are unchanged. + wide_ep: + decode_mem_fraction_static: 0.85 page_size: 256 max_running_requests: 256 chunked_prefill_size: 8192