31 lines
1.0 KiB
Bash
31 lines
1.0 KiB
Bash
#!/usr/bin/env bash
|
|
# Reset the mooncake L3 SSD KV cache before an experiment.
|
|
#
|
|
# MUST run on b300-01 only after the ds41-gist-exp workload is fully stopped:
|
|
# 1) kmctl stop cw ds41-gist-exp --wait ...
|
|
# 2) kubectl get cw ... -> Terminated 0/0, nvidia-smi -> 0 MiB on GPU0-7
|
|
# 3) bash clean_l3ssd_cache.sh
|
|
# 4) kmctl update/start the new manifest
|
|
#
|
|
# The script refuses to delete anything while a mooncake master or sglang
|
|
# server process is still visible on the host.
|
|
set -euo pipefail
|
|
|
|
SSD_DIR="${SSD_DIR:-/data/ymk/mooncake-ssd}"
|
|
|
|
if pgrep -af '[m]ooncake_master|[s]glang serve' >/dev/null 2>&1; then
|
|
echo "refusing: mooncake_master or sglang serve is still running" >&2
|
|
pgrep -af '[m]ooncake_master|[s]glang serve' >&2 || true
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -d "$SSD_DIR" ]; then
|
|
echo "SSD cache dir does not exist: $SSD_DIR"
|
|
exit 0
|
|
fi
|
|
|
|
echo "before: $(du -sh "$SSD_DIR" | awk '{print $1}') ($SSD_DIR)"
|
|
find "$SSD_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
|
|
echo "after: $(du -sh "$SSD_DIR" | awk '{print $1}') ($SSD_DIR)"
|
|
df -h /data | tail -1
|