Files
sglang/deploy/b300-k8s/clean_l3ssd_cache.sh

31 lines
1.0 KiB
Bash

#!/usr/bin/env bash
# Reset the mooncake L3 SSD KV cache before an experiment.
#
# MUST run on b300-01 only after the ds41-gist-exp workload is fully stopped:
# 1) kmctl stop cw ds41-gist-exp --wait ...
# 2) kubectl get cw ... -> Terminated 0/0, nvidia-smi -> 0 MiB on GPU0-7
# 3) bash clean_l3ssd_cache.sh
# 4) kmctl update/start the new manifest
#
# The script refuses to delete anything while a mooncake master or sglang
# server process is still visible on the host.
set -euo pipefail
SSD_DIR="${SSD_DIR:-/data/ymk/mooncake-ssd}"
if pgrep -af '[m]ooncake_master|[s]glang serve' >/dev/null 2>&1; then
echo "refusing: mooncake_master or sglang serve is still running" >&2
pgrep -af '[m]ooncake_master|[s]glang serve' >&2 || true
exit 1
fi
if [ ! -d "$SSD_DIR" ]; then
echo "SSD cache dir does not exist: $SSD_DIR"
exit 0
fi
echo "before: $(du -sh "$SSD_DIR" | awk '{print $1}') ($SSD_DIR)"
find "$SSD_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
echo "after: $(du -sh "$SSD_DIR" | awk '{print $1}') ($SSD_DIR)"
df -h /data | tail -1