From 8f53c1fb736d2647288a5ca6f56ad3cf9583b7f7 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Sun, 15 Nov 2026 20:33:38 +0800 Subject: [PATCH] deploy(l3ssd): offload_on_evict=false, 4TB cap, L3 reset script --- deploy/b300-k8s/README.md | 23 ++++++++++++++ deploy/b300-k8s/clean_l3ssd_cache.sh | 30 +++++++++++++++++++ ...s41-gist-exp-l3ssd-fastokens-20261115.yaml | 6 ++-- 3 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 deploy/b300-k8s/clean_l3ssd_cache.sh diff --git a/deploy/b300-k8s/README.md b/deploy/b300-k8s/README.md index e5b09c2536..0d34d2f242 100644 --- a/deploy/b300-k8s/README.md +++ b/deploy/b300-k8s/README.md @@ -114,3 +114,26 @@ P/D 的 `sglang serve` 参数**完全一致**;差异只在 mooncake: - fresh 轮均值:A rps 23.67 / p95 5.04s;B rps 24.04 / p95 3.56s (B +1.6% rps、p95 −29%)。 - 结论:保留 fastokens;之前未重启的 B2/B3(p95 5.99/6.69s)仅作历史参考、不纳入。 + +## 8. L3 SSD 4TB + offload_on_evict=false(2026-11-15) + +- 启动清单 `ds41-gist-exp-l3ssd-fastokens-20261115.yaml` 已改:master + `--offload_on_evict=false`;P 的 `MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE` / + `MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES` 改为 1 TiB/rank,4 rank 合计 + master 报 `SSD Storage: 0 B / 4.00 TB`。 +- 每次实验前必须重置 L3:stop cw → 确认 Terminated 0/0 + GPU 0 MiB → + `bash clean_l3ssd_cache.sh`(删除 `/data/ymk/mooncake-ssd` 下的 rank_*)→ update/start。 + 本次清理释放 1.2T,`/data` 可用 6.4T→7.5T。 +- c580(1200s,warmup 256,seed 42,fastokens,干净 L3): + rps 24.003、in 2.285M tok/s、out 22,447、TTFT p50/p95/p99 1.768/3.500/7.549s、 + ITL 8.15/11.74/15.31ms、29,764 请求、0 错误、valid。 + 对比 B1/B2:24.045/1.746/3.676/6.820 与 24.040/1.708/3.451/7.105—— + 吞吐与尾延迟均在方差内,**没有可归因于 offload_on_evict/4TB/清理的收益或回归**。 +- **重要:`offload_on_evict=false` 并没有停止 SSD 写入**。run 后 master + `SSD Storage: 533.81 GB / 4.00 TB`,`du /data/ymk/mooncake-ssd = 535G`; + 这是 `--hicache-write-policy write_through` 的 backup 写入,和内存 eviction 无关。 + 若要真正避免 SSD 写入,需要改 `write_back` 或关闭 `--enable_offload`,但那会改变 + L3 语义,需另做 A/B。 +- run 后 P cgroup `memory.current=2,198,904,565,760` ≈ 2047.5 GiB,已顶到 2 TiB + limit;宿主 MemAvailable 1.2 TiB。下一次实验前必须先停/清/重启;不要在同一实例 + 上连续跑,否则 L3 与 cgroup 都处于顶格状态。 diff --git a/deploy/b300-k8s/clean_l3ssd_cache.sh b/deploy/b300-k8s/clean_l3ssd_cache.sh new file mode 100644 index 0000000000..8baa64ac88 --- /dev/null +++ b/deploy/b300-k8s/clean_l3ssd_cache.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Reset the mooncake L3 SSD KV cache before an experiment. +# +# MUST run on b300-01 only after the ds41-gist-exp workload is fully stopped: +# 1) kmctl stop cw ds41-gist-exp --wait ... +# 2) kubectl get cw ... -> Terminated 0/0, nvidia-smi -> 0 MiB on GPU0-7 +# 3) bash clean_l3ssd_cache.sh +# 4) kmctl update/start the new manifest +# +# The script refuses to delete anything while a mooncake master or sglang +# server process is still visible on the host. +set -euo pipefail + +SSD_DIR="${SSD_DIR:-/data/ymk/mooncake-ssd}" + +if pgrep -af '[m]ooncake_master|[s]glang serve' >/dev/null 2>&1; then + echo "refusing: mooncake_master or sglang serve is still running" >&2 + pgrep -af '[m]ooncake_master|[s]glang serve' >&2 || true + exit 1 +fi + +if [ ! -d "$SSD_DIR" ]; then + echo "SSD cache dir does not exist: $SSD_DIR" + exit 0 +fi + +echo "before: $(du -sh "$SSD_DIR" | awk '{print $1}') ($SSD_DIR)" +find "$SSD_DIR" -mindepth 1 -maxdepth 1 -exec rm -rf {} + +echo "after: $(du -sh "$SSD_DIR" | awk '{print $1}') ($SSD_DIR)" +df -h /data | tail -1 diff --git a/deploy/b300-k8s/ds41-gist-exp-l3ssd-fastokens-20261115.yaml b/deploy/b300-k8s/ds41-gist-exp-l3ssd-fastokens-20261115.yaml index 6913dcd139..bdad38e8f8 100644 --- a/deploy/b300-k8s/ds41-gist-exp-l3ssd-fastokens-20261115.yaml +++ b/deploy/b300-k8s/ds41-gist-exp-l3ssd-fastokens-20261115.yaml @@ -36,7 +36,7 @@ spec: - command: - /bin/bash - -c - - "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=true --promotion_on_hit=false --offload_force_evict=false\n" + - "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n" image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 imagePullPolicy: IfNotPresent name: mooncake-master @@ -199,9 +199,9 @@ spec: - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR value: bucket_storage_backend - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE - value: '549755813888' + value: '1099511627776' - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES - value: '549755813888' + value: '1099511627776' - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY value: fifo - name: TMPDIR