deploy: bench_suite_v2 (warmup/prefill冷热双轮/decode阶梯80%cg/L3语料溢出) + 单独P/D基线compose
This commit is contained in:
@@ -0,0 +1,201 @@
|
||||
#!/bin/bash
|
||||
# bench_suite_v2.sh — DeepSeek-V4.1-Flash on b300-01,2026-10-09 新版基线方法
|
||||
#
|
||||
# 相对 v1 的方法变化:
|
||||
# 1. 服务刚起数据不准 → 每个测试模式自带 warmup 轮(结果丢弃,标记 WARM)
|
||||
# 2. prefill 冷/热双轮:冷轮写入 hicache(L1/L3),同 seed 热轮测纯 cache-hit TPS
|
||||
# 3. decode 阶梯并发:步长 32,上限 = cuda-graph-max-bs-decode 的 80%;
|
||||
# 每档先 warm(灌 L1)再测,prefill 全 cache hit;
|
||||
# 报告 TPOT + 整实例输出 TPS + 推算实际并发(= 输出TPS × TPOT / 1000)
|
||||
# 4. l3 模式(3p1d 经 router):大语料溢出 L1 后重放,验证 L3 命中
|
||||
#
|
||||
# 在宿主机执行,bench 客户端通过 docker exec 进容器跑(需要 /data 挂载)。
|
||||
#
|
||||
# 用法:
|
||||
# bench_suite_v2.sh warmup <container> <port>
|
||||
# bench_suite_v2.sh prefill <tag> <container> <port>
|
||||
# bench_suite_v2.sh decode <tag> <container> <port> [cg_max_bs=256]
|
||||
# bench_suite_v2.sh l3 <tag> <container> <port> [num_prompts=24]
|
||||
# bench_suite_v2.sh report <log>
|
||||
# 产物: /data/ymk/ds41/bench_v2_<tag>.log
|
||||
set -u
|
||||
|
||||
BASE=/data/ymk/ds41
|
||||
MODEL=${MODEL:-/data/models/DeepSeek-V4.1-Flash}
|
||||
HDR="Authorization=Bearer deepseek-v4-flash-0731-key"
|
||||
|
||||
usage() { sed -n 2,20p "$0" >&2; exit 1; }
|
||||
|
||||
# $1=container $2=port $3=tag -> 生成可执行脚本路径并 echo
|
||||
emit_common() {
|
||||
cat <<EOF
|
||||
cd $BASE
|
||||
HDR="$HDR"
|
||||
PORT=$2
|
||||
run() {
|
||||
local name="\$1"; shift
|
||||
echo "===== [\$(date +%H:%M:%S)] START: \$name ====="
|
||||
python3 -m sglang.bench_serving "\$@"
|
||||
echo "===== [\$(date +%H:%M:%S)] END(exit=\$?): \$name ====="
|
||||
sleep 5
|
||||
}
|
||||
EOF
|
||||
}
|
||||
|
||||
# ---------------- warmup ----------------
|
||||
cmd_warmup() {
|
||||
local c=${1:?container} port=${2:?port}
|
||||
local gen=$BASE/.bench_v2_warmup.sh
|
||||
{ emit_common "$c" "$port"; cat <<'EOF'
|
||||
run WARMUP_a --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777001 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
run WARMUP_b --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 16000 --random-output-len 1 \
|
||||
--num-prompts 2 --max-concurrency 1 --seed 777002 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
echo "===== [$(date +%H:%M:%S)] ALL DONE ====="
|
||||
EOF
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
docker exec "$c" bash "$gen"
|
||||
}
|
||||
|
||||
# ---------------- prefill(冷/热双轮) ----------------
|
||||
cmd_prefill() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port}
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
{ emit_common "$c" "$port"
|
||||
# warmup(丢弃)
|
||||
cat <<'EOF'
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
EOF
|
||||
# 冷轮
|
||||
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
|
||||
set -- $stage
|
||||
cat <<EOF
|
||||
run COLD_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
|
||||
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
# 热轮(同 seed → cache hit)
|
||||
for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do
|
||||
set -- $stage
|
||||
cat <<EOF
|
||||
run HIT_$1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len $2 --random-output-len 1 \\
|
||||
--num-prompts $4 --max-concurrency $3 --seed ${p}$5 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! seeds=${p}1..3 log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- decode 阶梯 ----------------
|
||||
cmd_decode() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port} cg=${4:-256}
|
||||
local maxc=$(( cg * 80 / 100 / 32 * 32 )) # 80% 向下取整到 32 的倍数
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
echo "ladder: 32..$maxc step 32 (cg_max_bs=$cg)"
|
||||
{ emit_common "$c" "$port"
|
||||
cat <<'EOF'
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 128 \
|
||||
--num-prompts 16 --max-concurrency 16 --seed 777000 --tokenize-prompt \
|
||||
--random-range-ratio 1.0 --header "$HDR"
|
||||
EOF
|
||||
# bs1 延迟地板(warm + measure)
|
||||
cat <<EOF
|
||||
run WARM_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run DEC_bs1 --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts 4 --max-concurrency 1 --seed ${p}01 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
local cc
|
||||
for (( cc=32; cc<=maxc; cc+=32 )); do
|
||||
local np=$(( cc * 2 ))
|
||||
cat <<EOF
|
||||
run WARM_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run DEC_c${cc} --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 2048 --random-output-len 512 \\
|
||||
--num-prompts $np --max-concurrency $cc --seed ${p}1${cc} --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
done
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} | sed "s|MODELPH|$MODEL|g" > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- L3 语料溢出(3p1d 经 router) ----------------
|
||||
cmd_l3() {
|
||||
local tag=${1:?tag} c=${2:?container} port=${3:?port} np=${4:-24}
|
||||
local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh
|
||||
{ emit_common "$c" "$port"
|
||||
cat <<EOF
|
||||
run WARMUP --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 4096 --random-output-len 16 \\
|
||||
--num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run L3_COLD_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
|
||||
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
run L3_HIT_160k --backend sglang --host 127.0.0.1 --port \$PORT --model $MODEL \\
|
||||
--dataset-name random-ids --random-input-len 160000 --random-output-len 1 \\
|
||||
--num-prompts $np --max-concurrency 8 --seed ${p}1 --tokenize-prompt \\
|
||||
--random-range-ratio 1.0 --header "\$HDR"
|
||||
EOF
|
||||
echo 'echo "===== [$(date +%H:%M:%S)] ALL DONE ====="'
|
||||
} > "$gen"
|
||||
chmod +x "$gen"
|
||||
nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 &
|
||||
echo "launched pid=$! corpus=${np}x160k log=$BASE/bench_v2_${tag}.log"
|
||||
}
|
||||
|
||||
# ---------------- report ----------------
|
||||
cmd_report() {
|
||||
local log=${1:?log required}
|
||||
awk '
|
||||
/START:/ { name=$4; sub(/:$/, "", name); tps=""; tpot=""; ttft=""; acc="" }
|
||||
/Output token throughput/ { tps=$NF }
|
||||
/Mean TPOT/ { tpot=$(NF-1) }
|
||||
/Mean TTFT/ { ttft=$(NF-1) }
|
||||
/[Aa]ccept/ { acc=$0; gsub(/^ +/, "", acc) }
|
||||
/END\(exit/ {
|
||||
line=sprintf "%-22s out_tps=%-10s tpot_ms=%-8s ttft_ms=%-8s", name, tps, tpot, ttft
|
||||
if (tps != "" && tpot != "" && tpot+0 > 0)
|
||||
line=line sprintf " eff_concurrency=%.1f", tps * tpot / 1000
|
||||
print line
|
||||
if (acc != "") print " " acc
|
||||
}
|
||||
' "$log"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
warmup) shift; cmd_warmup "$@" ;;
|
||||
prefill) shift; cmd_prefill "$@" ;;
|
||||
decode) shift; cmd_decode "$@" ;;
|
||||
l3) shift; cmd_l3 "$@" ;;
|
||||
report) shift; cmd_report "$@" ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
@@ -0,0 +1,49 @@
|
||||
# 单独 D 节点基线(非 PD role):3p1d 的 d1 配置去掉 disaggregation,直接对外服务
|
||||
# 用途:测单 D 节点 decode 基线(阶梯并发 32..80%*cuda-graph-max-bs-decode,
|
||||
# 报 TPOT / 实际并发 / 整实例输出 TPS;prefill 走 L1 radix cache-hit)
|
||||
# GPU 6,7;端口 30001;无 hicache 依赖(L1 即可支撑重复 prompt 命中)
|
||||
# 2026-10-09 建:配合 bench_suite_v2.sh decode
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
d1s:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-d1-standalone
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
@@ -0,0 +1,60 @@
|
||||
# 单独 P 节点基线(非 PD role):3p1d 的 p1 配置去掉 disaggregation,直接对外服务
|
||||
# 用途:测单 P 节点 prefill 基线吞吐(冷)+ hicache cache-hit TPS(热)
|
||||
# GPU 0,1;端口 30000;依赖 mooncake-master(hicache L3,127.0.0.1:50051)
|
||||
# 2026-10-09 建:配合 bench_suite_v2.sh prefill
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
services:
|
||||
p1s:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-p1-standalone
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
Reference in New Issue
Block a user