diff --git a/deploy/b300-ds41/bench_suite_v2.sh b/deploy/b300-ds41/bench_suite_v2.sh new file mode 100644 index 0000000000..d69f2c6088 --- /dev/null +++ b/deploy/b300-ds41/bench_suite_v2.sh @@ -0,0 +1,201 @@ +#!/bin/bash +# bench_suite_v2.sh — DeepSeek-V4.1-Flash on b300-01,2026-10-09 新版基线方法 +# +# 相对 v1 的方法变化: +# 1. 服务刚起数据不准 → 每个测试模式自带 warmup 轮(结果丢弃,标记 WARM) +# 2. prefill 冷/热双轮:冷轮写入 hicache(L1/L3),同 seed 热轮测纯 cache-hit TPS +# 3. decode 阶梯并发:步长 32,上限 = cuda-graph-max-bs-decode 的 80%; +# 每档先 warm(灌 L1)再测,prefill 全 cache hit; +# 报告 TPOT + 整实例输出 TPS + 推算实际并发(= 输出TPS × TPOT / 1000) +# 4. l3 模式(3p1d 经 router):大语料溢出 L1 后重放,验证 L3 命中 +# +# 在宿主机执行,bench 客户端通过 docker exec 进容器跑(需要 /data 挂载)。 +# +# 用法: +# bench_suite_v2.sh warmup +# bench_suite_v2.sh prefill +# bench_suite_v2.sh decode [cg_max_bs=256] +# bench_suite_v2.sh l3 [num_prompts=24] +# bench_suite_v2.sh report +# 产物: /data/ymk/ds41/bench_v2_.log +set -u + +BASE=/data/ymk/ds41 +MODEL=${MODEL:-/data/models/DeepSeek-V4.1-Flash} +HDR="Authorization=Bearer deepseek-v4-flash-0731-key" + +usage() { sed -n 2,20p "$0" >&2; exit 1; } + +# $1=container $2=port $3=tag -> 生成可执行脚本路径并 echo +emit_common() { + cat < "$gen" + chmod +x "$gen" + docker exec "$c" bash "$gen" +} + +# ---------------- prefill(冷/热双轮) ---------------- +cmd_prefill() { + local tag=${1:?tag} c=${2:?container} port=${3:?port} + local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh + { emit_common "$c" "$port" + # warmup(丢弃) + cat <<'EOF' +run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \ + --dataset-name random-ids --random-input-len 4096 --random-output-len 16 \ + --num-prompts 8 --max-concurrency 8 --seed 777000 --tokenize-prompt \ + --random-range-ratio 1.0 --header "$HDR" +EOF + # 冷轮 + for stage in "bs1_16000 16000 1 4 1" "bs1_160000 160000 1 4 2" "bs16_4096 4096 16 32 3"; do + set -- $stage + cat < "$gen" + chmod +x "$gen" + nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 & + echo "launched pid=$! seeds=${p}1..3 log=$BASE/bench_v2_${tag}.log" +} + +# ---------------- decode 阶梯 ---------------- +cmd_decode() { + local tag=${1:?tag} c=${2:?container} port=${3:?port} cg=${4:-256} + local maxc=$(( cg * 80 / 100 / 32 * 32 )) # 80% 向下取整到 32 的倍数 + local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh + echo "ladder: 32..$maxc step 32 (cg_max_bs=$cg)" + { emit_common "$c" "$port" + cat <<'EOF' +run WARMUP --backend sglang --host 127.0.0.1 --port $PORT --model MODELPH \ + --dataset-name random-ids --random-input-len 2048 --random-output-len 128 \ + --num-prompts 16 --max-concurrency 16 --seed 777000 --tokenize-prompt \ + --random-range-ratio 1.0 --header "$HDR" +EOF + # bs1 延迟地板(warm + measure) + cat < "$gen" + chmod +x "$gen" + nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 & + echo "launched pid=$! log=$BASE/bench_v2_${tag}.log" +} + +# ---------------- L3 语料溢出(3p1d 经 router) ---------------- +cmd_l3() { + local tag=${1:?tag} c=${2:?container} port=${3:?port} np=${4:-24} + local p=$(date +%H%M%S) gen=$BASE/.bench_v2_gen_${tag}.sh + { emit_common "$c" "$port" + cat < "$gen" + chmod +x "$gen" + nohup docker exec "$c" bash "$gen" > "$BASE/bench_v2_${tag}.log" 2>&1 & + echo "launched pid=$! corpus=${np}x160k log=$BASE/bench_v2_${tag}.log" +} + +# ---------------- report ---------------- +cmd_report() { + local log=${1:?log required} + awk ' + /START:/ { name=$4; sub(/:$/, "", name); tps=""; tpot=""; ttft=""; acc="" } + /Output token throughput/ { tps=$NF } + /Mean TPOT/ { tpot=$(NF-1) } + /Mean TTFT/ { ttft=$(NF-1) } + /[Aa]ccept/ { acc=$0; gsub(/^ +/, "", acc) } + /END\(exit/ { + line=sprintf "%-22s out_tps=%-10s tpot_ms=%-8s ttft_ms=%-8s", name, tps, tpot, ttft + if (tps != "" && tpot != "" && tpot+0 > 0) + line=line sprintf " eff_concurrency=%.1f", tps * tpot / 1000 + print line + if (acc != "") print " " acc + } + ' "$log" +} + +case "${1:-}" in + warmup) shift; cmd_warmup "$@" ;; + prefill) shift; cmd_prefill "$@" ;; + decode) shift; cmd_decode "$@" ;; + l3) shift; cmd_l3 "$@" ;; + report) shift; cmd_report "$@" ;; + *) usage ;; +esac diff --git a/deploy/b300-ds41/dockerserve-d-standalone.yml b/deploy/b300-ds41/dockerserve-d-standalone.yml new file mode 100644 index 0000000000..3dee5602a6 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-d-standalone.yml @@ -0,0 +1,49 @@ +# 单独 D 节点基线(非 PD role):3p1d 的 d1 配置去掉 disaggregation,直接对外服务 +# 用途:测单 D 节点 decode 基线(阶梯并发 32..80%*cuda-graph-max-bs-decode, +# 报 TPOT / 实际并发 / 整实例输出 TPS;prefill 走 L1 radix cache-hit) +# GPU 6,7;端口 30001;无 hicache 依赖(L1 即可支撑重复 prompt 命中) +# 2026-10-09 建:配合 bench_suite_v2.sh decode +x-sglang-common: &sglang-common + image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831 + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + +services: + d1s: + <<: *sglang-common + container_name: ds41-d1-standalone + environment: + SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_RAGGED_VERIFY_MODE: static + CUDA_VISIBLE_DEVICES: "6,7" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 --dp-size 2 + --enable-dp-attention --enable-dp-lm-head + --mem-fraction-static 0.80 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 256 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 256 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30001 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 diff --git a/deploy/b300-ds41/dockerserve-p-standalone.yml b/deploy/b300-ds41/dockerserve-p-standalone.yml new file mode 100644 index 0000000000..87713257c1 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-p-standalone.yml @@ -0,0 +1,60 @@ +# 单独 P 节点基线(非 PD role):3p1d 的 p1 配置去掉 disaggregation,直接对外服务 +# 用途:测单 P 节点 prefill 基线吞吐(冷)+ hicache cache-hit TPS(热) +# GPU 0,1;端口 30000;依赖 mooncake-master(hicache L3,127.0.0.1:50051) +# 2026-10-09 建:配合 bench_suite_v2.sh prefill +x-sglang-common: &sglang-common + image: ymkymx/sglang:dsv41-pd-visioncp-1068926cd-local-20261008-0831 + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + +services: + p1s: + <<: *sglang-common + container_name: ds41-p1-standalone + environment: + SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_RAGGED_VERIFY_MODE: static + MC_INTRANODE_NVLINK: "true" + MC_INTRA_NVLINK: "true" + SGLANG_MOONCAKE_SEND_AUX_TCP: "1" + SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json + CUDA_VISIBLE_DEVICES: "0,1" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 256 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30000 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0