diff --git a/.gitea/workflows/build-image.yaml b/.gitea/workflows/build-image.yaml index 2c32ae2b9..d00e806c5 100644 --- a/.gitea/workflows/build-image.yaml +++ b/.gitea/workflows/build-image.yaml @@ -13,6 +13,11 @@ name: build-sglang-image on: push: branches: [dsv41-pd, dsv41-pd-visioncp] + # compose/部署配置 与 workflow 自身的改动不触发镜像构建(省 runner 与推送带宽)。 + # 改了 workflow 想重建镜像时,需伴随任意代码改动或手动触发。 + paths-ignore: + - 'deploy/**' + - '.gitea/**' env: # 直接写死 Gitea 自带 registry(vars context 在该实例上求值异常会导致回退 docker.io) diff --git a/deploy/b300-ds41/README.md b/deploy/b300-ds41/README.md new file mode 100644 index 000000000..fb19e8cc4 --- /dev/null +++ b/deploy/b300-ds41/README.md @@ -0,0 +1,33 @@ +# B300 ds41 部署 compose 档案 + +b300-01 上 DeepSeek-V4.1-Flash 各部署方案的 docker compose 存档。 +**本目录是 source of truth**;b300 上的运行目录 `/data/ymk/ds41/` 是工作副本。 + +方案说明、参数矩阵、特殊 env、已知坑见飞书部署手册(组内共享): +《B300 sglang 部署手册》 https://u04wb5irxz.feishu.cn/docx/FOi8dbnybodHr0xIfkPcQ6gtnRx + +## 同步流程 + +```bash +# 本机(D:\B300\sglang) +git add deploy/ && git commit -m "deploy: ..." +git push origin dsv41-pd-visioncp # Gitea(paths-ignore,不触发镜像构建) +git push b300 dsv41-pd-visioncp # b300 裸仓库 + +# b300-01 +git -C /data/ymk/sglang pull # deploy/ 出现在 /data/ymk/sglang/deploy/b300-ds41/ +``` + +## CI 说明 + +`.gitea/workflows/build-image.yaml` 的 push 触发器带 `paths-ignore: ['deploy/**', '.gitea/**']`, +改本目录不会触发镜像构建。注意:workflow 自身的改动也不再自动触发,需要重建镜像时 +伴随代码改动 push 或手动触发。 + +## 目录规则 + +- 命名:`dockerserve-<拓扑>-<特性>-<卡位>.yml`(如 `-b4` = 后 4 卡) +- 退役文件不要留在主目录:移入 `archive-YYYYMMDD/`(b300 侧同样执行) +- 禁止提交 `.bak` 备份文件 +- `mooncake-store.json` 是 hicache L3 mooncake store 配置,被 + `dockerserve-pd2-p-r1/r2.yml` 通过 `SGLANG_HICACHE_MOONCAKE_CONFIG_PATH` 引用 diff --git a/deploy/b300-ds41/dockerserve-3cp2-pd.yml b/deploy/b300-ds41/dockerserve-3cp2-pd.yml new file mode 100644 index 000000000..d9115e12d --- /dev/null +++ b/deploy/b300-ds41/dockerserve-3cp2-pd.yml @@ -0,0 +1,171 @@ +# 3x cp2-P + 1x dp2-D PD 分离 + L3 mooncake 互联缓存 + engram host table +# 2026-09-23 实验;基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md +# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7 +# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002 +# bootstrap: P1=8998 P2=8999 P3=9000 D=9001 +x-sglang-common: &sglang-common + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + +x-p-environment: &p-environment + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_RAGGED_VERIFY_MODE: static + MC_INTRANODE_NVLINK: "true" + MC_INTRA_NVLINK: "true" + SGLANG_MOONCAKE_SEND_AUX_TCP: "1" + SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json + SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" + +services: + p1: + <<: *sglang-common + container_name: ds41-cp2-p1 + environment: + <<: *p-environment + CUDA_VISIBLE_DEVICES: "0,1" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 64 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30000 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 8998 + + p2: + <<: *sglang-common + container_name: ds41-cp2-p2 + environment: + <<: *p-environment + CUDA_VISIBLE_DEVICES: "2,3" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 64 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30010 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 8999 + + p3: + <<: *sglang-common + container_name: ds41-cp2-p3 + environment: + <<: *p-environment + CUDA_VISIBLE_DEVICES: "4,5" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 64 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30020 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 9000 + + d: + <<: *sglang-common + container_name: ds41-cp2-d + environment: + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_RAGGED_VERIFY_MODE: static + MC_INTRANODE_NVLINK: "true" + MC_INTRA_NVLINK: "true" + SGLANG_MOONCAKE_SEND_AUX_TCP: "1" + SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" + CUDA_VISIBLE_DEVICES: "6,7" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 --dp-size 2 + --enable-dp-attention --enable-dp-lm-head + --mem-fraction-static 0.80 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 64 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30001 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --disaggregation-mode decode + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 9001 diff --git a/deploy/b300-ds41/dockerserve-b4-dp.yml b/deploy/b300-ds41/dockerserve-b4-dp.yml new file mode 100644 index 000000000..3805f3f49 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-b4-dp.yml @@ -0,0 +1,43 @@ +services: + sglang: + image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142 + container_name: ds41-sglang-b4 + shm_size: "32gb" + ipc: host + privileged: true + ports: + - "30001:30000" + environment: + - CUDA_VISIBLE_DEVICES=4,5,6,7 + - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --mem-fraction-static 0.60 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --max-running-requests 64 + --host 0.0.0.0 + --port 30000 + --dp-size 4 + --enable-dp-attention + + --enable-cache-report + --enable-metrics + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-b4-tp.yml b/deploy/b300-ds41/dockerserve-b4-tp.yml new file mode 100644 index 000000000..2d8664f88 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-b4-tp.yml @@ -0,0 +1,43 @@ +services: + sglang: + image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142 + container_name: ds41-sglang-b4 + shm_size: "32gb" + ipc: host + privileged: true + ports: + - "30001:30000" + environment: + - CUDA_VISIBLE_DEVICES=4,5,6,7 + - PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --mem-fraction-static 0.60 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --enable-decoder-swa-bounded-replay + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --max-running-requests 64 + --host 0.0.0.0 + --port 30000 + + + --enable-cache-report + --enable-metrics + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-b4.yml b/deploy/b300-ds41/dockerserve-b4.yml new file mode 100644 index 000000000..a4d51bfc3 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-b4.yml @@ -0,0 +1,42 @@ +services: + sglang: + image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142 + container_name: ds41-sglang-b4 + shm_size: "32gb" + ipc: host + privileged: true + ports: + - "30001:30000" + environment: + - CUDA_VISIBLE_DEVICES=4,5,6,7 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --enable-decoder-swa-bounded-replay + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --max-running-requests 64 + --host 0.0.0.0 + --port 30000 + --enable-prefill-cp + --cp-strategy interleave + --enable-cache-report + --enable-metrics + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-cp2x4.yml b/deploy/b300-ds41/dockerserve-cp2x4.yml new file mode 100644 index 000000000..ff3a8a620 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-cp2x4.yml @@ -0,0 +1,160 @@ +# 4×cp2 独立实例(非 PD)+ hicache L3(mooncake) + dspark + engram host table 卸载(2026-09-23 晚) +# 用户指定组合:4 cp2 hicache l3 dspark roundrobin loadbalance engram offload +# 每实例:tp2 ep2 + interleave CP2 + dspark b5 + hicache L3 write_through(size 0) + engram host table +# (tp2 权重必须靠 engram 卸载才放得下,见 goals-track G1.9 反转) +# GPU: a=0,1 b=2,3 c=4,5 d=6,7;端口: 30000/30010/30020/30030;rr router=30002 +# 参考:dockerserve-cpdspark-l3.yml(hicache 参数)、dockerserve-3cp2-pd.yml(cp2/engram 参数) +x-common: &common + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + shm_size: "32gb" + ipc: host + privileged: true + network_mode: host + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + # 热补丁(2026-09-23):CP 对称 idle 检查,修 health-check × hicache drain 死锁 + # 源文件:/data/ymk/sglang(dsv41-pd-visioncp 分支工作区已改);补丁脚本见 + # D:\B300\experiments\cp2x4\patch_healthcheck_cp_idle.py + - /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py + environment: &env + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_RAGGED_VERIFY_MODE: static + MC_MS_AUTO_DISC: "0" + MOONCAKE_MASTER: 127.0.0.1:50051 + MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE + MOONCAKE_PROTOCOL: tcp + MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + +services: + a: + <<: *common + container_name: ds41-cp2-a + environment: + <<: *env + CUDA_VISIBLE_DEVICES: "0,1" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 64 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30000 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + b: + <<: *common + container_name: ds41-cp2-b + environment: + <<: *env + CUDA_VISIBLE_DEVICES: "2,3" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 64 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30010 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + c: + <<: *common + container_name: ds41-cp2-c + environment: + <<: *env + CUDA_VISIBLE_DEVICES: "4,5" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 64 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30020 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + d: + <<: *common + container_name: ds41-cp2-d2 + environment: + <<: *env + CUDA_VISIBLE_DEVICES: "6,7" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 64 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30030 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 diff --git a/deploy/b300-ds41/dockerserve-cp8.yml b/deploy/b300-ds41/dockerserve-cp8.yml new file mode 100644 index 000000000..2bfa48a15 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-cp8.yml @@ -0,0 +1,44 @@ +# cp8 单机非 PD 测试(2026-09-23 晚) +# 背景:dp8 单机 dspark 健康但长上下文 prefill 结构性慢(单请求只落 1 rank), +# 换 cp8(tp8 + interleave prefill CP8,单请求 prefill 切到 8 卡)对照。 +# 关 engram、无 hicache L3、无 PD。cp+dspark 无 replay 是已知健康组合 +# (cp4+dspark+replay 三元组必崩,本配置不开 replay)。 +# GPU 0-7,端口 30000,容器 ds41-cp8,project cp8 +services: + cp8: + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + container_name: ds41-cp8 + shm_size: "32gb" + ipc: host + privileged: true + network_mode: host + environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + CUDA_VISIBLE_DEVICES: "0,1,2,3,4,5,6,7" + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 8 --ep-size 8 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.80 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 64 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30000 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 diff --git a/deploy/b300-ds41/dockerserve-cpdspark-l3-b4-nocap.yml b/deploy/b300-ds41/dockerserve-cpdspark-l3-b4-nocap.yml new file mode 100644 index 000000000..dc9593780 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-cpdspark-l3-b4-nocap.yml @@ -0,0 +1,50 @@ +services: + sglang: + image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536 + container_name: ds41-cpdspark-l3-b + shm_size: "32gb" + ipc: host + privileged: true + network_mode: host + environment: + - CUDA_VISIBLE_DEVICES=4,5,6,7 + - MC_MS_AUTO_DISC=0 + - MOONCAKE_MASTER=127.0.0.1:50051 + - MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE + - MOONCAKE_PROTOCOL=tcp + - MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 --ep-size 4 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto --tool-call-parser auto + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30001 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --enable-prefill-cp --cp-strategy interleave + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-cpdspark-l3-b4.yml b/deploy/b300-ds41/dockerserve-cpdspark-l3-b4.yml new file mode 100644 index 000000000..95033cc80 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-cpdspark-l3-b4.yml @@ -0,0 +1,51 @@ +services: + sglang: + image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536 + container_name: ds41-cpdspark-l3-b + shm_size: "32gb" + ipc: host + privileged: true + network_mode: host + environment: + - CUDA_VISIBLE_DEVICES=4,5,6,7 + - MC_MS_AUTO_DISC=0 + - MOONCAKE_MASTER=127.0.0.1:50051 + - MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE + - MOONCAKE_PROTOCOL=tcp + - MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 --ep-size 4 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto --tool-call-parser auto + --max-running-requests 64 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30001 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --enable-prefill-cp --cp-strategy interleave + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-cpdspark-l3-nocap.yml b/deploy/b300-ds41/dockerserve-cpdspark-l3-nocap.yml new file mode 100644 index 000000000..bad14d803 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-cpdspark-l3-nocap.yml @@ -0,0 +1,50 @@ +services: + sglang: + image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536 + container_name: ds41-cpdspark-l3-a + shm_size: "32gb" + ipc: host + privileged: true + network_mode: host + environment: + - CUDA_VISIBLE_DEVICES=0,1,2,3 + - MC_MS_AUTO_DISC=0 + - MOONCAKE_MASTER=127.0.0.1:50051 + - MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE + - MOONCAKE_PROTOCOL=tcp + - MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 --ep-size 4 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto --tool-call-parser auto + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30000 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --enable-prefill-cp --cp-strategy interleave + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-cpdspark-l3.yml b/deploy/b300-ds41/dockerserve-cpdspark-l3.yml new file mode 100644 index 000000000..b57274630 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-cpdspark-l3.yml @@ -0,0 +1,51 @@ +services: + sglang: + image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536 + container_name: ds41-cpdspark-l3-a + shm_size: "32gb" + ipc: host + privileged: true + network_mode: host + environment: + - CUDA_VISIBLE_DEVICES=0,1,2,3 + - MC_MS_AUTO_DISC=0 + - MOONCAKE_MASTER=127.0.0.1:50051 + - MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE + - MOONCAKE_PROTOCOL=tcp + - MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 --ep-size 4 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto --tool-call-parser auto + --max-running-requests 64 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30000 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --enable-prefill-cp --cp-strategy interleave + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-cpdspark-nr-b4.yml b/deploy/b300-ds41/dockerserve-cpdspark-nr-b4.yml new file mode 100644 index 000000000..1bd9254b4 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-cpdspark-nr-b4.yml @@ -0,0 +1,44 @@ +services: + sglang: + image: ymkymx/sglang:dsv41-pd-92632a60b-20260922-2100 + container_name: ds41-cpdspark-nr-b4 + shm_size: "32gb" + ipc: host + privileged: true + ports: + - "30001:30001" + environment: + - CUDA_VISIBLE_DEVICES=4,5,6,7 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --max-running-requests 64 + --host 0.0.0.0 + --port 30000 + --enable-cache-report + --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --enable-prefill-cp + --cp-strategy interleave + + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-cpdspark-nr.yml b/deploy/b300-ds41/dockerserve-cpdspark-nr.yml new file mode 100644 index 000000000..c9f5a7236 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-cpdspark-nr.yml @@ -0,0 +1,44 @@ +services: + sglang: + image: ymkymx/sglang:dsv41-pd-92632a60b-20260922-2100 + container_name: ds41-cpdspark-nr + shm_size: "32gb" + ipc: host + privileged: true + ports: + - "30000:30000" + environment: + - CUDA_VISIBLE_DEVICES=0,1,2,3 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --max-running-requests 64 + --host 0.0.0.0 + --port 30000 + --enable-cache-report + --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --enable-prefill-cp + --cp-strategy interleave + + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-cpdspark.yml b/deploy/b300-ds41/dockerserve-cpdspark.yml new file mode 100644 index 000000000..24ab5bbad --- /dev/null +++ b/deploy/b300-ds41/dockerserve-cpdspark.yml @@ -0,0 +1,44 @@ +services: + sglang: + image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142 + container_name: ds41-cpdspark + shm_size: "32gb" + ipc: host + privileged: true + ports: + - "30000:30000" + environment: + - CUDA_VISIBLE_DEVICES=0,1,2,3 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --max-running-requests 64 + --host 0.0.0.0 + --port 30000 + --enable-cache-report + --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --enable-prefill-cp + --cp-strategy interleave + --enable-decoder-swa-bounded-replay + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-dp2.yml b/deploy/b300-ds41/dockerserve-dp2.yml new file mode 100644 index 000000000..5153fc232 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-dp2.yml @@ -0,0 +1,62 @@ +# dp2 对照(2026-09-24 上午):验证 dp2(dp-attention)下 dspark 是否正常。 +# 与 dockerserve-tp2.yml 逐参数对齐,唯一差异 = 加 --dp-size 2 --enable-dp-attention --enable-dp-lm-head。 +# GPU 6-7,端口 30030 直连。镜像/补丁挂载/L3/engram/tok8 全部相同。 +# 注:dp+dspark 不能开 --enable-decoder-swa-bounded-replay(本配置本来就没开)。 +x-common: &common + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + shm_size: "32gb" + ipc: host + privileged: true + network_mode: host + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + - /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py + environment: &env + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_RAGGED_VERIFY_MODE: static + MC_MS_AUTO_DISC: "0" + MOONCAKE_MASTER: 127.0.0.1:50051 + MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE + MOONCAKE_PROTOCOL: tcp + MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + +services: + dp2: + <<: *common + container_name: ds41-dp2 + environment: + <<: *env + CUDA_VISIBLE_DEVICES: "6,7" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --dp-size 2 --enable-dp-attention --enable-dp-lm-head + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 64 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30030 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 diff --git a/deploy/b300-ds41/dockerserve-dp4test.yml b/deploy/b300-ds41/dockerserve-dp4test.yml new file mode 100644 index 000000000..fc7f01577 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-dp4test.yml @@ -0,0 +1,42 @@ +services: + sglang: + image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142 + container_name: ds41-dp4test + shm_size: "32gb" + ipc: host + privileged: true + ports: + - "30000:30000" + environment: + - CUDA_VISIBLE_DEVICES=0,1,2,3 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --enable-decoder-swa-bounded-replay + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --max-running-requests 64 + --host 0.0.0.0 + --port 30000 + --enable-prefill-cp + --cp-strategy interleave + --enable-cache-report + --enable-metrics + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-dp8.yml b/deploy/b300-ds41/dockerserve-dp8.yml new file mode 100644 index 000000000..d976f90b3 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-dp8.yml @@ -0,0 +1,44 @@ +# dp8+tp8 单机非 PD 对照测试(2026-09-23 晚) +# 背景:3cp2+dp2 PD 部署 dspark accept 回归(32k ctx accept len 3.14→1.31, +# 见 experiments/3cp2-pd/bench-20260923.md「未决问题」)。本配置退回单机验证 +# dspark 健康度:engram host table 关闭、无 hicache L3、无 PD、无 mooncake。 +# 参数基准:dockerserve-3cp2-pd.yml 的 d 服务(dp 路径),去掉 PD/L3/engram 相关。 +# GPU 0-7,端口 30000,容器 ds41-dp8,project dp8 +services: + dp8: + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + container_name: ds41-dp8 + shm_size: "32gb" + ipc: host + privileged: true + network_mode: host + environment: + SGLANG_RAGGED_VERIFY_MODE: "static" + CUDA_VISIBLE_DEVICES: "0,1,2,3,4,5,6,7" + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 8 --ep-size 8 --dp-size 8 + --enable-dp-attention --enable-dp-lm-head + --mem-fraction-static 0.80 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 64 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30000 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 diff --git a/deploy/b300-ds41/dockerserve-p.yml b/deploy/b300-ds41/dockerserve-p.yml new file mode 100644 index 000000000..3c8ae85b3 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-p.yml @@ -0,0 +1,47 @@ +services: + sglang: + image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142 + container_name: ds41-prefill + shm_size: "32gb" + ipc: host + privileged: true + ports: + - "30000:30000" + environment: + - CUDA_VISIBLE_DEVICES=0,1,2,3 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --max-running-requests 64 + --host 0.0.0.0 + --port 30000 + --enable-cache-report + --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --enable-decoder-swa-bounded-replay + --enable-hierarchical-cache + --hicache-ratio 2.5 + --hicache-write-policy write_back + --disaggregation-mode prefill + --optimistic-prefill-attempts 4 + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-pd-d-vision.yml b/deploy/b300-ds41/dockerserve-pd-d-vision.yml new file mode 100644 index 000000000..b3fdf5253 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-pd-d-vision.yml @@ -0,0 +1,51 @@ +services: + sglang: + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + container_name: ds41-pd-decode + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + + environment: + - CUDA_VISIBLE_DEVICES=4,5,6,7 + - SGLANG_RAGGED_VERIFY_MODE=static + - MC_INTRANODE_NVLINK=true + - MC_INTRA_NVLINK=true + - SGLANG_MOONCAKE_SEND_AUX_TCP=1 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --dp-size 4 + --enable-dp-attention + --enable-dp-lm-head + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --tokenizer-worker-num 8 + --host 0.0.0.0 + --port 30001 + --enable-cache-report + --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --disaggregation-mode decode + --disaggregation-transfer-backend mooncake diff --git a/deploy/b300-ds41/dockerserve-pd-d.yml b/deploy/b300-ds41/dockerserve-pd-d.yml new file mode 100644 index 000000000..445264391 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-pd-d.yml @@ -0,0 +1,50 @@ +services: + sglang: + image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536 + container_name: ds41-pd-decode + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + + environment: + - CUDA_VISIBLE_DEVICES=4,5,6,7 + - SGLANG_RAGGED_VERIFY_MODE=static + - MC_INTRANODE_NVLINK=true + - MC_INTRA_NVLINK=true + - SGLANG_MOONCAKE_SEND_AUX_TCP=1 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --dp-size 4 + --enable-dp-attention + --enable-dp-lm-head + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --tokenizer-worker-num 8 + --host 0.0.0.0 + --port 30001 + --enable-cache-report + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --disaggregation-mode decode + --disaggregation-transfer-backend mooncake + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-pd-p-dp.yml b/deploy/b300-ds41/dockerserve-pd-p-dp.yml new file mode 100644 index 000000000..5f0c311ea --- /dev/null +++ b/deploy/b300-ds41/dockerserve-pd-p-dp.yml @@ -0,0 +1,55 @@ +services: + sglang: + image: ymkymx/sglang:dsv41-pd-c74a4037f-20260921-1300 + container_name: ds41-pd-prefill-dp + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + + environment: + - CUDA_VISIBLE_DEVICES=0,1,2,3 + - SGLANG_RAGGED_VERIFY_MODE=static + - MC_INTRANODE_NVLINK=true + - MC_INTRA_NVLINK=true + - SGLANG_MOONCAKE_SEND_AUX_TCP=1 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --dp-size 4 + --enable-dp-attention + --enable-dp-lm-head + --load-balance-method total_tokens + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --max-running-requests 64 + --host 0.0.0.0 + --port 30010 + --enable-cache-report + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2.5 + --hicache-write-policy write_back + --disaggregation-mode prefill + --disaggregation-bootstrap-port 8918 + --disaggregation-transfer-backend mooncake + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-pd-p-vision.yml b/deploy/b300-ds41/dockerserve-pd-p-vision.yml new file mode 100644 index 000000000..034e8bd23 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-pd-p-vision.yml @@ -0,0 +1,53 @@ +services: + sglang: + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + container_name: ds41-pd-prefill + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + + environment: + - CUDA_VISIBLE_DEVICES=0,1,2,3 + - SGLANG_RAGGED_VERIFY_MODE=static + - MC_INTRANODE_NVLINK=true + - MC_INTRA_NVLINK=true + - SGLANG_MOONCAKE_SEND_AUX_TCP=1 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --tokenizer-worker-num 8 + --host 0.0.0.0 + --port 30000 + --enable-cache-report + --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2.5 + --hicache-write-policy write_back + --enable-prefill-cp + --cp-strategy interleave + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake diff --git a/deploy/b300-ds41/dockerserve-pd-p.yml b/deploy/b300-ds41/dockerserve-pd-p.yml new file mode 100644 index 000000000..5ece8c967 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-pd-p.yml @@ -0,0 +1,52 @@ +services: + sglang: + image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536 + container_name: ds41-pd-prefill + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + + environment: + - CUDA_VISIBLE_DEVICES=0,1,2,3 + - SGLANG_RAGGED_VERIFY_MODE=static + - MC_INTRANODE_NVLINK=true + - MC_INTRA_NVLINK=true + - SGLANG_MOONCAKE_SEND_AUX_TCP=1 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --tokenizer-worker-num 8 + --host 0.0.0.0 + --port 30000 + --enable-cache-report + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2.5 + --hicache-write-policy write_back + --enable-prefill-cp + --cp-strategy interleave + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-pd2-d-r2.yml b/deploy/b300-ds41/dockerserve-pd2-d-r2.yml new file mode 100644 index 000000000..5acb81228 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-pd2-d-r2.yml @@ -0,0 +1,57 @@ +# PD2 二分 Round 2 D 侧(2026-09-24):干净 D + 嫌疑项② +# SGLANG_DISAGGREGATION_QUEUE_SIZE=16 + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32 +services: + sglang: + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + container_name: ds41-pd2-decode + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + + environment: + - CUDA_VISIBLE_DEVICES=6,7 + - SGLANG_RAGGED_VERIFY_MODE=static + - MC_INTRANODE_NVLINK=true + - MC_INTRA_NVLINK=true + - SGLANG_MOONCAKE_SEND_AUX_TCP=1 + - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 + - SGLANG_DISAGGREGATION_QUEUE_SIZE=16 + - SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + - /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 + --ep-size 2 + --dp-size 2 + --enable-dp-attention + --enable-dp-lm-head + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto + --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --tokenizer-worker-num 8 + --host 0.0.0.0 + --port 30030 + --enable-cache-report + --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --disaggregation-mode decode + --disaggregation-transfer-backend mooncake diff --git a/deploy/b300-ds41/dockerserve-pd2-d.yml b/deploy/b300-ds41/dockerserve-pd2-d.yml new file mode 100644 index 000000000..98d29e42d --- /dev/null +++ b/deploy/b300-ds41/dockerserve-pd2-d.yml @@ -0,0 +1,54 @@ +# PD 干净对照实验 D 节点(2026-09-24 上午):dp2 decode,配 dockerserve-pd2-p.yml 使用。 +services: + sglang: + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + container_name: ds41-pd2-decode + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + + environment: + - CUDA_VISIBLE_DEVICES=6,7 + - SGLANG_RAGGED_VERIFY_MODE=static + - MC_INTRANODE_NVLINK=true + - MC_INTRA_NVLINK=true + - SGLANG_MOONCAKE_SEND_AUX_TCP=1 + - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + - /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 + --ep-size 2 + --dp-size 2 + --enable-dp-attention + --enable-dp-lm-head + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto + --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --tokenizer-worker-num 8 + --host 0.0.0.0 + --port 30030 + --enable-cache-report + --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --disaggregation-mode decode + --disaggregation-transfer-backend mooncake diff --git a/deploy/b300-ds41/dockerserve-pd2-p-r1.yml b/deploy/b300-ds41/dockerserve-pd2-p-r1.yml new file mode 100644 index 000000000..e260c73e9 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-pd2-p-r1.yml @@ -0,0 +1,67 @@ +# PD2 二分 Round 1(2026-09-24):干净基线 + 嫌疑项①「P 侧 L3 hicache 块 + mooncake-store.json」 +# 相对 dockerserve-pd2-p.yml 的差异: +# env 加 SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json +# hicache 参数块换成坏部署同款:page_first_direct + direct + write_through + mooncake + +# wait_complete + size 0(替代 write_back L2) +# D 侧不变(坏部署 D 本无 L3)。探针:bs16 low_entropy decode 看 accept len。 +services: + sglang: + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + container_name: ds41-pd2-prefill + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + + environment: + - CUDA_VISIBLE_DEVICES=4,5 + - SGLANG_RAGGED_VERIFY_MODE=static + - MC_INTRANODE_NVLINK=true + - MC_INTRA_NVLINK=true + - SGLANG_MOONCAKE_SEND_AUX_TCP=1 + - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 + - SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + - /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 + --ep-size 2 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto + --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --tokenizer-worker-num 8 + --host 0.0.0.0 + --port 30020 + --enable-cache-report + --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --enable-prefill-cp + --cp-strategy interleave + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake diff --git a/deploy/b300-ds41/dockerserve-pd2-p-r2.yml b/deploy/b300-ds41/dockerserve-pd2-p-r2.yml new file mode 100644 index 000000000..9716fe4c4 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-pd2-p-r2.yml @@ -0,0 +1,65 @@ +# PD2 二分 Round 2 P 侧(2026-09-24):R1(L3 块)之上再加嫌疑项② +# SGLANG_DISAGGREGATION_QUEUE_SIZE=16 + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32 +services: + sglang: + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + container_name: ds41-pd2-prefill + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + + environment: + - CUDA_VISIBLE_DEVICES=4,5 + - SGLANG_RAGGED_VERIFY_MODE=static + - MC_INTRANODE_NVLINK=true + - MC_INTRA_NVLINK=true + - SGLANG_MOONCAKE_SEND_AUX_TCP=1 + - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 + - SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json + - SGLANG_DISAGGREGATION_QUEUE_SIZE=16 + - SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + - /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 + --ep-size 2 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto + --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --tokenizer-worker-num 8 + --host 0.0.0.0 + --port 30020 + --enable-cache-report + --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --enable-prefill-cp + --cp-strategy interleave + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake diff --git a/deploy/b300-ds41/dockerserve-pd2-p.yml b/deploy/b300-ds41/dockerserve-pd2-p.yml new file mode 100644 index 000000000..c8b1cefaa --- /dev/null +++ b/deploy/b300-ds41/dockerserve-pd2-p.yml @@ -0,0 +1,59 @@ +# PD 干净对照实验(2026-09-24 上午):验证「cp2-P → dp2-D」传输路径下 dspark 是否正常。 +# 用户疑问:单机六形态健康不等于 PD 链路健康,可能两边传输没对齐。 +# 以已验证可用的 dockerserve-pd-{p,d}-vision.yml 为底,缩到 2+2 卡;P 加 engram host table +# (2 卡权重放不下,见 G1.9)。P=GPU4-5 端口 30020,D=GPU6-7 端口 30030,mini_lb=30004。 +services: + sglang: + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + container_name: ds41-pd2-prefill + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + + environment: + - CUDA_VISIBLE_DEVICES=4,5 + - SGLANG_RAGGED_VERIFY_MODE=static + - MC_INTRANODE_NVLINK=true + - MC_INTRA_NVLINK=true + - SGLANG_MOONCAKE_SEND_AUX_TCP=1 + - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + - /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 + --ep-size 2 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto + --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --tokenizer-worker-num 8 + --host 0.0.0.0 + --port 30020 + --enable-cache-report + --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2.5 + --hicache-write-policy write_back + --enable-prefill-cp + --cp-strategy interleave + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake diff --git a/deploy/b300-ds41/dockerserve-replay-off.yml b/deploy/b300-ds41/dockerserve-replay-off.yml new file mode 100644 index 000000000..7eb10a72d --- /dev/null +++ b/deploy/b300-ds41/dockerserve-replay-off.yml @@ -0,0 +1,42 @@ +services: + sglang: + image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142 + container_name: ds41-replay-off + shm_size: "32gb" + ipc: host + privileged: true + ports: + - "30000:30000" + environment: + - CUDA_VISIBLE_DEVICES=0,1,2,3 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --max-running-requests 64 + --host 0.0.0.0 + --port 30000 + --enable-cache-report + --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-replay-on.yml b/deploy/b300-ds41/dockerserve-replay-on.yml new file mode 100644 index 000000000..8eb585946 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-replay-on.yml @@ -0,0 +1,42 @@ +services: + sglang: + image: ymkymx/sglang:main-ee5fcdf0d-20260920-0142 + container_name: ds41-replay-on + shm_size: "32gb" + ipc: host + privileged: true + ports: + - "30000:30000" + environment: + - CUDA_VISIBLE_DEVICES=0,1,2,3 + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --max-running-requests 64 + --host 0.0.0.0 + --port 30000 + --enable-cache-report + --enable-metrics + --speculative-algorithm DSPARK + --speculative-dspark-block-size 5 + --enable-decoder-swa-bounded-replay + --json-model-override-args '{"vision_n_layers": 0}' diff --git a/deploy/b300-ds41/dockerserve-tp2.yml b/deploy/b300-ds41/dockerserve-tp2.yml new file mode 100644 index 000000000..cbe412fc9 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-tp2.yml @@ -0,0 +1,92 @@ +# 纯 tp2 单实例对照(2026-09-24 早):与 dockerserve-cp2x4.yml 逐参数对齐,唯一差异 = 去掉 +# --enable-prefill-cp --cp-strategy interleave(即无 CP),用于测「tp2 vs cp2」的 prefill 效率差。 +# GPU 4-5(a)/6-7(b),端口 30020/30030;2×tp2 聚合经 rr router 30003。 +# 镜像/补丁挂载/L3/engram/tok8 全部与 cp2x4 相同。 +x-common: &common + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + shm_size: "32gb" + ipc: host + privileged: true + network_mode: host + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + # 与 cp2x4 保持同一份 scheduler.py(含未 commit 的 CP idle 补丁),保证唯一变量是 CP 开关 + - /data/ymk/sglang/python/sglang/srt/managers/scheduler.py:/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py + environment: &env + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_RAGGED_VERIFY_MODE: static + MC_MS_AUTO_DISC: "0" + MOONCAKE_MASTER: 127.0.0.1:50051 + MOONCAKE_TE_META_DATA_SERVER: P2PHANDSHAKE + MOONCAKE_PROTOCOL: tcp + MOONCAKE_GLOBAL_SEGMENT_SIZE: 300gb + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + +services: + tp2: + <<: *common + container_name: ds41-tp2 + environment: + <<: *env + CUDA_VISIBLE_DEVICES: "4,5" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 64 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30020 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + b: + <<: *common + container_name: ds41-tp2-b + environment: + <<: *env + CUDA_VISIBLE_DEVICES: "6,7" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 64 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30030 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 diff --git a/deploy/b300-ds41/dockerserve.yml b/deploy/b300-ds41/dockerserve.yml new file mode 100644 index 000000000..01a69a639 --- /dev/null +++ b/deploy/b300-ds41/dockerserve.yml @@ -0,0 +1,43 @@ +services: + sglang: + image: uhub.service.ucloud.cn/umirror/sglang:dev-dsv41 + shm_size: "32gb" + ipc: host + privileged: true + + ports: + - "30000:30000" + + volumes: + - /data:/data + + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + + command: >- + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 4 + --ep-size 4 + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --enable-decoder-swa-bounded-replay + --cuda-graph-max-bs-decode 64 + --reasoning-parser auto + --tool-call-parser auto + --max-running-requests 64 + --host 0.0.0.0 + --port 30000 + --enable-cache-report + --enable-metrics + --enable-prefill-cp + --cp-strategy interleave + --json-model-override-args '{"vision_n_layers": 0}' + diff --git a/deploy/b300-ds41/mooncake-store.json b/deploy/b300-ds41/mooncake-store.json new file mode 100644 index 000000000..20a9fd52d --- /dev/null +++ b/deploy/b300-ds41/mooncake-store.json @@ -0,0 +1,6 @@ +{ + "master_server_address": "127.0.0.1:50051", + "metadata_server": "P2PHANDSHAKE", + "global_segment_size": "300gb", + "protocol": "tcp" +}