From 5507fff133bead8a5c580d5526610c2a9c1c6447 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Sun, 27 Sep 2026 12:10:00 +0800 Subject: [PATCH] deploy: add 3p1d soak/prof compose + otlp tracing variant (jaeger badger 6h) --- deploy/b300-ds41/dockerserve-3p1d-otlp.yml | 195 +++++++++++++++++++++ deploy/b300-ds41/dockerserve-3p1d-prof.yml | 173 ++++++++++++++++++ deploy/b300-ds41/dockerserve-3p1d-soak.yml | 173 ++++++++++++++++++ 3 files changed, 541 insertions(+) create mode 100644 deploy/b300-ds41/dockerserve-3p1d-otlp.yml create mode 100644 deploy/b300-ds41/dockerserve-3p1d-prof.yml create mode 100644 deploy/b300-ds41/dockerserve-3p1d-soak.yml diff --git a/deploy/b300-ds41/dockerserve-3p1d-otlp.yml b/deploy/b300-ds41/dockerserve-3p1d-otlp.yml new file mode 100644 index 000000000..f4581f196 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-3p1d-otlp.yml @@ -0,0 +1,195 @@ +# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace（2026-09-27） +# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake +# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名> +# 依赖: b300-01 k8s cw jaeger-trace（jaegertracing/all-in-one:1.76.0，badger 6h TTL，hostNetwork 4317） +# 安全性: BatchSpanProcessor 导出失败只丢 span；Jaeger 宕机不影响推理服务 +# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1（异步 span 创建）, SGLANG_TRACE_LEVEL（默认 3） +# router（ds41-router-3p1d，rust）无 OTLP 支持，不产生 span，追踪从各 sglang HTTP 层开始 +# 2026-09-23 实验；基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md +# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7 +# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002 +# bootstrap: P1=8998 P2=8999 P3=9000 D=9001 +x-sglang-common: &sglang-common + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + +x-p-environment: &p-environment + SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_RAGGED_VERIFY_MODE: static + MC_INTRANODE_NVLINK: "true" + MC_INTRA_NVLINK: "true" + SGLANG_MOONCAKE_SEND_AUX_TCP: "1" + SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json + SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" + +services: + p1: + <<: *sglang-common + container_name: ds41-3p1d-p1 + environment: + <<: *p-environment + CUDA_VISIBLE_DEVICES: "0,1" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 256 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30000 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 8998 + --enable-trace + --trace-modules request,mooncake + --otlp-traces-endpoint 127.0.0.1:4317 + --otlp-service-name ds41-3p1d-p1 + + p2: + <<: *sglang-common + container_name: ds41-3p1d-p2 + environment: + <<: *p-environment + CUDA_VISIBLE_DEVICES: "2,3" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 256 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30010 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 8999 + --enable-trace + --trace-modules request,mooncake + --otlp-traces-endpoint 127.0.0.1:4317 + --otlp-service-name ds41-3p1d-p2 + + p3: + <<: *sglang-common + container_name: ds41-3p1d-p3 + environment: + <<: *p-environment + CUDA_VISIBLE_DEVICES: "4,5" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 256 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30020 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 9000 + --enable-trace + --trace-modules request,mooncake + --otlp-traces-endpoint 127.0.0.1:4317 + --otlp-service-name ds41-3p1d-p3 + + d: + <<: *sglang-common + container_name: ds41-3p1d-d1 + environment: + SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_RAGGED_VERIFY_MODE: static + MC_INTRANODE_NVLINK: "true" + MC_INTRA_NVLINK: "true" + SGLANG_MOONCAKE_SEND_AUX_TCP: "1" + SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" + CUDA_VISIBLE_DEVICES: "6,7" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 --dp-size 2 + --enable-dp-attention --enable-dp-lm-head + --mem-fraction-static 0.80 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 256 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 256 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30001 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --disaggregation-mode decode + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 9001 + --enable-trace + --trace-modules request,mooncake + --otlp-traces-endpoint 127.0.0.1:4317 + --otlp-service-name ds41-3p1d-d1 diff --git a/deploy/b300-ds41/dockerserve-3p1d-prof.yml b/deploy/b300-ds41/dockerserve-3p1d-prof.yml new file mode 100644 index 000000000..d52fab4db --- /dev/null +++ b/deploy/b300-ds41/dockerserve-3p1d-prof.yml @@ -0,0 +1,173 @@ +# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram（2026-09-25，mr256，nsys on p1/d1） +# 2026-09-23 实验；基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md +# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7 +# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002 +# bootstrap: P1=8998 P2=8999 P3=9000 D=9001 +x-sglang-common: &sglang-common + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + +x-p-environment: &p-environment + SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_RAGGED_VERIFY_MODE: static + MC_INTRANODE_NVLINK: "true" + MC_INTRA_NVLINK: "true" + SGLANG_MOONCAKE_SEND_AUX_TCP: "1" + SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json + SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" + +services: + p1: + <<: *sglang-common + container_name: ds41-3p1d-prof-p1 + environment: + <<: *p-environment + CUDA_VISIBLE_DEVICES: "0,1" + command: > + nsys launch --session-new=p1prof --cuda-graph-trace=node sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 256 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30000 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 8998 + + p2: + <<: *sglang-common + container_name: ds41-3p1d-prof-p2 + environment: + <<: *p-environment + CUDA_VISIBLE_DEVICES: "2,3" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 256 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30010 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 8999 + + p3: + <<: *sglang-common + container_name: ds41-3p1d-prof-p3 + environment: + <<: *p-environment + CUDA_VISIBLE_DEVICES: "4,5" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 256 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30020 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 9000 + + d: + <<: *sglang-common + container_name: ds41-3p1d-prof-d1 + environment: + SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_RAGGED_VERIFY_MODE: static + MC_INTRANODE_NVLINK: "true" + MC_INTRA_NVLINK: "true" + SGLANG_MOONCAKE_SEND_AUX_TCP: "1" + SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" + CUDA_VISIBLE_DEVICES: "6,7" + command: > + nsys launch --session-new=d1prof --cuda-graph-trace=node sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 --dp-size 2 + --enable-dp-attention --enable-dp-lm-head + --mem-fraction-static 0.80 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 256 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 256 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30001 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --disaggregation-mode decode + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 9001 diff --git a/deploy/b300-ds41/dockerserve-3p1d-soak.yml b/deploy/b300-ds41/dockerserve-3p1d-soak.yml new file mode 100644 index 000000000..16d25fcc3 --- /dev/null +++ b/deploy/b300-ds41/dockerserve-3p1d-soak.yml @@ -0,0 +1,173 @@ +# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram（2026-09-25，mr256，nsys on p1/d1） +# 2026-09-23 实验；基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md +# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7 +# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002 +# bootstrap: P1=8998 P2=8999 P3=9000 D=9001 +x-sglang-common: &sglang-common + image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix + shm_size: "32gb" + ipc: host + pid: host + privileged: true + network_mode: host + volumes: + - /data:/data + - /data/ymk/cache/sglang:/root/.cache/sglang + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + +x-p-environment: &p-environment + SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_RAGGED_VERIFY_MODE: static + MC_INTRANODE_NVLINK: "true" + MC_INTRA_NVLINK: "true" + SGLANG_MOONCAKE_SEND_AUX_TCP: "1" + SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json + SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" + +services: + p1: + <<: *sglang-common + container_name: ds41-3p1d-p1 + environment: + <<: *p-environment + CUDA_VISIBLE_DEVICES: "0,1" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 256 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30000 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 8998 + + p2: + <<: *sglang-common + container_name: ds41-3p1d-p2 + environment: + <<: *p-environment + CUDA_VISIBLE_DEVICES: "2,3" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 256 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30010 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 8999 + + p3: + <<: *sglang-common + container_name: ds41-3p1d-p3 + environment: + <<: *p-environment + CUDA_VISIBLE_DEVICES: "4,5" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 + --enable-prefill-cp --cp-strategy interleave + --mem-fraction-static 0.75 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 32 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 256 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30020 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --enable-hierarchical-cache + --hicache-ratio 2 + --hicache-mem-layout page_first_direct + --hicache-io-backend direct + --hicache-write-policy write_through + --hicache-storage-backend mooncake + --hicache-storage-prefetch-policy wait_complete + --hicache-size 0 + --disaggregation-mode prefill + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 9000 + + d: + <<: *sglang-common + container_name: ds41-3p1d-d1 + environment: + SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch + SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1" + SGLANG_RAGGED_VERIFY_MODE: static + MC_INTRANODE_NVLINK: "true" + MC_INTRA_NVLINK: "true" + SGLANG_MOONCAKE_SEND_AUX_TCP: "1" + SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" + SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" + CUDA_VISIBLE_DEVICES: "6,7" + command: > + sglang serve + --trust-remote-code + --model-path /data/models/DeepSeek-V4.1-Flash + --tp 2 --ep-size 2 --dp-size 2 + --enable-dp-attention --enable-dp-lm-head + --mem-fraction-static 0.80 + --attention-backend dsv4 + --moe-runner-backend flashinfer_mxfp4 + --cuda-graph-max-bs-decode 256 + --reasoning-parser auto --tool-call-parser auto + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' + --max-running-requests 256 + --tokenizer-worker-num 8 + --host 0.0.0.0 --port 30001 + --enable-cache-report --enable-metrics + --speculative-algorithm DSPARK --speculative-dspark-block-size 5 + --disaggregation-mode decode + --disaggregation-transfer-backend mooncake + --disaggregation-bootstrap-port 9001