deploy: add 3p1d soak/prof compose + otlp tracing variant (jaeger badger 6h)
This commit is contained in:
@@ -0,0 +1,195 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram + OTLP trace（2026-09-27）
|
||||
# 基于 dockerserve-3p1d-soak.yml 增加: --enable-trace --trace-modules request,mooncake
|
||||
# --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name ds41-3p1d-<实例名>
|
||||
# 依赖: b300-01 k8s cw jaeger-trace（jaegertracing/all-in-one:1.76.0，badger 6h TTL，hostNetwork 4317）
|
||||
# 安全性: BatchSpanProcessor 导出失败只丢 span；Jaeger 宕机不影响推理服务
|
||||
# 调优旋钮(默认未开): SGLANG_TRACE_ASYNC=1（异步 span 创建）, SGLANG_TRACE_LEVEL（默认 3）
|
||||
# router（ds41-router-3p1d，rust）无 OTLP 支持，不产生 span，追踪从各 sglang HTTP 层开始
|
||||
# 2026-09-23 实验；基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p1
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p2
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-p3
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
--enable-trace
|
||||
--trace-modules request,mooncake
|
||||
--otlp-traces-endpoint 127.0.0.1:4317
|
||||
--otlp-service-name ds41-3p1d-d1
|
||||
@@ -0,0 +1,173 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram（2026-09-25，mr256，nsys on p1/d1）
|
||||
# 2026-09-23 实验；基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
nsys launch --session-new=p1prof --cuda-graph-trace=node sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-prof-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
nsys launch --session-new=d1prof --cuda-graph-trace=node sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
@@ -0,0 +1,173 @@
|
||||
# 3x cp2-P + 1x dp2-D(cg256) PD 分离 + L3 mooncake + engram（2026-09-25，mr256，nsys on p1/d1）
|
||||
# 2026-09-23 实验；基线见 archive-20260923/ 与 D:\B300\experiments\3cp2-pd\baseline-20260923.md
|
||||
# GPU: P1=0,1 P2=2,3 P3=4,5 D=6,7
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-db7d2cb7d-fix
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
privileged: true
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /data:/data
|
||||
- /data/ymk/cache/sglang:/root/.cache/sglang
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
x-p-environment: &p-environment
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
|
||||
services:
|
||||
p1:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p1
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "0,1"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30000
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8998
|
||||
|
||||
p2:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p2
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "2,3"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30010
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 8999
|
||||
|
||||
p3:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-p3
|
||||
environment:
|
||||
<<: *p-environment
|
||||
CUDA_VISIBLE_DEVICES: "4,5"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2
|
||||
--enable-prefill-cp --cp-strategy interleave
|
||||
--mem-fraction-static 0.75
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 32
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30020
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--enable-hierarchical-cache
|
||||
--hicache-ratio 2
|
||||
--hicache-mem-layout page_first_direct
|
||||
--hicache-io-backend direct
|
||||
--hicache-write-policy write_through
|
||||
--hicache-storage-backend mooncake
|
||||
--hicache-storage-prefetch-policy wait_complete
|
||||
--hicache-size 0
|
||||
--disaggregation-mode prefill
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9000
|
||||
|
||||
d:
|
||||
<<: *sglang-common
|
||||
container_name: ds41-3p1d-d1
|
||||
environment:
|
||||
SGLANG_TORCH_PROFILER_DIR: /data/ymk/prof/torch
|
||||
SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: "1"
|
||||
SGLANG_RAGGED_VERIFY_MODE: static
|
||||
MC_INTRANODE_NVLINK: "true"
|
||||
MC_INTRA_NVLINK: "true"
|
||||
SGLANG_MOONCAKE_SEND_AUX_TCP: "1"
|
||||
SGLANG_DISAGGREGATION_QUEUE_SIZE: "16"
|
||||
SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32"
|
||||
CUDA_VISIBLE_DEVICES: "6,7"
|
||||
command: >
|
||||
sglang serve
|
||||
--trust-remote-code
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash
|
||||
--tp 2 --ep-size 2 --dp-size 2
|
||||
--enable-dp-attention --enable-dp-lm-head
|
||||
--mem-fraction-static 0.80
|
||||
--attention-backend dsv4
|
||||
--moe-runner-backend flashinfer_mxfp4
|
||||
--cuda-graph-max-bs-decode 256
|
||||
--reasoning-parser auto --tool-call-parser auto
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}'
|
||||
--max-running-requests 256
|
||||
--tokenizer-worker-num 8
|
||||
--host 0.0.0.0 --port 30001
|
||||
--enable-cache-report --enable-metrics
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 5
|
||||
--disaggregation-mode decode
|
||||
--disaggregation-transfer-backend mooncake
|
||||
--disaggregation-bootstrap-port 9001
|
||||
Reference in New Issue
Block a user