52 lines
1.6 KiB
YAML
52 lines
1.6 KiB
YAML
services:
|
|
sglang:
|
|
image: ymkymx/sglang:dsv41-pd-ddf520763-local-20260923-0536
|
|
container_name: ds41-cpdspark-l3-a
|
|
shm_size: "32gb"
|
|
ipc: host
|
|
privileged: true
|
|
network_mode: host
|
|
environment:
|
|
- CUDA_VISIBLE_DEVICES=0,1,2,3
|
|
- MC_MS_AUTO_DISC=0
|
|
- MOONCAKE_MASTER=127.0.0.1:50051
|
|
- MOONCAKE_TE_META_DATA_SERVER=P2PHANDSHAKE
|
|
- MOONCAKE_PROTOCOL=tcp
|
|
- MOONCAKE_GLOBAL_SEGMENT_SIZE=400gb
|
|
volumes:
|
|
- /data:/data
|
|
- /data/ymk/cache/sglang:/root/.cache/sglang
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities: [gpu]
|
|
command: >-
|
|
sglang serve
|
|
--trust-remote-code
|
|
--model-path /data/models/DeepSeek-V4.1-Flash
|
|
--tp 4 --ep-size 4
|
|
--mem-fraction-static 0.75
|
|
--attention-backend dsv4
|
|
--moe-runner-backend flashinfer_mxfp4
|
|
--cuda-graph-max-bs-decode 64
|
|
--reasoning-parser auto --tool-call-parser auto
|
|
--max-running-requests 64
|
|
--tokenizer-worker-num 8
|
|
--host 0.0.0.0 --port 30000
|
|
--enable-cache-report --enable-metrics
|
|
--speculative-algorithm DSPARK
|
|
--speculative-dspark-block-size 5
|
|
--enable-prefill-cp --cp-strategy interleave
|
|
--enable-hierarchical-cache
|
|
--hicache-ratio 2
|
|
--hicache-mem-layout page_first_direct
|
|
--hicache-io-backend direct
|
|
--hicache-write-policy write_through
|
|
--hicache-storage-backend mooncake
|
|
--hicache-storage-prefetch-policy wait_complete
|
|
--hicache-size 0
|
|
--json-model-override-args '{"vision_n_layers": 0}'
|