deploy: record 2026-11-17 c580 baseline (D16+fastokens) and experiments
This commit is contained in:
@@ -0,0 +1,10 @@
|
|||||||
|
# 2026-11-17 c580 基线:D tokenizer 16 + fastokens
|
||||||
|
|
||||||
|
- manifest: `manifest.yaml`(= `ds41-gist-exp-20261117-dtok16-fastokens.yaml`)
|
||||||
|
- P: cache / 无 optimistic / tokenizer-worker 16 / fastokens / hicache ratio 4 / L3 SSD 4T
|
||||||
|
- D: tokenizer-worker 16 / fastokens
|
||||||
|
- Gateway: 正常(无 P admission cap,health/circuit 开启,decode-max-running 512)
|
||||||
|
- L3:每轮前清空;完整重启
|
||||||
|
- c580 结果:rps 24.045,in 2.290M tok/s,out 22.449k tok/s,
|
||||||
|
TTFT 1.743/3.772/6.580 s,ITL p95 10.74 ms,29,816 req,valid=true。
|
||||||
|
- 原始结果:`experiments/agentx-align/results-c580-dtok16-fastokens-20261117/`
|
||||||
@@ -0,0 +1,488 @@
|
|||||||
|
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||||
|
kind: RoleBasedGroup
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
rbg.workloads.x-k8s.io/discovery-config-mode: refine
|
||||||
|
labels:
|
||||||
|
app: ds41-gist-exp
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
sglang-model: ds-v41-flash
|
||||||
|
name: ds41-gist-exp
|
||||||
|
namespace: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
spec:
|
||||||
|
roles:
|
||||||
|
- minReadySeconds: 0
|
||||||
|
name: mooncake-master
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '19003'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-mooncake
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n"
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: mooncake-master
|
||||||
|
ports:
|
||||||
|
- containerPort: 19003
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 60
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 5
|
||||||
|
tcpSocket:
|
||||||
|
port: 50051
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '4'
|
||||||
|
memory: 64Gi
|
||||||
|
requests:
|
||||||
|
cpu: '1'
|
||||||
|
memory: 8Gi
|
||||||
|
env:
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/master-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/master-xdg
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
- dependencies:
|
||||||
|
- mooncake-master
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: p
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '30000'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-p
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n"
|
||||||
|
env:
|
||||||
|
- name: CUDA_VISIBLE_DEVICES
|
||||||
|
value: 0,1,2,3
|
||||||
|
- name: SGLANG_LOG_MS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK
|
||||||
|
value: '65536'
|
||||||
|
- name: SGLANG_DSV4_KV_LAYOUT
|
||||||
|
value: v41_new
|
||||||
|
- name: SGLANG_DP_USE_GATHERV
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||||
|
value: static
|
||||||
|
- name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_NUMA_BIND_V2
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_DEFAULT_THINKING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||||
|
value: high
|
||||||
|
- name: SGLANG_TIMEOUT_KEEP_ALIVE
|
||||||
|
value: '900'
|
||||||
|
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||||
|
value: '1'
|
||||||
|
- name: MC_INTRANODE_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: MC_INTRA_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||||
|
value: /data/ymk/ds41/mooncake-store-l3ssd.json
|
||||||
|
- name: SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||||
|
value: '4'
|
||||||
|
- name: SGLANG_HICACHE_BACKUP_IO_THREADS
|
||||||
|
value: '4'
|
||||||
|
- name: SGLANG_HICACHE_TIMING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_TELEMETRY
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DEVICE_TIMER
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL
|
||||||
|
value: '100'
|
||||||
|
- name: SGLANG_MM_TRACE
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_SPAN_MIN_MS
|
||||||
|
value: '200'
|
||||||
|
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||||
|
value: '0.05'
|
||||||
|
- name: SGLANG_BATCH_SPAN_MAX_LINKS
|
||||||
|
value: '8'
|
||||||
|
- name: SGLANG_BATCH_STAGE_TIMING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_PASS_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_PASS_SPAN_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_RECV_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_RECV_SPAN_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_MM_TRACE_SYNC
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||||
|
value: '600'
|
||||||
|
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||||
|
value: '16'
|
||||||
|
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||||
|
value: '32'
|
||||||
|
- name: TZ
|
||||||
|
value: Asia/Shanghai
|
||||||
|
- name: SGLANG_TRACE_LOOP_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_TRACE_LOOP_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_FWD_TIMELINE
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_TRACE_LOOP_DETAIL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_TRACE_MOONCAKE_DETAIL
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_TRACE_BACKUP_DETAIL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ASYNC_BACKUP_SUBMIT
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH
|
||||||
|
value: '1'
|
||||||
|
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||||
|
value: '1024'
|
||||||
|
- name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE
|
||||||
|
value: '9400'
|
||||||
|
- name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR
|
||||||
|
value: bucket_storage_backend
|
||||||
|
- name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE
|
||||||
|
value: '1099511627776'
|
||||||
|
- name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES
|
||||||
|
value: '1099511627776'
|
||||||
|
- name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY
|
||||||
|
value: fifo
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/p-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/p-xdg
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /data/ymk/l3ssd/p-hf
|
||||||
|
- name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS
|
||||||
|
value: '0'
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: sglang
|
||||||
|
ports:
|
||||||
|
- containerPort: 30000
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 8998
|
||||||
|
name: bootstrap
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-0
|
||||||
|
containerPort: 9400
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-1
|
||||||
|
containerPort: 9401
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-2
|
||||||
|
containerPort: 9402
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-3
|
||||||
|
containerPort: 9403
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 120
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: metrics
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '96'
|
||||||
|
memory: 2Ti
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
requests:
|
||||||
|
cpu: '32'
|
||||||
|
memory: 512Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /root/.cache/sglang
|
||||||
|
name: sglang-cache
|
||||||
|
- mountPath: /dev/shm
|
||||||
|
name: dshm
|
||||||
|
- mountPath: /root/.cargo/bin/cargo
|
||||||
|
name: fake-cargo
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/cache/sglang
|
||||||
|
type: Directory
|
||||||
|
name: sglang-cache
|
||||||
|
- emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 64Gi
|
||||||
|
name: dshm
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/ds41/fake-cargo
|
||||||
|
type: File
|
||||||
|
name: fake-cargo
|
||||||
|
- dependencies:
|
||||||
|
- mooncake-master
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: d
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '30300'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-d
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 --tokenizer-worker-num 16 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n"
|
||||||
|
env:
|
||||||
|
- name: CUDA_VISIBLE_DEVICES
|
||||||
|
value: 4,5,6,7
|
||||||
|
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||||
|
value: static
|
||||||
|
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV4_KV_LAYOUT
|
||||||
|
value: v41_new
|
||||||
|
- name: SGLANG_DEFAULT_THINKING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||||
|
value: high
|
||||||
|
- name: MC_INTRANODE_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: MC_INTRA_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_TELEMETRY
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||||
|
value: '0.1'
|
||||||
|
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||||
|
value: '600'
|
||||||
|
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||||
|
value: '16'
|
||||||
|
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||||
|
value: '32'
|
||||||
|
- name: TZ
|
||||||
|
value: Asia/Shanghai
|
||||||
|
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||||
|
value: '1024'
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/d-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/d-xdg
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /data/ymk/l3ssd/d-hf
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: sglang
|
||||||
|
ports:
|
||||||
|
- containerPort: 30300
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 150
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: metrics
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '96'
|
||||||
|
memory: 550Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
requests:
|
||||||
|
cpu: '32'
|
||||||
|
memory: 256Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /root/.cache/sglang
|
||||||
|
name: sglang-cache
|
||||||
|
- mountPath: /dev/shm
|
||||||
|
name: dshm
|
||||||
|
- mountPath: /root/.cargo/bin/cargo
|
||||||
|
name: fake-cargo
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/cache/sglang
|
||||||
|
type: Directory
|
||||||
|
name: sglang-cache
|
||||||
|
- emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 64Gi
|
||||||
|
name: dshm
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/ds41/fake-cargo
|
||||||
|
type: File
|
||||||
|
name: fake-cargo
|
||||||
|
- dependencies:
|
||||||
|
- p
|
||||||
|
- d
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: gateway
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '29000'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
prometheus.io/scrape: 'true'
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-gw
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 512 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n"
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: gateway
|
||||||
|
ports:
|
||||||
|
- containerPort: 30400
|
||||||
|
name: http
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 29000
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 60
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 20
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '16'
|
||||||
|
memory: 32Gi
|
||||||
|
requests:
|
||||||
|
cpu: '2'
|
||||||
|
memory: 8Gi
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /opt/sglang/bin/sgl-model-gateway
|
||||||
|
name: gw-bin
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- name: gw-bin
|
||||||
|
hostPath:
|
||||||
|
path: /data/ymk/cache/gw-est-td/sgl-model-gateway
|
||||||
|
type: File
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"rps": 24.04512015569562, "in_tput": 2290117.7612668574, "out_tput": 22449.32127276801, "ttft_p50": 1743.335193, "ttft_p95": 3771.88552025, "ttft_p99": 6579.86958935, "itl_p50": 8.162268671611512, "itl_p95": 10.741317697249674, "itl_p99": 11.806237803568294, "lat_p95": 31884.86991075, "requests": 29816.0, "isl_p50": 79909.0, "osl_p50": 339.0, "cache_pct": NaN, "valid": true}
|
||||||
@@ -0,0 +1,488 @@
|
|||||||
|
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||||
|
kind: RoleBasedGroup
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
rbg.workloads.x-k8s.io/discovery-config-mode: refine
|
||||||
|
labels:
|
||||||
|
app: ds41-gist-exp
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
sglang-model: ds-v41-flash
|
||||||
|
name: ds41-gist-exp
|
||||||
|
namespace: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
spec:
|
||||||
|
roles:
|
||||||
|
- minReadySeconds: 0
|
||||||
|
name: mooncake-master
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '19003'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-mooncake
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n"
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: mooncake-master
|
||||||
|
ports:
|
||||||
|
- containerPort: 19003
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 60
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 5
|
||||||
|
tcpSocket:
|
||||||
|
port: 50051
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '4'
|
||||||
|
memory: 64Gi
|
||||||
|
requests:
|
||||||
|
cpu: '1'
|
||||||
|
memory: 8Gi
|
||||||
|
env:
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/master-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/master-xdg
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
- dependencies:
|
||||||
|
- mooncake-master
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: p
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '30000'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-p
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 --hicache-host-memory-mode buffer_only \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n"
|
||||||
|
env:
|
||||||
|
- name: CUDA_VISIBLE_DEVICES
|
||||||
|
value: 0,1,2,3
|
||||||
|
- name: SGLANG_LOG_MS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK
|
||||||
|
value: '65536'
|
||||||
|
- name: SGLANG_DSV4_KV_LAYOUT
|
||||||
|
value: v41_new
|
||||||
|
- name: SGLANG_DP_USE_GATHERV
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||||
|
value: static
|
||||||
|
- name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_NUMA_BIND_V2
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_DEFAULT_THINKING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||||
|
value: high
|
||||||
|
- name: SGLANG_TIMEOUT_KEEP_ALIVE
|
||||||
|
value: '900'
|
||||||
|
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||||
|
value: '1'
|
||||||
|
- name: MC_INTRANODE_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: MC_INTRA_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||||
|
value: /data/ymk/ds41/mooncake-store-l3ssd.json
|
||||||
|
- name: SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||||
|
value: '4'
|
||||||
|
- name: SGLANG_HICACHE_BACKUP_IO_THREADS
|
||||||
|
value: '4'
|
||||||
|
- name: SGLANG_HICACHE_TIMING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_TELEMETRY
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DEVICE_TIMER
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL
|
||||||
|
value: '100'
|
||||||
|
- name: SGLANG_MM_TRACE
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_SPAN_MIN_MS
|
||||||
|
value: '200'
|
||||||
|
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||||
|
value: '0.05'
|
||||||
|
- name: SGLANG_BATCH_SPAN_MAX_LINKS
|
||||||
|
value: '8'
|
||||||
|
- name: SGLANG_BATCH_STAGE_TIMING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_PASS_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_PASS_SPAN_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_RECV_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_RECV_SPAN_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_MM_TRACE_SYNC
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||||
|
value: '600'
|
||||||
|
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||||
|
value: '16'
|
||||||
|
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||||
|
value: '32'
|
||||||
|
- name: TZ
|
||||||
|
value: Asia/Shanghai
|
||||||
|
- name: SGLANG_TRACE_LOOP_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_TRACE_LOOP_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_FWD_TIMELINE
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_TRACE_LOOP_DETAIL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_TRACE_MOONCAKE_DETAIL
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_TRACE_BACKUP_DETAIL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ASYNC_BACKUP_SUBMIT
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH
|
||||||
|
value: '1'
|
||||||
|
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||||
|
value: '1024'
|
||||||
|
- name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE
|
||||||
|
value: '9400'
|
||||||
|
- name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR
|
||||||
|
value: bucket_storage_backend
|
||||||
|
- name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE
|
||||||
|
value: '1099511627776'
|
||||||
|
- name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES
|
||||||
|
value: '1099511627776'
|
||||||
|
- name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY
|
||||||
|
value: fifo
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/p-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/p-xdg
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /data/ymk/l3ssd/p-hf
|
||||||
|
- name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS
|
||||||
|
value: '0'
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: sglang
|
||||||
|
ports:
|
||||||
|
- containerPort: 30000
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 8998
|
||||||
|
name: bootstrap
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-0
|
||||||
|
containerPort: 9400
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-1
|
||||||
|
containerPort: 9401
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-2
|
||||||
|
containerPort: 9402
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-3
|
||||||
|
containerPort: 9403
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 120
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: metrics
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '96'
|
||||||
|
memory: 2Ti
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
requests:
|
||||||
|
cpu: '32'
|
||||||
|
memory: 512Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /root/.cache/sglang
|
||||||
|
name: sglang-cache
|
||||||
|
- mountPath: /dev/shm
|
||||||
|
name: dshm
|
||||||
|
- mountPath: /root/.cargo/bin/cargo
|
||||||
|
name: fake-cargo
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/cache/sglang
|
||||||
|
type: Directory
|
||||||
|
name: sglang-cache
|
||||||
|
- emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 64Gi
|
||||||
|
name: dshm
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/ds41/fake-cargo
|
||||||
|
type: File
|
||||||
|
name: fake-cargo
|
||||||
|
- dependencies:
|
||||||
|
- mooncake-master
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: d
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '30300'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-d
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 --tokenizer-worker-num 16 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n"
|
||||||
|
env:
|
||||||
|
- name: CUDA_VISIBLE_DEVICES
|
||||||
|
value: 4,5,6,7
|
||||||
|
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||||
|
value: static
|
||||||
|
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV4_KV_LAYOUT
|
||||||
|
value: v41_new
|
||||||
|
- name: SGLANG_DEFAULT_THINKING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||||
|
value: high
|
||||||
|
- name: MC_INTRANODE_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: MC_INTRA_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_TELEMETRY
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||||
|
value: '0.1'
|
||||||
|
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||||
|
value: '600'
|
||||||
|
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||||
|
value: '16'
|
||||||
|
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||||
|
value: '32'
|
||||||
|
- name: TZ
|
||||||
|
value: Asia/Shanghai
|
||||||
|
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||||
|
value: '1024'
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/d-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/d-xdg
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /data/ymk/l3ssd/d-hf
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: sglang
|
||||||
|
ports:
|
||||||
|
- containerPort: 30300
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 150
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: metrics
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '96'
|
||||||
|
memory: 550Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
requests:
|
||||||
|
cpu: '32'
|
||||||
|
memory: 256Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /root/.cache/sglang
|
||||||
|
name: sglang-cache
|
||||||
|
- mountPath: /dev/shm
|
||||||
|
name: dshm
|
||||||
|
- mountPath: /root/.cargo/bin/cargo
|
||||||
|
name: fake-cargo
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/cache/sglang
|
||||||
|
type: Directory
|
||||||
|
name: sglang-cache
|
||||||
|
- emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 64Gi
|
||||||
|
name: dshm
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/ds41/fake-cargo
|
||||||
|
type: File
|
||||||
|
name: fake-cargo
|
||||||
|
- dependencies:
|
||||||
|
- p
|
||||||
|
- d
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: gateway
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '29000'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
prometheus.io/scrape: 'true'
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-gw
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 512 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n"
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: gateway
|
||||||
|
ports:
|
||||||
|
- containerPort: 30400
|
||||||
|
name: http
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 29000
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 60
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 20
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '16'
|
||||||
|
memory: 32Gi
|
||||||
|
requests:
|
||||||
|
cpu: '2'
|
||||||
|
memory: 8Gi
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /opt/sglang/bin/sgl-model-gateway
|
||||||
|
name: gw-bin
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- name: gw-bin
|
||||||
|
hostPath:
|
||||||
|
path: /data/ymk/cache/gw-est-td/sgl-model-gateway
|
||||||
|
type: File
|
||||||
@@ -0,0 +1,488 @@
|
|||||||
|
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||||
|
kind: RoleBasedGroup
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
rbg.workloads.x-k8s.io/discovery-config-mode: refine
|
||||||
|
labels:
|
||||||
|
app: ds41-gist-exp
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
sglang-model: ds-v41-flash
|
||||||
|
name: ds41-gist-exp
|
||||||
|
namespace: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
spec:
|
||||||
|
roles:
|
||||||
|
- minReadySeconds: 0
|
||||||
|
name: mooncake-master
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '19003'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-mooncake
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n"
|
||||||
|
env:
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/master-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/master-xdg
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: mooncake-master
|
||||||
|
ports:
|
||||||
|
- containerPort: 19003
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 60
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 5
|
||||||
|
tcpSocket:
|
||||||
|
port: 50051
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '4'
|
||||||
|
memory: 64Gi
|
||||||
|
requests:
|
||||||
|
cpu: '1'
|
||||||
|
memory: 8Gi
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
- dependencies:
|
||||||
|
- mooncake-master
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: p
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '30000'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-p
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n"
|
||||||
|
env:
|
||||||
|
- name: CUDA_VISIBLE_DEVICES
|
||||||
|
value: 0,1,2,3
|
||||||
|
- name: SGLANG_LOG_MS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK
|
||||||
|
value: '65536'
|
||||||
|
- name: SGLANG_DSV4_KV_LAYOUT
|
||||||
|
value: v41_new
|
||||||
|
- name: SGLANG_DP_USE_GATHERV
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||||
|
value: static
|
||||||
|
- name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_NUMA_BIND_V2
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_DEFAULT_THINKING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||||
|
value: high
|
||||||
|
- name: SGLANG_TIMEOUT_KEEP_ALIVE
|
||||||
|
value: '900'
|
||||||
|
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||||
|
value: '1'
|
||||||
|
- name: MC_INTRANODE_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: MC_INTRA_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||||
|
value: /data/ymk/ds41/mooncake-store-l3ssd.json
|
||||||
|
- name: SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||||
|
value: '4'
|
||||||
|
- name: SGLANG_HICACHE_BACKUP_IO_THREADS
|
||||||
|
value: '4'
|
||||||
|
- name: SGLANG_HICACHE_TIMING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_TELEMETRY
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DEVICE_TIMER
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL
|
||||||
|
value: '100'
|
||||||
|
- name: SGLANG_MM_TRACE
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_SPAN_MIN_MS
|
||||||
|
value: '200'
|
||||||
|
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||||
|
value: '0.05'
|
||||||
|
- name: SGLANG_BATCH_SPAN_MAX_LINKS
|
||||||
|
value: '8'
|
||||||
|
- name: SGLANG_BATCH_STAGE_TIMING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_PASS_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_PASS_SPAN_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_RECV_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_RECV_SPAN_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_MM_TRACE_SYNC
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||||
|
value: '600'
|
||||||
|
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||||
|
value: '16'
|
||||||
|
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||||
|
value: '32'
|
||||||
|
- name: TZ
|
||||||
|
value: Asia/Shanghai
|
||||||
|
- name: SGLANG_TRACE_LOOP_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_TRACE_LOOP_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_FWD_TIMELINE
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_TRACE_LOOP_DETAIL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_TRACE_MOONCAKE_DETAIL
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_TRACE_BACKUP_DETAIL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ASYNC_BACKUP_SUBMIT
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH
|
||||||
|
value: '1'
|
||||||
|
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||||
|
value: '1024'
|
||||||
|
- name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE
|
||||||
|
value: '9400'
|
||||||
|
- name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR
|
||||||
|
value: bucket_storage_backend
|
||||||
|
- name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE
|
||||||
|
value: '1099511627776'
|
||||||
|
- name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES
|
||||||
|
value: '1099511627776'
|
||||||
|
- name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY
|
||||||
|
value: fifo
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/p-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/p-xdg
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /data/ymk/l3ssd/p-hf
|
||||||
|
- name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS
|
||||||
|
value: '0'
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: sglang
|
||||||
|
ports:
|
||||||
|
- containerPort: 30000
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 8998
|
||||||
|
name: bootstrap
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 9400
|
||||||
|
name: mc-0
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 9401
|
||||||
|
name: mc-1
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 9402
|
||||||
|
name: mc-2
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 9403
|
||||||
|
name: mc-3
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 120
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: metrics
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '96'
|
||||||
|
memory: 2Ti
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
requests:
|
||||||
|
cpu: '32'
|
||||||
|
memory: 512Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /root/.cache/sglang
|
||||||
|
name: sglang-cache
|
||||||
|
- mountPath: /dev/shm
|
||||||
|
name: dshm
|
||||||
|
- mountPath: /root/.cargo/bin/cargo
|
||||||
|
name: fake-cargo
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/cache/sglang
|
||||||
|
type: Directory
|
||||||
|
name: sglang-cache
|
||||||
|
- emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 64Gi
|
||||||
|
name: dshm
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/ds41/fake-cargo
|
||||||
|
type: File
|
||||||
|
name: fake-cargo
|
||||||
|
- dependencies:
|
||||||
|
- mooncake-master
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: d
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '30300'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-d
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 --tokenizer-worker-num 16 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n"
|
||||||
|
env:
|
||||||
|
- name: CUDA_VISIBLE_DEVICES
|
||||||
|
value: 4,5,6,7
|
||||||
|
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||||
|
value: static
|
||||||
|
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV4_KV_LAYOUT
|
||||||
|
value: v41_new
|
||||||
|
- name: SGLANG_DEFAULT_THINKING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||||
|
value: high
|
||||||
|
- name: MC_INTRANODE_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: MC_INTRA_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_TELEMETRY
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||||
|
value: '0.1'
|
||||||
|
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||||
|
value: '600'
|
||||||
|
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||||
|
value: '16'
|
||||||
|
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||||
|
value: '32'
|
||||||
|
- name: TZ
|
||||||
|
value: Asia/Shanghai
|
||||||
|
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||||
|
value: '1024'
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/d-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/d-xdg
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /data/ymk/l3ssd/d-hf
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: sglang
|
||||||
|
ports:
|
||||||
|
- containerPort: 30300
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 150
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: metrics
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '96'
|
||||||
|
memory: 550Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
requests:
|
||||||
|
cpu: '32'
|
||||||
|
memory: 256Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /root/.cache/sglang
|
||||||
|
name: sglang-cache
|
||||||
|
- mountPath: /dev/shm
|
||||||
|
name: dshm
|
||||||
|
- mountPath: /root/.cargo/bin/cargo
|
||||||
|
name: fake-cargo
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/cache/sglang
|
||||||
|
type: Directory
|
||||||
|
name: sglang-cache
|
||||||
|
- emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 64Gi
|
||||||
|
name: dshm
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/ds41/fake-cargo
|
||||||
|
type: File
|
||||||
|
name: fake-cargo
|
||||||
|
- dependencies:
|
||||||
|
- p
|
||||||
|
- d
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: gateway
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '29000'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
prometheus.io/scrape: 'true'
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-gw
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 --g9i-est-td-enabled --g9i-shadow-enabled --g9i-td-admit off --g9i-p-queue-reqs-cap 0 --g9i-p-queue-token-cap 0 --g9i-p-qps 0 --g9i-p-refresh-ms 250 --g9i-window-fixed 256 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 1000000 \\\n --pool-idle-timeout-secs 2 --pool-max-idle-per-host 0 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 --disable-circuit-breaker --disable-health-check \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n"
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: gateway
|
||||||
|
ports:
|
||||||
|
- containerPort: 30400
|
||||||
|
name: http
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 29000
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 60
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 20
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '16'
|
||||||
|
memory: 32Gi
|
||||||
|
requests:
|
||||||
|
cpu: '2'
|
||||||
|
memory: 8Gi
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /opt/sglang/bin/sgl-model-gateway
|
||||||
|
name: gw-bin
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/gw-ns-est-td/sgl-model-gateway
|
||||||
|
type: File
|
||||||
|
name: gw-bin
|
||||||
@@ -0,0 +1,488 @@
|
|||||||
|
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||||
|
kind: RoleBasedGroup
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
rbg.workloads.x-k8s.io/discovery-config-mode: refine
|
||||||
|
labels:
|
||||||
|
app: ds41-gist-exp
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
sglang-model: ds-v41-flash
|
||||||
|
name: ds41-gist-exp
|
||||||
|
namespace: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
spec:
|
||||||
|
roles:
|
||||||
|
- minReadySeconds: 0
|
||||||
|
name: mooncake-master
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '19003'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-mooncake
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n"
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: mooncake-master
|
||||||
|
ports:
|
||||||
|
- containerPort: 19003
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 60
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 5
|
||||||
|
tcpSocket:
|
||||||
|
port: 50051
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '4'
|
||||||
|
memory: 64Gi
|
||||||
|
requests:
|
||||||
|
cpu: '1'
|
||||||
|
memory: 8Gi
|
||||||
|
env:
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/master-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/master-xdg
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
- dependencies:
|
||||||
|
- mooncake-master
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: p
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '30000'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-p
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n"
|
||||||
|
env:
|
||||||
|
- name: CUDA_VISIBLE_DEVICES
|
||||||
|
value: 0,1,2,3
|
||||||
|
- name: SGLANG_LOG_MS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK
|
||||||
|
value: '65536'
|
||||||
|
- name: SGLANG_DSV4_KV_LAYOUT
|
||||||
|
value: v41_new
|
||||||
|
- name: SGLANG_DP_USE_GATHERV
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||||
|
value: static
|
||||||
|
- name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_NUMA_BIND_V2
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_DEFAULT_THINKING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||||
|
value: high
|
||||||
|
- name: SGLANG_TIMEOUT_KEEP_ALIVE
|
||||||
|
value: '900'
|
||||||
|
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||||
|
value: '1'
|
||||||
|
- name: MC_INTRANODE_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: MC_INTRA_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||||
|
value: /data/ymk/ds41/mooncake-store-l3ssd.json
|
||||||
|
- name: SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||||
|
value: '4'
|
||||||
|
- name: SGLANG_HICACHE_BACKUP_IO_THREADS
|
||||||
|
value: '4'
|
||||||
|
- name: SGLANG_HICACHE_TIMING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_TELEMETRY
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DEVICE_TIMER
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL
|
||||||
|
value: '100'
|
||||||
|
- name: SGLANG_MM_TRACE
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_SPAN_MIN_MS
|
||||||
|
value: '200'
|
||||||
|
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||||
|
value: '0.05'
|
||||||
|
- name: SGLANG_BATCH_SPAN_MAX_LINKS
|
||||||
|
value: '8'
|
||||||
|
- name: SGLANG_BATCH_STAGE_TIMING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_PASS_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_PASS_SPAN_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_RECV_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_RECV_SPAN_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_MM_TRACE_SYNC
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||||
|
value: '600'
|
||||||
|
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||||
|
value: '16'
|
||||||
|
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||||
|
value: '32'
|
||||||
|
- name: TZ
|
||||||
|
value: Asia/Shanghai
|
||||||
|
- name: SGLANG_TRACE_LOOP_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_TRACE_LOOP_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_FWD_TIMELINE
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_TRACE_LOOP_DETAIL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_TRACE_MOONCAKE_DETAIL
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_TRACE_BACKUP_DETAIL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ASYNC_BACKUP_SUBMIT
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH
|
||||||
|
value: '1'
|
||||||
|
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||||
|
value: '1024'
|
||||||
|
- name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE
|
||||||
|
value: '9400'
|
||||||
|
- name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR
|
||||||
|
value: bucket_storage_backend
|
||||||
|
- name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE
|
||||||
|
value: '1099511627776'
|
||||||
|
- name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES
|
||||||
|
value: '1099511627776'
|
||||||
|
- name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY
|
||||||
|
value: fifo
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/p-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/p-xdg
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /data/ymk/l3ssd/p-hf
|
||||||
|
- name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS
|
||||||
|
value: '0'
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: sglang
|
||||||
|
ports:
|
||||||
|
- containerPort: 30000
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 8998
|
||||||
|
name: bootstrap
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-0
|
||||||
|
containerPort: 9400
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-1
|
||||||
|
containerPort: 9401
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-2
|
||||||
|
containerPort: 9402
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-3
|
||||||
|
containerPort: 9403
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 120
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: metrics
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '96'
|
||||||
|
memory: 2Ti
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
requests:
|
||||||
|
cpu: '32'
|
||||||
|
memory: 512Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /root/.cache/sglang
|
||||||
|
name: sglang-cache
|
||||||
|
- mountPath: /dev/shm
|
||||||
|
name: dshm
|
||||||
|
- mountPath: /root/.cargo/bin/cargo
|
||||||
|
name: fake-cargo
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/cache/sglang
|
||||||
|
type: Directory
|
||||||
|
name: sglang-cache
|
||||||
|
- emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 64Gi
|
||||||
|
name: dshm
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/ds41/fake-cargo
|
||||||
|
type: File
|
||||||
|
name: fake-cargo
|
||||||
|
- dependencies:
|
||||||
|
- mooncake-master
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: d
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '30300'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-d
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 --tokenizer-worker-num 16 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n"
|
||||||
|
env:
|
||||||
|
- name: CUDA_VISIBLE_DEVICES
|
||||||
|
value: 4,5,6,7
|
||||||
|
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||||
|
value: static
|
||||||
|
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV4_KV_LAYOUT
|
||||||
|
value: v41_new
|
||||||
|
- name: SGLANG_DEFAULT_THINKING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||||
|
value: high
|
||||||
|
- name: MC_INTRANODE_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: MC_INTRA_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_TELEMETRY
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||||
|
value: '0.1'
|
||||||
|
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||||
|
value: '600'
|
||||||
|
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||||
|
value: '16'
|
||||||
|
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||||
|
value: '32'
|
||||||
|
- name: TZ
|
||||||
|
value: Asia/Shanghai
|
||||||
|
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||||
|
value: '1024'
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/d-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/d-xdg
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /data/ymk/l3ssd/d-hf
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: sglang
|
||||||
|
ports:
|
||||||
|
- containerPort: 30300
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 150
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: metrics
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '96'
|
||||||
|
memory: 550Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
requests:
|
||||||
|
cpu: '32'
|
||||||
|
memory: 256Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /root/.cache/sglang
|
||||||
|
name: sglang-cache
|
||||||
|
- mountPath: /dev/shm
|
||||||
|
name: dshm
|
||||||
|
- mountPath: /root/.cargo/bin/cargo
|
||||||
|
name: fake-cargo
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/cache/sglang
|
||||||
|
type: Directory
|
||||||
|
name: sglang-cache
|
||||||
|
- emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 64Gi
|
||||||
|
name: dshm
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/ds41/fake-cargo
|
||||||
|
type: File
|
||||||
|
name: fake-cargo
|
||||||
|
- dependencies:
|
||||||
|
- p
|
||||||
|
- d
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: gateway
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '29000'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
prometheus.io/scrape: 'true'
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-gw
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 512 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n"
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: gateway
|
||||||
|
ports:
|
||||||
|
- containerPort: 30400
|
||||||
|
name: http
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 29000
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 60
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 20
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '16'
|
||||||
|
memory: 32Gi
|
||||||
|
requests:
|
||||||
|
cpu: '2'
|
||||||
|
memory: 8Gi
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /opt/sglang/bin/sgl-model-gateway
|
||||||
|
name: gw-bin
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- name: gw-bin
|
||||||
|
hostPath:
|
||||||
|
path: /data/ymk/cache/gw-est-td/sgl-model-gateway
|
||||||
|
type: File
|
||||||
@@ -0,0 +1,488 @@
|
|||||||
|
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||||
|
kind: RoleBasedGroup
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
rbg.workloads.x-k8s.io/discovery-config-mode: refine
|
||||||
|
labels:
|
||||||
|
app: ds41-gist-exp
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
sglang-model: ds-v41-flash
|
||||||
|
name: ds41-gist-exp
|
||||||
|
namespace: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
spec:
|
||||||
|
roles:
|
||||||
|
- minReadySeconds: 0
|
||||||
|
name: mooncake-master
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '19003'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-mooncake
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n"
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: mooncake-master
|
||||||
|
ports:
|
||||||
|
- containerPort: 19003
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 60
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 5
|
||||||
|
tcpSocket:
|
||||||
|
port: 50051
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '4'
|
||||||
|
memory: 64Gi
|
||||||
|
requests:
|
||||||
|
cpu: '1'
|
||||||
|
memory: 8Gi
|
||||||
|
env:
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/master-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/master-xdg
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
- dependencies:
|
||||||
|
- mooncake-master
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: p
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '30000'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-p
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec sglang serve \\\n --trust-remote-code --tokenizer-backend huggingface --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n"
|
||||||
|
env:
|
||||||
|
- name: CUDA_VISIBLE_DEVICES
|
||||||
|
value: 0,1,2,3
|
||||||
|
- name: SGLANG_LOG_MS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK
|
||||||
|
value: '65536'
|
||||||
|
- name: SGLANG_DSV4_KV_LAYOUT
|
||||||
|
value: v41_new
|
||||||
|
- name: SGLANG_DP_USE_GATHERV
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||||
|
value: static
|
||||||
|
- name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_NUMA_BIND_V2
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_DEFAULT_THINKING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||||
|
value: high
|
||||||
|
- name: SGLANG_TIMEOUT_KEEP_ALIVE
|
||||||
|
value: '900'
|
||||||
|
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||||
|
value: '1'
|
||||||
|
- name: MC_INTRANODE_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: MC_INTRA_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||||
|
value: /data/ymk/ds41/mooncake-store-l3ssd.json
|
||||||
|
- name: SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||||
|
value: '4'
|
||||||
|
- name: SGLANG_HICACHE_BACKUP_IO_THREADS
|
||||||
|
value: '4'
|
||||||
|
- name: SGLANG_HICACHE_TIMING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_TELEMETRY
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DEVICE_TIMER
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL
|
||||||
|
value: '100'
|
||||||
|
- name: SGLANG_MM_TRACE
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_SPAN_MIN_MS
|
||||||
|
value: '200'
|
||||||
|
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||||
|
value: '0.05'
|
||||||
|
- name: SGLANG_BATCH_SPAN_MAX_LINKS
|
||||||
|
value: '8'
|
||||||
|
- name: SGLANG_BATCH_STAGE_TIMING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_PASS_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_PASS_SPAN_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_RECV_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_RECV_SPAN_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_MM_TRACE_SYNC
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||||
|
value: '600'
|
||||||
|
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||||
|
value: '16'
|
||||||
|
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||||
|
value: '32'
|
||||||
|
- name: TZ
|
||||||
|
value: Asia/Shanghai
|
||||||
|
- name: SGLANG_TRACE_LOOP_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_TRACE_LOOP_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_FWD_TIMELINE
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_TRACE_LOOP_DETAIL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_TRACE_MOONCAKE_DETAIL
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_TRACE_BACKUP_DETAIL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ASYNC_BACKUP_SUBMIT
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH
|
||||||
|
value: '1'
|
||||||
|
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||||
|
value: '1024'
|
||||||
|
- name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE
|
||||||
|
value: '9400'
|
||||||
|
- name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR
|
||||||
|
value: bucket_storage_backend
|
||||||
|
- name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE
|
||||||
|
value: '1099511627776'
|
||||||
|
- name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES
|
||||||
|
value: '1099511627776'
|
||||||
|
- name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY
|
||||||
|
value: fifo
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/p-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/p-xdg
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /data/ymk/l3ssd/p-hf
|
||||||
|
- name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS
|
||||||
|
value: '0'
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: sglang
|
||||||
|
ports:
|
||||||
|
- containerPort: 30000
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 8998
|
||||||
|
name: bootstrap
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-0
|
||||||
|
containerPort: 9400
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-1
|
||||||
|
containerPort: 9401
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-2
|
||||||
|
containerPort: 9402
|
||||||
|
protocol: TCP
|
||||||
|
- name: mc-3
|
||||||
|
containerPort: 9403
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 120
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: metrics
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '96'
|
||||||
|
memory: 2Ti
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
requests:
|
||||||
|
cpu: '32'
|
||||||
|
memory: 512Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /root/.cache/sglang
|
||||||
|
name: sglang-cache
|
||||||
|
- mountPath: /dev/shm
|
||||||
|
name: dshm
|
||||||
|
- mountPath: /root/.cargo/bin/cargo
|
||||||
|
name: fake-cargo
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/cache/sglang
|
||||||
|
type: Directory
|
||||||
|
name: sglang-cache
|
||||||
|
- emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 64Gi
|
||||||
|
name: dshm
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/ds41/fake-cargo
|
||||||
|
type: File
|
||||||
|
name: fake-cargo
|
||||||
|
- dependencies:
|
||||||
|
- mooncake-master
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: d
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '30300'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-d
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec sglang serve \\\n --trust-remote-code --tokenizer-backend huggingface --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 --tokenizer-worker-num 16 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n"
|
||||||
|
env:
|
||||||
|
- name: CUDA_VISIBLE_DEVICES
|
||||||
|
value: 4,5,6,7
|
||||||
|
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||||
|
value: static
|
||||||
|
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV4_KV_LAYOUT
|
||||||
|
value: v41_new
|
||||||
|
- name: SGLANG_DEFAULT_THINKING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||||
|
value: high
|
||||||
|
- name: MC_INTRANODE_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: MC_INTRA_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_TELEMETRY
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||||
|
value: '0.1'
|
||||||
|
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||||
|
value: '600'
|
||||||
|
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||||
|
value: '16'
|
||||||
|
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||||
|
value: '32'
|
||||||
|
- name: TZ
|
||||||
|
value: Asia/Shanghai
|
||||||
|
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||||
|
value: '1024'
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/d-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/d-xdg
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /data/ymk/l3ssd/d-hf
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: sglang
|
||||||
|
ports:
|
||||||
|
- containerPort: 30300
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 150
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: metrics
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '96'
|
||||||
|
memory: 550Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
requests:
|
||||||
|
cpu: '32'
|
||||||
|
memory: 256Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /root/.cache/sglang
|
||||||
|
name: sglang-cache
|
||||||
|
- mountPath: /dev/shm
|
||||||
|
name: dshm
|
||||||
|
- mountPath: /root/.cargo/bin/cargo
|
||||||
|
name: fake-cargo
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/cache/sglang
|
||||||
|
type: Directory
|
||||||
|
name: sglang-cache
|
||||||
|
- emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 64Gi
|
||||||
|
name: dshm
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/ds41/fake-cargo
|
||||||
|
type: File
|
||||||
|
name: fake-cargo
|
||||||
|
- dependencies:
|
||||||
|
- p
|
||||||
|
- d
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: gateway
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '29000'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
prometheus.io/scrape: 'true'
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-gw
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 512 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n"
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: gateway
|
||||||
|
ports:
|
||||||
|
- containerPort: 30400
|
||||||
|
name: http
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 29000
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 60
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 20
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '16'
|
||||||
|
memory: 32Gi
|
||||||
|
requests:
|
||||||
|
cpu: '2'
|
||||||
|
memory: 8Gi
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /opt/sglang/bin/sgl-model-gateway
|
||||||
|
name: gw-bin
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- name: gw-bin
|
||||||
|
hostPath:
|
||||||
|
path: /data/ymk/cache/gw-est-td/sgl-model-gateway
|
||||||
|
type: File
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
<h1>当前基线:ds41-gist-exp(4T L3 SSD,2026-11-17)</h1>
|
||||||
|
<callout emoji="📌" background-color="light-blue"><p>本节是 2026-11-17 定稿的 c580/c620 基线,后续 Dynamo 叠加以此为底座。旧的 ds41-gist-1p1d 章节已归档,不再作为当前部署。</p></callout>
|
||||||
|
|
||||||
|
<h2>1. 基线身份与镜像</h2>
|
||||||
|
<ul>
|
||||||
|
<li><b>工作负载</b>:<code>ds41-gist-exp</code> / <code>cw-dblft6sgrhagmpdt8ef0</code>,namespace <code>prj-d7te67bj0sq2fgdrmbig</code>,b300-01。</li>
|
||||||
|
<li><b>镜像</b>:<code>sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115</code>(40.1GB,含 Mooncake 0.3.14 + L3 SSD 支持)。</li>
|
||||||
|
<li><b>manifest</b>:<code>deploy/b300-k8s/ds41-gist-exp-20261117-dtok16-fastokens.yaml</code>;基线副本 <code>deploy/b300-k8s/baselines/20261117-dtok16-fastokens/manifest.yaml</code>;b300 副本在 <code>/data/ymk/deploy/b300-k8s/</code>。</li>
|
||||||
|
<li><b>形态</b>:RBG 三角色 <code>p</code> / <code>d</code> / <code>gateway</code> + <code>mooncake-master</code>;全 hostNetwork / hostPID / privileged;P=GPU0-3,D=GPU4-7。</li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<h2>2. 部署方式与端口</h2>
|
||||||
|
<table>
|
||||||
|
<thead><tr><th><p>角色</p></th><th><p>端口</p></th><th><p>说明</p></th></tr></thead>
|
||||||
|
<tbody>
|
||||||
|
<tr><td><p>P</p></td><td><p>30000 / 8998 / 9400-9403</p></td><td><p>HTTP / bootstrap / Mooncake client metrics</p></td></tr>
|
||||||
|
<tr><td><p>D</p></td><td><p>30300</p></td><td><p>HTTP</p></td></tr>
|
||||||
|
<tr><td><p>gateway</p></td><td><p>30400 / 29000</p></td><td><p>业务 / metrics</p></td></tr>
|
||||||
|
<tr><td><p>mooncake-master</p></td><td><p>50051 / 18081 / 19003</p></td><td><p>store / metadata / metrics</p></td></tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
<p>默认入口:<code>http://127.0.0.1:30400</code>;当前 new-api/AgentX 直接使用该入口。</p>
|
||||||
|
|
||||||
|
<h2>3. 关键参数</h2>
|
||||||
|
<p><b>P(prefill)</b>:tp4 / dp4 / ep4,megamoe + flashmla_mega,ep-redundant 32,DSpark block 3,decoder SWA bounded replay;chunked-prefill 131072 / max-prefill 131072,max-running 512;fastokens + tokenizer-worker 16;HiCache ratio 4、page_first_direct、direct、write_through、Mooncake L3;<b>cache 模式,无 buffer_only、无 optimistic</b>;mem-fraction 0.80。</p>
|
||||||
|
<p><b>D(decode)</b>:tp4 / dp4 / ep4,ep-redundant 64,DSpark block 3,stream-interval 32,swa-prefix-tails 2048;max-running 512,cuda-graph-max-bs-decode 512;chunked 32768 / max-prefill 8192;fastokens + tokenizer-worker 16;mem-fraction 0.80。</p>
|
||||||
|
<p><b>gateway</b>:镜像内 <code>sgl-model-gateway</code>;<b>无 P admission cap</b>(无 p-qps / p-queue-reqs-cap / p-queue-token-cap,td-admit 默认 off);保留 health check + circuit breaker;<code>--g9i-decode-max-running 512</code>。这是 c580 能跑通的关键。</p>
|
||||||
|
<p><b>L3</b>:Mooncake master 4TB SSD offload;<code>offload_on_evict=false</code>;L3 配置见 <code>/data/ymk/ds41/mooncake-store-l3ssd.json</code>。</p>
|
||||||
|
|
||||||
|
<h2>4. 启动 / 重启流程</h2>
|
||||||
|
<pre lang="bash"><code># 1) 停完整工作负载并确认 GPU 0
|
||||||
|
MSYS_NO_PATHCONV=1 wsl -e /home/ubuntu/.local/bin/kmctl stop cw ds41-gist-exp --wait --timeout 10m --request-timeout 180s
|
||||||
|
ssh 24.125.93.200.207 "nvidia-smi --query-gpu=index,memory.used --format=csv,noheader"
|
||||||
|
|
||||||
|
# 2) 清 L3 SSD(实验/基线前必须)
|
||||||
|
ssh 24.125.93.200.207 'bash /data/ymk/deploy/b300-k8s/clean_l3ssd_cache.sh'
|
||||||
|
|
||||||
|
# 3) 用完整平台 manifest 启动(不要只 kubectl patch gateway)
|
||||||
|
MSYS_NO_PATHCONV=1 wsl -e /home/ubuntu/.local/bin/kmctl update cw ds41-gist-exp \
|
||||||
|
-f /mnt/d/B300/deploy/b300-k8s/ds41-gist-exp-20261117-dtok16-fastokens.yaml
|
||||||
|
|
||||||
|
# 4) 等 4 Pod 1/1,检查 P/D/gateway health 200,再 smoke
|
||||||
|
curl -sS http://127.0.0.1:30400/v1/chat/completions -H 'Content-Type: application/json' \
|
||||||
|
-d '{"model":"/data/models/DeepSeek-V4.1-Flash","messages":[{"role":"user","content":"1+1=?"}],"max_tokens":8}'
|
||||||
|
</code></pre>
|
||||||
|
<p>启动基线:P <code>max_total_num_tokens≈16,158,208</code> / 128 per-rank / avail ≈23GB;D <code>58,859,520</code> / 128 per-rank / avail ≈79GB。正常 warmup 6,385 条约 6.5 分钟。</p>
|
||||||
|
|
||||||
|
<h2>5. 基线结果</h2>
|
||||||
|
<table>
|
||||||
|
<thead><tr><th><p>场景</p></th><th><p>RPS</p></th><th><p>TTFT p50/p95/p99</p></th><th><p>ITL p95</p></th><th><p>结论</p></th></tr></thead>
|
||||||
|
<tbody>
|
||||||
|
<tr><td><p>c580 基线</p></td><td><p>24.045</p></td><td><p>1.743 / 3.772 / 6.580 s</p></td><td><p>10.74 ms</p></td><td><p>推荐工作点</p></td></tr>
|
||||||
|
<tr><td><p>c620 首轮(清 L3)</p></td><td><p>25.998</p></td><td><p>1.814 / 4.041 / 8.257 s</p></td><td><p>11.69 ms</p></td><td><p>吞吐 +8%,尾部变差</p></td></tr>
|
||||||
|
<tr><td><p>c620 不重启 r2/r3</p></td><td><p>25.85 / 25.61</p></td><td><p>p95 5.39 / 6.44 s,p99 9.56 / 10.79 s</p></td><td><p>10.96 / 10.68 ms</p></td><td><p>D KV 接近/打满,TTFT 尾部持续劣化,不建议连测</p></td></tr>
|
||||||
|
<tr><td><p>buffer_only</p></td><td><p>23.749</p></td><td><p>2.153 / 4.904 / 10.483 s</p></td><td><p>10.32 ms</p></td><td><p>吞吐 -1.2%、TTFT 更差,不采用</p></td></tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
<p>结论:<b>c580 是稳定基线</b>;c620 只适合短时冲吞吐,且每轮之间必须重启/清池。buffer_only 不作为基线。</p>
|
||||||
|
|
||||||
|
<h2>6. 作为 Dynamo 底座</h2>
|
||||||
|
<p>当前镜像是 <b>SGLang + Mooncake 0.3.14 + L3 SSD</b> 基线,<b>不包含 Dynamo/nixl</b>。后续套 Dynamo 需要以本镜像为 base 另装 Dynamo 1.6.x / nixl,旧 <code>gist-nvlink-7f6fe2244-dynamo-local-20261026</code> 是更早的 10-23 代码,不是本基线。</p>
|
||||||
|
<p>Dynamo 叠加时保留本页的 P/D 参数、L3 SSD 和 gateway 入口;先做单机 4P4D 冒烟,再评估 Dynamo 的 P/D 角色、discovery 和 KV 传输配置。</p>
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
# 2026-11-17 cache + gateway no-limit 实验参数(失败记录)
|
||||||
|
|
||||||
|
- P: hicache_host_memory_mode=cache, optimistic_prefill_attempts=0, tokenizer-worker-num=16,
|
||||||
|
hicache ratio 4 / write_through / L3 mooncake, L3 SSD 4T.
|
||||||
|
- D: tokenizer-worker-num=16.
|
||||||
|
- Gateway: g9i-td-admit off, p-queue-reqs-cap 0, p-queue-token-cap 0, p-qps 0,
|
||||||
|
decode-max-running 1000000, disable-circuit-breaker, disable-health-check.
|
||||||
|
- Image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
- c580 run c580-cache-nolimit-20261117: warmup 4,720/6,385 (74%), 02:33:32 失败
|
||||||
|
(ServerDisconnected/warmup_failure;P/D/gateway 0 restart,无 503)。
|
||||||
|
- 观察:P 单批 32k/23k chunk 正常,但每 rank 同时仅 1 个请求;gateway
|
||||||
|
smg_worker_requests_active 峰值 1,840,P queue 峰值 503,gateway 约 1 核。
|
||||||
|
- 结论:该 no-limit/no-cb 网关配置不适合作为 c580 基线;仅作异常记录。
|
||||||
@@ -0,0 +1,488 @@
|
|||||||
|
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||||
|
kind: RoleBasedGroup
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
rbg.workloads.x-k8s.io/discovery-config-mode: refine
|
||||||
|
labels:
|
||||||
|
app: ds41-gist-exp
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
sglang-model: ds-v41-flash
|
||||||
|
name: ds41-gist-exp
|
||||||
|
namespace: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
spec:
|
||||||
|
roles:
|
||||||
|
- minReadySeconds: 0
|
||||||
|
name: mooncake-master
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '19003'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-mooncake
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n"
|
||||||
|
env:
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/master-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/master-xdg
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: mooncake-master
|
||||||
|
ports:
|
||||||
|
- containerPort: 19003
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 60
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 5
|
||||||
|
tcpSocket:
|
||||||
|
port: 50051
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '4'
|
||||||
|
memory: 64Gi
|
||||||
|
requests:
|
||||||
|
cpu: '1'
|
||||||
|
memory: 8Gi
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
- dependencies:
|
||||||
|
- mooncake-master
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: p
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '30000'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-p
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n"
|
||||||
|
env:
|
||||||
|
- name: CUDA_VISIBLE_DEVICES
|
||||||
|
value: 0,1,2,3
|
||||||
|
- name: SGLANG_LOG_MS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK
|
||||||
|
value: '65536'
|
||||||
|
- name: SGLANG_DSV4_KV_LAYOUT
|
||||||
|
value: v41_new
|
||||||
|
- name: SGLANG_DP_USE_GATHERV
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||||
|
value: static
|
||||||
|
- name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_NUMA_BIND_V2
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_DEFAULT_THINKING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||||
|
value: high
|
||||||
|
- name: SGLANG_TIMEOUT_KEEP_ALIVE
|
||||||
|
value: '900'
|
||||||
|
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||||
|
value: '1'
|
||||||
|
- name: MC_INTRANODE_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: MC_INTRA_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||||
|
value: /data/ymk/ds41/mooncake-store-l3ssd.json
|
||||||
|
- name: SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||||
|
value: '4'
|
||||||
|
- name: SGLANG_HICACHE_BACKUP_IO_THREADS
|
||||||
|
value: '4'
|
||||||
|
- name: SGLANG_HICACHE_TIMING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_TELEMETRY
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DEVICE_TIMER
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL
|
||||||
|
value: '100'
|
||||||
|
- name: SGLANG_MM_TRACE
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_SPAN_MIN_MS
|
||||||
|
value: '200'
|
||||||
|
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||||
|
value: '0.05'
|
||||||
|
- name: SGLANG_BATCH_SPAN_MAX_LINKS
|
||||||
|
value: '8'
|
||||||
|
- name: SGLANG_BATCH_STAGE_TIMING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_PASS_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_PASS_SPAN_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_RECV_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_RECV_SPAN_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_MM_TRACE_SYNC
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||||
|
value: '600'
|
||||||
|
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||||
|
value: '16'
|
||||||
|
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||||
|
value: '32'
|
||||||
|
- name: TZ
|
||||||
|
value: Asia/Shanghai
|
||||||
|
- name: SGLANG_TRACE_LOOP_SPAN
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_TRACE_LOOP_SAMPLE
|
||||||
|
value: '1.0'
|
||||||
|
- name: SGLANG_FWD_TIMELINE
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_TRACE_LOOP_DETAIL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_TRACE_MOONCAKE_DETAIL
|
||||||
|
value: '0'
|
||||||
|
- name: SGLANG_TRACE_BACKUP_DETAIL
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_ASYNC_BACKUP_SUBMIT
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH
|
||||||
|
value: '1'
|
||||||
|
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||||
|
value: '1024'
|
||||||
|
- name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE
|
||||||
|
value: '9400'
|
||||||
|
- name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR
|
||||||
|
value: bucket_storage_backend
|
||||||
|
- name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE
|
||||||
|
value: '1099511627776'
|
||||||
|
- name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES
|
||||||
|
value: '1099511627776'
|
||||||
|
- name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY
|
||||||
|
value: fifo
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/p-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/p-xdg
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /data/ymk/l3ssd/p-hf
|
||||||
|
- name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS
|
||||||
|
value: '0'
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: sglang
|
||||||
|
ports:
|
||||||
|
- containerPort: 30000
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 8998
|
||||||
|
name: bootstrap
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 9400
|
||||||
|
name: mc-0
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 9401
|
||||||
|
name: mc-1
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 9402
|
||||||
|
name: mc-2
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 9403
|
||||||
|
name: mc-3
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 120
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: metrics
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '96'
|
||||||
|
memory: 2Ti
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
requests:
|
||||||
|
cpu: '32'
|
||||||
|
memory: 512Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /root/.cache/sglang
|
||||||
|
name: sglang-cache
|
||||||
|
- mountPath: /dev/shm
|
||||||
|
name: dshm
|
||||||
|
- mountPath: /root/.cargo/bin/cargo
|
||||||
|
name: fake-cargo
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/cache/sglang
|
||||||
|
type: Directory
|
||||||
|
name: sglang-cache
|
||||||
|
- emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 64Gi
|
||||||
|
name: dshm
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/ds41/fake-cargo
|
||||||
|
type: File
|
||||||
|
name: fake-cargo
|
||||||
|
- dependencies:
|
||||||
|
- mooncake-master
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: d
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '30300'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-d
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 --tokenizer-worker-num 16 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n"
|
||||||
|
env:
|
||||||
|
- name: CUDA_VISIBLE_DEVICES
|
||||||
|
value: 4,5,6,7
|
||||||
|
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||||
|
value: static
|
||||||
|
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV4_KV_LAYOUT
|
||||||
|
value: v41_new
|
||||||
|
- name: SGLANG_DEFAULT_THINKING
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||||
|
value: high
|
||||||
|
- name: MC_INTRANODE_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: MC_INTRA_NVLINK
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||||
|
value: 'true'
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_TELEMETRY
|
||||||
|
value: '1'
|
||||||
|
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||||
|
value: '0.1'
|
||||||
|
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||||
|
value: '600'
|
||||||
|
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||||
|
value: '16'
|
||||||
|
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||||
|
value: '32'
|
||||||
|
- name: TZ
|
||||||
|
value: Asia/Shanghai
|
||||||
|
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||||
|
value: '1024'
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/d-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/d-xdg
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /data/ymk/l3ssd/d-hf
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: sglang
|
||||||
|
ports:
|
||||||
|
- containerPort: 30300
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 150
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: metrics
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '96'
|
||||||
|
memory: 550Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
requests:
|
||||||
|
cpu: '32'
|
||||||
|
memory: 256Gi
|
||||||
|
nvidia.com/gpu: '4'
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /root/.cache/sglang
|
||||||
|
name: sglang-cache
|
||||||
|
- mountPath: /dev/shm
|
||||||
|
name: dshm
|
||||||
|
- mountPath: /root/.cargo/bin/cargo
|
||||||
|
name: fake-cargo
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/cache/sglang
|
||||||
|
type: Directory
|
||||||
|
name: sglang-cache
|
||||||
|
- emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 64Gi
|
||||||
|
name: dshm
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/ds41/fake-cargo
|
||||||
|
type: File
|
||||||
|
name: fake-cargo
|
||||||
|
- dependencies:
|
||||||
|
- p
|
||||||
|
- d
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: gateway
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: '29000'
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
prometheus.io/scrape: 'true'
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-gw
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 --g9i-est-td-enabled --g9i-shadow-enabled --g9i-td-admit off --g9i-p-queue-reqs-cap 0 --g9i-p-queue-token-cap 0 --g9i-p-qps 0 --g9i-p-refresh-ms 250 --g9i-window-fixed 256 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 1000000 \\\n --pool-idle-timeout-secs 2 --pool-max-idle-per-host 0 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 --disable-circuit-breaker --disable-health-check \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n"
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: gateway
|
||||||
|
ports:
|
||||||
|
- containerPort: 30400
|
||||||
|
name: http
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 29000
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 60
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 20
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: '16'
|
||||||
|
memory: 32Gi
|
||||||
|
requests:
|
||||||
|
cpu: '2'
|
||||||
|
memory: 8Gi
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /opt/sglang/bin/sgl-model-gateway
|
||||||
|
name: gw-bin
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/gw-ns-est-td/sgl-model-gateway
|
||||||
|
type: File
|
||||||
|
name: gw-bin
|
||||||
@@ -0,0 +1,608 @@
|
|||||||
|
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||||
|
kind: RoleBasedGroup
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
rbg.workloads.x-k8s.io/discovery-config-mode: refine
|
||||||
|
creationTimestamp: "2026-11-16T13:54:16Z"
|
||||||
|
generation: 6
|
||||||
|
labels:
|
||||||
|
app: ds41-gist-exp
|
||||||
|
kongming.io/history-revision: hrev-b90013fa6cc5a2a0e6515115d188135ea41e48b7
|
||||||
|
kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0
|
||||||
|
sglang-model: ds-v41-flash
|
||||||
|
name: ds41-gist-exp
|
||||||
|
namespace: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
resourceVersion: "31040141"
|
||||||
|
uid: 242c2504-0c97-4414-ae60-8376f7d63de0
|
||||||
|
spec:
|
||||||
|
roles:
|
||||||
|
- minReadySeconds: 0
|
||||||
|
name: mooncake-master
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: "19003"
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-mooncake
|
||||||
|
kongming.io/history-revision: hrev-b90013fa6cc5a2a0e6515115d188135ea41e48b7
|
||||||
|
kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- |
|
||||||
|
# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;
|
||||||
|
# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。
|
||||||
|
exec /opt/sglang/bin/mooncake_master \
|
||||||
|
--port 50051 --enable_http_metadata_server=true \
|
||||||
|
--http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false
|
||||||
|
env:
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/master-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/master-xdg
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: mooncake-master
|
||||||
|
ports:
|
||||||
|
- containerPort: 19003
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 60
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 5
|
||||||
|
tcpSocket:
|
||||||
|
port: 50051
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: "4"
|
||||||
|
memory: 64Gi
|
||||||
|
requests:
|
||||||
|
cpu: "1"
|
||||||
|
memory: 8Gi
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
- dependencies:
|
||||||
|
- mooncake-master
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: p
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: "30000"
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-p
|
||||||
|
kongming.io/history-revision: hrev-b90013fa6cc5a2a0e6515115d188135ea41e48b7
|
||||||
|
kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- |
|
||||||
|
exec sglang serve \
|
||||||
|
--trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \
|
||||||
|
--reasoning-parser auto --tool-call-parser auto \
|
||||||
|
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||||
|
--tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \
|
||||||
|
--moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \
|
||||||
|
--ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \
|
||||||
|
--init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \
|
||||||
|
--mem-fraction-static 0.80 \
|
||||||
|
--chunked-prefill-size 131072 --max-prefill-tokens 131072 \
|
||||||
|
--max-running-requests 512 --tokenizer-worker-num 16 \
|
||||||
|
--load-balance-method total_tokens \
|
||||||
|
--speculative-algorithm DSPARK --speculative-dspark-block-size 3 \
|
||||||
|
--enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \
|
||||||
|
--cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \
|
||||||
|
--cuda-graph-backend-prefill breakable \
|
||||||
|
--cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \
|
||||||
|
--cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \
|
||||||
|
--enable-hierarchical-cache --hicache-ratio 4 \
|
||||||
|
--hicache-mem-layout page_first_direct --hicache-io-backend direct \
|
||||||
|
--hicache-write-policy write_through --hicache-storage-backend mooncake \
|
||||||
|
--hicache-storage-prefetch-policy wait_complete \
|
||||||
|
--disaggregation-mode prefill --disaggregation-transfer-backend mooncake \
|
||||||
|
--disaggregation-bootstrap-port 8998 \
|
||||||
|
--expert-balancedness-report-mode prometheus \
|
||||||
|
--expert-distribution-recorder-buffer-size 64 \
|
||||||
|
--host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \
|
||||||
|
--enable-trace --trace-modules request,mooncake \
|
||||||
|
--otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \
|
||||||
|
--watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast
|
||||||
|
env:
|
||||||
|
- name: CUDA_VISIBLE_DEVICES
|
||||||
|
value: 0,1,2,3
|
||||||
|
- name: SGLANG_LOG_MS
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK
|
||||||
|
value: "65536"
|
||||||
|
- name: SGLANG_DSV4_KV_LAYOUT
|
||||||
|
value: v41_new
|
||||||
|
- name: SGLANG_DP_USE_GATHERV
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||||
|
value: static
|
||||||
|
- name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION
|
||||||
|
value: "0"
|
||||||
|
- name: SGLANG_NUMA_BIND_V2
|
||||||
|
value: "0"
|
||||||
|
- name: SGLANG_DEFAULT_THINKING
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||||
|
value: high
|
||||||
|
- name: SGLANG_TIMEOUT_KEEP_ALIVE
|
||||||
|
value: "900"
|
||||||
|
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||||
|
value: "1"
|
||||||
|
- name: MC_INTRANODE_NVLINK
|
||||||
|
value: "true"
|
||||||
|
- name: MC_INTRA_NVLINK
|
||||||
|
value: "true"
|
||||||
|
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||||
|
value: "true"
|
||||||
|
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||||
|
value: /data/ymk/ds41/mooncake-store-l3ssd.json
|
||||||
|
- name: SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||||
|
value: "4"
|
||||||
|
- name: SGLANG_HICACHE_BACKUP_IO_THREADS
|
||||||
|
value: "4"
|
||||||
|
- name: SGLANG_HICACHE_TIMING
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_BATCH_TELEMETRY
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DEVICE_TIMER
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL
|
||||||
|
value: "100"
|
||||||
|
- name: SGLANG_MM_TRACE
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_BATCH_SPAN_MIN_MS
|
||||||
|
value: "200"
|
||||||
|
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||||
|
value: "0.05"
|
||||||
|
- name: SGLANG_BATCH_SPAN_MAX_LINKS
|
||||||
|
value: "8"
|
||||||
|
- name: SGLANG_BATCH_STAGE_TIMING
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_BATCH_PASS_SPAN
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_BATCH_PASS_SPAN_SAMPLE
|
||||||
|
value: "1.0"
|
||||||
|
- name: SGLANG_RECV_SPAN
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_RECV_SPAN_SAMPLE
|
||||||
|
value: "1.0"
|
||||||
|
- name: SGLANG_MM_TRACE_SYNC
|
||||||
|
value: "0"
|
||||||
|
- name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||||
|
value: "600"
|
||||||
|
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||||
|
value: "16"
|
||||||
|
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||||
|
value: "32"
|
||||||
|
- name: TZ
|
||||||
|
value: Asia/Shanghai
|
||||||
|
- name: SGLANG_TRACE_LOOP_SPAN
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_TRACE_LOOP_SAMPLE
|
||||||
|
value: "1.0"
|
||||||
|
- name: SGLANG_FWD_TIMELINE
|
||||||
|
value: "0"
|
||||||
|
- name: SGLANG_TRACE_LOOP_DETAIL
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_TRACE_MOONCAKE_DETAIL
|
||||||
|
value: "0"
|
||||||
|
- name: SGLANG_TRACE_BACKUP_DETAIL
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_ASYNC_BACKUP_SUBMIT
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH
|
||||||
|
value: "1"
|
||||||
|
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||||
|
value: "1024"
|
||||||
|
- name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE
|
||||||
|
value: "9400"
|
||||||
|
- name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR
|
||||||
|
value: bucket_storage_backend
|
||||||
|
- name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE
|
||||||
|
value: "1099511627776"
|
||||||
|
- name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES
|
||||||
|
value: "1099511627776"
|
||||||
|
- name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY
|
||||||
|
value: fifo
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/p-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/p-xdg
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /data/ymk/l3ssd/p-hf
|
||||||
|
- name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS
|
||||||
|
value: "0"
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: sglang
|
||||||
|
ports:
|
||||||
|
- containerPort: 30000
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 8998
|
||||||
|
name: bootstrap
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 9400
|
||||||
|
name: mc-0
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 9401
|
||||||
|
name: mc-1
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 9402
|
||||||
|
name: mc-2
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 9403
|
||||||
|
name: mc-3
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 120
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: metrics
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: "96"
|
||||||
|
memory: 2Ti
|
||||||
|
nvidia.com/gpu: "4"
|
||||||
|
requests:
|
||||||
|
cpu: "32"
|
||||||
|
memory: 512Gi
|
||||||
|
nvidia.com/gpu: "4"
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /root/.cache/sglang
|
||||||
|
name: sglang-cache
|
||||||
|
- mountPath: /dev/shm
|
||||||
|
name: dshm
|
||||||
|
- mountPath: /root/.cargo/bin/cargo
|
||||||
|
name: fake-cargo
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/cache/sglang
|
||||||
|
type: Directory
|
||||||
|
name: sglang-cache
|
||||||
|
- emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 64Gi
|
||||||
|
name: dshm
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/ds41/fake-cargo
|
||||||
|
type: File
|
||||||
|
name: fake-cargo
|
||||||
|
- dependencies:
|
||||||
|
- mooncake-master
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: d
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: "30300"
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-d
|
||||||
|
kongming.io/history-revision: hrev-b90013fa6cc5a2a0e6515115d188135ea41e48b7
|
||||||
|
kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- |
|
||||||
|
exec sglang serve \
|
||||||
|
--trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \
|
||||||
|
--reasoning-parser auto --tool-call-parser auto \
|
||||||
|
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||||
|
--tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \
|
||||||
|
--enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \
|
||||||
|
--moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \
|
||||||
|
--ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \
|
||||||
|
--init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \
|
||||||
|
--mem-fraction-static 0.80 \
|
||||||
|
--speculative-algorithm DSPARK --speculative-dspark-block-size 3 \
|
||||||
|
--stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \
|
||||||
|
--max-running-requests 512 --cuda-graph-max-bs-decode 512 --tokenizer-worker-num 16 \
|
||||||
|
--chunked-prefill-size 32768 --max-prefill-tokens 8192 \
|
||||||
|
--disaggregation-mode decode --disaggregation-transfer-backend mooncake \
|
||||||
|
--host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \
|
||||||
|
--enable-trace --trace-modules request,mooncake \
|
||||||
|
--otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \
|
||||||
|
--watchdog-timeout 3600
|
||||||
|
env:
|
||||||
|
- name: CUDA_VISIBLE_DEVICES
|
||||||
|
value: 4,5,6,7
|
||||||
|
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||||
|
value: static
|
||||||
|
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_DSV4_KV_LAYOUT
|
||||||
|
value: v41_new
|
||||||
|
- name: SGLANG_DEFAULT_THINKING
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||||
|
value: high
|
||||||
|
- name: MC_INTRANODE_NVLINK
|
||||||
|
value: "true"
|
||||||
|
- name: MC_INTRA_NVLINK
|
||||||
|
value: "true"
|
||||||
|
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||||
|
value: "true"
|
||||||
|
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_BATCH_TELEMETRY
|
||||||
|
value: "1"
|
||||||
|
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||||
|
value: "0.1"
|
||||||
|
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||||
|
value: "600"
|
||||||
|
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||||
|
value: "16"
|
||||||
|
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||||
|
value: "32"
|
||||||
|
- name: TZ
|
||||||
|
value: Asia/Shanghai
|
||||||
|
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||||
|
value: "1024"
|
||||||
|
- name: TMPDIR
|
||||||
|
value: /data/ymk/l3ssd/d-tmp
|
||||||
|
- name: XDG_CACHE_HOME
|
||||||
|
value: /data/ymk/l3ssd/d-xdg
|
||||||
|
- name: HF_HOME
|
||||||
|
value: /data/ymk/l3ssd/d-hf
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: sglang
|
||||||
|
ports:
|
||||||
|
- containerPort: 30300
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 150
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: metrics
|
||||||
|
initialDelaySeconds: 60
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: "96"
|
||||||
|
memory: 550Gi
|
||||||
|
nvidia.com/gpu: "4"
|
||||||
|
requests:
|
||||||
|
cpu: "32"
|
||||||
|
memory: 256Gi
|
||||||
|
nvidia.com/gpu: "4"
|
||||||
|
securityContext:
|
||||||
|
privileged: true
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /root/.cache/sglang
|
||||||
|
name: sglang-cache
|
||||||
|
- mountPath: /dev/shm
|
||||||
|
name: dshm
|
||||||
|
- mountPath: /root/.cargo/bin/cargo
|
||||||
|
name: fake-cargo
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
hostPID: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/cache/sglang
|
||||||
|
type: Directory
|
||||||
|
name: sglang-cache
|
||||||
|
- emptyDir:
|
||||||
|
medium: Memory
|
||||||
|
sizeLimit: 64Gi
|
||||||
|
name: dshm
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/ds41/fake-cargo
|
||||||
|
type: File
|
||||||
|
name: fake-cargo
|
||||||
|
- dependencies:
|
||||||
|
- p
|
||||||
|
- d
|
||||||
|
minReadySeconds: 0
|
||||||
|
name: gateway
|
||||||
|
podManagementPolicy: Parallel
|
||||||
|
replicas: 1
|
||||||
|
standalonePattern:
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
annotations:
|
||||||
|
prometheus.io/path: /metrics
|
||||||
|
prometheus.io/port: "29000"
|
||||||
|
prometheus.io/scheme: http
|
||||||
|
prometheus.io/scrape: "true"
|
||||||
|
labels:
|
||||||
|
app: ds41-l3ssd-gw
|
||||||
|
kongming.io/history-revision: hrev-b90013fa6cc5a2a0e6515115d188135ea41e48b7
|
||||||
|
kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7
|
||||||
|
kongming.io/managed-by: kongming
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0
|
||||||
|
kongming.io/resource-type: custom-workload
|
||||||
|
kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- command:
|
||||||
|
- /bin/bash
|
||||||
|
- -c
|
||||||
|
- |
|
||||||
|
exec /opt/sglang/bin/sgl-model-gateway launch \
|
||||||
|
--pd-disaggregation \
|
||||||
|
--prefill http://127.0.0.1:30000 8998 \
|
||||||
|
--decode http://127.0.0.1:30300 \
|
||||||
|
--prefill-policy cache_aware --decode-policy cache_aware \
|
||||||
|
--g9i-enabled --g9i-max-new-tokens 4096 --g9i-est-td-enabled --g9i-shadow-enabled --g9i-td-admit off --g9i-p-queue-reqs-cap 0 --g9i-p-queue-token-cap 0 --g9i-p-qps 0 --g9i-p-refresh-ms 250 --g9i-window-fixed 256 \
|
||||||
|
\
|
||||||
|
\
|
||||||
|
--g9i-regular-p-queue-reject-secs 0 \
|
||||||
|
--g9i-decode-max-running 1000000 \
|
||||||
|
--pool-idle-timeout-secs 2 --pool-max-idle-per-host 0 \
|
||||||
|
--health-check-timeout-secs 30 --health-check-interval-secs 30 \
|
||||||
|
--cb-failure-threshold 50 --disable-circuit-breaker --disable-health-check \
|
||||||
|
--model-path /data/models/DeepSeek-V4.1-Flash-1017 \
|
||||||
|
--tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \
|
||||||
|
--host 0.0.0.0 --port 30400
|
||||||
|
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
name: gateway
|
||||||
|
ports:
|
||||||
|
- containerPort: 30400
|
||||||
|
name: http
|
||||||
|
protocol: TCP
|
||||||
|
- containerPort: 29000
|
||||||
|
name: metrics
|
||||||
|
protocol: TCP
|
||||||
|
readinessProbe:
|
||||||
|
failureThreshold: 60
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: http
|
||||||
|
initialDelaySeconds: 20
|
||||||
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 5
|
||||||
|
resources:
|
||||||
|
limits:
|
||||||
|
cpu: "16"
|
||||||
|
memory: 32Gi
|
||||||
|
requests:
|
||||||
|
cpu: "2"
|
||||||
|
memory: 8Gi
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /data
|
||||||
|
name: data
|
||||||
|
- mountPath: /opt/sglang/bin/sgl-model-gateway
|
||||||
|
name: gw-bin
|
||||||
|
readOnly: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
hostNetwork: true
|
||||||
|
nodeSelector:
|
||||||
|
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||||
|
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||||
|
kubernetes.io/hostname: b300-01
|
||||||
|
volumes:
|
||||||
|
- hostPath:
|
||||||
|
path: /data
|
||||||
|
type: Directory
|
||||||
|
name: data
|
||||||
|
- hostPath:
|
||||||
|
path: /data/ymk/gw-ns-est-td/sgl-model-gateway
|
||||||
|
type: File
|
||||||
|
name: gw-bin
|
||||||
|
status:
|
||||||
|
conditions:
|
||||||
|
- lastTransitionTime: "2026-11-17T01:58:12Z"
|
||||||
|
message: All roles are ready
|
||||||
|
observedGeneration: 6
|
||||||
|
reason: AllRolesReady
|
||||||
|
status: "True"
|
||||||
|
type: Ready
|
||||||
|
observedGeneration: 6
|
||||||
|
roleStatuses:
|
||||||
|
- name: mooncake-master
|
||||||
|
readyReplicas: 1
|
||||||
|
replicas: 1
|
||||||
|
updatedReplicas: 1
|
||||||
|
- name: p
|
||||||
|
readyReplicas: 1
|
||||||
|
replicas: 1
|
||||||
|
updatedReplicas: 1
|
||||||
|
- name: d
|
||||||
|
readyReplicas: 1
|
||||||
|
replicas: 1
|
||||||
|
updatedReplicas: 1
|
||||||
|
- name: gateway
|
||||||
|
readyReplicas: 1
|
||||||
|
replicas: 1
|
||||||
|
updatedReplicas: 1
|
||||||
Reference in New Issue
Block a user