deploy: record 11/16 pre-optimistic stable config
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
# 11/16 上线前稳定配置(ds41-gist-exp)
|
||||
|
||||
记录时间:2026-11-16 21:49 CST(13:49Z),在切换 `buffer_only + optimistic` 之前。
|
||||
|
||||
## 身份
|
||||
|
||||
- 平台 cw:`ds41-gist-exp` / `cw-dblft6sgrhagmpdt8ef0`
|
||||
- RBG:`ds41-gist-exp`(namespace `prj-d7te67bj0sq2fgdrmbig`)
|
||||
- 角色:`mooncake-master` / `p` / `d` / `gateway`
|
||||
- 镜像:`sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115`
|
||||
- P:GPU 0-3;D:GPU 4-7;全 hostNetwork / hostPID / privileged,钉 b300-01
|
||||
|
||||
## 文件
|
||||
|
||||
- `rbg-live-raw.yaml`:`kubectl get rbg -o yaml` 原始快照(含 status,仅存档)
|
||||
- `cw-ds41-gist-exp-11-16-pre-optimistic.yaml`:清理过平台字段的 RBG,可直接用于回滚更新
|
||||
- `cw-platform.yaml`:`kmctl get cw -o yaml` 的平台清单(`Spec.Manifest` 是旧值,见下方漂移)
|
||||
- 本说明
|
||||
|
||||
## 生效配置摘要
|
||||
|
||||
P:
|
||||
|
||||
- `tp4 / dp4 / ep4`、`megamoe`、`flashmla_mega`、`ep-redundant 32`
|
||||
- DSPARK block 3、decoder SWA bounded replay、`swa-full-tokens-ratio 0.12`
|
||||
- `chunked-prefill-size 131072`、`max-prefill-tokens 131072`
|
||||
- `max-running-requests 512`、`tokenizer-worker-num 16`
|
||||
- `hicache-ratio 4`、`hicache-mem-layout page_first_direct`、`hicache-io-backend direct`
|
||||
- `hicache-write-policy write_through`、`hicache-storage-backend mooncake`
|
||||
- **默认 host memory mode(cache),没有 `--hicache-host-memory-mode buffer_only`**
|
||||
- **没有 `--optimistic-prefill-attempts`**(`#optimistic-req` 为 0)
|
||||
- L3 SSD:4 TB `mooncake-ssd`,master `--offload_on_evict=false`
|
||||
|
||||
D:
|
||||
|
||||
- 同并行 tp4/dp4/ep4、`ep-redundant 64`、DSPARK block 3
|
||||
- `max-running-requests 512`、`cuda-graph-max-bs-decode 512`
|
||||
- `chunked-prefill-size 32768`、`max-prefill-tokens 8192`
|
||||
- **没有 `--tokenizer-worker-num`**(默认 1)
|
||||
- `mem-fraction-static 0.80`
|
||||
|
||||
Gateway:
|
||||
|
||||
- 实际 live:`--g9i-p-queue-reqs-cap 64 --g9i-p-queue-token-cap 32000000 --g9i-p-qps 40 --g9i-p-refresh-ms 250 --g9i-window-fixed 256`
|
||||
- `--g9i-regular-p-queue-reject-secs 0`、`--g9i-decode-max-running 512`
|
||||
- 网关二进制 hostPath:`/data/ymk/gw-ns-est-td/sgl-model-gateway`
|
||||
|
||||
> ⚠️ 平台 `cw-platform.yaml` 的 `Spec.Manifest` 里 gateway 仍是旧的
|
||||
> `p-queue-reqs-cap 32 / token-cap 16000000 / p-qps 20`;**live RBG 是 64 / 32M / 40**。
|
||||
> 本次快照与回滚文件都以 live RBG 为准,避免回滚时把网关参数打回旧值。
|
||||
|
||||
## 切换前状态
|
||||
|
||||
- 所有 Pod `1/1 Running`;P/D/gateway `/health` 200。
|
||||
- 为切 `buffer_only` 做 L3 SSD 清理前,`/data/ymk/mooncake-ssd` 占用 **2.3T**;清理后 4.0K,`/data` 7.5T available。
|
||||
- 启动基线(切换后复核):P `max_total_num_tokens=16,158,208` / 128 per-rank / `available_gpu_mem=23.00 GB`;
|
||||
D `58,859,520` / 128 per-rank / `available_gpu_mem=79.27 GB`。
|
||||
|
||||
## 回滚
|
||||
|
||||
```bash
|
||||
MSYS_NO_PATHCONV=1 wsl -e /home/ubuntu/.local/bin/kmctl stop cw ds41-gist-exp --wait --timeout 10m --request-timeout 180s
|
||||
ssh 24.125.93.200.207 'bash /data/ymk/deploy/b300-k8s/clean_l3ssd_cache.sh'
|
||||
MSYS_NO_PATHCONV=1 wsl -e /home/ubuntu/.local/bin/kmctl update cw ds41-gist-exp \
|
||||
-f /mnt/d/B300/deploy/b300-k8s/snapshots/20261116-pre-optimistic/cw-ds41-gist-exp-11-16-pre-optimistic.yaml
|
||||
```
|
||||
|
||||
回滚前确认 `kmctl get cw ds41-gist-exp` 为 `Terminated 0/0`、GPU 0-7 为 0 MiB。
|
||||
+488
@@ -0,0 +1,488 @@
|
||||
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||
kind: RoleBasedGroup
|
||||
metadata:
|
||||
annotations:
|
||||
rbg.workloads.x-k8s.io/discovery-config-mode: refine
|
||||
labels:
|
||||
app: ds41-gist-exp
|
||||
kongming.io/managed-by: kongming
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kongming.io/resource-type: custom-workload
|
||||
sglang-model: ds-v41-flash
|
||||
name: ds41-gist-exp
|
||||
namespace: prj-d7te67bj0sq2fgdrmbig
|
||||
spec:
|
||||
roles:
|
||||
- minReadySeconds: 0
|
||||
name: mooncake-master
|
||||
podManagementPolicy: Parallel
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: '19003'
|
||||
prometheus.io/scheme: http
|
||||
labels:
|
||||
app: ds41-l3ssd-mooncake
|
||||
kongming.io/managed-by: kongming
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kongming.io/resource-type: custom-workload
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n"
|
||||
env:
|
||||
- name: TMPDIR
|
||||
value: /data/ymk/l3ssd/master-tmp
|
||||
- name: XDG_CACHE_HOME
|
||||
value: /data/ymk/l3ssd/master-xdg
|
||||
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: mooncake-master
|
||||
ports:
|
||||
- containerPort: 19003
|
||||
name: metrics
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 60
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
tcpSocket:
|
||||
port: 50051
|
||||
resources:
|
||||
limits:
|
||||
cpu: '4'
|
||||
memory: 64Gi
|
||||
requests:
|
||||
cpu: '1'
|
||||
memory: 8Gi
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
nodeSelector:
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kubernetes.io/hostname: b300-01
|
||||
- dependencies:
|
||||
- mooncake-master
|
||||
minReadySeconds: 0
|
||||
name: p
|
||||
podManagementPolicy: Parallel
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: '30000'
|
||||
prometheus.io/scheme: http
|
||||
labels:
|
||||
app: ds41-l3ssd-p
|
||||
kongming.io/managed-by: kongming
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kongming.io/resource-type: custom-workload
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n"
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: 0,1,2,3
|
||||
- name: SGLANG_LOG_MS
|
||||
value: '1'
|
||||
- name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS
|
||||
value: '1'
|
||||
- name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL
|
||||
value: '1'
|
||||
- name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS
|
||||
value: '1'
|
||||
- name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK
|
||||
value: '65536'
|
||||
- name: SGLANG_DSV4_KV_LAYOUT
|
||||
value: v41_new
|
||||
- name: SGLANG_DP_USE_GATHERV
|
||||
value: '1'
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION
|
||||
value: '0'
|
||||
- name: SGLANG_NUMA_BIND_V2
|
||||
value: '0'
|
||||
- name: SGLANG_DEFAULT_THINKING
|
||||
value: '1'
|
||||
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||
value: high
|
||||
- name: SGLANG_TIMEOUT_KEEP_ALIVE
|
||||
value: '900'
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: '1'
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: 'true'
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: 'true'
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: 'true'
|
||||
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||
value: /data/ymk/ds41/mooncake-store-l3ssd.json
|
||||
- name: SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||
value: '4'
|
||||
- name: SGLANG_HICACHE_BACKUP_IO_THREADS
|
||||
value: '4'
|
||||
- name: SGLANG_HICACHE_TIMING
|
||||
value: '1'
|
||||
- name: SGLANG_BATCH_TELEMETRY
|
||||
value: '1'
|
||||
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||
value: '1'
|
||||
- name: SGLANG_ENABLE_METRICS_DEVICE_TIMER
|
||||
value: '1'
|
||||
- name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL
|
||||
value: '100'
|
||||
- name: SGLANG_MM_TRACE
|
||||
value: '1'
|
||||
- name: SGLANG_BATCH_SPAN_MIN_MS
|
||||
value: '200'
|
||||
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||
value: '0.05'
|
||||
- name: SGLANG_BATCH_SPAN_MAX_LINKS
|
||||
value: '8'
|
||||
- name: SGLANG_BATCH_STAGE_TIMING
|
||||
value: '1'
|
||||
- name: SGLANG_BATCH_PASS_SPAN
|
||||
value: '1'
|
||||
- name: SGLANG_BATCH_PASS_SPAN_SAMPLE
|
||||
value: '1.0'
|
||||
- name: SGLANG_RECV_SPAN
|
||||
value: '1'
|
||||
- name: SGLANG_RECV_SPAN_SAMPLE
|
||||
value: '1.0'
|
||||
- name: SGLANG_MM_TRACE_SYNC
|
||||
value: '0'
|
||||
- name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK
|
||||
value: '1'
|
||||
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||
value: '600'
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: '16'
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: '32'
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
- name: SGLANG_TRACE_LOOP_SPAN
|
||||
value: '1'
|
||||
- name: SGLANG_TRACE_LOOP_SAMPLE
|
||||
value: '1.0'
|
||||
- name: SGLANG_FWD_TIMELINE
|
||||
value: '0'
|
||||
- name: SGLANG_TRACE_LOOP_DETAIL
|
||||
value: '1'
|
||||
- name: SGLANG_TRACE_MOONCAKE_DETAIL
|
||||
value: '0'
|
||||
- name: SGLANG_TRACE_BACKUP_DETAIL
|
||||
value: '1'
|
||||
- name: SGLANG_ASYNC_BACKUP_SUBMIT
|
||||
value: '1'
|
||||
- name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH
|
||||
value: '1'
|
||||
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||
value: '1024'
|
||||
- name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE
|
||||
value: '9400'
|
||||
- name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR
|
||||
value: bucket_storage_backend
|
||||
- name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE
|
||||
value: '1099511627776'
|
||||
- name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES
|
||||
value: '1099511627776'
|
||||
- name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY
|
||||
value: fifo
|
||||
- name: TMPDIR
|
||||
value: /data/ymk/l3ssd/p-tmp
|
||||
- name: XDG_CACHE_HOME
|
||||
value: /data/ymk/l3ssd/p-xdg
|
||||
- name: HF_HOME
|
||||
value: /data/ymk/l3ssd/p-hf
|
||||
- name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS
|
||||
value: '0'
|
||||
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: sglang
|
||||
ports:
|
||||
- containerPort: 30000
|
||||
name: metrics
|
||||
protocol: TCP
|
||||
- containerPort: 8998
|
||||
name: bootstrap
|
||||
protocol: TCP
|
||||
- containerPort: 9400
|
||||
name: mc-0
|
||||
protocol: TCP
|
||||
- containerPort: 9401
|
||||
name: mc-1
|
||||
protocol: TCP
|
||||
- containerPort: 9402
|
||||
name: mc-2
|
||||
protocol: TCP
|
||||
- containerPort: 9403
|
||||
name: mc-3
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 120
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
limits:
|
||||
cpu: '96'
|
||||
memory: 2Ti
|
||||
nvidia.com/gpu: '4'
|
||||
requests:
|
||||
cpu: '32'
|
||||
memory: 512Gi
|
||||
nvidia.com/gpu: '4'
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- mountPath: /data
|
||||
name: data
|
||||
- mountPath: /root/.cache/sglang
|
||||
name: sglang-cache
|
||||
- mountPath: /dev/shm
|
||||
name: dshm
|
||||
- mountPath: /root/.cargo/bin/cargo
|
||||
name: fake-cargo
|
||||
readOnly: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
nodeSelector:
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kubernetes.io/hostname: b300-01
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
name: data
|
||||
- hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
name: sglang-cache
|
||||
- emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 64Gi
|
||||
name: dshm
|
||||
- hostPath:
|
||||
path: /data/ymk/ds41/fake-cargo
|
||||
type: File
|
||||
name: fake-cargo
|
||||
- dependencies:
|
||||
- mooncake-master
|
||||
minReadySeconds: 0
|
||||
name: d
|
||||
podManagementPolicy: Parallel
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: '30300'
|
||||
prometheus.io/scheme: http
|
||||
labels:
|
||||
app: ds41-l3ssd-d
|
||||
kongming.io/managed-by: kongming
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kongming.io/resource-type: custom-workload
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n"
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: 4,5,6,7
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: '1'
|
||||
- name: SGLANG_DSV4_KV_LAYOUT
|
||||
value: v41_new
|
||||
- name: SGLANG_DEFAULT_THINKING
|
||||
value: '1'
|
||||
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||
value: high
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: 'true'
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: 'true'
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: 'true'
|
||||
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||
value: '1'
|
||||
- name: SGLANG_BATCH_TELEMETRY
|
||||
value: '1'
|
||||
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||
value: '0.1'
|
||||
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||
value: '600'
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: '16'
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: '32'
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||
value: '1024'
|
||||
- name: TMPDIR
|
||||
value: /data/ymk/l3ssd/d-tmp
|
||||
- name: XDG_CACHE_HOME
|
||||
value: /data/ymk/l3ssd/d-xdg
|
||||
- name: HF_HOME
|
||||
value: /data/ymk/l3ssd/d-hf
|
||||
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: sglang
|
||||
ports:
|
||||
- containerPort: 30300
|
||||
name: metrics
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 150
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
limits:
|
||||
cpu: '96'
|
||||
memory: 550Gi
|
||||
nvidia.com/gpu: '4'
|
||||
requests:
|
||||
cpu: '32'
|
||||
memory: 256Gi
|
||||
nvidia.com/gpu: '4'
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- mountPath: /data
|
||||
name: data
|
||||
- mountPath: /root/.cache/sglang
|
||||
name: sglang-cache
|
||||
- mountPath: /dev/shm
|
||||
name: dshm
|
||||
- mountPath: /root/.cargo/bin/cargo
|
||||
name: fake-cargo
|
||||
readOnly: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
nodeSelector:
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kubernetes.io/hostname: b300-01
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
name: data
|
||||
- hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
name: sglang-cache
|
||||
- emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 64Gi
|
||||
name: dshm
|
||||
- hostPath:
|
||||
path: /data/ymk/ds41/fake-cargo
|
||||
type: File
|
||||
name: fake-cargo
|
||||
- dependencies:
|
||||
- p
|
||||
- d
|
||||
minReadySeconds: 0
|
||||
name: gateway
|
||||
podManagementPolicy: Parallel
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: '29000'
|
||||
prometheus.io/scheme: http
|
||||
prometheus.io/scrape: 'true'
|
||||
labels:
|
||||
app: ds41-l3ssd-gw
|
||||
kongming.io/managed-by: kongming
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kongming.io/resource-type: custom-workload
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 --g9i-est-td-enabled --g9i-shadow-enabled --g9i-td-admit enforce --g9i-p-queue-reqs-cap 64 --g9i-p-queue-token-cap 32000000 --g9i-p-qps 40 --g9i-p-refresh-ms 250 --g9i-window-fixed 256 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 512 \\\n --pool-idle-timeout-secs 2 --pool-max-idle-per-host 0 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n"
|
||||
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: gateway
|
||||
ports:
|
||||
- containerPort: 30400
|
||||
name: http
|
||||
protocol: TCP
|
||||
- containerPort: 29000
|
||||
name: metrics
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 60
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
limits:
|
||||
cpu: '16'
|
||||
memory: 32Gi
|
||||
requests:
|
||||
cpu: '2'
|
||||
memory: 8Gi
|
||||
volumeMounts:
|
||||
- mountPath: /data
|
||||
name: data
|
||||
- mountPath: /opt/sglang/bin/sgl-model-gateway
|
||||
name: gw-bin
|
||||
readOnly: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
nodeSelector:
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kubernetes.io/hostname: b300-01
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
name: data
|
||||
- hostPath:
|
||||
path: /data/ymk/gw-ns-est-td/sgl-model-gateway
|
||||
type: File
|
||||
name: gw-bin
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,608 @@
|
||||
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||
kind: RoleBasedGroup
|
||||
metadata:
|
||||
annotations:
|
||||
rbg.workloads.x-k8s.io/discovery-config-mode: refine
|
||||
creationTimestamp: "2026-11-16T00:16:46Z"
|
||||
generation: 9
|
||||
labels:
|
||||
app: ds41-gist-exp
|
||||
kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571
|
||||
kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7
|
||||
kongming.io/managed-by: kongming
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0
|
||||
kongming.io/resource-type: custom-workload
|
||||
kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0
|
||||
sglang-model: ds-v41-flash
|
||||
name: ds41-gist-exp
|
||||
namespace: prj-d7te67bj0sq2fgdrmbig
|
||||
resourceVersion: "30735810"
|
||||
uid: cc94a410-abf2-41ec-b711-52f7c87598e1
|
||||
spec:
|
||||
roles:
|
||||
- minReadySeconds: 0
|
||||
name: mooncake-master
|
||||
podManagementPolicy: Parallel
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "19003"
|
||||
prometheus.io/scheme: http
|
||||
labels:
|
||||
app: ds41-l3ssd-mooncake
|
||||
kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571
|
||||
kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7
|
||||
kongming.io/managed-by: kongming
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0
|
||||
kongming.io/resource-type: custom-workload
|
||||
kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;
|
||||
# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。
|
||||
exec /opt/sglang/bin/mooncake_master \
|
||||
--port 50051 --enable_http_metadata_server=true \
|
||||
--http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false
|
||||
env:
|
||||
- name: TMPDIR
|
||||
value: /data/ymk/l3ssd/master-tmp
|
||||
- name: XDG_CACHE_HOME
|
||||
value: /data/ymk/l3ssd/master-xdg
|
||||
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: mooncake-master
|
||||
ports:
|
||||
- containerPort: 19003
|
||||
name: metrics
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 60
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
tcpSocket:
|
||||
port: 50051
|
||||
resources:
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: 64Gi
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 8Gi
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
nodeSelector:
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kubernetes.io/hostname: b300-01
|
||||
- dependencies:
|
||||
- mooncake-master
|
||||
minReadySeconds: 0
|
||||
name: p
|
||||
podManagementPolicy: Parallel
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "30000"
|
||||
prometheus.io/scheme: http
|
||||
labels:
|
||||
app: ds41-l3ssd-p
|
||||
kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571
|
||||
kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7
|
||||
kongming.io/managed-by: kongming
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0
|
||||
kongming.io/resource-type: custom-workload
|
||||
kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \
|
||||
--moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \
|
||||
--ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \
|
||||
--init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \
|
||||
--mem-fraction-static 0.80 \
|
||||
--chunked-prefill-size 131072 --max-prefill-tokens 131072 \
|
||||
--max-running-requests 512 --tokenizer-worker-num 16 \
|
||||
--load-balance-method total_tokens \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 3 \
|
||||
--enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \
|
||||
--cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \
|
||||
--cuda-graph-backend-prefill breakable \
|
||||
--cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \
|
||||
--cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \
|
||||
--enable-hierarchical-cache --hicache-ratio 4 \
|
||||
--hicache-mem-layout page_first_direct --hicache-io-backend direct \
|
||||
--hicache-write-policy write_through --hicache-storage-backend mooncake \
|
||||
--hicache-storage-prefetch-policy wait_complete \
|
||||
--disaggregation-mode prefill --disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port 8998 \
|
||||
--expert-balancedness-report-mode prometheus \
|
||||
--expert-distribution-recorder-buffer-size 64 \
|
||||
--host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \
|
||||
--enable-trace --trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \
|
||||
--watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: 0,1,2,3
|
||||
- name: SGLANG_LOG_MS
|
||||
value: "1"
|
||||
- name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS
|
||||
value: "1"
|
||||
- name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL
|
||||
value: "1"
|
||||
- name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS
|
||||
value: "1"
|
||||
- name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK
|
||||
value: "65536"
|
||||
- name: SGLANG_DSV4_KV_LAYOUT
|
||||
value: v41_new
|
||||
- name: SGLANG_DP_USE_GATHERV
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION
|
||||
value: "0"
|
||||
- name: SGLANG_NUMA_BIND_V2
|
||||
value: "0"
|
||||
- name: SGLANG_DEFAULT_THINKING
|
||||
value: "1"
|
||||
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||
value: high
|
||||
- name: SGLANG_TIMEOUT_KEEP_ALIVE
|
||||
value: "900"
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "true"
|
||||
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||
value: /data/ymk/ds41/mooncake-store-l3ssd.json
|
||||
- name: SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||
value: "4"
|
||||
- name: SGLANG_HICACHE_BACKUP_IO_THREADS
|
||||
value: "4"
|
||||
- name: SGLANG_HICACHE_TIMING
|
||||
value: "1"
|
||||
- name: SGLANG_BATCH_TELEMETRY
|
||||
value: "1"
|
||||
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||
value: "1"
|
||||
- name: SGLANG_ENABLE_METRICS_DEVICE_TIMER
|
||||
value: "1"
|
||||
- name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL
|
||||
value: "100"
|
||||
- name: SGLANG_MM_TRACE
|
||||
value: "1"
|
||||
- name: SGLANG_BATCH_SPAN_MIN_MS
|
||||
value: "200"
|
||||
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||
value: "0.05"
|
||||
- name: SGLANG_BATCH_SPAN_MAX_LINKS
|
||||
value: "8"
|
||||
- name: SGLANG_BATCH_STAGE_TIMING
|
||||
value: "1"
|
||||
- name: SGLANG_BATCH_PASS_SPAN
|
||||
value: "1"
|
||||
- name: SGLANG_BATCH_PASS_SPAN_SAMPLE
|
||||
value: "1.0"
|
||||
- name: SGLANG_RECV_SPAN
|
||||
value: "1"
|
||||
- name: SGLANG_RECV_SPAN_SAMPLE
|
||||
value: "1.0"
|
||||
- name: SGLANG_MM_TRACE_SYNC
|
||||
value: "0"
|
||||
- name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK
|
||||
value: "1"
|
||||
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||
value: "600"
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
- name: SGLANG_TRACE_LOOP_SPAN
|
||||
value: "1"
|
||||
- name: SGLANG_TRACE_LOOP_SAMPLE
|
||||
value: "1.0"
|
||||
- name: SGLANG_FWD_TIMELINE
|
||||
value: "0"
|
||||
- name: SGLANG_TRACE_LOOP_DETAIL
|
||||
value: "1"
|
||||
- name: SGLANG_TRACE_MOONCAKE_DETAIL
|
||||
value: "0"
|
||||
- name: SGLANG_TRACE_BACKUP_DETAIL
|
||||
value: "1"
|
||||
- name: SGLANG_ASYNC_BACKUP_SUBMIT
|
||||
value: "1"
|
||||
- name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH
|
||||
value: "1"
|
||||
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||
value: "1024"
|
||||
- name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE
|
||||
value: "9400"
|
||||
- name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR
|
||||
value: bucket_storage_backend
|
||||
- name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE
|
||||
value: "1099511627776"
|
||||
- name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES
|
||||
value: "1099511627776"
|
||||
- name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY
|
||||
value: fifo
|
||||
- name: TMPDIR
|
||||
value: /data/ymk/l3ssd/p-tmp
|
||||
- name: XDG_CACHE_HOME
|
||||
value: /data/ymk/l3ssd/p-xdg
|
||||
- name: HF_HOME
|
||||
value: /data/ymk/l3ssd/p-hf
|
||||
- name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS
|
||||
value: "0"
|
||||
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: sglang
|
||||
ports:
|
||||
- containerPort: 30000
|
||||
name: metrics
|
||||
protocol: TCP
|
||||
- containerPort: 8998
|
||||
name: bootstrap
|
||||
protocol: TCP
|
||||
- containerPort: 9400
|
||||
name: mc-0
|
||||
protocol: TCP
|
||||
- containerPort: 9401
|
||||
name: mc-1
|
||||
protocol: TCP
|
||||
- containerPort: 9402
|
||||
name: mc-2
|
||||
protocol: TCP
|
||||
- containerPort: 9403
|
||||
name: mc-3
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 120
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 2Ti
|
||||
nvidia.com/gpu: "4"
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 512Gi
|
||||
nvidia.com/gpu: "4"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- mountPath: /data
|
||||
name: data
|
||||
- mountPath: /root/.cache/sglang
|
||||
name: sglang-cache
|
||||
- mountPath: /dev/shm
|
||||
name: dshm
|
||||
- mountPath: /root/.cargo/bin/cargo
|
||||
name: fake-cargo
|
||||
readOnly: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
nodeSelector:
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kubernetes.io/hostname: b300-01
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
name: data
|
||||
- hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
name: sglang-cache
|
||||
- emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 64Gi
|
||||
name: dshm
|
||||
- hostPath:
|
||||
path: /data/ymk/ds41/fake-cargo
|
||||
type: File
|
||||
name: fake-cargo
|
||||
- dependencies:
|
||||
- mooncake-master
|
||||
minReadySeconds: 0
|
||||
name: d
|
||||
podManagementPolicy: Parallel
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "30300"
|
||||
prometheus.io/scheme: http
|
||||
labels:
|
||||
app: ds41-l3ssd-d
|
||||
kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571
|
||||
kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7
|
||||
kongming.io/managed-by: kongming
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0
|
||||
kongming.io/resource-type: custom-workload
|
||||
kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \
|
||||
--enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \
|
||||
--moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \
|
||||
--ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \
|
||||
--init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \
|
||||
--mem-fraction-static 0.80 \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 3 \
|
||||
--stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \
|
||||
--max-running-requests 512 --cuda-graph-max-bs-decode 512 \
|
||||
--chunked-prefill-size 32768 --max-prefill-tokens 8192 \
|
||||
--disaggregation-mode decode --disaggregation-transfer-backend mooncake \
|
||||
--host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \
|
||||
--enable-trace --trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \
|
||||
--watchdog-timeout 3600
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: 4,5,6,7
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_DSV4_KV_LAYOUT
|
||||
value: v41_new
|
||||
- name: SGLANG_DEFAULT_THINKING
|
||||
value: "1"
|
||||
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||
value: high
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "true"
|
||||
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||
value: "1"
|
||||
- name: SGLANG_BATCH_TELEMETRY
|
||||
value: "1"
|
||||
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||
value: "0.1"
|
||||
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||
value: "600"
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||
value: "1024"
|
||||
- name: TMPDIR
|
||||
value: /data/ymk/l3ssd/d-tmp
|
||||
- name: XDG_CACHE_HOME
|
||||
value: /data/ymk/l3ssd/d-xdg
|
||||
- name: HF_HOME
|
||||
value: /data/ymk/l3ssd/d-hf
|
||||
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: sglang
|
||||
ports:
|
||||
- containerPort: 30300
|
||||
name: metrics
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 150
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 550Gi
|
||||
nvidia.com/gpu: "4"
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 256Gi
|
||||
nvidia.com/gpu: "4"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- mountPath: /data
|
||||
name: data
|
||||
- mountPath: /root/.cache/sglang
|
||||
name: sglang-cache
|
||||
- mountPath: /dev/shm
|
||||
name: dshm
|
||||
- mountPath: /root/.cargo/bin/cargo
|
||||
name: fake-cargo
|
||||
readOnly: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
nodeSelector:
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kubernetes.io/hostname: b300-01
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
name: data
|
||||
- hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
name: sglang-cache
|
||||
- emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 64Gi
|
||||
name: dshm
|
||||
- hostPath:
|
||||
path: /data/ymk/ds41/fake-cargo
|
||||
type: File
|
||||
name: fake-cargo
|
||||
- dependencies:
|
||||
- p
|
||||
- d
|
||||
minReadySeconds: 0
|
||||
name: gateway
|
||||
podManagementPolicy: Parallel
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "29000"
|
||||
prometheus.io/scheme: http
|
||||
prometheus.io/scrape: "true"
|
||||
labels:
|
||||
app: ds41-l3ssd-gw
|
||||
kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571
|
||||
kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7
|
||||
kongming.io/managed-by: kongming
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0
|
||||
kongming.io/resource-type: custom-workload
|
||||
kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec /opt/sglang/bin/sgl-model-gateway launch \
|
||||
--pd-disaggregation \
|
||||
--prefill http://127.0.0.1:30000 8998 \
|
||||
--decode http://127.0.0.1:30300 \
|
||||
--prefill-policy cache_aware --decode-policy cache_aware \
|
||||
--g9i-enabled --g9i-max-new-tokens 4096 --g9i-est-td-enabled --g9i-shadow-enabled --g9i-td-admit enforce --g9i-p-queue-reqs-cap 64 --g9i-p-queue-token-cap 32000000 --g9i-p-qps 40 --g9i-p-refresh-ms 250 --g9i-window-fixed 256 \
|
||||
\
|
||||
\
|
||||
--g9i-regular-p-queue-reject-secs 0 \
|
||||
--g9i-decode-max-running 512 \
|
||||
--pool-idle-timeout-secs 2 --pool-max-idle-per-host 0 \
|
||||
--health-check-timeout-secs 30 --health-check-interval-secs 30 \
|
||||
--cb-failure-threshold 50 \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash-1017 \
|
||||
--tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \
|
||||
--host 0.0.0.0 --port 30400
|
||||
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: gateway
|
||||
ports:
|
||||
- containerPort: 30400
|
||||
name: http
|
||||
protocol: TCP
|
||||
- containerPort: 29000
|
||||
name: metrics
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 60
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
limits:
|
||||
cpu: "16"
|
||||
memory: 32Gi
|
||||
requests:
|
||||
cpu: "2"
|
||||
memory: 8Gi
|
||||
volumeMounts:
|
||||
- mountPath: /data
|
||||
name: data
|
||||
- mountPath: /opt/sglang/bin/sgl-model-gateway
|
||||
name: gw-bin
|
||||
readOnly: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
nodeSelector:
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kubernetes.io/hostname: b300-01
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
name: data
|
||||
- hostPath:
|
||||
path: /data/ymk/gw-ns-est-td/sgl-model-gateway
|
||||
type: File
|
||||
name: gw-bin
|
||||
status:
|
||||
conditions:
|
||||
- lastTransitionTime: "2026-11-16T12:36:12Z"
|
||||
message: All roles are ready
|
||||
observedGeneration: 9
|
||||
reason: AllRolesReady
|
||||
status: "True"
|
||||
type: Ready
|
||||
observedGeneration: 9
|
||||
roleStatuses:
|
||||
- name: mooncake-master
|
||||
readyReplicas: 1
|
||||
replicas: 1
|
||||
updatedReplicas: 1
|
||||
- name: p
|
||||
readyReplicas: 1
|
||||
replicas: 1
|
||||
updatedReplicas: 1
|
||||
- name: d
|
||||
readyReplicas: 1
|
||||
replicas: 1
|
||||
updatedReplicas: 1
|
||||
- name: gateway
|
||||
readyReplicas: 1
|
||||
replicas: 1
|
||||
updatedReplicas: 1
|
||||
@@ -0,0 +1,608 @@
|
||||
apiVersion: workloads.x-k8s.io/v1alpha2
|
||||
kind: RoleBasedGroup
|
||||
metadata:
|
||||
annotations:
|
||||
rbg.workloads.x-k8s.io/discovery-config-mode: refine
|
||||
creationTimestamp: "2026-11-16T00:16:46Z"
|
||||
generation: 9
|
||||
labels:
|
||||
app: ds41-gist-exp
|
||||
kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571
|
||||
kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7
|
||||
kongming.io/managed-by: kongming
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0
|
||||
kongming.io/resource-type: custom-workload
|
||||
kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0
|
||||
sglang-model: ds-v41-flash
|
||||
name: ds41-gist-exp
|
||||
namespace: prj-d7te67bj0sq2fgdrmbig
|
||||
resourceVersion: "30735810"
|
||||
uid: cc94a410-abf2-41ec-b711-52f7c87598e1
|
||||
spec:
|
||||
roles:
|
||||
- minReadySeconds: 0
|
||||
name: mooncake-master
|
||||
podManagementPolicy: Parallel
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "19003"
|
||||
prometheus.io/scheme: http
|
||||
labels:
|
||||
app: ds41-l3ssd-mooncake
|
||||
kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571
|
||||
kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7
|
||||
kongming.io/managed-by: kongming
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0
|
||||
kongming.io/resource-type: custom-workload
|
||||
kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;
|
||||
# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。
|
||||
exec /opt/sglang/bin/mooncake_master \
|
||||
--port 50051 --enable_http_metadata_server=true \
|
||||
--http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false
|
||||
env:
|
||||
- name: TMPDIR
|
||||
value: /data/ymk/l3ssd/master-tmp
|
||||
- name: XDG_CACHE_HOME
|
||||
value: /data/ymk/l3ssd/master-xdg
|
||||
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: mooncake-master
|
||||
ports:
|
||||
- containerPort: 19003
|
||||
name: metrics
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 60
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 5
|
||||
tcpSocket:
|
||||
port: 50051
|
||||
resources:
|
||||
limits:
|
||||
cpu: "4"
|
||||
memory: 64Gi
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 8Gi
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
nodeSelector:
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kubernetes.io/hostname: b300-01
|
||||
- dependencies:
|
||||
- mooncake-master
|
||||
minReadySeconds: 0
|
||||
name: p
|
||||
podManagementPolicy: Parallel
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "30000"
|
||||
prometheus.io/scheme: http
|
||||
labels:
|
||||
app: ds41-l3ssd-p
|
||||
kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571
|
||||
kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7
|
||||
kongming.io/managed-by: kongming
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0
|
||||
kongming.io/resource-type: custom-workload
|
||||
kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \
|
||||
--moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \
|
||||
--ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \
|
||||
--init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \
|
||||
--mem-fraction-static 0.80 \
|
||||
--chunked-prefill-size 131072 --max-prefill-tokens 131072 \
|
||||
--max-running-requests 512 --tokenizer-worker-num 16 \
|
||||
--load-balance-method total_tokens \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 3 \
|
||||
--enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \
|
||||
--cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \
|
||||
--cuda-graph-backend-prefill breakable \
|
||||
--cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \
|
||||
--cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \
|
||||
--enable-hierarchical-cache --hicache-ratio 4 \
|
||||
--hicache-mem-layout page_first_direct --hicache-io-backend direct \
|
||||
--hicache-write-policy write_through --hicache-storage-backend mooncake \
|
||||
--hicache-storage-prefetch-policy wait_complete \
|
||||
--disaggregation-mode prefill --disaggregation-transfer-backend mooncake \
|
||||
--disaggregation-bootstrap-port 8998 \
|
||||
--expert-balancedness-report-mode prometheus \
|
||||
--expert-distribution-recorder-buffer-size 64 \
|
||||
--host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \
|
||||
--enable-trace --trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \
|
||||
--watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: 0,1,2,3
|
||||
- name: SGLANG_LOG_MS
|
||||
value: "1"
|
||||
- name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS
|
||||
value: "1"
|
||||
- name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL
|
||||
value: "1"
|
||||
- name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS
|
||||
value: "1"
|
||||
- name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK
|
||||
value: "65536"
|
||||
- name: SGLANG_DSV4_KV_LAYOUT
|
||||
value: v41_new
|
||||
- name: SGLANG_DP_USE_GATHERV
|
||||
value: "1"
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION
|
||||
value: "0"
|
||||
- name: SGLANG_NUMA_BIND_V2
|
||||
value: "0"
|
||||
- name: SGLANG_DEFAULT_THINKING
|
||||
value: "1"
|
||||
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||
value: high
|
||||
- name: SGLANG_TIMEOUT_KEEP_ALIVE
|
||||
value: "900"
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "true"
|
||||
- name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH
|
||||
value: /data/ymk/ds41/mooncake-store-l3ssd.json
|
||||
- name: SGLANG_HICACHE_PREFETCH_IO_THREADS
|
||||
value: "4"
|
||||
- name: SGLANG_HICACHE_BACKUP_IO_THREADS
|
||||
value: "4"
|
||||
- name: SGLANG_HICACHE_TIMING
|
||||
value: "1"
|
||||
- name: SGLANG_BATCH_TELEMETRY
|
||||
value: "1"
|
||||
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||
value: "1"
|
||||
- name: SGLANG_ENABLE_METRICS_DEVICE_TIMER
|
||||
value: "1"
|
||||
- name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL
|
||||
value: "100"
|
||||
- name: SGLANG_MM_TRACE
|
||||
value: "1"
|
||||
- name: SGLANG_BATCH_SPAN_MIN_MS
|
||||
value: "200"
|
||||
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||
value: "0.05"
|
||||
- name: SGLANG_BATCH_SPAN_MAX_LINKS
|
||||
value: "8"
|
||||
- name: SGLANG_BATCH_STAGE_TIMING
|
||||
value: "1"
|
||||
- name: SGLANG_BATCH_PASS_SPAN
|
||||
value: "1"
|
||||
- name: SGLANG_BATCH_PASS_SPAN_SAMPLE
|
||||
value: "1.0"
|
||||
- name: SGLANG_RECV_SPAN
|
||||
value: "1"
|
||||
- name: SGLANG_RECV_SPAN_SAMPLE
|
||||
value: "1.0"
|
||||
- name: SGLANG_MM_TRACE_SYNC
|
||||
value: "0"
|
||||
- name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK
|
||||
value: "1"
|
||||
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||
value: "600"
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
- name: SGLANG_TRACE_LOOP_SPAN
|
||||
value: "1"
|
||||
- name: SGLANG_TRACE_LOOP_SAMPLE
|
||||
value: "1.0"
|
||||
- name: SGLANG_FWD_TIMELINE
|
||||
value: "0"
|
||||
- name: SGLANG_TRACE_LOOP_DETAIL
|
||||
value: "1"
|
||||
- name: SGLANG_TRACE_MOONCAKE_DETAIL
|
||||
value: "0"
|
||||
- name: SGLANG_TRACE_BACKUP_DETAIL
|
||||
value: "1"
|
||||
- name: SGLANG_ASYNC_BACKUP_SUBMIT
|
||||
value: "1"
|
||||
- name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH
|
||||
value: "1"
|
||||
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||
value: "1024"
|
||||
- name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE
|
||||
value: "9400"
|
||||
- name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR
|
||||
value: bucket_storage_backend
|
||||
- name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE
|
||||
value: "1099511627776"
|
||||
- name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES
|
||||
value: "1099511627776"
|
||||
- name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY
|
||||
value: fifo
|
||||
- name: TMPDIR
|
||||
value: /data/ymk/l3ssd/p-tmp
|
||||
- name: XDG_CACHE_HOME
|
||||
value: /data/ymk/l3ssd/p-xdg
|
||||
- name: HF_HOME
|
||||
value: /data/ymk/l3ssd/p-hf
|
||||
- name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS
|
||||
value: "0"
|
||||
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: sglang
|
||||
ports:
|
||||
- containerPort: 30000
|
||||
name: metrics
|
||||
protocol: TCP
|
||||
- containerPort: 8998
|
||||
name: bootstrap
|
||||
protocol: TCP
|
||||
- containerPort: 9400
|
||||
name: mc-0
|
||||
protocol: TCP
|
||||
- containerPort: 9401
|
||||
name: mc-1
|
||||
protocol: TCP
|
||||
- containerPort: 9402
|
||||
name: mc-2
|
||||
protocol: TCP
|
||||
- containerPort: 9403
|
||||
name: mc-3
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 120
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 2Ti
|
||||
nvidia.com/gpu: "4"
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 512Gi
|
||||
nvidia.com/gpu: "4"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- mountPath: /data
|
||||
name: data
|
||||
- mountPath: /root/.cache/sglang
|
||||
name: sglang-cache
|
||||
- mountPath: /dev/shm
|
||||
name: dshm
|
||||
- mountPath: /root/.cargo/bin/cargo
|
||||
name: fake-cargo
|
||||
readOnly: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
nodeSelector:
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kubernetes.io/hostname: b300-01
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
name: data
|
||||
- hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
name: sglang-cache
|
||||
- emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 64Gi
|
||||
name: dshm
|
||||
- hostPath:
|
||||
path: /data/ymk/ds41/fake-cargo
|
||||
type: File
|
||||
name: fake-cargo
|
||||
- dependencies:
|
||||
- mooncake-master
|
||||
minReadySeconds: 0
|
||||
name: d
|
||||
podManagementPolicy: Parallel
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "30300"
|
||||
prometheus.io/scheme: http
|
||||
labels:
|
||||
app: ds41-l3ssd-d
|
||||
kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571
|
||||
kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7
|
||||
kongming.io/managed-by: kongming
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0
|
||||
kongming.io/resource-type: custom-workload
|
||||
kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec sglang serve \
|
||||
--trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \
|
||||
--reasoning-parser auto --tool-call-parser auto \
|
||||
--default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \
|
||||
--tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \
|
||||
--enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \
|
||||
--moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \
|
||||
--ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \
|
||||
--init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \
|
||||
--mem-fraction-static 0.80 \
|
||||
--speculative-algorithm DSPARK --speculative-dspark-block-size 3 \
|
||||
--stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \
|
||||
--max-running-requests 512 --cuda-graph-max-bs-decode 512 \
|
||||
--chunked-prefill-size 32768 --max-prefill-tokens 8192 \
|
||||
--disaggregation-mode decode --disaggregation-transfer-backend mooncake \
|
||||
--host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \
|
||||
--enable-trace --trace-modules request,mooncake \
|
||||
--otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \
|
||||
--watchdog-timeout 3600
|
||||
env:
|
||||
- name: CUDA_VISIBLE_DEVICES
|
||||
value: 4,5,6,7
|
||||
- name: SGLANG_RAGGED_VERIFY_MODE
|
||||
value: static
|
||||
- name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE
|
||||
value: "1"
|
||||
- name: SGLANG_DSV4_KV_LAYOUT
|
||||
value: v41_new
|
||||
- name: SGLANG_DEFAULT_THINKING
|
||||
value: "1"
|
||||
- name: SGLANG_DSV41_REASONING_EFFORT
|
||||
value: high
|
||||
- name: MC_INTRANODE_NVLINK
|
||||
value: "true"
|
||||
- name: MC_INTRA_NVLINK
|
||||
value: "true"
|
||||
- name: SGLANG_MOONCAKE_SEND_AUX_TCP
|
||||
value: "true"
|
||||
- name: SGLANG_ENABLE_METRICS_DP_ATTENTION
|
||||
value: "1"
|
||||
- name: SGLANG_BATCH_TELEMETRY
|
||||
value: "1"
|
||||
- name: SGLANG_BATCH_SPAN_SAMPLE
|
||||
value: "0.1"
|
||||
- name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT
|
||||
value: "600"
|
||||
- name: SGLANG_DISAGGREGATION_QUEUE_SIZE
|
||||
value: "16"
|
||||
- name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE
|
||||
value: "32"
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
- name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT
|
||||
value: "1024"
|
||||
- name: TMPDIR
|
||||
value: /data/ymk/l3ssd/d-tmp
|
||||
- name: XDG_CACHE_HOME
|
||||
value: /data/ymk/l3ssd/d-xdg
|
||||
- name: HF_HOME
|
||||
value: /data/ymk/l3ssd/d-hf
|
||||
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: sglang
|
||||
ports:
|
||||
- containerPort: 30300
|
||||
name: metrics
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 150
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
limits:
|
||||
cpu: "96"
|
||||
memory: 550Gi
|
||||
nvidia.com/gpu: "4"
|
||||
requests:
|
||||
cpu: "32"
|
||||
memory: 256Gi
|
||||
nvidia.com/gpu: "4"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- mountPath: /data
|
||||
name: data
|
||||
- mountPath: /root/.cache/sglang
|
||||
name: sglang-cache
|
||||
- mountPath: /dev/shm
|
||||
name: dshm
|
||||
- mountPath: /root/.cargo/bin/cargo
|
||||
name: fake-cargo
|
||||
readOnly: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
nodeSelector:
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kubernetes.io/hostname: b300-01
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
name: data
|
||||
- hostPath:
|
||||
path: /data/ymk/cache/sglang
|
||||
type: Directory
|
||||
name: sglang-cache
|
||||
- emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 64Gi
|
||||
name: dshm
|
||||
- hostPath:
|
||||
path: /data/ymk/ds41/fake-cargo
|
||||
type: File
|
||||
name: fake-cargo
|
||||
- dependencies:
|
||||
- p
|
||||
- d
|
||||
minReadySeconds: 0
|
||||
name: gateway
|
||||
podManagementPolicy: Parallel
|
||||
replicas: 1
|
||||
standalonePattern:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "29000"
|
||||
prometheus.io/scheme: http
|
||||
prometheus.io/scrape: "true"
|
||||
labels:
|
||||
app: ds41-l3ssd-gw
|
||||
kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571
|
||||
kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7
|
||||
kongming.io/managed-by: kongming
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0
|
||||
kongming.io/resource-type: custom-workload
|
||||
kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0
|
||||
spec:
|
||||
containers:
|
||||
- command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
exec /opt/sglang/bin/sgl-model-gateway launch \
|
||||
--pd-disaggregation \
|
||||
--prefill http://127.0.0.1:30000 8998 \
|
||||
--decode http://127.0.0.1:30300 \
|
||||
--prefill-policy cache_aware --decode-policy cache_aware \
|
||||
--g9i-enabled --g9i-max-new-tokens 4096 --g9i-est-td-enabled --g9i-shadow-enabled --g9i-td-admit enforce --g9i-p-queue-reqs-cap 64 --g9i-p-queue-token-cap 32000000 --g9i-p-qps 40 --g9i-p-refresh-ms 250 --g9i-window-fixed 256 \
|
||||
\
|
||||
\
|
||||
--g9i-regular-p-queue-reject-secs 0 \
|
||||
--g9i-decode-max-running 512 \
|
||||
--pool-idle-timeout-secs 2 --pool-max-idle-per-host 0 \
|
||||
--health-check-timeout-secs 30 --health-check-interval-secs 30 \
|
||||
--cb-failure-threshold 50 \
|
||||
--model-path /data/models/DeepSeek-V4.1-Flash-1017 \
|
||||
--tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \
|
||||
--host 0.0.0.0 --port 30400
|
||||
image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115
|
||||
imagePullPolicy: IfNotPresent
|
||||
name: gateway
|
||||
ports:
|
||||
- containerPort: 30400
|
||||
name: http
|
||||
protocol: TCP
|
||||
- containerPort: 29000
|
||||
name: metrics
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
failureThreshold: 60
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
resources:
|
||||
limits:
|
||||
cpu: "16"
|
||||
memory: 32Gi
|
||||
requests:
|
||||
cpu: "2"
|
||||
memory: 8Gi
|
||||
volumeMounts:
|
||||
- mountPath: /data
|
||||
name: data
|
||||
- mountPath: /opt/sglang/bin/sgl-model-gateway
|
||||
name: gw-bin
|
||||
readOnly: true
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
hostNetwork: true
|
||||
nodeSelector:
|
||||
kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg
|
||||
kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig
|
||||
kubernetes.io/hostname: b300-01
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /data
|
||||
type: Directory
|
||||
name: data
|
||||
- hostPath:
|
||||
path: /data/ymk/gw-ns-est-td/sgl-model-gateway
|
||||
type: File
|
||||
name: gw-bin
|
||||
status:
|
||||
conditions:
|
||||
- lastTransitionTime: "2026-11-16T12:36:12Z"
|
||||
message: All roles are ready
|
||||
observedGeneration: 9
|
||||
reason: AllRolesReady
|
||||
status: "True"
|
||||
type: Ready
|
||||
observedGeneration: 9
|
||||
roleStatuses:
|
||||
- name: mooncake-master
|
||||
readyReplicas: 1
|
||||
replicas: 1
|
||||
updatedReplicas: 1
|
||||
- name: p
|
||||
readyReplicas: 1
|
||||
replicas: 1
|
||||
updatedReplicas: 1
|
||||
- name: d
|
||||
readyReplicas: 1
|
||||
replicas: 1
|
||||
updatedReplicas: 1
|
||||
- name: gateway
|
||||
readyReplicas: 1
|
||||
replicas: 1
|
||||
updatedReplicas: 1
|
||||
Reference in New Issue
Block a user