diff --git a/deploy/b300-k8s/Dockerfile.mooncake-0.3.14 b/deploy/b300-k8s/Dockerfile.mooncake-0.3.14 new file mode 100644 index 0000000000..6b706fc029 --- /dev/null +++ b/deploy/b300-k8s/Dockerfile.mooncake-0.3.14 @@ -0,0 +1,25 @@ +# L3 SSD image: recipe-ns engine + official mooncake 0.3.14 (CUDA 13, cp312). +# +# Build context: this directory plus the 0.3.14 wheel +# mooncake_transfer_engine_cuda13-0.3.14-cp312-cp312-manylinux_2_28_x86_64.whl +# Use build-l3ssd-image.sh, which downloads and verifies the wheel. +# +# Base tag recipe-ns-433257c7b-baked-20261114 = b300/integration @ 433257c7b +# (recipe namespace/chat-ns + gateway binary baked). Rebuild it from that commit +# if the local tag is missing. +FROM sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-433257c7b-baked-20261114 + +COPY mooncake_transfer_engine_cuda13-0.3.14-cp312-cp312-manylinux_2_28_x86_64.whl /tmp/ +RUN python3 -m pip install --no-deps --force-reinstall \ + /tmp/mooncake_transfer_engine_cuda13-0.3.14-cp312-cp312-manylinux_2_28_x86_64.whl \ + && rm -f /tmp/mooncake_transfer_engine_cuda13-0.3.14-cp312-cp312-manylinux_2_28_x86_64.whl \ + # The wheel's master binary uses $ORIGIN/../mooncake_transfer_engine_cuda13.libs; + # keep it in place and expose a wrapper at the legacy path used by the manifests. + && printf '%s\n' \ + '#!/bin/sh' \ + 'export LD_LIBRARY_PATH=/opt/sglang/lib/python3.12/site-packages/mooncake_transfer_engine_cuda13.libs:$LD_LIBRARY_PATH' \ + 'exec /opt/sglang/lib/python3.12/site-packages/mooncake/mooncake_master "$@"' \ + > /opt/sglang/bin/mooncake_master \ + && chmod +x /opt/sglang/bin/mooncake_master \ + && python3 -c "import importlib.metadata as m; print('mooncake:', m.version('mooncake-transfer-engine-cuda13'))" \ + && /opt/sglang/bin/mooncake_master --version | head -1 diff --git a/deploy/b300-k8s/README.md b/deploy/b300-k8s/README.md new file mode 100644 index 0000000000..b79675d78e --- /dev/null +++ b/deploy/b300-k8s/README.md @@ -0,0 +1,77 @@ +# B300 L3 Mooncake SSD 基线部署(2026-11-15) + +> 以同事在 `ds41-gist-exp` 上验证的 **L3 Mooncake SSD** 配置为基础,使用我们的 +> `b300/integration` 引擎代码 + 官方 **mooncake 0.3.14**,不再依赖 hostPath +> `/opt/mooncake-latest`。 + +## 1. 版本要求(重要) + +- **mooncake `0.3.14`**(`mooncake-transfer-engine-cuda13`,CUDA 13 / cp312 wheel)。 + - 镜像内原为 `0.3.13.post1`;0.3.14 含上游 SSD offload 相关修复/增强。 + - wheel 与 sha256 见 `requirements-mooncake.txt`,安装必须 `--no-deps`(基底已有运行时依赖)。 +- 引擎:`b300/integration` @ `433257c7b`(recipe namespace/chat-ns + 新 gateway)。 +- 镜像:`sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115` + (40.1GB,已 `ctr -n k8s.io images import`;本地 docker tag `ymkymx/sglang:...` 同名)。 +- 构建/装载:`./build-l3ssd-image.sh`(下载校验 wheel → docker build → ctr import)。 + +镜像内 `/opt/sglang/bin/mooncake_master` 是 wrapper:设置 wheel `.libs` 的 +`LD_LIBRARY_PATH` 后 exec 包内 master(已验证 `0.3.14 (git e1ea5d3)`)。 + +## 2. L3 SSD 配置 + +`mooncake-store-l3ssd.json`: + +```json +{"master_server_address":"127.0.0.1:50051","metadata_server":"P2PHANDSHAKE", + "global_segment_size":"200gb","protocol":"tcp", + "enable_ssd_offload":true,"ssd_offload_path":"/data/ymk/mooncake-ssd"} +``` + +- 部署时把该文件放到节点 `/data/ymk/ds41/mooncake-store-l3ssd.json`,SSD 目录 + `/data/ymk/mooncake-ssd` 需存在。 +- `MOONCAKE_OFFLOAD_*`:`bucket_storage_backend` / 512GiB bucket 上限 / `fifo` / + `flush_partial=0`(见 manifest)。 +- master 参数必须带 + `--enable_offload=true --offload_on_evict=true --promotion_on_hit=false --offload_force_evict=false`。 + +## 3. manifest + +| 文件 | 用途 | +|---|---| +| `ds41-l3ssd-test-baked-20261115.yaml` | 新建 cw `ds41-l3ssd-test`(推荐先做基线验证) | +| `ds41-gist-exp-l3ssd-baked-20261115.yaml` | 原名 `ds41-gist-exp`,用于直接更新原栈 | + +已通过 `kmctl create/update --dry-run` 校验。启动前先停掉占用 8 卡的 stack +(`ds41-recipe-ns-test` 或旧 `ds41-gist-exp`),确认 GPU 归零: + +```bash +kmctl create cw ds41-l3ssd-test --node-group B300 \ + -f deploy/b300-k8s/ds41-l3ssd-test-baked-20261115.yaml +# 或替换原栈: +# kmctl update cw ds41-gist-exp -f deploy/b300-k8s/ds41-gist-exp-l3ssd-baked-20261115.yaml +``` + +端口/角色与旧 `ds41-gist-exp` 一致:master 50051、P 30000、D 30300、gateway 30400。 +启动约 10-15 分钟;核对 P/D `max_total_num_tokens` 和 master 日志中的 SSD offload 初始化。 + +## 4. 与上一版配置的差异 + +P/D 的 `sglang serve` 参数**完全一致**;差异只在 mooncake: + +| 项 | 旧(11-09) | 本基线 | +|---|---|---| +| mooncake | 镜像内 0.3.13.post1 | 镜像内 **0.3.14** | +| master | `/opt/sglang/bin/mooncake_master`,无 offload | 同路径(0.3.14 wrapper),带 offload 参数 | +| P/D PYTHONPATH | 无 | 无(0.3.14 已烘入) | +| L3 配置 | 纯 TCP,无 SSD | `enable_ssd_offload` + `ssd_offload_path` | +| `MOONCAKE_OFFLOAD_*` | 无 | bucket backend / 512GiB / fifo / flush_partial=0 | +| TMPDIR/XDG/HF | 默认/`/data/ymk` | `/data/ymk/l3ssd/{master,p,d}-*` | + +## 5. 注意 + +- 同事原配置使用 `SGLANG_TRACE_LOOP_SAMPLE=1.0`(全量 detail tracing,约 2% 吞吐 + + p95 开销)。本基线为对齐其配置保留 1.0;正式上线建议改回 `0.05`。 +- `global_segment_size=200gb` + 512GiB SSD offload:上线前按宿主内存总账复核。 +- SSD 目录增长需监控(`/data` 当前约 7.6T 可用)。 +- 旧镜像 `recipe-ns-433257c7b-baked-20261114` 与 `recipe-ns-a7c668a93-baked-20261114` + 保留作回滚/对照。 diff --git a/deploy/b300-k8s/build-l3ssd-image.sh b/deploy/b300-k8s/build-l3ssd-image.sh new file mode 100644 index 0000000000..c9a434eb05 --- /dev/null +++ b/deploy/b300-k8s/build-l3ssd-image.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Build and load the L3 SSD image (mooncake 0.3.14) on b300-01. +set -euo pipefail +cd "$(dirname "$0")" + +WHEEL=mooncake_transfer_engine_cuda13-0.3.14-cp312-cp312-manylinux_2_28_x86_64.whl +URL=https://files.pythonhosted.org/packages/b9/bf/a117c9c9f892e86893caf0dd8b5bfae1d11edbfb0173dcd30108a7168fc3/$WHEEL +SHA=74fa5f0684b143167503e33f85cccb9ca687d3f9843d4b594dbd5342c7df3e17 +TAG=recipe-ns-l3ssd-433257c7b-baked-20261115 +REPO=sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang + +if [ ! -f "$WHEEL" ]; then + curl -fL -o "$WHEEL" "$URL" +fi +echo "$SHA $WHEEL" | sha256sum -c - + +docker build -t "ymkymx/sglang:$TAG" -f Dockerfile.mooncake-0.3.14 . +docker tag "ymkymx/sglang:$TAG" "$REPO:$TAG" +docker save "$REPO:$TAG" | ctr -n k8s.io images import - +echo "loaded $REPO:$TAG" diff --git a/deploy/b300-k8s/ds41-gist-exp-l3ssd-baked-20261115.yaml b/deploy/b300-k8s/ds41-gist-exp-l3ssd-baked-20261115.yaml new file mode 100644 index 0000000000..c578398dbe --- /dev/null +++ b/deploy/b300-k8s/ds41-gist-exp-l3ssd-baked-20261115.yaml @@ -0,0 +1,488 @@ +apiVersion: workloads.x-k8s.io/v1alpha2 +kind: RoleBasedGroup +metadata: + annotations: + rbg.workloads.x-k8s.io/discovery-config-mode: refine + labels: + app: ds41-gist-exp + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + sglang-model: ds-v41-flash + name: ds41-gist-exp + namespace: prj-d7te67bj0sq2fgdrmbig +spec: + roles: + - minReadySeconds: 0 + name: mooncake-master + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '19003' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-mooncake + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=true --promotion_on_hit=false --offload_force_evict=false\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: mooncake-master + ports: + - containerPort: 19003 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + initialDelaySeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 50051 + resources: + limits: + cpu: '4' + memory: 64Gi + requests: + cpu: '1' + memory: 8Gi + env: + - name: TMPDIR + value: /data/ymk/l3ssd/master-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/master-xdg + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: p + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30000' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-p + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 0,1,2,3 + - name: SGLANG_LOG_MS + value: '1' + - name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS + value: '1' + - name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL + value: '1' + - name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS + value: '1' + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: '65536' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DP_USE_GATHERV + value: '1' + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION + value: '0' + - name: SGLANG_NUMA_BIND_V2 + value: '0' + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: SGLANG_TIMEOUT_KEEP_ALIVE + value: '900' + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH + value: /data/ymk/ds41/mooncake-store-l3ssd.json + - name: SGLANG_HICACHE_PREFETCH_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_BACKUP_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_TIMING + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_ENABLE_METRICS_DEVICE_TIMER + value: '1' + - name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL + value: '100' + - name: SGLANG_MM_TRACE + value: '1' + - name: SGLANG_BATCH_SPAN_MIN_MS + value: '200' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.05' + - name: SGLANG_BATCH_SPAN_MAX_LINKS + value: '8' + - name: SGLANG_BATCH_STAGE_TIMING + value: '1' + - name: SGLANG_BATCH_PASS_SPAN + value: '1' + - name: SGLANG_BATCH_PASS_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_RECV_SPAN + value: '1' + - name: SGLANG_RECV_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_MM_TRACE_SYNC + value: '0' + - name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK + value: '1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: SGLANG_TRACE_LOOP_SPAN + value: '1' + - name: SGLANG_TRACE_LOOP_SAMPLE + value: '1.0' + - name: SGLANG_FWD_TIMELINE + value: '0' + - name: SGLANG_TRACE_LOOP_DETAIL + value: '1' + - name: SGLANG_TRACE_MOONCAKE_DETAIL + value: '0' + - name: SGLANG_TRACE_BACKUP_DETAIL + value: '1' + - name: SGLANG_ASYNC_BACKUP_SUBMIT + value: '1' + - name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH + value: '1' + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE + value: '9400' + - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR + value: bucket_storage_backend + - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE + value: '549755813888' + - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES + value: '549755813888' + - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY + value: fifo + - name: TMPDIR + value: /data/ymk/l3ssd/p-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/p-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/p-hf + - name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS + value: '0' + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30000 + name: metrics + protocol: TCP + - containerPort: 8998 + name: bootstrap + protocol: TCP + - name: mc-0 + containerPort: 9400 + protocol: TCP + - name: mc-1 + containerPort: 9401 + protocol: TCP + - name: mc-2 + containerPort: 9402 + protocol: TCP + - name: mc-3 + containerPort: 9403 + protocol: TCP + readinessProbe: + failureThreshold: 120 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 2Ti + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 512Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/l3ssd/model-cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: d + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30300' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-d + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 4,5,6,7 + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: TMPDIR + value: /data/ymk/l3ssd/d-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/d-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/d-hf + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30300 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 150 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 550Gi + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 256Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/l3ssd/model-cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - p + - d + minReadySeconds: 0 + name: gateway + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '29000' + prometheus.io/scheme: http + prometheus.io/scrape: 'true' + labels: + app: ds41-l3ssd-gw + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 512 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: gateway + ports: + - containerPort: 30400 + name: http + protocol: TCP + - containerPort: 29000 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '16' + memory: 32Gi + requests: + cpu: '2' + memory: 8Gi + volumeMounts: + - mountPath: /data + name: data + - mountPath: /opt/sglang/bin/sgl-model-gateway + name: gw-bin + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - name: gw-bin + hostPath: + path: /data/ymk/cache/gw-est-td/sgl-model-gateway + type: File diff --git a/deploy/b300-k8s/ds41-l3ssd-test-baked-20261115.yaml b/deploy/b300-k8s/ds41-l3ssd-test-baked-20261115.yaml new file mode 100644 index 0000000000..614b1b9dc3 --- /dev/null +++ b/deploy/b300-k8s/ds41-l3ssd-test-baked-20261115.yaml @@ -0,0 +1,488 @@ +apiVersion: workloads.x-k8s.io/v1alpha2 +kind: RoleBasedGroup +metadata: + annotations: + rbg.workloads.x-k8s.io/discovery-config-mode: refine + labels: + app: ds41-gist-exp + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + sglang-model: ds-v41-flash + name: ds41-l3ssd-test + namespace: prj-d7te67bj0sq2fgdrmbig +spec: + roles: + - minReadySeconds: 0 + name: mooncake-master + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '19003' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-mooncake + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=true --promotion_on_hit=false --offload_force_evict=false\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: mooncake-master + ports: + - containerPort: 19003 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + initialDelaySeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 50051 + resources: + limits: + cpu: '4' + memory: 64Gi + requests: + cpu: '1' + memory: 8Gi + env: + - name: TMPDIR + value: /data/ymk/l3ssd/master-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/master-xdg + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: p + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30000' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-p + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 0,1,2,3 + - name: SGLANG_LOG_MS + value: '1' + - name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS + value: '1' + - name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL + value: '1' + - name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS + value: '1' + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: '65536' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DP_USE_GATHERV + value: '1' + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION + value: '0' + - name: SGLANG_NUMA_BIND_V2 + value: '0' + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: SGLANG_TIMEOUT_KEEP_ALIVE + value: '900' + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH + value: /data/ymk/ds41/mooncake-store-l3ssd.json + - name: SGLANG_HICACHE_PREFETCH_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_BACKUP_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_TIMING + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_ENABLE_METRICS_DEVICE_TIMER + value: '1' + - name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL + value: '100' + - name: SGLANG_MM_TRACE + value: '1' + - name: SGLANG_BATCH_SPAN_MIN_MS + value: '200' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.05' + - name: SGLANG_BATCH_SPAN_MAX_LINKS + value: '8' + - name: SGLANG_BATCH_STAGE_TIMING + value: '1' + - name: SGLANG_BATCH_PASS_SPAN + value: '1' + - name: SGLANG_BATCH_PASS_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_RECV_SPAN + value: '1' + - name: SGLANG_RECV_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_MM_TRACE_SYNC + value: '0' + - name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK + value: '1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: SGLANG_TRACE_LOOP_SPAN + value: '1' + - name: SGLANG_TRACE_LOOP_SAMPLE + value: '1.0' + - name: SGLANG_FWD_TIMELINE + value: '0' + - name: SGLANG_TRACE_LOOP_DETAIL + value: '1' + - name: SGLANG_TRACE_MOONCAKE_DETAIL + value: '0' + - name: SGLANG_TRACE_BACKUP_DETAIL + value: '1' + - name: SGLANG_ASYNC_BACKUP_SUBMIT + value: '1' + - name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH + value: '1' + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE + value: '9400' + - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR + value: bucket_storage_backend + - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE + value: '549755813888' + - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES + value: '549755813888' + - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY + value: fifo + - name: TMPDIR + value: /data/ymk/l3ssd/p-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/p-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/p-hf + - name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS + value: '0' + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30000 + name: metrics + protocol: TCP + - containerPort: 8998 + name: bootstrap + protocol: TCP + - name: mc-0 + containerPort: 9400 + protocol: TCP + - name: mc-1 + containerPort: 9401 + protocol: TCP + - name: mc-2 + containerPort: 9402 + protocol: TCP + - name: mc-3 + containerPort: 9403 + protocol: TCP + readinessProbe: + failureThreshold: 120 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 2Ti + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 512Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/l3ssd/model-cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: d + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30300' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-d + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 4,5,6,7 + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: TMPDIR + value: /data/ymk/l3ssd/d-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/d-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/d-hf + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30300 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 150 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 550Gi + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 256Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/l3ssd/model-cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - p + - d + minReadySeconds: 0 + name: gateway + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '29000' + prometheus.io/scheme: http + prometheus.io/scrape: 'true' + labels: + app: ds41-l3ssd-gw + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 512 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: gateway + ports: + - containerPort: 30400 + name: http + protocol: TCP + - containerPort: 29000 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '16' + memory: 32Gi + requests: + cpu: '2' + memory: 8Gi + volumeMounts: + - mountPath: /data + name: data + - mountPath: /opt/sglang/bin/sgl-model-gateway + name: gw-bin + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - name: gw-bin + hostPath: + path: /data/ymk/cache/gw-est-td/sgl-model-gateway + type: File diff --git a/deploy/b300-k8s/mooncake-store-l3ssd.json b/deploy/b300-k8s/mooncake-store-l3ssd.json new file mode 100644 index 0000000000..13a8367ebd --- /dev/null +++ b/deploy/b300-k8s/mooncake-store-l3ssd.json @@ -0,0 +1,8 @@ +{ + "master_server_address": "127.0.0.1:50051", + "metadata_server": "P2PHANDSHAKE", + "global_segment_size": "200gb", + "protocol": "tcp", + "enable_ssd_offload": true, + "ssd_offload_path": "/data/ymk/mooncake-ssd" +} diff --git a/deploy/b300-k8s/requirements-mooncake.txt b/deploy/b300-k8s/requirements-mooncake.txt new file mode 100644 index 0000000000..955314a110 --- /dev/null +++ b/deploy/b300-k8s/requirements-mooncake.txt @@ -0,0 +1,6 @@ +# L3 Mooncake SSD baseline requires the official 0.3.14 wheel. +# Previous image shipped 0.3.13.post1; 0.3.14 contains the upstream SSD-offload fixes. +# Wheel: https://files.pythonhosted.org/packages/b9/bf/a117c9c9f892e86893caf0dd8b5bfae1d11edbfb0173dcd30108a7168fc3/mooncake_transfer_engine_cuda13-0.3.14-cp312-cp312-manylinux_2_28_x86_64.whl +# sha256: 74fa5f0684b143167503e33f85cccb9ca687d3f9843d4b594dbd5342c7df3e17 +# Installed with --no-deps; the base image already provides the runtime deps. +mooncake-transfer-engine-cuda13==0.3.14