From f363a6ab7bbb87cd72dae7940ef12dd8c5e75439 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Mon, 16 Nov 2026 22:03:11 +0800 Subject: [PATCH] deploy: record 11/16 pre-optimistic stable config --- .../20261116-pre-optimistic/NOTES.md | 68 ++ ...cw-ds41-gist-exp-11-16-pre-optimistic.yaml | 488 ++++++++++++++ .../20261116-pre-optimistic/cw-platform.yaml | 139 ++++ .../20261116-pre-optimistic/rbg-live-raw.yaml | 608 ++++++++++++++++++ .../20261116-pre-optimistic/rbg-live.yaml | 608 ++++++++++++++++++ 5 files changed, 1911 insertions(+) create mode 100644 deploy/b300-k8s/snapshots/20261116-pre-optimistic/NOTES.md create mode 100644 deploy/b300-k8s/snapshots/20261116-pre-optimistic/cw-ds41-gist-exp-11-16-pre-optimistic.yaml create mode 100644 deploy/b300-k8s/snapshots/20261116-pre-optimistic/cw-platform.yaml create mode 100644 deploy/b300-k8s/snapshots/20261116-pre-optimistic/rbg-live-raw.yaml create mode 100644 deploy/b300-k8s/snapshots/20261116-pre-optimistic/rbg-live.yaml diff --git a/deploy/b300-k8s/snapshots/20261116-pre-optimistic/NOTES.md b/deploy/b300-k8s/snapshots/20261116-pre-optimistic/NOTES.md new file mode 100644 index 0000000000..5a8b30fc1d --- /dev/null +++ b/deploy/b300-k8s/snapshots/20261116-pre-optimistic/NOTES.md @@ -0,0 +1,68 @@ +# 11/16 上线前稳定配置(ds41-gist-exp) + +记录时间:2026-11-16 21:49 CST(13:49Z),在切换 `buffer_only + optimistic` 之前。 + +## 身份 + +- 平台 cw:`ds41-gist-exp` / `cw-dblft6sgrhagmpdt8ef0` +- RBG:`ds41-gist-exp`(namespace `prj-d7te67bj0sq2fgdrmbig`) +- 角色:`mooncake-master` / `p` / `d` / `gateway` +- 镜像:`sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115` +- P:GPU 0-3;D:GPU 4-7;全 hostNetwork / hostPID / privileged,钉 b300-01 + +## 文件 + +- `rbg-live-raw.yaml`:`kubectl get rbg -o yaml` 原始快照(含 status,仅存档) +- `cw-ds41-gist-exp-11-16-pre-optimistic.yaml`:清理过平台字段的 RBG,可直接用于回滚更新 +- `cw-platform.yaml`:`kmctl get cw -o yaml` 的平台清单(`Spec.Manifest` 是旧值,见下方漂移) +- 本说明 + +## 生效配置摘要 + +P: + +- `tp4 / dp4 / ep4`、`megamoe`、`flashmla_mega`、`ep-redundant 32` +- DSPARK block 3、decoder SWA bounded replay、`swa-full-tokens-ratio 0.12` +- `chunked-prefill-size 131072`、`max-prefill-tokens 131072` +- `max-running-requests 512`、`tokenizer-worker-num 16` +- `hicache-ratio 4`、`hicache-mem-layout page_first_direct`、`hicache-io-backend direct` +- `hicache-write-policy write_through`、`hicache-storage-backend mooncake` +- **默认 host memory mode(cache),没有 `--hicache-host-memory-mode buffer_only`** +- **没有 `--optimistic-prefill-attempts`**(`#optimistic-req` 为 0) +- L3 SSD:4 TB `mooncake-ssd`,master `--offload_on_evict=false` + +D: + +- 同并行 tp4/dp4/ep4、`ep-redundant 64`、DSPARK block 3 +- `max-running-requests 512`、`cuda-graph-max-bs-decode 512` +- `chunked-prefill-size 32768`、`max-prefill-tokens 8192` +- **没有 `--tokenizer-worker-num`**(默认 1) +- `mem-fraction-static 0.80` + +Gateway: + +- 实际 live:`--g9i-p-queue-reqs-cap 64 --g9i-p-queue-token-cap 32000000 --g9i-p-qps 40 --g9i-p-refresh-ms 250 --g9i-window-fixed 256` +- `--g9i-regular-p-queue-reject-secs 0`、`--g9i-decode-max-running 512` +- 网关二进制 hostPath:`/data/ymk/gw-ns-est-td/sgl-model-gateway` + +> ⚠️ 平台 `cw-platform.yaml` 的 `Spec.Manifest` 里 gateway 仍是旧的 +> `p-queue-reqs-cap 32 / token-cap 16000000 / p-qps 20`;**live RBG 是 64 / 32M / 40**。 +> 本次快照与回滚文件都以 live RBG 为准,避免回滚时把网关参数打回旧值。 + +## 切换前状态 + +- 所有 Pod `1/1 Running`;P/D/gateway `/health` 200。 +- 为切 `buffer_only` 做 L3 SSD 清理前,`/data/ymk/mooncake-ssd` 占用 **2.3T**;清理后 4.0K,`/data` 7.5T available。 +- 启动基线(切换后复核):P `max_total_num_tokens=16,158,208` / 128 per-rank / `available_gpu_mem=23.00 GB`; + D `58,859,520` / 128 per-rank / `available_gpu_mem=79.27 GB`。 + +## 回滚 + +```bash +MSYS_NO_PATHCONV=1 wsl -e /home/ubuntu/.local/bin/kmctl stop cw ds41-gist-exp --wait --timeout 10m --request-timeout 180s +ssh 24.125.93.200.207 'bash /data/ymk/deploy/b300-k8s/clean_l3ssd_cache.sh' +MSYS_NO_PATHCONV=1 wsl -e /home/ubuntu/.local/bin/kmctl update cw ds41-gist-exp \ + -f /mnt/d/B300/deploy/b300-k8s/snapshots/20261116-pre-optimistic/cw-ds41-gist-exp-11-16-pre-optimistic.yaml +``` + +回滚前确认 `kmctl get cw ds41-gist-exp` 为 `Terminated 0/0`、GPU 0-7 为 0 MiB。 diff --git a/deploy/b300-k8s/snapshots/20261116-pre-optimistic/cw-ds41-gist-exp-11-16-pre-optimistic.yaml b/deploy/b300-k8s/snapshots/20261116-pre-optimistic/cw-ds41-gist-exp-11-16-pre-optimistic.yaml new file mode 100644 index 0000000000..1092870d3c --- /dev/null +++ b/deploy/b300-k8s/snapshots/20261116-pre-optimistic/cw-ds41-gist-exp-11-16-pre-optimistic.yaml @@ -0,0 +1,488 @@ +apiVersion: workloads.x-k8s.io/v1alpha2 +kind: RoleBasedGroup +metadata: + annotations: + rbg.workloads.x-k8s.io/discovery-config-mode: refine + labels: + app: ds41-gist-exp + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + sglang-model: ds-v41-flash + name: ds41-gist-exp + namespace: prj-d7te67bj0sq2fgdrmbig +spec: + roles: + - minReadySeconds: 0 + name: mooncake-master + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '19003' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-mooncake + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n" + env: + - name: TMPDIR + value: /data/ymk/l3ssd/master-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/master-xdg + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: mooncake-master + ports: + - containerPort: 19003 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + initialDelaySeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 50051 + resources: + limits: + cpu: '4' + memory: 64Gi + requests: + cpu: '1' + memory: 8Gi + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: p + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30000' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-p + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 0,1,2,3 + - name: SGLANG_LOG_MS + value: '1' + - name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS + value: '1' + - name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL + value: '1' + - name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS + value: '1' + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: '65536' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DP_USE_GATHERV + value: '1' + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION + value: '0' + - name: SGLANG_NUMA_BIND_V2 + value: '0' + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: SGLANG_TIMEOUT_KEEP_ALIVE + value: '900' + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH + value: /data/ymk/ds41/mooncake-store-l3ssd.json + - name: SGLANG_HICACHE_PREFETCH_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_BACKUP_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_TIMING + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_ENABLE_METRICS_DEVICE_TIMER + value: '1' + - name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL + value: '100' + - name: SGLANG_MM_TRACE + value: '1' + - name: SGLANG_BATCH_SPAN_MIN_MS + value: '200' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.05' + - name: SGLANG_BATCH_SPAN_MAX_LINKS + value: '8' + - name: SGLANG_BATCH_STAGE_TIMING + value: '1' + - name: SGLANG_BATCH_PASS_SPAN + value: '1' + - name: SGLANG_BATCH_PASS_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_RECV_SPAN + value: '1' + - name: SGLANG_RECV_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_MM_TRACE_SYNC + value: '0' + - name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK + value: '1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: SGLANG_TRACE_LOOP_SPAN + value: '1' + - name: SGLANG_TRACE_LOOP_SAMPLE + value: '1.0' + - name: SGLANG_FWD_TIMELINE + value: '0' + - name: SGLANG_TRACE_LOOP_DETAIL + value: '1' + - name: SGLANG_TRACE_MOONCAKE_DETAIL + value: '0' + - name: SGLANG_TRACE_BACKUP_DETAIL + value: '1' + - name: SGLANG_ASYNC_BACKUP_SUBMIT + value: '1' + - name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH + value: '1' + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE + value: '9400' + - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR + value: bucket_storage_backend + - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY + value: fifo + - name: TMPDIR + value: /data/ymk/l3ssd/p-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/p-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/p-hf + - name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS + value: '0' + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30000 + name: metrics + protocol: TCP + - containerPort: 8998 + name: bootstrap + protocol: TCP + - containerPort: 9400 + name: mc-0 + protocol: TCP + - containerPort: 9401 + name: mc-1 + protocol: TCP + - containerPort: 9402 + name: mc-2 + protocol: TCP + - containerPort: 9403 + name: mc-3 + protocol: TCP + readinessProbe: + failureThreshold: 120 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 2Ti + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 512Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: d + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30300' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-d + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 4,5,6,7 + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: TMPDIR + value: /data/ymk/l3ssd/d-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/d-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/d-hf + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30300 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 150 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 550Gi + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 256Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - p + - d + minReadySeconds: 0 + name: gateway + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '29000' + prometheus.io/scheme: http + prometheus.io/scrape: 'true' + labels: + app: ds41-l3ssd-gw + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 --g9i-est-td-enabled --g9i-shadow-enabled --g9i-td-admit enforce --g9i-p-queue-reqs-cap 64 --g9i-p-queue-token-cap 32000000 --g9i-p-qps 40 --g9i-p-refresh-ms 250 --g9i-window-fixed 256 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 512 \\\n --pool-idle-timeout-secs 2 --pool-max-idle-per-host 0 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: gateway + ports: + - containerPort: 30400 + name: http + protocol: TCP + - containerPort: 29000 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '16' + memory: 32Gi + requests: + cpu: '2' + memory: 8Gi + volumeMounts: + - mountPath: /data + name: data + - mountPath: /opt/sglang/bin/sgl-model-gateway + name: gw-bin + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/gw-ns-est-td/sgl-model-gateway + type: File + name: gw-bin diff --git a/deploy/b300-k8s/snapshots/20261116-pre-optimistic/cw-platform.yaml b/deploy/b300-k8s/snapshots/20261116-pre-optimistic/cw-platform.yaml new file mode 100644 index 0000000000..72f70d551f --- /dev/null +++ b/deploy/b300-k8s/snapshots/20261116-pre-optimistic/cw-platform.yaml @@ -0,0 +1,139 @@ +Meta: + Id: cw-dblft6sgrhagmpdt8ef0 + ClusterId: cls-damg5fkgrhalvvbgbbj0 + Name: ds41-gist-exp + ProjectId: prj-d7te67bj0sq2fgdrmbig + NodeGroupId: ng-damg6d4grhalvvbgbbrg + NodeGroupName: B300 + CreatorId: usr-d7upbr3j0sq2fgdrmcr0 + CreatorName: minke.yu + VisibleRange: All + CreatedAt: 1793785511 + UpdatedAt: 1794788208 +Spec: + Status: Running + Ready: 4/4 + Manifest: "apiVersion: workloads.x-k8s.io/v1alpha2\r\nkind: RoleBasedGroup\r\nmetadata:\r\n annotations:\r\n rbg.workloads.x-k8s.io/discovery-config-mode: refine\r\n labels:\r\n app: ds41-gist-exp\r\n kongming.io/managed-by: kongming\r\n kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg\r\n kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig\r\n kongming.io/resource-type: custom-workload\r\n sglang-model: ds-v41-flash\r\n name: ds41-gist-exp\r\n namespace: prj-d7te67bj0sq2fgdrmbig\r\nspec:\r\n roles:\r\n - minReadySeconds: 0\r\n name: mooncake-master\r\n podManagementPolicy: Parallel\r\n replicas: 1\r\n standalonePattern:\r\n template:\r\n metadata:\r\n annotations:\r\n prometheus.io/path: /metrics\r\n prometheus.io/port: '19003'\r\n prometheus.io/scheme: http\r\n labels:\r\n app: ds41-l3ssd-mooncake\r\n kongming.io/managed-by: kongming\r\n kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg\r\n kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig\r\n kongming.io/resource-type: custom-workload\r\n spec:\r\n containers:\r\n - command:\r\n - /bin/bash\r\n - -c\r\n - \"# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\\n# 我们的 P/D\\\r\n \\ 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\\nexec /opt/sglang/bin/mooncake_master\\\r\n \\ \\\\\\n --port 50051 --enable_http_metadata_server=true \\\\\\n --http_metadata_server_port=18081\\\r\n \\ --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true\\\r\n \\ --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\\n\"\r\n image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115\r\n imagePullPolicy: IfNotPresent\r\n name: mooncake-master\r\n ports:\r\n - containerPort: 19003\r\n name: metrics\r\n protocol: TCP\r\n readinessProbe:\r\n failureThreshold: 60\r\n initialDelaySeconds: 5\r\n periodSeconds: 5\r\n tcpSocket:\r\n port: 50051\r\n resources:\r\n limits:\r\n cpu: '4'\r\n memory: 64Gi\r\n requests:\r\n cpu: '1'\r\n memory: 8Gi\r\n env:\r\n - name: TMPDIR\r\n value: /data/ymk/l3ssd/master-tmp\r\n - name: XDG_CACHE_HOME\r\n value: /data/ymk/l3ssd/master-xdg\r\n dnsPolicy: ClusterFirstWithHostNet\r\n hostNetwork: true\r\n nodeSelector:\r\n kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg\r\n kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig\r\n kubernetes.io/hostname: b300-01\r\n - dependencies:\r\n - mooncake-master\r\n minReadySeconds: 0\r\n name: p\r\n podManagementPolicy: Parallel\r\n replicas: 1\r\n standalonePattern:\r\n template:\r\n metadata:\r\n annotations:\r\n prometheus.io/path: /metrics\r\n prometheus.io/port: '30000'\r\n prometheus.io/scheme: http\r\n labels:\r\n app: ds41-l3ssd-p\r\n kongming.io/managed-by: kongming\r\n kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg\r\n kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig\r\n kongming.io/resource-type: custom-workload\r\n spec:\r\n containers:\r\n - command:\r\n - /bin/bash\r\n - -c\r\n - \"exec sglang serve \\\\\\n --trust-remote-code --tokenizer-backend fastokens\\\r\n \\ --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name\\\r\n \\ /data/models/DeepSeek-V4.1-Flash \\\\\\n --reasoning-parser auto --tool-call-parser\\\r\n \\ auto \\\\\\n --default-chat-template-kwargs '{\\\"thinking\\\": true, \\\"\\\r\n reasoning_effort\\\": \\\"high\\\"}' \\\\\\n --tp 4 --dp-size 4 --ep-size 4\\\r\n \\ --enable-dp-attention --enable-dp-lm-head \\\\\\n --moe-a2a-backend\\\r\n \\ megamoe --dsv4-attn-backend flashmla_mega \\\\\\n --ep-num-redundant-experts\\\r\n \\ 32 --ep-dispatch-algorithm dynamic \\\\\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json\\\r\n \\ \\\\\\n --mem-fraction-static 0.80 \\\\\\n --chunked-prefill-size 131072\\\r\n \\ --max-prefill-tokens 131072 \\\\\\n --max-running-requests 512 --tokenizer-worker-num\\\r\n \\ 16 \\\\\\n --load-balance-method total_tokens \\\\\\n --speculative-algorithm\\\r\n \\ DSPARK --speculative-dspark-block-size 3 \\\\\\n --enable-decoder-swa-bounded-replay\\\r\n \\ --swa-full-tokens-ratio 0.12 \\\\\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph\\\r\n \\ \\\\\\n --cuda-graph-backend-prefill breakable \\\\\\n --cuda-graph-bs-prefill\\\r\n \\ 1024 2048 4096 8192 16384 32768 \\\\\\n --cuda-graph-prefill-max-context\\\r\n \\ 262144 --cuda-graph-max-seq-len-prefill 262144 \\\\\\n --enable-hierarchical-cache\\\r\n \\ --hicache-ratio 4 \\\\\\n --hicache-mem-layout page_first_direct --hicache-io-backend\\\r\n \\ direct \\\\\\n --hicache-write-policy write_through --hicache-storage-backend\\\r\n \\ mooncake \\\\\\n --hicache-storage-prefetch-policy wait_complete \\\\\\n\\\r\n \\ --disaggregation-mode prefill --disaggregation-transfer-backend mooncake\\\r\n \\ \\\\\\n --disaggregation-bootstrap-port 8998 \\\\\\n --expert-balancedness-report-mode\\\r\n \\ prometheus \\\\\\n --expert-distribution-recorder-buffer-size 64 \\\\\\n\\\r\n \\ --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report\\\r\n \\ \\\\\\n --enable-trace --trace-modules request,mooncake \\\\\\n --otlp-traces-endpoint\\\r\n \\ 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\\\\n --watchdog-timeout\\\r\n \\ 3600 --enable-dp-attention-local-control-broadcast\\n\"\r\n env:\r\n - name: CUDA_VISIBLE_DEVICES\r\n value: 0,1,2,3\r\n - name: SGLANG_LOG_MS\r\n value: '1'\r\n - name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS\r\n value: '1'\r\n - name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL\r\n value: '1'\r\n - name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS\r\n value: '1'\r\n - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK\r\n value: '65536'\r\n - name: SGLANG_DSV4_KV_LAYOUT\r\n value: v41_new\r\n - name: SGLANG_DP_USE_GATHERV\r\n value: '1'\r\n - name: SGLANG_RAGGED_VERIFY_MODE\r\n value: static\r\n - name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION\r\n value: '0'\r\n - name: SGLANG_NUMA_BIND_V2\r\n value: '0'\r\n - name: SGLANG_DEFAULT_THINKING\r\n value: '1'\r\n - name: SGLANG_DSV41_REASONING_EFFORT\r\n value: high\r\n - name: SGLANG_TIMEOUT_KEEP_ALIVE\r\n value: '900'\r\n - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE\r\n value: '1'\r\n - name: MC_INTRANODE_NVLINK\r\n value: 'true'\r\n - name: MC_INTRA_NVLINK\r\n value: 'true'\r\n - name: SGLANG_MOONCAKE_SEND_AUX_TCP\r\n value: 'true'\r\n - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH\r\n value: /data/ymk/ds41/mooncake-store-l3ssd.json\r\n - name: SGLANG_HICACHE_PREFETCH_IO_THREADS\r\n value: '4'\r\n - name: SGLANG_HICACHE_BACKUP_IO_THREADS\r\n value: '4'\r\n - name: SGLANG_HICACHE_TIMING\r\n value: '1'\r\n - name: SGLANG_BATCH_TELEMETRY\r\n value: '1'\r\n - name: SGLANG_ENABLE_METRICS_DP_ATTENTION\r\n value: '1'\r\n - name: SGLANG_ENABLE_METRICS_DEVICE_TIMER\r\n value: '1'\r\n - name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL\r\n value: '100'\r\n - name: SGLANG_MM_TRACE\r\n value: '1'\r\n - name: SGLANG_BATCH_SPAN_MIN_MS\r\n value: '200'\r\n - name: SGLANG_BATCH_SPAN_SAMPLE\r\n value: '0.05'\r\n - name: SGLANG_BATCH_SPAN_MAX_LINKS\r\n value: '8'\r\n - name: SGLANG_BATCH_STAGE_TIMING\r\n value: '1'\r\n - name: SGLANG_BATCH_PASS_SPAN\r\n value: '1'\r\n - name: SGLANG_BATCH_PASS_SPAN_SAMPLE\r\n value: '1.0'\r\n - name: SGLANG_RECV_SPAN\r\n value: '1'\r\n - name: SGLANG_RECV_SPAN_SAMPLE\r\n value: '1.0'\r\n - name: SGLANG_MM_TRACE_SYNC\r\n value: '0'\r\n - name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK\r\n value: '1'\r\n - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT\r\n value: '600'\r\n - name: SGLANG_DISAGGREGATION_QUEUE_SIZE\r\n value: '16'\r\n - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE\r\n value: '32'\r\n - name: TZ\r\n value: Asia/Shanghai\r\n - name: SGLANG_TRACE_LOOP_SPAN\r\n value: '1'\r\n - name: SGLANG_TRACE_LOOP_SAMPLE\r\n value: '1.0'\r\n - name: SGLANG_FWD_TIMELINE\r\n value: '0'\r\n - name: SGLANG_TRACE_LOOP_DETAIL\r\n value: '1'\r\n - name: SGLANG_TRACE_MOONCAKE_DETAIL\r\n value: '0'\r\n - name: SGLANG_TRACE_BACKUP_DETAIL\r\n value: '1'\r\n - name: SGLANG_ASYNC_BACKUP_SUBMIT\r\n value: '1'\r\n - name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH\r\n value: '1'\r\n - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT\r\n value: '1024'\r\n - name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE\r\n value: '9400'\r\n - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR\r\n value: bucket_storage_backend\r\n - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE\r\n value: '1099511627776'\r\n - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES\r\n value: '1099511627776'\r\n - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY\r\n value: fifo\r\n - name: TMPDIR\r\n value: /data/ymk/l3ssd/p-tmp\r\n - name: XDG_CACHE_HOME\r\n value: /data/ymk/l3ssd/p-xdg\r\n - name: HF_HOME\r\n value: /data/ymk/l3ssd/p-hf\r\n - name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS\r\n value: '0'\r\n image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115\r\n imagePullPolicy: IfNotPresent\r\n name: sglang\r\n ports:\r\n - containerPort: 30000\r\n name: metrics\r\n protocol: TCP\r\n - containerPort: 8998\r\n name: bootstrap\r\n protocol: TCP\r\n - name: mc-0\r\n containerPort: 9400\r\n protocol: TCP\r\n - name: mc-1\r\n containerPort: 9401\r\n protocol: TCP\r\n - name: mc-2\r\n containerPort: 9402\r\n protocol: TCP\r\n - name: mc-3\r\n containerPort: 9403\r\n protocol: TCP\r\n readinessProbe:\r\n failureThreshold: 120\r\n httpGet:\r\n path: /health\r\n port: metrics\r\n initialDelaySeconds: 60\r\n periodSeconds: 10\r\n timeoutSeconds: 5\r\n resources:\r\n limits:\r\n cpu: '96'\r\n memory: 2Ti\r\n nvidia.com/gpu: '4'\r\n requests:\r\n cpu: '32'\r\n memory: 512Gi\r\n nvidia.com/gpu: '4'\r\n securityContext:\r\n privileged: true\r\n volumeMounts:\r\n - mountPath: /data\r\n name: data\r\n - mountPath: /root/.cache/sglang\r\n name: sglang-cache\r\n - mountPath: /dev/shm\r\n name: dshm\r\n - mountPath: /root/.cargo/bin/cargo\r\n name: fake-cargo\r\n readOnly: true\r\n dnsPolicy: ClusterFirstWithHostNet\r\n hostNetwork: true\r\n hostPID: true\r\n nodeSelector:\r\n kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg\r\n kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig\r\n kubernetes.io/hostname: b300-01\r\n volumes:\r\n - hostPath:\r\n path: /data\r\n type: Directory\r\n name: data\r\n - hostPath:\r\n path: /data/ymk/cache/sglang\r\n type: Directory\r\n name: sglang-cache\r\n - emptyDir:\r\n medium: Memory\r\n sizeLimit: 64Gi\r\n name: dshm\r\n - hostPath:\r\n path: /data/ymk/ds41/fake-cargo\r\n type: File\r\n name: fake-cargo\r\n - dependencies:\r\n - mooncake-master\r\n minReadySeconds: 0\r\n name: d\r\n podManagementPolicy: Parallel\r\n replicas: 1\r\n standalonePattern:\r\n template:\r\n metadata:\r\n annotations:\r\n prometheus.io/path: /metrics\r\n prometheus.io/port: '30300'\r\n prometheus.io/scheme: http\r\n labels:\r\n app: ds41-l3ssd-d\r\n kongming.io/managed-by: kongming\r\n kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg\r\n kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig\r\n kongming.io/resource-type: custom-workload\r\n spec:\r\n containers:\r\n - command:\r\n - /bin/bash\r\n - -c\r\n - \"exec sglang serve \\\\\\n --trust-remote-code --tokenizer-backend fastokens\\\r\n \\ --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name\\\r\n \\ /data/models/DeepSeek-V4.1-Flash \\\\\\n --reasoning-parser auto --tool-call-parser\\\r\n \\ auto \\\\\\n --default-chat-template-kwargs '{\\\"thinking\\\": true, \\\"\\\r\n reasoning_effort\\\": \\\"high\\\"}' \\\\\\n --tp 4 --dp-size 4 --ep-size 4\\\r\n \\ --enable-dp-attention --enable-dp-lm-head \\\\\\n --enable-fp32-lm-head\\\r\n \\ --enable-dp-attention-local-control-broadcast \\\\\\n --moe-a2a-backend\\\r\n \\ megamoe --dsv4-attn-backend flashmla_mega \\\\\\n --ep-num-redundant-experts\\\r\n \\ 64 --ep-dispatch-algorithm dynamic \\\\\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json\\\r\n \\ \\\\\\n --mem-fraction-static 0.80 \\\\\\n --speculative-algorithm DSPARK\\\r\n \\ --speculative-dspark-block-size 3 \\\\\\n --stream-interval 32 --swa-prefix-tails\\\r\n \\ 2048 --enable-decoder-swa-bounded-replay \\\\\\n --max-running-requests\\\r\n \\ 512 --cuda-graph-max-bs-decode 512 \\\\\\n --chunked-prefill-size 32768\\\r\n \\ --max-prefill-tokens 8192 \\\\\\n --disaggregation-mode decode --disaggregation-transfer-backend\\\r\n \\ mooncake \\\\\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report\\\r\n \\ \\\\\\n --enable-trace --trace-modules request,mooncake \\\\\\n --otlp-traces-endpoint\\\r\n \\ 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\\\\n --watchdog-timeout\\\r\n \\ 3600\\n\"\r\n env:\r\n - name: CUDA_VISIBLE_DEVICES\r\n value: 4,5,6,7\r\n - name: SGLANG_RAGGED_VERIFY_MODE\r\n value: static\r\n - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE\r\n value: '1'\r\n - name: SGLANG_DSV4_KV_LAYOUT\r\n value: v41_new\r\n - name: SGLANG_DEFAULT_THINKING\r\n value: '1'\r\n - name: SGLANG_DSV41_REASONING_EFFORT\r\n value: high\r\n - name: MC_INTRANODE_NVLINK\r\n value: 'true'\r\n - name: MC_INTRA_NVLINK\r\n value: 'true'\r\n - name: SGLANG_MOONCAKE_SEND_AUX_TCP\r\n value: 'true'\r\n - name: SGLANG_ENABLE_METRICS_DP_ATTENTION\r\n value: '1'\r\n - name: SGLANG_BATCH_TELEMETRY\r\n value: '1'\r\n - name: SGLANG_BATCH_SPAN_SAMPLE\r\n value: '0.1'\r\n - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT\r\n value: '600'\r\n - name: SGLANG_DISAGGREGATION_QUEUE_SIZE\r\n value: '16'\r\n - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE\r\n value: '32'\r\n - name: TZ\r\n value: Asia/Shanghai\r\n - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT\r\n value: '1024'\r\n - name: TMPDIR\r\n value: /data/ymk/l3ssd/d-tmp\r\n - name: XDG_CACHE_HOME\r\n value: /data/ymk/l3ssd/d-xdg\r\n - name: HF_HOME\r\n value: /data/ymk/l3ssd/d-hf\r\n image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115\r\n imagePullPolicy: IfNotPresent\r\n name: sglang\r\n ports:\r\n - containerPort: 30300\r\n name: metrics\r\n protocol: TCP\r\n readinessProbe:\r\n failureThreshold: 150\r\n httpGet:\r\n path: /health\r\n port: metrics\r\n initialDelaySeconds: 60\r\n periodSeconds: 10\r\n timeoutSeconds: 5\r\n resources:\r\n limits:\r\n cpu: '96'\r\n memory: 550Gi\r\n nvidia.com/gpu: '4'\r\n requests:\r\n cpu: '32'\r\n memory: 256Gi\r\n nvidia.com/gpu: '4'\r\n securityContext:\r\n privileged: true\r\n volumeMounts:\r\n - mountPath: /data\r\n name: data\r\n - mountPath: /root/.cache/sglang\r\n name: sglang-cache\r\n - mountPath: /dev/shm\r\n name: dshm\r\n - mountPath: /root/.cargo/bin/cargo\r\n name: fake-cargo\r\n readOnly: true\r\n dnsPolicy: ClusterFirstWithHostNet\r\n hostNetwork: true\r\n hostPID: true\r\n nodeSelector:\r\n kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg\r\n kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig\r\n kubernetes.io/hostname: b300-01\r\n volumes:\r\n - hostPath:\r\n path: /data\r\n type: Directory\r\n name: data\r\n - hostPath:\r\n path: /data/ymk/cache/sglang\r\n type: Directory\r\n name: sglang-cache\r\n - emptyDir:\r\n medium: Memory\r\n sizeLimit: 64Gi\r\n name: dshm\r\n - hostPath:\r\n path: /data/ymk/ds41/fake-cargo\r\n type: File\r\n name: fake-cargo\r\n - dependencies:\r\n - p\r\n - d\r\n minReadySeconds: 0\r\n name: gateway\r\n podManagementPolicy: Parallel\r\n replicas: 1\r\n standalonePattern:\r\n template:\r\n metadata:\r\n annotations:\r\n prometheus.io/path: /metrics\r\n prometheus.io/port: '29000'\r\n prometheus.io/scheme: http\r\n prometheus.io/scrape: 'true'\r\n labels:\r\n app: ds41-l3ssd-gw\r\n kongming.io/managed-by: kongming\r\n kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg\r\n kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig\r\n kongming.io/resource-type: custom-workload\r\n spec:\r\n containers:\r\n - command:\r\n - /bin/bash\r\n - -c\r\n - \"exec /opt/sglang/bin/sgl-model-gateway launch \\\\\\n --pd-disaggregation\\\r\n \\ \\\\\\n --prefill http://127.0.0.1:30000 8998 \\\\\\n --decode http://127.0.0.1:30300\\\r\n \\ \\\\\\n --prefill-policy cache_aware --decode-policy cache_aware \\\\\\n\\\r\n \\ --g9i-enabled --g9i-max-new-tokens 4096 --g9i-est-td-enabled --g9i-shadow-enabled\\\r\n \\ --g9i-td-admit enforce --g9i-p-queue-reqs-cap 32 --g9i-p-queue-token-cap\\\r\n \\ 16000000 --g9i-p-qps 20 --g9i-p-refresh-ms 250 \\\\\\n \\\\\\n \\\\\\n --g9i-regular-p-queue-reject-secs\\\r\n \\ 0 \\\\\\n --g9i-decode-max-running 512 \\\\\\n --health-check-timeout-secs\\\r\n \\ 30 --health-check-interval-secs 30 \\\\\\n --cb-failure-threshold 50\\\r\n \\ \\\\\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\\\\n --tokenizer-path\\\r\n \\ /data/models/DeepSeek-V4.1-Flash-1017 \\\\\\n --host 0.0.0.0 --port\\\r\n \\ 30400\\n\"\r\n image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115\r\n imagePullPolicy: IfNotPresent\r\n name: gateway\r\n ports:\r\n - containerPort: 30400\r\n name: http\r\n protocol: TCP\r\n - containerPort: 29000\r\n name: metrics\r\n protocol: TCP\r\n readinessProbe:\r\n failureThreshold: 60\r\n httpGet:\r\n path: /health\r\n port: http\r\n initialDelaySeconds: 20\r\n periodSeconds: 10\r\n timeoutSeconds: 5\r\n resources:\r\n limits:\r\n cpu: '16'\r\n memory: 32Gi\r\n requests:\r\n cpu: '2'\r\n memory: 8Gi\r\n volumeMounts:\r\n - mountPath: /data\r\n name: data\r\n - mountPath: /opt/sglang/bin/sgl-model-gateway\r\n name: gw-bin\r\n readOnly: true\r\n dnsPolicy: ClusterFirstWithHostNet\r\n hostNetwork: true\r\n nodeSelector:\r\n kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg\r\n kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig\r\n kubernetes.io/hostname: b300-01\r\n volumes:\r\n - hostPath:\r\n path: /data\r\n type: Directory\r\n name: data\r\n - name: gw-bin\r\n hostPath:\r\n path: /data/ymk/gw-ns-est-td/sgl-model-gateway\r\n type: File\r\n" + InstanceSpec: + Count: 4 + Cpu: 67 + Memory: 784 + Gpu: + nvidia.com/gpu: 8 +Pods: + - Name: ds41-gist-exp-d-0 + Status: Running + StartTime: 1794788212 + FinishedAt: 0 + Node: b300-01 + NodeId: nd-damg5gkgrhalvvbgbbjg + Ip: 192.168.1.243 + NodeIP: 192.168.1.243 + Ready: true + Restarts: 0 + PodAge: 13h + ContainerAge: 13h + Containers: + - Name: sglang + Image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + Ready: true + State: "" + Init: false + Restarts: 0 + MonitorUrl: http://125.93.200.207:31008/d/pod-gpu-resource?from=now-1h&kiosk=true&orgId=1&refresh=10s&timezone=browser&to=now&var-datasource=default&var-namespace=prj-d7te67bj0sq2fgdrmbig&var-pod=ds41-gist-exp-d-0 + Resources: + Requests: + Cpu: 32 + Memory: 256 + Gpu: + nvidia.com/gpu: 4 + Limits: + Cpu: 96 + Memory: 550 + Gpu: + nvidia.com/gpu: 4 + - Name: ds41-gist-exp-gateway-0 + Status: Running + StartTime: 1794832538 + FinishedAt: 0 + Node: b300-01 + NodeId: nd-damg5gkgrhalvvbgbbjg + Ip: 192.168.1.243 + NodeIP: 192.168.1.243 + Ready: true + Restarts: 0 + PodAge: 74m + ContainerAge: 74m + Containers: + - Name: gateway + Image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + Ready: true + State: "" + Init: false + Restarts: 0 + MonitorUrl: http://125.93.200.207:31008/d/pod-gpu-resource?from=now-1h&kiosk=true&orgId=1&refresh=10s&timezone=browser&to=now&var-datasource=default&var-namespace=prj-d7te67bj0sq2fgdrmbig&var-pod=ds41-gist-exp-gateway-0 + Resources: + Requests: + Cpu: 2 + Memory: 8 + Limits: + Cpu: 16 + Memory: 32 + - Name: ds41-gist-exp-mooncake-master-0 + Status: Running + StartTime: 1794788206 + FinishedAt: 0 + Node: b300-01 + NodeId: nd-damg5gkgrhalvvbgbbjg + Ip: 192.168.1.243 + NodeIP: 192.168.1.243 + Ready: true + Restarts: 0 + PodAge: 13h + ContainerAge: 13h + Containers: + - Name: mooncake-master + Image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + Ready: true + State: "" + Init: false + Restarts: 0 + MonitorUrl: http://125.93.200.207:31008/d/pod-gpu-resource?from=now-1h&kiosk=true&orgId=1&refresh=10s&timezone=browser&to=now&var-datasource=default&var-namespace=prj-d7te67bj0sq2fgdrmbig&var-pod=ds41-gist-exp-mooncake-master-0 + Resources: + Requests: + Cpu: 1 + Memory: 8 + Limits: + Cpu: 4 + Memory: 64 + - Name: ds41-gist-exp-p-0 + Status: Running + StartTime: 1794788212 + FinishedAt: 0 + Node: b300-01 + NodeId: nd-damg5gkgrhalvvbgbbjg + Ip: 192.168.1.243 + NodeIP: 192.168.1.243 + Ready: true + Restarts: 0 + PodAge: 13h + ContainerAge: 13h + Containers: + - Name: sglang + Image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + Ready: true + State: "" + Init: false + Restarts: 0 + MonitorUrl: http://125.93.200.207:31008/d/pod-gpu-resource?from=now-1h&kiosk=true&orgId=1&refresh=10s&timezone=browser&to=now&var-datasource=default&var-namespace=prj-d7te67bj0sq2fgdrmbig&var-pod=ds41-gist-exp-p-0 + Resources: + Requests: + Cpu: 32 + Memory: 512 + Gpu: + nvidia.com/gpu: 4 + Limits: + Cpu: 96 + Memory: 2048 + Gpu: + nvidia.com/gpu: 4 diff --git a/deploy/b300-k8s/snapshots/20261116-pre-optimistic/rbg-live-raw.yaml b/deploy/b300-k8s/snapshots/20261116-pre-optimistic/rbg-live-raw.yaml new file mode 100644 index 0000000000..633c5c8fee --- /dev/null +++ b/deploy/b300-k8s/snapshots/20261116-pre-optimistic/rbg-live-raw.yaml @@ -0,0 +1,608 @@ +apiVersion: workloads.x-k8s.io/v1alpha2 +kind: RoleBasedGroup +metadata: + annotations: + rbg.workloads.x-k8s.io/discovery-config-mode: refine + creationTimestamp: "2026-11-16T00:16:46Z" + generation: 9 + labels: + app: ds41-gist-exp + kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571 + kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7 + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0 + kongming.io/resource-type: custom-workload + kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0 + sglang-model: ds-v41-flash + name: ds41-gist-exp + namespace: prj-d7te67bj0sq2fgdrmbig + resourceVersion: "30735810" + uid: cc94a410-abf2-41ec-b711-52f7c87598e1 +spec: + roles: + - minReadySeconds: 0 + name: mooncake-master + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "19003" + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-mooncake + kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571 + kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7 + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0 + kongming.io/resource-type: custom-workload + kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0 + spec: + containers: + - command: + - /bin/bash + - -c + - | + # 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用; + # 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。 + exec /opt/sglang/bin/mooncake_master \ + --port 50051 --enable_http_metadata_server=true \ + --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false + env: + - name: TMPDIR + value: /data/ymk/l3ssd/master-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/master-xdg + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: mooncake-master + ports: + - containerPort: 19003 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + initialDelaySeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 50051 + resources: + limits: + cpu: "4" + memory: 64Gi + requests: + cpu: "1" + memory: 8Gi + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: p + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "30000" + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-p + kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571 + kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7 + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0 + kongming.io/resource-type: custom-workload + kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0 + spec: + containers: + - command: + - /bin/bash + - -c + - | + exec sglang serve \ + --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \ + --reasoning-parser auto --tool-call-parser auto \ + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \ + --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \ + --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \ + --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \ + --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \ + --mem-fraction-static 0.80 \ + --chunked-prefill-size 131072 --max-prefill-tokens 131072 \ + --max-running-requests 512 --tokenizer-worker-num 16 \ + --load-balance-method total_tokens \ + --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \ + --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \ + --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \ + --cuda-graph-backend-prefill breakable \ + --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \ + --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \ + --enable-hierarchical-cache --hicache-ratio 4 \ + --hicache-mem-layout page_first_direct --hicache-io-backend direct \ + --hicache-write-policy write_through --hicache-storage-backend mooncake \ + --hicache-storage-prefetch-policy wait_complete \ + --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \ + --disaggregation-bootstrap-port 8998 \ + --expert-balancedness-report-mode prometheus \ + --expert-distribution-recorder-buffer-size 64 \ + --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \ + --enable-trace --trace-modules request,mooncake \ + --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \ + --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast + env: + - name: CUDA_VISIBLE_DEVICES + value: 0,1,2,3 + - name: SGLANG_LOG_MS + value: "1" + - name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS + value: "1" + - name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL + value: "1" + - name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "65536" + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DP_USE_GATHERV + value: "1" + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION + value: "0" + - name: SGLANG_NUMA_BIND_V2 + value: "0" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: SGLANG_TIMEOUT_KEEP_ALIVE + value: "900" + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: "1" + - name: MC_INTRANODE_NVLINK + value: "true" + - name: MC_INTRA_NVLINK + value: "true" + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: "true" + - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH + value: /data/ymk/ds41/mooncake-store-l3ssd.json + - name: SGLANG_HICACHE_PREFETCH_IO_THREADS + value: "4" + - name: SGLANG_HICACHE_BACKUP_IO_THREADS + value: "4" + - name: SGLANG_HICACHE_TIMING + value: "1" + - name: SGLANG_BATCH_TELEMETRY + value: "1" + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: "1" + - name: SGLANG_ENABLE_METRICS_DEVICE_TIMER + value: "1" + - name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL + value: "100" + - name: SGLANG_MM_TRACE + value: "1" + - name: SGLANG_BATCH_SPAN_MIN_MS + value: "200" + - name: SGLANG_BATCH_SPAN_SAMPLE + value: "0.05" + - name: SGLANG_BATCH_SPAN_MAX_LINKS + value: "8" + - name: SGLANG_BATCH_STAGE_TIMING + value: "1" + - name: SGLANG_BATCH_PASS_SPAN + value: "1" + - name: SGLANG_BATCH_PASS_SPAN_SAMPLE + value: "1.0" + - name: SGLANG_RECV_SPAN + value: "1" + - name: SGLANG_RECV_SPAN_SAMPLE + value: "1.0" + - name: SGLANG_MM_TRACE_SYNC + value: "0" + - name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK + value: "1" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "600" + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: "16" + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: "32" + - name: TZ + value: Asia/Shanghai + - name: SGLANG_TRACE_LOOP_SPAN + value: "1" + - name: SGLANG_TRACE_LOOP_SAMPLE + value: "1.0" + - name: SGLANG_FWD_TIMELINE + value: "0" + - name: SGLANG_TRACE_LOOP_DETAIL + value: "1" + - name: SGLANG_TRACE_MOONCAKE_DETAIL + value: "0" + - name: SGLANG_TRACE_BACKUP_DETAIL + value: "1" + - name: SGLANG_ASYNC_BACKUP_SUBMIT + value: "1" + - name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH + value: "1" + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: "1024" + - name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE + value: "9400" + - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR + value: bucket_storage_backend + - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE + value: "1099511627776" + - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES + value: "1099511627776" + - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY + value: fifo + - name: TMPDIR + value: /data/ymk/l3ssd/p-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/p-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/p-hf + - name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS + value: "0" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30000 + name: metrics + protocol: TCP + - containerPort: 8998 + name: bootstrap + protocol: TCP + - containerPort: 9400 + name: mc-0 + protocol: TCP + - containerPort: 9401 + name: mc-1 + protocol: TCP + - containerPort: 9402 + name: mc-2 + protocol: TCP + - containerPort: 9403 + name: mc-3 + protocol: TCP + readinessProbe: + failureThreshold: 120 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: "96" + memory: 2Ti + nvidia.com/gpu: "4" + requests: + cpu: "32" + memory: 512Gi + nvidia.com/gpu: "4" + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: d + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "30300" + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-d + kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571 + kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7 + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0 + kongming.io/resource-type: custom-workload + kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0 + spec: + containers: + - command: + - /bin/bash + - -c + - | + exec sglang serve \ + --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \ + --reasoning-parser auto --tool-call-parser auto \ + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \ + --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \ + --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \ + --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \ + --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \ + --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \ + --mem-fraction-static 0.80 \ + --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \ + --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \ + --max-running-requests 512 --cuda-graph-max-bs-decode 512 \ + --chunked-prefill-size 32768 --max-prefill-tokens 8192 \ + --disaggregation-mode decode --disaggregation-transfer-backend mooncake \ + --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \ + --enable-trace --trace-modules request,mooncake \ + --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \ + --watchdog-timeout 3600 + env: + - name: CUDA_VISIBLE_DEVICES + value: 4,5,6,7 + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: "1" + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: MC_INTRANODE_NVLINK + value: "true" + - name: MC_INTRA_NVLINK + value: "true" + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: "true" + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: "1" + - name: SGLANG_BATCH_TELEMETRY + value: "1" + - name: SGLANG_BATCH_SPAN_SAMPLE + value: "0.1" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "600" + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: "16" + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: "32" + - name: TZ + value: Asia/Shanghai + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: "1024" + - name: TMPDIR + value: /data/ymk/l3ssd/d-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/d-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/d-hf + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30300 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 150 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: "96" + memory: 550Gi + nvidia.com/gpu: "4" + requests: + cpu: "32" + memory: 256Gi + nvidia.com/gpu: "4" + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - p + - d + minReadySeconds: 0 + name: gateway + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "29000" + prometheus.io/scheme: http + prometheus.io/scrape: "true" + labels: + app: ds41-l3ssd-gw + kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571 + kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7 + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0 + kongming.io/resource-type: custom-workload + kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0 + spec: + containers: + - command: + - /bin/bash + - -c + - | + exec /opt/sglang/bin/sgl-model-gateway launch \ + --pd-disaggregation \ + --prefill http://127.0.0.1:30000 8998 \ + --decode http://127.0.0.1:30300 \ + --prefill-policy cache_aware --decode-policy cache_aware \ + --g9i-enabled --g9i-max-new-tokens 4096 --g9i-est-td-enabled --g9i-shadow-enabled --g9i-td-admit enforce --g9i-p-queue-reqs-cap 64 --g9i-p-queue-token-cap 32000000 --g9i-p-qps 40 --g9i-p-refresh-ms 250 --g9i-window-fixed 256 \ + \ + \ + --g9i-regular-p-queue-reject-secs 0 \ + --g9i-decode-max-running 512 \ + --pool-idle-timeout-secs 2 --pool-max-idle-per-host 0 \ + --health-check-timeout-secs 30 --health-check-interval-secs 30 \ + --cb-failure-threshold 50 \ + --model-path /data/models/DeepSeek-V4.1-Flash-1017 \ + --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \ + --host 0.0.0.0 --port 30400 + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: gateway + ports: + - containerPort: 30400 + name: http + protocol: TCP + - containerPort: 29000 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: "16" + memory: 32Gi + requests: + cpu: "2" + memory: 8Gi + volumeMounts: + - mountPath: /data + name: data + - mountPath: /opt/sglang/bin/sgl-model-gateway + name: gw-bin + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/gw-ns-est-td/sgl-model-gateway + type: File + name: gw-bin +status: + conditions: + - lastTransitionTime: "2026-11-16T12:36:12Z" + message: All roles are ready + observedGeneration: 9 + reason: AllRolesReady + status: "True" + type: Ready + observedGeneration: 9 + roleStatuses: + - name: mooncake-master + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 + - name: p + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 + - name: d + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 + - name: gateway + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 diff --git a/deploy/b300-k8s/snapshots/20261116-pre-optimistic/rbg-live.yaml b/deploy/b300-k8s/snapshots/20261116-pre-optimistic/rbg-live.yaml new file mode 100644 index 0000000000..633c5c8fee --- /dev/null +++ b/deploy/b300-k8s/snapshots/20261116-pre-optimistic/rbg-live.yaml @@ -0,0 +1,608 @@ +apiVersion: workloads.x-k8s.io/v1alpha2 +kind: RoleBasedGroup +metadata: + annotations: + rbg.workloads.x-k8s.io/discovery-config-mode: refine + creationTimestamp: "2026-11-16T00:16:46Z" + generation: 9 + labels: + app: ds41-gist-exp + kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571 + kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7 + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0 + kongming.io/resource-type: custom-workload + kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0 + sglang-model: ds-v41-flash + name: ds41-gist-exp + namespace: prj-d7te67bj0sq2fgdrmbig + resourceVersion: "30735810" + uid: cc94a410-abf2-41ec-b711-52f7c87598e1 +spec: + roles: + - minReadySeconds: 0 + name: mooncake-master + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "19003" + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-mooncake + kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571 + kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7 + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0 + kongming.io/resource-type: custom-workload + kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0 + spec: + containers: + - command: + - /bin/bash + - -c + - | + # 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用; + # 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。 + exec /opt/sglang/bin/mooncake_master \ + --port 50051 --enable_http_metadata_server=true \ + --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false + env: + - name: TMPDIR + value: /data/ymk/l3ssd/master-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/master-xdg + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: mooncake-master + ports: + - containerPort: 19003 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + initialDelaySeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 50051 + resources: + limits: + cpu: "4" + memory: 64Gi + requests: + cpu: "1" + memory: 8Gi + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: p + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "30000" + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-p + kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571 + kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7 + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0 + kongming.io/resource-type: custom-workload + kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0 + spec: + containers: + - command: + - /bin/bash + - -c + - | + exec sglang serve \ + --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \ + --reasoning-parser auto --tool-call-parser auto \ + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \ + --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \ + --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \ + --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \ + --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \ + --mem-fraction-static 0.80 \ + --chunked-prefill-size 131072 --max-prefill-tokens 131072 \ + --max-running-requests 512 --tokenizer-worker-num 16 \ + --load-balance-method total_tokens \ + --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \ + --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \ + --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \ + --cuda-graph-backend-prefill breakable \ + --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \ + --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \ + --enable-hierarchical-cache --hicache-ratio 4 \ + --hicache-mem-layout page_first_direct --hicache-io-backend direct \ + --hicache-write-policy write_through --hicache-storage-backend mooncake \ + --hicache-storage-prefetch-policy wait_complete \ + --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \ + --disaggregation-bootstrap-port 8998 \ + --expert-balancedness-report-mode prometheus \ + --expert-distribution-recorder-buffer-size 64 \ + --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \ + --enable-trace --trace-modules request,mooncake \ + --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \ + --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast + env: + - name: CUDA_VISIBLE_DEVICES + value: 0,1,2,3 + - name: SGLANG_LOG_MS + value: "1" + - name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS + value: "1" + - name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL + value: "1" + - name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "65536" + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DP_USE_GATHERV + value: "1" + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION + value: "0" + - name: SGLANG_NUMA_BIND_V2 + value: "0" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: SGLANG_TIMEOUT_KEEP_ALIVE + value: "900" + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: "1" + - name: MC_INTRANODE_NVLINK + value: "true" + - name: MC_INTRA_NVLINK + value: "true" + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: "true" + - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH + value: /data/ymk/ds41/mooncake-store-l3ssd.json + - name: SGLANG_HICACHE_PREFETCH_IO_THREADS + value: "4" + - name: SGLANG_HICACHE_BACKUP_IO_THREADS + value: "4" + - name: SGLANG_HICACHE_TIMING + value: "1" + - name: SGLANG_BATCH_TELEMETRY + value: "1" + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: "1" + - name: SGLANG_ENABLE_METRICS_DEVICE_TIMER + value: "1" + - name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL + value: "100" + - name: SGLANG_MM_TRACE + value: "1" + - name: SGLANG_BATCH_SPAN_MIN_MS + value: "200" + - name: SGLANG_BATCH_SPAN_SAMPLE + value: "0.05" + - name: SGLANG_BATCH_SPAN_MAX_LINKS + value: "8" + - name: SGLANG_BATCH_STAGE_TIMING + value: "1" + - name: SGLANG_BATCH_PASS_SPAN + value: "1" + - name: SGLANG_BATCH_PASS_SPAN_SAMPLE + value: "1.0" + - name: SGLANG_RECV_SPAN + value: "1" + - name: SGLANG_RECV_SPAN_SAMPLE + value: "1.0" + - name: SGLANG_MM_TRACE_SYNC + value: "0" + - name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK + value: "1" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "600" + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: "16" + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: "32" + - name: TZ + value: Asia/Shanghai + - name: SGLANG_TRACE_LOOP_SPAN + value: "1" + - name: SGLANG_TRACE_LOOP_SAMPLE + value: "1.0" + - name: SGLANG_FWD_TIMELINE + value: "0" + - name: SGLANG_TRACE_LOOP_DETAIL + value: "1" + - name: SGLANG_TRACE_MOONCAKE_DETAIL + value: "0" + - name: SGLANG_TRACE_BACKUP_DETAIL + value: "1" + - name: SGLANG_ASYNC_BACKUP_SUBMIT + value: "1" + - name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH + value: "1" + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: "1024" + - name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE + value: "9400" + - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR + value: bucket_storage_backend + - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE + value: "1099511627776" + - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES + value: "1099511627776" + - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY + value: fifo + - name: TMPDIR + value: /data/ymk/l3ssd/p-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/p-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/p-hf + - name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS + value: "0" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30000 + name: metrics + protocol: TCP + - containerPort: 8998 + name: bootstrap + protocol: TCP + - containerPort: 9400 + name: mc-0 + protocol: TCP + - containerPort: 9401 + name: mc-1 + protocol: TCP + - containerPort: 9402 + name: mc-2 + protocol: TCP + - containerPort: 9403 + name: mc-3 + protocol: TCP + readinessProbe: + failureThreshold: 120 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: "96" + memory: 2Ti + nvidia.com/gpu: "4" + requests: + cpu: "32" + memory: 512Gi + nvidia.com/gpu: "4" + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: d + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "30300" + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-d + kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571 + kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7 + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0 + kongming.io/resource-type: custom-workload + kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0 + spec: + containers: + - command: + - /bin/bash + - -c + - | + exec sglang serve \ + --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \ + --reasoning-parser auto --tool-call-parser auto \ + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \ + --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \ + --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \ + --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \ + --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \ + --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \ + --mem-fraction-static 0.80 \ + --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \ + --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \ + --max-running-requests 512 --cuda-graph-max-bs-decode 512 \ + --chunked-prefill-size 32768 --max-prefill-tokens 8192 \ + --disaggregation-mode decode --disaggregation-transfer-backend mooncake \ + --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \ + --enable-trace --trace-modules request,mooncake \ + --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \ + --watchdog-timeout 3600 + env: + - name: CUDA_VISIBLE_DEVICES + value: 4,5,6,7 + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: "1" + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: MC_INTRANODE_NVLINK + value: "true" + - name: MC_INTRA_NVLINK + value: "true" + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: "true" + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: "1" + - name: SGLANG_BATCH_TELEMETRY + value: "1" + - name: SGLANG_BATCH_SPAN_SAMPLE + value: "0.1" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "600" + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: "16" + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: "32" + - name: TZ + value: Asia/Shanghai + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: "1024" + - name: TMPDIR + value: /data/ymk/l3ssd/d-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/d-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/d-hf + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30300 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 150 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: "96" + memory: 550Gi + nvidia.com/gpu: "4" + requests: + cpu: "32" + memory: 256Gi + nvidia.com/gpu: "4" + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - p + - d + minReadySeconds: 0 + name: gateway + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "29000" + prometheus.io/scheme: http + prometheus.io/scrape: "true" + labels: + app: ds41-l3ssd-gw + kongming.io/history-revision: hrev-0d055a5880fbf263aac7967a2d38378d47209571 + kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7 + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0 + kongming.io/resource-type: custom-workload + kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0 + spec: + containers: + - command: + - /bin/bash + - -c + - | + exec /opt/sglang/bin/sgl-model-gateway launch \ + --pd-disaggregation \ + --prefill http://127.0.0.1:30000 8998 \ + --decode http://127.0.0.1:30300 \ + --prefill-policy cache_aware --decode-policy cache_aware \ + --g9i-enabled --g9i-max-new-tokens 4096 --g9i-est-td-enabled --g9i-shadow-enabled --g9i-td-admit enforce --g9i-p-queue-reqs-cap 64 --g9i-p-queue-token-cap 32000000 --g9i-p-qps 40 --g9i-p-refresh-ms 250 --g9i-window-fixed 256 \ + \ + \ + --g9i-regular-p-queue-reject-secs 0 \ + --g9i-decode-max-running 512 \ + --pool-idle-timeout-secs 2 --pool-max-idle-per-host 0 \ + --health-check-timeout-secs 30 --health-check-interval-secs 30 \ + --cb-failure-threshold 50 \ + --model-path /data/models/DeepSeek-V4.1-Flash-1017 \ + --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \ + --host 0.0.0.0 --port 30400 + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: gateway + ports: + - containerPort: 30400 + name: http + protocol: TCP + - containerPort: 29000 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: "16" + memory: 32Gi + requests: + cpu: "2" + memory: 8Gi + volumeMounts: + - mountPath: /data + name: data + - mountPath: /opt/sglang/bin/sgl-model-gateway + name: gw-bin + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/gw-ns-est-td/sgl-model-gateway + type: File + name: gw-bin +status: + conditions: + - lastTransitionTime: "2026-11-16T12:36:12Z" + message: All roles are ready + observedGeneration: 9 + reason: AllRolesReady + status: "True" + type: Ready + observedGeneration: 9 + roleStatuses: + - name: mooncake-master + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 + - name: p + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 + - name: d + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 + - name: gateway + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1