From 310ad35430a98c24f67d2a74f8d84eb061139c75 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Tue, 17 Nov 2026 20:05:11 +0800 Subject: [PATCH] deploy: record 2026-11-17 c580 baseline (D16+fastokens) and experiments --- .../20261117-dtok16-fastokens/README.md | 10 + .../20261117-dtok16-fastokens/manifest.yaml | 488 ++++++++++++++ .../20261117-dtok16-fastokens/summary.json | 1 + ...-20261117-bufferonly-dtok16-fastokens.yaml | 488 ++++++++++++++ .../ds41-gist-exp-20261117-cache-nolimit.yaml | 488 ++++++++++++++ ...41-gist-exp-20261117-dtok16-fastokens.yaml | 488 ++++++++++++++ .../ds41-gist-exp-20261117-dtok16-hf.yaml | 488 ++++++++++++++ deploy/b300-k8s/feishu-baseline-20261117.xml | 62 ++ .../snapshots/20261117-cache-nolimit/NOTES.md | 13 + .../20261117-cache-nolimit/manifest.yaml | 488 ++++++++++++++ .../20261117-cache-nolimit/rbg-live.yaml | 608 ++++++++++++++++++ 11 files changed, 3622 insertions(+) create mode 100644 deploy/b300-k8s/baselines/20261117-dtok16-fastokens/README.md create mode 100644 deploy/b300-k8s/baselines/20261117-dtok16-fastokens/manifest.yaml create mode 100644 deploy/b300-k8s/baselines/20261117-dtok16-fastokens/summary.json create mode 100644 deploy/b300-k8s/ds41-gist-exp-20261117-bufferonly-dtok16-fastokens.yaml create mode 100644 deploy/b300-k8s/ds41-gist-exp-20261117-cache-nolimit.yaml create mode 100644 deploy/b300-k8s/ds41-gist-exp-20261117-dtok16-fastokens.yaml create mode 100644 deploy/b300-k8s/ds41-gist-exp-20261117-dtok16-hf.yaml create mode 100644 deploy/b300-k8s/feishu-baseline-20261117.xml create mode 100644 deploy/b300-k8s/snapshots/20261117-cache-nolimit/NOTES.md create mode 100644 deploy/b300-k8s/snapshots/20261117-cache-nolimit/manifest.yaml create mode 100644 deploy/b300-k8s/snapshots/20261117-cache-nolimit/rbg-live.yaml diff --git a/deploy/b300-k8s/baselines/20261117-dtok16-fastokens/README.md b/deploy/b300-k8s/baselines/20261117-dtok16-fastokens/README.md new file mode 100644 index 0000000000..77187fd5a3 --- /dev/null +++ b/deploy/b300-k8s/baselines/20261117-dtok16-fastokens/README.md @@ -0,0 +1,10 @@ +# 2026-11-17 c580 基线:D tokenizer 16 + fastokens + +- manifest: `manifest.yaml`(= `ds41-gist-exp-20261117-dtok16-fastokens.yaml`) +- P: cache / 无 optimistic / tokenizer-worker 16 / fastokens / hicache ratio 4 / L3 SSD 4T +- D: tokenizer-worker 16 / fastokens +- Gateway: 正常(无 P admission cap,health/circuit 开启,decode-max-running 512) +- L3:每轮前清空;完整重启 +- c580 结果:rps 24.045,in 2.290M tok/s,out 22.449k tok/s, + TTFT 1.743/3.772/6.580 s,ITL p95 10.74 ms,29,816 req,valid=true。 +- 原始结果:`experiments/agentx-align/results-c580-dtok16-fastokens-20261117/` diff --git a/deploy/b300-k8s/baselines/20261117-dtok16-fastokens/manifest.yaml b/deploy/b300-k8s/baselines/20261117-dtok16-fastokens/manifest.yaml new file mode 100644 index 0000000000..14e6cfa1f0 --- /dev/null +++ b/deploy/b300-k8s/baselines/20261117-dtok16-fastokens/manifest.yaml @@ -0,0 +1,488 @@ +apiVersion: workloads.x-k8s.io/v1alpha2 +kind: RoleBasedGroup +metadata: + annotations: + rbg.workloads.x-k8s.io/discovery-config-mode: refine + labels: + app: ds41-gist-exp + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + sglang-model: ds-v41-flash + name: ds41-gist-exp + namespace: prj-d7te67bj0sq2fgdrmbig +spec: + roles: + - minReadySeconds: 0 + name: mooncake-master + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '19003' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-mooncake + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: mooncake-master + ports: + - containerPort: 19003 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + initialDelaySeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 50051 + resources: + limits: + cpu: '4' + memory: 64Gi + requests: + cpu: '1' + memory: 8Gi + env: + - name: TMPDIR + value: /data/ymk/l3ssd/master-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/master-xdg + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: p + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30000' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-p + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 0,1,2,3 + - name: SGLANG_LOG_MS + value: '1' + - name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS + value: '1' + - name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL + value: '1' + - name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS + value: '1' + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: '65536' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DP_USE_GATHERV + value: '1' + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION + value: '0' + - name: SGLANG_NUMA_BIND_V2 + value: '0' + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: SGLANG_TIMEOUT_KEEP_ALIVE + value: '900' + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH + value: /data/ymk/ds41/mooncake-store-l3ssd.json + - name: SGLANG_HICACHE_PREFETCH_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_BACKUP_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_TIMING + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_ENABLE_METRICS_DEVICE_TIMER + value: '1' + - name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL + value: '100' + - name: SGLANG_MM_TRACE + value: '1' + - name: SGLANG_BATCH_SPAN_MIN_MS + value: '200' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.05' + - name: SGLANG_BATCH_SPAN_MAX_LINKS + value: '8' + - name: SGLANG_BATCH_STAGE_TIMING + value: '1' + - name: SGLANG_BATCH_PASS_SPAN + value: '1' + - name: SGLANG_BATCH_PASS_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_RECV_SPAN + value: '1' + - name: SGLANG_RECV_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_MM_TRACE_SYNC + value: '0' + - name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK + value: '1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: SGLANG_TRACE_LOOP_SPAN + value: '1' + - name: SGLANG_TRACE_LOOP_SAMPLE + value: '1.0' + - name: SGLANG_FWD_TIMELINE + value: '0' + - name: SGLANG_TRACE_LOOP_DETAIL + value: '1' + - name: SGLANG_TRACE_MOONCAKE_DETAIL + value: '0' + - name: SGLANG_TRACE_BACKUP_DETAIL + value: '1' + - name: SGLANG_ASYNC_BACKUP_SUBMIT + value: '1' + - name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH + value: '1' + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE + value: '9400' + - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR + value: bucket_storage_backend + - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY + value: fifo + - name: TMPDIR + value: /data/ymk/l3ssd/p-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/p-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/p-hf + - name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS + value: '0' + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30000 + name: metrics + protocol: TCP + - containerPort: 8998 + name: bootstrap + protocol: TCP + - name: mc-0 + containerPort: 9400 + protocol: TCP + - name: mc-1 + containerPort: 9401 + protocol: TCP + - name: mc-2 + containerPort: 9402 + protocol: TCP + - name: mc-3 + containerPort: 9403 + protocol: TCP + readinessProbe: + failureThreshold: 120 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 2Ti + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 512Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: d + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30300' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-d + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 --tokenizer-worker-num 16 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 4,5,6,7 + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: TMPDIR + value: /data/ymk/l3ssd/d-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/d-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/d-hf + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30300 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 150 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 550Gi + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 256Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - p + - d + minReadySeconds: 0 + name: gateway + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '29000' + prometheus.io/scheme: http + prometheus.io/scrape: 'true' + labels: + app: ds41-l3ssd-gw + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 512 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: gateway + ports: + - containerPort: 30400 + name: http + protocol: TCP + - containerPort: 29000 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '16' + memory: 32Gi + requests: + cpu: '2' + memory: 8Gi + volumeMounts: + - mountPath: /data + name: data + - mountPath: /opt/sglang/bin/sgl-model-gateway + name: gw-bin + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - name: gw-bin + hostPath: + path: /data/ymk/cache/gw-est-td/sgl-model-gateway + type: File diff --git a/deploy/b300-k8s/baselines/20261117-dtok16-fastokens/summary.json b/deploy/b300-k8s/baselines/20261117-dtok16-fastokens/summary.json new file mode 100644 index 0000000000..5823d9b0a1 --- /dev/null +++ b/deploy/b300-k8s/baselines/20261117-dtok16-fastokens/summary.json @@ -0,0 +1 @@ +{"rps": 24.04512015569562, "in_tput": 2290117.7612668574, "out_tput": 22449.32127276801, "ttft_p50": 1743.335193, "ttft_p95": 3771.88552025, "ttft_p99": 6579.86958935, "itl_p50": 8.162268671611512, "itl_p95": 10.741317697249674, "itl_p99": 11.806237803568294, "lat_p95": 31884.86991075, "requests": 29816.0, "isl_p50": 79909.0, "osl_p50": 339.0, "cache_pct": NaN, "valid": true} diff --git a/deploy/b300-k8s/ds41-gist-exp-20261117-bufferonly-dtok16-fastokens.yaml b/deploy/b300-k8s/ds41-gist-exp-20261117-bufferonly-dtok16-fastokens.yaml new file mode 100644 index 0000000000..e493a9c217 --- /dev/null +++ b/deploy/b300-k8s/ds41-gist-exp-20261117-bufferonly-dtok16-fastokens.yaml @@ -0,0 +1,488 @@ +apiVersion: workloads.x-k8s.io/v1alpha2 +kind: RoleBasedGroup +metadata: + annotations: + rbg.workloads.x-k8s.io/discovery-config-mode: refine + labels: + app: ds41-gist-exp + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + sglang-model: ds-v41-flash + name: ds41-gist-exp + namespace: prj-d7te67bj0sq2fgdrmbig +spec: + roles: + - minReadySeconds: 0 + name: mooncake-master + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '19003' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-mooncake + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: mooncake-master + ports: + - containerPort: 19003 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + initialDelaySeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 50051 + resources: + limits: + cpu: '4' + memory: 64Gi + requests: + cpu: '1' + memory: 8Gi + env: + - name: TMPDIR + value: /data/ymk/l3ssd/master-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/master-xdg + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: p + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30000' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-p + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 --hicache-host-memory-mode buffer_only \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 0,1,2,3 + - name: SGLANG_LOG_MS + value: '1' + - name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS + value: '1' + - name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL + value: '1' + - name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS + value: '1' + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: '65536' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DP_USE_GATHERV + value: '1' + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION + value: '0' + - name: SGLANG_NUMA_BIND_V2 + value: '0' + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: SGLANG_TIMEOUT_KEEP_ALIVE + value: '900' + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH + value: /data/ymk/ds41/mooncake-store-l3ssd.json + - name: SGLANG_HICACHE_PREFETCH_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_BACKUP_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_TIMING + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_ENABLE_METRICS_DEVICE_TIMER + value: '1' + - name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL + value: '100' + - name: SGLANG_MM_TRACE + value: '1' + - name: SGLANG_BATCH_SPAN_MIN_MS + value: '200' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.05' + - name: SGLANG_BATCH_SPAN_MAX_LINKS + value: '8' + - name: SGLANG_BATCH_STAGE_TIMING + value: '1' + - name: SGLANG_BATCH_PASS_SPAN + value: '1' + - name: SGLANG_BATCH_PASS_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_RECV_SPAN + value: '1' + - name: SGLANG_RECV_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_MM_TRACE_SYNC + value: '0' + - name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK + value: '1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: SGLANG_TRACE_LOOP_SPAN + value: '1' + - name: SGLANG_TRACE_LOOP_SAMPLE + value: '1.0' + - name: SGLANG_FWD_TIMELINE + value: '0' + - name: SGLANG_TRACE_LOOP_DETAIL + value: '1' + - name: SGLANG_TRACE_MOONCAKE_DETAIL + value: '0' + - name: SGLANG_TRACE_BACKUP_DETAIL + value: '1' + - name: SGLANG_ASYNC_BACKUP_SUBMIT + value: '1' + - name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH + value: '1' + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE + value: '9400' + - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR + value: bucket_storage_backend + - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY + value: fifo + - name: TMPDIR + value: /data/ymk/l3ssd/p-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/p-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/p-hf + - name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS + value: '0' + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30000 + name: metrics + protocol: TCP + - containerPort: 8998 + name: bootstrap + protocol: TCP + - name: mc-0 + containerPort: 9400 + protocol: TCP + - name: mc-1 + containerPort: 9401 + protocol: TCP + - name: mc-2 + containerPort: 9402 + protocol: TCP + - name: mc-3 + containerPort: 9403 + protocol: TCP + readinessProbe: + failureThreshold: 120 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 2Ti + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 512Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: d + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30300' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-d + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 --tokenizer-worker-num 16 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 4,5,6,7 + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: TMPDIR + value: /data/ymk/l3ssd/d-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/d-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/d-hf + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30300 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 150 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 550Gi + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 256Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - p + - d + minReadySeconds: 0 + name: gateway + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '29000' + prometheus.io/scheme: http + prometheus.io/scrape: 'true' + labels: + app: ds41-l3ssd-gw + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 512 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: gateway + ports: + - containerPort: 30400 + name: http + protocol: TCP + - containerPort: 29000 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '16' + memory: 32Gi + requests: + cpu: '2' + memory: 8Gi + volumeMounts: + - mountPath: /data + name: data + - mountPath: /opt/sglang/bin/sgl-model-gateway + name: gw-bin + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - name: gw-bin + hostPath: + path: /data/ymk/cache/gw-est-td/sgl-model-gateway + type: File diff --git a/deploy/b300-k8s/ds41-gist-exp-20261117-cache-nolimit.yaml b/deploy/b300-k8s/ds41-gist-exp-20261117-cache-nolimit.yaml new file mode 100644 index 0000000000..31cc39cba2 --- /dev/null +++ b/deploy/b300-k8s/ds41-gist-exp-20261117-cache-nolimit.yaml @@ -0,0 +1,488 @@ +apiVersion: workloads.x-k8s.io/v1alpha2 +kind: RoleBasedGroup +metadata: + annotations: + rbg.workloads.x-k8s.io/discovery-config-mode: refine + labels: + app: ds41-gist-exp + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + sglang-model: ds-v41-flash + name: ds41-gist-exp + namespace: prj-d7te67bj0sq2fgdrmbig +spec: + roles: + - minReadySeconds: 0 + name: mooncake-master + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '19003' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-mooncake + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n" + env: + - name: TMPDIR + value: /data/ymk/l3ssd/master-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/master-xdg + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: mooncake-master + ports: + - containerPort: 19003 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + initialDelaySeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 50051 + resources: + limits: + cpu: '4' + memory: 64Gi + requests: + cpu: '1' + memory: 8Gi + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: p + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30000' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-p + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 0,1,2,3 + - name: SGLANG_LOG_MS + value: '1' + - name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS + value: '1' + - name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL + value: '1' + - name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS + value: '1' + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: '65536' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DP_USE_GATHERV + value: '1' + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION + value: '0' + - name: SGLANG_NUMA_BIND_V2 + value: '0' + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: SGLANG_TIMEOUT_KEEP_ALIVE + value: '900' + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH + value: /data/ymk/ds41/mooncake-store-l3ssd.json + - name: SGLANG_HICACHE_PREFETCH_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_BACKUP_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_TIMING + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_ENABLE_METRICS_DEVICE_TIMER + value: '1' + - name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL + value: '100' + - name: SGLANG_MM_TRACE + value: '1' + - name: SGLANG_BATCH_SPAN_MIN_MS + value: '200' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.05' + - name: SGLANG_BATCH_SPAN_MAX_LINKS + value: '8' + - name: SGLANG_BATCH_STAGE_TIMING + value: '1' + - name: SGLANG_BATCH_PASS_SPAN + value: '1' + - name: SGLANG_BATCH_PASS_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_RECV_SPAN + value: '1' + - name: SGLANG_RECV_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_MM_TRACE_SYNC + value: '0' + - name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK + value: '1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: SGLANG_TRACE_LOOP_SPAN + value: '1' + - name: SGLANG_TRACE_LOOP_SAMPLE + value: '1.0' + - name: SGLANG_FWD_TIMELINE + value: '0' + - name: SGLANG_TRACE_LOOP_DETAIL + value: '1' + - name: SGLANG_TRACE_MOONCAKE_DETAIL + value: '0' + - name: SGLANG_TRACE_BACKUP_DETAIL + value: '1' + - name: SGLANG_ASYNC_BACKUP_SUBMIT + value: '1' + - name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH + value: '1' + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE + value: '9400' + - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR + value: bucket_storage_backend + - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY + value: fifo + - name: TMPDIR + value: /data/ymk/l3ssd/p-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/p-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/p-hf + - name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS + value: '0' + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30000 + name: metrics + protocol: TCP + - containerPort: 8998 + name: bootstrap + protocol: TCP + - containerPort: 9400 + name: mc-0 + protocol: TCP + - containerPort: 9401 + name: mc-1 + protocol: TCP + - containerPort: 9402 + name: mc-2 + protocol: TCP + - containerPort: 9403 + name: mc-3 + protocol: TCP + readinessProbe: + failureThreshold: 120 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 2Ti + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 512Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: d + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30300' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-d + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 --tokenizer-worker-num 16 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 4,5,6,7 + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: TMPDIR + value: /data/ymk/l3ssd/d-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/d-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/d-hf + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30300 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 150 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 550Gi + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 256Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - p + - d + minReadySeconds: 0 + name: gateway + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '29000' + prometheus.io/scheme: http + prometheus.io/scrape: 'true' + labels: + app: ds41-l3ssd-gw + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 --g9i-est-td-enabled --g9i-shadow-enabled --g9i-td-admit off --g9i-p-queue-reqs-cap 0 --g9i-p-queue-token-cap 0 --g9i-p-qps 0 --g9i-p-refresh-ms 250 --g9i-window-fixed 256 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 1000000 \\\n --pool-idle-timeout-secs 2 --pool-max-idle-per-host 0 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 --disable-circuit-breaker --disable-health-check \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: gateway + ports: + - containerPort: 30400 + name: http + protocol: TCP + - containerPort: 29000 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '16' + memory: 32Gi + requests: + cpu: '2' + memory: 8Gi + volumeMounts: + - mountPath: /data + name: data + - mountPath: /opt/sglang/bin/sgl-model-gateway + name: gw-bin + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/gw-ns-est-td/sgl-model-gateway + type: File + name: gw-bin diff --git a/deploy/b300-k8s/ds41-gist-exp-20261117-dtok16-fastokens.yaml b/deploy/b300-k8s/ds41-gist-exp-20261117-dtok16-fastokens.yaml new file mode 100644 index 0000000000..14e6cfa1f0 --- /dev/null +++ b/deploy/b300-k8s/ds41-gist-exp-20261117-dtok16-fastokens.yaml @@ -0,0 +1,488 @@ +apiVersion: workloads.x-k8s.io/v1alpha2 +kind: RoleBasedGroup +metadata: + annotations: + rbg.workloads.x-k8s.io/discovery-config-mode: refine + labels: + app: ds41-gist-exp + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + sglang-model: ds-v41-flash + name: ds41-gist-exp + namespace: prj-d7te67bj0sq2fgdrmbig +spec: + roles: + - minReadySeconds: 0 + name: mooncake-master + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '19003' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-mooncake + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: mooncake-master + ports: + - containerPort: 19003 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + initialDelaySeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 50051 + resources: + limits: + cpu: '4' + memory: 64Gi + requests: + cpu: '1' + memory: 8Gi + env: + - name: TMPDIR + value: /data/ymk/l3ssd/master-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/master-xdg + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: p + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30000' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-p + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 0,1,2,3 + - name: SGLANG_LOG_MS + value: '1' + - name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS + value: '1' + - name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL + value: '1' + - name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS + value: '1' + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: '65536' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DP_USE_GATHERV + value: '1' + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION + value: '0' + - name: SGLANG_NUMA_BIND_V2 + value: '0' + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: SGLANG_TIMEOUT_KEEP_ALIVE + value: '900' + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH + value: /data/ymk/ds41/mooncake-store-l3ssd.json + - name: SGLANG_HICACHE_PREFETCH_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_BACKUP_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_TIMING + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_ENABLE_METRICS_DEVICE_TIMER + value: '1' + - name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL + value: '100' + - name: SGLANG_MM_TRACE + value: '1' + - name: SGLANG_BATCH_SPAN_MIN_MS + value: '200' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.05' + - name: SGLANG_BATCH_SPAN_MAX_LINKS + value: '8' + - name: SGLANG_BATCH_STAGE_TIMING + value: '1' + - name: SGLANG_BATCH_PASS_SPAN + value: '1' + - name: SGLANG_BATCH_PASS_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_RECV_SPAN + value: '1' + - name: SGLANG_RECV_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_MM_TRACE_SYNC + value: '0' + - name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK + value: '1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: SGLANG_TRACE_LOOP_SPAN + value: '1' + - name: SGLANG_TRACE_LOOP_SAMPLE + value: '1.0' + - name: SGLANG_FWD_TIMELINE + value: '0' + - name: SGLANG_TRACE_LOOP_DETAIL + value: '1' + - name: SGLANG_TRACE_MOONCAKE_DETAIL + value: '0' + - name: SGLANG_TRACE_BACKUP_DETAIL + value: '1' + - name: SGLANG_ASYNC_BACKUP_SUBMIT + value: '1' + - name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH + value: '1' + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE + value: '9400' + - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR + value: bucket_storage_backend + - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY + value: fifo + - name: TMPDIR + value: /data/ymk/l3ssd/p-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/p-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/p-hf + - name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS + value: '0' + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30000 + name: metrics + protocol: TCP + - containerPort: 8998 + name: bootstrap + protocol: TCP + - name: mc-0 + containerPort: 9400 + protocol: TCP + - name: mc-1 + containerPort: 9401 + protocol: TCP + - name: mc-2 + containerPort: 9402 + protocol: TCP + - name: mc-3 + containerPort: 9403 + protocol: TCP + readinessProbe: + failureThreshold: 120 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 2Ti + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 512Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: d + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30300' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-d + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 --tokenizer-worker-num 16 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 4,5,6,7 + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: TMPDIR + value: /data/ymk/l3ssd/d-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/d-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/d-hf + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30300 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 150 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 550Gi + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 256Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - p + - d + minReadySeconds: 0 + name: gateway + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '29000' + prometheus.io/scheme: http + prometheus.io/scrape: 'true' + labels: + app: ds41-l3ssd-gw + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 512 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: gateway + ports: + - containerPort: 30400 + name: http + protocol: TCP + - containerPort: 29000 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '16' + memory: 32Gi + requests: + cpu: '2' + memory: 8Gi + volumeMounts: + - mountPath: /data + name: data + - mountPath: /opt/sglang/bin/sgl-model-gateway + name: gw-bin + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - name: gw-bin + hostPath: + path: /data/ymk/cache/gw-est-td/sgl-model-gateway + type: File diff --git a/deploy/b300-k8s/ds41-gist-exp-20261117-dtok16-hf.yaml b/deploy/b300-k8s/ds41-gist-exp-20261117-dtok16-hf.yaml new file mode 100644 index 0000000000..69ceb3a67f --- /dev/null +++ b/deploy/b300-k8s/ds41-gist-exp-20261117-dtok16-hf.yaml @@ -0,0 +1,488 @@ +apiVersion: workloads.x-k8s.io/v1alpha2 +kind: RoleBasedGroup +metadata: + annotations: + rbg.workloads.x-k8s.io/discovery-config-mode: refine + labels: + app: ds41-gist-exp + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + sglang-model: ds-v41-flash + name: ds41-gist-exp + namespace: prj-d7te67bj0sq2fgdrmbig +spec: + roles: + - minReadySeconds: 0 + name: mooncake-master + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '19003' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-mooncake + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: mooncake-master + ports: + - containerPort: 19003 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + initialDelaySeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 50051 + resources: + limits: + cpu: '4' + memory: 64Gi + requests: + cpu: '1' + memory: 8Gi + env: + - name: TMPDIR + value: /data/ymk/l3ssd/master-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/master-xdg + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: p + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30000' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-p + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend huggingface --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 0,1,2,3 + - name: SGLANG_LOG_MS + value: '1' + - name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS + value: '1' + - name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL + value: '1' + - name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS + value: '1' + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: '65536' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DP_USE_GATHERV + value: '1' + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION + value: '0' + - name: SGLANG_NUMA_BIND_V2 + value: '0' + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: SGLANG_TIMEOUT_KEEP_ALIVE + value: '900' + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH + value: /data/ymk/ds41/mooncake-store-l3ssd.json + - name: SGLANG_HICACHE_PREFETCH_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_BACKUP_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_TIMING + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_ENABLE_METRICS_DEVICE_TIMER + value: '1' + - name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL + value: '100' + - name: SGLANG_MM_TRACE + value: '1' + - name: SGLANG_BATCH_SPAN_MIN_MS + value: '200' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.05' + - name: SGLANG_BATCH_SPAN_MAX_LINKS + value: '8' + - name: SGLANG_BATCH_STAGE_TIMING + value: '1' + - name: SGLANG_BATCH_PASS_SPAN + value: '1' + - name: SGLANG_BATCH_PASS_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_RECV_SPAN + value: '1' + - name: SGLANG_RECV_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_MM_TRACE_SYNC + value: '0' + - name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK + value: '1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: SGLANG_TRACE_LOOP_SPAN + value: '1' + - name: SGLANG_TRACE_LOOP_SAMPLE + value: '1.0' + - name: SGLANG_FWD_TIMELINE + value: '0' + - name: SGLANG_TRACE_LOOP_DETAIL + value: '1' + - name: SGLANG_TRACE_MOONCAKE_DETAIL + value: '0' + - name: SGLANG_TRACE_BACKUP_DETAIL + value: '1' + - name: SGLANG_ASYNC_BACKUP_SUBMIT + value: '1' + - name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH + value: '1' + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE + value: '9400' + - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR + value: bucket_storage_backend + - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY + value: fifo + - name: TMPDIR + value: /data/ymk/l3ssd/p-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/p-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/p-hf + - name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS + value: '0' + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30000 + name: metrics + protocol: TCP + - containerPort: 8998 + name: bootstrap + protocol: TCP + - name: mc-0 + containerPort: 9400 + protocol: TCP + - name: mc-1 + containerPort: 9401 + protocol: TCP + - name: mc-2 + containerPort: 9402 + protocol: TCP + - name: mc-3 + containerPort: 9403 + protocol: TCP + readinessProbe: + failureThreshold: 120 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 2Ti + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 512Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: d + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30300' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-d + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend huggingface --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 --tokenizer-worker-num 16 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 4,5,6,7 + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: TMPDIR + value: /data/ymk/l3ssd/d-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/d-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/d-hf + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30300 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 150 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 550Gi + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 256Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - p + - d + minReadySeconds: 0 + name: gateway + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '29000' + prometheus.io/scheme: http + prometheus.io/scrape: 'true' + labels: + app: ds41-l3ssd-gw + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 512 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: gateway + ports: + - containerPort: 30400 + name: http + protocol: TCP + - containerPort: 29000 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '16' + memory: 32Gi + requests: + cpu: '2' + memory: 8Gi + volumeMounts: + - mountPath: /data + name: data + - mountPath: /opt/sglang/bin/sgl-model-gateway + name: gw-bin + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - name: gw-bin + hostPath: + path: /data/ymk/cache/gw-est-td/sgl-model-gateway + type: File diff --git a/deploy/b300-k8s/feishu-baseline-20261117.xml b/deploy/b300-k8s/feishu-baseline-20261117.xml new file mode 100644 index 0000000000..08cae57057 --- /dev/null +++ b/deploy/b300-k8s/feishu-baseline-20261117.xml @@ -0,0 +1,62 @@ +

当前基线:ds41-gist-exp(4T L3 SSD,2026-11-17)

+

本节是 2026-11-17 定稿的 c580/c620 基线,后续 Dynamo 叠加以此为底座。旧的 ds41-gist-1p1d 章节已归档,不再作为当前部署。

+ +

1. 基线身份与镜像

+ + +

2. 部署方式与端口

+ + + + + + + + +

角色

端口

说明

P

30000 / 8998 / 9400-9403

HTTP / bootstrap / Mooncake client metrics

D

30300

HTTP

gateway

30400 / 29000

业务 / metrics

mooncake-master

50051 / 18081 / 19003

store / metadata / metrics

+

默认入口:http://127.0.0.1:30400;当前 new-api/AgentX 直接使用该入口。

+ +

3. 关键参数

+

P(prefill):tp4 / dp4 / ep4,megamoe + flashmla_mega,ep-redundant 32,DSpark block 3,decoder SWA bounded replay;chunked-prefill 131072 / max-prefill 131072,max-running 512;fastokens + tokenizer-worker 16;HiCache ratio 4、page_first_direct、direct、write_through、Mooncake L3;cache 模式,无 buffer_only、无 optimistic;mem-fraction 0.80。

+

D(decode):tp4 / dp4 / ep4,ep-redundant 64,DSpark block 3,stream-interval 32,swa-prefix-tails 2048;max-running 512,cuda-graph-max-bs-decode 512;chunked 32768 / max-prefill 8192;fastokens + tokenizer-worker 16;mem-fraction 0.80。

+

gateway:镜像内 sgl-model-gateway;无 P admission cap(无 p-qps / p-queue-reqs-cap / p-queue-token-cap,td-admit 默认 off);保留 health check + circuit breaker;--g9i-decode-max-running 512。这是 c580 能跑通的关键。

+

L3:Mooncake master 4TB SSD offload;offload_on_evict=false;L3 配置见 /data/ymk/ds41/mooncake-store-l3ssd.json。

+ +

4. 启动 / 重启流程

+
# 1) 停完整工作负载并确认 GPU 0
+MSYS_NO_PATHCONV=1 wsl -e /home/ubuntu/.local/bin/kmctl stop cw ds41-gist-exp --wait --timeout 10m --request-timeout 180s
+ssh 24.125.93.200.207 "nvidia-smi --query-gpu=index,memory.used --format=csv,noheader"
+
+# 2) 清 L3 SSD(实验/基线前必须)
+ssh 24.125.93.200.207 'bash /data/ymk/deploy/b300-k8s/clean_l3ssd_cache.sh'
+
+# 3) 用完整平台 manifest 启动(不要只 kubectl patch gateway)
+MSYS_NO_PATHCONV=1 wsl -e /home/ubuntu/.local/bin/kmctl update cw ds41-gist-exp \
+  -f /mnt/d/B300/deploy/b300-k8s/ds41-gist-exp-20261117-dtok16-fastokens.yaml
+
+# 4) 等 4 Pod 1/1,检查 P/D/gateway health 200,再 smoke
+curl -sS http://127.0.0.1:30400/v1/chat/completions -H 'Content-Type: application/json' \
+  -d '{"model":"/data/models/DeepSeek-V4.1-Flash","messages":[{"role":"user","content":"1+1=?"}],"max_tokens":8}'
+
+

启动基线:P max_total_num_tokens≈16,158,208 / 128 per-rank / avail ≈23GB;D 58,859,520 / 128 per-rank / avail ≈79GB。正常 warmup 6,385 条约 6.5 分钟。

+ +

5. 基线结果

+ + + + + + + + +

场景

RPS

TTFT p50/p95/p99

ITL p95

结论

c580 基线

24.045

1.743 / 3.772 / 6.580 s

10.74 ms

推荐工作点

c620 首轮(清 L3)

25.998

1.814 / 4.041 / 8.257 s

11.69 ms

吞吐 +8%,尾部变差

c620 不重启 r2/r3

25.85 / 25.61

p95 5.39 / 6.44 s,p99 9.56 / 10.79 s

10.96 / 10.68 ms

D KV 接近/打满,TTFT 尾部持续劣化,不建议连测

buffer_only

23.749

2.153 / 4.904 / 10.483 s

10.32 ms

吞吐 -1.2%、TTFT 更差,不采用

+

结论:c580 是稳定基线;c620 只适合短时冲吞吐,且每轮之间必须重启/清池。buffer_only 不作为基线。

+ +

6. 作为 Dynamo 底座

+

当前镜像是 SGLang + Mooncake 0.3.14 + L3 SSD 基线,不包含 Dynamo/nixl。后续套 Dynamo 需要以本镜像为 base 另装 Dynamo 1.6.x / nixl,旧 gist-nvlink-7f6fe2244-dynamo-local-20261026 是更早的 10-23 代码,不是本基线。

+

Dynamo 叠加时保留本页的 P/D 参数、L3 SSD 和 gateway 入口;先做单机 4P4D 冒烟,再评估 Dynamo 的 P/D 角色、discovery 和 KV 传输配置。

diff --git a/deploy/b300-k8s/snapshots/20261117-cache-nolimit/NOTES.md b/deploy/b300-k8s/snapshots/20261117-cache-nolimit/NOTES.md new file mode 100644 index 0000000000..3fa13030c1 --- /dev/null +++ b/deploy/b300-k8s/snapshots/20261117-cache-nolimit/NOTES.md @@ -0,0 +1,13 @@ +# 2026-11-17 cache + gateway no-limit 实验参数(失败记录) + +- P: hicache_host_memory_mode=cache, optimistic_prefill_attempts=0, tokenizer-worker-num=16, + hicache ratio 4 / write_through / L3 mooncake, L3 SSD 4T. +- D: tokenizer-worker-num=16. +- Gateway: g9i-td-admit off, p-queue-reqs-cap 0, p-queue-token-cap 0, p-qps 0, + decode-max-running 1000000, disable-circuit-breaker, disable-health-check. +- Image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 +- c580 run c580-cache-nolimit-20261117: warmup 4,720/6,385 (74%), 02:33:32 失败 + (ServerDisconnected/warmup_failure;P/D/gateway 0 restart,无 503)。 +- 观察:P 单批 32k/23k chunk 正常,但每 rank 同时仅 1 个请求;gateway + smg_worker_requests_active 峰值 1,840,P queue 峰值 503,gateway 约 1 核。 +- 结论:该 no-limit/no-cb 网关配置不适合作为 c580 基线;仅作异常记录。 diff --git a/deploy/b300-k8s/snapshots/20261117-cache-nolimit/manifest.yaml b/deploy/b300-k8s/snapshots/20261117-cache-nolimit/manifest.yaml new file mode 100644 index 0000000000..31cc39cba2 --- /dev/null +++ b/deploy/b300-k8s/snapshots/20261117-cache-nolimit/manifest.yaml @@ -0,0 +1,488 @@ +apiVersion: workloads.x-k8s.io/v1alpha2 +kind: RoleBasedGroup +metadata: + annotations: + rbg.workloads.x-k8s.io/discovery-config-mode: refine + labels: + app: ds41-gist-exp + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + sglang-model: ds-v41-flash + name: ds41-gist-exp + namespace: prj-d7te67bj0sq2fgdrmbig +spec: + roles: + - minReadySeconds: 0 + name: mooncake-master + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '19003' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-mooncake + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n" + env: + - name: TMPDIR + value: /data/ymk/l3ssd/master-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/master-xdg + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: mooncake-master + ports: + - containerPort: 19003 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + initialDelaySeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 50051 + resources: + limits: + cpu: '4' + memory: 64Gi + requests: + cpu: '1' + memory: 8Gi + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: p + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30000' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-p + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 0,1,2,3 + - name: SGLANG_LOG_MS + value: '1' + - name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS + value: '1' + - name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL + value: '1' + - name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS + value: '1' + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: '65536' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DP_USE_GATHERV + value: '1' + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION + value: '0' + - name: SGLANG_NUMA_BIND_V2 + value: '0' + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: SGLANG_TIMEOUT_KEEP_ALIVE + value: '900' + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH + value: /data/ymk/ds41/mooncake-store-l3ssd.json + - name: SGLANG_HICACHE_PREFETCH_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_BACKUP_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_TIMING + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_ENABLE_METRICS_DEVICE_TIMER + value: '1' + - name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL + value: '100' + - name: SGLANG_MM_TRACE + value: '1' + - name: SGLANG_BATCH_SPAN_MIN_MS + value: '200' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.05' + - name: SGLANG_BATCH_SPAN_MAX_LINKS + value: '8' + - name: SGLANG_BATCH_STAGE_TIMING + value: '1' + - name: SGLANG_BATCH_PASS_SPAN + value: '1' + - name: SGLANG_BATCH_PASS_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_RECV_SPAN + value: '1' + - name: SGLANG_RECV_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_MM_TRACE_SYNC + value: '0' + - name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK + value: '1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: SGLANG_TRACE_LOOP_SPAN + value: '1' + - name: SGLANG_TRACE_LOOP_SAMPLE + value: '1.0' + - name: SGLANG_FWD_TIMELINE + value: '0' + - name: SGLANG_TRACE_LOOP_DETAIL + value: '1' + - name: SGLANG_TRACE_MOONCAKE_DETAIL + value: '0' + - name: SGLANG_TRACE_BACKUP_DETAIL + value: '1' + - name: SGLANG_ASYNC_BACKUP_SUBMIT + value: '1' + - name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH + value: '1' + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE + value: '9400' + - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR + value: bucket_storage_backend + - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY + value: fifo + - name: TMPDIR + value: /data/ymk/l3ssd/p-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/p-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/p-hf + - name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS + value: '0' + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30000 + name: metrics + protocol: TCP + - containerPort: 8998 + name: bootstrap + protocol: TCP + - containerPort: 9400 + name: mc-0 + protocol: TCP + - containerPort: 9401 + name: mc-1 + protocol: TCP + - containerPort: 9402 + name: mc-2 + protocol: TCP + - containerPort: 9403 + name: mc-3 + protocol: TCP + readinessProbe: + failureThreshold: 120 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 2Ti + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 512Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: d + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30300' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-d + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 --tokenizer-worker-num 16 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 4,5,6,7 + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: TMPDIR + value: /data/ymk/l3ssd/d-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/d-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/d-hf + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30300 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 150 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 550Gi + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 256Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - p + - d + minReadySeconds: 0 + name: gateway + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '29000' + prometheus.io/scheme: http + prometheus.io/scrape: 'true' + labels: + app: ds41-l3ssd-gw + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 --g9i-est-td-enabled --g9i-shadow-enabled --g9i-td-admit off --g9i-p-queue-reqs-cap 0 --g9i-p-queue-token-cap 0 --g9i-p-qps 0 --g9i-p-refresh-ms 250 --g9i-window-fixed 256 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 1000000 \\\n --pool-idle-timeout-secs 2 --pool-max-idle-per-host 0 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 --disable-circuit-breaker --disable-health-check \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: gateway + ports: + - containerPort: 30400 + name: http + protocol: TCP + - containerPort: 29000 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '16' + memory: 32Gi + requests: + cpu: '2' + memory: 8Gi + volumeMounts: + - mountPath: /data + name: data + - mountPath: /opt/sglang/bin/sgl-model-gateway + name: gw-bin + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/gw-ns-est-td/sgl-model-gateway + type: File + name: gw-bin diff --git a/deploy/b300-k8s/snapshots/20261117-cache-nolimit/rbg-live.yaml b/deploy/b300-k8s/snapshots/20261117-cache-nolimit/rbg-live.yaml new file mode 100644 index 0000000000..36ae143ff4 --- /dev/null +++ b/deploy/b300-k8s/snapshots/20261117-cache-nolimit/rbg-live.yaml @@ -0,0 +1,608 @@ +apiVersion: workloads.x-k8s.io/v1alpha2 +kind: RoleBasedGroup +metadata: + annotations: + rbg.workloads.x-k8s.io/discovery-config-mode: refine + creationTimestamp: "2026-11-16T13:54:16Z" + generation: 6 + labels: + app: ds41-gist-exp + kongming.io/history-revision: hrev-b90013fa6cc5a2a0e6515115d188135ea41e48b7 + kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7 + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0 + kongming.io/resource-type: custom-workload + kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0 + sglang-model: ds-v41-flash + name: ds41-gist-exp + namespace: prj-d7te67bj0sq2fgdrmbig + resourceVersion: "31040141" + uid: 242c2504-0c97-4414-ae60-8376f7d63de0 +spec: + roles: + - minReadySeconds: 0 + name: mooncake-master + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "19003" + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-mooncake + kongming.io/history-revision: hrev-b90013fa6cc5a2a0e6515115d188135ea41e48b7 + kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7 + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0 + kongming.io/resource-type: custom-workload + kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0 + spec: + containers: + - command: + - /bin/bash + - -c + - | + # 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用; + # 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。 + exec /opt/sglang/bin/mooncake_master \ + --port 50051 --enable_http_metadata_server=true \ + --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false + env: + - name: TMPDIR + value: /data/ymk/l3ssd/master-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/master-xdg + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: mooncake-master + ports: + - containerPort: 19003 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + initialDelaySeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 50051 + resources: + limits: + cpu: "4" + memory: 64Gi + requests: + cpu: "1" + memory: 8Gi + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: p + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "30000" + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-p + kongming.io/history-revision: hrev-b90013fa6cc5a2a0e6515115d188135ea41e48b7 + kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7 + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0 + kongming.io/resource-type: custom-workload + kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0 + spec: + containers: + - command: + - /bin/bash + - -c + - | + exec sglang serve \ + --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \ + --reasoning-parser auto --tool-call-parser auto \ + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \ + --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \ + --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \ + --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \ + --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \ + --mem-fraction-static 0.80 \ + --chunked-prefill-size 131072 --max-prefill-tokens 131072 \ + --max-running-requests 512 --tokenizer-worker-num 16 \ + --load-balance-method total_tokens \ + --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \ + --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \ + --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \ + --cuda-graph-backend-prefill breakable \ + --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \ + --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \ + --enable-hierarchical-cache --hicache-ratio 4 \ + --hicache-mem-layout page_first_direct --hicache-io-backend direct \ + --hicache-write-policy write_through --hicache-storage-backend mooncake \ + --hicache-storage-prefetch-policy wait_complete \ + --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \ + --disaggregation-bootstrap-port 8998 \ + --expert-balancedness-report-mode prometheus \ + --expert-distribution-recorder-buffer-size 64 \ + --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \ + --enable-trace --trace-modules request,mooncake \ + --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \ + --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast + env: + - name: CUDA_VISIBLE_DEVICES + value: 0,1,2,3 + - name: SGLANG_LOG_MS + value: "1" + - name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS + value: "1" + - name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL + value: "1" + - name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS + value: "1" + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: "65536" + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DP_USE_GATHERV + value: "1" + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION + value: "0" + - name: SGLANG_NUMA_BIND_V2 + value: "0" + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: SGLANG_TIMEOUT_KEEP_ALIVE + value: "900" + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: "1" + - name: MC_INTRANODE_NVLINK + value: "true" + - name: MC_INTRA_NVLINK + value: "true" + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: "true" + - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH + value: /data/ymk/ds41/mooncake-store-l3ssd.json + - name: SGLANG_HICACHE_PREFETCH_IO_THREADS + value: "4" + - name: SGLANG_HICACHE_BACKUP_IO_THREADS + value: "4" + - name: SGLANG_HICACHE_TIMING + value: "1" + - name: SGLANG_BATCH_TELEMETRY + value: "1" + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: "1" + - name: SGLANG_ENABLE_METRICS_DEVICE_TIMER + value: "1" + - name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL + value: "100" + - name: SGLANG_MM_TRACE + value: "1" + - name: SGLANG_BATCH_SPAN_MIN_MS + value: "200" + - name: SGLANG_BATCH_SPAN_SAMPLE + value: "0.05" + - name: SGLANG_BATCH_SPAN_MAX_LINKS + value: "8" + - name: SGLANG_BATCH_STAGE_TIMING + value: "1" + - name: SGLANG_BATCH_PASS_SPAN + value: "1" + - name: SGLANG_BATCH_PASS_SPAN_SAMPLE + value: "1.0" + - name: SGLANG_RECV_SPAN + value: "1" + - name: SGLANG_RECV_SPAN_SAMPLE + value: "1.0" + - name: SGLANG_MM_TRACE_SYNC + value: "0" + - name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK + value: "1" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "600" + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: "16" + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: "32" + - name: TZ + value: Asia/Shanghai + - name: SGLANG_TRACE_LOOP_SPAN + value: "1" + - name: SGLANG_TRACE_LOOP_SAMPLE + value: "1.0" + - name: SGLANG_FWD_TIMELINE + value: "0" + - name: SGLANG_TRACE_LOOP_DETAIL + value: "1" + - name: SGLANG_TRACE_MOONCAKE_DETAIL + value: "0" + - name: SGLANG_TRACE_BACKUP_DETAIL + value: "1" + - name: SGLANG_ASYNC_BACKUP_SUBMIT + value: "1" + - name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH + value: "1" + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: "1024" + - name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE + value: "9400" + - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR + value: bucket_storage_backend + - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE + value: "1099511627776" + - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES + value: "1099511627776" + - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY + value: fifo + - name: TMPDIR + value: /data/ymk/l3ssd/p-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/p-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/p-hf + - name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS + value: "0" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30000 + name: metrics + protocol: TCP + - containerPort: 8998 + name: bootstrap + protocol: TCP + - containerPort: 9400 + name: mc-0 + protocol: TCP + - containerPort: 9401 + name: mc-1 + protocol: TCP + - containerPort: 9402 + name: mc-2 + protocol: TCP + - containerPort: 9403 + name: mc-3 + protocol: TCP + readinessProbe: + failureThreshold: 120 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: "96" + memory: 2Ti + nvidia.com/gpu: "4" + requests: + cpu: "32" + memory: 512Gi + nvidia.com/gpu: "4" + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: d + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "30300" + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-d + kongming.io/history-revision: hrev-b90013fa6cc5a2a0e6515115d188135ea41e48b7 + kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7 + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0 + kongming.io/resource-type: custom-workload + kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0 + spec: + containers: + - command: + - /bin/bash + - -c + - | + exec sglang serve \ + --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \ + --reasoning-parser auto --tool-call-parser auto \ + --default-chat-template-kwargs '{"thinking": true, "reasoning_effort": "high"}' \ + --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \ + --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \ + --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \ + --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \ + --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \ + --mem-fraction-static 0.80 \ + --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \ + --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \ + --max-running-requests 512 --cuda-graph-max-bs-decode 512 --tokenizer-worker-num 16 \ + --chunked-prefill-size 32768 --max-prefill-tokens 8192 \ + --disaggregation-mode decode --disaggregation-transfer-backend mooncake \ + --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \ + --enable-trace --trace-modules request,mooncake \ + --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \ + --watchdog-timeout 3600 + env: + - name: CUDA_VISIBLE_DEVICES + value: 4,5,6,7 + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: "1" + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DEFAULT_THINKING + value: "1" + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: MC_INTRANODE_NVLINK + value: "true" + - name: MC_INTRA_NVLINK + value: "true" + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: "true" + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: "1" + - name: SGLANG_BATCH_TELEMETRY + value: "1" + - name: SGLANG_BATCH_SPAN_SAMPLE + value: "0.1" + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: "600" + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: "16" + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: "32" + - name: TZ + value: Asia/Shanghai + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: "1024" + - name: TMPDIR + value: /data/ymk/l3ssd/d-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/d-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/d-hf + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30300 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 150 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: "96" + memory: 550Gi + nvidia.com/gpu: "4" + requests: + cpu: "32" + memory: 256Gi + nvidia.com/gpu: "4" + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - p + - d + minReadySeconds: 0 + name: gateway + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "29000" + prometheus.io/scheme: http + prometheus.io/scrape: "true" + labels: + app: ds41-l3ssd-gw + kongming.io/history-revision: hrev-b90013fa6cc5a2a0e6515115d188135ea41e48b7 + kongming.io/history-run: hrun-4ca5f278e80f5e20e03b98351874ad7cd859d5e7 + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-id: cw-dblft6sgrhagmpdt8ef0 + kongming.io/resource-type: custom-workload + kongming.io/user-id: usr-d7upbr3j0sq2fgdrmcr0 + spec: + containers: + - command: + - /bin/bash + - -c + - | + exec /opt/sglang/bin/sgl-model-gateway launch \ + --pd-disaggregation \ + --prefill http://127.0.0.1:30000 8998 \ + --decode http://127.0.0.1:30300 \ + --prefill-policy cache_aware --decode-policy cache_aware \ + --g9i-enabled --g9i-max-new-tokens 4096 --g9i-est-td-enabled --g9i-shadow-enabled --g9i-td-admit off --g9i-p-queue-reqs-cap 0 --g9i-p-queue-token-cap 0 --g9i-p-qps 0 --g9i-p-refresh-ms 250 --g9i-window-fixed 256 \ + \ + \ + --g9i-regular-p-queue-reject-secs 0 \ + --g9i-decode-max-running 1000000 \ + --pool-idle-timeout-secs 2 --pool-max-idle-per-host 0 \ + --health-check-timeout-secs 30 --health-check-interval-secs 30 \ + --cb-failure-threshold 50 --disable-circuit-breaker --disable-health-check \ + --model-path /data/models/DeepSeek-V4.1-Flash-1017 \ + --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \ + --host 0.0.0.0 --port 30400 + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: gateway + ports: + - containerPort: 30400 + name: http + protocol: TCP + - containerPort: 29000 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: "16" + memory: 32Gi + requests: + cpu: "2" + memory: 8Gi + volumeMounts: + - mountPath: /data + name: data + - mountPath: /opt/sglang/bin/sgl-model-gateway + name: gw-bin + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/gw-ns-est-td/sgl-model-gateway + type: File + name: gw-bin +status: + conditions: + - lastTransitionTime: "2026-11-17T01:58:12Z" + message: All roles are ready + observedGeneration: 6 + reason: AllRolesReady + status: "True" + type: Ready + observedGeneration: 6 + roleStatuses: + - name: mooncake-master + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 + - name: p + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 + - name: d + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 + - name: gateway + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1