From 2eb163ad73cb5cf4a36d0ab32cbf28f8d6811d0d Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Sun, 15 Nov 2026 14:23:52 +0800 Subject: [PATCH] fix(deploy): point sglang cache at the shared warm cache --- .../ds41-gist-exp-l3ssd-baked-20261115.yaml | 4 +- ...s41-gist-exp-l3ssd-fastokens-20261115.yaml | 488 ++++++++++++++++++ .../ds41-l3ssd-test-baked-20261115.yaml | 4 +- 3 files changed, 492 insertions(+), 4 deletions(-) create mode 100644 deploy/b300-k8s/ds41-gist-exp-l3ssd-fastokens-20261115.yaml diff --git a/deploy/b300-k8s/ds41-gist-exp-l3ssd-baked-20261115.yaml b/deploy/b300-k8s/ds41-gist-exp-l3ssd-baked-20261115.yaml index c578398dbe..a1545f77ea 100644 --- a/deploy/b300-k8s/ds41-gist-exp-l3ssd-baked-20261115.yaml +++ b/deploy/b300-k8s/ds41-gist-exp-l3ssd-baked-20261115.yaml @@ -276,7 +276,7 @@ spec: type: Directory name: data - hostPath: - path: /data/ymk/l3ssd/model-cache/sglang + path: /data/ymk/cache/sglang type: Directory name: sglang-cache - emptyDir: @@ -402,7 +402,7 @@ spec: type: Directory name: data - hostPath: - path: /data/ymk/l3ssd/model-cache/sglang + path: /data/ymk/cache/sglang type: Directory name: sglang-cache - emptyDir: diff --git a/deploy/b300-k8s/ds41-gist-exp-l3ssd-fastokens-20261115.yaml b/deploy/b300-k8s/ds41-gist-exp-l3ssd-fastokens-20261115.yaml new file mode 100644 index 0000000000..6913dcd139 --- /dev/null +++ b/deploy/b300-k8s/ds41-gist-exp-l3ssd-fastokens-20261115.yaml @@ -0,0 +1,488 @@ +apiVersion: workloads.x-k8s.io/v1alpha2 +kind: RoleBasedGroup +metadata: + annotations: + rbg.workloads.x-k8s.io/discovery-config-mode: refine + labels: + app: ds41-gist-exp + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + sglang-model: ds-v41-flash + name: ds41-gist-exp + namespace: prj-d7te67bj0sq2fgdrmbig +spec: + roles: + - minReadySeconds: 0 + name: mooncake-master + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '19003' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-mooncake + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081 --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true --offload_on_evict=true --promotion_on_hit=false --offload_force_evict=false\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: mooncake-master + ports: + - containerPort: 19003 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + initialDelaySeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 50051 + resources: + limits: + cpu: '4' + memory: 64Gi + requests: + cpu: '1' + memory: 8Gi + env: + - name: TMPDIR + value: /data/ymk/l3ssd/master-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/master-xdg + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: p + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30000' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-p + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072 --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend direct \\\n --hicache-write-policy write_through --hicache-storage-backend mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n --disaggregation-mode prefill --disaggregation-transfer-backend mooncake \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout 3600 --enable-dp-attention-local-control-broadcast\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 0,1,2,3 + - name: SGLANG_LOG_MS + value: '1' + - name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS + value: '1' + - name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL + value: '1' + - name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS + value: '1' + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: '65536' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DP_USE_GATHERV + value: '1' + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION + value: '0' + - name: SGLANG_NUMA_BIND_V2 + value: '0' + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: SGLANG_TIMEOUT_KEEP_ALIVE + value: '900' + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH + value: /data/ymk/ds41/mooncake-store-l3ssd.json + - name: SGLANG_HICACHE_PREFETCH_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_BACKUP_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_TIMING + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_ENABLE_METRICS_DEVICE_TIMER + value: '1' + - name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL + value: '100' + - name: SGLANG_MM_TRACE + value: '1' + - name: SGLANG_BATCH_SPAN_MIN_MS + value: '200' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.05' + - name: SGLANG_BATCH_SPAN_MAX_LINKS + value: '8' + - name: SGLANG_BATCH_STAGE_TIMING + value: '1' + - name: SGLANG_BATCH_PASS_SPAN + value: '1' + - name: SGLANG_BATCH_PASS_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_RECV_SPAN + value: '1' + - name: SGLANG_RECV_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_MM_TRACE_SYNC + value: '0' + - name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK + value: '1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: SGLANG_TRACE_LOOP_SPAN + value: '1' + - name: SGLANG_TRACE_LOOP_SAMPLE + value: '1.0' + - name: SGLANG_FWD_TIMELINE + value: '0' + - name: SGLANG_TRACE_LOOP_DETAIL + value: '1' + - name: SGLANG_TRACE_MOONCAKE_DETAIL + value: '0' + - name: SGLANG_TRACE_BACKUP_DETAIL + value: '1' + - name: SGLANG_ASYNC_BACKUP_SUBMIT + value: '1' + - name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH + value: '1' + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE + value: '9400' + - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR + value: bucket_storage_backend + - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE + value: '549755813888' + - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES + value: '549755813888' + - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY + value: fifo + - name: TMPDIR + value: /data/ymk/l3ssd/p-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/p-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/p-hf + - name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS + value: '0' + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30000 + name: metrics + protocol: TCP + - containerPort: 8998 + name: bootstrap + protocol: TCP + - name: mc-0 + containerPort: 9400 + protocol: TCP + - name: mc-1 + containerPort: 9401 + protocol: TCP + - name: mc-2 + containerPort: 9402 + protocol: TCP + - name: mc-3 + containerPort: 9403 + protocol: TCP + readinessProbe: + failureThreshold: 120 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 2Ti + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 512Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: d + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30300' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-d + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4 --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests 512 --cuda-graph-max-bs-decode 512 \\\n --chunked-prefill-size 32768 --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout 3600\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 4,5,6,7 + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: TMPDIR + value: /data/ymk/l3ssd/d-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/d-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/d-hf + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30300 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 150 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 550Gi + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 256Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - p + - d + minReadySeconds: 0 + name: gateway + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '29000' + prometheus.io/scheme: http + prometheus.io/scrape: 'true' + labels: + app: ds41-l3ssd-gw + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300 \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n --g9i-enabled --g9i-max-new-tokens 4096 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs 0 \\\n --g9i-decode-max-running 512 \\\n --health-check-timeout-secs 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50 \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port 30400\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: gateway + ports: + - containerPort: 30400 + name: http + protocol: TCP + - containerPort: 29000 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '16' + memory: 32Gi + requests: + cpu: '2' + memory: 8Gi + volumeMounts: + - mountPath: /data + name: data + - mountPath: /opt/sglang/bin/sgl-model-gateway + name: gw-bin + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - name: gw-bin + hostPath: + path: /data/ymk/cache/gw-est-td/sgl-model-gateway + type: File diff --git a/deploy/b300-k8s/ds41-l3ssd-test-baked-20261115.yaml b/deploy/b300-k8s/ds41-l3ssd-test-baked-20261115.yaml index 614b1b9dc3..520f0f71dd 100644 --- a/deploy/b300-k8s/ds41-l3ssd-test-baked-20261115.yaml +++ b/deploy/b300-k8s/ds41-l3ssd-test-baked-20261115.yaml @@ -276,7 +276,7 @@ spec: type: Directory name: data - hostPath: - path: /data/ymk/l3ssd/model-cache/sglang + path: /data/ymk/cache/sglang type: Directory name: sglang-cache - emptyDir: @@ -402,7 +402,7 @@ spec: type: Directory name: data - hostPath: - path: /data/ymk/l3ssd/model-cache/sglang + path: /data/ymk/cache/sglang type: Directory name: sglang-cache - emptyDir: