diff --git a/deploy/b300-k8s/ds41-gist-exp-20261115-g9i-t26-qps20.yaml b/deploy/b300-k8s/ds41-gist-exp-20261115-g9i-t26-qps20.yaml new file mode 100644 index 0000000000..4564c93f0f --- /dev/null +++ b/deploy/b300-k8s/ds41-gist-exp-20261115-g9i-t26-qps20.yaml @@ -0,0 +1,544 @@ +apiVersion: workloads.x-k8s.io/v1alpha2 +kind: RoleBasedGroup +metadata: + annotations: + rbg.workloads.x-k8s.io/discovery-config-mode: refine + labels: + app: ds41-gist-exp + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + sglang-model: ds-v41-flash + name: ds41-gist-exp + namespace: prj-d7te67bj0sq2fgdrmbig +spec: + roles: + - minReadySeconds: 0 + name: mooncake-master + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '19003' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-mooncake + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "# 18081: 宿主机 18080 被 AgentX fake upstream(./fakeup)长期占用;\n# 我们的 P/D\ + \ 用 metadata_server=P2PHANDSHAKE,此 HTTP 端口只是 master 可选项。\nexec /opt/sglang/bin/mooncake_master\ + \ \\\n --port 50051 --enable_http_metadata_server=true \\\n --http_metadata_server_port=18081\ + \ --metrics_port=19003 --eviction_high_watermark_ratio=0.90 --enable_offload=true\ + \ --offload_on_evict=false --promotion_on_hit=false --offload_force_evict=false\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: mooncake-master + ports: + - containerPort: 19003 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + initialDelaySeconds: 5 + periodSeconds: 5 + tcpSocket: + port: 50051 + resources: + limits: + cpu: '4' + memory: 64Gi + requests: + cpu: '1' + memory: 8Gi + env: + - name: TMPDIR + value: /data/ymk/l3ssd/master-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/master-xdg + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: p + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30000' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-p + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens\ + \ --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name\ + \ /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser\ + \ auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"\ + reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4\ + \ --enable-dp-attention --enable-dp-lm-head \\\n --moe-a2a-backend\ + \ megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts\ + \ 32 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r32_ep4.json\ + \ \\\n --mem-fraction-static 0.80 \\\n --chunked-prefill-size 131072\ + \ --max-prefill-tokens 131072 \\\n --max-running-requests 512 --tokenizer-worker-num\ + \ 16 \\\n --load-balance-method total_tokens \\\n --speculative-algorithm\ + \ DSPARK --speculative-dspark-block-size 3 \\\n --enable-decoder-swa-bounded-replay\ + \ --swa-full-tokens-ratio 0.12 \\\n --cuda-graph-max-bs-decode 1 --disable-decode-cuda-graph\ + \ \\\n --cuda-graph-backend-prefill breakable \\\n --cuda-graph-bs-prefill\ + \ 1024 2048 4096 8192 16384 32768 \\\n --cuda-graph-prefill-max-context\ + \ 262144 --cuda-graph-max-seq-len-prefill 262144 \\\n --enable-hierarchical-cache\ + \ --hicache-ratio 4 \\\n --hicache-mem-layout page_first_direct --hicache-io-backend\ + \ direct \\\n --hicache-write-policy write_through --hicache-storage-backend\ + \ mooncake \\\n --hicache-storage-prefetch-policy wait_complete \\\n\ + \ --disaggregation-mode prefill --disaggregation-transfer-backend mooncake\ + \ \\\n --disaggregation-bootstrap-port 8998 \\\n --expert-balancedness-report-mode\ + \ prometheus \\\n --expert-distribution-recorder-buffer-size 64 \\\n\ + \ --host 0.0.0.0 --port 30000 --enable-metrics --enable-cache-report\ + \ \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint\ + \ 127.0.0.1:4317 --otlp-service-name gist-exp-p \\\n --watchdog-timeout\ + \ 3600 --enable-dp-attention-local-control-broadcast\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 0,1,2,3 + - name: SGLANG_LOG_MS + value: '1' + - name: SGLANG_OPT_DSV41_ENGRAM_PEER_SHARDS + value: '1' + - name: SGLANG_OPT_DSV41_MEGA_MHC_PREFILL + value: '1' + - name: SGLANG_OPT_DSV41_BF16_INDEX_LOGITS + value: '1' + - name: SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK + value: '65536' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DP_USE_GATHERV + value: '1' + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION + value: '0' + - name: SGLANG_NUMA_BIND_V2 + value: '0' + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: SGLANG_TIMEOUT_KEEP_ALIVE + value: '900' + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH + value: /data/ymk/ds41/mooncake-store-l3ssd.json + - name: SGLANG_HICACHE_PREFETCH_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_BACKUP_IO_THREADS + value: '4' + - name: SGLANG_HICACHE_TIMING + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_ENABLE_METRICS_DEVICE_TIMER + value: '1' + - name: SGLANG_EPLB_HEATMAP_COLLECTION_INTERVAL + value: '100' + - name: SGLANG_MM_TRACE + value: '1' + - name: SGLANG_BATCH_SPAN_MIN_MS + value: '200' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.05' + - name: SGLANG_BATCH_SPAN_MAX_LINKS + value: '8' + - name: SGLANG_BATCH_STAGE_TIMING + value: '1' + - name: SGLANG_BATCH_PASS_SPAN + value: '1' + - name: SGLANG_BATCH_PASS_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_RECV_SPAN + value: '1' + - name: SGLANG_RECV_SPAN_SAMPLE + value: '1.0' + - name: SGLANG_MM_TRACE_SYNC + value: '0' + - name: SGLANG_DISAGGREGATION_FORCE_QUERY_PREFILL_DP_RANK + value: '1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: SGLANG_TRACE_LOOP_SPAN + value: '1' + - name: SGLANG_TRACE_LOOP_SAMPLE + value: '1.0' + - name: SGLANG_FWD_TIMELINE + value: '0' + - name: SGLANG_TRACE_LOOP_DETAIL + value: '1' + - name: SGLANG_TRACE_MOONCAKE_DETAIL + value: '0' + - name: SGLANG_TRACE_BACKUP_DETAIL + value: '1' + - name: SGLANG_ASYNC_BACKUP_SUBMIT + value: '1' + - name: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH + value: '1' + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: SGLANG_MOONCAKE_CLIENT_METRICS_PORT_BASE + value: '9400' + - name: MOONCAKE_OFFLOAD_STORAGE_BACKEND_DESCRIPTOR + value: bucket_storage_backend + - name: MOONCAKE_OFFLOAD_BUCKET_MAX_TOTAL_SIZE + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_TOTAL_SIZE_LIMIT_BYTES + value: '1099511627776' + - name: MOONCAKE_OFFLOAD_BUCKET_EVICTION_POLICY + value: fifo + - name: TMPDIR + value: /data/ymk/l3ssd/p-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/p-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/p-hf + - name: MOONCAKE_OFFLOAD_FLUSH_PARTIAL_BUCKETS + value: '0' + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30000 + name: metrics + protocol: TCP + - containerPort: 8998 + name: bootstrap + protocol: TCP + - name: mc-0 + containerPort: 9400 + protocol: TCP + - name: mc-1 + containerPort: 9401 + protocol: TCP + - name: mc-2 + containerPort: 9402 + protocol: TCP + - name: mc-3 + containerPort: 9403 + protocol: TCP + readinessProbe: + failureThreshold: 120 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 2Ti + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 512Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - mooncake-master + minReadySeconds: 0 + name: d + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '30300' + prometheus.io/scheme: http + labels: + app: ds41-l3ssd-d + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec sglang serve \\\n --trust-remote-code --tokenizer-backend fastokens\ + \ --model-path /data/models/DeepSeek-V4.1-Flash-1017 --served-model-name\ + \ /data/models/DeepSeek-V4.1-Flash \\\n --reasoning-parser auto --tool-call-parser\ + \ auto \\\n --default-chat-template-kwargs '{\"thinking\": true, \"\ + reasoning_effort\": \"high\"}' \\\n --tp 4 --dp-size 4 --ep-size 4\ + \ --enable-dp-attention --enable-dp-lm-head \\\n --enable-fp32-lm-head\ + \ --enable-dp-attention-local-control-broadcast \\\n --moe-a2a-backend\ + \ megamoe --dsv4-attn-backend flashmla_mega \\\n --ep-num-redundant-experts\ + \ 64 --ep-dispatch-algorithm dynamic \\\n --init-expert-location /data/ymk/gist-dsv41/placement_r64_ep4.json\ + \ \\\n --mem-fraction-static 0.80 \\\n --speculative-algorithm DSPARK\ + \ --speculative-dspark-block-size 3 \\\n --stream-interval 32 --swa-prefix-tails\ + \ 2048 --enable-decoder-swa-bounded-replay \\\n --max-running-requests\ + \ 512 --cuda-graph-max-bs-decode 512 \\\n --chunked-prefill-size 32768\ + \ --max-prefill-tokens 8192 \\\n --disaggregation-mode decode --disaggregation-transfer-backend\ + \ mooncake \\\n --host 0.0.0.0 --port 30300 --enable-metrics --enable-cache-report\ + \ \\\n --enable-trace --trace-modules request,mooncake \\\n --otlp-traces-endpoint\ + \ 127.0.0.1:4317 --otlp-service-name gist-exp-d \\\n --watchdog-timeout\ + \ 3600\n" + env: + - name: CUDA_VISIBLE_DEVICES + value: 4,5,6,7 + - name: SGLANG_RAGGED_VERIFY_MODE + value: static + - name: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE + value: '1' + - name: SGLANG_DSV4_KV_LAYOUT + value: v41_new + - name: SGLANG_DEFAULT_THINKING + value: '1' + - name: SGLANG_DSV41_REASONING_EFFORT + value: high + - name: MC_INTRANODE_NVLINK + value: 'true' + - name: MC_INTRA_NVLINK + value: 'true' + - name: SGLANG_MOONCAKE_SEND_AUX_TCP + value: 'true' + - name: SGLANG_ENABLE_METRICS_DP_ATTENTION + value: '1' + - name: SGLANG_BATCH_TELEMETRY + value: '1' + - name: SGLANG_BATCH_SPAN_SAMPLE + value: '0.1' + - name: SGLANG_DISAGGREGATION_WAITING_TIMEOUT + value: '600' + - name: SGLANG_DISAGGREGATION_QUEUE_SIZE + value: '16' + - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE + value: '32' + - name: TZ + value: Asia/Shanghai + - name: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + value: '1024' + - name: TMPDIR + value: /data/ymk/l3ssd/d-tmp + - name: XDG_CACHE_HOME + value: /data/ymk/l3ssd/d-xdg + - name: HF_HOME + value: /data/ymk/l3ssd/d-hf + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: sglang + ports: + - containerPort: 30300 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 150 + httpGet: + path: /health + port: metrics + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '96' + memory: 550Gi + nvidia.com/gpu: '4' + requests: + cpu: '32' + memory: 256Gi + nvidia.com/gpu: '4' + securityContext: + privileged: true + volumeMounts: + - mountPath: /data + name: data + - mountPath: /root/.cache/sglang + name: sglang-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /root/.cargo/bin/cargo + name: fake-cargo + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - hostPath: + path: /data/ymk/cache/sglang + type: Directory + name: sglang-cache + - emptyDir: + medium: Memory + sizeLimit: 64Gi + name: dshm + - hostPath: + path: /data/ymk/ds41/fake-cargo + type: File + name: fake-cargo + - dependencies: + - p + - d + minReadySeconds: 0 + name: gateway + podManagementPolicy: Parallel + replicas: 1 + standalonePattern: + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: '29000' + prometheus.io/scheme: http + prometheus.io/scrape: 'true' + labels: + app: ds41-l3ssd-gw + kongming.io/managed-by: kongming + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kongming.io/resource-type: custom-workload + spec: + containers: + - command: + - /bin/bash + - -c + - "exec /opt/sglang/bin/sgl-model-gateway launch \\\n --pd-disaggregation\ + \ \\\n --prefill http://127.0.0.1:30000 8998 \\\n --decode http://127.0.0.1:30300\ + \ \\\n --prefill-policy cache_aware --decode-policy cache_aware \\\n\ + \ --g9i-enabled --g9i-max-new-tokens 4096 --g9i-est-td-enabled --g9i-shadow-enabled\ + \ --g9i-td-admit enforce --g9i-p-queue-reqs-cap 32 --g9i-p-queue-token-cap\ + \ 16000000 --g9i-p-qps 20 --g9i-p-refresh-ms 250 \\\n \\\n \\\n --g9i-regular-p-queue-reject-secs\ + \ 0 \\\n --g9i-decode-max-running 512 \\\n --health-check-timeout-secs\ + \ 30 --health-check-interval-secs 30 \\\n --cb-failure-threshold 50\ + \ \\\n --model-path /data/models/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-path\ + \ /data/models/DeepSeek-V4.1-Flash-1017 \\\n --host 0.0.0.0 --port\ + \ 30400\n" + image: sealos.hub:5000/prj-d7te67bj0sq2fgdrmbig/sglang:recipe-ns-l3ssd-433257c7b-baked-20261115 + imagePullPolicy: IfNotPresent + name: gateway + ports: + - containerPort: 30400 + name: http + protocol: TCP + - containerPort: 29000 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 60 + httpGet: + path: /health + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: '16' + memory: 32Gi + requests: + cpu: '2' + memory: 8Gi + volumeMounts: + - mountPath: /data + name: data + - mountPath: /opt/sglang/bin/sgl-model-gateway + name: gw-bin + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + kongming.io/node-group-id: ng-damg6d4grhalvvbgbbrg + kongming.io/project-id: prj-d7te67bj0sq2fgdrmbig + kubernetes.io/hostname: b300-01 + volumes: + - hostPath: + path: /data + type: Directory + name: data + - name: gw-bin + hostPath: + path: /data/ymk/gw-ns-est-td/sgl-model-gateway + type: File