diff --git a/deploy/b300-ds41/dockerserve-3cp2-pd.yml b/deploy/b300-ds41/dockerserve-3cp2-pd.yml index d9115e12dc..ee2c3c160b 100644 --- a/deploy/b300-ds41/dockerserve-3cp2-pd.yml +++ b/deploy/b300-ds41/dockerserve-3cp2-pd.yml @@ -28,6 +28,7 @@ x-p-environment: &p-environment MC_INTRA_NVLINK: "true" SGLANG_MOONCAKE_SEND_AUX_TCP: "1" SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json + SGLANG_HICACHE_PREFETCH_IO_THREADS: "4" SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" diff --git a/deploy/b300-ds41/dockerserve-3p1d-l3dummy.yml b/deploy/b300-ds41/dockerserve-3p1d-l3dummy.yml index c576c42677..f7fd4e1f29 100644 --- a/deploy/b300-ds41/dockerserve-3p1d-l3dummy.yml +++ b/deploy/b300-ds41/dockerserve-3p1d-l3dummy.yml @@ -48,6 +48,7 @@ x-p-environment: &p-environment MC_INTRA_NVLINK: "true" SGLANG_MOONCAKE_SEND_AUX_TCP: "1" SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json + SGLANG_HICACHE_PREFETCH_IO_THREADS: "4" SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" diff --git a/deploy/b300-ds41/dockerserve-3p1d-l3test.yml b/deploy/b300-ds41/dockerserve-3p1d-l3test.yml index 5ed81daa39..fb187561f4 100644 --- a/deploy/b300-ds41/dockerserve-3p1d-l3test.yml +++ b/deploy/b300-ds41/dockerserve-3p1d-l3test.yml @@ -39,6 +39,7 @@ x-p-environment: &p-environment MC_INTRA_NVLINK: "true" SGLANG_MOONCAKE_SEND_AUX_TCP: "1" SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json + SGLANG_HICACHE_PREFETCH_IO_THREADS: "4" SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" diff --git a/deploy/b300-ds41/dockerserve-3p1d-otlp-cs32k.yml b/deploy/b300-ds41/dockerserve-3p1d-otlp-cs32k.yml index f6d6c8bfcc..8a060f542f 100644 --- a/deploy/b300-ds41/dockerserve-3p1d-otlp-cs32k.yml +++ b/deploy/b300-ds41/dockerserve-3p1d-otlp-cs32k.yml @@ -35,6 +35,7 @@ x-p-environment: &p-environment MC_INTRA_NVLINK: "true" SGLANG_MOONCAKE_SEND_AUX_TCP: "1" SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json + SGLANG_HICACHE_PREFETCH_IO_THREADS: "4" SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" diff --git a/deploy/b300-ds41/dockerserve-3p1d-otlp.yml b/deploy/b300-ds41/dockerserve-3p1d-otlp.yml index 056228c81b..2175c2af1a 100644 --- a/deploy/b300-ds41/dockerserve-3p1d-otlp.yml +++ b/deploy/b300-ds41/dockerserve-3p1d-otlp.yml @@ -35,6 +35,7 @@ x-p-environment: &p-environment MC_INTRA_NVLINK: "true" SGLANG_MOONCAKE_SEND_AUX_TCP: "1" SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json + SGLANG_HICACHE_PREFETCH_IO_THREADS: "4" SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" diff --git a/deploy/b300-ds41/dockerserve-3p1d-prof.yml b/deploy/b300-ds41/dockerserve-3p1d-prof.yml index d52fab4db9..613132f1b1 100644 --- a/deploy/b300-ds41/dockerserve-3p1d-prof.yml +++ b/deploy/b300-ds41/dockerserve-3p1d-prof.yml @@ -29,6 +29,7 @@ x-p-environment: &p-environment MC_INTRA_NVLINK: "true" SGLANG_MOONCAKE_SEND_AUX_TCP: "1" SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json + SGLANG_HICACHE_PREFETCH_IO_THREADS: "4" SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" diff --git a/deploy/b300-ds41/dockerserve-3p1d-soak.yml b/deploy/b300-ds41/dockerserve-3p1d-soak.yml index 16d25fcc3c..db450c2954 100644 --- a/deploy/b300-ds41/dockerserve-3p1d-soak.yml +++ b/deploy/b300-ds41/dockerserve-3p1d-soak.yml @@ -29,6 +29,7 @@ x-p-environment: &p-environment MC_INTRA_NVLINK: "true" SGLANG_MOONCAKE_SEND_AUX_TCP: "1" SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json + SGLANG_HICACHE_PREFETCH_IO_THREADS: "4" SGLANG_DISAGGREGATION_QUEUE_SIZE: "16" SGLANG_DISAGGREGATION_THREAD_POOL_SIZE: "32" diff --git a/deploy/b300-ds41/dockerserve-p-standalone.yml b/deploy/b300-ds41/dockerserve-p-standalone.yml index 87713257c1..8f1d3747f2 100644 --- a/deploy/b300-ds41/dockerserve-p-standalone.yml +++ b/deploy/b300-ds41/dockerserve-p-standalone.yml @@ -32,6 +32,7 @@ services: MC_INTRA_NVLINK: "true" SGLANG_MOONCAKE_SEND_AUX_TCP: "1" SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store.json + SGLANG_HICACHE_PREFETCH_IO_THREADS: "4" CUDA_VISIBLE_DEVICES: "0,1" command: > sglang serve diff --git a/deploy/b300-ds41/dockerserve-pd2-p-r1.yml b/deploy/b300-ds41/dockerserve-pd2-p-r1.yml index e260c73e95..fa67c8a868 100644 --- a/deploy/b300-ds41/dockerserve-pd2-p-r1.yml +++ b/deploy/b300-ds41/dockerserve-pd2-p-r1.yml @@ -22,6 +22,7 @@ services: - SGLANG_MOONCAKE_SEND_AUX_TCP=1 - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 - SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json + - SGLANG_HICACHE_PREFETCH_IO_THREADS=4 volumes: - /data:/data - /data/ymk/cache/sglang:/root/.cache/sglang diff --git a/deploy/b300-ds41/dockerserve-pd2-p-r2.yml b/deploy/b300-ds41/dockerserve-pd2-p-r2.yml index 9716fe4c47..6214b82a7a 100644 --- a/deploy/b300-ds41/dockerserve-pd2-p-r2.yml +++ b/deploy/b300-ds41/dockerserve-pd2-p-r2.yml @@ -18,6 +18,7 @@ services: - SGLANG_MOONCAKE_SEND_AUX_TCP=1 - SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE=1 - SGLANG_HICACHE_MOONCAKE_CONFIG_PATH=/data/ymk/ds41/mooncake-store.json + - SGLANG_HICACHE_PREFETCH_IO_THREADS=4 - SGLANG_DISAGGREGATION_QUEUE_SIZE=16 - SGLANG_DISAGGREGATION_THREAD_POOL_SIZE=32 volumes: diff --git a/deploy/b300-ds41/dockerserve-tier-cp2.yml b/deploy/b300-ds41/dockerserve-tier-cp2.yml index 2d788a02e1..e5e4529e52 100644 --- a/deploy/b300-ds41/dockerserve-tier-cp2.yml +++ b/deploy/b300-ds41/dockerserve-tier-cp2.yml @@ -32,6 +32,7 @@ services: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: 1 SGLANG_RAGGED_VERIFY_MODE: static SGLANG_HICACHE_MOONCAKE_CONFIG_PATH: /data/ymk/ds41/mooncake-store-dummy.json + SGLANG_HICACHE_PREFETCH_IO_THREADS: 4 SGLANG_HICACHE_TIMING: 1 SGLANG_HICACHE_STORAGE_BATCH_SIZE: 512 CUDA_VISIBLE_DEVICES: 0,1 diff --git a/deploy/b300-ds41/ds41-3p1d-rbg-cw.yaml b/deploy/b300-ds41/ds41-3p1d-rbg-cw.yaml index bc81b150f5..382611cb1a 100644 --- a/deploy/b300-ds41/ds41-3p1d-rbg-cw.yaml +++ b/deploy/b300-ds41/ds41-3p1d-rbg-cw.yaml @@ -85,6 +85,8 @@ spec: value: "1" - name: SGLANG_HICACHE_MOONCAKE_CONFIG_PATH value: /data/ymk/ds41/mooncake-store.json + - name: SGLANG_HICACHE_PREFETCH_IO_THREADS + value: "4" - name: SGLANG_DISAGGREGATION_QUEUE_SIZE value: "16" - name: SGLANG_DISAGGREGATION_THREAD_POOL_SIZE