[Docs] GLM-5.2: move GB300 AgentX recipe to the agentic HiCache section and drop no-op settings (#42167)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
21c9bbdf2c
commit
f2f0be731c
@@ -303,3 +303,74 @@ python3 -m sglang.launch_server \
|
||||
```
|
||||
|
||||
**B200 (TP8):** same command, with `--mem-fraction-static 0.83` and `--hicache-size 169`. Lower concurrency (c1/c4/c8) uses `--hicache-ratio 0.75` instead.
|
||||
|
||||
**GB300 (TP4, single node):** one 4-GPU GB300 node serves the whole model. All three commands offload KV cache to host DRAM with `--hicache-size 135` (GB of host memory per rank, 540 GB per node). GLM-5.2 is a DSA model, so on Blackwell SGLang already defaults to an FP8 E4M3 KV cache and TRT-LLM DSA attention; the commands do not need to set them.
|
||||
|
||||
Low-latency (TP4, MTP 5-1-6, up to 16 concurrent requests):
|
||||
```bash Command
|
||||
sglang serve \
|
||||
--model-path nvidia/GLM-5.2-NVFP4 \
|
||||
--trust-remote-code \
|
||||
--tp 4 \
|
||||
--quantization modelopt_fp4 \
|
||||
--fp4-gemm-backend flashinfer_trtllm \
|
||||
--max-running-requests 16 \
|
||||
--cuda-graph-max-bs-decode 16 \
|
||||
--chunked-prefill-size 8192 \
|
||||
--max-prefill-tokens 8192 \
|
||||
--mem-fraction-static 0.8 \
|
||||
--tool-call-parser glm47 \
|
||||
--reasoning-parser glm45 \
|
||||
--speculative-algorithm EAGLE \
|
||||
--speculative-num-steps 5 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 6 \
|
||||
--enable-hierarchical-cache \
|
||||
--hicache-size 135 \
|
||||
--hicache-write-policy write_back \
|
||||
--hicache-io-backend direct
|
||||
```
|
||||
|
||||
Balanced (DP attention, MTP 2-1-3, up to 256 concurrent requests):
|
||||
```bash Command
|
||||
sglang serve \
|
||||
--model-path nvidia/GLM-5.2-NVFP4 \
|
||||
--trust-remote-code \
|
||||
--tp 4 \
|
||||
--attn-dp-size 4 \
|
||||
--quantization modelopt_fp4 \
|
||||
--max-running-requests 256 \
|
||||
--chunked-prefill-size 8192 \
|
||||
--mem-fraction-static 0.92 \
|
||||
--tool-call-parser glm47 \
|
||||
--reasoning-parser glm45 \
|
||||
--speculative-algorithm EAGLE \
|
||||
--speculative-num-steps 2 \
|
||||
--speculative-eagle-topk 1 \
|
||||
--speculative-num-draft-tokens 3 \
|
||||
--enable-hierarchical-cache \
|
||||
--hicache-size 135 \
|
||||
--hicache-write-policy write_back \
|
||||
--hicache-io-backend direct
|
||||
```
|
||||
|
||||
High-throughput (DP attention, no MTP, up to 512 concurrent requests):
|
||||
```bash Command
|
||||
sglang serve \
|
||||
--model-path nvidia/GLM-5.2-NVFP4 \
|
||||
--trust-remote-code \
|
||||
--tp 4 \
|
||||
--attn-dp-size 4 \
|
||||
--quantization modelopt_fp4 \
|
||||
--max-running-requests 512 \
|
||||
--chunked-prefill-size 8192 \
|
||||
--mem-fraction-static 0.92 \
|
||||
--tool-call-parser glm47 \
|
||||
--reasoning-parser glm45 \
|
||||
--enable-hierarchical-cache \
|
||||
--hicache-size 135 \
|
||||
--hicache-write-policy write_back \
|
||||
--hicache-io-backend direct
|
||||
```
|
||||
|
||||
With `--attn-dp-size 4`, `--chunked-prefill-size` is a global budget divided across the attention DP ranks, so the balanced and high-throughput commands prefill 2048 tokens per rank per chunk.
|
||||
|
||||
@@ -817,40 +817,20 @@ sgl-eval run aime25 \\
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK=1",
|
||||
"SGLANG_ENABLE_THINKING=1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_HICACHE_DEBUG_LOG=1",
|
||||
"SGLANG_HICACHE_DEBUG_SAMPLE_RATE=16384",
|
||||
"SGLANG_MOE_NVFP4_DISPATCH=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--tp 4",
|
||||
"--quantization modelopt_fp4",
|
||||
"--kv-cache-dtype fp8_e4m3",
|
||||
"--dsa-prefill-backend trtllm",
|
||||
"--dsa-decode-backend trtllm",
|
||||
"--moe-runner-backend flashinfer_trtllm",
|
||||
"--fp4-gemm-backend flashinfer_trtllm",
|
||||
"--disable-shared-experts-fusion",
|
||||
"--flashinfer-allreduce-fusion-backend auto",
|
||||
// MTP 4-1-5 and HiCache (write_back, --hicache-size 135) match the InferenceX
|
||||
// GB300 AgentX low-concurrency recipe (SemiAnalysisAI/InferenceX#3178).
|
||||
"--speculative-algorithm EAGLE",
|
||||
"--speculative-num-steps 4",
|
||||
"--speculative-num-steps 5",
|
||||
"--speculative-eagle-topk 1",
|
||||
"--speculative-num-draft-tokens 5",
|
||||
"--speculative-num-draft-tokens 6",
|
||||
"--chunked-prefill-size 8192",
|
||||
"--max-prefill-tokens 8192",
|
||||
"--mem-fraction-static 0.8",
|
||||
"--mem-fraction-static 0.85",
|
||||
"--max-running-requests 16",
|
||||
"--cuda-graph-max-bs-decode 16",
|
||||
"--enable-hierarchical-cache",
|
||||
"--hicache-size 135",
|
||||
"--hicache-write-policy write_back",
|
||||
"--hicache-io-backend direct",
|
||||
"--max-prefill-tokens 8192",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
@@ -858,14 +838,7 @@ sgl-eval run aime25 \\
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK=1",
|
||||
"SGLANG_ENABLE_THINKING=1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_HICACHE_DEBUG_LOG=1",
|
||||
"SGLANG_HICACHE_DEBUG_SAMPLE_RATE=16384",
|
||||
"SGLANG_MOE_NVFP4_DISPATCH=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--tp 4",
|
||||
@@ -887,14 +860,7 @@ sgl-eval run aime25 \\
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK=1",
|
||||
"SGLANG_ENABLE_THINKING=1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
"SGLANG_HICACHE_DEBUG_LOG=1",
|
||||
"SGLANG_HICACHE_DEBUG_SAMPLE_RATE=16384",
|
||||
"SGLANG_MOE_NVFP4_DISPATCH=1",
|
||||
],
|
||||
env: [],
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--tp 4",
|
||||
|
||||
Reference in New Issue
Block a user