diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx index 103f23cd96..988f95a528 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx @@ -303,3 +303,74 @@ python3 -m sglang.launch_server \ ``` **B200 (TP8):** same command, with `--mem-fraction-static 0.83` and `--hicache-size 169`. Lower concurrency (c1/c4/c8) uses `--hicache-ratio 0.75` instead. + +**GB300 (TP4, single node):** one 4-GPU GB300 node serves the whole model. All three commands offload KV cache to host DRAM with `--hicache-size 135` (GB of host memory per rank, 540 GB per node). GLM-5.2 is a DSA model, so on Blackwell SGLang already defaults to an FP8 E4M3 KV cache and TRT-LLM DSA attention; the commands do not need to set them. + +Low-latency (TP4, MTP 5-1-6, up to 16 concurrent requests): +```bash Command +sglang serve \ + --model-path nvidia/GLM-5.2-NVFP4 \ + --trust-remote-code \ + --tp 4 \ + --quantization modelopt_fp4 \ + --fp4-gemm-backend flashinfer_trtllm \ + --max-running-requests 16 \ + --cuda-graph-max-bs-decode 16 \ + --chunked-prefill-size 8192 \ + --max-prefill-tokens 8192 \ + --mem-fraction-static 0.8 \ + --tool-call-parser glm47 \ + --reasoning-parser glm45 \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 5 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 6 \ + --enable-hierarchical-cache \ + --hicache-size 135 \ + --hicache-write-policy write_back \ + --hicache-io-backend direct +``` + +Balanced (DP attention, MTP 2-1-3, up to 256 concurrent requests): +```bash Command +sglang serve \ + --model-path nvidia/GLM-5.2-NVFP4 \ + --trust-remote-code \ + --tp 4 \ + --attn-dp-size 4 \ + --quantization modelopt_fp4 \ + --max-running-requests 256 \ + --chunked-prefill-size 8192 \ + --mem-fraction-static 0.92 \ + --tool-call-parser glm47 \ + --reasoning-parser glm45 \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 2 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 3 \ + --enable-hierarchical-cache \ + --hicache-size 135 \ + --hicache-write-policy write_back \ + --hicache-io-backend direct +``` + +High-throughput (DP attention, no MTP, up to 512 concurrent requests): +```bash Command +sglang serve \ + --model-path nvidia/GLM-5.2-NVFP4 \ + --trust-remote-code \ + --tp 4 \ + --attn-dp-size 4 \ + --quantization modelopt_fp4 \ + --max-running-requests 512 \ + --chunked-prefill-size 8192 \ + --mem-fraction-static 0.92 \ + --tool-call-parser glm47 \ + --reasoning-parser glm45 \ + --enable-hierarchical-cache \ + --hicache-size 135 \ + --hicache-write-policy write_back \ + --hicache-io-backend direct +``` + +With `--attn-dp-size 4`, `--chunked-prefill-size` is a global budget divided across the attention DP ranks, so the balanced and high-throughput commands prefill 2048 tokens per rank per chunk. diff --git a/docs/src/snippets/configs/zai-org/glm-5.2.jsx b/docs/src/snippets/configs/zai-org/glm-5.2.jsx index f14e317268..9fa84902fe 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.2.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.2.jsx @@ -817,40 +817,20 @@ sgl-eval run aime25 \\ { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, verified: true, - env: [ - "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK=1", - "SGLANG_ENABLE_THINKING=1", - "SGLANG_ENABLE_UNIFIED_RADIX_TREE=1", - "SGLANG_HICACHE_DEBUG_LOG=1", - "SGLANG_HICACHE_DEBUG_SAMPLE_RATE=16384", - "SGLANG_MOE_NVFP4_DISPATCH=1", - ], + env: [], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", "--quantization modelopt_fp4", - "--kv-cache-dtype fp8_e4m3", - "--dsa-prefill-backend trtllm", - "--dsa-decode-backend trtllm", - "--moe-runner-backend flashinfer_trtllm", - "--fp4-gemm-backend flashinfer_trtllm", - "--disable-shared-experts-fusion", - "--flashinfer-allreduce-fusion-backend auto", - // MTP 4-1-5 and HiCache (write_back, --hicache-size 135) match the InferenceX - // GB300 AgentX low-concurrency recipe (SemiAnalysisAI/InferenceX#3178). "--speculative-algorithm EAGLE", - "--speculative-num-steps 4", + "--speculative-num-steps 5", "--speculative-eagle-topk 1", - "--speculative-num-draft-tokens 5", + "--speculative-num-draft-tokens 6", "--chunked-prefill-size 8192", - "--max-prefill-tokens 8192", - "--mem-fraction-static 0.8", + "--mem-fraction-static 0.85", "--max-running-requests 16", "--cuda-graph-max-bs-decode 16", - "--enable-hierarchical-cache", - "--hicache-size 135", - "--hicache-write-policy write_back", - "--hicache-io-backend direct", + "--max-prefill-tokens 8192", "--host {{HOST_IP}}", "--port {{PORT}}", ], @@ -858,14 +838,7 @@ sgl-eval run aime25 \\ { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, verified: true, - env: [ - "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK=1", - "SGLANG_ENABLE_THINKING=1", - "SGLANG_ENABLE_UNIFIED_RADIX_TREE=1", - "SGLANG_HICACHE_DEBUG_LOG=1", - "SGLANG_HICACHE_DEBUG_SAMPLE_RATE=16384", - "SGLANG_MOE_NVFP4_DISPATCH=1", - ], + env: [], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4", @@ -887,14 +860,7 @@ sgl-eval run aime25 \\ { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, verified: true, - env: [ - "SGLANG_DISABLE_TP_MEMORY_INBALANCE_CHECK=1", - "SGLANG_ENABLE_THINKING=1", - "SGLANG_ENABLE_UNIFIED_RADIX_TREE=1", - "SGLANG_HICACHE_DEBUG_LOG=1", - "SGLANG_HICACHE_DEBUG_SAMPLE_RATE=16384", - "SGLANG_MOE_NVFP4_DISPATCH=1", - ], + env: [], flags: [ "--model-path {{MODEL_NAME}}", "--tp 4",