diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx index 61e4227a4..7ddbeb2c8 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx @@ -92,7 +92,7 @@ The deployment recipes use the checkpoint's generation configuration. Override s ### Choose a strategy -Start with **Low Latency** for chat and agent workloads. Adaptive MTP changes the draft depth as acceptance changes, reducing unnecessary draft work when the server is busy. Measure **High Throughput** for heavily batched traffic where disabling speculative decoding can be more efficient. SGLang serves MTP through `--speculative-algorithm EAGLE`, so generated commands use that flag value. +Start with **Low Latency** for chat and agent workloads. Adaptive MTP changes the draft depth as acceptance changes, reducing unnecessary draft work when the server is busy. Measure **High Throughput** for heavily batched traffic where disabling speculative decoding can be more efficient. SGLang serves MTP through `--speculative-algorithm EAGLE` (upstream folds the older NEXTN spelling into EAGLE), so generated commands use that flag value. Strategy labels describe the workload goal. Both strategies stay available on NVIDIA GPUs; the AMD ROCm recipes expose only High Throughput until MTP speculative decoding is validated there. @@ -106,10 +106,14 @@ Keep the checkpoint's KDA lower-bound setting unchanged. In particular, do not o ### Keep the KV and DSA backends paired -On Blackwell, the recipes default to an FP8 KV cache with TRT-LLM DSA: on GB300 this pairing measured 2.3–5.5% higher throughput and about 1.8x the KV token capacity at identical pool bytes, with GSM8K accuracy within noise of BF16. BF16 KV with TileLang DSA remains selectable in the deployment panel and is the default on H100 and H200, where FP8 KV with TRT-LLM DSA is disabled. Switch the dtype and both DSA backends together; TileLang DSA with FP8 KV is not a valid CUDA combination. +On Blackwell, the recipes default to an FP8 KV cache with TRT-LLM DSA: on GB300 this pairing measured 2.9–5.7% higher throughput and about 1.8x the KV token capacity at identical pool bytes, with GSM8K accuracy within noise of BF16. BF16 KV with TileLang DSA remains selectable in the deployment panel and is the default on H100 and H200, where FP8 KV with TRT-LLM DSA is disabled. Switch the dtype and both DSA backends together; TileLang DSA with FP8 KV is not a valid CUDA combination. On AMD ROCm, use BF16 KV cache with TileLang DSA, set `SGLANG_USE_AITER=1`, keep the MoE runner on Triton, and disable CUDA graphs. The ROCm recipe uses TP8 on a single eight-GPU node. MI300X and MI325X both use gfx942, but the MI325X entry remains explicitly unverified because it is inferred from MI300X rather than measured directly. AMD validation covers text generation and GSM8K only; multimodal serving remains unverified. +### Decode context parallelism + +Decode context parallelism (DCP) shards the KV across GPUs during decode to cut long-context latency. The **Context Parallelism** row offers DCP4, validated on 4x GB300 (TP4/EP4) with both KV pairings and adaptive MTP 5/1/6; it requires the current release image, which carries the TileLang LSE fix. Other platforms and attention backends are unvalidated, and draft-extend v2 is unsupported under DCP. + ### Extend the cache hierarchy Keep **HiCache** off when GPU memory is sufficient. Select **L1 + L2** to spill reusable cache entries into host memory. Select **+ L3** only after configuring Mooncake on every serving node; the generated command exposes the required configuration path. These options remain selectable but are marked **Not Verified** until the resulting command is validated on the chosen hardware. diff --git a/docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx b/docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx index 521df67b9..e433138b2 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx @@ -1,7 +1,7 @@ export const benchmarks = [ { match: { hw: "gb300", strategy: "low-latency" }, - sglang_version: "f13cb6f6a7", + sglang_version: "d6ab04bdf1", latencyPercentile: "Mean", speed: [ { @@ -12,18 +12,18 @@ export const benchmarks = [ max_concurrency: 16, num_prompts: 80, }, - ttft_ms: 599.91, - tpot_ms: 6.48, - tokens_per_sec_per_gpu: 2266.08, + ttft_ms: 598.41, + tpot_ms: 6.43, + tokens_per_sec_per_gpu: 2280.83, }, ], accuracy: { gsm8k_pct: 97.50 }, notes: - "Measured on 4x GB300 (TP4/EP4) with the final weights (zai-org/GLM-5.3-Flash, c5b82b63e37b) at the rc2 cut (f13cb6f6a7), adaptive MTP 5/1/6 with SGLANG_SIMULATE_ACC_LEN=3 (accept length confirmed 3.00 in the bench summary and server log): 80 random requests at 1,024 input / 256 output tokens and concurrency 16 produced 1,812.86 aggregate output tok/s after two discarded warmups. Simulated accept length makes this a throughput-mechanism number. Accuracy is from the shared non-simulated full GSM8K gate: 97.50% with a 100% stop rate over all 1,319 problems.", + "Measured on 4x GB300 (TP4/EP4) with the final weights (zai-org/GLM-5.3-Flash, c5b82b63e37b) on the current release-image tree (d6ab04bdf1), adaptive MTP 5/1/6 with SGLANG_SIMULATE_ACC_LEN=3 (accept length confirmed 3.00 in the bench summary and server log): 80 random requests at 1,024 input / 256 output tokens and concurrency 16 produced 1,824.66 aggregate output tok/s after two discarded warmups. Simulated accept length makes this a throughput-mechanism number. Accuracy is from the shared non-simulated full GSM8K gate: 97.50% with a 100% stop rate over all 1,319 problems.", }, { match: { hw: "gb300", strategy: "low-latency", kvDsaPair: "fp8-trtllm" }, - sglang_version: "f13cb6f6a7", + sglang_version: "d6ab04bdf1", latencyPercentile: "Mean", speed: [ { @@ -34,17 +34,17 @@ export const benchmarks = [ max_concurrency: 16, num_prompts: 80, }, - ttft_ms: 604.2, - tpot_ms: 6.19, - tokens_per_sec_per_gpu: 2338.53, + ttft_ms: 609.9, + tpot_ms: 6.25, + tokens_per_sec_per_gpu: 2317.35, }, ], notes: - "The Low Latency recipe with FP8 KV + TRT-LLM DSA on 4x GB300, final weights (c5b82b63e37b) at rc2 (f13cb6f6a7), adaptive MTP 5/1/6 with SGLANG_SIMULATE_ACC_LEN=3 (accept 3.00): 80 random requests at 1,024 input / 256 output tokens and concurrency 16 produced 1,870.82 aggregate output tok/s — 3.2% above the BF16 + TileLang Low Latency row, with mean TPOT 6.19 ms vs 6.48 ms. Draft and target full-graph capture succeeded for this combination.", + "The Low Latency recipe with FP8 KV + TRT-LLM DSA on 4x GB300, final weights (c5b82b63e37b) on the current release-image tree (d6ab04bdf1), adaptive MTP 5/1/6 with SGLANG_SIMULATE_ACC_LEN=3 (accept 3.00): 80 random requests at 1,024 input / 256 output tokens and concurrency 16 produced 1,853.88 aggregate output tok/s — 1.6% above the BF16 + TileLang Low Latency row, with mean TPOT 6.25 ms vs 6.43 ms. Draft and target full-graph capture succeeded for this combination. The speed rows were measured with the NEXTN spelling and --disable-shared-experts-fusion, which resolve to the same runtime path as the published command on this tree.", }, { - match: { hw: "gb300", strategy: "high-throughput" }, - sglang_version: "f13cb6f6a7", + match: { hw: "gb300", strategy: "low-latency", kvDsaPair: "fp8-trtllm", dcp: "4" }, + sglang_version: "d6ab04bdf1", latencyPercentile: "Mean", speed: [ { @@ -55,9 +55,51 @@ export const benchmarks = [ max_concurrency: 16, num_prompts: 80, }, - ttft_ms: 684.63, - tpot_ms: 11.53, - tokens_per_sec_per_gpu: 1410.4, + ttft_ms: 534.1, + tpot_ms: 7.31, + tokens_per_sec_per_gpu: 2100.76, + }, + ], + notes: + "The Low Latency recipe with FP8 KV + TRT-LLM DSA and DCP4 (--dcp-size 4 --dcp-comm-backend a2a --dcp-replicate-q-proj) on 4x GB300, final weights (c5b82b63e37b) on the d6ab04bdf1 tree, adaptive MTP 5/1/6 with full decode graph: 80 random requests at 1,024 input / 256 output tokens and concurrency 16 produced 1,680.61 aggregate output tok/s at a 3.937 accept length — about 10% below the non-DCP FP8 Low Latency row. TRT-LLM DSA DCP decode returns the LSE natively, so this arm needs no patch.", + }, + { + match: { hw: "gb300", strategy: "low-latency", kvDsaPair: "bf16-tilelang", dcp: "4" }, + sglang_version: "d6ab04bdf1", + latencyPercentile: "Mean", + speed: [ + { + workload: { + dataset: "random", + isl: 1024, + osl: 256, + max_concurrency: 16, + num_prompts: 80, + }, + ttft_ms: 510.0, + tpot_ms: 8.0, + tokens_per_sec_per_gpu: 1957.25, + }, + ], + notes: + "The Low Latency recipe with BF16 KV + TileLang DSA and DCP4 on 4x GB300, final weights (c5b82b63e37b) on the d6ab04bdf1 tree, adaptive MTP 5/1/6 with full decode graph: 80 random requests at 1,024 input / 256 output tokens and concurrency 16 produced 1,565.8 aggregate output tok/s at a 3.90 accept length. TileLang DSA DCP decode needs the LSE fix that ships in the current release image.", + }, + { + match: { hw: "gb300", strategy: "high-throughput" }, + sglang_version: "d6ab04bdf1", + latencyPercentile: "Mean", + speed: [ + { + workload: { + dataset: "random", + isl: 1024, + osl: 256, + max_concurrency: 16, + num_prompts: 80, + }, + ttft_ms: 586.33, + tpot_ms: 11.51, + tokens_per_sec_per_gpu: 1451.53, }, { workload: { @@ -67,9 +109,9 @@ export const benchmarks = [ max_concurrency: 64, num_prompts: 320, }, - ttft_ms: 1691.64, - tpot_ms: 19.88, - tokens_per_sec_per_gpu: 3023.31, + ttft_ms: 1461.23, + tpot_ms: 18.36, + tokens_per_sec_per_gpu: 3325.3, }, { workload: { @@ -79,18 +121,18 @@ export const benchmarks = [ max_concurrency: 256, num_prompts: 1280, }, - ttft_ms: 5192.23, - tpot_ms: 43.35, - tokens_per_sec_per_gpu: 4856.19, + ttft_ms: 3912.74, + tpot_ms: 37.69, + tokens_per_sec_per_gpu: 6035.41, }, ], accuracy: { gsm8k_pct: 97.50 }, notes: - "Measured on 4x GB300 (TP4/EP4) with the final weights (zai-org/GLM-5.3-Flash, c5b82b63e37b) at the rc2 cut (f13cb6f6a7), speculative decoding off, after two discarded warmups per row: 1,128.32 / 2,418.65 / 3,884.95 aggregate output tok/s at concurrency 16 / 64 / 256 (80 / 320 / 1,280 random requests at 1,024 input / 256 output tokens). The measured server added --max-running-requests 256, --cuda-graph-backend-decode full, --cuda-graph-max-bs-decode 256, --chunked-prefill-size 8192, --max-prefill-tokens 8192, and --disable-shared-experts-fusion to the cell command — the concurrency 64/256 rows are not reproducible at the cell's automatic sizing. Throughput at 256 is still scaling but sublinear (prefill queueing). Accuracy is the full GSM8K gate on the same server: 97.50% with a 100% stop rate over all 1,319 problems. With HiCache L1+L2 (32 GB host tier) the same protocol measured 1,125.45 / 2,403.57 / 3,839.57 tok/s — a 0.3-1.2% overhead; the random dataset has no prefix reuse, so L2 benefit was not exercised.", + "Measured on 4x GB300 (TP4/EP4) with the final weights (zai-org/GLM-5.3-Flash, c5b82b63e37b) on the current release-image tree (d6ab04bdf1), speculative decoding off, after two discarded warmups per row: 1,161.22 / 2,660.24 / 4,828.33 aggregate output tok/s at concurrency 16 / 64 / 256 (80 / 320 / 1,280 random requests at 1,024 input / 256 output tokens). The server ran exactly the published cell command. Throughput at 256 is still scaling but sublinear (prefill queueing). Accuracy is from the shared non-simulated full GSM8K gate: 97.50% with a 100% stop rate over all 1,319 problems. With HiCache L1+L2 (32 GB host tier, 16k prefill chunks) the same protocol measured 1,202.07 / 2,696.20 / 4,634.47 tok/s — within 4% of the non-HiCache rows; the random dataset has no prefix reuse, so L2 benefit was not exercised.", }, { match: { hw: "gb300", strategy: "high-throughput", kvDsaPair: "fp8-trtllm" }, - sglang_version: "f13cb6f6a7", + sglang_version: "d6ab04bdf1", latencyPercentile: "Mean", speed: [ { @@ -101,9 +143,9 @@ export const benchmarks = [ max_concurrency: 16, num_prompts: 80, }, - ttft_ms: 681.3, - tpot_ms: 10.81, - tokens_per_sec_per_gpu: 1487.45, + ttft_ms: 581.52, + tpot_ms: 10.79, + tokens_per_sec_per_gpu: 1533.84, }, { workload: { @@ -113,9 +155,9 @@ export const benchmarks = [ max_concurrency: 64, num_prompts: 320, }, - ttft_ms: 1693.9, - tpot_ms: 19.25, - tokens_per_sec_per_gpu: 3096.09, + ttft_ms: 1459.62, + tpot_ms: 17.68, + tokens_per_sec_per_gpu: 3423.26, }, { workload: { @@ -125,14 +167,14 @@ export const benchmarks = [ max_concurrency: 256, num_prompts: 1280, }, - ttft_ms: 5208.7, - tpot_ms: 41.96, - tokens_per_sec_per_gpu: 4965.65, + ttft_ms: 3908.3, + tpot_ms: 36.12, + tokens_per_sec_per_gpu: 6221.28, }, ], accuracy: { gsm8k_pct: 97.35 }, notes: - "FP8 KV cache with TRT-LLM DSA on 4x GB300, final weights (c5b82b63e37b) at rc2 (f13cb6f6a7), same protocol as the BF16 rows: 1,189.96 / 2,476.87 / 3,972.52 aggregate output tok/s at concurrency 16 / 64 / 256 — 2.3–5.5% above BF16 + TileLang across the curve, and the FP8 pool holds 13.5M tokens per rank vs 7.5M at BF16 (1.8x capacity at identical pool bytes). Accuracy is the full GSM8K gate on this variant: 97.35% vs 97.50% on BF16 KV, a 0.15-point gap inside sampling noise, with a 99.92% stop rate (one truncated run of 1,319). With HiCache L1+L2 (32 GB host tier) the same protocol measured 1,187.13 / 2,464.89 / 3,941.37 tok/s — a 0.2-0.8% overhead; the random dataset has no prefix reuse, so L2 benefit was not exercised.", + "FP8 KV cache with TRT-LLM DSA on 4x GB300, final weights (c5b82b63e37b) on the current release-image tree (d6ab04bdf1), same protocol as the BF16 rows: 1,227.07 / 2,738.61 / 4,977.02 aggregate output tok/s at concurrency 16 / 64 / 256 — 2.9–5.7% above BF16 + TileLang across the curve, and the FP8 pool holds 12.6M tokens per rank vs 7.0M at BF16 (1.8x capacity at identical pool bytes). Accuracy is the full GSM8K gate on this variant: 97.35% vs 97.50% on BF16 KV, a 0.15-point gap inside sampling noise, with a 100% stop rate over all 1,319 problems. With HiCache L1+L2 (32 GB host tier, 16k prefill chunks) the same protocol measured 1,263.85 / 2,763.31 / 4,773.95 tok/s — within 5% of the non-HiCache rows; the random dataset has no prefix reuse, so L2 benefit was not exercised.", }, { match: { hw: "h100", strategy: "low-latency" }, diff --git a/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx b/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx index 43a2382b2..f04aae83e 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx @@ -30,7 +30,8 @@ export const config = { return ( s.kvDsaPair === pairing && s.mmTransport === "auto" && - s.hicache === "off" + s.hicache === "off" && + s.dcp === "off" ); }, @@ -90,8 +91,6 @@ export const config = { label: "L1 + L2", subtitle: "Host memory", flags: ["--enable-hierarchical-cache", "--hicache-size 32"], - disabled: (s) => s.strategy === "low-latency", - disableReason: "HiCache with MTP speculative decoding crashes at startup in the current build (DSA draft pool lacks full_kv_pool); use it with High Throughput only.", hints: ["32 GB host tier; the default ratio can demand more host RAM than the node has free."], }, { @@ -100,12 +99,26 @@ export const config = { subtitle: "Mooncake", flags: ["--enable-hierarchical-cache", "--hicache-size 32", "--hicache-storage-backend mooncake"], env: ["SGLANG_HICACHE_MOONCAKE_CONFIG_PATH={{MOONCAKE_CONFIG}}"], - disabled: (s) => s.strategy === "low-latency", - disableReason: "HiCache with MTP speculative decoding crashes at startup in the current build (DSA draft pool lacks full_kv_pool); use it with High Throughput only.", hints: ["Start Mooncake and place the configuration file on every serving node."], }, ], }, + { + id: "dcp", + title: "Context Parallelism", + default: "off", + options: [ + { id: "off", label: "Off" }, + { + id: "4", + label: "DCP 4", + disabled: (s) => s.hw !== "gb300", + disableReason: "DCP is validated only on 4x GB300 TP4/EP4 for now.", + flags: ["--dcp-size 4", "--dcp-comm-backend a2a", "--dcp-replicate-q-proj"], + hints: ["Measured on 4x GB300 with both KV/DSA pairings, adaptive MTP 5/1/6, full decode graph."], + }, + ], + }, ], modelNames: { @@ -264,7 +277,8 @@ sgl-eval run gsm8k \\ verificationStatus: (s) => ["bf16-tilelang", "fp8-trtllm"].includes(s.kvDsaPair) && s.mmTransport === "auto" && - s.hicache === "off" + s.hicache === "off" && + ["off", "4"].includes(s.dcp) ? "verified" : "unverified", env: [], @@ -276,7 +290,6 @@ sgl-eval run gsm8k \\ "--dsa-decode-backend trtllm", "--kv-cache-dtype fp8_e4m3", "--moe-runner-backend deep_gemm", - "--disable-shared-experts-fusion", "--speculative-algorithm EAGLE", "--speculative-num-steps 5", "--speculative-eagle-topk 1", @@ -295,7 +308,8 @@ sgl-eval run gsm8k \\ verificationStatus: (s) => ["bf16-tilelang", "fp8-trtllm"].includes(s.kvDsaPair) && s.mmTransport === "auto" && - s.hicache === "off" + s.hicache === "off" && + s.dcp === "off" ? "verified" : "unverified", env: [], @@ -307,7 +321,6 @@ sgl-eval run gsm8k \\ "--dsa-decode-backend trtllm", "--kv-cache-dtype fp8_e4m3", "--moe-runner-backend deep_gemm", - "--disable-shared-experts-fusion", "--reasoning-parser glm45", "--tool-call-parser glm47", "--host {{HOST_IP}}", @@ -332,7 +345,6 @@ sgl-eval run gsm8k \\ "--dsa-decode-backend tilelang", "--kv-cache-dtype bfloat16", "--moe-runner-backend deep_gemm", - "--disable-shared-experts-fusion", "--speculative-algorithm EAGLE", "--speculative-num-steps 5", "--speculative-eagle-topk 1", @@ -360,7 +372,6 @@ sgl-eval run gsm8k \\ "--dsa-decode-backend tilelang", "--kv-cache-dtype bfloat16", "--moe-runner-backend deep_gemm", - "--disable-shared-experts-fusion", "--reasoning-parser glm45", "--tool-call-parser glm47", "--host {{HOST_IP}}", @@ -385,7 +396,6 @@ sgl-eval run gsm8k \\ "--dsa-decode-backend tilelang", "--kv-cache-dtype bfloat16", "--moe-runner-backend deep_gemm", - "--disable-shared-experts-fusion", "--speculative-algorithm EAGLE", "--speculative-num-steps 5", "--speculative-eagle-topk 1", @@ -412,7 +422,6 @@ sgl-eval run gsm8k \\ "--dsa-decode-backend tilelang", "--kv-cache-dtype bfloat16", "--moe-runner-backend deep_gemm", - "--disable-shared-experts-fusion", "--reasoning-parser glm45", "--tool-call-parser glm47", "--host {{HOST_IP}}", @@ -433,7 +442,6 @@ sgl-eval run gsm8k \\ "--dsa-decode-backend trtllm", "--kv-cache-dtype fp8_e4m3", "--moe-runner-backend deep_gemm", - "--disable-shared-experts-fusion", "--speculative-algorithm EAGLE", "--speculative-num-steps 5", "--speculative-eagle-topk 1", @@ -460,7 +468,6 @@ sgl-eval run gsm8k \\ "--dsa-decode-backend trtllm", "--kv-cache-dtype fp8_e4m3", "--moe-runner-backend deep_gemm", - "--disable-shared-experts-fusion", "--reasoning-parser glm45", "--tool-call-parser glm47", "--host {{HOST_IP}}", @@ -481,7 +488,6 @@ sgl-eval run gsm8k \\ "--dsa-decode-backend trtllm", "--kv-cache-dtype fp8_e4m3", "--moe-runner-backend deep_gemm", - "--disable-shared-experts-fusion", "--speculative-algorithm EAGLE", "--speculative-num-steps 5", "--speculative-eagle-topk 1", @@ -508,7 +514,6 @@ sgl-eval run gsm8k \\ "--dsa-decode-backend trtllm", "--kv-cache-dtype fp8_e4m3", "--moe-runner-backend deep_gemm", - "--disable-shared-experts-fusion", "--reasoning-parser glm45", "--tool-call-parser glm47", "--host {{HOST_IP}}", @@ -528,7 +533,6 @@ sgl-eval run gsm8k \\ "--dsa-decode-backend trtllm", "--kv-cache-dtype fp8_e4m3", "--moe-runner-backend deep_gemm", - "--disable-shared-experts-fusion", "--speculative-algorithm EAGLE", "--speculative-num-steps 5", "--speculative-eagle-topk 1", @@ -553,7 +557,6 @@ sgl-eval run gsm8k \\ "--dsa-decode-backend trtllm", "--kv-cache-dtype fp8_e4m3", "--moe-runner-backend deep_gemm", - "--disable-shared-experts-fusion", "--reasoning-parser glm45", "--tool-call-parser glm47", "--host {{HOST_IP}}",