diff --git a/docs/src/snippets/_deployment.jsx b/docs/src/snippets/_deployment.jsx index 7b0ccd88f..0af3634df 100644 --- a/docs/src/snippets/_deployment.jsx +++ b/docs/src/snippets/_deployment.jsx @@ -578,8 +578,14 @@ export const Deployment = ({ config, benchmarks }) => { const findCell = (cells, sel) => cells.find((c) => DIMENSIONS.every((d) => c.match[d] === sel[d])); - const findBenchmark = (list, sel) => - (list || []).find((b) => DIMENSIONS.every((d) => b.match[d] === sel[d])) || null; + // Entries may also key on overlay dims (e.g. kvDsaPair): an entry applies + // only when every declared key equals the selection, and the most specific + // match wins, so plain hw×strategy entries stay the fallback. + const findBenchmark = (list, sel) => { + const hits = (list || []).filter((b) => + Object.entries(b.match || {}).every(([k, v]) => sel[k] === v)); + return hits.sort((a, b) => Object.keys(b.match).length - Object.keys(a.match).length)[0] || null; + }; // Accepts a single measurement object or an array; always returns an array. const normalizeSpeed = (speed) => { diff --git a/docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx b/docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx index 1a3281752..ace887a66 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx @@ -21,6 +21,27 @@ export const benchmarks = [ notes: "Measured on 4x GB300 (TP4/EP4) with the final weights (zai-org/GLM-5.3-Flash, c5b82b63e37b) at the rc2 cut (f13cb6f6a7), adaptive MTP 5/1/6 with SGLANG_SIMULATE_ACC_LEN=3 (accept length confirmed 3.00 in the bench summary and server log): 80 random requests at 1,024 input / 256 output tokens and concurrency 16 produced 1,821.97 aggregate output tok/s after two discarded warmups. Simulated accept length makes this a throughput-mechanism number. Accuracy is from the shared non-simulated full GSM8K gate: 97.50% with a 100% stop rate over all 1,319 problems.", }, + { + match: { hw: "gb300", strategy: "low-latency", kvDsaPair: "fp8-trtllm" }, + sglang_version: "f13cb6f6a7", + latencyPercentile: "Mean", + speed: [ + { + workload: { + dataset: "random", + isl: 1024, + osl: 256, + max_concurrency: 16, + num_prompts: 80, + }, + ttft_ms: 583.36, + tpot_ms: 6.21, + tokens_per_sec_per_gpu: 2357.1, + }, + ], + notes: + "The Low Latency recipe with FP8 KV + TRT-LLM DSA on 4x GB300, final weights (c5b82b63e37b) at rc2 (f13cb6f6a7), adaptive MTP 5/1/6 with SGLANG_SIMULATE_ACC_LEN=3 (accept 3.00): 80 random requests at 1,024 input / 256 output tokens and concurrency 16 produced 1,885.68 aggregate output tok/s — 3.5% above the BF16 + TileLang Low Latency row, with mean TPOT 6.21 ms vs 6.48 ms. Draft and target full-graph capture succeeded for this combination.", + }, { match: { hw: "gb300", strategy: "high-throughput" }, sglang_version: "f13cb6f6a7", @@ -67,6 +88,51 @@ export const benchmarks = [ notes: "Measured on 4x GB300 (TP4/EP4) with the final weights (zai-org/GLM-5.3-Flash, c5b82b63e37b) at the rc2 cut (f13cb6f6a7), speculative decoding off, after two discarded warmups per row: 1,128.32 / 2,418.65 / 3,884.95 aggregate output tok/s at concurrency 16 / 64 / 256 (80 / 320 / 1,280 random requests at 1,024 input / 256 output tokens; the high-concurrency rows with --max-running-requests 256 and decode graph batch 256). Throughput at 256 is still scaling but sublinear (prefill queueing). Accuracy is the full GSM8K gate on the same server: 97.50% with a 100% stop rate over all 1,319 problems.", }, + { + match: { hw: "gb300", strategy: "high-throughput", kvDsaPair: "fp8-trtllm" }, + sglang_version: "f13cb6f6a7", + latencyPercentile: "Mean", + speed: [ + { + workload: { + dataset: "random", + isl: 1024, + osl: 256, + max_concurrency: 16, + num_prompts: 80, + }, + ttft_ms: 681.3, + tpot_ms: 10.81, + tokens_per_sec_per_gpu: 1487.45, + }, + { + workload: { + dataset: "random", + isl: 1024, + osl: 256, + max_concurrency: 64, + num_prompts: 320, + }, + ttft_ms: 1693.9, + tpot_ms: 19.25, + tokens_per_sec_per_gpu: 3096.09, + }, + { + workload: { + dataset: "random", + isl: 1024, + osl: 256, + max_concurrency: 256, + num_prompts: 1280, + }, + ttft_ms: 5208.7, + tpot_ms: 41.96, + tokens_per_sec_per_gpu: 4965.65, + }, + ], + notes: + "FP8 KV cache with TRT-LLM DSA on 4x GB300, final weights (c5b82b63e37b) at rc2 (f13cb6f6a7), same protocol as the BF16 rows: 1,189.96 / 2,476.87 / 3,972.52 aggregate output tok/s at concurrency 16 / 64 / 256 — 2.3–5.5% above BF16 + TileLang across the curve, and the FP8 pool holds 13.5M tokens per rank vs 7.5M at BF16 (1.8x capacity at identical pool bytes). Sanity requests answered correctly and stopped cleanly; accuracy was not re-run for this variant (the 97.50% GSM8K gate used BF16 KV).", + }, { match: { hw: "h100", strategy: "low-latency" } }, { match: { hw: "h100", strategy: "high-throughput" } }, { match: { hw: "h200", strategy: "low-latency" } }, diff --git a/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx b/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx index 833561e45..0edc69018 100644 --- a/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx +++ b/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx @@ -239,7 +239,11 @@ sgl-eval run gsm8k \\ match: { hw: "gb300", strategy: "low-latency" }, nnodes: 1, verified: true, - verificationStatus: (s) => config.isRecommendedSelection(s) ? "verified" : "unverified", + verificationStatus: (s) => + config.isRecommendedSelection(s) || + (s.kvDsaPair === "fp8-trtllm" && s.mmTransport === "auto" && s.hicache === "off") + ? "verified" + : "unverified", env: [], flags: [ "--model-path {{MODEL_NAME}}", @@ -264,7 +268,11 @@ sgl-eval run gsm8k \\ match: { hw: "gb300", strategy: "high-throughput" }, nnodes: 1, verified: true, - verificationStatus: (s) => config.isRecommendedSelection(s) ? "verified" : "unverified", + verificationStatus: (s) => + config.isRecommendedSelection(s) || + (s.kvDsaPair === "fp8-trtllm" && s.mmTransport === "auto" && s.hicache === "off") + ? "verified" + : "unverified", env: [], flags: [ "--model-path {{MODEL_NAME}}",