From 3b64169f9d6f6b96425ad7eb83202316a2440599 Mon Sep 17 00:00:00 2001 From: zijiexia <37504505+zijiexia@users.noreply.github.com> Date: Fri, 4 Sep 2026 17:44:00 -0700 Subject: [PATCH] [Cookbook] Kimi-K3: add measured B300 1x8 Unified 8k/1k speed numbers (#37878) Co-authored-by: Claude Opus 5 --- .../autoregressive/Moonshotai/Kimi-K3.mdx | 21 ++++-- .../configs/moonshotai/kimi-k3-benchmarks.jsx | 72 +++++++++++++++++++ .../snippets/configs/moonshotai/kimi-k3.jsx | 8 +-- 3 files changed, 91 insertions(+), 10 deletions(-) diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx index b65e5cc2b..ea3f81d0e 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx @@ -81,6 +81,16 @@ import { KimiK3MambaRatioCalculator } from "/src/snippets/_kimi_k3_mamba_ratio_c + + B300 1×8 Unified speed numbers are measured on `v0.5.18 @ 71de97b2` with `--random-range-ratio 1.0`, + `--warmup-requests 64`, `--flush-cache`, at ISL 8192 / OSL 1024. DSPARK cells pin the acceptance + length via the serve env `SGLANG_SIMULATE_ACC_LEN=4.5` — they report what block size 7 delivers at + that acceptance, not a measured acceptance rate for this workload. Balanced DSPARK adds + `--max-running-requests 256`; without it speculation resets the cap to 48. The KDA state pool still + clamps admission below that (101 / 68 / 91 / 60 concurrent requests for MXFP4 NOSPEC / MXFP4 DSPARK / + NVFP4 NOSPEC / NVFP4 DSPARK), which is why no point past concurrency 64 is published for Balanced. + + ### Mamba ratio calculator @@ -124,9 +134,10 @@ are scheduled to release by July 27, 2026**. The recipes on this page were valid repository (`moonshotai/Kimi-K3`) and a public `lmsysorg/sglang` image with K3 support will be available at launch. -Every cell in the Deploy panel above is currently marked **Final Verification In Progress**: the -recipe runs, but its serving round on the final weights and current code is still open. Re-measure -throughput and accuracy before you rely on any of them. +The B300 1×8 `Unified` Low-Latency and Balanced cells are **Verified** — a speed round on the final +weights is published below. Every other cell is still marked **Final Verification In Progress**: the +recipe runs, but its serving round on the final weights and current code is still open. Accuracy has +not been re-measured on any cell — re-measure before you rely on one. **Recommended generation:** `temperature=1.0`, `top_p=0.95`, `presence_penalty=0`, `frequency_penalty=0` (fixed by the model; informational — do not hardcode in sample code). @@ -155,7 +166,7 @@ Speculation: DSPARK holds block size + 1 (= 8) intermediate states per request **Context length.** `--context-length` bounds the longest accepted request plus some context-scaled buffers; it does not size the KV pool. For long context the lever that adds capacity is `fp8_e4m3` KV. -**DSPARK.** Adds `--speculative-algorithm DSPARK` plus the draft checkpoint on top of the showing strategy. Leave `--speculative-draft-attention-backend` unset. No serving round on the final draft checkpoint has landed — measure against the same recipe running NOSPEC before adopting. +**DSPARK.** Adds `--speculative-algorithm DSPARK` plus the draft checkpoint on top of the showing strategy. Leave `--speculative-draft-attention-backend` unset. The published B300 DSPARK numbers pin the acceptance length with `SGLANG_SIMULATE_ACC_LEN`, so no measured acceptance rate exists for a real workload yet — measure against the same recipe running NOSPEC before adopting. **Per-platform notes:** @@ -179,7 +190,7 @@ Speculation: DSPARK holds block size + 1 (= 8) intermediate states per request - Calculator ratios run well above 1 here (`r > 1` is legal): `bfloat16` state buys admission, `fp8` KV buys context. - Don't use EP with an a2a backend: a2a buffers reclaim the KV that DCP buys. Compose only to measure. a2a backend is set when `--moe-a2a-backend` is set. -No cell has a serving round in this exact shape — treat them as starting points to verify. +Outside the two verified B300 1×8 `Unified` cells, no cell has a serving round in this exact shape — treat those as starting points to verify. diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3-benchmarks.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3-benchmarks.jsx index 525e0aeba..e3434ac20 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3-benchmarks.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3-benchmarks.jsx @@ -19,4 +19,76 @@ export const benchmarks = [ { match: { hw: "gb200", pdMode: "unified", strategy: "low-latency" } }, { match: { hw: "gb200", pdMode: "unified", strategy: "balanced" } }, { match: { hw: "gb200", pdMode: "unified", strategy: "high-throughput" } }, + { + match: { hw: "b300", pdMode: "unified", strategy: "low-latency", quant: "mxfp4", spec: "none" }, + sglang_version: "v0.5.18 @ 71de97b2", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 378, tpot_ms: 8.51, tokens_per_sec_per_gpu: 127 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 3539, tpot_ms: 19.47, tokens_per_sec_per_gpu: 785 }, + ], + }, + { + match: { hw: "b300", pdMode: "unified", strategy: "low-latency", quant: "mxfp4", spec: "dspark" }, + sglang_version: "v0.5.18 @ 71de97b2", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 389, tpot_ms: 2.84, tokens_per_sec_per_gpu: 351 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 3942, tpot_ms: 9.88, tokens_per_sec_per_gpu: 1319 }, + ], + }, + { + match: { hw: "b300", pdMode: "unified", strategy: "low-latency", quant: "nvfp4", spec: "none" }, + sglang_version: "v0.5.18 @ 71de97b2", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 369, tpot_ms: 10.12, tokens_per_sec_per_gpu: 107 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 3387, tpot_ms: 20.94, tokens_per_sec_per_gpu: 742 }, + ], + }, + { + match: { hw: "b300", pdMode: "unified", strategy: "low-latency", quant: "nvfp4", spec: "dspark" }, + sglang_version: "v0.5.18 @ 71de97b2", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 380, tpot_ms: 3.24, tokens_per_sec_per_gpu: 313 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 3765, tpot_ms: 9.77, tokens_per_sec_per_gpu: 1345 }, + ], + }, + { + match: { hw: "b300", pdMode: "unified", strategy: "balanced", quant: "mxfp4", spec: "none" }, + sglang_version: "v0.5.18 @ 71de97b2", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 11635, tpot_ms: 40.19, tokens_per_sec_per_gpu: 1395 }, + ], + }, + { + match: { hw: "b300", pdMode: "unified", strategy: "balanced", quant: "mxfp4", spec: "dspark" }, + sglang_version: "v0.5.18 @ 71de97b2", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 12038, tpot_ms: 24.47, tokens_per_sec_per_gpu: 1987 }, + ], + }, + { + match: { hw: "b300", pdMode: "unified", strategy: "balanced", quant: "nvfp4", spec: "none" }, + sglang_version: "v0.5.18 @ 71de97b2", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 11063, tpot_ms: 41.55, tokens_per_sec_per_gpu: 1373 }, + ], + }, + { + match: { hw: "b300", pdMode: "unified", strategy: "balanced", quant: "nvfp4", spec: "dspark" }, + sglang_version: "v0.5.18 @ 71de97b2", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 11664, tpot_ms: 22.17, tokens_per_sec_per_gpu: 1946 }, + ], + }, ]; diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx index 17ee2f4c5..da32d9af0 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx @@ -454,7 +454,7 @@ export const config = { --dataset-name {{DATASET}} \\ --random-input-len {{ISL}} --random-output-len {{OSL}} --random-range-ratio 1.0 \\ --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\ - --flush-cache`, + --warmup-requests 64 --flush-cache`, // num_prompts = 5 × concurrency (measured floor 16). numPromptsByConc: { 1: 16, 16: 80, 64: 320, 256: 1280, 1024: 5120 }, }, @@ -1013,8 +1013,7 @@ export const config = { { match: { hw: "b300", pdMode: "unified", strategy: "low-latency" }, nnodes: 1, - verified: false, - verificationStatus: "in-progress", + verified: true, env: [], // No --enable-symm-mem: it makes the fused all-reduce auto-probe skip. flags: [ @@ -1031,8 +1030,7 @@ export const config = { { match: { hw: "b300", pdMode: "unified", strategy: "balanced" }, nnodes: 1, - verified: false, - verificationStatus: "in-progress", + verified: true, env: [], flags: [ "--trust-remote-code",