minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Cookbook] Kimi-K3: add measured B300 1x8 Unified 8k/1k speed numbers (#37878)

Browse Source 
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
zijiexia
2026-09-04 17:44:00 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent a44bb397a9
commit 3b64169f9d
3 changed files with 91 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/src/snippets/configs/moonshotai/kimi-k3-benchmarks.jsx
+72
View File
@@ -19,4 +19,76 @@ export const benchmarks = [
{ match: { hw: "gb200", pdMode: "unified", strategy: "low-latency" } },
{ match: { hw: "gb200", pdMode: "unified", strategy: "balanced" } },
{ match: { hw: "gb200", pdMode: "unified", strategy: "high-throughput" } },
{
match: { hw: "b300", pdMode: "unified", strategy: "low-latency", quant: "mxfp4", spec: "none" },
sglang_version: "v0.5.18 @ 71de97b2",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 378, tpot_ms: 8.51, tokens_per_sec_per_gpu: 127 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 3539, tpot_ms: 19.47, tokens_per_sec_per_gpu: 785 },
],
},
{
match: { hw: "b300", pdMode: "unified", strategy: "low-latency", quant: "mxfp4", spec: "dspark" },
sglang_version: "v0.5.18 @ 71de97b2",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 389, tpot_ms: 2.84, tokens_per_sec_per_gpu: 351 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 3942, tpot_ms: 9.88, tokens_per_sec_per_gpu: 1319 },
],
},
{
match: { hw: "b300", pdMode: "unified", strategy: "low-latency", quant: "nvfp4", spec: "none" },
sglang_version: "v0.5.18 @ 71de97b2",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 369, tpot_ms: 10.12, tokens_per_sec_per_gpu: 107 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 3387, tpot_ms: 20.94, tokens_per_sec_per_gpu: 742 },
],
},
{
match: { hw: "b300", pdMode: "unified", strategy: "low-latency", quant: "nvfp4", spec: "dspark" },
sglang_version: "v0.5.18 @ 71de97b2",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 380, tpot_ms: 3.24, tokens_per_sec_per_gpu: 313 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 3765, tpot_ms: 9.77, tokens_per_sec_per_gpu: 1345 },
],
},
{
match: { hw: "b300", pdMode: "unified", strategy: "balanced", quant: "mxfp4", spec: "none" },
sglang_version: "v0.5.18 @ 71de97b2",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
ttft_ms: 11635, tpot_ms: 40.19, tokens_per_sec_per_gpu: 1395 },
],
},
{
match: { hw: "b300", pdMode: "unified", strategy: "balanced", quant: "mxfp4", spec: "dspark" },
sglang_version: "v0.5.18 @ 71de97b2",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
ttft_ms: 12038, tpot_ms: 24.47, tokens_per_sec_per_gpu: 1987 },
],
},
{
match: { hw: "b300", pdMode: "unified", strategy: "balanced", quant: "nvfp4", spec: "none" },
sglang_version: "v0.5.18 @ 71de97b2",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
ttft_ms: 11063, tpot_ms: 41.55, tokens_per_sec_per_gpu: 1373 },
],
},
{
match: { hw: "b300", pdMode: "unified", strategy: "balanced", quant: "nvfp4", spec: "dspark" },
sglang_version: "v0.5.18 @ 71de97b2",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
ttft_ms: 11664, tpot_ms: 22.17, tokens_per_sec_per_gpu: 1946 },
],
},
];
docs/src/snippets/configs/moonshotai/kimi-k3.jsx
+3 -5
View File
@@ -454,7 +454,7 @@ export const config = {
--dataset-name {{DATASET}} \\
--random-input-len {{ISL}} --random-output-len {{OSL}} --random-range-ratio 1.0 \\
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\
--flush-cache`,
--warmup-requests 64 --flush-cache`,
// num_prompts = 5 × concurrency (measured floor 16).
numPromptsByConc: { 1: 16, 16: 80, 64: 320, 256: 1280, 1024: 5120 },
},
@@ -1013,8 +1013,7 @@ export const config = {
{
match: { hw: "b300", pdMode: "unified", strategy: "low-latency" },
nnodes: 1,
verified: false,
verificationStatus: "in-progress",
verified: true,
env: [],
// No --enable-symm-mem: it makes the fused all-reduce auto-probe skip.
flags: [
@@ -1031,8 +1030,7 @@ export const config = {
{
match: { hw: "b300", pdMode: "unified", strategy: "balanced" },
nnodes: 1,
verified: false,
verificationStatus: "in-progress",
verified: true,
env: [],
flags: [
"--trust-remote-code",