[Docs] Add GB300 cells and benchmarks for Qwen3.8-27B (#34863)
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com> Co-authored-by: yhyang201 <yhyang201@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
zijiexia
co-authored by
Xinyuan Tong
yhyang201
Claude Opus 5
Xinyuan Tong
yhyang201
Claude Opus 5
parent
29c6be15a4
commit
70e291b70f
@@ -161,9 +161,9 @@ checkpoint's calibration scales automatically.
|
||||
requires a FlashInfer build whose prefill `plan` accepts `uniform_q_len`
|
||||
(newer than 0.6.15.post1); otherwise run spec with `--attention-backend triton`.
|
||||
On DGX Spark the 128GB is unified memory shared with the host CPU, so all
|
||||
three checkpoints fit; its cells use 8192-token prefill chunks and
|
||||
`--mem-fraction-static 0.95`. The SM121 recipe is not yet validated on that
|
||||
platform.
|
||||
three checkpoints fit; its cells use 8192-token prefill chunks,
|
||||
`--mem-fraction-static 0.95`, and `--disable-prefill-cuda-graph`. The SM121
|
||||
recipe is not yet validated on that platform.
|
||||
- **H200 (SM90)**: BF16 and FP8 only — the card has no FP4 tensor cores, so the
|
||||
NVFP4 checkpoint's MLP would fall back to the Marlin W4A16 weight-only path
|
||||
and its cell is greyed out. The H200 recipes use 32768-token prefill chunks
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
// Qwen3.8-27B per-cell benchmark numbers, keyed by the same `match` tuple as
|
||||
// qwen3.8-27b.jsx cells. See _deployment.jsx for the speed/accuracy schema.
|
||||
//
|
||||
// All six rows are ONE-BATCH measurements (sglang.bench_serving --flush-cache,
|
||||
// random dataset, ISL=1024 / OSL=1024, --random-range-ratio 1, request-rate inf,
|
||||
// max_concurrency 1/16/64, n=64/64/256 respectively) on a single GB300 GPU
|
||||
// (Blackwell Ultra SM103, 288GB HBM) on 2026-08-14. SGLang binary:
|
||||
// lmsysorg/sglang:dev resolving to commit c4271c3fe1262fc2adbd162c33b25de5255251c5.
|
||||
// With no --attention-backend pin, that commit on GB300 resolves attention to
|
||||
// triton; a newer sglang (c7c03ec+) resolves trtllm_mha. Same binary and
|
||||
// protocol for all six cells so the numbers are apples-to-apples. MTP rows
|
||||
// show mean speculative accept_length across all decode batches.
|
||||
//
|
||||
// Accuracy (sgl-eval run gsm8k, full 1319, stop_rate 1.0, truncated_rate 0.0):
|
||||
// only the W4A4-NVFP4 checkpoint has been GSM8K'd on a GB300 single-GPU box —
|
||||
// via a sibling experiment pinned to sglang c7c03ec53b1e664c2d415db4f02e43f86661f31d
|
||||
// with an explicit `--kv-cache-dtype` A/B. The bf16-KV row (96.82%) is the better
|
||||
// baseline to cite; the default fp8-KV the NVFP4 checkpoint auto-enables scores
|
||||
// 96.44%. FP8 / BF16 GB300 GSM8K has not been measured and stays null below
|
||||
// (see journal 2026-08-14-1048-claude-jrn_f6c1265be8cbdf86c44fe36c).
|
||||
export const benchmarks = [
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "lmsysorg/sglang:dev @ c4271c3fe",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 1, num_prompts: 64 },
|
||||
ttft_ms: 79, tpot_ms: 6.4, tokens_per_sec_per_gpu: 155 },
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 16, num_prompts: 64 },
|
||||
ttft_ms: 715, tpot_ms: 8.5, tokens_per_sec_per_gpu: 1742 },
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 64, num_prompts: 256 },
|
||||
ttft_ms: 1216, tpot_ms: 13.6, tokens_per_sec_per_gpu: 4316 },
|
||||
],
|
||||
accuracy: { gsm8k_pct: 96.44 },
|
||||
notes: "NVFP4 = RadixArk W4A4-0811 (private). KV auto fp8_e4m3 from ckpt-declared kv_cache_quant_algo. GSM8K fp8-KV = 96.44 / bf16-KV = 96.82 (sgl-eval, `c7c03ec`, sibling experiment on `/scratch/qwen38-w4a4-kv-ab-0814` on `qwen38-27b-nvfp4-convert-0812`); both stop_rate 100% / truncated 0% on full 1319.",
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "lmsysorg/sglang:dev @ c4271c3fe",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 1, num_prompts: 64 },
|
||||
ttft_ms: 73, tpot_ms: 2.3, tokens_per_sec_per_gpu: 415 },
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 16, num_prompts: 64 },
|
||||
ttft_ms: 216, tpot_ms: 4.4, tokens_per_sec_per_gpu: 3256 },
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 64, num_prompts: 256 },
|
||||
ttft_ms: 3085, tpot_ms: 8.6, tokens_per_sec_per_gpu: 5155 },
|
||||
],
|
||||
accuracy: { gsm8k_pct: 96.44 },
|
||||
notes: "MTP accept_length mean ~3.31 across conc. Suspected cap is `--speculative-num-draft-tokens 4` (max 4 accepted/reject step).",
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "lmsysorg/sglang:dev @ c4271c3fe",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 1, num_prompts: 64 },
|
||||
ttft_ms: 73, tpot_ms: 8.2, tokens_per_sec_per_gpu: 121 },
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 16, num_prompts: 64 },
|
||||
ttft_ms: 586, tpot_ms: 11.2, tokens_per_sec_per_gpu: 1355 },
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 64, num_prompts: 256 },
|
||||
ttft_ms: 1539, tpot_ms: 19.7, tokens_per_sec_per_gpu: 3012 },
|
||||
],
|
||||
accuracy: { gsm8k_pct: null },
|
||||
notes: "FP8 = Qwen/Qwen3.8-27B-FP8 revision 032ba94c96507998d543d5a43f7b4ebcfa6b4fa9 (private). GSM8K not yet measured on GB300 for FP8; TP4 B300 sibling experiment scored it 1276/1319 (96.74%) under thinking sampling.",
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "lmsysorg/sglang:dev @ c4271c3fe",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 1, num_prompts: 64 },
|
||||
ttft_ms: 74, tpot_ms: 3.2, tokens_per_sec_per_gpu: 302 },
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 16, num_prompts: 64 },
|
||||
ttft_ms: 174, tpot_ms: 5.3, tokens_per_sec_per_gpu: 2771 },
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 64, num_prompts: 256 },
|
||||
ttft_ms: 3376, tpot_ms: 9.5, tokens_per_sec_per_gpu: 4599 },
|
||||
],
|
||||
accuracy: { gsm8k_pct: null },
|
||||
notes: "MTP accept_length mean ~3.16.",
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "lmsysorg/sglang:dev @ c4271c3fe",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 1, num_prompts: 64 },
|
||||
ttft_ms: 72, tpot_ms: 10.3, tokens_per_sec_per_gpu: 97 },
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 16, num_prompts: 64 },
|
||||
ttft_ms: 490, tpot_ms: 12.5, tokens_per_sec_per_gpu: 1235 },
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 64, num_prompts: 256 },
|
||||
ttft_ms: 1622, tpot_ms: 18.4, tokens_per_sec_per_gpu: 3208 },
|
||||
],
|
||||
accuracy: { gsm8k_pct: null },
|
||||
notes: "BF16 = Qwen/Qwen3.8-27B revision 08cb37595cfe9ce298e6f172c6b50d814ef42413 (private). GSM8K not yet measured on GB300 for BF16.",
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "lmsysorg/sglang:dev @ c4271c3fe",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 1, num_prompts: 64 },
|
||||
ttft_ms: 76, tpot_ms: 4.0, tokens_per_sec_per_gpu: 245 },
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 16, num_prompts: 64 },
|
||||
ttft_ms: 211, tpot_ms: 6.3, tokens_per_sec_per_gpu: 2306 },
|
||||
{ workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 64, num_prompts: 256 },
|
||||
ttft_ms: 3894, tpot_ms: 10.9, tokens_per_sec_per_gpu: 3995 },
|
||||
],
|
||||
accuracy: { gsm8k_pct: null },
|
||||
notes: "MTP accept_length mean ~3.22.",
|
||||
},
|
||||
];
|
||||
@@ -37,7 +37,7 @@
|
||||
export const config = {
|
||||
modelName: "Qwen3.8-27B",
|
||||
|
||||
supportedHardware: ["h200", "rtx6000", "rtx5090", "dgx-spark"],
|
||||
supportedHardware: ["h200", "rtx6000", "rtx5090", "dgx-spark", "gb300"],
|
||||
|
||||
// RTX PRO 6000 and RTX 5090 (SM120 / Blackwell Desktop) are workstation and
|
||||
// consumer cards, not datacenter GPUs, so they are not in the shared catalog.
|
||||
@@ -69,6 +69,7 @@ export const config = {
|
||||
// Playground's speculative axis instead of splitting the strategy dimension.
|
||||
strategies: [
|
||||
{ id: "balanced", label: "Balanced" },
|
||||
{ id: "high-throughput", label: "High-Throughput" },
|
||||
],
|
||||
nodesOptions: [
|
||||
{ id: "single", label: "Single Node" },
|
||||
@@ -130,6 +131,7 @@ export const config = {
|
||||
// TODO: verify an arm64 build of this tag for DGX Spark (GB10 is aarch64);
|
||||
// the x86-only tag will not pull there.
|
||||
"dgx-spark": "lmsysorg/sglang:qwen38-27b",
|
||||
gb300: "lmsysorg/sglang:dev",
|
||||
},
|
||||
|
||||
github: {
|
||||
@@ -384,8 +386,9 @@ export const config = {
|
||||
// DGX Spark (GB10, SM121): single node, 128GB coherent unified memory
|
||||
// shared with the CPU — every checkpoint fits, so all three quants get a
|
||||
// cell. FlashInfer attention comes from the SM120 pair; the platform gets
|
||||
// its own operating point at 8192-token prefill chunks and 0.95 static
|
||||
// fraction. Unvalidated on SM121 / aarch64.
|
||||
// its own operating point at 8192-token prefill chunks, 0.95 static
|
||||
// fraction, and prefill CUDA graphs disabled. Unvalidated on SM121 /
|
||||
// aarch64.
|
||||
{
|
||||
match: { hw: "dgx-spark", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
env: [],
|
||||
@@ -395,6 +398,7 @@ export const config = {
|
||||
"--mem-fraction-static 0.95",
|
||||
"--attention-backend flashinfer",
|
||||
"--chunked-prefill-size 8192",
|
||||
"--disable-prefill-cuda-graph",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--host {{HOST_IP}}",
|
||||
@@ -410,6 +414,7 @@ export const config = {
|
||||
"--mem-fraction-static 0.95",
|
||||
"--attention-backend flashinfer",
|
||||
"--chunked-prefill-size 8192",
|
||||
"--disable-prefill-cuda-graph",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--host {{HOST_IP}}",
|
||||
@@ -425,11 +430,121 @@ export const config = {
|
||||
"--mem-fraction-static 0.95",
|
||||
"--attention-backend flashinfer",
|
||||
"--chunked-prefill-size 8192",
|
||||
"--disable-prefill-cuda-graph",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
// GB300 (SM103), single 288GB GPU. All six cells measured on a 4xGB300
|
||||
// devbox on 2026-08-14 against lmsysorg/sglang:dev @ c4271c3fe1262fc2adbd162c33b25de5255251c5.
|
||||
// With no --attention-backend pin, :dev on GB300 resolves attention to
|
||||
// triton (the newer c7c03ec resolves trtllm_mha); the cells keep engine-default
|
||||
// resolution so the benchmark card and the cell see the same kernel. The
|
||||
// `high-throughput` strategy adds the in-checkpoint MTP head
|
||||
// (EAGLE / NEXTN semantics, num-steps 3, topk 1, draft-tokens 4).
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--mem-fraction-static 0.85",
|
||||
"--chunked-prefill-size 2048",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--mem-fraction-static 0.85",
|
||||
"--chunked-prefill-size 2048",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--speculative-algorithm EAGLE",
|
||||
"--speculative-num-steps 3",
|
||||
"--speculative-eagle-topk 1",
|
||||
"--speculative-num-draft-tokens 4",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--mem-fraction-static 0.85",
|
||||
"--chunked-prefill-size 2048",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--mem-fraction-static 0.85",
|
||||
"--chunked-prefill-size 2048",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--speculative-algorithm EAGLE",
|
||||
"--speculative-num-steps 3",
|
||||
"--speculative-eagle-topk 1",
|
||||
"--speculative-num-draft-tokens 4",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--mem-fraction-static 0.85",
|
||||
"--chunked-prefill-size 2048",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--mem-fraction-static 0.85",
|
||||
"--chunked-prefill-size 2048",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--speculative-algorithm EAGLE",
|
||||
"--speculative-num-steps 3",
|
||||
"--speculative-eagle-topk 1",
|
||||
"--speculative-num-draft-tokens 4",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user