diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx index 79e25946d..9846d5bfc 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx @@ -161,9 +161,9 @@ checkpoint's calibration scales automatically. requires a FlashInfer build whose prefill `plan` accepts `uniform_q_len` (newer than 0.6.15.post1); otherwise run spec with `--attention-backend triton`. On DGX Spark the 128GB is unified memory shared with the host CPU, so all - three checkpoints fit; its cells use 8192-token prefill chunks and - `--mem-fraction-static 0.95`. The SM121 recipe is not yet validated on that - platform. + three checkpoints fit; its cells use 8192-token prefill chunks, + `--mem-fraction-static 0.95`, and `--disable-prefill-cuda-graph`. The SM121 + recipe is not yet validated on that platform. - **H200 (SM90)**: BF16 and FP8 only — the card has no FP4 tensor cores, so the NVFP4 checkpoint's MLP would fall back to the Marlin W4A16 weight-only path and its cell is greyed out. The H200 recipes use 32768-token prefill chunks diff --git a/docs/src/snippets/configs/Qwen/qwen3.8-27b-benchmarks.jsx b/docs/src/snippets/configs/Qwen/qwen3.8-27b-benchmarks.jsx new file mode 100644 index 000000000..a5d3ff290 --- /dev/null +++ b/docs/src/snippets/configs/Qwen/qwen3.8-27b-benchmarks.jsx @@ -0,0 +1,106 @@ +// Qwen3.8-27B per-cell benchmark numbers, keyed by the same `match` tuple as +// qwen3.8-27b.jsx cells. See _deployment.jsx for the speed/accuracy schema. +// +// All six rows are ONE-BATCH measurements (sglang.bench_serving --flush-cache, +// random dataset, ISL=1024 / OSL=1024, --random-range-ratio 1, request-rate inf, +// max_concurrency 1/16/64, n=64/64/256 respectively) on a single GB300 GPU +// (Blackwell Ultra SM103, 288GB HBM) on 2026-08-14. SGLang binary: +// lmsysorg/sglang:dev resolving to commit c4271c3fe1262fc2adbd162c33b25de5255251c5. +// With no --attention-backend pin, that commit on GB300 resolves attention to +// triton; a newer sglang (c7c03ec+) resolves trtllm_mha. Same binary and +// protocol for all six cells so the numbers are apples-to-apples. MTP rows +// show mean speculative accept_length across all decode batches. +// +// Accuracy (sgl-eval run gsm8k, full 1319, stop_rate 1.0, truncated_rate 0.0): +// only the W4A4-NVFP4 checkpoint has been GSM8K'd on a GB300 single-GPU box — +// via a sibling experiment pinned to sglang c7c03ec53b1e664c2d415db4f02e43f86661f31d +// with an explicit `--kv-cache-dtype` A/B. The bf16-KV row (96.82%) is the better +// baseline to cite; the default fp8-KV the NVFP4 checkpoint auto-enables scores +// 96.44%. FP8 / BF16 GB300 GSM8K has not been measured and stays null below +// (see journal 2026-08-14-1048-claude-jrn_f6c1265be8cbdf86c44fe36c). +export const benchmarks = [ + { + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, + sglang_version: "lmsysorg/sglang:dev @ c4271c3fe", + speed: [ + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 1, num_prompts: 64 }, + ttft_ms: 79, tpot_ms: 6.4, tokens_per_sec_per_gpu: 155 }, + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 16, num_prompts: 64 }, + ttft_ms: 715, tpot_ms: 8.5, tokens_per_sec_per_gpu: 1742 }, + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 64, num_prompts: 256 }, + ttft_ms: 1216, tpot_ms: 13.6, tokens_per_sec_per_gpu: 4316 }, + ], + accuracy: { gsm8k_pct: 96.44 }, + notes: "NVFP4 = RadixArk W4A4-0811 (private). KV auto fp8_e4m3 from ckpt-declared kv_cache_quant_algo. GSM8K fp8-KV = 96.44 / bf16-KV = 96.82 (sgl-eval, `c7c03ec`, sibling experiment on `/scratch/qwen38-w4a4-kv-ab-0814` on `qwen38-27b-nvfp4-convert-0812`); both stop_rate 100% / truncated 0% on full 1319.", + }, + { + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "lmsysorg/sglang:dev @ c4271c3fe", + speed: [ + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 1, num_prompts: 64 }, + ttft_ms: 73, tpot_ms: 2.3, tokens_per_sec_per_gpu: 415 }, + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 16, num_prompts: 64 }, + ttft_ms: 216, tpot_ms: 4.4, tokens_per_sec_per_gpu: 3256 }, + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 64, num_prompts: 256 }, + ttft_ms: 3085, tpot_ms: 8.6, tokens_per_sec_per_gpu: 5155 }, + ], + accuracy: { gsm8k_pct: 96.44 }, + notes: "MTP accept_length mean ~3.31 across conc. Suspected cap is `--speculative-num-draft-tokens 4` (max 4 accepted/reject step).", + }, + { + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, + sglang_version: "lmsysorg/sglang:dev @ c4271c3fe", + speed: [ + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 1, num_prompts: 64 }, + ttft_ms: 73, tpot_ms: 8.2, tokens_per_sec_per_gpu: 121 }, + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 16, num_prompts: 64 }, + ttft_ms: 586, tpot_ms: 11.2, tokens_per_sec_per_gpu: 1355 }, + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 64, num_prompts: 256 }, + ttft_ms: 1539, tpot_ms: 19.7, tokens_per_sec_per_gpu: 3012 }, + ], + accuracy: { gsm8k_pct: null }, + notes: "FP8 = Qwen/Qwen3.8-27B-FP8 revision 032ba94c96507998d543d5a43f7b4ebcfa6b4fa9 (private). GSM8K not yet measured on GB300 for FP8; TP4 B300 sibling experiment scored it 1276/1319 (96.74%) under thinking sampling.", + }, + { + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + sglang_version: "lmsysorg/sglang:dev @ c4271c3fe", + speed: [ + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 1, num_prompts: 64 }, + ttft_ms: 74, tpot_ms: 3.2, tokens_per_sec_per_gpu: 302 }, + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 16, num_prompts: 64 }, + ttft_ms: 174, tpot_ms: 5.3, tokens_per_sec_per_gpu: 2771 }, + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 64, num_prompts: 256 }, + ttft_ms: 3376, tpot_ms: 9.5, tokens_per_sec_per_gpu: 4599 }, + ], + accuracy: { gsm8k_pct: null }, + notes: "MTP accept_length mean ~3.16.", + }, + { + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, + sglang_version: "lmsysorg/sglang:dev @ c4271c3fe", + speed: [ + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 1, num_prompts: 64 }, + ttft_ms: 72, tpot_ms: 10.3, tokens_per_sec_per_gpu: 97 }, + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 16, num_prompts: 64 }, + ttft_ms: 490, tpot_ms: 12.5, tokens_per_sec_per_gpu: 1235 }, + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 64, num_prompts: 256 }, + ttft_ms: 1622, tpot_ms: 18.4, tokens_per_sec_per_gpu: 3208 }, + ], + accuracy: { gsm8k_pct: null }, + notes: "BF16 = Qwen/Qwen3.8-27B revision 08cb37595cfe9ce298e6f172c6b50d814ef42413 (private). GSM8K not yet measured on GB300 for BF16.", + }, + { + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + sglang_version: "lmsysorg/sglang:dev @ c4271c3fe", + speed: [ + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 1, num_prompts: 64 }, + ttft_ms: 76, tpot_ms: 4.0, tokens_per_sec_per_gpu: 245 }, + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 16, num_prompts: 64 }, + ttft_ms: 211, tpot_ms: 6.3, tokens_per_sec_per_gpu: 2306 }, + { workload: { dataset: "random", isl: 1024, osl: 1024, max_concurrency: 64, num_prompts: 256 }, + ttft_ms: 3894, tpot_ms: 10.9, tokens_per_sec_per_gpu: 3995 }, + ], + accuracy: { gsm8k_pct: null }, + notes: "MTP accept_length mean ~3.22.", + }, +]; diff --git a/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx b/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx index 41f5252aa..33f75733f 100644 --- a/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx +++ b/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx @@ -37,7 +37,7 @@ export const config = { modelName: "Qwen3.8-27B", - supportedHardware: ["h200", "rtx6000", "rtx5090", "dgx-spark"], + supportedHardware: ["h200", "rtx6000", "rtx5090", "dgx-spark", "gb300"], // RTX PRO 6000 and RTX 5090 (SM120 / Blackwell Desktop) are workstation and // consumer cards, not datacenter GPUs, so they are not in the shared catalog. @@ -69,6 +69,7 @@ export const config = { // Playground's speculative axis instead of splitting the strategy dimension. strategies: [ { id: "balanced", label: "Balanced" }, + { id: "high-throughput", label: "High-Throughput" }, ], nodesOptions: [ { id: "single", label: "Single Node" }, @@ -130,6 +131,7 @@ export const config = { // TODO: verify an arm64 build of this tag for DGX Spark (GB10 is aarch64); // the x86-only tag will not pull there. "dgx-spark": "lmsysorg/sglang:qwen38-27b", + gb300: "lmsysorg/sglang:dev", }, github: { @@ -384,8 +386,9 @@ export const config = { // DGX Spark (GB10, SM121): single node, 128GB coherent unified memory // shared with the CPU — every checkpoint fits, so all three quants get a // cell. FlashInfer attention comes from the SM120 pair; the platform gets - // its own operating point at 8192-token prefill chunks and 0.95 static - // fraction. Unvalidated on SM121 / aarch64. + // its own operating point at 8192-token prefill chunks, 0.95 static + // fraction, and prefill CUDA graphs disabled. Unvalidated on SM121 / + // aarch64. { match: { hw: "dgx-spark", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, env: [], @@ -395,6 +398,7 @@ export const config = { "--mem-fraction-static 0.95", "--attention-backend flashinfer", "--chunked-prefill-size 8192", + "--disable-prefill-cuda-graph", "--reasoning-parser qwen3", "--tool-call-parser qwen3_coder", "--host {{HOST_IP}}", @@ -410,6 +414,7 @@ export const config = { "--mem-fraction-static 0.95", "--attention-backend flashinfer", "--chunked-prefill-size 8192", + "--disable-prefill-cuda-graph", "--reasoning-parser qwen3", "--tool-call-parser qwen3_coder", "--host {{HOST_IP}}", @@ -425,11 +430,121 @@ export const config = { "--mem-fraction-static 0.95", "--attention-backend flashinfer", "--chunked-prefill-size 8192", + "--disable-prefill-cuda-graph", "--reasoning-parser qwen3", "--tool-call-parser qwen3_coder", "--host {{HOST_IP}}", "--port {{PORT}}", ], }, + // GB300 (SM103), single 288GB GPU. All six cells measured on a 4xGB300 + // devbox on 2026-08-14 against lmsysorg/sglang:dev @ c4271c3fe1262fc2adbd162c33b25de5255251c5. + // With no --attention-backend pin, :dev on GB300 resolves attention to + // triton (the newer c7c03ec resolves trtllm_mha); the cells keep engine-default + // resolution so the benchmark card and the cell see the same kernel. The + // `high-throughput` strategy adds the in-checkpoint MTP head + // (EAGLE / NEXTN semantics, num-steps 3, topk 1, draft-tokens 4). + { + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--mem-fraction-static 0.85", + "--chunked-prefill-size 2048", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--mem-fraction-static 0.85", + "--chunked-prefill-size 2048", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--mem-fraction-static 0.85", + "--chunked-prefill-size 2048", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--mem-fraction-static 0.85", + "--chunked-prefill-size 2048", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--mem-fraction-static 0.85", + "--chunked-prefill-size 2048", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--mem-fraction-static 0.85", + "--chunked-prefill-size 2048", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, ], };