minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[docs] Re-measure the Qwen3.8-27B RTX 5090, RTX PRO 6000 and DGX Spark grids on 1cf2b8c (#35825)

Browse Source 
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jimmy Shong
2026-08-22 13:03:18 +08:00
committed by GitHub
co-authored by Claude Fable 5
parent 96bfd2476c
commit 4cb5aebfe0
2 changed files with 118 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx
+73 -73
View File
@@ -83,12 +83,14 @@ export const config = {
...(["rtx5090", "rtx6000", "dgx-spark"].includes(sel.hw)
? ["--enable-linear-replayssm-spec"]
: []),
// Measured on the 5090: bf16 state serves at 0.92, fp32 needs
// 0.94 (an fp32 slot is 146.81 MiB vs bf16's 74.81).
// Measured on the 5090 at commit 1cf2b8c: fp32 serves at 0.94,
// bf16 at 0.93. bf16 moved UP from 0.92 with the dense-lm_head
// checkpoint -- the heavier weights need a larger static budget
// before the state pool fits.
...(sel.hw === "rtx5090"
? [sel.ssmDtype === "float32"
? "--mem-fraction-static 0.94"
: "--mem-fraction-static 0.92"]
: "--mem-fraction-static 0.93"]
: []),
],
},
@@ -107,13 +109,13 @@ export const config = {
"--speculative-algorithm DSPARK",
"--speculative-draft-model-path RadixArk/Qwen3.8-27B-DSpark",
"--speculative-draft-attention-backend flashinfer",
// Measured on the 5090: bf16 state serves at 0.90, fp32 needs
// 0.92 — the opposite correction to EAGLE's (see above).
...(sel.hw === "rtx5090"
? [sel.ssmDtype === "float32"
? "--mem-fraction-static 0.92"
: "--mem-fraction-static 0.90"]
: []),
// Measured on the 5090 at commit 1cf2b8c: bf16 serves at 0.88,
// below the 0.90 this recipe carried when it was measured on an
// older build, because a draft model plus the automatic prefill
// CUDA-graph capture no longer fit there. fp32 is greyed out by the
// SSM dtype row. EAGLE and no-speculation are unaffected: replayssm
// keeps EAGLE's state pool tiny and no-spec loads no draft weights.
...(sel.hw === "rtx5090" ? ["--mem-fraction-static 0.88"] : []),
],
},
{
@@ -128,31 +130,21 @@ export const config = {
disabled: (sel) => sel.hw === "rtx5090" && sel.quant !== "nvfp4",
disableReason:
"On the 32GB RTX 5090 the DFlash2 draft model only fits on top of the NVFP4 weights",
// fp32 is the one case that needs the balanced ratio overridden, so
// that family is stripped too and re-emitted below.
stripPrefixes: (sel) =>
sel.hw === "rtx5090"
? sel.ssmDtype === "float32"
? ["--mem-fraction-static", "--mamba-full-memory-ratio"]
: ["--mem-fraction-static"]
: [],
sel.hw === "rtx5090" ? ["--mem-fraction-static"] : [],
flags: (sel) => [
"--speculative-algorithm DFLASH",
"--speculative-draft-model-path incoai/Qwen3.8-27B-DFlash2",
"--speculative-num-draft-tokens 8",
// Measured on the 5090 at commit 1cf2b8c, the build the Install
// accordion pins for this pick. bf16 serves at 0.88 on the balanced
// ratio (0.90, DSPARK's pin, OOMs on the first request). fp32 needs
// 0.895 AND the balanced ratio overridden to 10: these cells pin
// --max-running-requests 1, so the balanced value provisions KV for
// concurrency this recipe never uses, starving the state pool of the
// slots fp32 needs. Only High-Throughput reaches fp32 (S=4); the SSM
// dtype row greys fp32 out for Low-Latency (S=5).
// accordion pins. This is the only cell on the page that also needs
// a prefill chunk smaller than the engine default: at 0.91 the pools
// fit but a 2048-token chunk's activations do not. The pair together
// is the fastest recipe on this card (4.92ms median TPOT, 4.29
// accept length). fp32 is greyed out by the SSM dtype row.
...(sel.hw === "rtx5090"
? sel.ssmDtype === "float32"
? ["--mem-fraction-static 0.895",
"--mamba-full-memory-ratio 10"]
: ["--mem-fraction-static 0.88"]
? ["--mem-fraction-static 0.91",
"--chunked-prefill-size 1024"]
: []),
],
},
@@ -198,21 +190,22 @@ export const config = {
// serves, and is the faster cell there anyway.
{
id: "float32", label: "float32",
// Only the Low-Latency tier is out of reach: it needs S=5 fp32 slots
// (735MB) plus >=9216 KV tokens for one request, and no mem-fraction
// holds both -- at 0.8975/r14 the pool buys the 5th slot but KV falls
// to 7752 tokens and generation stops after one token, while every
// mem-fraction with a big enough pool (>=0.90) dies in graph capture.
// High-Throughput needs one slot fewer and does fit; see the DFLASH2
// option's pins.
// On the 32GB RTX 5090 a draft model plus an fp32 state pool no longer
// fit together: the checkpoint's dense lm_head adds ~3.2GB of weights,
// which pushes the pools up into the mem-fraction range where prefill
// CUDA-graph capture no longer fits. Measured across 0.86-0.96 at both
// prefill chunk sizes, plus balanced-ratio overrides up to 20: below
// ~0.92 the state pool never reaches the tier's slot count, and at or
// above it capture or the first request OOMs. An fp32 slot is 154MB
// against bfloat16's 78MB, which is why only fp32 is caught. EAGLE and
// no-speculation are unaffected -- replayssm keeps EAGLE's pool tiny
// and no-spec loads no draft weights at all.
disabled: (sel) =>
sel.hw === "rtx5090" &&
sel.spec === "dflash" &&
sel.tier === "low-latency",
(sel.spec === "dflash" || sel.spec === "dspark"),
disableReason:
"On the 32GB RTX 5090 the Low-Latency tier cannot hold five fp32 state " +
"slots and a full request's KV at once — use bfloat16, or the " +
"High-Throughput tier which fits fp32",
"On the 32GB RTX 5090 an fp32 GDN state pool and a speculative draft model " +
"do not fit together — use bfloat16",
flags: ["--mamba-ssm-dtype float32"],
},
{
@@ -274,11 +267,14 @@ export const config = {
dockerImages: {
h200: "lmsysorg/sglang:qwen38-27b",
rtx6000: "lmsysorg/sglang:qwen38-27b",
rtx5090: "lmsysorg/sglang:qwen38-27b",
// Both SM120 cards are validated on this image (built from 1cf2b8c, the
// commit every pin on those cards was measured against).
rtx6000: "lmsysorg/sglang:dev-qwen38-27b-dflash2",
rtx5090: "lmsysorg/sglang:dev-qwen38-27b-dflash2",
// Multi-arch: this tag ships both linux/amd64 and linux/arm64, so it pulls
// natively on DGX Spark (GB10 is aarch64).
"dgx-spark": "lmsysorg/sglang:qwen38-27b",
// Multi-arch (linux/amd64 + linux/arm64), so GB10 pulls it natively.
"dgx-spark": "lmsysorg/sglang:dev-qwen38-27b-dflash2",
gb300: "lmsysorg/sglang:dev",
},
@@ -558,32 +554,41 @@ export const config = {
},
// DGX Spark (GB10, SM121): single node, 128GB coherent unified memory
// shared with the CPU — every checkpoint fits, so all three quants get a
// cell. These cells reuse the RTX PRO 6000 recipe verbatim rather than a
// separate SM121 operating point: both cards are SM12x Blackwell, and
// GB10's 128GB unified pool is larger than the 6000's 96GB, so a recipe
// that fits the smaller card has headroom here.
// cell. These cells reuse the RTX PRO 6000 recipe at one lower
// mem-fraction rather than a separate SM121 operating point: both cards
// are SM12x Blackwell, and GB10's 128GB unified pool is larger than the
// 6000's 96GB, so a recipe that fits the smaller card has headroom here.
//
// Validated on GB10 (SM121 / aarch64): all 36 configurations booted and
// served at ISL 8192 / OSL 1024, concurrency 1. Boot-and-serve only -- no
// throughput or acceptance-length numbers were taken, so this is a weaker
// standard than the SM120 pair's validation, and the Deploy-panel Note says
// so.
// Why 0.80 and not the 0.85 every other SM12x cell pins: the pool is
// unified, so mem-fraction prices the HOST's memory too. 0.85 of 128GB
// leaves ~8GB for the OS — exactly DGX OS earlyoom's SIGTERM threshold —
// and the first long prefill or boot-time graph capture dips under it and
// gets the scheduler killed (exit code -15, no traceback; check
// `journalctl -u earlyoom`). Re-measured on 1cf2b8c (2026-08-21): at 0.85,
// 15 of 48 cells were SIGTERMed, and which 15 is margin noise, biased
// toward the big-state configs (bfloat16 SSM, DSPARK/DFLASH2 ratios); at
// 0.80 every cell served on every attempt.
//
// Validated on GB10 (SM121 / aarch64) at 1cf2b8c: all 48 configurations —
// DFLASH2 included — booted and served at ISL 8192 / OSL 1024,
// concurrency 1. Boot-and-serve only -- no throughput or acceptance-length
// numbers were taken, so this is a weaker standard than the SM120 pair's
// validation, and the Deploy-panel Note says so. NVFP4 was exercised with
// the BF16-LMHead export (same as the SM120 re-measurement); the
// packed-head export also served its DFLASH2 cells on this platform in the
// 12-cell DFLASH2 pass.
{
match: { hw: "dgx-spark", variant: "default", quant: "nvfp4", nodes: "single" },
// All 12 overlay combinations served on GB10. DSPARK here also
// exercises the 4-bit `lm_head` this checkpoint quantizes, with no shape
// error.
// All 16 overlay combinations served on GB10 at 1cf2b8c, DFLASH2
// included — its selector folded into the draft CUDA graph in all four
// of its cells here.
verified: true,
// DFLASH2 has not been exercised on this platform; every other overlay
// pick keeps this cell's original validation.
verificationStatus: (sel) =>
sel.spec === "dflash" ? "in-progress" : "verified",
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--kv-cache-dtype fp8_e4m3",
"--mem-fraction-static 0.85",
"--mem-fraction-static 0.80",
"--attention-backend flashinfer",
"--chunked-prefill-size 2048",
"--reasoning-parser qwen3",
@@ -594,18 +599,16 @@ export const config = {
},
{
match: { hw: "dgx-spark", variant: "default", quant: "fp8", nodes: "single" },
// All 12 overlay combinations served on GB10.
// All 16 overlay combinations served on GB10 at 1cf2b8c, DFLASH2
// included. This checkpoint held the sweep's most earlyoom-prone cells
// at 0.85 (every bfloat16-SSM pick was killed); all clean at 0.80.
verified: true,
// DFLASH2 has not been exercised on this platform; every other overlay
// pick keeps this cell's original validation.
verificationStatus: (sel) =>
sel.spec === "dflash" ? "in-progress" : "verified",
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--kv-cache-dtype fp8_e4m3",
"--mem-fraction-static 0.85",
"--mem-fraction-static 0.80",
"--attention-backend flashinfer",
"--chunked-prefill-size 2048",
"--reasoning-parser qwen3",
@@ -616,19 +619,16 @@ export const config = {
},
{
match: { hw: "dgx-spark", variant: "default", quant: "bf16", nodes: "single" },
// All 12 overlay combinations served on GB10. Heaviest checkpoint, so
// it holds the sweep's tightest cell: DSPARK + float32 + extra_buffer.
// All 16 overlay combinations served on GB10 at 1cf2b8c, DFLASH2
// included. Heaviest checkpoint (52GB, ~6.5 min to load its 18 shards
// from NVMe — budget ~10 min to READY before calling a boot hung).
verified: true,
// DFLASH2 has not been exercised on this platform; every other overlay
// pick keeps this cell's original validation.
verificationStatus: (sel) =>
sel.spec === "dflash" ? "in-progress" : "verified",
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--kv-cache-dtype fp8_e4m3",
"--mem-fraction-static 0.85",
"--mem-fraction-static 0.80",
"--attention-backend flashinfer",
"--chunked-prefill-size 2048",
"--reasoning-parser qwen3",