[Docs] Add Qwen3.8-27B cookbook page (#34860)
Co-authored-by: BBuf <1182563586@qq.com> Co-authored-by: yhyang201 <yhyang201@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
zijiexia
co-authored by
BBuf
yhyang201
Claude Opus 5
BBuf
yhyang201
Claude Opus 5
parent
9c9a3273be
commit
29c6be15a4
@@ -0,0 +1,359 @@
|
||||
// Qwen3.8-27B-only calculator, live-coupled to the Deploy panel (same wiring as
|
||||
// _kimi_k3_mamba_ratio_calculator.jsx): every serving parameter except the
|
||||
// average request length and the target concurrency is derived from the
|
||||
// effective config the Playground broadcasts (base cell + Deploy overlays +
|
||||
// Playground overrides), and the computed --mamba-full-memory-ratio is
|
||||
// broadcast back for the Deploy panel to pin into its command.
|
||||
//
|
||||
// Geometry constants are validated against boot logs on RTX PRO 6000
|
||||
// (state 153.9 MB/slot at fp32, KV 32.8 KB/token at fp8). Byte-exact against
|
||||
// the boot log once the pool's +1 padding slot is counted: the log's
|
||||
// "ssm_state size: 27.00GB" at max_mamba_cache_size 191 is 192 slots x
|
||||
// 150,994,944 B = 27.0000 GiB — divide by 191 and you get the wrong 154.7.
|
||||
|
||||
export const Qwen38MambaRatioCalculator = () => {
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
const [requestLength, setRequestLength] = useState("5120");
|
||||
const [targetConcurrency, setTargetConcurrency] = useState("64");
|
||||
const [copied, setCopied] = useState(false);
|
||||
// Effective serving config; empty until the Playground's first broadcast
|
||||
// (the parse below then falls back to the stock defaults: fp32 state,
|
||||
// extra_buffer, no spec).
|
||||
const [cfg, setCfg] = useState({ flags: [], env: [], baseFlags: [], baseEnv: [] });
|
||||
|
||||
useEffect(() => {
|
||||
const checkTheme = () => {
|
||||
const html = document.documentElement;
|
||||
setIsDark(
|
||||
html.classList.contains("dark") ||
|
||||
html.getAttribute("data-theme") === "dark" ||
|
||||
html.style.colorScheme === "dark"
|
||||
);
|
||||
};
|
||||
checkTheme();
|
||||
const observer = new MutationObserver(checkTheme);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ["class", "data-theme", "style"],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const onCfg = (e) =>
|
||||
setCfg({
|
||||
flags: (e.detail && e.detail.flags) || [],
|
||||
env: (e.detail && e.detail.env) || [],
|
||||
baseFlags: (e.detail && e.detail.baseFlags) || [],
|
||||
baseEnv: (e.detail && e.detail.baseEnv) || [],
|
||||
});
|
||||
window.addEventListener("sglang-k3-effective-config", onCfg);
|
||||
return () => window.removeEventListener("sglang-k3-effective-config", onCfg);
|
||||
}, []);
|
||||
|
||||
// The Deploy panel's live selection. Needed because the effective-config
|
||||
// broadcast carries the RAW cell flags — `--model-path` is still the
|
||||
// unresolved `{{MODEL_NAME}}` there — so the checkpoint precision, which
|
||||
// decides what `--kv-cache-dtype auto` resolves to, is only knowable from the
|
||||
// selection's quant.
|
||||
const [quant, setQuant] = useState("nvfp4");
|
||||
useEffect(() => {
|
||||
const onSel = (e) => {
|
||||
if (e.detail && e.detail.quant) setQuant(e.detail.quant);
|
||||
};
|
||||
window.addEventListener("sglang-deploy-sel", onSel);
|
||||
return () => window.removeEventListener("sglang-deploy-sel", onSel);
|
||||
}, []);
|
||||
|
||||
const L = Number.parseFloat(requestLength);
|
||||
const C = Number.parseFloat(targetConcurrency);
|
||||
|
||||
// Derive the serving parameters from a flag list and evaluate the balance
|
||||
// formula, written as a per-request cost ratio:
|
||||
//
|
||||
// r = (S + D) x state_bytes / (L x kv_bytes_per_token)
|
||||
//
|
||||
const derive = (flags) => {
|
||||
const flagArg = (name) => {
|
||||
for (const f of flags) {
|
||||
const parts = f.split(/\s+/);
|
||||
if (parts[0] === name) return parts[1];
|
||||
}
|
||||
return null;
|
||||
};
|
||||
const hasFlag = (name) => flags.some((f) => f.split(/[\s=]/)[0] === name);
|
||||
|
||||
// Read both spellings: the cookbook convention is --tp, but some configs
|
||||
// still emit --tp-size. The geometry below is TP1-only (this is a
|
||||
// single-GPU page and no cell carries a TP flag), so anything else is
|
||||
// reported as out of range rather than silently mis-computed.
|
||||
const tp = Number(flagArg("--tp")) || Number(flagArg("--tp-size")) || 1;
|
||||
|
||||
// The NVFP4 checkpoint declares kv_cache_quant_algo: FP8, so the default
|
||||
// --kv-cache-dtype auto lands on fp8_e4m3 there with no flag present; the
|
||||
// BF16 / FP8 checkpoints keep a bf16 KV pool under the same default. An
|
||||
// explicit flag always wins over the checkpoint's declaration.
|
||||
const kvFlag = flagArg("--kv-cache-dtype");
|
||||
const kvDtype =
|
||||
kvFlag === "fp8_e4m3"
|
||||
? "fp8_e4m3"
|
||||
: kvFlag === "bfloat16" || kvFlag === "bf16"
|
||||
? "bfloat16"
|
||||
: quant === "nvfp4"
|
||||
? "fp8_e4m3"
|
||||
: "bfloat16";
|
||||
|
||||
const ssmFlag = flagArg("--mamba-ssm-dtype");
|
||||
const ssmDtype =
|
||||
ssmFlag === "bfloat16" || ssmFlag === "float16" ? ssmFlag : "float32";
|
||||
|
||||
const radixOff = hasFlag("--disable-radix-cache");
|
||||
// "auto" (and anything unrecognized) resolves to extra_buffer.
|
||||
const strategyFlag = flagArg("--mamba-radix-cache-strategy");
|
||||
const strategy =
|
||||
strategyFlag === "no_buffer" || strategyFlag === "extra_buffer_lazy"
|
||||
? strategyFlag
|
||||
: "extra_buffer";
|
||||
|
||||
// S mirrors kv_cache_configurator._calculate_mamba_ratio (single GPU,
|
||||
// overlap scheduler on): extra_buffer=5, extra_buffer_lazy=4,
|
||||
// no_buffer=3, radix cache disabled=1.
|
||||
const slots = radixOff
|
||||
? 1
|
||||
: strategy === "no_buffer"
|
||||
? 3
|
||||
: strategy === "extra_buffer_lazy"
|
||||
? 4
|
||||
: 5;
|
||||
|
||||
// Verify intermediates under speculative decoding: the draft-token count
|
||||
// (4 at the recommended EAGLE/MTP 3/1/4), 0 when spec is off.
|
||||
const specOn = hasFlag("--speculative-algorithm");
|
||||
const drafts = specOn ? Number(flagArg("--speculative-num-draft-tokens")) || 4 : 0;
|
||||
|
||||
// Fixed Qwen3.8-27B geometry (TP1):
|
||||
// GDN: 48 layers, 48 value heads x 128 x 128 SSM state (--mamba-ssm-dtype),
|
||||
// conv state 10240 x 3 always bf16.
|
||||
// Full attention: 16 layers, GQA 4 kv heads x head_dim 256, K+V.
|
||||
const ssmBytes = ssmDtype === "float32" ? 4 : 2;
|
||||
const kvBytes = kvDtype === "fp8_e4m3" ? 1 : 2;
|
||||
const stateBytesPerSlot = 48 * (48 * 128 * 128 * ssmBytes + 10240 * 3 * 2);
|
||||
const kvBytesPerToken = 16 * 4 * 256 * 2 * kvBytes;
|
||||
|
||||
const ratio = ((slots + drafts) * stateBytesPerSlot) / (kvBytesPerToken * L);
|
||||
return { ratio, tp, kvDtype, ssmDtype, radixOff, strategy, slots, specOn,
|
||||
drafts, stateBytesPerSlot, kvBytesPerToken };
|
||||
};
|
||||
|
||||
// Two evaluations: `eff` matches the Playground's composed command, `bs`
|
||||
// matches the Deploy command (cell + overlays only).
|
||||
const eff = derive(cfg.flags);
|
||||
const bs = derive(cfg.baseFlags.length ? cfg.baseFlags : cfg.flags);
|
||||
const { ratio, tp, kvDtype, ssmDtype, radixOff, strategy, slots, specOn,
|
||||
drafts, stateBytesPerSlot, kvBytesPerToken } = eff;
|
||||
|
||||
const valid = Number.isFinite(ratio) && ratio > 0 && L > 0 && tp === 1;
|
||||
const baseValid = Number.isFinite(bs.ratio) && bs.ratio > 0 && L > 0 && bs.tp === 1;
|
||||
const pin = Math.ceil(C * (slots + drafts));
|
||||
const pinValid = valid && Number.isFinite(pin) && pin > 0 && C > 0;
|
||||
|
||||
const formatRatio = (value) => (Math.round(value * 100) / 100).toString();
|
||||
const ratioStr = valid ? formatRatio(ratio) : "—";
|
||||
const baseRatioStr = baseValid ? formatRatio(bs.ratio) : "—";
|
||||
|
||||
const flagText = valid
|
||||
? pinValid
|
||||
? `--mamba-full-memory-ratio ${ratioStr} # or: --max-mamba-cache-size ${pin}`
|
||||
: `--mamba-full-memory-ratio ${ratioStr}`
|
||||
: "";
|
||||
|
||||
// Broadcast both results: the Deploy command takes the base-config value,
|
||||
// the Playground's composed command takes the effective one.
|
||||
useEffect(() => {
|
||||
window.dispatchEvent(
|
||||
new CustomEvent("sglang-k3-mamba-ratio", {
|
||||
detail: {
|
||||
ratio: valid ? ratioStr : null,
|
||||
baseRatio: baseValid ? baseRatioStr : null,
|
||||
},
|
||||
})
|
||||
);
|
||||
}, [ratioStr, valid, baseRatioStr, baseValid]);
|
||||
|
||||
const copy = () => {
|
||||
if (!valid) return;
|
||||
navigator.clipboard.writeText(flagText).then(() => {
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 1200);
|
||||
});
|
||||
};
|
||||
|
||||
const colors = {
|
||||
border: isDark ? "#374151" : "#e5e7eb",
|
||||
panel: isDark ? "#1f2937" : "#ffffff",
|
||||
input: isDark ? "#111827" : "#f8fafc",
|
||||
text: isDark ? "#e5e7eb" : "#1f2937",
|
||||
muted: isDark ? "#9ca3af" : "#64748b",
|
||||
accent: isDark ? "#E85D4D" : "#D45D44",
|
||||
error: isDark ? "#fca5a5" : "#b91c1c",
|
||||
};
|
||||
|
||||
const inputStyle = {
|
||||
width: "100%",
|
||||
boxSizing: "border-box",
|
||||
padding: "8px 10px",
|
||||
border: `1px solid ${colors.border}`,
|
||||
borderRadius: "5px",
|
||||
background: colors.input,
|
||||
color: colors.text,
|
||||
fontSize: "13px",
|
||||
};
|
||||
const labelStyle = {
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
gap: "5px",
|
||||
fontSize: "12px",
|
||||
fontWeight: 600,
|
||||
};
|
||||
const chipStyle = {
|
||||
padding: "3px 9px",
|
||||
border: `1px solid ${colors.border}`,
|
||||
borderRadius: "999px",
|
||||
background: colors.input,
|
||||
color: colors.text,
|
||||
fontSize: "12px",
|
||||
whiteSpace: "nowrap",
|
||||
};
|
||||
|
||||
// Everything the ratio depends on except L and the concurrency target, read
|
||||
// back from the panels so the reader can see what the number was derived from.
|
||||
const derivedChips = [
|
||||
`KV ${kvDtype === "fp8_e4m3" ? "FP8" : "BF16"}`,
|
||||
`State ${ssmDtype === "float32" ? "FP32" : ssmDtype === "bfloat16" ? "BF16" : "FP16"}`,
|
||||
radixOff ? "Radix off (S = 1)" : `${strategy} (S = ${slots})`,
|
||||
specOn ? `Spec on (D = ${drafts})` : "NOSPEC",
|
||||
];
|
||||
|
||||
return (
|
||||
<div
|
||||
className="not-prose"
|
||||
style={{
|
||||
display: "grid",
|
||||
gap: "12px",
|
||||
padding: "14px",
|
||||
border: `1px solid ${colors.border}`,
|
||||
borderRadius: "8px",
|
||||
background: colors.panel,
|
||||
color: colors.text,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
display: "grid",
|
||||
gridTemplateColumns: "minmax(180px, 240px) minmax(150px, 200px) 1fr",
|
||||
gap: "14px",
|
||||
alignItems: "start",
|
||||
}}
|
||||
>
|
||||
<label htmlFor="qwen38-ratio-length" style={labelStyle}>
|
||||
Average request length
|
||||
<input
|
||||
id="qwen38-ratio-length"
|
||||
type="number"
|
||||
min="1"
|
||||
step="1"
|
||||
value={requestLength}
|
||||
onChange={(event) => setRequestLength(event.target.value)}
|
||||
style={inputStyle}
|
||||
/>
|
||||
<span style={{ color: colors.muted, fontSize: "11px", fontWeight: 400 }}>
|
||||
Input + output tokens — the ratio's only free parameter
|
||||
</span>
|
||||
</label>
|
||||
|
||||
<label htmlFor="qwen38-ratio-concurrency" style={labelStyle}>
|
||||
Target concurrency
|
||||
<input
|
||||
id="qwen38-ratio-concurrency"
|
||||
type="number"
|
||||
min="1"
|
||||
step="1"
|
||||
value={targetConcurrency}
|
||||
onChange={(event) => setTargetConcurrency(event.target.value)}
|
||||
style={inputStyle}
|
||||
/>
|
||||
<span style={{ color: colors.muted, fontSize: "11px", fontWeight: 400 }}>
|
||||
Requests in flight — sizes the explicit pin, not the ratio
|
||||
</span>
|
||||
</label>
|
||||
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: "6px" }}>
|
||||
<span style={{ fontSize: "12px", fontWeight: 600 }}>
|
||||
Serving configuration (follows the Deploy panel and Playground)
|
||||
</span>
|
||||
<div style={{ display: "flex", flexWrap: "wrap", gap: "6px" }}>
|
||||
{derivedChips.map((c) => (
|
||||
<span key={c} style={chipStyle}>{c}</span>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!valid ? (
|
||||
<div style={{ color: colors.error, fontSize: "12px" }}>
|
||||
{tp !== 1
|
||||
? `This calculator models the single-GPU (TP1) geometry; the panels are at TP${tp}.`
|
||||
: "Enter a valid request length."}
|
||||
</div>
|
||||
) : (
|
||||
<div
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: "12px",
|
||||
paddingTop: "12px",
|
||||
borderTop: `1px solid ${colors.border}`,
|
||||
flexWrap: "wrap",
|
||||
}}
|
||||
>
|
||||
<div>
|
||||
<div style={{ color: colors.muted, fontSize: "11px" }}>
|
||||
Balanced ratio — pinned into the commands above
|
||||
</div>
|
||||
<div style={{ fontSize: "26px", fontWeight: 700 }}>{ratioStr}</div>
|
||||
{baseValid && baseRatioStr !== ratioStr ? (
|
||||
<div style={{ color: colors.muted, fontSize: "11px" }}>
|
||||
Deploy command (without Playground overrides): {baseRatioStr}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
<code style={{ flex: 1, minWidth: "240px", color: colors.text }}>
|
||||
{flagText}
|
||||
</code>
|
||||
<button
|
||||
type="button"
|
||||
onClick={copy}
|
||||
style={{
|
||||
padding: "7px 11px",
|
||||
border: 0,
|
||||
borderRadius: "5px",
|
||||
background: colors.accent,
|
||||
color: "#ffffff",
|
||||
fontSize: "12px",
|
||||
fontWeight: 600,
|
||||
cursor: "pointer",
|
||||
}}
|
||||
>
|
||||
{copied ? "Copied" : "Copy flag"}
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
<div style={{ color: colors.muted, fontSize: "11px" }}>
|
||||
state/slot {(stateBytesPerSlot / 1e6).toFixed(1)} MB · KV/token{" "}
|
||||
{(kvBytesPerToken / 1e3).toFixed(1)} KB · {slots + drafts} state slots per
|
||||
request, so {targetConcurrency || "N"} concurrent requests need{" "}
|
||||
{pinValid ? pin : "—"} slots.
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,435 @@
|
||||
// Single `export const config` literal — no spreads/calls/IIFE (Mintlify re-evals at hydration).
|
||||
// Cells are denormalized: no `--nnodes`/`--node-rank`/`--dist-init-addr`/`--host`/`--port` literals — engine injects them.
|
||||
//
|
||||
// Qwen3.8-27B: DENSE hybrid Gated Delta Networks VISION-LANGUAGE model — a 27B
|
||||
// causal LM plus a vision encoder, served through SGLang's Qwen3-VL path
|
||||
// (Qwen3_5ForConditionalGeneration extends Qwen3VLForConditionalGeneration and
|
||||
// is registered in the multimodal arch lists). 64 layers as 16 repeats of
|
||||
// 3 x (Gated DeltaNet -> FFN) then 1 x (Gated Attention -> FFN): 48
|
||||
// linear-attention layers to 16 full-attention. GDN runs 48 value heads and 16
|
||||
// QK heads at head_dim 128; attention is GQA 24/4 at head_dim 256. An MTP head
|
||||
// trained with multiple steps ships in-checkpoint. Context 262,144 native,
|
||||
// extensible to 1,000,000. Dense, so there is no MoE axis.
|
||||
//
|
||||
// Single-GPU on every supported card — H200 (SM90 datacenter), the SM120
|
||||
// workstation pair (RTX PRO 6000 Blackwell / RTX 5090), and DGX Spark (GB10,
|
||||
// SM121, 128GB unified memory) — hence one node and no parallelism flags in
|
||||
// any cell.
|
||||
//
|
||||
// PROVENANCE — every flag and value below is transcribed from the
|
||||
// pre-migration prose page (one unconditional launch command + its Configuration
|
||||
// Tips). Model ids are the exception: BF16/FP8 point at the official Qwen
|
||||
// checkpoints, NVFP4 at the RadixArk W4A4 build. That page pinned
|
||||
// no sglang version for its measurements, so under the
|
||||
// migration skill's reproducible-anchor rule NO measured numbers were carried
|
||||
// over: there is no sibling `-benchmarks.jsx`. Cells are nevertheless marked
|
||||
// `verified: true` at the maintainers' direction — the badge there reflects
|
||||
// their own unpublished validation, not measured data carried by this page. The
|
||||
// DGX Spark cells are the exception and stay unverified: that recipe is
|
||||
// unvalidated on SM121 / aarch64, as both §2 and the cell comment below say.
|
||||
// `benchmarkCommands` below records the page's measurement protocol so the
|
||||
// numbers can be re-measured against a pinned build and then land as a
|
||||
// benchmarks file.
|
||||
//
|
||||
// A hardware x quantization combination with no launch recipe has no cell, and
|
||||
// the engine greys it out.
|
||||
|
||||
export const config = {
|
||||
modelName: "Qwen3.8-27B",
|
||||
|
||||
supportedHardware: ["h200", "rtx6000", "rtx5090", "dgx-spark"],
|
||||
|
||||
// RTX PRO 6000 and RTX 5090 (SM120 / Blackwell Desktop) are workstation and
|
||||
// consumer cards, not datacenter GPUs, so they are not in the shared catalog.
|
||||
// Ids/labels match the DeepSeek-V4 config's entries for the same two cards.
|
||||
// DGX Spark needs no entry here: it is already in the shared catalog
|
||||
// (_deployment.jsx HARDWARE_CATALOG), with its multi-node Docker flags.
|
||||
hardware: [
|
||||
{ id: "rtx6000", label: "RTX PRO 6000", vram: "96GB", vendor: "blackwell" },
|
||||
{ id: "rtx5090", label: "RTX 5090", vram: "32GB", vendor: "blackwell" },
|
||||
],
|
||||
|
||||
variants: [
|
||||
{ id: "default", label: "Default" },
|
||||
],
|
||||
// BF16/FP8 are the official Qwen checkpoints; NVFP4 is the RadixArk
|
||||
// W4A4 build. NVFP4 is W4A4 with FP8 projections and declares
|
||||
// `kv_cache_quant_algo: FP8`, so under the default `--kv-cache-dtype auto` its
|
||||
// KV pool runs fp8_e4m3 off the checkpoint's own calibration scales — no
|
||||
// `--kv-cache-dtype` flag in the recipe, and nothing accuracy-degrading added
|
||||
// by the cell.
|
||||
quantizations: [
|
||||
{ id: "bf16", label: "BF16" },
|
||||
{ id: "fp8", label: "FP8" },
|
||||
{ id: "nvfp4", label: "NVFP4" },
|
||||
],
|
||||
// The source page documents ONE operating point: a single general-purpose
|
||||
// launch command with no latency/throughput toggle. MTP is described as an
|
||||
// opt-in in the tips, not as a second named recipe, so it rides the
|
||||
// Playground's speculative axis instead of splitting the strategy dimension.
|
||||
strategies: [
|
||||
{ id: "balanced", label: "Balanced" },
|
||||
],
|
||||
nodesOptions: [
|
||||
{ id: "single", label: "Single Node" },
|
||||
],
|
||||
|
||||
modelNames: {
|
||||
"default|bf16": "Qwen/Qwen3.8-27B",
|
||||
"default|fp8": "Qwen/Qwen3.8-27B-FP8",
|
||||
"default|nvfp4": "RadixArk/Qwen3.8-27B-NVFP4",
|
||||
},
|
||||
|
||||
placeholders: {
|
||||
HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" },
|
||||
PORT: { target: "command", label: "Bind port", default: "30000" },
|
||||
HF_TOKEN: { target: "command", label: "HF token (Docker)", default: "<your-hf-token>" },
|
||||
CURL_HOST: { target: "curl", label: "Server host", default: "localhost" },
|
||||
CURL_PORT: { target: "curl", label: "Server port", default: "30000" },
|
||||
},
|
||||
|
||||
curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\
|
||||
-H 'Content-Type: application/json' \\
|
||||
-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`,
|
||||
|
||||
// The measurement protocol the source page described, kept so its numbers can
|
||||
// be reproduced against a pinned build. --random-range-ratio 1 pins ISL
|
||||
// exactly rather than drawing a range; --flush-cache measures cache-cold
|
||||
// (bench_serving's `random` prompts are deterministic, so a warm rerun would
|
||||
// hit the radix cache and inflate throughput) — the page's own "prefix caching
|
||||
// disabled" protocol.
|
||||
benchmarkCommands: {
|
||||
speed:
|
||||
`python3 -m sglang.bench_serving \\
|
||||
--backend sglang-oai \\
|
||||
--host {{CURL_HOST}} --port {{CURL_PORT}} \\
|
||||
--model {{MODEL_NAME}} \\
|
||||
--dataset-name {{DATASET}} \\
|
||||
--random-input-len {{ISL}} --random-output-len {{OSL}} --random-range-ratio 1 \\
|
||||
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\
|
||||
--request-rate inf \\
|
||||
--flush-cache`,
|
||||
accuracy: {
|
||||
gsm8k_pct:
|
||||
`python3 -m sglang.test.run_eval \\
|
||||
--host http://{{CURL_HOST}} --port {{CURL_PORT}} \\
|
||||
--model {{MODEL_NAME}} \\
|
||||
--eval-name gsm8k \\
|
||||
--num-examples 1319`,
|
||||
},
|
||||
},
|
||||
|
||||
accuracyLabels: [
|
||||
["gsm8k_pct", "GSM8K", "%"],
|
||||
],
|
||||
|
||||
dockerImages: {
|
||||
h200: "lmsysorg/sglang:qwen38-27b",
|
||||
rtx6000: "lmsysorg/sglang:qwen38-27b",
|
||||
rtx5090: "lmsysorg/sglang:qwen38-27b",
|
||||
// TODO: verify an arm64 build of this tag for DGX Spark (GB10 is aarch64);
|
||||
// the x86-only tag will not pull there.
|
||||
"dgx-spark": "lmsysorg/sglang:qwen38-27b",
|
||||
},
|
||||
|
||||
github: {
|
||||
cookbookModel: "Qwen/Qwen3.8-27B",
|
||||
},
|
||||
|
||||
playgroundFeatures: {
|
||||
|
||||
// No "Attention Parallelism" card. The source page is single-GPU
|
||||
// throughout and no cell carries a parallelism flag, so there is nothing to
|
||||
// override: DP-Attention targets MLA models, prefill-CP has no model-side
|
||||
// integration for this architecture, and a TP knob would desync the ratio
|
||||
// calculator below (its geometry is TP1-only, so it would stop emitting and
|
||||
// the command would silently fall back to the 0.9 default this page warns
|
||||
// about). Re-add it together with TP-aware geometry in the calculator.
|
||||
|
||||
// ----- Card: "Parsers" -----
|
||||
// Same parser pair the Qwen3.8 flagship page ships, and baked into every
|
||||
// cell: this model is used through agent harnesses, and a deploy command
|
||||
// without them returns tool calls as raw text instead of structured
|
||||
// `tool_calls`. So this card is an opt-OUT — the handler derives both chips
|
||||
// as already-on from the cell and strips the flag when one is toggled off.
|
||||
parsers: {
|
||||
items: [
|
||||
{ id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser qwen3" },
|
||||
{ id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser qwen3_coder" },
|
||||
],
|
||||
},
|
||||
|
||||
// ----- Card: "Speculative Decoding" -----
|
||||
// The in-checkpoint 1-layer MTP head. The source page wrote the preset as
|
||||
// `--speculative-algorithm NEXTN`; NEXTN is an alias of EAGLE, so it is
|
||||
// normalized here (the Playground strips/derives by the first token, and an
|
||||
// alias would survive toggles and double up). DSpark is the trained draft
|
||||
// model, a separate checkpoint.
|
||||
speculative: {
|
||||
options: [
|
||||
{ id: "current", label: "Inherited from base" },
|
||||
{ id: "off", label: "Off (greedy)" },
|
||||
{ id: "mtp", label: "EAGLE / MTP",
|
||||
flags: ["--speculative-algorithm EAGLE", "--speculative-num-steps 3",
|
||||
"--speculative-eagle-topk 1", "--speculative-num-draft-tokens 4"] },
|
||||
{ id: "dspark", label: "DSpark",
|
||||
flags: ["--speculative-algorithm DSPARK",
|
||||
"--speculative-draft-model-path RadixArk/Qwen3.8-27B-DSpark"] },
|
||||
],
|
||||
},
|
||||
|
||||
// ----- Card: single-selects over one flag family each -----
|
||||
flagSelects: [
|
||||
{
|
||||
// trtllm_mha is SM100-only, so every cell bakes flashinfer (SM90 and
|
||||
// SM120 alike). FA3 is the SM90 alternative — measured slightly faster
|
||||
// at bs=1 on H200. Triton is the documented fallback when MTP runs on a
|
||||
// FlashInfer build whose prefill `plan` predates `uniform_q_len`
|
||||
// (<= 0.6.15.post1).
|
||||
id: "attnBackend", title: "Attention Backend",
|
||||
stripPrefixes: ["--attention-backend"],
|
||||
options: [
|
||||
{ id: "flashinfer", label: "FlashInfer (default)",
|
||||
flags: ["--attention-backend flashinfer"] },
|
||||
{ id: "fa3", label: "FlashAttention-3 (SM90 only)",
|
||||
flags: ["--attention-backend fa3"] },
|
||||
{ id: "triton", label: "Triton — MTP fallback on older FlashInfer",
|
||||
flags: ["--attention-backend triton"] },
|
||||
],
|
||||
},
|
||||
{
|
||||
// Halving kv_bytes_per_token (65.5 KB bf16 -> 32.8 KB fp8) doubles the
|
||||
// KV pool at a fixed --mamba-full-memory-ratio. Accuracy-degrading over
|
||||
// a bf16-KV checkpoint, so it stays an opt-in and is never in a cell —
|
||||
// the NVFP4 checkpoint gets fp8 KV on its own via kv_cache_quant_algo.
|
||||
id: "kvCacheDtype", title: "KV Cache Precision",
|
||||
stripPrefixes: ["--kv-cache-dtype"],
|
||||
options: [
|
||||
{ id: "auto", label: "Auto (checkpoint-declared)" },
|
||||
{ id: "fp8", label: "FP8 (E4M3) — halves KV memory", flags: ["--kv-cache-dtype fp8_e4m3"] },
|
||||
{ id: "bf16", label: "BFloat16", flags: ["--kv-cache-dtype bfloat16"] },
|
||||
],
|
||||
},
|
||||
{
|
||||
// One state slot is 154.7 MB at fp32, 79.2 MB at bf16 — the single
|
||||
// biggest lever on the GDN state pool, which is what bounds concurrency
|
||||
// on small-VRAM cards.
|
||||
id: "mambaSsmDtype", title: "GDN State Precision",
|
||||
stripPrefixes: ["--mamba-ssm-dtype"],
|
||||
options: [
|
||||
{ id: "auto", label: "Auto (FP32)" },
|
||||
{ id: "bf16", label: "BFloat16 — halves state memory", flags: ["--mamba-ssm-dtype bfloat16"] },
|
||||
],
|
||||
},
|
||||
{
|
||||
// Whole-model prefix cache. Off drops the per-request state cost to
|
||||
// S=1 slot, which is the cheapest way to buy concurrency on a 32GB card
|
||||
// when the traffic has no shared prefixes (offline batch, evals).
|
||||
id: "prefixCache", title: "Prefix Cache",
|
||||
stripPrefixes: ["--disable-radix-cache"],
|
||||
options: [
|
||||
{ id: "on", label: "On" },
|
||||
{ id: "off", label: "Off (S=1)", flags: ["--disable-radix-cache"] },
|
||||
],
|
||||
},
|
||||
{
|
||||
// How GDN state buffers for radix reuse — slot cost per request S:
|
||||
// extra_buffer 5, extra_buffer_lazy 4, no_buffer 3. No strategy exists
|
||||
// with the prefix cache off, so the row hides (and stops emitting) there.
|
||||
// Changing S changes the balanced ratio — recompute it in the page's
|
||||
// calculator after picking a strategy here.
|
||||
id: "mambaRadix", title: "GDN Radix Cache Strategy",
|
||||
showWhen: (b, v, d) => (((v && v.prefixCache) ?? (d && d.prefixCache)) !== "off"),
|
||||
stripPrefixes: ["--mamba-radix-cache-strategy"],
|
||||
options: [
|
||||
{ id: "auto", label: "Auto (extra_buffer, S=5)" },
|
||||
{ id: "lazy", label: "extra_buffer_lazy (S=4)", flags: ["--mamba-radix-cache-strategy extra_buffer_lazy"] },
|
||||
{ id: "nobuf", label: "no_buffer (S=3)", flags: ["--mamba-radix-cache-strategy no_buffer"] },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
|
||||
// Every cell is the source page's single launch command with only the model id
|
||||
// varying. The H200 and SM120 cells are `verified: true` at the maintainers'
|
||||
// direction; the DGX Spark cells are not, matching the unvalidated-on-SM121
|
||||
// note on those cells. The page carries no measured data of its own (no
|
||||
// `-benchmarks.jsx`), so a badge rests on validation held outside this page —
|
||||
// re-check it against the per-platform notes in §2 before trusting a cell.
|
||||
//
|
||||
// Cells carry NO --mamba-full-memory-ratio. The source page's worked 4.6 held
|
||||
// only for the NVFP4 recipe at 4096-in/1024-out; the ratio is a function of
|
||||
// the workload, of S (radix-cache strategy / prefix cache), of D (spec) and of
|
||||
// kv_bytes_per_token, all of which the Playground can change. So the page's
|
||||
// ratio calculator computes it live from the effective config and broadcasts
|
||||
// it, and the engines pin it into the rendered command — which they only do
|
||||
// while the cell itself stays ratio-free (_deployment.jsx `cellWithRatio`).
|
||||
// Adding the flag back here would silently freeze the value again.
|
||||
cells: [
|
||||
{
|
||||
// H200 141GB, FP8 blockwise (~28.5GB of weights). The 32768-token chunk
|
||||
// is the H200-validated setting: SM90 prefill is fast enough that a big
|
||||
// chunk stalls decode far less than on SM120, and the SM90 FlashInfer GDN
|
||||
// prefill default engages under it (fp32 state pool, chunk <= 32768).
|
||||
// No NVFP4 cell on this card: SM90 has no FP4 tensor cores, so the W4A4
|
||||
// checkpoint's MLP would fall back to the Marlin W4A16 weight-only path —
|
||||
// runnable, but not a recipe this page ships.
|
||||
match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--mem-fraction-static 0.85",
|
||||
"--attention-backend flashinfer",
|
||||
"--chunked-prefill-size 32768",
|
||||
"--max-prefill-tokens 32768",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
// H200, BF16 reference checkpoint (~54GB of weights).
|
||||
match: { hw: "h200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--mem-fraction-static 0.85",
|
||||
"--attention-backend flashinfer",
|
||||
"--chunked-prefill-size 32768",
|
||||
"--max-prefill-tokens 32768",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
// The page's headline recipe: NVFP4 W4A4 on the 96GB workstation card,
|
||||
// ~16.5GB of weights, fp8 KV auto-enabled by the checkpoint.
|
||||
match: { hw: "rtx6000", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--mem-fraction-static 0.85",
|
||||
"--attention-backend flashinfer",
|
||||
"--chunked-prefill-size 2048",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
// FP8 blockwise, ~28.5GB of weights — comfortable on 96GB.
|
||||
match: { hw: "rtx6000", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--mem-fraction-static 0.85",
|
||||
"--attention-backend flashinfer",
|
||||
"--chunked-prefill-size 2048",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
// BF16, the reference checkpoint.
|
||||
match: { hw: "rtx6000", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--mem-fraction-static 0.85",
|
||||
"--attention-backend flashinfer",
|
||||
"--chunked-prefill-size 2048",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
// RTX 5090 32GB. NVFP4 is the only checkpoint that fits with room to
|
||||
// serve (~16.5GB); FP8 at ~28.5GB is not serviceable past bs<=2 and BF16
|
||||
// does not fit, so neither has a cell. On this card the GDN state pool —
|
||||
// not KV — bounds concurrency: lower S with the Playground's radix-cache
|
||||
// strategy (or turn the prefix cache off for S=1) and recompute the ratio.
|
||||
match: { hw: "rtx5090", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
verified: true,
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--mem-fraction-static 0.85",
|
||||
"--attention-backend flashinfer",
|
||||
"--chunked-prefill-size 2048",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
// DGX Spark (GB10, SM121): single node, 128GB coherent unified memory
|
||||
// shared with the CPU — every checkpoint fits, so all three quants get a
|
||||
// cell. FlashInfer attention comes from the SM120 pair; the platform gets
|
||||
// its own operating point at 8192-token prefill chunks and 0.95 static
|
||||
// fraction. Unvalidated on SM121 / aarch64.
|
||||
{
|
||||
match: { hw: "dgx-spark", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--mem-fraction-static 0.95",
|
||||
"--attention-backend flashinfer",
|
||||
"--chunked-prefill-size 8192",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "dgx-spark", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--mem-fraction-static 0.95",
|
||||
"--attention-backend flashinfer",
|
||||
"--chunked-prefill-size 8192",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "dgx-spark", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
|
||||
env: [],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--mem-fraction-static 0.95",
|
||||
"--attention-backend flashinfer",
|
||||
"--chunked-prefill-size 8192",
|
||||
"--reasoning-parser qwen3",
|
||||
"--tool-call-parser qwen3_coder",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
@@ -12,6 +12,23 @@
|
||||
// paraphrasing that page's own opening.
|
||||
|
||||
export const popularModels = [
|
||||
{
|
||||
name: "Qwen3.8-27B",
|
||||
vendor: "Qwen",
|
||||
href: "/cookbook/autoregressive/Qwen/Qwen3.8-27B",
|
||||
logo: "/cards/logos/qwen.png",
|
||||
badge: "New",
|
||||
tags: ["4 platforms", "Hybrid GDN", "BF16 / FP8 / NVFP4"],
|
||||
hero: {
|
||||
eyebrow: "Featured model \u00b7 New",
|
||||
headline: "Meet Qwen3.8-27B on SGLang",
|
||||
blurb:
|
||||
"A dense hybrid Gated Delta Networks model \u2014 48 GDN linear-attention layers interleaved with 16 full-attention, an in-checkpoint MTP head, and a native 262,144-token context. The cookbook covers single-GPU serving on H200 and RTX PRO 6000 / 5090.",
|
||||
tags: ["Dense 27B", "262K context", "Single-GPU"],
|
||||
cta: "Open the Qwen3.8-27B cookbook",
|
||||
caption: "Qwen3.8-27B deployment guide",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Kimi-K3",
|
||||
vendor: "Moonshot AI",
|
||||
|
||||
Reference in New Issue
Block a user