From 29c6be15a4ef08c0ecc239b313a706334c471e1e Mon Sep 17 00:00:00 2001 From: zijiexia <37504505+zijiexia@users.noreply.github.com> Date: Fri, 14 Aug 2026 08:23:39 -0700 Subject: [PATCH] [Docs] Add Qwen3.8-27B cookbook page (#34860) Co-authored-by: BBuf <1182563586@qq.com> Co-authored-by: yhyang201 Co-authored-by: Claude Opus 5 --- .../autoregressive/Qwen/Qwen3.8-27B.mdx | 391 ++++++++++++++++ docs/cookbook/autoregressive/Qwen/Qwen3.8.mdx | 1 - docs/docs.json | 1 + .../_qwen38_mamba_ratio_calculator.jsx | 359 +++++++++++++++ .../src/snippets/configs/Qwen/qwen3.8-27b.jsx | 435 ++++++++++++++++++ docs/src/snippets/configs/popular-models.jsx | 17 + 6 files changed, 1203 insertions(+), 1 deletion(-) create mode 100644 docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx create mode 100644 docs/src/snippets/_qwen38_mamba_ratio_calculator.jsx create mode 100644 docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx new file mode 100644 index 000000000..79e25946d --- /dev/null +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx @@ -0,0 +1,391 @@ +--- +title: Qwen3.8-27B +description: "Deploy Qwen3.8-27B with SGLang — dense hybrid GDN vision-language model with BF16/FP8/NVFP4 W4A4 checkpoints and in-checkpoint MTP, single-GPU on H200, RTX PRO 6000, RTX 5090 and DGX Spark." +tag: NEW +--- + +## Deployment + + + + + +For all methods and hardware platforms, see the [official SGLang installation guide](../../../docs/get-started/install). The two paths below match the **Python / Docker** toggle in the command panel. + + + + + +```bash Command +pip install --upgrade pip +pip install uv +uv pip install sglang +``` + +Then run the **Python** output of the command panel below in that environment. + + + + + +```bash Command +docker pull lmsysorg/sglang:qwen38-27b +``` + +For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces. + + + + + + + +Pick your card + checkpoint precision to generate the launch command. The model runs single-GPU on every supported card — H200, RTX PRO 6000, RTX 5090 and DGX Spark — and ships one operating point. + + +`--mamba-full-memory-ratio` is the one sizing flag that matters for throughput +on hybrid GDN models: the default (0.9) over-provisions the KV pool and silently +clamps concurrency. Set your average request length in the +[Mamba ratio calculator](#mamba-ratio-calculator) below; everything else follows +the panels, and the computed value is pinned into the command. + + +import { Deployment } from "/src/snippets/_deployment.jsx"; +import { config } from "/src/snippets/configs/Qwen/qwen3.8-27b.jsx"; +import { Qwen38MambaRatioCalculator } from "/src/snippets/_qwen38_mamba_ratio_calculator.jsx"; + + + +### Mamba ratio calculator + + + + + +Hybrid GDN models split post-weight memory into a worst-case-reserved **GDN +state pool** (sets the concurrency ceiling) and a paged **attention KV pool**, +divided by `--mamba-full-memory-ratio`. Every parameter below except `L` and the +target concurrency is read live from the Deploy panel and Playground selection; +the balanced value is the per-request cost ratio: + +```text Formula +ratio = (S + D) x state_bytes / (L x kv_bytes_per_token) +``` + +- `S` — state slots per running request: `extra_buffer=5` (default), + `extra_buffer_lazy=4`, `no_buffer=3`, disabled radix cache `=1`. +- `D` — verify intermediate states under speculative decoding: + `--speculative-num-draft-tokens` (4 at the recommended EAGLE 3/1/4), 0 otherwise. +- `state_bytes` — one state slot, from the fixed geometry + (48 GDN layers x 48 heads x 128 x 128 at `--mamba-ssm-dtype`, plus bf16 conv + state): 153.9 MB at fp32, 78.4 MB at bf16. +- `kv_bytes_per_token` — 16 attention layers x GQA 4 x 256 x K+V: + 32.8 KB at fp8, 65.5 KB at bf16. +- `L` — average total request length in tokens: input + output. + +`--max-mamba-cache-size = target_concurrency x (S + D)` is the equivalent +explicit pin and overrides the ratio; the calculator emits it alongside. After +boot, verify with the `max_running_requests` line in the server log — it should +not be capped below your target concurrency. + + + +## Playground + +The Playground is where you experiment with **SGLang features beyond the recipes above**. The Deploy panel emits this model's documented launch recipes; the Playground lets you turn on additional knobs on top of whichever cell the Deploy panel is currently showing. + +import { Playground } from "/src/snippets/_playground.jsx"; + + + +## 1. Model Introduction + +**Qwen3.8-27B** is a dense hybrid Gated Delta Networks (GDN) **vision-language** +model: a 27B causal language model paired with a vision encoder, with native +image and video understanding alongside text. SGLang serves it through the +Qwen3-VL path, so the vision tower is live on the recipes below. + +The language model is 64 layers, laid out as 16 repeats of *3 × (Gated DeltaNet +→ FFN)* followed by *1 × (Gated Attention → FFN)* — 48 linear-attention layers +to 16 full-attention ones. Gated DeltaNet runs 48 value heads and 16 QK heads at +head_dim 128; Gated Attention is GQA 24/4 at head_dim 256 with a 64-dim rotary +slice. Hidden size is 5120 over a 17,408-dim FFN, and the checkpoint ships an +MTP head trained with multiple steps. Context is 262,144 tokens natively, +extensible to 1,000,000. The serving-relevant architecture is identical to +Qwen3.6-27B. + +Thinking mode is on by default and can be disabled per request; reasoning depth +is tunable with `reasoning_effort`, and `preserve_thinking` retains reasoning +context from earlier messages. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModelQuantizationWeights
Qwen3.8-27BBF16Qwen/Qwen3.8-27B
Qwen3.8-27B-FP8FP8 (blockwise)Qwen/Qwen3.8-27B-FP8
Qwen3.8-27B-NVFP4NVFP4 W4A4 + FP8 projectionsRadixArk/Qwen3.8-27B-NVFP4
+ +The NVFP4 checkpoint declares `kv_cache_quant_algo: FP8`; SGLang's default +`--kv-cache-dtype auto` honors it, so the KV pool runs in `fp8_e4m3` with the +checkpoint's calibration scales automatically. + +## 2. Configuration Tips + +- **SM120/SM121 (RTX PRO 6000 Blackwell, RTX 5090, DGX Spark)**: use `--attention-backend + flashinfer`; `trtllm_mha` is SM100-only. MTP with the FlashInfer backend + requires a FlashInfer build whose prefill `plan` accepts `uniform_q_len` + (newer than 0.6.15.post1); otherwise run spec with `--attention-backend triton`. + On DGX Spark the 128GB is unified memory shared with the host CPU, so all + three checkpoints fit; its cells use 8192-token prefill chunks and + `--mem-fraction-static 0.95`. The SM121 recipe is not yet validated on that + platform. +- **H200 (SM90)**: BF16 and FP8 only — the card has no FP4 tensor cores, so the + NVFP4 checkpoint's MLP would fall back to the Marlin W4A16 weight-only path + and its cell is greyed out. The H200 recipes use 32768-token prefill chunks + (SM90 prefill is fast enough that a big chunk barely stalls decode, unlike + the SM120 guidance below), and the FlashInfer GDN prefill backend engages by + default under them. `--attention-backend fa3` is a valid alternative, + measured slightly faster at bs=1. +- **MTP**: `--speculative-algorithm EAGLE --speculative-num-steps 3 + --speculative-eagle-topk 1 --speculative-num-draft-tokens 4` uses the + in-checkpoint MTP head. (This recipe was originally documented with `NEXTN`, + an alias of `EAGLE` — same algorithm.) +- **DSpark**: the trained draft model is a separate checkpoint — add + `--speculative-algorithm DSPARK --speculative-draft-model-path + RadixArk/Qwen3.8-27B-DSpark` (the Playground's Speculative Decoding card + emits this pair). +- **Hardware fit**: FP8 weights ~28.5GB (not serviceable beyond bs≤2 on + 32GB cards); NVFP4 weights ~16.5GB (recommended for RTX 5090-class GPUs). +- `--mamba-radix-cache-strategy extra_buffer_lazy` lowers the state cost per + request from 5 slots to 4 at no accuracy cost. On small-VRAM cards (RTX 5090 + 32GB) the state pool bounds concurrency long before KV does — prefer lowering + `S` (lazy strategy, or `--disable-radix-cache` for S=1); the + [calculator](#mamba-ratio-calculator) re-derives the ratio for the new `S`. + The balanced ratio itself is VRAM-independent. +- `--chunked-prefill-size 2048`: decode steps stall behind each prefill chunk + on hybrid GDN models, and 8192-token chunks stall them ~600ms at a time. + 2048 keeps decode inter-token latency smooth under mixed load and also + improves single-wave TTFT. (DGX Spark is the exception: its cells run + 8192-token chunks.) + +## 3. Agent Harnesses + +Agent harnesses drive the model through the OpenAI-compatible endpoint — or, for +Claude Code, through SGLang's Anthropic-compatible one — so any of them works +once three things line up. + +**The parsers ship in the command.** Every recipe above carries +`--reasoning-parser qwen3 --tool-call-parser qwen3_coder`, because without them a +harness receives tool calls as raw text instead of structured `tool_calls`. The +**Parsers** card in the [Playground](#playground) is therefore an opt-out — both +chips start on, and turning one off strips its flag. + +`qwen3_coder` is the right tool-call parser for this checkpoint: its chat +template instructs the model to reply with an inner `` / +`` block nested in ``, which is exactly what +that parser decodes. The Hermes parser (`--tool-call-parser hermes`) reads a +*different* payload — bare JSON inside `` — so pointing a Hermes-format +harness at this model without switching the flag yields tool calls that never +parse. `--reasoning-parser qwen3` matches the template's `enable_thinking` +toggle, which defaults to on. + +**Endpoint and model id.** The base URL is `http://:30000/v1`. The `model` +string a harness sends must equal the server's `--model-path` — the OpenAI +`/v1/models` name defaults to it — unless you override it with +`--served-model-name`, which is usually worth doing to keep harness configs short. + +SGLang also serves an Anthropic-compatible `/v1/messages`, which is what +[§3.3](#3-3-claude-code) uses. It converts each request to the OpenAI shape, +hands it to the same chat-serving path, and converts the response back — so the +parser flags above apply there identically. + +**Auth.** `--api-key` is unset by default, so the server accepts unauthenticated +requests. Harnesses that insist on a key can send any placeholder; set +`--api-key` on the server if the endpoint is reachable beyond localhost. + +### 3.1 OpenCode + +[OpenCode](https://opencode.ai/docs/providers/) reaches a self-hosted endpoint +through a provider entry in `opencode.json`. + + + +Store the credential first — pick **Other**, give the provider an id, and enter +any placeholder when the server has no `--api-key`: + +```bash Command +opencode +/connect +``` + +Then declare the provider in `opencode.json`: + +```json Config +{ + "$schema": "https://opencode.ai/config.json", + "provider": { + "sglang": { + "npm": "@ai-sdk/openai-compatible", + "name": "SGLang (Qwen3.8-27B)", + "options": { + "baseURL": "http://localhost:30000/v1" + }, + "models": { + "RadixArk/Qwen3.8-27B-NVFP4": { + "name": "Qwen3.8-27B NVFP4" + } + } + } + } +} +``` + +`npm` selects the transport — `@ai-sdk/openai-compatible` is the one for a plain +OpenAI-shaped endpoint. `apiKey` is optional and takes a `"{env:VAR_NAME}"` +reference rather than a literal. The `models` keys are the ids sent on the wire, +so they must match the served model name. Confirm with `/models`. + + + +### 3.2 Pi + +[Pi](https://pi.dev/docs/latest/custom-provider) +(`@earendil-works/pi-coding-agent`) registers providers from an extension rather +than a config file. + + + +```javascript Extension +pi.registerProvider("sglang", { + baseUrl: "http://localhost:30000/v1", + api: "openai-completions", + apiKey: "$SGLANG_API_KEY", + models: [ + { + id: "RadixArk/Qwen3.8-27B-NVFP4", + name: "Qwen3.8-27B", + reasoning: true, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 262144, + maxTokens: 32768, + }, + ], +}); +``` + +`api: "openai-completions"` is what selects the OpenAI-compatible transport, and +`apiKey` takes a `$ENV_VAR` reference rather than a literal. `contextWindow` is +the checkpoint's native 262,144; set `maxTokens` to whatever output cap you want +per turn. Confirm registration with `pi --list-models`. + + + +### 3.3 Claude Code + +Claude Code speaks the Anthropic API, so it points at SGLang's `/v1/messages` +rather than the OpenAI endpoint. + + +Anthropic documents that routing Claude Code to non-Claude models through a +gateway is **not supported**. The wiring below works because SGLang implements +the Anthropic message format, but it sits outside what Claude Code is tested +against — expect newer Claude Code features to degrade or fail. + + + + +`ANTHROPIC_BASE_URL` is the server origin — Claude Code appends `/v1/messages` +itself, so leave the `/v1` suffix off: + +```bash Command +export ANTHROPIC_BASE_URL=http://localhost:30000 +export ANTHROPIC_AUTH_TOKEN=placeholder +``` + +The two credential variables travel in different headers: +`ANTHROPIC_AUTH_TOKEN` goes out as `Authorization: Bearer`, `ANTHROPIC_API_KEY` +as `x-api-key`. Either satisfies a server started without `--api-key`; with +`--api-key` set, pick the variable matching the header your server reads. A +credential variable also takes precedence over a saved claude.ai login for that +session. + +The same pair can live in a settings file instead, which persists across shells +and wins over a shell export: + +```json Config +{ + "env": { + "ANTHROPIC_BASE_URL": "http://localhost:30000", + "ANTHROPIC_AUTH_TOKEN": "placeholder" + } +} +``` + +Run `/status` in Claude Code to confirm which base URL and credential source the +session picked up. + + + +### 3.4 Hermes Agent + +[Hermes Agent](https://github.com/NousResearch/hermes-agent) (Nous Research, MIT) +selects a self-hosted endpoint through its setup wizard or its config file. + + + +```bash Command +hermes model +# choose "Custom endpoint (self-hosted / VLLM / etc.)", then enter the +# base URL, an API key (blank for a local server) and the model name +``` + +Equivalently, in `~/.hermes/config.yaml`: + +```yaml Config +model: + default: RadixArk/Qwen3.8-27B-NVFP4 + provider: custom + base_url: http://localhost:30000/v1 + api_key: "" + context_length: 262144 +``` + +For several endpoints at once, declare them under `providers:` and switch with +`/model custom:` mid-session: + +```yaml Config +providers: + workstation: + api: http://localhost:30000/v1 + server: + api: https://gpu-host.internal:30000/v1 + key_env: SGLANG_API_KEY +``` + + diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8.mdx index d2685d2b1..5a4e9b286 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.8.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8.mdx @@ -1,7 +1,6 @@ --- title: Qwen3.8 description: "Deploy Qwen3.8 with SGLang — day-0 recipes for Qwen's 2.4T-parameter (95B active) hybrid GDN/GQA Mixture-of-Experts model on NVIDIA and AMD." -tag: NEW --- ## Deployment diff --git a/docs/docs.json b/docs/docs.json index 6f69e673f..85fd6e39d 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1242,6 +1242,7 @@ "group": "Qwen", "pages": [ "cookbook/autoregressive/Qwen/Qwen3.8", + "cookbook/autoregressive/Qwen/Qwen3.8-27B", "cookbook/autoregressive/Qwen/Qwen3.6", "cookbook/autoregressive/Qwen/Qwen3.5", "cookbook/autoregressive/Qwen/Qwen3", diff --git a/docs/src/snippets/_qwen38_mamba_ratio_calculator.jsx b/docs/src/snippets/_qwen38_mamba_ratio_calculator.jsx new file mode 100644 index 000000000..5633622f5 --- /dev/null +++ b/docs/src/snippets/_qwen38_mamba_ratio_calculator.jsx @@ -0,0 +1,359 @@ +// Qwen3.8-27B-only calculator, live-coupled to the Deploy panel (same wiring as +// _kimi_k3_mamba_ratio_calculator.jsx): every serving parameter except the +// average request length and the target concurrency is derived from the +// effective config the Playground broadcasts (base cell + Deploy overlays + +// Playground overrides), and the computed --mamba-full-memory-ratio is +// broadcast back for the Deploy panel to pin into its command. +// +// Geometry constants are validated against boot logs on RTX PRO 6000 +// (state 153.9 MB/slot at fp32, KV 32.8 KB/token at fp8). Byte-exact against +// the boot log once the pool's +1 padding slot is counted: the log's +// "ssm_state size: 27.00GB" at max_mamba_cache_size 191 is 192 slots x +// 150,994,944 B = 27.0000 GiB — divide by 191 and you get the wrong 154.7. + +export const Qwen38MambaRatioCalculator = () => { + const [isDark, setIsDark] = useState(false); + const [requestLength, setRequestLength] = useState("5120"); + const [targetConcurrency, setTargetConcurrency] = useState("64"); + const [copied, setCopied] = useState(false); + // Effective serving config; empty until the Playground's first broadcast + // (the parse below then falls back to the stock defaults: fp32 state, + // extra_buffer, no spec). + const [cfg, setCfg] = useState({ flags: [], env: [], baseFlags: [], baseEnv: [] }); + + useEffect(() => { + const checkTheme = () => { + const html = document.documentElement; + setIsDark( + html.classList.contains("dark") || + html.getAttribute("data-theme") === "dark" || + html.style.colorScheme === "dark" + ); + }; + checkTheme(); + const observer = new MutationObserver(checkTheme); + observer.observe(document.documentElement, { + attributes: true, + attributeFilter: ["class", "data-theme", "style"], + }); + return () => observer.disconnect(); + }, []); + + useEffect(() => { + const onCfg = (e) => + setCfg({ + flags: (e.detail && e.detail.flags) || [], + env: (e.detail && e.detail.env) || [], + baseFlags: (e.detail && e.detail.baseFlags) || [], + baseEnv: (e.detail && e.detail.baseEnv) || [], + }); + window.addEventListener("sglang-k3-effective-config", onCfg); + return () => window.removeEventListener("sglang-k3-effective-config", onCfg); + }, []); + + // The Deploy panel's live selection. Needed because the effective-config + // broadcast carries the RAW cell flags — `--model-path` is still the + // unresolved `{{MODEL_NAME}}` there — so the checkpoint precision, which + // decides what `--kv-cache-dtype auto` resolves to, is only knowable from the + // selection's quant. + const [quant, setQuant] = useState("nvfp4"); + useEffect(() => { + const onSel = (e) => { + if (e.detail && e.detail.quant) setQuant(e.detail.quant); + }; + window.addEventListener("sglang-deploy-sel", onSel); + return () => window.removeEventListener("sglang-deploy-sel", onSel); + }, []); + + const L = Number.parseFloat(requestLength); + const C = Number.parseFloat(targetConcurrency); + + // Derive the serving parameters from a flag list and evaluate the balance + // formula, written as a per-request cost ratio: + // + // r = (S + D) x state_bytes / (L x kv_bytes_per_token) + // + const derive = (flags) => { + const flagArg = (name) => { + for (const f of flags) { + const parts = f.split(/\s+/); + if (parts[0] === name) return parts[1]; + } + return null; + }; + const hasFlag = (name) => flags.some((f) => f.split(/[\s=]/)[0] === name); + + // Read both spellings: the cookbook convention is --tp, but some configs + // still emit --tp-size. The geometry below is TP1-only (this is a + // single-GPU page and no cell carries a TP flag), so anything else is + // reported as out of range rather than silently mis-computed. + const tp = Number(flagArg("--tp")) || Number(flagArg("--tp-size")) || 1; + + // The NVFP4 checkpoint declares kv_cache_quant_algo: FP8, so the default + // --kv-cache-dtype auto lands on fp8_e4m3 there with no flag present; the + // BF16 / FP8 checkpoints keep a bf16 KV pool under the same default. An + // explicit flag always wins over the checkpoint's declaration. + const kvFlag = flagArg("--kv-cache-dtype"); + const kvDtype = + kvFlag === "fp8_e4m3" + ? "fp8_e4m3" + : kvFlag === "bfloat16" || kvFlag === "bf16" + ? "bfloat16" + : quant === "nvfp4" + ? "fp8_e4m3" + : "bfloat16"; + + const ssmFlag = flagArg("--mamba-ssm-dtype"); + const ssmDtype = + ssmFlag === "bfloat16" || ssmFlag === "float16" ? ssmFlag : "float32"; + + const radixOff = hasFlag("--disable-radix-cache"); + // "auto" (and anything unrecognized) resolves to extra_buffer. + const strategyFlag = flagArg("--mamba-radix-cache-strategy"); + const strategy = + strategyFlag === "no_buffer" || strategyFlag === "extra_buffer_lazy" + ? strategyFlag + : "extra_buffer"; + + // S mirrors kv_cache_configurator._calculate_mamba_ratio (single GPU, + // overlap scheduler on): extra_buffer=5, extra_buffer_lazy=4, + // no_buffer=3, radix cache disabled=1. + const slots = radixOff + ? 1 + : strategy === "no_buffer" + ? 3 + : strategy === "extra_buffer_lazy" + ? 4 + : 5; + + // Verify intermediates under speculative decoding: the draft-token count + // (4 at the recommended EAGLE/MTP 3/1/4), 0 when spec is off. + const specOn = hasFlag("--speculative-algorithm"); + const drafts = specOn ? Number(flagArg("--speculative-num-draft-tokens")) || 4 : 0; + + // Fixed Qwen3.8-27B geometry (TP1): + // GDN: 48 layers, 48 value heads x 128 x 128 SSM state (--mamba-ssm-dtype), + // conv state 10240 x 3 always bf16. + // Full attention: 16 layers, GQA 4 kv heads x head_dim 256, K+V. + const ssmBytes = ssmDtype === "float32" ? 4 : 2; + const kvBytes = kvDtype === "fp8_e4m3" ? 1 : 2; + const stateBytesPerSlot = 48 * (48 * 128 * 128 * ssmBytes + 10240 * 3 * 2); + const kvBytesPerToken = 16 * 4 * 256 * 2 * kvBytes; + + const ratio = ((slots + drafts) * stateBytesPerSlot) / (kvBytesPerToken * L); + return { ratio, tp, kvDtype, ssmDtype, radixOff, strategy, slots, specOn, + drafts, stateBytesPerSlot, kvBytesPerToken }; + }; + + // Two evaluations: `eff` matches the Playground's composed command, `bs` + // matches the Deploy command (cell + overlays only). + const eff = derive(cfg.flags); + const bs = derive(cfg.baseFlags.length ? cfg.baseFlags : cfg.flags); + const { ratio, tp, kvDtype, ssmDtype, radixOff, strategy, slots, specOn, + drafts, stateBytesPerSlot, kvBytesPerToken } = eff; + + const valid = Number.isFinite(ratio) && ratio > 0 && L > 0 && tp === 1; + const baseValid = Number.isFinite(bs.ratio) && bs.ratio > 0 && L > 0 && bs.tp === 1; + const pin = Math.ceil(C * (slots + drafts)); + const pinValid = valid && Number.isFinite(pin) && pin > 0 && C > 0; + + const formatRatio = (value) => (Math.round(value * 100) / 100).toString(); + const ratioStr = valid ? formatRatio(ratio) : "—"; + const baseRatioStr = baseValid ? formatRatio(bs.ratio) : "—"; + + const flagText = valid + ? pinValid + ? `--mamba-full-memory-ratio ${ratioStr} # or: --max-mamba-cache-size ${pin}` + : `--mamba-full-memory-ratio ${ratioStr}` + : ""; + + // Broadcast both results: the Deploy command takes the base-config value, + // the Playground's composed command takes the effective one. + useEffect(() => { + window.dispatchEvent( + new CustomEvent("sglang-k3-mamba-ratio", { + detail: { + ratio: valid ? ratioStr : null, + baseRatio: baseValid ? baseRatioStr : null, + }, + }) + ); + }, [ratioStr, valid, baseRatioStr, baseValid]); + + const copy = () => { + if (!valid) return; + navigator.clipboard.writeText(flagText).then(() => { + setCopied(true); + setTimeout(() => setCopied(false), 1200); + }); + }; + + const colors = { + border: isDark ? "#374151" : "#e5e7eb", + panel: isDark ? "#1f2937" : "#ffffff", + input: isDark ? "#111827" : "#f8fafc", + text: isDark ? "#e5e7eb" : "#1f2937", + muted: isDark ? "#9ca3af" : "#64748b", + accent: isDark ? "#E85D4D" : "#D45D44", + error: isDark ? "#fca5a5" : "#b91c1c", + }; + + const inputStyle = { + width: "100%", + boxSizing: "border-box", + padding: "8px 10px", + border: `1px solid ${colors.border}`, + borderRadius: "5px", + background: colors.input, + color: colors.text, + fontSize: "13px", + }; + const labelStyle = { + display: "flex", + flexDirection: "column", + gap: "5px", + fontSize: "12px", + fontWeight: 600, + }; + const chipStyle = { + padding: "3px 9px", + border: `1px solid ${colors.border}`, + borderRadius: "999px", + background: colors.input, + color: colors.text, + fontSize: "12px", + whiteSpace: "nowrap", + }; + + // Everything the ratio depends on except L and the concurrency target, read + // back from the panels so the reader can see what the number was derived from. + const derivedChips = [ + `KV ${kvDtype === "fp8_e4m3" ? "FP8" : "BF16"}`, + `State ${ssmDtype === "float32" ? "FP32" : ssmDtype === "bfloat16" ? "BF16" : "FP16"}`, + radixOff ? "Radix off (S = 1)" : `${strategy} (S = ${slots})`, + specOn ? `Spec on (D = ${drafts})` : "NOSPEC", + ]; + + return ( +
+
+ + + + +
+ + Serving configuration (follows the Deploy panel and Playground) + +
+ {derivedChips.map((c) => ( + {c} + ))} +
+
+
+ + {!valid ? ( +
+ {tp !== 1 + ? `This calculator models the single-GPU (TP1) geometry; the panels are at TP${tp}.` + : "Enter a valid request length."} +
+ ) : ( +
+
+
+ Balanced ratio — pinned into the commands above +
+
{ratioStr}
+ {baseValid && baseRatioStr !== ratioStr ? ( +
+ Deploy command (without Playground overrides): {baseRatioStr} +
+ ) : null} +
+ + {flagText} + + +
+ )} +
+ state/slot {(stateBytesPerSlot / 1e6).toFixed(1)} MB · KV/token{" "} + {(kvBytesPerToken / 1e3).toFixed(1)} KB · {slots + drafts} state slots per + request, so {targetConcurrency || "N"} concurrent requests need{" "} + {pinValid ? pin : "—"} slots. +
+
+ ); +}; diff --git a/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx b/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx new file mode 100644 index 000000000..41f5252aa --- /dev/null +++ b/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx @@ -0,0 +1,435 @@ +// Single `export const config` literal — no spreads/calls/IIFE (Mintlify re-evals at hydration). +// Cells are denormalized: no `--nnodes`/`--node-rank`/`--dist-init-addr`/`--host`/`--port` literals — engine injects them. +// +// Qwen3.8-27B: DENSE hybrid Gated Delta Networks VISION-LANGUAGE model — a 27B +// causal LM plus a vision encoder, served through SGLang's Qwen3-VL path +// (Qwen3_5ForConditionalGeneration extends Qwen3VLForConditionalGeneration and +// is registered in the multimodal arch lists). 64 layers as 16 repeats of +// 3 x (Gated DeltaNet -> FFN) then 1 x (Gated Attention -> FFN): 48 +// linear-attention layers to 16 full-attention. GDN runs 48 value heads and 16 +// QK heads at head_dim 128; attention is GQA 24/4 at head_dim 256. An MTP head +// trained with multiple steps ships in-checkpoint. Context 262,144 native, +// extensible to 1,000,000. Dense, so there is no MoE axis. +// +// Single-GPU on every supported card — H200 (SM90 datacenter), the SM120 +// workstation pair (RTX PRO 6000 Blackwell / RTX 5090), and DGX Spark (GB10, +// SM121, 128GB unified memory) — hence one node and no parallelism flags in +// any cell. +// +// PROVENANCE — every flag and value below is transcribed from the +// pre-migration prose page (one unconditional launch command + its Configuration +// Tips). Model ids are the exception: BF16/FP8 point at the official Qwen +// checkpoints, NVFP4 at the RadixArk W4A4 build. That page pinned +// no sglang version for its measurements, so under the +// migration skill's reproducible-anchor rule NO measured numbers were carried +// over: there is no sibling `-benchmarks.jsx`. Cells are nevertheless marked +// `verified: true` at the maintainers' direction — the badge there reflects +// their own unpublished validation, not measured data carried by this page. The +// DGX Spark cells are the exception and stay unverified: that recipe is +// unvalidated on SM121 / aarch64, as both §2 and the cell comment below say. +// `benchmarkCommands` below records the page's measurement protocol so the +// numbers can be re-measured against a pinned build and then land as a +// benchmarks file. +// +// A hardware x quantization combination with no launch recipe has no cell, and +// the engine greys it out. + +export const config = { + modelName: "Qwen3.8-27B", + + supportedHardware: ["h200", "rtx6000", "rtx5090", "dgx-spark"], + + // RTX PRO 6000 and RTX 5090 (SM120 / Blackwell Desktop) are workstation and + // consumer cards, not datacenter GPUs, so they are not in the shared catalog. + // Ids/labels match the DeepSeek-V4 config's entries for the same two cards. + // DGX Spark needs no entry here: it is already in the shared catalog + // (_deployment.jsx HARDWARE_CATALOG), with its multi-node Docker flags. + hardware: [ + { id: "rtx6000", label: "RTX PRO 6000", vram: "96GB", vendor: "blackwell" }, + { id: "rtx5090", label: "RTX 5090", vram: "32GB", vendor: "blackwell" }, + ], + + variants: [ + { id: "default", label: "Default" }, + ], + // BF16/FP8 are the official Qwen checkpoints; NVFP4 is the RadixArk + // W4A4 build. NVFP4 is W4A4 with FP8 projections and declares + // `kv_cache_quant_algo: FP8`, so under the default `--kv-cache-dtype auto` its + // KV pool runs fp8_e4m3 off the checkpoint's own calibration scales — no + // `--kv-cache-dtype` flag in the recipe, and nothing accuracy-degrading added + // by the cell. + quantizations: [ + { id: "bf16", label: "BF16" }, + { id: "fp8", label: "FP8" }, + { id: "nvfp4", label: "NVFP4" }, + ], + // The source page documents ONE operating point: a single general-purpose + // launch command with no latency/throughput toggle. MTP is described as an + // opt-in in the tips, not as a second named recipe, so it rides the + // Playground's speculative axis instead of splitting the strategy dimension. + strategies: [ + { id: "balanced", label: "Balanced" }, + ], + nodesOptions: [ + { id: "single", label: "Single Node" }, + ], + + modelNames: { + "default|bf16": "Qwen/Qwen3.8-27B", + "default|fp8": "Qwen/Qwen3.8-27B-FP8", + "default|nvfp4": "RadixArk/Qwen3.8-27B-NVFP4", + }, + + placeholders: { + HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" }, + PORT: { target: "command", label: "Bind port", default: "30000" }, + HF_TOKEN: { target: "command", label: "HF token (Docker)", default: "" }, + CURL_HOST: { target: "curl", label: "Server host", default: "localhost" }, + CURL_PORT: { target: "curl", label: "Server port", default: "30000" }, + }, + + curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\ +-H 'Content-Type: application/json' \\ +-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`, + + // The measurement protocol the source page described, kept so its numbers can + // be reproduced against a pinned build. --random-range-ratio 1 pins ISL + // exactly rather than drawing a range; --flush-cache measures cache-cold + // (bench_serving's `random` prompts are deterministic, so a warm rerun would + // hit the radix cache and inflate throughput) — the page's own "prefix caching + // disabled" protocol. + benchmarkCommands: { + speed: +`python3 -m sglang.bench_serving \\ + --backend sglang-oai \\ + --host {{CURL_HOST}} --port {{CURL_PORT}} \\ + --model {{MODEL_NAME}} \\ + --dataset-name {{DATASET}} \\ + --random-input-len {{ISL}} --random-output-len {{OSL}} --random-range-ratio 1 \\ + --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\ + --request-rate inf \\ + --flush-cache`, + accuracy: { + gsm8k_pct: +`python3 -m sglang.test.run_eval \\ + --host http://{{CURL_HOST}} --port {{CURL_PORT}} \\ + --model {{MODEL_NAME}} \\ + --eval-name gsm8k \\ + --num-examples 1319`, + }, + }, + + accuracyLabels: [ + ["gsm8k_pct", "GSM8K", "%"], + ], + + dockerImages: { + h200: "lmsysorg/sglang:qwen38-27b", + rtx6000: "lmsysorg/sglang:qwen38-27b", + rtx5090: "lmsysorg/sglang:qwen38-27b", + // TODO: verify an arm64 build of this tag for DGX Spark (GB10 is aarch64); + // the x86-only tag will not pull there. + "dgx-spark": "lmsysorg/sglang:qwen38-27b", + }, + + github: { + cookbookModel: "Qwen/Qwen3.8-27B", + }, + + playgroundFeatures: { + + // No "Attention Parallelism" card. The source page is single-GPU + // throughout and no cell carries a parallelism flag, so there is nothing to + // override: DP-Attention targets MLA models, prefill-CP has no model-side + // integration for this architecture, and a TP knob would desync the ratio + // calculator below (its geometry is TP1-only, so it would stop emitting and + // the command would silently fall back to the 0.9 default this page warns + // about). Re-add it together with TP-aware geometry in the calculator. + + // ----- Card: "Parsers" ----- + // Same parser pair the Qwen3.8 flagship page ships, and baked into every + // cell: this model is used through agent harnesses, and a deploy command + // without them returns tool calls as raw text instead of structured + // `tool_calls`. So this card is an opt-OUT — the handler derives both chips + // as already-on from the cell and strips the flag when one is toggled off. + parsers: { + items: [ + { id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser qwen3" }, + { id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser qwen3_coder" }, + ], + }, + + // ----- Card: "Speculative Decoding" ----- + // The in-checkpoint 1-layer MTP head. The source page wrote the preset as + // `--speculative-algorithm NEXTN`; NEXTN is an alias of EAGLE, so it is + // normalized here (the Playground strips/derives by the first token, and an + // alias would survive toggles and double up). DSpark is the trained draft + // model, a separate checkpoint. + speculative: { + options: [ + { id: "current", label: "Inherited from base" }, + { id: "off", label: "Off (greedy)" }, + { id: "mtp", label: "EAGLE / MTP", + flags: ["--speculative-algorithm EAGLE", "--speculative-num-steps 3", + "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 4"] }, + { id: "dspark", label: "DSpark", + flags: ["--speculative-algorithm DSPARK", + "--speculative-draft-model-path RadixArk/Qwen3.8-27B-DSpark"] }, + ], + }, + + // ----- Card: single-selects over one flag family each ----- + flagSelects: [ + { + // trtllm_mha is SM100-only, so every cell bakes flashinfer (SM90 and + // SM120 alike). FA3 is the SM90 alternative — measured slightly faster + // at bs=1 on H200. Triton is the documented fallback when MTP runs on a + // FlashInfer build whose prefill `plan` predates `uniform_q_len` + // (<= 0.6.15.post1). + id: "attnBackend", title: "Attention Backend", + stripPrefixes: ["--attention-backend"], + options: [ + { id: "flashinfer", label: "FlashInfer (default)", + flags: ["--attention-backend flashinfer"] }, + { id: "fa3", label: "FlashAttention-3 (SM90 only)", + flags: ["--attention-backend fa3"] }, + { id: "triton", label: "Triton — MTP fallback on older FlashInfer", + flags: ["--attention-backend triton"] }, + ], + }, + { + // Halving kv_bytes_per_token (65.5 KB bf16 -> 32.8 KB fp8) doubles the + // KV pool at a fixed --mamba-full-memory-ratio. Accuracy-degrading over + // a bf16-KV checkpoint, so it stays an opt-in and is never in a cell — + // the NVFP4 checkpoint gets fp8 KV on its own via kv_cache_quant_algo. + id: "kvCacheDtype", title: "KV Cache Precision", + stripPrefixes: ["--kv-cache-dtype"], + options: [ + { id: "auto", label: "Auto (checkpoint-declared)" }, + { id: "fp8", label: "FP8 (E4M3) — halves KV memory", flags: ["--kv-cache-dtype fp8_e4m3"] }, + { id: "bf16", label: "BFloat16", flags: ["--kv-cache-dtype bfloat16"] }, + ], + }, + { + // One state slot is 154.7 MB at fp32, 79.2 MB at bf16 — the single + // biggest lever on the GDN state pool, which is what bounds concurrency + // on small-VRAM cards. + id: "mambaSsmDtype", title: "GDN State Precision", + stripPrefixes: ["--mamba-ssm-dtype"], + options: [ + { id: "auto", label: "Auto (FP32)" }, + { id: "bf16", label: "BFloat16 — halves state memory", flags: ["--mamba-ssm-dtype bfloat16"] }, + ], + }, + { + // Whole-model prefix cache. Off drops the per-request state cost to + // S=1 slot, which is the cheapest way to buy concurrency on a 32GB card + // when the traffic has no shared prefixes (offline batch, evals). + id: "prefixCache", title: "Prefix Cache", + stripPrefixes: ["--disable-radix-cache"], + options: [ + { id: "on", label: "On" }, + { id: "off", label: "Off (S=1)", flags: ["--disable-radix-cache"] }, + ], + }, + { + // How GDN state buffers for radix reuse — slot cost per request S: + // extra_buffer 5, extra_buffer_lazy 4, no_buffer 3. No strategy exists + // with the prefix cache off, so the row hides (and stops emitting) there. + // Changing S changes the balanced ratio — recompute it in the page's + // calculator after picking a strategy here. + id: "mambaRadix", title: "GDN Radix Cache Strategy", + showWhen: (b, v, d) => (((v && v.prefixCache) ?? (d && d.prefixCache)) !== "off"), + stripPrefixes: ["--mamba-radix-cache-strategy"], + options: [ + { id: "auto", label: "Auto (extra_buffer, S=5)" }, + { id: "lazy", label: "extra_buffer_lazy (S=4)", flags: ["--mamba-radix-cache-strategy extra_buffer_lazy"] }, + { id: "nobuf", label: "no_buffer (S=3)", flags: ["--mamba-radix-cache-strategy no_buffer"] }, + ], + }, + ], + }, + + // Every cell is the source page's single launch command with only the model id + // varying. The H200 and SM120 cells are `verified: true` at the maintainers' + // direction; the DGX Spark cells are not, matching the unvalidated-on-SM121 + // note on those cells. The page carries no measured data of its own (no + // `-benchmarks.jsx`), so a badge rests on validation held outside this page — + // re-check it against the per-platform notes in §2 before trusting a cell. + // + // Cells carry NO --mamba-full-memory-ratio. The source page's worked 4.6 held + // only for the NVFP4 recipe at 4096-in/1024-out; the ratio is a function of + // the workload, of S (radix-cache strategy / prefix cache), of D (spec) and of + // kv_bytes_per_token, all of which the Playground can change. So the page's + // ratio calculator computes it live from the effective config and broadcasts + // it, and the engines pin it into the rendered command — which they only do + // while the cell itself stays ratio-free (_deployment.jsx `cellWithRatio`). + // Adding the flag back here would silently freeze the value again. + cells: [ + { + // H200 141GB, FP8 blockwise (~28.5GB of weights). The 32768-token chunk + // is the H200-validated setting: SM90 prefill is fast enough that a big + // chunk stalls decode far less than on SM120, and the SM90 FlashInfer GDN + // prefill default engages under it (fp32 state pool, chunk <= 32768). + // No NVFP4 cell on this card: SM90 has no FP4 tensor cores, so the W4A4 + // checkpoint's MLP would fall back to the Marlin W4A16 weight-only path — + // runnable, but not a recipe this page ships. + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--mem-fraction-static 0.85", + "--attention-backend flashinfer", + "--chunked-prefill-size 32768", + "--max-prefill-tokens 32768", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // H200, BF16 reference checkpoint (~54GB of weights). + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--mem-fraction-static 0.85", + "--attention-backend flashinfer", + "--chunked-prefill-size 32768", + "--max-prefill-tokens 32768", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // The page's headline recipe: NVFP4 W4A4 on the 96GB workstation card, + // ~16.5GB of weights, fp8 KV auto-enabled by the checkpoint. + match: { hw: "rtx6000", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--mem-fraction-static 0.85", + "--attention-backend flashinfer", + "--chunked-prefill-size 2048", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // FP8 blockwise, ~28.5GB of weights — comfortable on 96GB. + match: { hw: "rtx6000", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--mem-fraction-static 0.85", + "--attention-backend flashinfer", + "--chunked-prefill-size 2048", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // BF16, the reference checkpoint. + match: { hw: "rtx6000", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--mem-fraction-static 0.85", + "--attention-backend flashinfer", + "--chunked-prefill-size 2048", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // RTX 5090 32GB. NVFP4 is the only checkpoint that fits with room to + // serve (~16.5GB); FP8 at ~28.5GB is not serviceable past bs<=2 and BF16 + // does not fit, so neither has a cell. On this card the GDN state pool — + // not KV — bounds concurrency: lower S with the Playground's radix-cache + // strategy (or turn the prefix cache off for S=1) and recompute the ratio. + match: { hw: "rtx5090", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--mem-fraction-static 0.85", + "--attention-backend flashinfer", + "--chunked-prefill-size 2048", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // DGX Spark (GB10, SM121): single node, 128GB coherent unified memory + // shared with the CPU — every checkpoint fits, so all three quants get a + // cell. FlashInfer attention comes from the SM120 pair; the platform gets + // its own operating point at 8192-token prefill chunks and 0.95 static + // fraction. Unvalidated on SM121 / aarch64. + { + match: { hw: "dgx-spark", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--mem-fraction-static 0.95", + "--attention-backend flashinfer", + "--chunked-prefill-size 8192", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "dgx-spark", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--mem-fraction-static 0.95", + "--attention-backend flashinfer", + "--chunked-prefill-size 8192", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "dgx-spark", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--mem-fraction-static 0.95", + "--attention-backend flashinfer", + "--chunked-prefill-size 8192", + "--reasoning-parser qwen3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + ], +}; diff --git a/docs/src/snippets/configs/popular-models.jsx b/docs/src/snippets/configs/popular-models.jsx index e646781dc..6c77df09a 100644 --- a/docs/src/snippets/configs/popular-models.jsx +++ b/docs/src/snippets/configs/popular-models.jsx @@ -12,6 +12,23 @@ // paraphrasing that page's own opening. export const popularModels = [ + { + name: "Qwen3.8-27B", + vendor: "Qwen", + href: "/cookbook/autoregressive/Qwen/Qwen3.8-27B", + logo: "/cards/logos/qwen.png", + badge: "New", + tags: ["4 platforms", "Hybrid GDN", "BF16 / FP8 / NVFP4"], + hero: { + eyebrow: "Featured model \u00b7 New", + headline: "Meet Qwen3.8-27B on SGLang", + blurb: + "A dense hybrid Gated Delta Networks model \u2014 48 GDN linear-attention layers interleaved with 16 full-attention, an in-checkpoint MTP head, and a native 262,144-token context. The cookbook covers single-GPU serving on H200 and RTX PRO 6000 / 5090.", + tags: ["Dense 27B", "262K context", "Single-GPU"], + cta: "Open the Qwen3.8-27B cookbook", + caption: "Qwen3.8-27B deployment guide", + }, + }, { name: "Kimi-K3", vendor: "Moonshot AI",