diff --git a/.claude/skills/cookbook-add-model/SKILL.md b/.claude/skills/cookbook-add-model/SKILL.md index 4b8da4ed5..3c3e10571 100644 --- a/.claude/skills/cookbook-add-model/SKILL.md +++ b/.claude/skills/cookbook-add-model/SKILL.md @@ -149,7 +149,8 @@ this table (RTX PRO 6000, GH200, future chips) goes in the model's own `config.h 4. **Fill `cells[]`** with the verified recipes from Phase 1 (replace every EXAMPLE cell; set `verified: true` only on tested combos), and `modelNames` with real HF slugs, `dockerImages` for your hw (use the Phase-1 tag, or default `lmsysorg/sglang:dev` — never - a guessed release), `multiNodeHints` only for fabric-specific hw (e.g. gb200). + a guessed release; key by `hw`, or `hw|quant` when one quant on a shared GPU needs its own + image), `multiNodeHints` only for fabric-specific hw (e.g. gb200). ### Site-wiring (do all three) diff --git a/.claude/skills/cookbook-add-model/references/authoring-reference.md b/.claude/skills/cookbook-add-model/references/authoring-reference.md index bb8f9cf94..5ddd7a2ad 100644 --- a/.claude/skills/cookbook-add-model/references/authoring-reference.md +++ b/.claude/skills/cookbook-add-model/references/authoring-reference.md @@ -44,7 +44,7 @@ the full contract): | Field | Type | Purpose | |---|---|---| | `multiNodeHints` | `{[hwId]: string[]}` | Lines prepended as `# ...` comments to multi-node commands (env-var hints). Per-hw, and only for hw whose **cluster fabric needs manual NIC config** (e.g. `gb200` NVL72/MNNVL → NVSHMEM/Gloo hints). NOT every multi-N hw needs an entry — standard-IB DeepEP (h200) auto-detects the HCA, and Marlin multi-node (h100) uses no DeepEP/NVSHMEM at all. | -| `dockerImages` | `{[hwId]: string}` | Per-hw image name for `docker run` framing. **Ask the user which sglang build the recipes ran on; don't guess a supporting release.** Falls back to `lmsysorg/sglang:dev` if missing — also the sensible default when unsure. | +| `dockerImages` | `{[key]: string}` | Image for `docker run` framing, keyed by `hw\|quant` (most specific) then `hw`. Use a `hw\|quant` key only when one quant on a shared GPU needs a different image (e.g. an NVFP4 dev build on b300/gb300 while FP8/BF16 stay on the release image); otherwise key by plain `hw`. **Ask the user which sglang build the recipes ran on; don't guess a supporting release.** Falls back to `lmsysorg/sglang:dev` if missing — also the sensible default when unsure. | | `playgroundFeatures` | `{[axisId]: {...}}` | Opts into the Playground widget. See §2.3. | | `benchmarkCommands` | `{speed: string, accuracy: {[accKey]: string \| {[variant]: string}}, numPromptsByConc?: {[c]: number}}` | Powers the benchmark card's **"⚡ Reproduce"** modal. `speed` is ONE `bench_serving` template; the engine fills `{{DATASET}}`/`{{ISL}}`/`{{OSL}}` from each cell's `speed[].workload`, the chip-picked `{{MAX_CONCURRENCY}}`, and `{{NUM_PROMPTS}}` (resolved `workload.num_prompts ?? numPromptsByConc[c] ?? max(c*2, 200)`). `accuracy` maps an accuracy field (e.g. `gsm8k_pct`) to a per-eval template — a string, OR a `{flash, pro, …}` object keyed by variant when the command differs per variant (e.g. GPQA/AIME `--max-tokens`). The modal renders a chip per eval (one command area, like Speed). Both also use `{{MODEL_NAME}}` + `{{CURL_HOST}}`/`{{CURL_PORT}}` like `curl`. Optional; the button only appears when this AND `benchmarks` are present. | | `defaultAccuracy` | `{[variant]: {[accKey]: number}}` | Model-level accuracy applied to **every** cell of a variant (e.g. GPQA Diamond / AIME25 — hardware-independent). Merged UNDER each cell's measured `accuracy` (a per-cell value wins), so you set a variant's score once instead of copying it onto every benchmark entry. Keys must match `accuracyLabels` (below) + `benchmarkCommands.accuracy`. | diff --git a/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl b/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl index 05d70ffd7..e3d482213 100644 --- a/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl +++ b/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl @@ -140,10 +140,12 @@ sgl-eval run gsm8k \\ ], }, - // Per-hw image for `docker run` framing. ASK the user which sglang build the recipes ran - // on; don't guess a supporting release. Default below is :dev (nightly) — replace the tag - // with the user's release if they give one. NVIDIA share one image; AMD uses ROCm tags. - // GB200/GB300/B300 may need a `-cu130` (CUDA 13) tag — confirm per release. + // Image for `docker run` framing, keyed by `hw` (or `hw|quant`, resolved first, when one + // quant on a shared GPU needs its own image — e.g. an FP4 dev build while FP8/BF16 use the + // release tag). ASK the user which sglang build the recipes ran on; don't guess a supporting + // release. Default below is :dev (nightly) — replace the tag with the user's release if they + // give one. NVIDIA share one image; AMD uses ROCm tags. GB200/GB300/B300 may need a + // `-cu130` (CUDA 13) tag — confirm per release. dockerImages: { h100: "lmsysorg/sglang:dev", h200: "lmsysorg/sglang:dev", diff --git a/.claude/skills/cookbook-review-pr/SKILL.md b/.claude/skills/cookbook-review-pr/SKILL.md index bfcbba50f..236878010 100644 --- a/.claude/skills/cookbook-review-pr/SKILL.md +++ b/.claude/skills/cookbook-review-pr/SKILL.md @@ -61,7 +61,9 @@ than restating. explicit maintainer-confirmed justification in the PR. The MDX strategy bullets describe serving semantics in the DSv4 style (single-user chat / typical multi-user / batch jobs), not internal toggles. -- `dockerImages` covers the hw ids that have cells (else users hit the `:dev` fallback). +- `dockerImages` covers the hw ids that have cells (else users hit the `:dev` fallback); a + `hw|quant` key (resolved before the plain `hw`) is valid when one quant on a shared GPU needs + a different image (e.g. an FP4 dev build) — don't flag those. - `multiNodeHints` present ONLY for hw whose fabric needs manual NIC env (e.g. `gb200` NVL72) — NOT every `multi-N` hw (standard-IB DeepEP / Marlin multi-node don't need it). - `github.cookbookModel` is set to the model's HF id (`/`). The issue diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx index 8275a54b9..79b8bcdcd 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx @@ -52,6 +52,10 @@ import { benchmarks } from "/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx + + **NVFP4 (B300 / GB300)** deploys on the dev image **`lmsysorg/sglang:dev-glm52-nvfp4`** — the command panel's **Docker** toggle selects it automatically. The FP8 / BF16 recipes use the release **`lmsysorg/sglang:latest`** (the `docker pull` in Install above). + + ## Playground The Playground is where you experiment with **SGLang features beyond the verified matrix**. The Deploy panel above only emits combinations the SGLang team has signed off on; the Playground lets you turn on additional knobs on top of whichever cell the Deploy panel is currently showing. @@ -62,7 +66,7 @@ import { Playground } from "/src/snippets/_playground.jsx"; ## 1. Model Introduction -**GLM-5.2** is Z.ai's flagship Mixture-of-Experts model built on **DeepSeek Sparse Attention (DSA)**: a lightning indexer selects a sparse set of key tokens per query (top-2048), so attention cost stays near-constant as context grows. It ships in two precisions — **FP8** (`zai-org/GLM-5.2-FP8`) and full **BF16** (`zai-org/GLM-5.2`) — both with **78 transformer layers**, **256 routed experts** (8 active per token), a **1M-token context window**, and a single **MTP (Multi-Token Prediction)** layer for built-in EAGLE-style speculative decoding. FP8 is the recommended deployment; BF16 (~1.5 TB) needs an 8×B300 node or a multi-node setup. +**GLM-5.2** is Z.ai's flagship Mixture-of-Experts model built on **DeepSeek Sparse Attention (DSA)**: a lightning indexer selects a sparse set of key tokens per query (top-2048), so attention cost stays near-constant as context grows. It ships in two precisions — **FP8** (`zai-org/GLM-5.2-FP8`) and full **BF16** (`zai-org/GLM-5.2`) — both with **78 transformer layers**, **256 routed experts** (8 active per token), a **1M-token context window**, and a single **MTP (Multi-Token Prediction)** layer for built-in EAGLE-style speculative decoding. FP8 is the recommended deployment; BF16 (~1.5 TB) needs an 8×B300 node or a multi-node setup. For Blackwell, NVIDIA also publishes an **NVFP4** build (`nvidia/GLM-5.2-NVFP4`) that quantizes only the MoE experts' linear weights and activations to 4-bit (the shared expert stays unquantized), holding accuracy within ~1 point of the FP8 baseline on GPQA Diamond, SciCode, and IFBench. @@ -83,12 +87,17 @@ import { Playground } from "/src/snippets/_playground.jsx"; + + + + +
MoE · DSA · 256 experts (top-8) · MTP · BF16 1,048,576
GLM-5.2-NVFP4MoE · DSA · 256 experts (top-8) · MTP · NVFP41,048,576
**Recommended generation:** `temperature=1.0`, `top_p=0.95` (the checkpoint's `generation_config.json` defaults; informational — do not hardcode in client code). -**Resources:** [GLM-5.2-FP8](https://huggingface.co/zai-org/GLM-5.2-FP8) · [GLM-5.2 (BF16)](https://huggingface.co/zai-org/GLM-5.2). +**Resources:** [GLM-5.2-FP8](https://huggingface.co/zai-org/GLM-5.2-FP8) · [GLM-5.2 (BF16)](https://huggingface.co/zai-org/GLM-5.2) · [GLM-5.2-NVFP4](https://huggingface.co/nvidia/GLM-5.2-NVFP4). ## 2. Configuration Tips diff --git a/docs_new/src/snippets/_deployment.jsx b/docs_new/src/snippets/_deployment.jsx index de8a915d4..907923fa0 100644 --- a/docs_new/src/snippets/_deployment.jsx +++ b/docs_new/src/snippets/_deployment.jsx @@ -24,7 +24,8 @@ // benchmark card + "⚡ Reproduce". NO engine default: // required whenever benchmarks carry accuracy data // multiNodeHints optional — {[hwId]: string[]} prepended as `# ...` lines -// dockerImages optional — per-hw image for `docker run` mode +// dockerImages optional — `docker run` image, keyed by `hw|quant` +// then `hw`; falls back to `lmsysorg/sglang:dev` // github optional — "Submit verified cell" issue-template overrides // playgroundFeatures optional — consumed by _playground.jsx (see its header) // @@ -500,8 +501,9 @@ export const Deployment = ({ config, benchmarks }) => { let cmd; if (mode === "docker") { - // Image picked by hardware; falls back to `:dev` if unmapped. - const image = (config.dockerImages && config.dockerImages[sel.hw]) || "lmsysorg/sglang:dev"; + // Image keyed by `hw|quant` (most specific) then `hw`; `:dev` if unmapped. + const di = config.dockerImages || {}; + const image = di[`${sel.hw}|${sel.quant}`] || di[sel.hw] || "lmsysorg/sglang:dev"; const portFlag = flags.find((x) => x.split(/[\s=]/)[0] === "--port"); const servePort = portFlag ? portFlag.slice("--port".length).trim() : "{{PORT}}"; const vendorOf = (hwId) => { diff --git a/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx b/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx index 79a3c341b..0324576a6 100644 --- a/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx +++ b/docs_new/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx @@ -171,4 +171,9 @@ export const benchmarks = [ { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "multi-2" } }, { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "multi-2" } }, { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "multi-2" } }, + // ---- NVFP4 (Blackwell Ultra) ---- benchmarks pending + { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" } }, + { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" } }, + { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" } }, ]; diff --git a/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx b/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx index 6a0decffc..cdc8dde54 100644 --- a/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx +++ b/docs_new/src/snippets/configs/zai-org/glm-5.2.jsx @@ -15,6 +15,7 @@ export const config = { quantizations: [ { id: "fp8", label: "FP8" }, { id: "bf16", label: "BF16" }, + { id: "nvfp4", label: "NVFP4" }, ], strategies: [ { id: "low-latency", label: "Low-Latency" }, @@ -29,6 +30,7 @@ export const config = { modelNames: { "default|fp8": "zai-org/GLM-5.2-FP8", "default|bf16": "zai-org/GLM-5.2", + "default|nvfp4": "nvidia/GLM-5.2-NVFP4", }, placeholders: { @@ -90,6 +92,9 @@ sgl-eval run aime25 \\ b200: "lmsysorg/sglang:latest", gb300: "lmsysorg/sglang:latest", b300: "lmsysorg/sglang:latest", + // NVFP4 needs the dev image with modelopt_fp4 support (per-quant override). + "b300|nvfp4": "lmsysorg/sglang:dev-glm52-nvfp4", + "gb300|nvfp4": "lmsysorg/sglang:dev-glm52-nvfp4", }, github: { @@ -628,5 +633,79 @@ sgl-eval run aime25 \\ "--port {{PORT}}", ], }, + + // ==================================================================== + // NVFP4 (Blackwell Ultra) — nvidia/GLM-5.2-NVFP4 (Model Optimizer). + // TP4 on B300 / GB300, low-latency + balanced. GB300 mirrors the B300 + // recipe (same TP4 / flags; the 4-GPU GB300 node fits the ~381 GB build). + // ==================================================================== + { + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--quantization modelopt_fp4", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 5", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 6", + "--chunked-prefill-size 131072", + "--mem-fraction-static 0.70", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--quantization modelopt_fp4", + "--chunked-prefill-size 131072", + "--mem-fraction-static 0.70", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--quantization modelopt_fp4", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 5", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 6", + "--chunked-prefill-size 131072", + "--mem-fraction-static 0.70", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--quantization modelopt_fp4", + "--chunked-prefill-size 131072", + "--mem-fraction-static 0.70", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, ], };