diff --git a/docs_new/cookbook/autoregressive/ThinkingMachines/Inkling.mdx b/docs_new/cookbook/autoregressive/ThinkingMachines/Inkling.mdx index 220cb3e9e..a272a5fc0 100644 --- a/docs_new/cookbook/autoregressive/ThinkingMachines/Inkling.mdx +++ b/docs_new/cookbook/autoregressive/ThinkingMachines/Inkling.mdx @@ -37,6 +37,7 @@ There are two multi-arch (amd64 / arm64) CUDA builds plus a ROCm build; pick the docker pull lmsysorg/sglang:inkling-cu13 # CUDA 13 docker pull lmsysorg/sglang:inkling-cu12 # CUDA 12 docker pull lmsysorg/sglang:inkling-rocm700-mi35x # AMD MI350X / MI355X +docker pull lmsysorg/sglang:dev-cu13-inkling-dspark # CUDA 13 + DSpark support ``` For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces. @@ -47,7 +48,7 @@ For how to launch the image, see [Install → Method 3: Using Docker](../../../d -Pick your hardware to generate the launch command. Each platform ships a **Balanced** recipe plus an **MTP** (speculative decoding) tier and a **Long Context (MXFP8 KV)** tier where validated; the **LoRA** variant serves adapters on top of the frozen base model. Set `MAX_LORAS` to the number of distinct adapters you serve (1 is fastest for single-adapter serving). +Pick your hardware to generate the launch command. Each platform ships a **Balanced** recipe plus **MTP** and **DSpark** (speculative decoding) tiers and a **Long Context (MXFP8 KV)** tier where validated; the **LoRA** variant serves adapters on top of the frozen base model. Set `MAX_LORAS` to the number of distinct adapters you serve (1 is fastest for single-adapter serving). import { Deployment } from "/src/snippets/_deployment.jsx"; import { config } from "/src/snippets/configs/thinkingmachines/inkling.jsx"; @@ -301,3 +302,9 @@ The **Long Context** deploy strategy adds `--kv-cache-dtype mxfp8` on top of the The tradeoff is not just a ~5% decode latency penalty from the extra quantize/dequantize work versus BF16 KV — storing KV in MXFP8 also introduces some accuracy loss at long context lengths. Treat it as a capacity lever, not a speed one — stay on **Balanced** if you have headroom in the memory pool and just want lower latency or maximum output quality. To try it, select the **Long Context** strategy in the Deploy panel above for any NVFP4 cell; the panel regenerates the launch command with `--kv-cache-dtype mxfp8` inserted. Verified end-to-end on B200, B300, and GB300. + +### 3.7 DSpark (Speculative Decoding) + +The **DSpark** deploy strategy is the second speculative-decoding path for Inkling. Unlike **MTP**, which drives Inkling's own multi-layer draft head, DSpark runs a **separate draft checkpoint** — `RadixArk/Inkling-DSpark-Preview` — served unquantized alongside the NVFP4 target. + +It needs an SGLang build with DSpark support — `lmsysorg/sglang:dev-cu13-inkling-dspark`, which the Deploy panel's **Docker** command uses for this tier; the `inkling-cu13` / `inkling-cu12` images don't carry DSpark yet. Verified end-to-end on B200 (TP=8, NVFP4). diff --git a/docs_new/src/snippets/_deployment.jsx b/docs_new/src/snippets/_deployment.jsx index c1b657ba3..5a74937dc 100644 --- a/docs_new/src/snippets/_deployment.jsx +++ b/docs_new/src/snippets/_deployment.jsx @@ -51,8 +51,9 @@ // override the page value per cell (entry → config → "P50"). // Legacy "Mean" data is being re-measured to P50; drop once done // multiNodeHints optional — {[hwId]: string[]} prepended as `# ...` lines -// dockerImages optional — `docker run` image, keyed by `hw|quant` -// then `hw`; falls back to `lmsysorg/sglang:dev` +// dockerImages optional — `docker run` image, keyed by +// `hw|quant|strategy` then `hw|quant` then `hw`; +// falls back to `lmsysorg/sglang:dev` // runModes optional — command output tabs to show (`python` and/or // `docker`); defaults to both, in that order // github optional — "Submit verified cell" issue-template overrides @@ -674,9 +675,12 @@ export const Deployment = ({ config, benchmarks }) => { let cmd; if (mode === "docker") { - // Image keyed by `hw|quant` (most specific) then `hw`; `:dev` if unmapped. + // Image keyed by `hw|quant|strategy` (most specific), then `hw|quant`, + // then `hw`; `:dev` if unmapped. The strategy key covers a tier that + // needs its own build (e.g. a spec-decoding preview image). const di = config.dockerImages || {}; - const image = di[`${sel.hw}|${sel.quant}`] || di[sel.hw] || "lmsysorg/sglang:dev"; + const image = di[`${sel.hw}|${sel.quant}|${sel.strategy}`] + || di[`${sel.hw}|${sel.quant}`] || di[sel.hw] || "lmsysorg/sglang:dev"; const portFlag = flags.find((x) => x.split(/[\s=]/)[0] === "--port"); const servePort = portFlag ? portFlag.slice("--port".length).trim() : "{{PORT}}"; const vendorOf = (hwId) => { diff --git a/docs_new/src/snippets/_playground.jsx b/docs_new/src/snippets/_playground.jsx index 926f1c577..cfb2020fb 100644 --- a/docs_new/src/snippets/_playground.jsx +++ b/docs_new/src/snippets/_playground.jsx @@ -1365,9 +1365,14 @@ export const Playground = ({ config }) => { } let cmd; if (mode === "docker") { - // Image keyed by `hw|quant` (most specific) then `hw`; `:dev` if unmapped (matches _deployment.jsx). + // Image keyed by `hw|quant|strategy` (most specific), then `hw|quant`, then + // `hw`; `:dev` if unmapped (matches _deployment.jsx). The strategy key covers + // a tier that needs its own build (e.g. a spec-decoding preview image), so the + // playground base must resolve it too or it hands back an image that cannot + // run the command. const di = config.dockerImages || {}; - const image = di[`${sel.hw}|${sel.quant}`] || di[sel.hw] || "lmsysorg/sglang:dev"; + const image = di[`${sel.hw}|${sel.quant}|${sel.strategy}`] + || di[`${sel.hw}|${sel.quant}`] || di[sel.hw] || "lmsysorg/sglang:dev"; const portFlag = f.find((x) => x.split(/[\s=]/)[0] === "--port"); const servePort = portFlag ? portFlag.slice("--port".length).trim() : "{{PORT}}"; const dockerLines = [ diff --git a/docs_new/src/snippets/configs/thinkingmachines/inkling-benchmarks.jsx b/docs_new/src/snippets/configs/thinkingmachines/inkling-benchmarks.jsx index 7aa9fda51..aac64cc35 100644 --- a/docs_new/src/snippets/configs/thinkingmachines/inkling-benchmarks.jsx +++ b/docs_new/src/snippets/configs/thinkingmachines/inkling-benchmarks.jsx @@ -32,6 +32,7 @@ export const benchmarks = [ { match: { hw: "gb200" , variant: "default" , quant: "nvfp4" , strategy: "mtp" , nodes: "single" } }, { match: { hw: "gb300" , variant: "default" , quant: "nvfp4" , strategy: "mtp" , nodes: "single" } }, { match: { hw: "h200" , variant: "default" , quant: "nvfp4" , strategy: "mtp" , nodes: "single" } }, + { match: { hw: "b200" , variant: "default" , quant: "nvfp4" , strategy: "dspark" , nodes: "single" } }, { match: { hw: "b200" , variant: "default" , quant: "nvfp4" , strategy: "long_context" , nodes: "single" } }, { match: { hw: "b300" , variant: "default" , quant: "nvfp4" , strategy: "long_context" , nodes: "single" } }, { match: { hw: "gb200" , variant: "default" , quant: "nvfp4" , strategy: "long_context" , nodes: "single" } }, diff --git a/docs_new/src/snippets/configs/thinkingmachines/inkling.jsx b/docs_new/src/snippets/configs/thinkingmachines/inkling.jsx index eb4e750ad..319f70320 100644 --- a/docs_new/src/snippets/configs/thinkingmachines/inkling.jsx +++ b/docs_new/src/snippets/configs/thinkingmachines/inkling.jsx @@ -26,6 +26,7 @@ export const config = { strategies: [ { id: "balanced", label: "Balanced" }, { id: "mtp", label: "MTP" }, + { id: "dspark", label: "DSpark" }, { id: "long_context", label: "Long Context (MXFP8 KV)" }, ], nodesOptions: [ @@ -73,7 +74,10 @@ export const config = { // NVIDIA: two multi-arch CUDA builds (inkling-cu12 / inkling-cu13) — pick by your // CUDA version, not by GPU. AMD: inkling-rocm700-mi35x. Panel defaults to cu13. + // The DSpark tier needs its own preview build (DSpark isn't in the inkling-cu1x + // images yet), so it takes a `hw|quant|strategy` key. dockerImages: { + "b200|nvfp4|dspark": "lmsysorg/sglang:dev-cu13-inkling-dspark", h200: "lmsysorg/sglang:inkling-cu13", b200: "lmsysorg/sglang:inkling-cu13", b300: "lmsysorg/sglang:inkling-cu13", @@ -682,6 +686,49 @@ export const config = { "--port {{PORT}}", ], }, + // ==================================================================== + // DSpark (speculative decoding) — separate draft checkpoint + // (RadixArk/Inkling-DSpark-Preview, served unquantized) instead of Inkling's + // own MTP head, so it needs a build carrying DSpark support: the + // `dev-cu13-inkling-dspark` image (see the dockerImages key above). + // The draft weights sit outside the FP4 target, hence mem-fraction 0.68. + // B200 verified end-to-end. + // ==================================================================== + { + match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "dspark", nodes: "single" }, + verified: true, + env: [ + "SGLANG_ENABLE_UNIFIED_RADIX_TREE=1", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--quantization modelopt_fp4", + "--attention-backend fa4", + "--page-size 128", + "--fp4-gemm-backend flashinfer_trtllm", + "--moe-runner-backend flashinfer_trtllm_routed", + "--enable-torch-symm-mem", + "--mamba-radix-cache-strategy extra_buffer", + "--mem-fraction-static 0.68", + "--swa-full-tokens-ratio 0.1", + "--mamba-full-memory-ratio 0.1", + "--max-running-requests 68", + "--reasoning-parser inkling", + "--tool-call-parser inkling", + "--skip-server-warmup", + "--speculative-algorithm DSPARK", + "--speculative-draft-model-path RadixArk/Inkling-DSpark-Preview", + "--speculative-draft-model-quantization unquant", + "--chunked-prefill-size 8192", + "--cuda-graph-max-bs-prefill 8192", + "--disable-flashinfer-autotune", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // ==================================================================== // GB300 BF16 — 2x GB300 nodes (4 GPUs each) over MNNVL. The NCCL_MNNVL / // NVLS / CUMEM envs are required: 2-node NCCL init hangs without them.