diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx index 8b88a8ec6..11fe7f8f1 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx @@ -1,6 +1,6 @@ --- title: Kimi-K3 -description: "Deploy Moonshot AI's Kimi-K3 with SGLang — a 2.8T-parameter hybrid Mixture-of-Experts vision-language model (Kimi Delta Attention + MLA, 16/896 active experts) with NVIDIA and AMD recipes." +description: "Deploy Moonshot AI's Kimi-K3 with SGLang — a 2.8T-parameter hybrid Mixture-of-Experts vision-language model (Kimi Delta Attention + MLA, 16/896 active experts) with NVIDIA, AMD, and NPU recipes." tag: NEW --- @@ -37,11 +37,26 @@ For how to launch the image, see [Install → Method 3: Using Docker](../../../d + + +```bash Command +docker pull quay.io/ascend/sglang:main-cann9.0.0-a3 +``` + +For host and platform setup, see the +[NPU installation guide](../../../docs/hardware-platforms/ascend-npus/getting-started/installation) and the +[quick start guide](../../../docs/hardware-platforms/ascend-npus/getting-started/quick_start). + +**Weights (NPU):** [Kimi-K3-W4A8](https://www.modelscope.cn/models/sgl-npu/Kimi-K3-W4A8) (W4A8, 1.49 TB) · +[Kimi-K3-DSpark](https://www.modelscope.cn/models/RadixArk/Kimi-K3-DSpark) (DSPARK draft, 4.5 GB) + + + -Pick your hardware, then the deployment shape and operating point. Node count follows the hardware recipe (B200 2×8, GB200 4×4, H100 4×8, B300 1×8, H200 2×8 — 4×8 on Unified High-Throughput, GB300 2×4, MI350X/MI355X 1×8), so it is not a separate choice. If you serve the NVFP4 checkpoint (`nvidia/Kimi-K3-NVFP4`, the **Quantization** row in the panel below), use the `lmsysorg/sglang:dev-dev-kimi-k3-nvfp4` image. +Pick your hardware, then the deployment shape and operating point. Node count follows the hardware recipe (B200 2×8, GB200 4×4, H100 4×8, B300 1×8, H200 2×8 — 4×8 on Unified High-Throughput, GB300 2×4, MI350X/MI355X 1×8, Atlas 800I A3 4×8 — 32 cards / 64 dies), so it is not a separate choice. If you serve the NVFP4 checkpoint (`nvidia/Kimi-K3-NVFP4`, the **Quantization** row in the panel below), use the `lmsysorg/sglang:dev-dev-kimi-k3-nvfp4` image. **PD Mode** — `Unified` serves prefill and decode together. `Prefill` / `Decode` split them into dedicated pools (see [PD disaggregation](#3-4-pd-disaggregation)); `Prefill` ships two strategies, both chunked at 16k. On the 8-GPU platforms (B300 1×8, GB300 2×4), `Default` is TP8 and `Long-Context` is `--pp-size 8 --tp-size 1`. On the 16-GPU platforms (B200 2×8, GB200 4×4), both are `--pp-size 16 --tp-size 1` and differ only in `--mem-fraction-static` (0.85 vs 0.90) — deep PP is the throughput shape there, not just the long-context one (see [Deep PP](#deep-pp-for-prefill)). @@ -56,7 +71,7 @@ Pick your hardware, then the deployment shape and operating point. Node count fo **Spec Decode** — layers onto the strategy without changing it, on every platform except B200. DSPARK proposes 7 draft tokens per step (tune in the Playground) and requires `pp_size == 1`, so on B200 it also drops the pipeline and re-lays the same 16 GPUs flat: PP2 × TP8 → TP16, PP2 × DCPEP8 → DCPEP16. DFLASH has no published draft checkpoint. The win is largest on short interactive traffic and fades as the prompt grows. -`--mamba-full-memory-ratio` is the one sizing flag, computed live: set your average request length in the [Mamba ratio calculator](#mamba-ratio-calculator); everything else follows the panels, and the result is pinned into the command. +`--mamba-full-memory-ratio` is the one sizing flag, computed live: set your average request length in the [Mamba ratio calculator](#mamba-ratio-calculator); everything else follows the panels, and the result is pinned into the command. (The Atlas 800I A3 uses `--max-mamba-cache-size` instead.) import { Deployment } from "/src/snippets/_deployment.jsx"; @@ -120,7 +135,7 @@ throughput and accuracy before you rely on any of them. ## 2. Configuration Tips -**Memory: two pools, one flag.** K3 splits static memory into a worst-case-reserved **KDA state pool** (it sets the concurrency ceiling) and a paged **MLA KV pool**, divided by `--mamba-full-memory-ratio`. The command panel pins that flag to the [calculator](#mamba-ratio-calculator)'s output — set your average request length there; every other calculator input follows the panels. After boot, read back `max_total_num_tokens` (the KV side) and the admitted-request cap (the state side). +**Memory: two pools, one flag.** K3 splits static memory into a worst-case-reserved **KDA state pool** (it sets the concurrency ceiling) and a paged **MLA KV pool**, divided by `--mamba-full-memory-ratio`. The command panel pins that flag to the [calculator](#mamba-ratio-calculator)'s output — set your average request length there; every other calculator input follows the panels. (On the Atlas 800I A3: `--max-mamba-cache-size`, no calculator.) After boot, read back `max_total_num_tokens` (the KV side) and the admitted-request cap (the state side). Capacity levers, all in the Playground. Each trades precision or cache behavior for capacity — re-verify accuracy on your workload: @@ -153,6 +168,7 @@ Speculation: DSPARK holds block size + 1 (= 8) intermediate states per request | H200 2×8 (4×8 on Unified High-Throughput) | TP16/EP16 + symm-mem, Marlin + FlashMLA; High-Throughput widens to TP32/EP32 over 4 nodes at mem-frac 0.90 with `extra_buffer_lazy` | same block on every node; export the cross-node NIC (`GLOO_SOCKET_IFNAME` / `NCCL_SOCKET_IFNAME`, `SGLANG_HOST_IP`); keep `NCCL_MNNVL_ENABLE=1 NCCL_CUMEM_ENABLE=1` | | H100 4×8 | TP32/EP32, Marlin + FlashMLA | SM90a build of the K3 image; pin NCCL/Gloo to the same NIC on all nodes; least post-weight headroom (80 GB) | | MI350X/MI355X 1×8 | TP8 ROCm/AITER | AITER A8W4 FlyDSL MoE, Triton attention (`SGLANG_MLA_DECODE_TUNE=1` for gfx950 MLA decode geometry), graph bs up to 256, fp8 kvcache; DSPARK supported | +| Atlas 800I A3 4×8 (32 cards / 64 dies) | TP64/DP4 + DeepEP | PD-mixed `Unified` only; DSPARK baked in; pin `GLOO`/`HCCL_SOCKET_IFNAME` on every node | **DCP notes** — the DCP cells are Balanced and High-Throughput on every Blackwell platform, in both the `Unified` and `Decode` roles: @@ -199,7 +215,7 @@ Pending update... ### 3.2 Tool Calling -Enable the `kimi_k3` tool-call parser (toggle **Tool Call Parser** in the **Parsers** card of the [Playground above](#playground)) to surface structured tool calls via `message.tool_calls`. Because K3 is a thinking model, the follow-up turn may put text in `reasoning_content` as well as `content` — print both. +Enable the `kimi_k3` tool-call parser (toggle **Tool Call Parser** in the **Parsers** card of the [Playground above](#playground)) to surface structured tool calls via `message.tool_calls`. Because K3 is a thinking model, the follow-up turn may put text in `reasoning_content` as well as `content` — print both. (Not yet supported on the Atlas 800I A3.) diff --git a/docs/src/snippets/_deployment.jsx b/docs/src/snippets/_deployment.jsx index 26d74ceac..23a82ebc1 100644 --- a/docs/src/snippets/_deployment.jsx +++ b/docs/src/snippets/_deployment.jsx @@ -9,7 +9,7 @@ // hardware optional — per-model GPUs the shared HARDWARE_CATALOG lacks: // {id, label, vram, vendor}[] merged into the catalog at render // (so a model-specific GPU never needs an engine-catalog edit); -// vendor picks the selector group: blackwell | hopper | amd. +// vendor picks the selector group: blackwell | hopper | amd | npu. // `multiNodeDockerFlags: string[]` (either source) adds // `docker run` flags the platform's fabric needs // groupHardware optional — set false to show one flat hardware row @@ -133,6 +133,11 @@ export const Deployment = ({ config, benchmarks }) => { { id: "mi350x", label: "MI350X", vram: "288GB" }, { id: "mi355x", label: "MI355X", vram: "288GB" }, ], + // Atlas 800I A3 (910C): 1 card = 2 dies, so --tp-size is 2× the card + // count (32 cards -> --tp-size 64). + npu: [ + { id: "a3", label: "Atlas 800I A3", vram: "64GB/die" }, + ], }; // ==== 2. Style helper (dark-mode-aware) ==== @@ -737,7 +742,7 @@ export const Deployment = ({ config, benchmarks }) => { // Insert the multi-node trio after the last parallelism flag, // falling back to right after --model-path. const PARALLELISM_ANCHORS = new Set([ - "--enable-dp-attention", "--dp", "--tp-size", "--tp", + "--enable-dp-attention", "--dp-size", "--dp", "--tp-size", "--tp", "--sp-degree", "--ulysses-degree", "--ring-degree", ]); let i = flags.reduce( @@ -800,6 +805,25 @@ export const Deployment = ({ config, benchmarks }) => { " --cap-add=SYS_PTRACE --security-opt seccomp=unconfined", " --shm-size 32g", ] + : vendorOf(sel.hw) === "npu" + ? [ + // NPU: --privileged grants the davinci devices (16 dies on an + // 8-card Atlas 800I A3 node); the host CANN driver/firmware/state + // must be mounted in. + "docker run --privileged --shm-size=16g", + " --device=/dev/davinci0 --device=/dev/davinci1 --device=/dev/davinci2 --device=/dev/davinci3", + " --device=/dev/davinci4 --device=/dev/davinci5 --device=/dev/davinci6 --device=/dev/davinci7", + " --device=/dev/davinci8 --device=/dev/davinci9 --device=/dev/davinci10 --device=/dev/davinci11", + " --device=/dev/davinci12 --device=/dev/davinci13 --device=/dev/davinci14 --device=/dev/davinci15", + " --device=/dev/davinci_manager", + " --device=/dev/hisi_hdc", + " -v /usr/local/sbin:/usr/local/sbin", + " -v /usr/local/Ascend/driver:/usr/local/Ascend/driver", + " -v /usr/local/Ascend/firmware:/usr/local/Ascend/firmware", + " -v /etc/ascend_install.info:/etc/ascend_install.info", + " -v /var/queue_schedule:/var/queue_schedule", + " -v ~/.cache/:/root/.cache/", + ] : [ "docker run --gpus all", " --shm-size 32g", @@ -811,7 +835,9 @@ export const Deployment = ({ config, benchmarks }) => { // just maps the serve port. hostNetwork ? " --network host" : ` -p ${servePort}:${servePort}`, ...(multinode ? fabricFlagsOf(sel.hw).map((f) => " " + f) : []), - " -v ~/.cache/huggingface:/root/.cache/huggingface", + // The NPU device block already mounts ~/.cache/. + ...(vendorOf(sel.hw) === "npu" + ? [] : [" -v ~/.cache/huggingface:/root/.cache/huggingface"]), ...(config.dockerMounts || []).map((mount) => ` -v ${mount}`), // HF token only for gated checkpoints — configs that declare an HF_TOKEN placeholder. ...(config.placeholders && config.placeholders.HF_TOKEN @@ -1348,10 +1374,12 @@ export const Deployment = ({ config, benchmarks }) => { const builderMeta = (cell && cell.builder) || {}; const verifyStatus = cellVerifyStatus(cell, sel); // Pin the calculator-computed ratio into the rendered command (before the - // host/port tail); cells themselves stay ratio-free. + // host/port tail); cells themselves stay ratio-free. Cells sizing the pool + // with --max-mamba-cache-size opt out. const cellWithRatio = (() => { if (!cell || !mambaRatio) return cell; - if (cell.flags.some((f) => f.startsWith("--mamba-full-memory-ratio"))) return cell; + if (cell.flags.some((f) => + f.startsWith("--mamba-full-memory-ratio") || f.startsWith("--max-mamba-cache-size"))) return cell; const flags = [...cell.flags]; const line = `--mamba-full-memory-ratio ${mambaRatio}`; const i = flags.findIndex((f) => f.startsWith("--host")); @@ -1638,7 +1666,9 @@ export const Deployment = ({ config, benchmarks }) => { }} title={ disabled - ? item.disableReason || "Not supported for current selection" + ? (typeof item.disableReason === "function" + ? item.disableReason(sel) + : item.disableReason) || "Not supported for current selection" : "" } onClick={(e) => { diff --git a/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx b/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx index 41088c1bb..3a996b2ae 100644 --- a/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx +++ b/docs/src/snippets/_kimi_k3_mamba_ratio_calculator.jsx @@ -135,6 +135,10 @@ export const KimiK3MambaRatioCalculator = () => { const eff = derive(cfg.flags, cfg.env); const bs = derive(cfg.baseFlags.length ? cfg.baseFlags : cfg.flags, cfg.baseFlags.length ? cfg.baseEnv : cfg.env); + // A --max-mamba-cache-size cell sizes the pool explicitly — no ratio to + // compute or broadcast. + const explicitSizing = (cfg.baseFlags.length ? cfg.baseFlags : cfg.flags) + .some((f) => f.startsWith("--max-mamba-cache-size")); const { ratio, tp, dp, attnTp, dcp, kvDtype, ssmDtype, radixOff, strategy, skipLock, slots, specOn, replaySpec, block, pdRole } = eff; const valid = Number.isFinite(ratio) && ratio > 0 && length > 0 && 96 % attnTp === 0; const baseValid = Number.isFinite(bs.ratio) && bs.ratio > 0 && length > 0; @@ -153,6 +157,7 @@ export const KimiK3MambaRatioCalculator = () => { // Broadcast both results: the Deploy command takes the base-config value, // the Playground's composed command takes the effective one. useEffect(() => { + if (explicitSizing) return; window.dispatchEvent( new CustomEvent("sglang-k3-mamba-ratio", { detail: { @@ -161,7 +166,7 @@ export const KimiK3MambaRatioCalculator = () => { }, }) ); - }, [result, valid, baseResult, baseValid]); + }, [result, valid, baseResult, baseValid, explicitSizing]); const copyFlag = () => { if (!cliFlag || typeof navigator === "undefined" || !navigator.clipboard) return; @@ -230,6 +235,14 @@ export const KimiK3MambaRatioCalculator = () => { specLabel, ].filter(Boolean); + if (explicitSizing) { + return ( +
+ This recipe sizes the KDA state pool explicitly with --max-mamba-cache-size, so the ratio calculator does not apply. +
+ ); + } + return (
{ return null; }; - // hw|variant|quant → variant|quant → hw|quant → quant → "". + // hw|variant|quant → variant|quant → hw|quant → quant → hw → default. const resolveModelName = (sel) => { const keys = [ `${sel.hw}|${sel.variant}|${sel.quant}`, `${sel.variant}|${sel.quant}`, `${sel.hw}|${sel.quant}`, sel.quant, + sel.hw, + "default", ]; for (const k of keys) { const hit = config.modelNames[k]; @@ -233,7 +235,9 @@ export const Playground = ({ config }) => { } const hidden = entry.hide ? matchConstraint(base, entry.hide) : false; let disabled = entry.disabled === true || entry.disable === true; - let disableReason = entry.disableReason || ""; + let disableReason = typeof entry.disableReason === "function" + ? entry.disableReason(base) + : entry.disableReason || ""; if (!disabled && entry.disable && typeof entry.disable === "object") { if (Array.isArray(entry.disable)) { for (const item of entry.disable) { @@ -336,11 +340,13 @@ export const Playground = ({ config }) => { return null; }; - // --tp / --ep spelling families: configs write either the canonical - // --tp-size / --ep-size or the short --tp / --ep. Parse and strip every - // spelling; when re-emitting, keep the spelling the base already uses. + // --tp / --ep / --dp spelling families: configs write either the canonical + // --tp-size / --ep-size / --dp-size, a short alias, or the long form. Parse + // and strip every spelling; when re-emitting, keep the spelling the base + // already uses. const TP_HEADS = ["--tp-size", "--tp", "--tensor-parallel-size"]; const EP_HEADS = ["--ep-size", "--ep", "--expert-parallel-size"]; + const DP_HEADS = ["--dp-size", "--dp", "--data-parallel-size"]; const parseIntFlagAny = (flags, heads) => { for (const head of heads) { const n = parseIntFlag(flags, head); @@ -356,17 +362,17 @@ export const Playground = ({ config }) => { // insertion still works in partial cells). const ANCHOR_NEAR_MODEL_PATH = ["--model-path"]; const ANCHOR_NEAR_TP = ["--tp-size", "--tp", "--model-path"]; - const ANCHOR_NEAR_DP = ["--dp", "--tp-size", "--tp", "--model-path"]; - const ANCHOR_NEAR_DPATTN = ["--enable-dp-attention", "--dp", "--tp-size", "--tp", "--model-path"]; + const ANCHOR_NEAR_DP = ["--dp-size", "--dp", "--tp-size", "--tp", "--model-path"]; + const ANCHOR_NEAR_DPATTN = ["--enable-dp-attention", "--dp-size", "--dp", "--tp-size", "--tp", "--model-path"]; const ANCHOR_NEAR_MOE = ["--moe-a2a-backend", "--moe-runner-backend", - "--enable-dp-attention", "--dp", "--tp-size", "--tp", "--model-path"]; + "--enable-dp-attention", "--dp-size", "--dp", "--tp-size", "--tp", "--model-path"]; // Helper bundle passed to every axis handler. const helpers = { matchConstraint, evaluateChip, findEntry, isHidden, stripFlagsByFirstToken, stripEnvByPrefix, insertBeforeTail, insertAfter, parseIntFlag, hasFlag, findFlagArg, - TP_HEADS, EP_HEADS, parseIntFlagAny, flagSpelling, + TP_HEADS, EP_HEADS, DP_HEADS, parseIntFlagAny, flagSpelling, ANCHOR_NEAR_MODEL_PATH, ANCHOR_NEAR_TP, ANCHOR_NEAR_DP, ANCHOR_NEAR_DPATTN, ANCHOR_NEAR_MOE, }; @@ -431,7 +437,7 @@ export const Playground = ({ config }) => { // baked strategy (legacy mode flags mapped to zigzag/interleave). deriveFromBase: (cell, fc, h) => { const flags = (cell && cell.flags) || []; - const dpVal = h.parseIntFlag(flags, "--dp"); + const dpVal = h.parseIntFlagAny(flags, h.DP_HEADS); const hasDpAttn = h.hasFlag(flags, "--enable-dp-attention"); let dpAttn; if (dpVal !== null) dpAttn = dpVal; @@ -468,7 +474,7 @@ export const Playground = ({ config }) => { const dpIntent = (value.dpAttn !== null && value.dpAttn !== undefined) ? value.dpAttn : (h.hasFlag(flags, "--enable-dp-attention") - ? (h.parseIntFlag(flags, "--dp") ?? 1) : false); + ? (h.parseIntFlagAny(flags, h.DP_HEADS) ?? 1) : false); if (typeof dpIntent === "number" && dpIntent > 1) return null; return h.parseIntFlagAny(flags, h.TP_HEADS); }; @@ -525,10 +531,13 @@ export const Playground = ({ config }) => { } if (value.dpAttn !== null && value.dpAttn !== undefined && !blocked("dpAttn", value.dpAttn)) { - flags = h.stripFlagsByFirstToken(flags, ["--dp", "--enable-dp-attention"]); + // Capture the spelling before stripping — the TP/EP handlers do the + // same, and a lookup on the stripped array always hits the fallback. + const dpHead = h.flagSpelling(flags, h.DP_HEADS, "--dp-size"); + flags = h.stripFlagsByFirstToken(flags, [...h.DP_HEADS, "--enable-dp-attention"]); if (typeof value.dpAttn === "number" && value.dpAttn > 0) { flags = h.insertAfter(flags, h.ANCHOR_NEAR_TP, [ - `--dp ${value.dpAttn}`, + `${dpHead} ${value.dpAttn}`, "--enable-dp-attention", ]); } @@ -707,6 +716,10 @@ export const Playground = ({ config }) => { && (!mmOpt.requiresHw || mmOpt.requiresHw.includes(base.hw)) && (!mmOpt.excludesStrategy || !mmOpt.excludesStrategy.includes(base.strategy)); const backendIsMega = slotDisplay("backend") === "megamoe"; + // `ep.showWhen` (function of base) drops the whole EP select on bases + // where EP is not a supported lever (e.g. the single-shape A3 recipe). + const epShown = !!fc.ep + && !(typeof fc.ep.showWhen === "function" && !fc.ep.showWhen(base)); return (
@@ -728,7 +741,7 @@ export const Playground = ({ config }) => { (v) => setSlot("mmQuant", v), base)} )} - {fc.ep && ( + {epShown && ( {fc.ep.label || "EP"} {renderSelect(slotDisplay("ep"), fc.ep.values || [null], @@ -1475,7 +1488,7 @@ export const Playground = ({ config }) => { if (multinode && !f.some((x) => x.startsWith("--nnodes"))) { // Insert the multi-node trio after the last parallelism flag (matches // _deployment.jsx so untouched-base output is byte-identical). - const PARALLELISM_ANCHORS = ["--enable-dp-attention", "--dp", "--tp-size", "--tp"]; + const PARALLELISM_ANCHORS = ["--enable-dp-attention", "--dp-size", "--dp", "--tp-size", "--tp"]; let at = -1; for (const anchor of PARALLELISM_ANCHORS) { at = f.findIndex((x) => x.split(/[\s=]/)[0] === anchor); @@ -2169,10 +2182,12 @@ export const Playground = ({ config }) => { let pgFlagsLatest = []; let pgEnvLatest = []; // Render-only ratio injection (before the host/port tail); skipped if the - // flags somehow already carry the family. + // flags already carry the family, or the base cell sizes the pool explicitly + // with --max-mamba-cache-size (the ratio would contradict its slot count). const withRatio = (fl, value) => { if (!value) return fl; - if (fl.some((f) => f.startsWith("--mamba-full-memory-ratio"))) return fl; + if (fl.some((f) => + f.startsWith("--mamba-full-memory-ratio") || f.startsWith("--max-mamba-cache-size"))) return fl; const out = [...fl]; const line = `--mamba-full-memory-ratio ${value}`; const i = out.findIndex((f) => f.startsWith("--host")); diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx index b38f44271..ce241bff8 100644 --- a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx +++ b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx @@ -15,7 +15,7 @@ export const config = { // re-lays the same 16 as flat TP16 / DCPEP16), GB200 (4×4 TP16 MNNVL), // H200 (2×8 TP16/EP16, or 4×8 TP32/EP32 for High-Throughput), H100 // (4×8 TP32/EP32), and MI350X/MI355X (1×8 TP8) have serving recipes. - supportedHardware: ["b300", "gb300", "b200", "gb200", "h200", "h100", "mi350x", "mi355x"], + supportedHardware: ["b300", "gb300", "b200", "gb200", "h200", "h100", "mi350x", "mi355x", "a3"], // ---- Cell introspection (config-internal; the engines ignore these keys) ---- // @@ -138,8 +138,18 @@ export const config = { title: "PD Mode", options: [ { id: "unified", label: "Unified" }, - { id: "prefill", label: "Prefill" }, - { id: "decode", label: "Decode" }, + { + id: "prefill", + label: "Prefill", + disabled: (s) => s.hw === "a3", + disableReason: (s) => (s.hw === "a3" ? "Only Unified PD is supported on this recipe." : ""), + }, + { + id: "decode", + label: "Decode", + disabled: (s) => s.hw === "a3", + disableReason: (s) => (s.hw === "a3" ? "Only Unified PD is supported on this recipe." : ""), + }, ], }, { @@ -148,9 +158,21 @@ export const config = { id: "strategy", title: "Strategy", options: [ - { id: "low-latency", label: "Low-Latency", showWhen: (s) => s.pdMode !== "prefill" }, + { + id: "low-latency", + label: "Low-Latency", + showWhen: (s) => s.pdMode !== "prefill", + disabled: (s) => s.hw === "a3", + disableReason: (s) => (s.hw === "a3" ? "Only the Balanced operating point is supported on this recipe." : ""), + }, { id: "balanced", label: "Balanced", showWhen: (s) => s.pdMode !== "prefill" }, - { id: "high-throughput", label: "High-Throughput", showWhen: (s) => s.pdMode !== "prefill" }, + { + id: "high-throughput", + label: "High-Throughput", + showWhen: (s) => s.pdMode !== "prefill", + disabled: (s) => s.hw === "a3", + disableReason: (s) => (s.hw === "a3" ? "Only the Balanced operating point is supported on this recipe." : ""), + }, { id: "default", label: "Default", showWhen: (s) => s.pdMode === "prefill" }, { id: "long-context", label: "Long-Context", showWhen: (s) => s.pdMode === "prefill" }, ], @@ -173,7 +195,17 @@ export const config = { title: "Quantization", default: "mxfp4", options: [ - { id: "mxfp4", label: "MXFP4", subtitle: "Moonshot AI checkpoint" }, + { id: "mxfp4", label: "MXFP4", subtitle: "Moonshot AI checkpoint", + disabled: (s) => s.hw === "a3", + disableReason: (s) => (s.hw === "a3" ? "Only Modelslim (W4A8) is supported on this recipe." : ""), + }, + { + // A3 only (NPU W4A8 checkpoint); hidden on the GPU recipes. + id: "modelslim", + label: "Modelslim (W4A8)", + subtitle: "ModelScope NPU checkpoint", + showWhen: (s) => s.hw === "a3", + }, { id: "nvfp4", label: "NVFP4", @@ -197,7 +229,7 @@ export const config = { id: "mmTransport", title: "VLM Transport", default: "auto", - showWhen: (s) => s.pdMode !== "decode", + showWhen: (s) => s.pdMode !== "decode" && s.hw !== "a3", options: [ { id: "auto", @@ -236,6 +268,8 @@ export const config = { options: [ { id: "none", label: "Non-Spec", env: (s) => (["mi350x", "mi355x"].includes(s.hw) ? ["SGLANG_MLA_DECODE_TUNE=1"] : []), + disabled: (s) => s.hw === "a3", + disableReason: (s) => (s.hw === "a3" ? "Only DSPARK is supported on this recipe." : ""), }, { id: "dspark", @@ -259,17 +293,29 @@ export const config = { config.specCollapses(s) ? ["--tp-size", "--pp-size", "--dcp-size", "--ep-size"] : [], - // Every DSPARK recipe layers ReplaySSM on: it moves the per-draft - // intermediate SSM states onto a fixed ring, lifting the concurrency - // the state pool admits (needs the Triton decode kernel, the K3 - // default). Only the PD prefill role opts out — it never runs verify - // and rejects the flag at startup. flags: (s) => [ ...(config.specCollapses(s) ? config.specCollapsedFlags(s) : []), "--speculative-algorithm DSPARK", "--speculative-draft-model-path RadixArk/Kimi-K3-DSpark", "--speculative-dspark-block-size 7", - ...(s.pdMode === "prefill" ? [] : ["--enable-linear-replayssm-spec"]), + // The NPU recipe adds the NPU draft path's own knobs (draft + // attention backend, topk 1, unquantized draft weights) on top of + // the common trio. + ...(s.hw === "a3" + ? [ + "--speculative-draft-attention-backend ascend", + "--speculative-eagle-topk 1", + "--speculative-draft-model-quantization unquant", + ] + : []), + // ReplaySSM moves the per-draft intermediate SSM states onto a + // fixed ring, lifting the concurrency the state pool admits (needs + // the Triton decode kernel, the K3 default). It is CUDA-only, and + // the PD prefill role opts out — it never runs verify and rejects + // the flag at startup. + ...(s.hw !== "a3" && s.pdMode !== "prefill" + ? ["--enable-linear-replayssm-spec"] + : []), ], }, { @@ -302,6 +348,10 @@ export const config = { { id: "l2", label: "L1+L2 (host)", + disabled: (s) => s.hw === "a3", + disableReason: (s) => (s.hw === "a3" + ? "NPU HiCache does not support mamba cache (K3's KDA state is one)." + : ""), flags: [ "--enable-hierarchical-cache", ], @@ -331,6 +381,10 @@ export const config = { { id: "l3", label: "+ L3 (Mooncake)", + disabled: (s) => s.hw === "a3", + disableReason: (s) => (s.hw === "a3" + ? "NPU HiCache does not support mamba cache (K3's KDA state is one)." + : ""), flags: [ "--enable-hierarchical-cache", "--hicache-storage-backend mooncake", @@ -369,6 +423,7 @@ export const config = { modelNames: { default: "moonshotai/Kimi-K3", nvfp4: "nvidia/Kimi-K3-NVFP4", + a3: "sgl-npu/Kimi-K3-W4A8", }, placeholders: { @@ -421,6 +476,7 @@ export const config = { "gb300|nvfp4": "lmsysorg/sglang:dev-dev-kimi-k3-nvfp4", "b200|nvfp4": "lmsysorg/sglang:dev-dev-kimi-k3-nvfp4", "gb200|nvfp4": "lmsysorg/sglang:dev-dev-kimi-k3-nvfp4", + a3: "quay.io/ascend/sglang:main-cann9.0.0-a3", }, // Pre-selects the issue template's `model` field on "Submit verified cell". github: { @@ -437,10 +493,23 @@ export const config = { attention: { knobs: [ { id: "tp", label: "TP", values: [ - null, 8, + null, + { + value: 8, + get disable() { return [ + { + when: { hw: ["a3"] }, + reason: "Only TP64 is supported on this recipe.", + }, + ]; }, + }, { value: 16, get disable() { return [ + { + when: { hw: ["a3"] }, + reason: "Only TP64 is supported on this recipe.", + }, { when: { hw: ["b300", "gb300"] }, reason: "TP=16 needs 16 ranks; the B300 and GB300 recipes have 8 ranks.", @@ -452,13 +521,41 @@ export const config = { ...config.pipelinedKnobDisableRules, ]; }, }, + { + // A3 only: 64 ranks (4 nodes × 8 cards × 2 dies); hidden on the GPU recipes. + value: 64, + hide: { hw: ["b300", "gb300", "b200", "gb200", "h200", "h100", "mi350x", "mi355x"] }, + }, ]}, { id: "dpAttn", label: "DP-Attention", values: [ - null, false, 2, 4, + null, + { + value: false, + get disable() { return [ + { + when: { hw: ["a3"] }, + reason: "Only DP-Attention=4 is supported on this recipe.", + }, + ]; }, + }, + { + value: 2, + get disable() { return [ + { + when: { hw: ["a3"] }, + reason: "Only DP-Attention=4 is supported on this recipe.", + }, + ]; }, + }, + 4, { value: 8, get disable() { return [ + { + when: { hw: ["a3"] }, + reason: "Only DP-Attention=4 is supported on this recipe.", + }, { when: { hw: ["b300", "gb300"] }, reason: "On an 8-rank deployment (B300 1×8, GB300 2×4) dp=8 leaves attn_tp=1, so each rank holds the full unsharded MLA KV and OOMs — prefer dp=2/attn_tp=4.", @@ -473,6 +570,10 @@ export const config = { { value: 16, get disable() { return [ + { + when: { hw: ["a3"] }, + reason: "Only DP-Attention=4 is supported on this recipe.", + }, { when: { hw: ["b300", "gb300"] }, reason: "DP-Attention=16 needs 16 TP ranks; the B300 and GB300 recipes have 8.", @@ -502,8 +603,12 @@ export const config = { requiresHw: ["b200", "b300", "gb200", "gb300"] }, // Blackwell-only: runs FlashInfer's official trtllm-gen SiTU kernels. { id: "flashinfer_mxfp4", label: "FlashInfer (MXFP4)", flags: ["--moe-runner-backend flashinfer_mxfp4"], - requiresHw: ["b200", "b300", "gb200", "gb300"] }, - { id: "marlin", label: "Marlin (W4A16)", flags: ["--moe-runner-backend marlin"] }, + requiresHw: ["b200", "b300", "gb200", "gb300"], + disable: [{ when: { hw: ["a3"] }, + reason: "FlashInfer is a CUDA kernel and is not supported on NPU." }] }, + { id: "marlin", label: "Marlin (W4A16)", flags: ["--moe-runner-backend marlin"], + disable: [{ when: { hw: ["a3"] }, + reason: "Marlin is a CUDA kernel and is not supported on NPU." }] }, ], }, // MegaMoE quantization sub-select — shown only when backend === "megamoe". @@ -517,30 +622,36 @@ export const config = { env: ["SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320"] }, ], }, - ep: { label: "EP", values: [ - null, 1, 2, 4, 8, - { - value: 16, - get disable() { return [ - { - when: { hw: ["b300", "gb300"] }, - reason: "EP=16 needs 16 TP ranks; the B300 and GB300 recipes have 8.", - }, - { - when: { hw: ["b200"], pdMode: ["unified"], spec: ["none"] }, - reason: "With Spec Decode off, the B200 Unified recipes use TP8 within each PP2 stage, so EP cannot exceed 8. Switch Spec Decode to DSPARK for the flat TP16 shape.", - }, - ...config.pipelinedKnobDisableRules, - ]; }, - }, - ]}, + ep: { + showWhen: (b) => b.hw !== "a3", + label: "EP", + values: [ + null, 1, 2, 4, 8, + { + value: 16, + get disable() { return [ + { + when: { hw: ["b300", "gb300"] }, + reason: "EP=16 needs 16 TP ranks; the B300 and GB300 recipes have 8.", + }, + { + when: { hw: ["b200"], pdMode: ["unified"], spec: ["none"] }, + reason: "With Spec Decode off, the B200 Unified recipes use TP8 within each PP2 stage, so EP cannot exceed 8. Switch Spec Decode to DSPARK for the flat TP16 shape.", + }, + ...config.pipelinedKnobDisableRules, + ]; }, + }, + ], + }, }, // ----- Card: "Parsers" ----- parsers: { items: [ { id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser kimi_k3" }, - { id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser kimi_k3" }, + // Tool calling is not yet supported on the NPU, so the item hides on a3. + { id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser kimi_k3", + hide: { hw: ["a3"] } }, ], }, @@ -617,7 +728,8 @@ export const config = { // EAGLE --speculative-num-steps N (chain; topk>1 is a tree) // Only DSPARK is selectable today, so only its form is emitted. id: "proposedDraftTokens", title: "Proposed Draft Tokens", - showWhen: (b) => b.spec === "dspark", + // The A3 recipe pins the shipped block size (7). + showWhen: (b) => b.spec === "dspark" && b.hw !== "a3", control: "slider", stripPrefixes: [ "--speculative-dspark-block-size", @@ -640,9 +752,10 @@ export const config = { // Spec-only, so gate the row on DSPARK; every DSPARK recipe (except the PD // prefill role) turns it on in the base, so this row derives to On and // exists mainly as the opt-out. - // Needs the Triton linear-attn decode backend (the K3 default). + // Needs the Triton linear-attn decode backend (the K3 default); the A3 + // script never sets it. id: "replaySsm", title: "ReplaySSM (spec)", - showWhen: (b) => b.spec === "dspark", + showWhen: (b) => b.spec === "dspark" && b.hw !== "a3", stripPrefixes: ["--enable-linear-replayssm-spec"], options: [ { id: "off", label: "Off" }, @@ -662,7 +775,8 @@ export const config = { // without --speculative-dspark-sps-table-path (every step still // verifies full width); fails fast with ReplaySSM or DCP > 1. id: "raggedVerify", title: "Ragged Verify Mode (spec)", - showWhen: (b) => b.spec === "dspark", + // The A3 recipe pins static. + showWhen: (b) => b.spec === "dspark" && b.hw !== "a3", stripEnv: ["SGLANG_RAGGED_VERIFY_MODE"], options: [ { id: "static", label: "Auto (static)" }, @@ -671,6 +785,7 @@ export const config = { }, { id: "kvCacheDtype", title: "KV Cache Precision", + showWhen: (b) => b.hw !== "a3", stripPrefixes: ["--kv-cache-dtype"], options: [ { id: "auto", label: "Auto (BF16)" }, @@ -679,6 +794,7 @@ export const config = { }, { id: "mambaSsmDtype", title: "KDA State Precision", + showWhen: (b) => b.hw !== "a3", stripPrefixes: ["--mamba-ssm-dtype"], options: [ { id: "auto", label: "Auto (FP32)" }, @@ -693,6 +809,7 @@ export const config = { // Off suits prefix-free traffic (offline batch, evals): 1 state slot // per request instead of 4-5. id: "prefixCache", title: "Prefix Cache", + showWhen: (b) => b.hw !== "a3", stripPrefixes: ["--disable-radix-cache"], options: [ { id: "on", label: "On" }, @@ -704,7 +821,8 @@ export const config = { // prefix cache off, so the row hides (and stops emitting) there. // Slot cost per request: extra_buffer 5, extra_buffer_lazy 4. id: "mambaRadix", title: "KDA Radix Cache Strategy", - showWhen: (b, v, d) => (((v && v.prefixCache) ?? (d && d.prefixCache)) !== "off"), + showWhen: (b, v, d) => (b.hw !== "a3") + && ((((v && v.prefixCache) ?? (d && d.prefixCache)) !== "off")), stripPrefixes: ["--mamba-radix-cache-strategy"], options: [ { id: "auto", label: "Auto (extra_buffer)" }, @@ -718,6 +836,7 @@ export const config = { // (extra_buffer 5→4, extra_buffer_lazy 4→3; no_buffer stays 3). Off by // default. Env var, not a flag, so it emits via env/stripEnv. id: "mambaSlotSaving", title: "KDA Slot Saving (experimental)", + showWhen: (b) => b.hw !== "a3", stripEnv: ["SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK"], options: [ { id: "off", label: "Off" }, @@ -727,6 +846,7 @@ export const config = { { // Only meaningful with EP a2a on (MoE card or a large-scale preset). id: "eplb", title: "Expert Rebalancing (EPLB)", + showWhen: (b) => b.hw !== "a3", stripPrefixes: ["--enable-eplb"], options: [ { id: "off", label: "Off" }, @@ -738,6 +858,7 @@ export const config = { // Validated on the no-a2a MXFP4 runner; untested against SBO (EP a2a) // and DP attention. id: "prefillGraph", title: "Prefill CUDA Graph", + showWhen: (b) => b.hw !== "a3", stripPrefixes: ["--cuda-graph-backend-prefill"], options: [ { id: "auto", label: "Auto (off)" }, @@ -760,7 +881,7 @@ export const config = { // resolves it into the full parallelism shape (tp/ep/dp/dcp; attn-tp = // tp/dp); pool sizing rides the calculator-driven ratio. id: "lsGpus", title: "Cluster Size (large-scale)", - showWhen: (b) => b.pdMode === undefined || b.pdMode === "unified", + showWhen: (b) => (b.hw !== "a3") && (b.pdMode === undefined || b.pdMode === "unified"), // Default follows the base cell's own GPU count (tp8 lanes -> 8, // tp16 lanes -> 16), so a preset starts from "same hardware, new shape". default: (b) => @@ -776,7 +897,7 @@ export const config = { }, { id: "lsPreset", title: "Large-Scale Preset", - showWhen: (b) => b.pdMode === undefined || b.pdMode === "unified", + showWhen: (b) => (b.hw !== "a3") && (b.pdMode === undefined || b.pdMode === "unified"), stripPrefixes: [ "--tp-size", "--tp", "--tensor-parallel-size", "--ep-size", "--ep", "--expert-parallel-size", @@ -2187,6 +2308,63 @@ export const config = { "--port {{PORT}}", ], }, + { + match: { hw: "a3", pdMode: "unified", strategy: "balanced" }, + nnodes: 4, + verified: false, + verificationStatus: "in-progress", + env: [ + "SGLANG_USE_MODELSCOPE=1", + "GLOO_SOCKET_IFNAME={{NETWORK_IFACE}}", + "HCCL_SOCKET_IFNAME={{NETWORK_IFACE}}", + "PYTORCH_NPU_ALLOC_CONF=expandable_segments:True", + "SGLANG_SET_CPU_AFFINITY=1", + "SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS=1", + "SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1", + "STREAMS_PER_DEVICE=32", + "DEEP_NORMAL_MODE_USE_INT8_QUANT=1", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128", + "HCCL_BUFFSIZE=200", + "HCCL_OP_EXPANSION_MODE=AIV", + "DEEPEP_NORMAL_LONG_SEQ_ROUND=64", + "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512", + "DEEPEP_HCCL_BUFFSIZE=1800", + "SGLANG_K3_SHARED_EXPERTS_ATTN_TP=1", + "SGLANG_K3_DENSE_MLP_ATTN_TP=1", + "SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE=1", + "SGLANG_ENABLE_SPEC_V2=1", + "SGLANG_RAGGED_VERIFY_MODE=static", + "SGLANG_DSPARK_FOLDED_PROPOSAL=0", + "SGLANG_DSPARK_FOLDED_SAMPLING=0", + "SGLANG_DSPARK_STACKED_CTX_KV=0", + "SGLANG_DSPARK_EMBED_IN_GRAPH=0", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tokenizer-path {{MODEL_NAME}}", + "--attention-backend ascend", + "--device npu", + "--quantization modelslim", + "--dtype bfloat16", + "--tp-size 64", + "--enable-dp-attention", + "--dp-size 4", + "--enable-dp-lm-head", + "--mem-fraction-static 0.78", + "--chunked-prefill-size 16384", + "--cuda-graph-bs 2 4 8 16", + "--max-running-requests 64", + "--max-mamba-cache-size 64", + "--moe-a2a-backend deepep", + "--deepep-mode auto", + "--reasoning-parser kimi_k3", + "--watchdog-timeout 9000", + "--model-loader-extra-config '{\"enable_multithread_load\": true}'", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, ], // Cross-node fabric env (substitute the NIC used by every rank). @@ -2221,5 +2399,13 @@ export const config = { " NCCL_SOCKET_IFNAME= # force NCCL off kube-ipvs0", " SGLANG_HOST_IP=", ], + a3: [ + "Run the same command on all four nodes with --node-rank 0/1/2/3.", + "NPU collectives use HCCL. Pin the cross-node NIC on EVERY node:", + " GLOO_SOCKET_IFNAME= # bootstrap interface", + " HCCL_SOCKET_IFNAME= # HCCL transport interface", + " SGLANG_HOST_IP= # this node's IP on that NIC", + "If running outside the official image, source set_env.sh on every node first.", + ], }, };