{
return null;
};
- // hw|variant|quant → variant|quant → hw|quant → quant → "".
+ // hw|variant|quant → variant|quant → hw|quant → quant → hw → default.
const resolveModelName = (sel) => {
const keys = [
`${sel.hw}|${sel.variant}|${sel.quant}`,
`${sel.variant}|${sel.quant}`,
`${sel.hw}|${sel.quant}`,
sel.quant,
+ sel.hw,
+ "default",
];
for (const k of keys) {
const hit = config.modelNames[k];
@@ -233,7 +235,9 @@ export const Playground = ({ config }) => {
}
const hidden = entry.hide ? matchConstraint(base, entry.hide) : false;
let disabled = entry.disabled === true || entry.disable === true;
- let disableReason = entry.disableReason || "";
+ let disableReason = typeof entry.disableReason === "function"
+ ? entry.disableReason(base)
+ : entry.disableReason || "";
if (!disabled && entry.disable && typeof entry.disable === "object") {
if (Array.isArray(entry.disable)) {
for (const item of entry.disable) {
@@ -336,11 +340,13 @@ export const Playground = ({ config }) => {
return null;
};
- // --tp / --ep spelling families: configs write either the canonical
- // --tp-size / --ep-size or the short --tp / --ep. Parse and strip every
- // spelling; when re-emitting, keep the spelling the base already uses.
+ // --tp / --ep / --dp spelling families: configs write either the canonical
+ // --tp-size / --ep-size / --dp-size, a short alias, or the long form. Parse
+ // and strip every spelling; when re-emitting, keep the spelling the base
+ // already uses.
const TP_HEADS = ["--tp-size", "--tp", "--tensor-parallel-size"];
const EP_HEADS = ["--ep-size", "--ep", "--expert-parallel-size"];
+ const DP_HEADS = ["--dp-size", "--dp", "--data-parallel-size"];
const parseIntFlagAny = (flags, heads) => {
for (const head of heads) {
const n = parseIntFlag(flags, head);
@@ -356,17 +362,17 @@ export const Playground = ({ config }) => {
// insertion still works in partial cells).
const ANCHOR_NEAR_MODEL_PATH = ["--model-path"];
const ANCHOR_NEAR_TP = ["--tp-size", "--tp", "--model-path"];
- const ANCHOR_NEAR_DP = ["--dp", "--tp-size", "--tp", "--model-path"];
- const ANCHOR_NEAR_DPATTN = ["--enable-dp-attention", "--dp", "--tp-size", "--tp", "--model-path"];
+ const ANCHOR_NEAR_DP = ["--dp-size", "--dp", "--tp-size", "--tp", "--model-path"];
+ const ANCHOR_NEAR_DPATTN = ["--enable-dp-attention", "--dp-size", "--dp", "--tp-size", "--tp", "--model-path"];
const ANCHOR_NEAR_MOE = ["--moe-a2a-backend", "--moe-runner-backend",
- "--enable-dp-attention", "--dp", "--tp-size", "--tp", "--model-path"];
+ "--enable-dp-attention", "--dp-size", "--dp", "--tp-size", "--tp", "--model-path"];
// Helper bundle passed to every axis handler.
const helpers = {
matchConstraint, evaluateChip, findEntry, isHidden,
stripFlagsByFirstToken, stripEnvByPrefix, insertBeforeTail, insertAfter,
parseIntFlag, hasFlag, findFlagArg,
- TP_HEADS, EP_HEADS, parseIntFlagAny, flagSpelling,
+ TP_HEADS, EP_HEADS, DP_HEADS, parseIntFlagAny, flagSpelling,
ANCHOR_NEAR_MODEL_PATH, ANCHOR_NEAR_TP, ANCHOR_NEAR_DP,
ANCHOR_NEAR_DPATTN, ANCHOR_NEAR_MOE,
};
@@ -431,7 +437,7 @@ export const Playground = ({ config }) => {
// baked strategy (legacy mode flags mapped to zigzag/interleave).
deriveFromBase: (cell, fc, h) => {
const flags = (cell && cell.flags) || [];
- const dpVal = h.parseIntFlag(flags, "--dp");
+ const dpVal = h.parseIntFlagAny(flags, h.DP_HEADS);
const hasDpAttn = h.hasFlag(flags, "--enable-dp-attention");
let dpAttn;
if (dpVal !== null) dpAttn = dpVal;
@@ -468,7 +474,7 @@ export const Playground = ({ config }) => {
const dpIntent = (value.dpAttn !== null && value.dpAttn !== undefined)
? value.dpAttn
: (h.hasFlag(flags, "--enable-dp-attention")
- ? (h.parseIntFlag(flags, "--dp") ?? 1) : false);
+ ? (h.parseIntFlagAny(flags, h.DP_HEADS) ?? 1) : false);
if (typeof dpIntent === "number" && dpIntent > 1) return null;
return h.parseIntFlagAny(flags, h.TP_HEADS);
};
@@ -525,10 +531,13 @@ export const Playground = ({ config }) => {
}
if (value.dpAttn !== null && value.dpAttn !== undefined
&& !blocked("dpAttn", value.dpAttn)) {
- flags = h.stripFlagsByFirstToken(flags, ["--dp", "--enable-dp-attention"]);
+ // Capture the spelling before stripping — the TP/EP handlers do the
+ // same, and a lookup on the stripped array always hits the fallback.
+ const dpHead = h.flagSpelling(flags, h.DP_HEADS, "--dp-size");
+ flags = h.stripFlagsByFirstToken(flags, [...h.DP_HEADS, "--enable-dp-attention"]);
if (typeof value.dpAttn === "number" && value.dpAttn > 0) {
flags = h.insertAfter(flags, h.ANCHOR_NEAR_TP, [
- `--dp ${value.dpAttn}`,
+ `${dpHead} ${value.dpAttn}`,
"--enable-dp-attention",
]);
}
@@ -707,6 +716,10 @@ export const Playground = ({ config }) => {
&& (!mmOpt.requiresHw || mmOpt.requiresHw.includes(base.hw))
&& (!mmOpt.excludesStrategy || !mmOpt.excludesStrategy.includes(base.strategy));
const backendIsMega = slotDisplay("backend") === "megamoe";
+ // `ep.showWhen` (function of base) drops the whole EP select on bases
+ // where EP is not a supported lever (e.g. the single-shape A3 recipe).
+ const epShown = !!fc.ep
+ && !(typeof fc.ep.showWhen === "function" && !fc.ep.showWhen(base));
return (
@@ -728,7 +741,7 @@ export const Playground = ({ config }) => {
(v) => setSlot("mmQuant", v), base)}
)}
- {fc.ep && (
+ {epShown && (
{fc.ep.label || "EP"}
{renderSelect(slotDisplay("ep"), fc.ep.values || [null],
@@ -1475,7 +1488,7 @@ export const Playground = ({ config }) => {
if (multinode && !f.some((x) => x.startsWith("--nnodes"))) {
// Insert the multi-node trio after the last parallelism flag (matches
// _deployment.jsx so untouched-base output is byte-identical).
- const PARALLELISM_ANCHORS = ["--enable-dp-attention", "--dp", "--tp-size", "--tp"];
+ const PARALLELISM_ANCHORS = ["--enable-dp-attention", "--dp-size", "--dp", "--tp-size", "--tp"];
let at = -1;
for (const anchor of PARALLELISM_ANCHORS) {
at = f.findIndex((x) => x.split(/[\s=]/)[0] === anchor);
@@ -2169,10 +2182,12 @@ export const Playground = ({ config }) => {
let pgFlagsLatest = [];
let pgEnvLatest = [];
// Render-only ratio injection (before the host/port tail); skipped if the
- // flags somehow already carry the family.
+ // flags already carry the family, or the base cell sizes the pool explicitly
+ // with --max-mamba-cache-size (the ratio would contradict its slot count).
const withRatio = (fl, value) => {
if (!value) return fl;
- if (fl.some((f) => f.startsWith("--mamba-full-memory-ratio"))) return fl;
+ if (fl.some((f) =>
+ f.startsWith("--mamba-full-memory-ratio") || f.startsWith("--max-mamba-cache-size"))) return fl;
const out = [...fl];
const line = `--mamba-full-memory-ratio ${value}`;
const i = out.findIndex((f) => f.startsWith("--host"));
diff --git a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx
index b38f44271..ce241bff8 100644
--- a/docs/src/snippets/configs/moonshotai/kimi-k3.jsx
+++ b/docs/src/snippets/configs/moonshotai/kimi-k3.jsx
@@ -15,7 +15,7 @@ export const config = {
// re-lays the same 16 as flat TP16 / DCPEP16), GB200 (4×4 TP16 MNNVL),
// H200 (2×8 TP16/EP16, or 4×8 TP32/EP32 for High-Throughput), H100
// (4×8 TP32/EP32), and MI350X/MI355X (1×8 TP8) have serving recipes.
- supportedHardware: ["b300", "gb300", "b200", "gb200", "h200", "h100", "mi350x", "mi355x"],
+ supportedHardware: ["b300", "gb300", "b200", "gb200", "h200", "h100", "mi350x", "mi355x", "a3"],
// ---- Cell introspection (config-internal; the engines ignore these keys) ----
//
@@ -138,8 +138,18 @@ export const config = {
title: "PD Mode",
options: [
{ id: "unified", label: "Unified" },
- { id: "prefill", label: "Prefill" },
- { id: "decode", label: "Decode" },
+ {
+ id: "prefill",
+ label: "Prefill",
+ disabled: (s) => s.hw === "a3",
+ disableReason: (s) => (s.hw === "a3" ? "Only Unified PD is supported on this recipe." : ""),
+ },
+ {
+ id: "decode",
+ label: "Decode",
+ disabled: (s) => s.hw === "a3",
+ disableReason: (s) => (s.hw === "a3" ? "Only Unified PD is supported on this recipe." : ""),
+ },
],
},
{
@@ -148,9 +158,21 @@ export const config = {
id: "strategy",
title: "Strategy",
options: [
- { id: "low-latency", label: "Low-Latency", showWhen: (s) => s.pdMode !== "prefill" },
+ {
+ id: "low-latency",
+ label: "Low-Latency",
+ showWhen: (s) => s.pdMode !== "prefill",
+ disabled: (s) => s.hw === "a3",
+ disableReason: (s) => (s.hw === "a3" ? "Only the Balanced operating point is supported on this recipe." : ""),
+ },
{ id: "balanced", label: "Balanced", showWhen: (s) => s.pdMode !== "prefill" },
- { id: "high-throughput", label: "High-Throughput", showWhen: (s) => s.pdMode !== "prefill" },
+ {
+ id: "high-throughput",
+ label: "High-Throughput",
+ showWhen: (s) => s.pdMode !== "prefill",
+ disabled: (s) => s.hw === "a3",
+ disableReason: (s) => (s.hw === "a3" ? "Only the Balanced operating point is supported on this recipe." : ""),
+ },
{ id: "default", label: "Default", showWhen: (s) => s.pdMode === "prefill" },
{ id: "long-context", label: "Long-Context", showWhen: (s) => s.pdMode === "prefill" },
],
@@ -173,7 +195,17 @@ export const config = {
title: "Quantization",
default: "mxfp4",
options: [
- { id: "mxfp4", label: "MXFP4", subtitle: "Moonshot AI checkpoint" },
+ { id: "mxfp4", label: "MXFP4", subtitle: "Moonshot AI checkpoint",
+ disabled: (s) => s.hw === "a3",
+ disableReason: (s) => (s.hw === "a3" ? "Only Modelslim (W4A8) is supported on this recipe." : ""),
+ },
+ {
+ // A3 only (NPU W4A8 checkpoint); hidden on the GPU recipes.
+ id: "modelslim",
+ label: "Modelslim (W4A8)",
+ subtitle: "ModelScope NPU checkpoint",
+ showWhen: (s) => s.hw === "a3",
+ },
{
id: "nvfp4",
label: "NVFP4",
@@ -197,7 +229,7 @@ export const config = {
id: "mmTransport",
title: "VLM Transport",
default: "auto",
- showWhen: (s) => s.pdMode !== "decode",
+ showWhen: (s) => s.pdMode !== "decode" && s.hw !== "a3",
options: [
{
id: "auto",
@@ -236,6 +268,8 @@ export const config = {
options: [
{ id: "none", label: "Non-Spec",
env: (s) => (["mi350x", "mi355x"].includes(s.hw) ? ["SGLANG_MLA_DECODE_TUNE=1"] : []),
+ disabled: (s) => s.hw === "a3",
+ disableReason: (s) => (s.hw === "a3" ? "Only DSPARK is supported on this recipe." : ""),
},
{
id: "dspark",
@@ -259,17 +293,29 @@ export const config = {
config.specCollapses(s)
? ["--tp-size", "--pp-size", "--dcp-size", "--ep-size"]
: [],
- // Every DSPARK recipe layers ReplaySSM on: it moves the per-draft
- // intermediate SSM states onto a fixed ring, lifting the concurrency
- // the state pool admits (needs the Triton decode kernel, the K3
- // default). Only the PD prefill role opts out — it never runs verify
- // and rejects the flag at startup.
flags: (s) => [
...(config.specCollapses(s) ? config.specCollapsedFlags(s) : []),
"--speculative-algorithm DSPARK",
"--speculative-draft-model-path RadixArk/Kimi-K3-DSpark",
"--speculative-dspark-block-size 7",
- ...(s.pdMode === "prefill" ? [] : ["--enable-linear-replayssm-spec"]),
+ // The NPU recipe adds the NPU draft path's own knobs (draft
+ // attention backend, topk 1, unquantized draft weights) on top of
+ // the common trio.
+ ...(s.hw === "a3"
+ ? [
+ "--speculative-draft-attention-backend ascend",
+ "--speculative-eagle-topk 1",
+ "--speculative-draft-model-quantization unquant",
+ ]
+ : []),
+ // ReplaySSM moves the per-draft intermediate SSM states onto a
+ // fixed ring, lifting the concurrency the state pool admits (needs
+ // the Triton decode kernel, the K3 default). It is CUDA-only, and
+ // the PD prefill role opts out — it never runs verify and rejects
+ // the flag at startup.
+ ...(s.hw !== "a3" && s.pdMode !== "prefill"
+ ? ["--enable-linear-replayssm-spec"]
+ : []),
],
},
{
@@ -302,6 +348,10 @@ export const config = {
{
id: "l2",
label: "L1+L2 (host)",
+ disabled: (s) => s.hw === "a3",
+ disableReason: (s) => (s.hw === "a3"
+ ? "NPU HiCache does not support mamba cache (K3's KDA state is one)."
+ : ""),
flags: [
"--enable-hierarchical-cache",
],
@@ -331,6 +381,10 @@ export const config = {
{
id: "l3",
label: "+ L3 (Mooncake)",
+ disabled: (s) => s.hw === "a3",
+ disableReason: (s) => (s.hw === "a3"
+ ? "NPU HiCache does not support mamba cache (K3's KDA state is one)."
+ : ""),
flags: [
"--enable-hierarchical-cache",
"--hicache-storage-backend mooncake",
@@ -369,6 +423,7 @@ export const config = {
modelNames: {
default: "moonshotai/Kimi-K3",
nvfp4: "nvidia/Kimi-K3-NVFP4",
+ a3: "sgl-npu/Kimi-K3-W4A8",
},
placeholders: {
@@ -421,6 +476,7 @@ export const config = {
"gb300|nvfp4": "lmsysorg/sglang:dev-dev-kimi-k3-nvfp4",
"b200|nvfp4": "lmsysorg/sglang:dev-dev-kimi-k3-nvfp4",
"gb200|nvfp4": "lmsysorg/sglang:dev-dev-kimi-k3-nvfp4",
+ a3: "quay.io/ascend/sglang:main-cann9.0.0-a3",
},
// Pre-selects the issue template's `model` field on "Submit verified cell".
github: {
@@ -437,10 +493,23 @@ export const config = {
attention: {
knobs: [
{ id: "tp", label: "TP", values: [
- null, 8,
+ null,
+ {
+ value: 8,
+ get disable() { return [
+ {
+ when: { hw: ["a3"] },
+ reason: "Only TP64 is supported on this recipe.",
+ },
+ ]; },
+ },
{
value: 16,
get disable() { return [
+ {
+ when: { hw: ["a3"] },
+ reason: "Only TP64 is supported on this recipe.",
+ },
{
when: { hw: ["b300", "gb300"] },
reason: "TP=16 needs 16 ranks; the B300 and GB300 recipes have 8 ranks.",
@@ -452,13 +521,41 @@ export const config = {
...config.pipelinedKnobDisableRules,
]; },
},
+ {
+ // A3 only: 64 ranks (4 nodes × 8 cards × 2 dies); hidden on the GPU recipes.
+ value: 64,
+ hide: { hw: ["b300", "gb300", "b200", "gb200", "h200", "h100", "mi350x", "mi355x"] },
+ },
]},
{ id: "dpAttn", label: "DP-Attention",
values: [
- null, false, 2, 4,
+ null,
+ {
+ value: false,
+ get disable() { return [
+ {
+ when: { hw: ["a3"] },
+ reason: "Only DP-Attention=4 is supported on this recipe.",
+ },
+ ]; },
+ },
+ {
+ value: 2,
+ get disable() { return [
+ {
+ when: { hw: ["a3"] },
+ reason: "Only DP-Attention=4 is supported on this recipe.",
+ },
+ ]; },
+ },
+ 4,
{
value: 8,
get disable() { return [
+ {
+ when: { hw: ["a3"] },
+ reason: "Only DP-Attention=4 is supported on this recipe.",
+ },
{
when: { hw: ["b300", "gb300"] },
reason: "On an 8-rank deployment (B300 1×8, GB300 2×4) dp=8 leaves attn_tp=1, so each rank holds the full unsharded MLA KV and OOMs — prefer dp=2/attn_tp=4.",
@@ -473,6 +570,10 @@ export const config = {
{
value: 16,
get disable() { return [
+ {
+ when: { hw: ["a3"] },
+ reason: "Only DP-Attention=4 is supported on this recipe.",
+ },
{
when: { hw: ["b300", "gb300"] },
reason: "DP-Attention=16 needs 16 TP ranks; the B300 and GB300 recipes have 8.",
@@ -502,8 +603,12 @@ export const config = {
requiresHw: ["b200", "b300", "gb200", "gb300"] },
// Blackwell-only: runs FlashInfer's official trtllm-gen SiTU kernels.
{ id: "flashinfer_mxfp4", label: "FlashInfer (MXFP4)", flags: ["--moe-runner-backend flashinfer_mxfp4"],
- requiresHw: ["b200", "b300", "gb200", "gb300"] },
- { id: "marlin", label: "Marlin (W4A16)", flags: ["--moe-runner-backend marlin"] },
+ requiresHw: ["b200", "b300", "gb200", "gb300"],
+ disable: [{ when: { hw: ["a3"] },
+ reason: "FlashInfer is a CUDA kernel and is not supported on NPU." }] },
+ { id: "marlin", label: "Marlin (W4A16)", flags: ["--moe-runner-backend marlin"],
+ disable: [{ when: { hw: ["a3"] },
+ reason: "Marlin is a CUDA kernel and is not supported on NPU." }] },
],
},
// MegaMoE quantization sub-select — shown only when backend === "megamoe".
@@ -517,30 +622,36 @@ export const config = {
env: ["SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320"] },
],
},
- ep: { label: "EP", values: [
- null, 1, 2, 4, 8,
- {
- value: 16,
- get disable() { return [
- {
- when: { hw: ["b300", "gb300"] },
- reason: "EP=16 needs 16 TP ranks; the B300 and GB300 recipes have 8.",
- },
- {
- when: { hw: ["b200"], pdMode: ["unified"], spec: ["none"] },
- reason: "With Spec Decode off, the B200 Unified recipes use TP8 within each PP2 stage, so EP cannot exceed 8. Switch Spec Decode to DSPARK for the flat TP16 shape.",
- },
- ...config.pipelinedKnobDisableRules,
- ]; },
- },
- ]},
+ ep: {
+ showWhen: (b) => b.hw !== "a3",
+ label: "EP",
+ values: [
+ null, 1, 2, 4, 8,
+ {
+ value: 16,
+ get disable() { return [
+ {
+ when: { hw: ["b300", "gb300"] },
+ reason: "EP=16 needs 16 TP ranks; the B300 and GB300 recipes have 8.",
+ },
+ {
+ when: { hw: ["b200"], pdMode: ["unified"], spec: ["none"] },
+ reason: "With Spec Decode off, the B200 Unified recipes use TP8 within each PP2 stage, so EP cannot exceed 8. Switch Spec Decode to DSPARK for the flat TP16 shape.",
+ },
+ ...config.pipelinedKnobDisableRules,
+ ]; },
+ },
+ ],
+ },
},
// ----- Card: "Parsers" -----
parsers: {
items: [
{ id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser kimi_k3" },
- { id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser kimi_k3" },
+ // Tool calling is not yet supported on the NPU, so the item hides on a3.
+ { id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser kimi_k3",
+ hide: { hw: ["a3"] } },
],
},
@@ -617,7 +728,8 @@ export const config = {
// EAGLE --speculative-num-steps N (chain; topk>1 is a tree)
// Only DSPARK is selectable today, so only its form is emitted.
id: "proposedDraftTokens", title: "Proposed Draft Tokens",
- showWhen: (b) => b.spec === "dspark",
+ // The A3 recipe pins the shipped block size (7).
+ showWhen: (b) => b.spec === "dspark" && b.hw !== "a3",
control: "slider",
stripPrefixes: [
"--speculative-dspark-block-size",
@@ -640,9 +752,10 @@ export const config = {
// Spec-only, so gate the row on DSPARK; every DSPARK recipe (except the PD
// prefill role) turns it on in the base, so this row derives to On and
// exists mainly as the opt-out.
- // Needs the Triton linear-attn decode backend (the K3 default).
+ // Needs the Triton linear-attn decode backend (the K3 default); the A3
+ // script never sets it.
id: "replaySsm", title: "ReplaySSM (spec)",
- showWhen: (b) => b.spec === "dspark",
+ showWhen: (b) => b.spec === "dspark" && b.hw !== "a3",
stripPrefixes: ["--enable-linear-replayssm-spec"],
options: [
{ id: "off", label: "Off" },
@@ -662,7 +775,8 @@ export const config = {
// without --speculative-dspark-sps-table-path (every step still
// verifies full width); fails fast with ReplaySSM or DCP > 1.
id: "raggedVerify", title: "Ragged Verify Mode (spec)",
- showWhen: (b) => b.spec === "dspark",
+ // The A3 recipe pins static.
+ showWhen: (b) => b.spec === "dspark" && b.hw !== "a3",
stripEnv: ["SGLANG_RAGGED_VERIFY_MODE"],
options: [
{ id: "static", label: "Auto (static)" },
@@ -671,6 +785,7 @@ export const config = {
},
{
id: "kvCacheDtype", title: "KV Cache Precision",
+ showWhen: (b) => b.hw !== "a3",
stripPrefixes: ["--kv-cache-dtype"],
options: [
{ id: "auto", label: "Auto (BF16)" },
@@ -679,6 +794,7 @@ export const config = {
},
{
id: "mambaSsmDtype", title: "KDA State Precision",
+ showWhen: (b) => b.hw !== "a3",
stripPrefixes: ["--mamba-ssm-dtype"],
options: [
{ id: "auto", label: "Auto (FP32)" },
@@ -693,6 +809,7 @@ export const config = {
// Off suits prefix-free traffic (offline batch, evals): 1 state slot
// per request instead of 4-5.
id: "prefixCache", title: "Prefix Cache",
+ showWhen: (b) => b.hw !== "a3",
stripPrefixes: ["--disable-radix-cache"],
options: [
{ id: "on", label: "On" },
@@ -704,7 +821,8 @@ export const config = {
// prefix cache off, so the row hides (and stops emitting) there.
// Slot cost per request: extra_buffer 5, extra_buffer_lazy 4.
id: "mambaRadix", title: "KDA Radix Cache Strategy",
- showWhen: (b, v, d) => (((v && v.prefixCache) ?? (d && d.prefixCache)) !== "off"),
+ showWhen: (b, v, d) => (b.hw !== "a3")
+ && ((((v && v.prefixCache) ?? (d && d.prefixCache)) !== "off")),
stripPrefixes: ["--mamba-radix-cache-strategy"],
options: [
{ id: "auto", label: "Auto (extra_buffer)" },
@@ -718,6 +836,7 @@ export const config = {
// (extra_buffer 5→4, extra_buffer_lazy 4→3; no_buffer stays 3). Off by
// default. Env var, not a flag, so it emits via env/stripEnv.
id: "mambaSlotSaving", title: "KDA Slot Saving (experimental)",
+ showWhen: (b) => b.hw !== "a3",
stripEnv: ["SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK"],
options: [
{ id: "off", label: "Off" },
@@ -727,6 +846,7 @@ export const config = {
{
// Only meaningful with EP a2a on (MoE card or a large-scale preset).
id: "eplb", title: "Expert Rebalancing (EPLB)",
+ showWhen: (b) => b.hw !== "a3",
stripPrefixes: ["--enable-eplb"],
options: [
{ id: "off", label: "Off" },
@@ -738,6 +858,7 @@ export const config = {
// Validated on the no-a2a MXFP4 runner; untested against SBO (EP a2a)
// and DP attention.
id: "prefillGraph", title: "Prefill CUDA Graph",
+ showWhen: (b) => b.hw !== "a3",
stripPrefixes: ["--cuda-graph-backend-prefill"],
options: [
{ id: "auto", label: "Auto (off)" },
@@ -760,7 +881,7 @@ export const config = {
// resolves it into the full parallelism shape (tp/ep/dp/dcp; attn-tp =
// tp/dp); pool sizing rides the calculator-driven ratio.
id: "lsGpus", title: "Cluster Size (large-scale)",
- showWhen: (b) => b.pdMode === undefined || b.pdMode === "unified",
+ showWhen: (b) => (b.hw !== "a3") && (b.pdMode === undefined || b.pdMode === "unified"),
// Default follows the base cell's own GPU count (tp8 lanes -> 8,
// tp16 lanes -> 16), so a preset starts from "same hardware, new shape".
default: (b) =>
@@ -776,7 +897,7 @@ export const config = {
},
{
id: "lsPreset", title: "Large-Scale Preset",
- showWhen: (b) => b.pdMode === undefined || b.pdMode === "unified",
+ showWhen: (b) => (b.hw !== "a3") && (b.pdMode === undefined || b.pdMode === "unified"),
stripPrefixes: [
"--tp-size", "--tp", "--tensor-parallel-size",
"--ep-size", "--ep", "--expert-parallel-size",
@@ -2187,6 +2308,63 @@ export const config = {
"--port {{PORT}}",
],
},
+ {
+ match: { hw: "a3", pdMode: "unified", strategy: "balanced" },
+ nnodes: 4,
+ verified: false,
+ verificationStatus: "in-progress",
+ env: [
+ "SGLANG_USE_MODELSCOPE=1",
+ "GLOO_SOCKET_IFNAME={{NETWORK_IFACE}}",
+ "HCCL_SOCKET_IFNAME={{NETWORK_IFACE}}",
+ "PYTORCH_NPU_ALLOC_CONF=expandable_segments:True",
+ "SGLANG_SET_CPU_AFFINITY=1",
+ "SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS=1",
+ "SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1",
+ "STREAMS_PER_DEVICE=32",
+ "DEEP_NORMAL_MODE_USE_INT8_QUANT=1",
+ "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=128",
+ "HCCL_BUFFSIZE=200",
+ "HCCL_OP_EXPANSION_MODE=AIV",
+ "DEEPEP_NORMAL_LONG_SEQ_ROUND=64",
+ "DEEPEP_NORMAL_LONG_SEQ_PER_ROUND_TOKENS=512",
+ "DEEPEP_HCCL_BUFFSIZE=1800",
+ "SGLANG_K3_SHARED_EXPERTS_ATTN_TP=1",
+ "SGLANG_K3_DENSE_MLP_ATTN_TP=1",
+ "SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE=1",
+ "SGLANG_ENABLE_SPEC_V2=1",
+ "SGLANG_RAGGED_VERIFY_MODE=static",
+ "SGLANG_DSPARK_FOLDED_PROPOSAL=0",
+ "SGLANG_DSPARK_FOLDED_SAMPLING=0",
+ "SGLANG_DSPARK_STACKED_CTX_KV=0",
+ "SGLANG_DSPARK_EMBED_IN_GRAPH=0",
+ ],
+ flags: [
+ "--trust-remote-code",
+ "--model-path {{MODEL_NAME}}",
+ "--tokenizer-path {{MODEL_NAME}}",
+ "--attention-backend ascend",
+ "--device npu",
+ "--quantization modelslim",
+ "--dtype bfloat16",
+ "--tp-size 64",
+ "--enable-dp-attention",
+ "--dp-size 4",
+ "--enable-dp-lm-head",
+ "--mem-fraction-static 0.78",
+ "--chunked-prefill-size 16384",
+ "--cuda-graph-bs 2 4 8 16",
+ "--max-running-requests 64",
+ "--max-mamba-cache-size 64",
+ "--moe-a2a-backend deepep",
+ "--deepep-mode auto",
+ "--reasoning-parser kimi_k3",
+ "--watchdog-timeout 9000",
+ "--model-loader-extra-config '{\"enable_multithread_load\": true}'",
+ "--host {{HOST_IP}}",
+ "--port {{PORT}}",
+ ],
+ },
],
// Cross-node fabric env (substitute the NIC used by every rank).
@@ -2221,5 +2399,13 @@ export const config = {
" NCCL_SOCKET_IFNAME= # force NCCL off kube-ipvs0",
" SGLANG_HOST_IP=",
],
+ a3: [
+ "Run the same command on all four nodes with --node-rank 0/1/2/3.",
+ "NPU collectives use HCCL. Pin the cross-node NIC on EVERY node:",
+ " GLOO_SOCKET_IFNAME= # bootstrap interface",
+ " HCCL_SOCKET_IFNAME= # HCCL transport interface",
+ " SGLANG_HOST_IP= # this node's IP on that NIC",
+ "If running outside the official image, source set_env.sh on every node first.",
+ ],
},
};