feat(cookbook): add DGX Spark support for Inkling-Small (#33131)
Co-authored-by: Zijie Xia <zijie.xia@radixark.ai> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
yvbbrjdr
co-authored by
Zijie Xia
Claude Opus 5
Zijie Xia
Claude Opus 5
parent
3e0f7c3f30
commit
e4c4faf8a2
@@ -31,11 +31,12 @@ Then run the **Python** output of the command panel below.
|
||||
|
||||
<Note>The Inkling-Small images are being published to [`lmsysorg/sglang`](https://hub.docker.com/r/lmsysorg/sglang/tags) — watch the tag list for status.</Note>
|
||||
|
||||
There are two multi-arch (amd64 / arm64) CUDA builds plus a ROCm build; pick the CUDA build by your CUDA version, not your GPU:
|
||||
There are two multi-arch (amd64 / arm64) CUDA builds plus a ROCm build; pick the CUDA build by your CUDA version, not your GPU. DGX Spark (GB10) uses a dedicated arm64 CUDA 13 image:
|
||||
|
||||
```bash Command
|
||||
docker pull lmsysorg/sglang:dev-inkling-dspark # CUDA 13
|
||||
docker pull lmsysorg/sglang:dev-cu12-inkling-dspark # CUDA 12
|
||||
docker pull lmsysorg/sglang:dev-inkling-small-dgx-spark # DGX Spark (GB10 / SM121)
|
||||
docker pull lmsysorg/sglang-rocm:dev-rocm720-mi35x-inkling-dspark # AMD MI350X / MI355X
|
||||
```
|
||||
|
||||
@@ -85,6 +86,8 @@ import { Playground } from "/src/snippets/_playground.jsx";
|
||||
|
||||
**Multimodal.** The recipes pass `--enable-multimodal` so the server accepts image and audio inputs alongside text — drop it for text-only serving.
|
||||
|
||||
**DGX Spark (2× GB10).** The verified cell runs NVFP4 with TP=2 across two Sparks over ConnectX-7 (1 GPU per node). Use the `dev-inkling-small-dgx-spark` image, Triton attention + Marlin FP4/MoE, and `--disable-prefill-cuda-graph`. The Docker command already carries the ConnectX-7 flags `--ulimit memlock=-1:-1 --cap-add IPC_LOCK --device /dev/infiniband`.
|
||||
|
||||
**Memory pool ratios.** `--swa-full-tokens-ratio` and `--mamba-full-memory-ratio` (both default `0.1`) size the SWA and Mamba/sconv state pools; tune them to your workload's usage.
|
||||
|
||||
**MTP needs `--enable-multi-layer-eagle`.** The MTP recipe drives Inkling-Small's multi-layer draft head; without this flag the standard EAGLE worker runs against it and outputs garbage.
|
||||
|
||||
@@ -9,7 +9,9 @@
|
||||
// hardware optional — per-model GPUs the shared HARDWARE_CATALOG lacks:
|
||||
// {id, label, vram, vendor}[] merged into the catalog at render
|
||||
// (so a model-specific GPU never needs an engine-catalog edit);
|
||||
// vendor picks the selector group: blackwell | hopper | amd
|
||||
// vendor picks the selector group: blackwell | hopper | amd.
|
||||
// `multiNodeDockerFlags: string[]` (either source) adds
|
||||
// `docker run` flags the platform's fabric needs
|
||||
// variants/quantizations/strategies/nodesOptions LEGACY 4-dim option lists,
|
||||
// used when `matchDims` is absent (nodesOptions id is
|
||||
// `single` or `multi-N` → --nnodes N)
|
||||
@@ -79,6 +81,12 @@ export const Deployment = ({ config, benchmarks }) => {
|
||||
{ id: "gb300", label: "GB300", vram: "288GB" },
|
||||
{ id: "b200", label: "B200", vram: "192GB" },
|
||||
{ id: "gb200", label: "GB200", vram: "192GB" },
|
||||
// GB10 Grace Blackwell — 128 GB coherent unified system memory (not discrete VRAM).
|
||||
// Multi-node runs over ConnectX-7 RDMA (pinned memory + IB passthrough).
|
||||
{ id: "dgx-spark", label: "DGX Spark", vram: "128GB",
|
||||
multiNodeDockerFlags: [
|
||||
"--ulimit memlock=-1:-1", "--cap-add IPC_LOCK", "--device /dev/infiniband",
|
||||
] },
|
||||
],
|
||||
hopper: [
|
||||
{ id: "h200", label: "H200", vram: "141GB" },
|
||||
@@ -690,6 +698,16 @@ export const Deployment = ({ config, benchmarks }) => {
|
||||
const extra = (config.hardware || []).find((h) => h.id === hwId);
|
||||
return (extra && extra.vendor) || "nvidia";
|
||||
};
|
||||
// `config.hardware` overrides by id, as in buildHardwareGroups.
|
||||
const fabricFlagsOf = (hwId) => {
|
||||
const extra = (config.hardware || []).find((h) => h.id === hwId);
|
||||
if (extra) return extra.multiNodeDockerFlags || [];
|
||||
for (const list of Object.values(HARDWARE_CATALOG)) {
|
||||
const hit = list.find((h) => h.id === hwId);
|
||||
if (hit) return hit.multiNodeDockerFlags || [];
|
||||
}
|
||||
return [];
|
||||
};
|
||||
const gpuAccessLines = vendorOf(sel.hw) === "amd"
|
||||
? [
|
||||
"docker run",
|
||||
@@ -708,6 +726,7 @@ export const Deployment = ({ config, benchmarks }) => {
|
||||
// (--dist-init-addr) and NCCL/GLOO traffic are reachable; single-node
|
||||
// just maps the serve port.
|
||||
multinode ? " --network host" : ` -p ${servePort}:${servePort}`,
|
||||
...(multinode ? fabricFlagsOf(sel.hw).map((f) => " " + f) : []),
|
||||
" -v ~/.cache/huggingface:/root/.cache/huggingface",
|
||||
// HF token only for gated checkpoints — configs that declare an HF_TOKEN placeholder.
|
||||
...(config.placeholders && config.placeholders.HF_TOKEN
|
||||
|
||||
@@ -1392,10 +1392,19 @@ export const Playground = ({ config }) => {
|
||||
|| di[`${sel.hw}|${sel.quant}`] || di[sel.hw] || "lmsysorg/sglang:dev";
|
||||
const portFlag = f.find((x) => x.split(/[\s=]/)[0] === "--port");
|
||||
const servePort = portFlag ? portFlag.slice("--port".length).trim() : "{{PORT}}";
|
||||
// Mirrors `multiNodeDockerFlags` on the _deployment.jsx HARDWARE_CATALOG
|
||||
// (Mintlify strips module state, so the engines cannot share it).
|
||||
const HW_MULTINODE_DOCKER_FLAGS = {
|
||||
"dgx-spark": [
|
||||
"--ulimit memlock=-1:-1", "--cap-add IPC_LOCK", "--device /dev/infiniband",
|
||||
],
|
||||
};
|
||||
const fabricFlags = HW_MULTINODE_DOCKER_FLAGS[sel.hw] || [];
|
||||
const dockerLines = [
|
||||
"docker run --gpus all",
|
||||
" --shm-size 32g",
|
||||
(multinode || pdMode) ? " --network host" : ` -p ${servePort}:${servePort}`,
|
||||
...(multinode ? fabricFlags.map((x) => " " + x) : []),
|
||||
" -v ~/.cache/huggingface:/root/.cache/huggingface",
|
||||
` --env "HF_TOKEN={{HF_TOKEN}}"`,
|
||||
...cellEnv.map((e) => ` --env ${e}`),
|
||||
|
||||
@@ -15,6 +15,8 @@ export const benchmarks = [
|
||||
{ match: { hw: "h200" , variant: "default" , quant: "nvfp4" , strategy: "balanced" , nodes: "single" },
|
||||
sglang_version: "dev-inkling-dspark (b7252cc)",
|
||||
accuracy: { aime26_pct: 95.00, bfcl_pct: 76.02, mmau_pct: 74.70 } },
|
||||
{ match: { hw: "dgx-spark", variant: "default" , quant: "nvfp4" , strategy: "balanced" , nodes: "multi-2" },
|
||||
sglang_version: "dev-inkling-small-dgx-spark" },
|
||||
{ match: { hw: "mi350x" , variant: "default" , quant: "bf16" , strategy: "balanced" , nodes: "single" } },
|
||||
{ match: { hw: "mi355x" , variant: "default" , quant: "bf16" , strategy: "balanced" , nodes: "single" } },
|
||||
{ match: { hw: "b200" , variant: "default" , quant: "nvfp4" , strategy: "mtp" , nodes: "single" },
|
||||
|
||||
@@ -9,6 +9,7 @@ export const config = {
|
||||
// Platform list inherited from the Inkling recipes (same architecture family).
|
||||
supportedHardware: [
|
||||
"h200", "b200", "b300", "gb200", "gb300",
|
||||
"dgx-spark",
|
||||
"mi350x", "mi355x",
|
||||
],
|
||||
|
||||
@@ -71,13 +72,15 @@ export const config = {
|
||||
// NVIDIA: two multi-arch CUDA builds (dev-inkling-dspark for CUDA 13,
|
||||
// dev-cu12-inkling-dspark for CUDA 12) — pick by your CUDA version, not by GPU.
|
||||
// Panel defaults to cu13. AMD: dev-rocm720-mi35x-inkling-dspark (sglang-rocm repo).
|
||||
// All tiers ship from the same images, DSpark included.
|
||||
// DGX Spark uses a dedicated arm64 CUDA 13 image with NCCL 2.30.7.
|
||||
// All tiers ship from the same images, DSpark included (except DGX Spark).
|
||||
dockerImages: {
|
||||
h200: "lmsysorg/sglang:dev-inkling-dspark",
|
||||
b200: "lmsysorg/sglang:dev-inkling-dspark",
|
||||
b300: "lmsysorg/sglang:dev-inkling-dspark",
|
||||
gb200: "lmsysorg/sglang:dev-inkling-dspark",
|
||||
gb300: "lmsysorg/sglang:dev-inkling-dspark",
|
||||
"dgx-spark": "lmsysorg/sglang:dev-inkling-small-dgx-spark",
|
||||
mi350x: "lmsysorg/sglang-rocm:dev-rocm720-mi35x-inkling-dspark",
|
||||
mi355x: "lmsysorg/sglang-rocm:dev-rocm720-mi35x-inkling-dspark",
|
||||
},
|
||||
@@ -89,12 +92,16 @@ export const config = {
|
||||
playgroundFeatures: {
|
||||
|
||||
// ----- Card: "Attention Parallelism" -----
|
||||
// TP only. Inkling-Small needs TP=8 to hold the 1M-token SWA + Mamba/sconv pools
|
||||
// (TP=4 can't fit — see §2). TP=16 is cross-node (multi-node path).
|
||||
// TP only. Datacenter NVFP4 recipes need TP=8 (or TP=4 on GB200/GB300) to hold
|
||||
// the 1M-token SWA + Mamba/sconv pools. DGX Spark uses TP=2 across 2 nodes
|
||||
// (1 GPU each). TP=16 is the cross-node datacenter path.
|
||||
attention: {
|
||||
knobs: [
|
||||
{ id: "tp", label: "TP", values: [
|
||||
null, 4, 8,
|
||||
null,
|
||||
{ value: 2, disable: { hw: ["h200", "b200", "b300", "gb200", "gb300", "mi350x", "mi355x"] },
|
||||
disableReason: "TP=2 is the DGX Spark multi-node recipe (1 GPU × 2 nodes)." },
|
||||
4, 8,
|
||||
{ value: 16, disable: { nodes: ["single"] },
|
||||
disableReason: "TP=16 requires 16 ranks — switch the Deploy panel's Nodes to Multi-Nodes first." },
|
||||
]},
|
||||
@@ -103,7 +110,7 @@ export const config = {
|
||||
|
||||
// ----- Card: "MoE Parallelism" -----
|
||||
// Blackwell (SM100) runs the FlashInfer TRT-LLM routed FP4 experts; Hopper (SM90)
|
||||
// has no FP4 runner and falls back to Marlin W4A16.
|
||||
// and DGX Spark (SM121) fall back to Marlin W4A16.
|
||||
moe: {
|
||||
backend: {
|
||||
options: [
|
||||
@@ -112,16 +119,16 @@ export const config = {
|
||||
{ id: "flashinfer_trtllm_routed", label: "FlashInfer TRT-LLM (routed FP4)",
|
||||
flags: ["--moe-runner-backend flashinfer_trtllm_routed"],
|
||||
requiresHw: ["b200", "b300", "gb200", "gb300"],
|
||||
hide: { hw: ["mi350x", "mi355x"] } },
|
||||
hide: { hw: ["mi350x", "mi355x", "dgx-spark"] } },
|
||||
{ id: "marlin", label: "Marlin (W4A16)",
|
||||
flags: ["--moe-runner-backend marlin"],
|
||||
hide: { hw: ["mi350x", "mi355x"] } },
|
||||
{ id: "aiter", label: "AITER",
|
||||
flags: ["--moe-runner-backend aiter"],
|
||||
hide: { hw: ["h200", "b200", "b300", "gb200", "gb300"] } },
|
||||
hide: { hw: ["h200", "b200", "b300", "gb200", "gb300", "dgx-spark"] } },
|
||||
{ id: "triton", label: "Triton",
|
||||
flags: ["--moe-runner-backend triton"],
|
||||
hide: { hw: ["h200", "b200", "b300", "gb200", "gb300"] } },
|
||||
hide: { hw: ["h200", "b200", "b300", "gb200", "gb300", "dgx-spark"] } },
|
||||
],
|
||||
},
|
||||
},
|
||||
@@ -146,12 +153,12 @@ export const config = {
|
||||
],
|
||||
},
|
||||
|
||||
// ----- Card: "PD Disaggregation" ----- NVIDIA only; Mooncake MNNVL env gated to GB200/GB300.
|
||||
// ----- Card: "PD Disaggregation" ----- NVIDIA datacenter only; Mooncake MNNVL env gated to GB200/GB300.
|
||||
pdDisagg: {
|
||||
modes: [
|
||||
{ id: "off", label: "Off" },
|
||||
{ id: "prefill", label: "Prefill role", hide: { hw: ["mi350x", "mi355x"] } },
|
||||
{ id: "decode", label: "Decode role", hide: { hw: ["mi350x", "mi355x"] } },
|
||||
{ id: "prefill", label: "Prefill role", hide: { hw: ["mi350x", "mi355x", "dgx-spark"] } },
|
||||
{ id: "decode", label: "Decode role", hide: { hw: ["mi350x", "mi355x", "dgx-spark"] } },
|
||||
],
|
||||
transferBackends: [
|
||||
{ id: "mooncake", label: "Mooncake",
|
||||
@@ -336,6 +343,38 @@ export const config = {
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
// ====================================================================
|
||||
// NVIDIA DGX Spark (GB10 / SM121) + NVFP4 — 2× Spark over ConnectX-7.
|
||||
// 1 GPU per node → TP=2 across 2 nodes. Marlin W4A16 + Triton attention;
|
||||
// prefill CUDA graphs disabled on this platform.
|
||||
// ====================================================================
|
||||
{
|
||||
match: { hw: "dgx-spark", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "multi-2" },
|
||||
verified: true,
|
||||
env: [
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE=1",
|
||||
],
|
||||
flags: [
|
||||
"--trust-remote-code",
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--tp 2",
|
||||
"--quantization modelopt_fp4",
|
||||
"--attention-backend triton",
|
||||
"--page-size 128",
|
||||
"--fp4-gemm-backend marlin",
|
||||
"--moe-runner-backend marlin",
|
||||
"--mamba-radix-cache-strategy extra_buffer",
|
||||
"--mem-fraction-static 0.85",
|
||||
"--swa-full-tokens-ratio 0.1",
|
||||
"--mamba-full-memory-ratio 0.1",
|
||||
"--enable-multimodal",
|
||||
"--disable-prefill-cuda-graph",
|
||||
"--reasoning-parser inkling",
|
||||
"--tool-call-parser inkling",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
// AMD ROCm (MI350X / MI355X) + BF16 — verified, TP=8. `--moe-runner-backend`
|
||||
// sits right after `--tp` so the Playground AITER override (re-inserted at
|
||||
// that anchor) reproduces this command exactly.
|
||||
|
||||
Reference in New Issue
Block a user