From ed82bea1464d8ef66ed2b3ff6d9fc06c2e18ee60 Mon Sep 17 00:00:00 2001 From: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> Date: Tue, 1 Sep 2026 15:42:39 -0700 Subject: [PATCH] [Cookbook] DeepSeek-V4: add DGX Spark (2x GB10) Flash Official FP4 recipe (#37479) Co-authored-by: Claude Fable 5.1 --- .../autoregressive/DeepSeek/DeepSeek-V4.mdx | 13 ++++- .../configs/deepseek-ai/deepseek-v4.jsx | 54 +++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index bafc0b5b1..65602f7ad 100644 --- a/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -32,7 +32,7 @@ For how to launch the image, see [Install → Method 3: Using Docker](../../../d **NVIDIA GPUs** -A single image — `lmsysorg/sglang:latest` — covers the **datacenter GPUs** in this cookbook (B200 / B300 / GB200 / GB300 / H100 / H200 / RTX PRO 6000). The one exception is **Flash Vision (Exp)**, whose support has not shipped in a release yet: its cells use the preview image `lmsysorg/sglang:dev-dsv4-flash-vision` (the command panel picks it automatically — see the [Flash Vision notes](#vision-note)). +A single image — `lmsysorg/sglang:latest` — covers the **datacenter GPUs** in this cookbook (B200 / B300 / GB200 / GB300 / H100 / H200 / RTX PRO 6000). The one exception is **Flash Vision (Exp)**, whose support has not shipped in a release yet: its cells use the preview image `lmsysorg/sglang:dev-dsv4-flash-vision` (the command panel picks it automatically — see the [Flash Vision notes](#vision-note)). **DGX Spark** is the other exception: its single Flash Official FP4 cell uses the DGX Spark–only preview image `lmsysorg/sglang:dev-v4f-2dgx` (the command panel picks it automatically — see the [DGX Spark notes](#spark-note)); do not use that image on any other hardware. ```bash Command docker pull lmsysorg/sglang:latest @@ -215,6 +215,17 @@ For the original Flash and Pro checkpoints: - `high-throughput`: MTP disabled — at saturation the verify step costs more than it saves. - MTP runs on the v2 speculative path. + + +**DGX Spark (2x GB10, Flash Official FP4)** + +The **DGX Spark** row has a single recipe: **Flash Official (0731) · FP4 · Balanced · Multi-Nodes** — the 284B checkpoint does not fit one 128GB GB10, so it runs TP=2 across two DGX Sparks connected over ConnectX-7 (RoCE). Every other DGX Spark combination is greyed out on purpose. + +- **Docker image** — the cell uses `lmsysorg/sglang:dev-v4f-2dgx`, a preview build made **only for DGX Spark**: it bakes in the SM12x `b12x` MoE (W4A8) and compressed-MLA attention kernels ([#34878](https://github.com/sgl-project/sglang/pull/34878), [#35899](https://github.com/sgl-project/sglang/pull/35899), [#34018](https://github.com/sgl-project/sglang/pull/34018)) plus the CuTeDSL and NCCL pins the GB10 pair needs. Do not use it on other hardware, and use the panel's Docker mode — the bare Python command needs the `b12x` kernel package this image ships. +- **Run the same command on both Sparks** with `--node-rank 0` / `--node-rank 1` and `--dist-init-addr` pointing at node 0 over the ConnectX-7 link. The `docker run` flags the panel emits (`--network host --ulimit memlock=-1:-1 --cap-add IPC_LOCK --device /dev/infiniband`) are what let NCCL use RDMA; without them NCCL silently falls back to TCP and decode slows by roughly 40%. +- **Env knobs** in the cell are part of the recipe: `SGLANG_SM120_FLASHMLA_BACKEND=b12x` selects the b12x attention path, `SGLANG_B12X_MAX_TOKENS` must equal `--chunked-prefill-size`, and `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` avoids unified-memory fragmentation OOMs on GB10. +- **Text only** — the image predates Flash Vision support, and the b12x kernels do not yet cover image prefill on SM12x; there is no DGX Spark cell for the Flash Vision variant. + **DeepSeek-V4-Flash-Vision-Exp (Experimental)** diff --git a/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index 88821e0bb..1afa2c99a 100644 --- a/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -9,6 +9,10 @@ export const config = { supportedHardware: [ "h100", "h200", "b200", "b300", "gb200", "gb300", "rtx6000", "rtx5090", + // NVIDIA DGX Spark (GB10, SM121) — Flash Official FP4 only, as a 2-node + // TP=2 pair over ConnectX-7 RoCE; the shared HARDWARE_CATALOG carries the + // entry and its multi-node Docker flags. + "dgx-spark", // AMD ROCm — MI300X (Flash FP8) + MI355X (Flash/Pro, FP4/FP8). "mi300x", "mi355x", ], @@ -192,6 +196,12 @@ sgl-eval run mmmu_pro \\ // (sgl-project/sglang#37253) — until it does, the variant needs this // preview build on every hardware. "flash-vision|fp4": "lmsysorg/sglang:dev-dsv4-flash-vision", + // DGX Spark ONLY. A dedicated preview build for the 2x GB10 pair: it bakes + // in the SM12x b12x MoE/attention kernels (sgl-project/sglang#34878, + // #35899, #34018) and CuTeDSL/NCCL pins the GB10 recipe needs, none of + // which are in `latest`. It is not built for, and must not be used on, any + // other hardware — every other row keeps its own image. + "dgx-spark|flash-official|fp4": "lmsysorg/sglang:dev-v4f-2dgx", // NVFP4 checkpoints crash at weight load on v0.5.18 (the MXFP4-packed MTP // layer's FP8 delegate needs the #36275 guard, merged 2026-08-26) — route // every NVFP4 cell to the nightly until a release contains that fix. @@ -3019,6 +3029,50 @@ sgl-eval run mmmu_pro \\ ], }, + // ==================================================================== + // DGX Spark (GB10 / SM121) — Flash Official FP4, 2-node TP=2, Balanced + // ==================================================================== + // One cell only: the verified GB10 recipe. It runs the SM12x b12x MoE + // (W4A8) + b12x compressed-MLA attention with DSpark, split TP=2 across two + // DGX Sparks over ConnectX-7 RoCE. Verified end to end on 2x DGX Spark with + // the `lmsysorg/sglang:dev-v4f-2dgx` image (GSM8K 96%, AgentX c1/c2 clean). + // Every other DGX Spark combination (other variants / quants / strategies / + // single node) is intentionally absent and greys out: a single 128GB GB10 + // cannot hold the checkpoint, and the b12x kernels are text-only today + // (image prefill for Flash Vision is unsupported on SM12x). + // Env: b12x attention + FP8 wo_a opt-in + MHC post/pre fusion are the GB10 + // tuning knobs; SGLANG_B12X_MAX_TOKENS must track --chunked-prefill-size; + // expandable_segments avoids unified-memory fragmentation OOMs. + { + match: { hw: "dgx-spark", variant: "flash-official", quant: "fp4", strategy: "balanced", nodes: "multi-2" }, + verified: true, + warn: "The Docker image lmsysorg/sglang:dev-v4f-2dgx is a DGX Spark-only preview build (2x GB10, TP=2 over ConnectX-7) — do not use it on other hardware. Use Docker mode: the bare Python command needs the b12x kernel package this image ships. See [DGX Spark notes](#spark-note).", + env: [ + "SGLANG_SM120_FLASHMLA_BACKEND=b12x", + "B12X_MLA_SM120_DSV4_H16_NATIVE=1", + "SGLANG_OPT_FUSE_MHC_POST_PRE=1", + "SGLANG_OPT_FP8_WO_A_GEMM=1", + "SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK=1", + "SGLANG_B12X_MAX_TOKENS=8192", + "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True", + ], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 2", + "--moe-runner-backend b12x", + "--speculative-algorithm DSPARK", + "--chunked-prefill-size 8192", + "--context-length 327680", + "--mem-fraction-static 0.80", + "--swa-full-tokens-ratio 0.2", + "--cuda-graph-max-bs-decode 32", + "--max-running-requests 32", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // ==================================================================== // B200 + FP4 — Flash Vision (Exp) //