From c3947eeadadb51ffe2fbaa1e85cba118eda81aad Mon Sep 17 00:00:00 2001 From: Mick Date: Tue, 25 Aug 2026 20:25:17 +0800 Subject: [PATCH] [diffusion] docs: desktop-safe 24 GB recipe and the DGX Spark tier (#36169) Co-authored-by: Claude Fable 5 --- .../cookbook/diffusion/MiniMax/MiniMax-H3.mdx | 19 ++++++-- .../snippets/configs/MiniMaxAI/minimax-h3.jsx | 47 ++++++++++++++++--- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx index 99e46a15e..187cfeba6 100644 --- a/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx +++ b/docs/cookbook/diffusion/MiniMax/MiniMax-H3.mdx @@ -1530,10 +1530,23 @@ Under that cap, Recipe A wins the whole request at every host size: | 48 GB host | 15.8 + 192.1 + 10.0 ≈ **218 s** | 246–267 s | | 64 GB host | 7.5 + 162.4 + 10.3 ≈ **180 s** | 194–195 s | -Same GPU, same load window, unpruned bf16 checkpoints, outputs verified. The +Same GPU, same load window, unpruned bf16 checkpoints, outputs verified. All +figures are anchored at 480P — activations grow with the pixel count, so at +768P drop the resident DiT layers to 0 first, then `video_vae` to 24 if the +decode still collides. And the host convention holds the weights in page +cache; a physical 32 GB machine re-reads them from disk each step, so the +page cache cannot hold the per-step weight sweep, so every step re-reads it +from disk and the drive becomes the denoise clock: a real desktop 4090 with a +990 Pro measured 38 s/step, reading 52.9 GB per step (faulted sequentially, so +almost none of it shows in majflt — measure `read_bytes`, not major faults). +Two things cut that read directly: resident DiT layers (~1 GB/step each — on a +physically small host raise them as far as VRAM allows, the opposite of the +capped-host guidance above), and more RAM (64 GB caches the sweep and returns +to the quoted times). The VRAM axis holds too: capped at 16 GiB the same recipe wins ~250 vs 292–301 s, -and at 24 GiB (with `--dit-layerwise-resident-layers 10`, which only a 24 GB -card has headroom for) ~230 vs 249–260 s. Four changes carry it: the VAE staying on its checkpoint mapping (#35862, root fix +and at 24 GiB (with `--dit-layerwise-resident-layers 6` — measured at a +22 GiB cap so a desktop's own allocations fit; a headless card can raise it +to 10 for under 1% more) ~8.5 s/step vs ComfyUI's 249–260 s requests. Four changes carry it: the VAE staying on its checkpoint mapping (#35862, root fix #35946), per-layer pinning with net-cost accounting (#35867), the courier thread that ships still-mapped layers through pinned slots (#35882), and decoder weights held in their decode dtype from load (#35967) — which is what diff --git a/docs/src/snippets/configs/MiniMaxAI/minimax-h3.jsx b/docs/src/snippets/configs/MiniMaxAI/minimax-h3.jsx index d7d8be5dc..8758ebb49 100644 --- a/docs/src/snippets/configs/MiniMaxAI/minimax-h3.jsx +++ b/docs/src/snippets/configs/MiniMaxAI/minimax-h3.jsx @@ -24,17 +24,24 @@ const CONSUMER_24G = ["rtx4090", "rtx3090"]; // their recipes are derived from the tier logic, not verified runs. const WORKSTATION_48G = ["rtx6000ada"]; const WORKSTATION_96G = ["rtxpro6000"]; +// GB10 unified memory: 128 GB shared between CPU and GPU, so the VRAM/host +// split that shapes every tier above does not exist. The whole 108 GB +// deployment fits, and an offloaded component's "copy to device" is an +// in-memory copy on the coherent bus. No hard-cap anchor was measured. +const UNIFIED_128G = ["dgx-spark"]; const CONSUMER_SINGLE = [ ...CONSUMER_12G, ...CONSUMER_16G, ...CONSUMER_24G, ...WORKSTATION_48G, ...WORKSTATION_96G, + ...UNIFIED_128G, ]; const CONSUMER_VRAM_16_PLUS = [...CONSUMER_16G, ...CONSUMER_24G]; const CONSUMER_AMPERE = ["rtx3060", "rtx3090"]; function consumerFlags(s) { + if (UNIFIED_128G.includes(s.hw)) return unified128Flags(); if (WORKSTATION_96G.includes(s.hw)) return workstation96Flags(); // The whole video decoder held for the decode only: residency arms at the // decoder's first block and releases when it finishes, so the denoise still @@ -50,12 +57,14 @@ function consumerFlags(s) { if (CONSUMER_VRAM_16_PLUS.includes(s.hw) && s.host_ram === "ram96") { flags.push("--dit-layerwise-resident-layers 4"); } - // A 24 GB card on a 32 GB host has allocator headroom to keep ten DiT - // layers resident (measured 10.4 vs 11.6 s/step); a 16 GB card does not -- - // there even four resident layers measured slower than none, so it keeps - // the plain recipe. + // A 24 GB card has headroom for resident DiT layers, but their benefit + // flattened once the decoder went fp16 and the courier overlapped the + // streaming: measured at a 22 GiB cap (2 GiB desktop headroom), r10/r6/r4 + // land at 8.41/8.48/8.51 s/step. Six layers keep ~2.4 GiB more free than + // ten for under 1% of speed -- the desktop-safe point. A 16 GB card keeps + // the plain recipe; even four resident layers measured slower there. if (CONSUMER_24G.includes(s.hw) && s.host_ram === "ram32") { - flags.push("--dit-layerwise-resident-layers 10"); + flags.push("--dit-layerwise-resident-layers 6"); } if (WORKSTATION_48G.includes(s.hw)) { flags.push("--dit-layerwise-resident-layers 40"); @@ -63,6 +72,17 @@ function consumerFlags(s) { return flags; } +function unified128Flags() { + // Unified memory holds the whole deployment: the memory manager pins every + // component (the budget sees ~128 GB), and streamed copies move at memory + // speed on the coherent bus. video_vae=36 still buys the fast decode. + return [ + "--performance-mode memory", + "--layerwise-offload-components dit,text_encoder,vae", + "--layerwise-resident-layers video_vae=36", + ]; +} + function workstation96Flags() { // 96 GB holds the whole 61.7 GB DiT; only the encoders and VAEs step aside. return [ @@ -79,13 +99,14 @@ function consumerHints(s) { if (bigHost) { if (CONSUMER_VRAM_16_PLUS.includes(s.hw)) { hints.push("verified end to end: ~6 s per denoise step, 13 s decode"); + hints.push("fewer resident layers than the 32 GB rows is not a typo: with the DiT pinned in a big host, streamed layers arrive at pinned-copy speed and GPU residency buys little; on a 32 GB host the stream is the bottleneck residency cuts"); } else { hints.push("~6 s per step once the host pins the DiT; the decode holds all 36 blocks in their fp16 decode dtype and takes ~10 s"); } return hints; } if (CONSUMER_24G.includes(s.hw)) { - hints.push("measured at 32 GB host: ~10.4 s per denoise step with ten resident layers, ~9.6 s decode, ~230 s per request -- ahead of ComfyUI (249-260 s) under the same hard 24 GiB cap"); + hints.push("measured at 32 GB host under a 22 GiB cap (desktop headroom): ~8.5 s per denoise step with six resident layers, ~9.6 s decode -- ahead of ComfyUI (249-260 s at the 24 GiB cap); a headless card can raise to ten layers for under 1% more"); } else if (CONSUMER_16G.includes(s.hw)) { hints.push("measured at 32 GB host: ~11.9 s per denoise step, ~11 s decode, ~250 s per request -- ahead of ComfyUI (292-301 s) under the same hard 16 GiB cap"); } else { @@ -94,6 +115,12 @@ function consumerHints(s) { if (CONSUMER_AMPERE.includes(s.hw)) { hints.push("the recipe and its memory behavior are tier-exact for this card; the step times above were measured on 40-series compute, and Ampere lands above them"); } + if (UNIFIED_128G.includes(s.hw)) { + return [ + "derived recipe, not yet verified: the 128 GB unified pool holds the whole 108 GB deployment, so the memory manager pins every component and offload copies run at memory speed on the coherent bus", + "expect step times above the discrete-GPU rows: the GB10's ~273 GB/s memory bandwidth is the denoise ceiling, not the placement", + ]; + } if (WORKSTATION_96G.includes(s.hw)) { hints.push("derived recipe, not yet verified: 96 GB holds the whole 61.7 GB DiT resident, so only the text encoder and VAEs stream -- expect near-datacenter step times rather than the offload figures above"); } @@ -107,6 +134,9 @@ function consumerHints(s) { hints.push("a 32 GB host cannot cache the 108 GB checkpoint: NVMe is required, and real runs land above the quoted step time"); } hints.push('the startup log should say "leaving ... GiB of weights on the checkpoint mapping" -- if it does not, the host is not the constraint you set'); + hints.push("every figure here is anchored at 480P: activations grow with the pixel count, so at 768P drop the resident DiT layers to 0 first, then video_vae to 24 if the decode still collides -- the flags trade speed for headroom in that order"); + hints.push("on a physical 32 GB host the page cache cannot hold the per-step weight sweep, so every step re-reads ~40-65 GB from disk and the drive is the denoise clock: a real desktop 4090 with a 990 Pro measured 38 s/step (52.9 GB read per step). Resident DiT layers cut that read directly (~1 GB/step each), so raise them as far as VRAM allows; 64 GB of RAM caches the sweep and returns to the quoted times"); + hints.push("on Windows run under WSL2, and keep the checkpoint inside the ext4 side (under ~), never on /mnt/c -- the NTFS bridge reads an order of magnitude slower and multiplies the disk clock"); return hints; } @@ -122,6 +152,7 @@ return { "mi355x", "rtxpro6000", "rtx6000ada", + "dgx-spark", "rtx5090", "rtx4090", "rtx3090", @@ -158,7 +189,8 @@ return { scope: "serve", description: "System memory decides where the DiT weights wait between steps: pinned when they fit, on the checkpoint mapping when they do not.", default: "ram32", - showWhen: (s) => CONSUMER_SINGLE.includes(s.hw), + showWhen: (s) => + CONSUMER_SINGLE.includes(s.hw) && !UNIFIED_128G.includes(s.hw), options: [ { id: "ram32", label: "32 GB" }, { id: "ram64", label: "48-64 GB" }, @@ -525,6 +557,7 @@ return { { id: "mi355x-resident-2", hw: "mi355x", nodes: 1, gpus_per_node: 2, placement: "resident", tp_size: 1, ulysses_degree: 2, ring_degree: 1, encoder: "auto" }, { id: "mi355x-resident-4", hw: "mi355x", nodes: 1, gpus_per_node: 4, placement: "resident", tp_size: 1, ulysses_degree: 4, ring_degree: 1, encoder: "auto" }, { id: "mi355x-resident-8", hw: "mi355x", nodes: 1, gpus_per_node: 8, placement: "resident", tp_size: 1, ulysses_degree: 8, ring_degree: 1, encoder: "auto", default: true }, + { id: "dgx-spark-offload-1", hw: "dgx-spark", nodes: 1, gpus_per_node: 1, placement: "offload", tp_size: 1, ulysses_degree: 1, ring_degree: 1, encoder: "auto", default: true, unverified: true }, { id: "rtxpro6000-offload-1", hw: "rtxpro6000", nodes: 1, gpus_per_node: 1, placement: "offload", tp_size: 1, ulysses_degree: 1, ring_degree: 1, encoder: "auto", default: true, unverified: true }, { id: "rtx6000ada-offload-1", hw: "rtx6000ada", nodes: 1, gpus_per_node: 1, placement: "offload", tp_size: 1, ulysses_degree: 1, ring_degree: 1, encoder: "auto", default: true, unverified: true }, { id: "rtx5090-offload-1", hw: "rtx5090", nodes: 1, gpus_per_node: 1, placement: "offload", tp_size: 1, ulysses_degree: 1, ring_degree: 1, encoder: "auto", default: true },