diff --git a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx index 89b7307c8..7b7078562 100644 --- a/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx +++ b/docs/cookbook/autoregressive/Qwen/Qwen3.8-27B.mdx @@ -59,7 +59,10 @@ import { Qwen38MambaRatioCalculator } from "/src/snippets/_qwen38_mamba_ratio_ca The RTX 5090 and RTX PRO 6000 cells above — including every Speculative Decoding / Serving Strategy / SSM dtype combination — were validated at - ISL 8192 / OSL 1024, concurrency 1. The other platforms' recipes carry their + ISL 8192 / OSL 1024, concurrency 1. The DGX Spark cells cover that same full + combination set, but to a weaker standard: each was confirmed to **boot and + serve** at ISL 8192 / OSL 1024, concurrency 1, with no throughput or + acceptance-length numbers taken. The remaining platforms' recipes carry their original validation, which covers the default overlay picks (plus MTP on GB300); non-default overlay picks there are valid but unmeasured. @@ -179,9 +182,17 @@ checkpoint's calibration scales automatically. requires a FlashInfer build whose prefill `plan` accepts `uniform_q_len` (newer than 0.6.15.post1); otherwise run spec with `--attention-backend triton`. On DGX Spark the 128GB is unified memory shared with the host CPU, so all - three checkpoints fit; its cells use 8192-token prefill chunks, - `--mem-fraction-static 0.95`, and `--disable-prefill-cuda-graph`. The SM121 - recipe is not yet validated on that platform. + three checkpoints fit, and its cells reuse the RTX PRO 6000 recipe verbatim + rather than a separate operating point. **Validated on SM121 / aarch64**: all + 36 configurations (3 checkpoints x Speculative Decoding x Serving Strategy x + Mamba SSM Dtype) booted and served on GB10 under `lmsysorg/sglang:qwen38-27b` + at ISL 8192 / OSL 1024, concurrency 1. That is boot-and-serve coverage only — + no throughput or acceptance-length numbers — and it includes the FlashInfer `plan` / + `uniform_q_len` path above, which raised no arity error on that image. Two + host quirks when reproducing on GB10: docker GPU access is CDI-only + (`--device nvidia.com/gpu=all`, as no `nvidia` runtime is registered), and + `nvidia-smi` reports `Not Supported` for memory because it is unified with the + CPU — gate a relaunch on `MemAvailable` in `/proc/meminfo` instead. - **H200 (SM90)**: BF16 and FP8 only — the card has no FP4 tensor cores, so the NVFP4 checkpoint's MLP would fall back to the Marlin W4A16 weight-only path and its cell is greyed out. The H200 recipes use 32768-token prefill chunks @@ -238,8 +249,7 @@ checkpoint's calibration scales automatically. - `--chunked-prefill-size 2048`: decode steps stall behind each prefill chunk on hybrid GDN models, and 8192-token chunks stall them ~600ms at a time. 2048 keeps decode inter-token latency smooth under mixed load and also - improves single-wave TTFT. (DGX Spark is the exception: its cells run - 8192-token chunks.) + improves single-wave TTFT. ## 3. Agent Harnesses diff --git a/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx b/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx index 7b9eb1c1a..4af730961 100644 --- a/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx +++ b/docs/src/snippets/configs/Qwen/qwen3.8-27b.jsx @@ -338,8 +338,9 @@ export const config = { // Verification: RTX 5090 / RTX PRO 6000 cells were measured across their // whole overlay envelope; the h200/gb300 badges carry the source page's // validation, which covers the overlay defaults (plus plain MTP on gb300) — - // non-default overlay picks there are valid but unmeasured. DGX Spark stays - // unverified (SM121 / aarch64 unvalidated). + // non-default overlay picks there are valid but unmeasured. DGX Spark was + // measured across its whole overlay envelope too, but to a weaker standard + // (boot-and-serve only — see the cell block comment below). // // Cells carry NO --mamba-full-memory-ratio: the ratio depends on workload, // S, D and kv_bytes_per_token, so the page's calculator computes it live @@ -480,21 +481,30 @@ export const config = { }, // DGX Spark (GB10, SM121): single node, 128GB coherent unified memory // shared with the CPU — every checkpoint fits, so all three quants get a - // cell. FlashInfer attention comes from the SM120 pair; the platform gets - // its own operating point at 8192-token prefill chunks, 0.95 static - // fraction, and prefill CUDA graphs disabled. Unvalidated on SM121 / - // aarch64. + // cell. These cells reuse the RTX PRO 6000 recipe verbatim rather than a + // separate SM121 operating point: both cards are SM12x Blackwell, and + // GB10's 128GB unified pool is larger than the 6000's 96GB, so a recipe + // that fits the smaller card has headroom here. + // + // Validated on GB10 (SM121 / aarch64): all 36 configurations booted and + // served at ISL 8192 / OSL 1024, concurrency 1. Boot-and-serve only -- no + // throughput or acceptance-length numbers were taken, so this is a weaker + // standard than the SM120 pair's validation, and the Deploy-panel Note says + // so. { match: { hw: "dgx-spark", variant: "default", quant: "nvfp4", nodes: "single" }, + // All 12 overlay combinations served on GB10. DSPARK here also + // exercises the 4-bit `lm_head` this checkpoint quantizes, with no shape + // error. + verified: true, env: [], flags: [ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--kv-cache-dtype fp8_e4m3", - "--mem-fraction-static 0.95", + "--mem-fraction-static 0.85", "--attention-backend flashinfer", - "--chunked-prefill-size 8192", - "--disable-prefill-cuda-graph", + "--chunked-prefill-size 2048", "--reasoning-parser qwen3", "--tool-call-parser qwen3_coder", "--host {{HOST_IP}}", @@ -503,15 +513,16 @@ export const config = { }, { match: { hw: "dgx-spark", variant: "default", quant: "fp8", nodes: "single" }, + // All 12 overlay combinations served on GB10. + verified: true, env: [], flags: [ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--kv-cache-dtype fp8_e4m3", - "--mem-fraction-static 0.95", + "--mem-fraction-static 0.85", "--attention-backend flashinfer", - "--chunked-prefill-size 8192", - "--disable-prefill-cuda-graph", + "--chunked-prefill-size 2048", "--reasoning-parser qwen3", "--tool-call-parser qwen3_coder", "--host {{HOST_IP}}", @@ -520,15 +531,17 @@ export const config = { }, { match: { hw: "dgx-spark", variant: "default", quant: "bf16", nodes: "single" }, + // All 12 overlay combinations served on GB10. Heaviest checkpoint, so + // it holds the sweep's tightest cell: DSPARK + float32 + extra_buffer. + verified: true, env: [], flags: [ "--trust-remote-code", "--model-path {{MODEL_NAME}}", "--kv-cache-dtype fp8_e4m3", - "--mem-fraction-static 0.95", + "--mem-fraction-static 0.85", "--attention-backend flashinfer", - "--chunked-prefill-size 8192", - "--disable-prefill-cuda-graph", + "--chunked-prefill-size 2048", "--reasoning-parser qwen3", "--tool-call-parser qwen3_coder", "--host {{HOST_IP}}",