From 4dbba37965aa202d3c96180d9e40077fae027e54 Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Fri, 18 Sep 2026 14:20:18 +0800 Subject: [PATCH] Verify the Ling-3.0-flash-VL FP4 lane on H200 and disable shared-expert fusion in quant recipes (#39419) --- .../InclusionAI/Ling-3.0-flash-VL.mdx | 4 ++-- .../inclusionAI/ling-3.0-flash-vl-benchmarks.jsx | 6 ++++++ .../configs/inclusionAI/ling-3.0-flash-vl.jsx | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash-VL.mdx b/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash-VL.mdx index 79908ae1e..53008fe10 100644 --- a/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash-VL.mdx +++ b/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash-VL.mdx @@ -95,8 +95,8 @@ It is a thinking model: the chat template turns chain-of-thought on by default a - Native context is 128K, and the generated recipes default to the 256K YaRN variant (`--context-length 262144` plus the `rope_scaling` override and `SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1`), matching the model card. For the native 128K window, drop all three from the command. - Audio content parts are rejected before preprocessing; send images as `image_url` and clips as `video_url` content parts (§3). - The FP8 variant (`inclusionAI/Ling-3.0-flash-VL-FP8`) uses 128×128 block quantization, so plain TP must keep `768 ÷ tp` divisible by 128 (TP 1, 2, 3, or 6 — TP=4 fails to load). FP8 recipes default to a single GPU where the ~126 GB weights fit (`--tp 1` on 288 GB-class and B200, verified on GB300) or `--tp 2` on tighter cards. As a manual alternative on four GPUs, `--tp 4 --ep 4` keeps experts whole per rank (measured on 4×GB300). The BF16 checkpoint can also be quantized online with `--quantization fp8`; measured accuracy and speed for both paths are in the benchmark cards below. -- The INT4 variant (`inclusionAI/Ling-3.0-flash-VL-int4`) is a GPTQ 4-bit export (compressed-tensors, group size 32). It runs on all listed hardware with single-GPU (`--tp 1`) recipes; GB300, H200, and DGX Spark cells are verified, the rest carry the unverified badge. -- The FP4 variant (`inclusionAI/Ling-3.0-flash-VL-fp4`) is an MXFP4 export (e2m1 routed-expert weights with e8m0 scales, group 32, plus FP8 e4m3 dynamic activations) and runs on Blackwell only (GB300, B300, B200, DGX Spark). The `flashinfer_mxfp4` MoE backend is selected automatically on the `lmsysorg/sglang:dev-Ling-3.0-flash-VL` image and on sglang with sgl-project/sglang#38526; on builds that already include the VL integration but predate the auto-selection, add `--moe-runner-backend flashinfer_mxfp4` explicitly. GB300 and DGX Spark cells are verified, the rest carry the unverified badge. +- The INT4 variant (`inclusionAI/Ling-3.0-flash-VL-int4`) is a GPTQ 4-bit export (compressed-tensors, group size 32). It runs on all listed hardware with single-GPU (`--tp 1`) recipes; GB300, H200, and DGX Spark cells are verified, the rest carry the unverified badge. Shared-expert fusion is auto-disabled for the INT4 and FP4 checkpoints by the current dev image and by sglang main; older images need `--disable-shared-experts-fusion` explicitly. +- The FP4 variant (`inclusionAI/Ling-3.0-flash-VL-fp4`) is an MXFP4 export (e2m1 routed-expert weights with e8m0 scales, group 32, plus FP8 e4m3 dynamic activations) and runs on Hopper (SM90 CUTLASS W4A16) and Blackwell (SM100/SM120): H200, GB300, B300, B200, and DGX Spark. The `flashinfer_mxfp4` MoE backend is selected automatically on the `lmsysorg/sglang:dev-Ling-3.0-flash-VL` image and on sglang main; on builds that already include the VL integration but predate the auto-selection, add `--moe-runner-backend flashinfer_mxfp4` explicitly. GB300, H200, and DGX Spark cells are verified, the rest carry the unverified badge. ## 3. Advanced Usage diff --git a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-vl-benchmarks.jsx b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-vl-benchmarks.jsx index ab0db5171..728684ccf 100644 --- a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-vl-benchmarks.jsx +++ b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-vl-benchmarks.jsx @@ -80,6 +80,12 @@ export const benchmarks = [ }, { match: { hw: "b300", variant: "default", quant: "fp4", strategy: "balanced", nodes: "single" } }, { match: { hw: "b200", variant: "default", quant: "fp4", strategy: "balanced", nodes: "single" } }, + { + match: { hw: "h200", variant: "default", quant: "fp4", strategy: "balanced", nodes: "single" }, + sglang_version: "dev @ bf254483a1", + accuracy: { mmmu_pro_pct: 76.24, gsm8k_pct: 96.66 }, + notes: "1×H200 (141 GB), TP=1, flashinfer_mxfp4 MoE backend (SM90 CUTLASS W4A16), auto-selected — verified without an explicit --moe-runner-backend flag (healthy in 330 s). Serving smoke verified: text request completes with finish_reason=stop and reasoning split. Accuracy measured on 2×B300, TP=2 (sgl-eval, single-shot, thinking on): MMMU-Pro 76.24% (1,730 examples, stop 98.96%, truncated 1.04%), GSM8K 96.66% (stop 100%).", + }, { match: { hw: "dgx-spark", variant: "default", quant: "fp4", strategy: "balanced", nodes: "single" }, sglang_version: "dev @ bf254483a1", diff --git a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-vl.jsx b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-vl.jsx index 838b088d1..fa35912d0 100644 --- a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-vl.jsx +++ b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-vl.jsx @@ -385,6 +385,22 @@ sgl-eval run gsm8k \\ "--port {{PORT}}", ], }, + { + match: { hw: "h200", variant: "default", quant: "fp4", strategy: "balanced", nodes: "single" }, + verified: true, + env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 1", + "--context-length 262144", + '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'', + "--reasoning-parser auto", + "--tool-call-parser auto", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, { match: { hw: "dgx-spark", variant: "default", quant: "fp4", strategy: "balanced", nodes: "single" }, verified: true,