Add INT4 and FP4 lanes to the Ling-3.0-flash-VL cookbook (#38527)

2026-09-10 15:23:50 +08:00
parent 1b77f498a0
commit 9a2f17f41d
3 changed files with 250 additions and 28 deletions
@@ -28,8 +28,8 @@ export const benchmarks = [
  {
    match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
    sglang_version: "dev @ bf254483a1",
-    accuracy: { mmmu_pro_pct: 76.01, gsm8k_pct: 97.19 },
-    notes: "4×GB300, TP=4. Measured with online dynamic FP8 (--quantization fp8 on the BF16 checkpoint), the same serving path the FP8 variant uses. Accuracy vs BF16 on the same box: MMMU-Pro 76.01% vs 77.86% (stop 99.36%), GSM8K 97.19% vs 97.35% (stop 100%). Speed (LL points, same protocol as the BF16 card): text 8192/1024 conc 1: TTFT 174.80 ms, TPOT 3.95 ms, 546 tok/s/GPU; conc 16: TTFT 981.25 ms, TPOT 7.49 ms, 4273 tok/s/GPU. Image 1024/1024 conc 1: TTFT 272.09 ms, TPOT 4.51 ms, 153 tok/s/GPU; conc 16: TTFT 1389.27 ms, TPOT 7.15 ms, 1369 tok/s/GPU. FP8 prefill (TTFT) is consistently faster than BF16 while TPOT is ~10% slower.",
+    accuracy: { mmmu_pro_pct: 77.34, gsm8k_pct: 97.19 },
+    notes: "GB300 TP=1, official FP8 checkpoint. MMMU-Pro 77.34% (stop 99.77%, truncated 0.23%, error 0) measured at TP=4 --ep 4 on 4×GB300; TP=1 serving smoke also verified on one GB300; GSM8K 97.19% (stop 100%) measured with online dynamic FP8 (--quantization fp8 on the BF16 checkpoint). BF16 on the same box: MMMU-Pro 75.78% (TP=4), GSM8K 97.35%. Speed (LL points, online-quantized measurement at TP=4, same protocol as the BF16 card): text 8192/1024 conc 1: TTFT 174.80 ms, TPOT 3.95 ms, 546 tok/s/GPU; conc 16: TTFT 981.25 ms, TPOT 7.49 ms, 4273 tok/s/GPU. Image 1024/1024 conc 1: TTFT 272.09 ms, TPOT 4.51 ms, 153 tok/s/GPU; conc 16: TTFT 1389.27 ms, TPOT 7.15 ms, 1369 tok/s/GPU. FP8 prefill (TTFT) is consistently faster than BF16 while TPOT is ~10% slower.",
  },
  { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" } },
  { match: { hw: "b200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" } },
@@ -50,6 +50,58 @@ export const benchmarks = [
  { match: { hw: "h100", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" } },
  { match: { hw: "b300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" } },
  { match: { hw: "b200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" } },
-  { match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" } },
+  {
+    match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
+    sglang_version: "dev @ bf254483a1",
+    accuracy: { mmmu_pro_pct: 77.34, gsm8k_pct: 97.19 },
+    notes: "2×H200, TP=2, E4M3 block-[128,128] kernels engaged (measured: 126.2 GB checkpoint, ~59 GB weights per GPU, healthy in 581 s including download and first-time DeepGEMM compile). Serving smoke verified: text and image requests complete with finish_reason=stop and reasoning split. Accuracy: MMMU-Pro 77.34% measured on the official FP8 checkpoint at TP=4 --ep 4 on 4×GB300; GSM8K 97.19% measured with online dynamic FP8 (--quantization fp8 on the BF16 checkpoint) on the same serving path.",
+  },
  { match: { hw: "h100", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" } },
+  {
+    match: { hw: "gb300", variant: "default", quant: "int4", strategy: "balanced", nodes: "single" },
+    sglang_version: "dev @ bf254483a1",
+    accuracy: { mmmu_pro_pct: 77.51, gsm8k_pct: 96.97 },
+    notes: "GB300 TP=1, auto-detected compressed-tensors int4 (group_size 32). Serving smoke verified at TP=1 and 2×GB300 TP=2: text and image requests complete with finish_reason=stop and reasoning split into reasoning_content. Accuracy measured on 2×B300, TP=2 (sgl-eval, single-shot, thinking on): MMMU-Pro 77.51% (1,730 examples, stop 98.96%, truncated 1.04%), GSM8K 96.97% (stop 100%).",
+  },
+  { match: { hw: "b300", variant: "default", quant: "int4", strategy: "balanced", nodes: "single" } },
+  { match: { hw: "b200", variant: "default", quant: "int4", strategy: "balanced", nodes: "single" } },
+  {
+    match: { hw: "h200", variant: "default", quant: "int4", strategy: "balanced", nodes: "single" },
+    sglang_version: "dev @ bf254483a1",
+    accuracy: { mmmu_pro_pct: 77.51, gsm8k_pct: 96.97 },
+    notes: "1×H200 (141 GB), TP=1, auto-detected compressed-tensors int4 (W4A16 Marlin MoE; 121 GB GPU memory in use incl. KV pool). Serving smoke verified: text and image requests complete with finish_reason=stop and reasoning split. Accuracy measured on 2×B300, TP=2 (sgl-eval, single-shot, thinking on): MMMU-Pro 77.51% (stop 98.96%), GSM8K 96.97% (stop 100%).",
+  },
+  { match: { hw: "h100", variant: "default", quant: "int4", strategy: "balanced", nodes: "single" } },
+  {
+    match: { hw: "gb300", variant: "default", quant: "fp4", strategy: "balanced", nodes: "single" },
+    sglang_version: "dev @ bf254483a1",
+    accuracy: { mmmu_pro_pct: 76.24, gsm8k_pct: 96.66 },
+    notes: "GB300 TP=1, flashinfer_mxfp4 MoE backend (auto-selected). Serving smoke verified on one GB300 at TP=1 and on 2×GB300 at TP=2: text and image requests complete with finish_reason=stop and reasoning split. Accuracy measured on 2×B300, TP=2, without an explicit --moe-runner-backend flag (sgl-eval, single-shot, thinking on): MMMU-Pro 76.24% (1,730 examples, stop 98.96%, truncated 1.04%), GSM8K 96.66% (stop 100%).",
+  },
+  { match: { hw: "b300", variant: "default", quant: "fp4", strategy: "balanced", nodes: "single" } },
+  { match: { hw: "b200", variant: "default", quant: "fp4", strategy: "balanced", nodes: "single" } },
+  {
+    match: { hw: "dgx-spark", variant: "default", quant: "fp4", strategy: "balanced", nodes: "single" },
+    sglang_version: "dev @ bf254483a1",
+    speed: [
+      { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 8 },
+        ttft_ms: 2538.64, tpot_ms: 30.18, tokens_per_sec_per_gpu: 272.44 },
+      { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16, num_prompts: 32 },
+        ttft_ms: 23622.31, tpot_ms: 111.61, tokens_per_sec_per_gpu: 1069.73 },
+    ],
+    accuracy: { mmmu_pro_pct: 76.24, gsm8k_pct: 96.66 },
+    notes: "DGX Spark GB10, TP=1, flashinfer_mxfp4 MoE backend (SM120 CUTLASS W4A8 path), auto-selected — verified on GB10 without an explicit --moe-runner-backend flag. Serving smoke verified on the 128GB unified-memory node with positive headroom: text and image requests complete with finish_reason=stop and reasoning split. Speed: bench_serving --flush-cache, temperature 0, TTFT/TPOT are P50; tok/s = total (input + output) token throughput on one GPU. Image workload (one 720p JPEG per request, 883 vision tokens, in/out=1024/1024): conc 1 (8 prompts): TTFT 318.70 ms, TPOT 30.70 ms, 96.26 tok/s; conc 16 (32 prompts): TTFT 9476.07 ms, TPOT 82.39 ms, 510.44 tok/s. Accuracy measured on 2×B300, TP=2 (sgl-eval, single-shot, thinking on): MMMU-Pro 76.24% (1,730 examples, stop 98.96%, truncated 1.04%), GSM8K 96.66% (stop 100%).",
+  },
+  {
+    match: { hw: "dgx-spark", variant: "default", quant: "int4", strategy: "balanced", nodes: "single" },
+    sglang_version: "dev @ bf254483a1",
+    speed: [
+      { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1, num_prompts: 8 },
+        ttft_ms: 2885.34, tpot_ms: 44.18, tokens_per_sec_per_gpu: 191.87 },
+      { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16, num_prompts: 32 },
+        ttft_ms: 173008.14, tpot_ms: 66.22, tokens_per_sec_per_gpu: 549.63 },
+    ],
+    accuracy: { mmmu_pro_pct: 77.51, gsm8k_pct: 96.97 },
+    notes: "DGX Spark GB10, TP=1, auto-detected compressed-tensors int4. Serving smoke verified on the 128GB unified-memory node with positive headroom: text and image requests complete with finish_reason=stop and reasoning split. Speed: bench_serving --flush-cache, temperature 0, TTFT/TPOT are P50; tok/s = total (input + output) token throughput on one GPU. Image workload (one 720p JPEG per request, 883 vision tokens, in/out=1024/1024): conc 1 (8 prompts): TTFT 1137.64 ms, TPOT 43.92 ms, 65.14 tok/s; conc 16 (32 prompts): TTFT 137419.53 ms, TPOT 60.71 ms, 214.28 tok/s. conc-16 TTFT is queue-dominated (two waves of 16; GB10 prefill runs at ~2.8k tok/s). Accuracy measured on 2×B300, TP=2 (sgl-eval, single-shot, thinking on): MMMU-Pro 77.51% (1,730 examples, stop 98.96%, truncated 1.04%), GSM8K 96.97% (stop 100%).",
+  },
 ];
@@ -1,13 +1,15 @@
 export const config = {
  modelName: "Ling-3.0-flash-VL",

-  supportedHardware: ["gb300", "b300", "b200", "h200", "h100"],
+  supportedHardware: ["gb300", "b300", "b200", "h200", "h100", "dgx-spark"],
  groupHardware: false,

  variants: [{ id: "default", label: "Ling-3.0-flash-VL" }],
  quantizations: [
    { id: "bf16", label: "BF16" },
    { id: "fp8", label: "FP8" },
+    { id: "int4", label: "INT4 (GPTQ)" },
+    { id: "fp4", label: "FP4 (MXFP4)" },
  ],
  strategies: [{ id: "balanced", label: "Balanced" }],
  nodesOptions: [{ id: "single", label: "Single Node" }],
@@ -15,12 +17,13 @@ export const config = {
  modelNames: {
    "default|bf16": "inclusionAI/Ling-3.0-flash-VL",
    "default|fp8": "inclusionAI/Ling-3.0-flash-VL-FP8",
+    "default|int4": "inclusionAI/Ling-3.0-flash-VL-int4",
+    "default|fp4": "inclusionAI/Ling-3.0-flash-VL-fp4",
  },

  placeholders: {
    HOST_IP:   { target: "command", label: "Bind host",         default: "0.0.0.0"         },
    PORT:      { target: "command", label: "Bind port",         default: "30000"           },
-    HF_TOKEN:  { target: "command", label: "HF token (Docker)", default: "<your-hf-token>" },
    CURL_HOST: { target: "curl",    label: "Server host",       default: "localhost"       },
    CURL_PORT: { target: "curl",    label: "Server port",       default: "30000"           },
  },
@@ -44,6 +47,7 @@ export const config = {
    b200:  "lmsysorg/sglang:dev-Ling-3.0-flash-VL",
    h200:  "lmsysorg/sglang:dev-Ling-3.0-flash-VL",
    h100:  "lmsysorg/sglang:dev-Ling-3.0-flash-VL",
+    "dgx-spark": "lmsysorg/sglang:dev-Ling-3.0-flash-VL",
  },

  benchmarkCommands: {
@@ -64,12 +68,18 @@ sgl-eval run mmmu_pro \\
  --model {{MODEL_NAME}} \\
  --temperature 0.0 --top-p 0.95 \\
  --num-threads 64`,
+      gsm8k_pct: `pip install sgl-eval
+sgl-eval run gsm8k \\
+  --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
+  --model {{MODEL_NAME}} \\
+  --num-threads 32`,
    },
    numPromptsByConc: { 1: 8, 16: 32, 64: 128 },
  },

  accuracyLabels: [
    ["mmmu_pro_pct", "MMMU-Pro", "%"],
+    ["gsm8k_pct", "GSM8K", "%"],
  ],

  github: {
@@ -94,7 +104,7 @@ sgl-eval run mmmu_pro \\
        "--model-path {{MODEL_NAME}}",
        "--tp 4",
        "--context-length 262144",
-        '--json-model-override-args {"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}',
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
        "--reasoning-parser auto",
        "--tool-call-parser auto",
        "--host {{HOST_IP}}",
@@ -110,10 +120,9 @@ sgl-eval run mmmu_pro \\
      flags: [
        "--trust-remote-code",
        "--model-path {{MODEL_NAME}}",
-        "--tp 4",
-        "--ep 4",
+        "--tp 1",
        "--context-length 262144",
-        '--json-model-override-args {"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}',
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
        "--reasoning-parser auto",
        "--tool-call-parser auto",
        "--host {{HOST_IP}}",
@@ -129,7 +138,7 @@ sgl-eval run mmmu_pro \\
        "--model-path {{MODEL_NAME}}",
        "--tp 4",
        "--context-length 262144",
-        '--json-model-override-args {"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}',
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
        "--reasoning-parser auto",
        "--tool-call-parser auto",
        "--host {{HOST_IP}}",
@@ -145,7 +154,7 @@ sgl-eval run mmmu_pro \\
        "--model-path {{MODEL_NAME}}",
        "--tp 4",
        "--context-length 262144",
-        '--json-model-override-args {"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}',
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
        "--reasoning-parser auto",
        "--tool-call-parser auto",
        "--host {{HOST_IP}}",
@@ -161,7 +170,7 @@ sgl-eval run mmmu_pro \\
        "--model-path {{MODEL_NAME}}",
        "--tp 4",
        "--context-length 262144",
-        '--json-model-override-args {"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}',
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
        "--reasoning-parser auto",
        "--tool-call-parser auto",
        "--host {{HOST_IP}}",
@@ -177,7 +186,7 @@ sgl-eval run mmmu_pro \\
        "--model-path {{MODEL_NAME}}",
        "--tp 8",
        "--context-length 262144",
-        '--json-model-override-args {"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}',
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
        "--reasoning-parser auto",
        "--tool-call-parser auto",
        "--host {{HOST_IP}}",
@@ -191,10 +200,9 @@ sgl-eval run mmmu_pro \\
      flags: [
        "--trust-remote-code",
        "--model-path {{MODEL_NAME}}",
-        "--tp 4",
-        "--ep 4",
+        "--tp 1",
        "--context-length 262144",
-        '--json-model-override-args {"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}',
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
        "--reasoning-parser auto",
        "--tool-call-parser auto",
        "--host {{HOST_IP}}",
@@ -208,10 +216,9 @@ sgl-eval run mmmu_pro \\
      flags: [
        "--trust-remote-code",
        "--model-path {{MODEL_NAME}}",
-        "--tp 4",
-        "--ep 4",
+        "--tp 1",
        "--context-length 262144",
-        '--json-model-override-args {"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}',
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
        "--reasoning-parser auto",
        "--tool-call-parser auto",
        "--host {{HOST_IP}}",
@@ -220,15 +227,14 @@ sgl-eval run mmmu_pro \\
    },
    {
      match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
-      verified: false,
+      verified: true,
      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
      flags: [
        "--trust-remote-code",
        "--model-path {{MODEL_NAME}}",
-        "--tp 4",
-        "--ep 4",
+        "--tp 2",
        "--context-length 262144",
-        '--json-model-override-args {"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}',
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
        "--reasoning-parser auto",
        "--tool-call-parser auto",
        "--host {{HOST_IP}}",
@@ -242,10 +248,169 @@ sgl-eval run mmmu_pro \\
      flags: [
        "--trust-remote-code",
        "--model-path {{MODEL_NAME}}",
-        "--tp 8",
-        "--ep 8",
+        "--tp 2",
        "--context-length 262144",
-        '--json-model-override-args {"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}',
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
+        "--reasoning-parser auto",
+        "--tool-call-parser auto",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "gb300", variant: "default", quant: "int4", strategy: "balanced", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--trust-remote-code",
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--context-length 262144",
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
+        "--reasoning-parser auto",
+        "--tool-call-parser auto",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "b300", variant: "default", quant: "int4", strategy: "balanced", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--trust-remote-code",
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--context-length 262144",
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
+        "--reasoning-parser auto",
+        "--tool-call-parser auto",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "b200", variant: "default", quant: "int4", strategy: "balanced", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--trust-remote-code",
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--context-length 262144",
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
+        "--reasoning-parser auto",
+        "--tool-call-parser auto",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h200", variant: "default", quant: "int4", strategy: "balanced", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--trust-remote-code",
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--context-length 262144",
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
+        "--reasoning-parser auto",
+        "--tool-call-parser auto",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h100", variant: "default", quant: "int4", strategy: "balanced", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--trust-remote-code",
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--context-length 262144",
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
+        "--reasoning-parser auto",
+        "--tool-call-parser auto",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "gb300", variant: "default", quant: "fp4", strategy: "balanced", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--trust-remote-code",
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--context-length 262144",
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
+        "--reasoning-parser auto",
+        "--tool-call-parser auto",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "b300", variant: "default", quant: "fp4", strategy: "balanced", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--trust-remote-code",
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--context-length 262144",
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
+        "--reasoning-parser auto",
+        "--tool-call-parser auto",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "b200", variant: "default", quant: "fp4", strategy: "balanced", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--trust-remote-code",
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--context-length 262144",
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
+        "--reasoning-parser auto",
+        "--tool-call-parser auto",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "dgx-spark", variant: "default", quant: "fp4", strategy: "balanced", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--trust-remote-code",
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--context-length 262144",
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
+        "--reasoning-parser auto",
+        "--tool-call-parser auto",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "dgx-spark", variant: "default", quant: "int4", strategy: "balanced", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--trust-remote-code",
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--context-length 262144",
+        '--json-model-override-args \'{"rope_scaling":{"rope_type":"yarn","factor":2.0,"rope_theta":6000000,"partial_rotary_factor":0.5,"original_max_position_embeddings":131072}}\'',
        "--reasoning-parser auto",
        "--tool-call-parser auto",
        "--host {{HOST_IP}}",