Add Ling-3.0-flash cookbook (#33556)

Co-authored-by: Zijie Xia <zijie.xia@radixark.ai>
2026-08-05 22:53:34 +08:00
parent 4e7209caa8
commit b3cdd016ba
8 changed files with 990 additions and 19 deletions
@@ -59,6 +59,7 @@
 //   dockerImages       optional — `docker run` image, keyed by
 //                      `hw|quant|strategy` then `hw|quant` then `hw`;
 //                      falls back to `lmsysorg/sglang:dev`
+//   dockerHostNetworkWhen optional — `(selection, {flags, env}) => boolean`
 //   dockerMounts       optional — additional `-v` mount specs
 //   dockerRunCommand   optional — command placed after the image and before
 //                      generated server flags; string or `(selection) => string`
@@ -100,6 +101,8 @@ export const Deployment = ({ config, benchmarks }) => {
    hopper: [
      { id: "h200",  label: "H200",  vram: "141GB" },
      { id: "h100",  label: "H100",  vram: "80GB"  },
+      { id: "h20-3e", label: "H20-3e", vram: "141GB" },
+      { id: "h800",  label: "H800",  vram: "80GB"  },
    ],
    amd: [
      { id: "mi300x", label: "MI300X", vram: "192GB" },
@@ -726,6 +729,8 @@ export const Deployment = ({ config, benchmarks }) => {
        : (config.dockerRunCommand || "sglang serve");
      const portFlag = flags.find((x) => x.split(/[\s=]/)[0] === "--port");
      const servePort = portFlag ? portFlag.slice("--port".length).trim() : "{{PORT}}";
+      const hostNetwork = multinode || (typeof config.dockerHostNetworkWhen === "function"
+        && config.dockerHostNetworkWhen(sel, { flags, env: cellEnv }));
      const vendorOf = (hwId) => {
        for (const [vendor, list] of Object.entries(HARDWARE_CATALOG)) {
          if (list.some((h) => h.id === hwId)) return vendor;
@@ -760,7 +765,7 @@ export const Deployment = ({ config, benchmarks }) => {
        // Multi-node needs host networking so the cross-node rendezvous port
        // (--dist-init-addr) and NCCL/GLOO traffic are reachable; single-node
        // just maps the serve port.
-        multinode ? "  --network host" : `  -p ${servePort}:${servePort}`,
+        hostNetwork ? "  --network host" : `  -p ${servePort}:${servePort}`,
        ...(multinode ? fabricFlagsOf(sel.hw).map((f) => "  " + f) : []),
        "  -v ~/.cache/huggingface:/root/.cache/huggingface",
        ...(config.dockerMounts || []).map((mount) => `  -v ${mount}`),
@@ -1061,12 +1061,21 @@ export const Playground = ({ config }) => {
    },

    // ---- Axis: Hierarchical KV Cache ----------------------------------------
-    // Enable + optional backend + write policy. Owns the `--hicache-*` family
-    // (unconditional strip).
+    // Enable + optional backend + write policy. `null` inherits the base cell,
+    // so a verified HiCache recipe remains byte-identical until it is changed.
    hicache: {
-      initState: () => ({ enable: false, backend: null, writePolicy: "auto" }),
+      initState: () => ({ enable: null, backend: null, writePolicy: "auto" }),

-      apply: ({ flags, env, value, fc, sel, h }) => {
+      deriveFromBase: (cell, fc, h) => {
+        const flags = (cell && cell.flags) || [];
+        return {
+          enable: h.hasFlag(flags, "--enable-hierarchical-cache"),
+          backend: h.findFlagArg(flags, "--hicache-storage-backend"),
+          writePolicy: h.findFlagArg(flags, "--hicache-write-policy") || "auto",
+        };
+      },
+
+      apply: ({ flags, env, value, fc, sel, h, derived }) => {
        if (fc.excludesHw && sel && fc.excludesHw.includes(sel.hw)) return { flags, env };
        // When the Deploy panel owns enablement (`showWhen`), the base already
        // carries a complete, verified hicache recipe. Rebuilding it from this
@@ -1083,12 +1092,34 @@ export const Playground = ({ config }) => {
          }
          return { flags, env };
        }
-        flags = h.stripFlagsByFirstToken(flags, [
+
+        const hasOverride = value.enable !== null
+          || value.backend !== null
+          || (value.writePolicy && value.writePolicy !== "auto");
+        if (!hasOverride) return { flags, env };
+
+        const backendOptions = fc.backends || [];
+        const ownedHeads = [
          "--enable-hierarchical-cache", "--hicache-ratio", "--hicache-size",
          "--hicache-write-policy", "--hicache-mem-layout", "--hicache-io-backend",
          "--hicache-storage-backend", "--hicache-storage-prefetch-policy",
-        ]);
-        if (value.enable) {
+          "--hicache-storage-backend-extra-config",
+          ...((fc.requiredFlags || []).map((f) => f.split(/\s/)[0])),
+          ...backendOptions.flatMap((o) => (o.flags || []).map((f) => f.split(/\s/)[0])),
+        ];
+        const ownedEnvKeys = [
+          ...(fc.requiredEnv || []),
+          ...backendOptions.flatMap((o) => o.env || []),
+        ].map((e) => e.split("=")[0]);
+        flags = h.stripFlagsByFirstToken(flags, ownedHeads);
+        if (ownedEnvKeys.length) env = h.stripEnvByPrefix(env, ownedEnvKeys);
+
+        const enabled = value.enable !== null
+          ? value.enable : !!(derived && derived.enable);
+        const backend = value.backend !== null
+          ? value.backend
+          : ((derived && derived.backend) || fc.defaultBackend || null);
+        if (enabled) {
          const isAmd = sel && /^mi\d/.test(sel.hw);
          const pdMode = h.findFlagArg(flags, "--disaggregation-mode") || "off";
          const pdBackend = h.findFlagArg(flags, "--disaggregation-transfer-backend");
@@ -1113,7 +1144,7 @@ export const Playground = ({ config }) => {
          if (useAmdIo) {
            adds.push(`--hicache-mem-layout ${amdIo.memLayout}`,
                      `--hicache-io-backend ${amdIo.ioBackend}`);
-          } else if (value.backend) {
+          } else if (backend) {
            adds.push("--hicache-mem-layout page_first_direct",
                      "--hicache-io-backend direct");
          }
@@ -1121,43 +1152,58 @@ export const Playground = ({ config }) => {
            ? value.writePolicy : ((amdIo && amdIo.writePolicy) || "write_through");
          adds.push(`--hicache-write-policy ${writePolicy}`);
          // When amdStorageFileOnly is set, AMD emits storage flags only for "file".
-          if ((isAmd && fc.amdStorageFileOnly) ? value.backend === "file" : !!value.backend) {
-            adds.push(`--hicache-storage-backend ${value.backend}`,
+          if ((isAmd && fc.amdStorageFileOnly) ? backend === "file" : !!backend) {
+            adds.push(`--hicache-storage-backend ${backend}`,
                      `--hicache-storage-prefetch-policy ${(amdIo && amdIo.prefetchPolicy) || "wait_complete"}`);
          } else if (amdIo && amdIo.prefetchPolicy) {
            adds.push(`--hicache-storage-prefetch-policy ${amdIo.prefetchPolicy}`);
          }
+          const backendOption = backendOptions.find((o) => o.id === backend);
+          adds.push(...(backendOption?.flags || []), ...(fc.requiredFlags || []));
          flags = h.insertBeforeTail(flags, adds);
+          env = [
+            ...env,
+            ...(backendOption?.env || []),
+            ...(fc.requiredEnv || []),
+          ];
        }
        return { flags, env };
      },

-      render: ({ axisId, value, setValue, fc, base, s, renderChip, renderSelect }) => {
+      render: ({ axisId, value, setValue, fc, base, s, renderChip, renderSelect, derived }) => {
        if (fc.excludesHw && fc.excludesHw.includes(base.hw)) return null;
        const setSlot = (k, v) => setValue({ ...value, [k]: v });
        const hasBackends = (fc.backends || []).length > 0;
        const hasPolicies = (fc.writePolicies || []).length > 0;
+        const enabled = value.enable !== null
+          ? value.enable : !!(derived && derived.enable);
+        const hasAutoBackend = (fc.backends || []).some((o) => o.id === null);
+        const backend = value.backend !== null
+          ? value.backend
+          : (hasAutoBackend ? null : ((derived && derived.backend) || fc.defaultBackend || null));
+        const writePolicy = value.writePolicy !== "auto"
+          ? value.writePolicy : ((derived && derived.writePolicy) || "auto");
        return (
          <div key={axisId} style={s.card}>
            <div style={s.compactRow}>
              <span style={s.axisTitle}>HiCache</span>
              {typeof fc.showWhen !== "function" && (
                <span style={s.field}>
-                  {renderChip("Enable", value.enable, true,
-                    () => setSlot("enable", !value.enable))}
+                  {renderChip("Enable", enabled, true,
+                    () => setSlot("enable", !enabled))}
                </span>
              )}
              {hasBackends && (
                <span style={s.field}>
                  <span style={s.fieldLabel}>Storage</span>
-                  {renderSelect(value.backend, fc.backends,
+                  {renderSelect(backend, fc.backends,
                    (v) => setSlot("backend", v), base)}
                </span>
              )}
              {hasPolicies && (
                <span style={s.field}>
                  <span style={s.fieldLabel}>Write</span>
-                  {renderSelect(value.writePolicy, fc.writePolicies,
+                  {renderSelect(writePolicy, fc.writePolicies,
                    (v) => setSlot("writePolicy", v), base)}
                </span>
              )}
@@ -1419,6 +1465,8 @@ export const Playground = ({ config }) => {
        : (config.dockerRunCommand || "sglang serve");
      const portFlag = f.find((x) => x.split(/[\s=]/)[0] === "--port");
      const servePort = portFlag ? portFlag.slice("--port".length).trim() : "{{PORT}}";
+      const hostNetwork = multinode || pdMode || (typeof config.dockerHostNetworkWhen === "function"
+        && config.dockerHostNetworkWhen(sel, { flags: f, env: cellEnv }));
      // Mirrors `multiNodeDockerFlags` on the _deployment.jsx HARDWARE_CATALOG
      // (Mintlify strips module state, so the engines cannot share it).
      const HW_MULTINODE_DOCKER_FLAGS = {
@@ -1430,7 +1478,7 @@ export const Playground = ({ config }) => {
      const dockerLines = [
        "docker run --gpus all",
        "  --shm-size 32g",
-        (multinode || pdMode) ? "  --network host" : `  -p ${servePort}:${servePort}`,
+        hostNetwork ? "  --network host" : `  -p ${servePort}:${servePort}`,
        ...(multinode ? fabricFlags.map((x) => "  " + x) : []),
        "  -v ~/.cache/huggingface:/root/.cache/huggingface",
        ...(config.dockerMounts || []).map((mount) => `  -v ${mount}`),
@@ -0,0 +1,97 @@
+// Ling-3.0-flash per-cell benchmark numbers, keyed by the same `match` tuple as
+// ling-3.0-flash.jsx cells. See _deployment.jsx for the speed/accuracy schema.
+//
+// Accuracy harness (one harness for the whole GSM8K column, per
+// config.benchmarkCommands.accuracy): sgl-eval run gsm8k, full 1319 questions,
+// --num-threads 32. Every filled entry below also recorded 100% stop /
+// 0% truncated / 0% error.
+//
+// Speed numbers are not measured yet — entries carry accuracy only.
+//
+// Cells with no entry (H20-3e / H800 / H100, both quantizations) had no matching
+// allocation and were never gated.
+export const benchmarks = [
+  // ====================================================================
+  // H200 + BF16 (TP4)
+  // ====================================================================
+  {
+    match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
+    sglang_version: "PR #33561 @ c5071ded",
+    accuracy: { gsm8k_pct: 96.59 },
+  },
+  {
+    // Rejected by the full GSM8K gate at request 1319: a no-EOS runaway generated
+    // >33k tokens. Recipe stays `verified: false` in ling-3.0-flash.jsx.
+    match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
+    accuracy: { gsm8k_pct: null },
+    notes: "Full GSM8K gate did not complete: one request ran away without emitting EOS (>33k generated tokens).",
+  },
+
+  // ====================================================================
+  // H200 + FP8 (TP4 + EP4)
+  // ====================================================================
+  {
+    match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
+    sglang_version: "PR #33561 @ e57e030b",
+    accuracy: { gsm8k_pct: 95.83 },
+  },
+  {
+    match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
+    sglang_version: "PR #33561 @ e57e030b",
+    accuracy: { gsm8k_pct: 96.51 },
+  },
+
+  // ====================================================================
+  // H200 + HiCache (Mooncake tiered cache)
+  // ====================================================================
+  {
+    match: { hw: "h200", variant: "default", quant: "bf16", strategy: "hicache", nodes: "single" },
+    sglang_version: "PR #33561 @ 51bcd89c",
+    accuracy: { gsm8k_pct: 96.44 },
+  },
+
+  // ====================================================================
+  // B200 + BF16 (TP4)
+  // ====================================================================
+  {
+    match: { hw: "b200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
+    sglang_version: "PR #33561 @ c5071ded",
+    accuracy: { gsm8k_pct: 96.44 },
+  },
+  {
+    match: { hw: "b200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
+    sglang_version: "PR #33561 @ c5071ded",
+    accuracy: { gsm8k_pct: 96.51 },
+  },
+
+  // ====================================================================
+  // B200 + FP8 (TP4 + EP4)
+  // ====================================================================
+  {
+    match: { hw: "b200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
+    sglang_version: "PR #33561 @ e57e030b",
+    accuracy: { gsm8k_pct: 96.59 },
+  },
+  {
+    match: { hw: "b200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
+    sglang_version: "PR #33561 @ e57e030b",
+    accuracy: { gsm8k_pct: 97.04 },
+  },
+
+  // ====================================================================
+  // GB300 + BF16 (TP4)
+  // ====================================================================
+  // TODO: both cells are `verified: true` (gated on the final head) but the GSM8K
+  // percentages were not recorded in the PR body or in the verifying commits —
+  // fill from the run logs.
+  { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" } },
+  { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" } },
+
+  // ====================================================================
+  // GB300 + FP8 (TP4 + EP4)
+  // ====================================================================
+  // TODO: both cells are `verified: true` on the final head; the 96.66% / 96.44%
+  // pair in the PR body predates the TP+EP change — fill with the re-measured values.
+  { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" } },
+  { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" } },
+];
@@ -0,0 +1,615 @@
+export const config = {
+  modelName: "Ling-3.0-flash",
+
+  supportedHardware: ["h20-3e", "h200", "h800", "h100", "b200", "gb300"],
+  groupHardware: false,
+
+  variants: [
+    { id: "default", label: "Ling-3.0-flash" },
+  ],
+  quantizations: [
+    { id: "bf16", label: "BF16" },
+    { id: "fp8", label: "FP8" },
+  ],
+  strategies: [
+    { id: "low-latency", label: "Low-Latency" },
+    { id: "high-throughput", label: "High-Throughput" },
+    { id: "hicache", label: "HiCache + Mooncake" },
+  ],
+  nodesOptions: [
+    { id: "single", label: "Single Node" },
+  ],
+
+  modelNames: {
+    "default|bf16": "inclusionAI/Ling-3.0-flash",
+    "default|fp8": "inclusionAI/Ling-3.0-flash-fp8",
+  },
+
+  placeholders: {
+    HOST_IP:      { target: "command", label: "Bind host",          default: "0.0.0.0"              },
+    PORT:         { target: "command", label: "Bind port",          default: "30000"                },
+    HF_TOKEN:     { target: "command", label: "HF token (Docker)",  default: "<your-hf-token>"      },
+    MOONCAKE_MASTER: { target: "command", label: "Mooncake master", default: "127.0.0.1:50171"      },
+    MOONCAKE_METADATA_SERVER: { target: "command", label: "Mooncake metadata", default: "http://127.0.0.1:8290/metadata" },
+    CURL_HOST:    { target: "curl",    label: "Server host",        default: "localhost"            },
+    CURL_PORT:    { target: "curl",    label: "Server port",        default: "30000"                },
+  },
+
+  curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\
+-H 'Content-Type: application/json' \\
+-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"What is the capital of France?"}] }'`,
+
+  dockerImages: {
+    "h20-3e": "lmsysorg/sglang:dev-Ling-3.0-flash",
+    "h200": "lmsysorg/sglang:dev-Ling-3.0-flash",
+    "h800": "lmsysorg/sglang:dev-Ling-3.0-flash",
+    "h100": "lmsysorg/sglang:dev-Ling-3.0-flash",
+    "b200": "lmsysorg/sglang:dev-Ling-3.0-flash",
+    "gb300": "lmsysorg/sglang:dev-Ling-3.0-flash",
+  },
+
+  dockerHostNetworkWhen: (_sel, { flags }) =>
+    flags.some((flag) => flag === "--hicache-storage-backend mooncake"),
+
+  benchmarkCommands: {
+    speed: `python3 -m sglang.bench_serving \\
+  --backend sglang \\
+  --host {{CURL_HOST}} --port {{CURL_PORT}} \\
+  --model {{MODEL_NAME}} \\
+  --dataset-name {{DATASET}} \\
+  --random-input-len {{ISL}} --random-output-len {{OSL}} \\
+  --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\
+  --flush-cache`,
+    accuracy: {
+      gsm8k_pct: `# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
+sgl-eval run gsm8k \\
+  --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
+  --num-threads 32`,
+    },
+  },
+
+  accuracyLabels: [
+    ["gsm8k_pct", "GSM8K", "%"],
+  ],
+
+  github: {
+    cookbookModel: "inclusionAI/Ling-3.0-flash",
+  },
+
+  playgroundFeatures: {
+    attention: {
+      knobs: [
+        { id: "tp", label: "TP", values: [null, 4, 8] },
+      ],
+    },
+    parsers: {
+      items: [
+        { id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser ling3" },
+        { id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser ling3" },
+      ],
+    },
+    speculative: {
+      options: [
+        { id: "current", label: "Inherited from base" },
+        { id: "off", label: "Off (greedy)" },
+        { id: "nextn", label: "NEXTN (built-in MTP)", flags: ["--speculative-algorithm NEXTN"] },
+      ],
+    },
+    hicache: {
+      defaultBackend: "mooncake",
+      requiredFlags: [
+        "--mamba-scheduler-strategy extra_buffer",
+        "--enable-cache-report",
+      ],
+      backends: [
+        {
+          id: "mooncake",
+          label: "Mooncake",
+          flags: [
+            "--hicache-storage-backend-extra-config '{\"hicache_storage_pass_prefix_keys\":true}'",
+          ],
+          env: [
+            "MOONCAKE_MASTER={{MOONCAKE_MASTER}}",
+            "MOONCAKE_PROTOCOL=tcp",
+            "MC_MS_AUTO_DISC=0",
+            "MOONCAKE_DEVICE=",
+            "MOONCAKE_TE_META_DATA_SERVER={{MOONCAKE_METADATA_SERVER}}",
+            "MOONCAKE_GLOBAL_SEGMENT_SIZE=0",
+          ],
+        },
+      ],
+    },
+  },
+
+  cells: [
+    {
+      match: { hw: "h20-3e", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h800", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 8",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h100", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 8",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "b200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h20-3e", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--ep-size 4",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--ep-size 4",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h800", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 8",
+        "--ep-size 8",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h100", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 8",
+        "--ep-size 8",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "b200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--ep-size 4",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--ep-size 4",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h20-3e", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--context-length 262144",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--context-length 262144",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h800", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 8",
+        "--context-length 262144",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h100", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 8",
+        "--context-length 262144",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "b200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--context-length 262144",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--context-length 262144",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h20-3e", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--ep-size 4",
+        "--context-length 262144",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--ep-size 4",
+        "--context-length 262144",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h800", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 8",
+        "--ep-size 8",
+        "--context-length 262144",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h100", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
+      verified: false,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 8",
+        "--ep-size 8",
+        "--context-length 262144",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "b200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--ep-size 4",
+        "--context-length 262144",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
+      verified: true,
+      env: ["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1"],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--ep-size 4",
+        "--context-length 262144",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+
+    // Hybrid KDA must pass prefix keys to Mooncake; otherwise storage writes are empty.
+    // Cold uncached extends above chunked_prefill_size skip write-through for that influx.
+    {
+      match: { hw: "h200", variant: "default", quant: "bf16", strategy: "hicache", nodes: "single" },
+      verified: true,
+      env: [
+        "SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1",
+        "MOONCAKE_MASTER={{MOONCAKE_MASTER}}",
+        "MOONCAKE_PROTOCOL=tcp",
+        "MC_MS_AUTO_DISC=0",
+        "MOONCAKE_DEVICE=",
+        "MOONCAKE_TE_META_DATA_SERVER={{MOONCAKE_METADATA_SERVER}}",
+        "MOONCAKE_GLOBAL_SEGMENT_SIZE=0",
+      ],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--enable-hierarchical-cache",
+        "--hicache-storage-backend mooncake",
+        "--hicache-io-backend direct",
+        "--hicache-mem-layout page_first_direct",
+        "--mamba-scheduler-strategy extra_buffer",
+        "--enable-cache-report",
+        "--hicache-storage-prefetch-policy wait_complete",
+        "--hicache-storage-backend-extra-config '{\"hicache_storage_pass_prefix_keys\":true}'",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h200", variant: "default", quant: "fp8", strategy: "hicache", nodes: "single" },
+      verified: false,
+      env: [
+        "SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1",
+        "MOONCAKE_MASTER={{MOONCAKE_MASTER}}",
+        "MOONCAKE_PROTOCOL=tcp",
+        "MC_MS_AUTO_DISC=0",
+        "MOONCAKE_DEVICE=",
+        "MOONCAKE_TE_META_DATA_SERVER={{MOONCAKE_METADATA_SERVER}}",
+        "MOONCAKE_GLOBAL_SEGMENT_SIZE=0",
+      ],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--ep-size 4",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--enable-hierarchical-cache",
+        "--hicache-storage-backend mooncake",
+        "--hicache-io-backend direct",
+        "--hicache-mem-layout page_first_direct",
+        "--mamba-scheduler-strategy extra_buffer",
+        "--enable-cache-report",
+        "--hicache-storage-prefetch-policy wait_complete",
+        "--hicache-storage-backend-extra-config '{\"hicache_storage_pass_prefix_keys\":true}'",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "hicache", nodes: "single" },
+      verified: false,
+      env: [
+        "SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1",
+        "MOONCAKE_MASTER={{MOONCAKE_MASTER}}",
+        "MOONCAKE_PROTOCOL=tcp",
+        "MC_MS_AUTO_DISC=0",
+        "MOONCAKE_DEVICE=",
+        "MOONCAKE_TE_META_DATA_SERVER={{MOONCAKE_METADATA_SERVER}}",
+        "MOONCAKE_GLOBAL_SEGMENT_SIZE=0",
+      ],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--enable-hierarchical-cache",
+        "--hicache-storage-backend mooncake",
+        "--hicache-io-backend direct",
+        "--hicache-mem-layout page_first_direct",
+        "--mamba-scheduler-strategy extra_buffer",
+        "--enable-cache-report",
+        "--hicache-storage-prefetch-policy wait_complete",
+        "--hicache-storage-backend-extra-config '{\"hicache_storage_pass_prefix_keys\":true}'",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "hicache", nodes: "single" },
+      verified: false,
+      env: [
+        "SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1",
+        "MOONCAKE_MASTER={{MOONCAKE_MASTER}}",
+        "MOONCAKE_PROTOCOL=tcp",
+        "MC_MS_AUTO_DISC=0",
+        "MOONCAKE_DEVICE=",
+        "MOONCAKE_TE_META_DATA_SERVER={{MOONCAKE_METADATA_SERVER}}",
+        "MOONCAKE_GLOBAL_SEGMENT_SIZE=0",
+      ],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 4",
+        "--ep-size 4",
+        "--context-length 262144",
+        "--speculative-algorithm NEXTN",
+        "--json-model-override-args '{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"rope_theta\":6000000,\"partial_rotary_factor\":0.5,\"original_max_position_embeddings\":131072}}'",
+        "--mem-fraction-static 0.8",
+        "--enable-hierarchical-cache",
+        "--hicache-storage-backend mooncake",
+        "--hicache-io-backend direct",
+        "--hicache-mem-layout page_first_direct",
+        "--mamba-scheduler-strategy extra_buffer",
+        "--enable-cache-report",
+        "--hicache-storage-prefetch-policy wait_complete",
+        "--hicache-storage-backend-extra-config '{\"hicache_storage_pass_prefix_keys\":true}'",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+  ],
+};