docs(cookbook): add IBM Granite 4.2 cookbook (#36286)

2026-08-25 22:54:55 +08:00
parent 46d9427b91
commit b760f7fb19
6 changed files with 545 additions and 0 deletions
@@ -0,0 +1,110 @@
+export const benchmarks = [
+  {
+    match: { hw: "h200", variant: "3b", quant: "bf16", strategy: "balanced", nodes: "single" },
+    sglang_version: "d59c1ddf7",
+    speed: [
+      {
+        workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
+        ttft_ms: 131.17,
+        tpot_ms: 3.51,
+        tokens_per_sec_per_gpu: 2472.96,
+      },
+      {
+        workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
+        ttft_ms: 1092.54,
+        tpot_ms: 7.23,
+        tokens_per_sec_per_gpu: 17416.41,
+      },
+    ],
+  },
+  {
+    match: { hw: "h200", variant: "8b", quant: "bf16", strategy: "balanced", nodes: "single" },
+    sglang_version: "d59c1ddf7",
+    speed: [
+      {
+        workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
+        ttft_ms: 260.59,
+        tpot_ms: 6.28,
+        tokens_per_sec_per_gpu: 1378.86,
+      },
+      {
+        workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
+        ttft_ms: 2168.05,
+        tpot_ms: 13.06,
+        tokens_per_sec_per_gpu: 9484.18,
+      },
+    ],
+  },
+  {
+    match: { hw: "h200", variant: "30b", quant: "bf16", strategy: "balanced", nodes: "single" },
+    sglang_version: "d59c1ddf7",
+    speed: [
+      {
+        workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
+        ttft_ms: 807.38,
+        tpot_ms: 17.14,
+        tokens_per_sec_per_gpu: 502.74,
+      },
+      {
+        workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
+        ttft_ms: 6860.18,
+        tpot_ms: 31.19,
+        tokens_per_sec_per_gpu: 3803.83,
+      },
+    ],
+  },
+  {
+    match: { hw: "b200", variant: "3b", quant: "bf16", strategy: "balanced", nodes: "single" },
+    sglang_version: "d10a656ad8",
+    speed: [
+      {
+        workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
+        ttft_ms: 74.93,
+        tpot_ms: 2.76,
+        tokens_per_sec_per_gpu: 3179.78,
+      },
+      {
+        workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
+        ttft_ms: 559.10,
+        tpot_ms: 4.66,
+        tokens_per_sec_per_gpu: 27563.95,
+      },
+    ],
+  },
+  {
+    match: { hw: "b200", variant: "8b", quant: "bf16", strategy: "balanced", nodes: "single" },
+    sglang_version: "d10a656ad8",
+    speed: [
+      {
+        workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
+        ttft_ms: 132.85,
+        tpot_ms: 4.44,
+        tokens_per_sec_per_gpu: 1969.35,
+      },
+      {
+        workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
+        ttft_ms: 1060.55,
+        tpot_ms: 8.08,
+        tokens_per_sec_per_gpu: 15800.59,
+      },
+    ],
+  },
+  {
+    match: { hw: "b200", variant: "30b", quant: "bf16", strategy: "balanced", nodes: "single" },
+    sglang_version: "d10a656ad8",
+    speed: [
+      {
+        workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
+        ttft_ms: 386.03,
+        tpot_ms: 12.01,
+        tokens_per_sec_per_gpu: 727.24,
+      },
+      {
+        workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
+        ttft_ms: 3391.21,
+        tpot_ms: 18.78,
+        tokens_per_sec_per_gpu: 6522.86,
+      },
+    ],
+  },
+];
@@ -0,0 +1,192 @@
+export const config = {
+  modelName: "Granite 4.2",
+
+  latencyPercentile: "P50",
+
+  supportedHardware: ["h200", "b200"],
+
+  variants: [
+    { id: "3b", label: "3B", subtitle: "Dense" },
+    { id: "8b", label: "8B", subtitle: "Dense" },
+    { id: "30b", label: "30B", subtitle: "Dense" },
+  ],
+  quantizations: [
+    { id: "bf16", label: "BF16" },
+  ],
+  strategies: [
+    { id: "balanced", label: "Balanced" },
+  ],
+  nodesOptions: [
+    { id: "single", label: "Single Node" },
+  ],
+
+  modelNames: {
+    "3b|bf16": "ibm-granite/granite-4.2-3b",
+    "8b|bf16": "ibm-granite/granite-4.2-8b",
+    "30b|bf16": "ibm-granite/granite-4.2-30b",
+  },
+
+  placeholders: {
+    HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" },
+    PORT: { target: "command", label: "Bind port", default: "30000" },
+    CURL_HOST: { target: "curl", label: "Server host", default: "localhost" },
+    CURL_PORT: { target: "curl", label: "Server port", default: "30000" },
+  },
+
+  curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\
+-H 'Content-Type: application/json' \\
+-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`,
+
+  benchmarkCommands: {
+    speed:
+`python -m sglang.benchmark.serving \\
+  --backend sglang-oai \\
+  --host {{CURL_HOST}} --port {{CURL_PORT}} \\
+  --model {{MODEL_NAME}} --tokenizer {{MODEL_NAME}} \\
+  --dataset-name {{DATASET}} \\
+  --random-input-len {{ISL}} --random-output-len {{OSL}} \\
+  --random-range-ratio 1.0 \\
+  --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\
+  --warmup-requests 8 --flush-cache \\
+  --temperature 0.0 --top-p 1.0 \\
+  --seed 123 --disable-tqdm --output-details`,
+    numPromptsByConc: { 1: 80, 16: 80 },
+  },
+
+  dockerImages: {
+    h200: "lmsysorg/sglang:dev",
+    b200: "lmsysorg/sglang:dev",
+  },
+
+  github: {
+    cookbookModel: "ibm-granite/granite-4.2-3b",
+  },
+
+  playgroundFeatures: {
+    attention: {
+      knobs: [
+        { id: "tp", label: "TP", values: [null, 1, 2, 4, 8] },
+      ],
+    },
+    parsers: {
+      items: [
+        { id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser nemotron_3" },
+        { id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser qwen3_coder" },
+      ],
+    },
+    pdDisagg: {
+      modes: [
+        { id: "off", label: "Off" },
+        { id: "prefill", label: "Prefill role" },
+        { id: "decode", label: "Decode role" },
+      ],
+      transferBackends: [
+        { id: "mooncake", label: "Mooncake" },
+        { id: "nixl", label: "NiXL" },
+      ],
+      ibDevices: [{ id: "auto", label: "Auto" }, "mlx5_0", "mlx5_7"],
+    },
+    hicache: {
+      backends: [
+        { id: null, label: "Auto" },
+        { id: "file", label: "File" },
+        { id: "mooncake", label: "Mooncake" },
+        { id: "hf3fs", label: "HF3FS" },
+        { id: "nixl", label: "NiXL" },
+      ],
+      writePolicies: [
+        { id: "auto", label: "Auto" },
+        { id: "write_through", label: "Write-through" },
+        { id: "write_back", label: "Write-back" },
+        { id: "write_through_selective", label: "Write-through (selective)" },
+      ],
+    },
+  },
+
+  cells: [
+    {
+      match: { hw: "h200", variant: "3b", quant: "bf16", strategy: "balanced", nodes: "single" },
+      verified: true,
+      env: [],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--mem-fraction-static 0.8",
+        "--reasoning-parser nemotron_3",
+        "--tool-call-parser qwen3_coder",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h200", variant: "8b", quant: "bf16", strategy: "balanced", nodes: "single" },
+      verified: true,
+      env: [],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--mem-fraction-static 0.8",
+        "--reasoning-parser nemotron_3",
+        "--tool-call-parser qwen3_coder",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h200", variant: "30b", quant: "bf16", strategy: "balanced", nodes: "single" },
+      verified: true,
+      env: [],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--mem-fraction-static 0.8",
+        "--reasoning-parser nemotron_3",
+        "--tool-call-parser qwen3_coder",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "b200", variant: "3b", quant: "bf16", strategy: "balanced", nodes: "single" },
+      verified: true,
+      env: [],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--mem-fraction-static 0.8",
+        "--reasoning-parser nemotron_3",
+        "--tool-call-parser qwen3_coder",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "b200", variant: "8b", quant: "bf16", strategy: "balanced", nodes: "single" },
+      verified: true,
+      env: [],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--mem-fraction-static 0.8",
+        "--reasoning-parser nemotron_3",
+        "--tool-call-parser qwen3_coder",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "b200", variant: "30b", quant: "bf16", strategy: "balanced", nodes: "single" },
+      verified: true,
+      env: [],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 1",
+        "--mem-fraction-static 0.8",
+        "--reasoning-parser nemotron_3",
+        "--tool-call-parser qwen3_coder",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+  ],
+};