minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Cookbook renovation (#26885)

Browse Source 
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
zijiexia
2026-06-07 22:04:54 -07:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 6365d6faee
commit d1777d1f6d
16 changed files with 6692 additions and 1693 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.claude/skills/cookbook-add-model/templates/benchmarks.jsx.tmpl
+29
View File
@@ -0,0 +1,29 @@
// TEMPLATE — instantiate via the cookbook-add-model skill. NOT a live cookbook.
// Copy to docs_new/src/snippets/configs/<hf-org>/<model-slug>-benchmarks.jsx and
// fill measured numbers — OR delete this file entirely if you have none yet (the
// MDX simply omits the `benchmarks` import/prop).
//
// One entry per cell `match` tuple (same 5 keys as config cells). The card stays
// "pending" until an entry has a non-null speed metric or accuracy. Speed shape:
// speed: [{ workload: {dataset, isl, osl, max_concurrency}, ttft_ms, tpot_ms,
// tokens_per_sec_per_gpu }, ...] // interactivity is derived (1000/TPOT)
// Per-cell `accuracy: { <key>: <pct> }` overrides the config's defaultAccuracy.
export const benchmarks = [
// EXAMPLE — one filled entry showing the shape; replace numbers, add one per cell.
{
match: { hw: "b200", variant: "default", quant: "fp4", strategy: "low-latency", nodes: "single" },
sglang_version: "0.0.0", // TODO: ASK the user for the sglang version these numbers were measured on — don't invent one
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: null, tpot_ms: null, tokens_per_sec_per_gpu: null },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: null, tpot_ms: null, tokens_per_sec_per_gpu: null },
],
},
// Bare-match stubs (no data yet) are fine — the card shows "pending" for these.
{ match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" } },
{ match: { hw: "h100", variant: "default", quant: "fp4", strategy: "high-throughput", nodes: "single" } },
{ match: { hw: "mi300x", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" } },
{ match: { hw: "b200", variant: "default", quant: "fp4", strategy: "high-throughput", nodes: "multi-2" } },
];
.claude/skills/cookbook-add-model/templates/config.jsx.tmpl
+374
View File
@@ -0,0 +1,374 @@
// TEMPLATE — instantiate via the cookbook-add-model skill. NOT a live cookbook.
// Copy to docs_new/src/snippets/configs/<hf-org>/<model-slug>.jsx, then:
// 1. replace every __TOKEN__,
// 2. fill cells[] with your verified recipes (the examples below show the shape),
// 3. DELETE the hardware / playground axes / quantizations your model lacks.
//
// Instantiation tokens (skill fills these; distinct from the engine's runtime
// {{PLACEHOLDER}} which MUST survive verbatim into the output):
// __MODEL_DISPLAY__ display name, e.g. "DeepSeek-V4"
// __MODEL_SLUG__ file slug, e.g. "deepseek-v4"
// __HF_ORG__ HuggingFace org, e.g. "deepseek-ai"
// __HF_REPO__ HuggingFace repo, e.g. "DeepSeek-V4-Flash"
// __REASONING_PARSER__ e.g. "deepseek-v4" (delete the parsers axis if none)
// __TOOLCALL_PARSER__ e.g. "deepseekv4" (delete the parsers axis if none)
//
// Mintlify: single `export const config = {...}` literal — no spreads/calls/IIFE,
// no `!(x in y)`. Cells are denormalized: no --nnodes/--node-rank/--dist-init-addr/
// --host/--port literals (the engine injects them).
export const config = {
modelName: "__MODEL_DISPLAY__",
// List ONLY hardware you ship a cell for; unlisted ids auto-grey-out. The full
// catalog is below — delete the families your model doesn't support (e.g. drop
// every `mi*` if there's no AMD recipe).
supportedHardware: [
"h100", "h200", "b200", "b300", "gb200", "gb300",
"mi300x", "mi325x", "mi350x", "mi355x",
],
// OPTIONAL — declare GPUs the shared HARDWARE_CATALOG (in _deployment.jsx) doesn't
// carry (workstation / desktop / future chips). The engine merges these in, so a
// model-specific GPU is config data, never an engine-catalog edit. Add the id to
// supportedHardware above too. Delete if you only use catalog GPUs.
// hardware: [
// { id: "rtx6000", label: "RTX PRO 6000", vram: "96GB", vendor: "nvidia" },
// ],
// 2nd dim. Single-element `default` if the model has no variant axis; else list
// real variants (e.g. {id:"flash",...},{id:"pro",...}) and key modelNames/
// defaultAccuracy by them.
variants: [
{ id: "default", label: "Default" },
],
// 3rd dim. Keep only what your model ships (BF16 / FP8 / FP4 / …).
quantizations: [
{ id: "bf16", label: "BF16" },
{ id: "fp8", label: "FP8" },
{ id: "fp4", label: "FP4" },
],
strategies: [
{ id: "low-latency", label: "Low-Latency" },
{ id: "balanced", label: "Balanced" },
{ id: "high-throughput", label: "High-Throughput" },
],
// `multi-N` id carries the node count for `--nnodes N`.
nodesOptions: [
{ id: "single", label: "Single Node" },
{ id: "multi-2", label: "Multi-Nodes" },
],
// HF slug lookup. Key by `variant|quant` (or `hw|variant|quant` for a per-hw
// repackaging, e.g. an FP8 conversion only valid on one platform).
modelNames: {
"default|bf16": "__HF_ORG__/__HF_REPO__",
"default|fp8": "__HF_ORG__/__HF_REPO__",
"default|fp4": "__HF_ORG__/__HF_REPO__",
},
placeholders: {
HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" },
PORT: { target: "command", label: "Bind port", default: "30000" },
NODE0_IP: { target: "command", label: "Head node IP", default: "<node0-ip>" },
NODE_RANK: { target: "command", label: "This node rank", default: "<node-rank>" },
HF_TOKEN: { target: "command", label: "HF token (Docker)", default: "<your-hf-token>" },
CURL_HOST: { target: "curl", label: "Server host", default: "localhost" },
CURL_PORT: { target: "curl", label: "Server port", default: "30000" },
},
curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\
-H 'Content-Type: application/json' \\
-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`,
// OPTIONAL — powers the benchmark card's "⚡ Reproduce" modal. Delete the whole
// block (and the benchmarks file) if you have no measured numbers yet.
benchmarkCommands: {
speed:
`python3 -m sglang.bench_serving \\
--backend sglang \\
--host {{CURL_HOST}} --port {{CURL_PORT}} \\
--model {{MODEL_NAME}} \\
--dataset-name {{DATASET}} \\
--random-input-len {{ISL}} --random-output-len {{OSL}} \\
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}}`,
// One entry per accuracy field. A value is a string, OR a {[variant]: string}
// object when the command differs per variant. Keys must match ACCURACY_LABELS
// in _deployment.jsx + the per-cell/defaultAccuracy keys.
accuracy: {
gsm8k_pct:
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 32`,
},
// {{NUM_PROMPTS}} fallback per concurrency (else max(c*2, 200)).
numPromptsByConc: { 1: 8, 16: 32, 64: 128, 256: 512, 1024: 2048, 4096: 4096 },
},
// OPTIONAL — per-variant accuracy applied to EVERY cell of a variant (hardware-
// independent, e.g. GPQA/AIME). Per-cell `accuracy` overrides. Keys must match
// ACCURACY_LABELS + benchmarkCommands.accuracy. Delete if no numbers yet.
defaultAccuracy: {
default: { gsm8k_pct: null },
},
// OPTIONAL — `# ...` hint lines prepended to multi-node commands, ONLY for hw
// whose fabric needs manual NIC env (e.g. gb200 NVL72/MNNVL). NOT every multi-N
// hw needs this — standard-IB DeepEP / Marlin multi-node don't. Delete if unused.
multiNodeHints: {
gb200: [
"The following env vars may be needed depending on your cluster:",
" GLOO_SOCKET_IFNAME=<your-nic>",
" NVSHMEM_ENABLE_NIC_PE_MAPPING=1",
" NVSHMEM_HCA_LIST=<your-hca-list>",
],
},
// Per-hw image for `docker run` framing. ASK the user which sglang build the recipes ran
// on; don't guess a supporting release. Default below is :dev (nightly) — replace the tag
// with the user's release if they give one. NVIDIA share one image; AMD uses ROCm tags.
// GB200/GB300/B300 may need a `-cu130` (CUDA 13) tag — confirm per release.
dockerImages: {
h100: "lmsysorg/sglang:dev",
h200: "lmsysorg/sglang:dev",
b200: "lmsysorg/sglang:dev",
b300: "lmsysorg/sglang:dev",
gb200: "lmsysorg/sglang:dev",
gb300: "lmsysorg/sglang:dev",
mi300x: "lmsysorg/sglang:dev-rocm720-mi30x",
mi325x: "lmsysorg/sglang:dev-rocm720-mi30x",
mi350x: "lmsysorg/sglang:dev-rocm720-mi35x",
mi355x: "lmsysorg/sglang:dev-rocm720-mi35x",
},
// Pre-selects the issue template's `model` dropdown on "Submit verified cell".
// Must match that dropdown's value (usually `<hf-org>/<model-slug>`).
github: {
cookbookModel: "__HF_ORG__/__MODEL_SLUG__",
},
// Opt-in per axis. DELETE any axis your model doesn't expose (don't leave a stub).
playgroundFeatures: {
// ----- Card: "Attention Parallelism" ----- KEEP if the model exposes TP/CP/DP
// knobs. DP-Attention is a combined knob: value = DP degree AND toggles `--enable-dp-attention`.
attention: {
knobs: [
{ id: "tp", label: "TP", values: [
null, 1, 2, 4, 8,
{ value: 16, disable: { nodes: ["single"] },
disableReason: "TP=16 requires 16 ranks — switch the Deploy panel's Nodes to Multi-Nodes first." },
]},
{ id: "cp", label: "CP", values: [null, 1, 2, 4] },
{ id: "dpAttn", label: "DP-Attention",
values: [
null, false, 1, 2, 4, 8,
{ value: 16, disable: { nodes: ["single"] },
disableReason: "DP-Attention=16 requires 16 ranks — switch the Deploy panel's Nodes to Multi-Nodes first." },
],
labels: { "auto": "Auto", "false": "Off" } },
],
},
// ----- Card: "MoE Parallelism" ----- KEEP if MoE + multiple `--moe-*-backend`
// choices. DELETE for dense models.
moe: {
backend: {
options: [
{ id: null, label: "Inherited" },
{ id: "deepep", label: "DeepEP", flags: ["--moe-a2a-backend deepep"] },
{ id: "flashinfer_mxfp4", label: "FlashInfer (MXFP4)", flags: ["--moe-runner-backend flashinfer_mxfp4"] },
{ id: "marlin", label: "Marlin (W4A16)", flags: ["--moe-runner-backend marlin"] },
],
},
ep: { label: "EP", values: [
null, 1, 2, 4, 8,
{ value: 16, disable: { nodes: ["single"] },
disableReason: "EP=16 requires 16 ranks — switch the Deploy panel's Nodes to Multi-Nodes first." },
]},
},
// ----- Card: "Parsers" ----- KEEP if the model has reasoning / tool-call
// parsers (set the slugs below). DELETE the axis if neither applies.
parsers: {
items: [
{ id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser __REASONING_PARSER__" },
{ id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser __TOOLCALL_PARSER__" },
],
},
// ----- Card: "Speculative Decoding" ----- KEEP if the model has spec-decoding
// presets. Drop options the model doesn't support.
speculative: {
options: [
{ id: "current", label: "Inherited from base" },
{ id: "off", label: "Off (greedy)" },
{ id: "mtp", label: "EAGLE / MTP",
flags: ["--speculative-algorithm EAGLE", "--speculative-num-steps 3",
"--speculative-eagle-topk 1", "--speculative-num-draft-tokens 4"] },
{ id: "ngram", label: "NGRAM",
flags: ["--speculative-algorithm NGRAM",
"--speculative-num-draft-tokens 16",
"--speculative-ngram-max-bfs-breadth 10"],
disable: { dpAttnOn: [true] },
disableReason: "NGRAM is incompatible with DP-Attention. Turn DP-Attention off in the Attention card above to use NGRAM." },
],
},
// ----- Card: "PD Disaggregation" ----- KEEP if the model supports prefill/
// decode disaggregation. Delete `router` if you have no router topology.
pdDisagg: {
modes: [
{ id: "off", label: "Off" },
{ id: "prefill", label: "Prefill role" },
{ id: "decode", label: "Decode role" },
],
transferBackends: [
{ id: "mooncake", label: "Mooncake",
env: ["NCCL_MNNVL_ENABLE=1", "NCCL_CUMEM_ENABLE=1"],
envWhen: { hw: ["gb200", "gb300"] } },
{ id: "nixl", label: "NiXL" },
],
// `auto` is a sentinel (emits no --disaggregation-ib-device flag).
ibDevices: [{ id: "auto", label: "Auto" }, "mlx5_0", "mlx5_7"],
// Router fronting prefill + decode; substitute <prefill-host>/<decode-host>.
router: {
port: 8000,
command:
`python3 -m sglang_router.launch_router \\
--pd-disaggregation \\
--prefill http://<prefill-host>:30000 \\
--decode http://<decode-host>:30001 \\
--host 0.0.0.0 --port 8000 \\
--disable-circuit-breaker \\
--health-check-interval-secs 999999`,
},
},
// ----- Card: "Hierarchical KV Cache" ----- KEEP if the model is large enough
// that hierarchical KV caching matters.
hicache: {
backends: [
{ id: null, label: "Auto" },
{ id: "file", label: "File" },
{ id: "mooncake", label: "Mooncake" },
{ id: "hf3fs", label: "HF3FS" },
{ id: "nixl", label: "NiXL" },
],
writePolicies: [
{ id: "auto", label: "Auto" },
{ id: "write_through", label: "Write-through" },
{ id: "write_back", label: "Write-back" },
{ id: "write_through_selective", label: "Write-through (selective)" },
],
},
// ----- Card: "HiSparse" ----- KEEP only for DSA-style sparse-attention models
// (DeepSeek-V3.2/V4, GLM-5). Decode-only: shown when live PD-Disagg mode is `decode`.
hisparse: {
requiredFlags: ["--disable-radix-cache"],
config: { top_k: 2048, device_buffer_size: 6144 },
hostRatios: [
{ id: 5, label: "5 (~1TB host)" },
{ id: 10, label: "10 (~2TB host)" },
],
defaultHostRatio: 10,
},
// ----- Card: "MegaMoE" ----- KEEP only for Blackwell MoE kernel-fusion models.
megamoe: {
requiresHw: ["b200", "b300", "gb200", "gb300"],
excludesStrategy: ["low-latency", "balanced"],
stripEnv: ["SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK"],
options: [
{ id: "disabled", label: "Disabled" },
{ id: "w4a8", label: "W4A8",
flags: ["--moe-a2a-backend megamoe"],
env: ["SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK=8320"] },
],
},
},
// EXAMPLE cells — one per hardware family to show the shape. REPLACE each with
// your model's verified recipe, or DELETE families you don't support. `match`
// MUST have exactly the 5 keys; env/flags are flat literals.
cells: [
// ==== NVIDIA Blackwell + FP4 (single node) ====
{
match: { hw: "b200", variant: "default", quant: "fp4", strategy: "low-latency", nodes: "single" },
verified: true, // EXAMPLE — set false / replace with your verified recipe
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 4",
"--moe-runner-backend flashinfer_mxfp4",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
// ==== NVIDIA Hopper + FP8 (single node, DP-attention + DeepEP) ====
{
match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
verified: true, // EXAMPLE
env: ["SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=256"],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 4",
"--dp 4",
"--enable-dp-attention",
"--moe-a2a-backend deepep",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
// ==== NVIDIA Hopper + FP4 (single node, Marlin W4A16 — Hopper has no FP4 runner) ====
{
match: { hw: "h100", variant: "default", quant: "fp4", strategy: "high-throughput", nodes: "single" },
verified: true, // EXAMPLE
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 8",
"--moe-runner-backend marlin",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
// ==== AMD + BF16 (single node) — Triton attention + AITER; EP == TP for MoE ====
{
match: { hw: "mi300x", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
verified: true, // EXAMPLE
env: ["SGLANG_USE_AITER=1", "SGLANG_ROCM_FUSED_DECODE_MLA=0"],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 8",
"--ep 8",
"--attention-backend triton",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
// ==== Multi-node example (2 nodes, TP=16) — engine injects --nnodes/--node-rank/
// --dist-init-addr from match.nodes; do NOT add them here. ====
{
match: { hw: "b200", variant: "default", quant: "fp4", strategy: "high-throughput", nodes: "multi-2" },
verified: true, // EXAMPLE
env: [],
flags: [
"--trust-remote-code",
"--model-path {{MODEL_NAME}}",
"--tp 16",
"--dp 16",
"--enable-dp-attention",
"--moe-a2a-backend deepep",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
],
};
.claude/skills/cookbook-add-model/templates/page.mdx.tmpl
+148
View File
@@ -0,0 +1,148 @@
---
title: __MODEL_DISPLAY__
description: "__ONE_LINER__"
tag: NEW
mode: wide
---
{/* TEMPLATE — instantiate via the cookbook-add-model skill, then DELETE this banner.
(Frontmatter MUST stay the first thing in the file, so this note lives below it.)
Replace every __TOKEN__, fill the TODO prose, delete the §3 subsections your model
lacks. Tokens: __MODEL_DISPLAY__ __ONE_LINER__ __HF_ORG__ __MODEL_SLUG__ __HF_REPO__
__REASONING_PARSER__ __TOOLCALL_PARSER__. MDX rules (JSX tables, labeled fences, no
Docusaurus/@site/GitHub-alert/pipe-tables):
.claude/skills/cookbook-add-model/references/mintlify-authoring.md */}
## Deployment
<a id="install" />
<Accordion title="Install SGLang">
For all methods and hardware platforms, see the [official SGLang installation guide](../../../docs/get-started/install). The two paths below match the **Python / Docker** toggle in the command panel.
<Tabs>
<Tab title="Python (pip / uv)">
```bash Command
pip install --upgrade pip
pip install uv
uv pip install sglang
```
Then run the **Python** output of the command panel below in that environment.
</Tab>
<Tab title="Docker">
```bash Command
docker pull lmsysorg/sglang:latest
```
For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces.
</Tab>
</Tabs>
</Accordion>
Pick your hardware + recipe to generate the launch command. The three serving strategies cover the common operating points:
- **Low-Latency** — fastest reply for a single user. Pick for chat.
- **Balanced** — good speed with several users at once. Use for typical multi-user serving.
- **High-Throughput** — most tokens per second across many users. Best for batch jobs.
import { Deployment } from "/src/snippets/_deployment.jsx";
import { config } from "/src/snippets/configs/__HF_ORG__/__MODEL_SLUG__.jsx";
import { benchmarks } from "/src/snippets/configs/__HF_ORG__/__MODEL_SLUG__-benchmarks.jsx";
<Deployment config={config} benchmarks={benchmarks} />
## Playground
The Playground is where you experiment with **SGLang features beyond the verified matrix**. The Deploy panel above only emits combinations the SGLang team has signed off on; the Playground lets you turn on additional knobs on top of whichever cell the Deploy panel is currently showing.
import { Playground } from "/src/snippets/_playground.jsx";
<Playground config={config} />
## 1. Model Introduction
{/* TODO: 1-2 paragraph intro from the HF card — what the model is, release date,
license, architecture highlights, context length. Keep it lean. */}
**__MODEL_DISPLAY__** is __ONE_LINER__.
{/* TODO: variants table (JSX, NOT a markdown pipe table). Drop the table if there's
a single variant and inline the HF link in the intro paragraph above instead. */}
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
<thead>
<tr style={{borderBottom: "2px solid #d55816"}}>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>Variant</th>
<th style={{textAlign: "right", padding: "10px 12px", fontWeight: 700}}>Total params</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>Use</th>
</tr>
</thead>
<tbody>
<tr>
<td style={{padding: "9px 12px"}}><strong><a href="https://huggingface.co/__HF_ORG__/__HF_REPO__">__MODEL_DISPLAY__</a></strong></td>
<td style={{padding: "9px 12px", textAlign: "right"}}>TODO</td>
<td style={{padding: "9px 12px"}}>TODO</td>
</tr>
</tbody>
</table>
**Recommended generation:** {/* TODO e.g. `temperature=1.0`, `top_p=1.0` (informational; do NOT hardcode in sample code) */}
**Resources:** [HuggingFace](https://huggingface.co/__HF_ORG__/__HF_REPO__).
## 2. Configuration Tips
{/* TODO: model/hardware-specific tuning notes, caveats, known issues. Delete if none. */}
## 3. Advanced Usage
{/* Keep only the subsections that apply. Each runnable block is followed by an
**Output Example:** + a ```text Output block with REAL server output. */}
### 3.1 Reasoning
Enable the `__REASONING_PARSER__` reasoning parser (toggle **Reasoning Parser** in the **Parsers** card of the [Playground above](#playground)) to separate thinking from the final answer.
{/* This example assumes a SEPARATE-FIELD parser (thinking → `reasoning_content`,
answer → `content`). If your parser emits inline `<think>...</think>` tags inside
`content`, parse the tags from `content` instead. */}
```python Example
from openai import OpenAI
client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY")
resp = client.chat.completions.create(
model="__HF_ORG__/__HF_REPO__",
messages=[{"role": "user", "content": "What is 15% of 240?"}],
extra_body={"chat_template_kwargs": {"thinking": True}},
)
msg = resp.choices[0].message
print("Reasoning:", getattr(msg, "reasoning_content", None))
print("Answer:", msg.content)
```
**Output Example:**
```text Output
TODO: paste real server output here.
```
### 3.2 Tool Calling
Enable the `__TOOLCALL_PARSER__` tool-call parser (toggle **Tool Call Parser** in the **Parsers** card of the [Playground above](#playground)) to surface structured tool calls via `message.tool_calls`.
{/* TODO: tool-calling example + **Output Example:**. On thinking-mode models the
follow-up may put text in `reasoning_content`; print both that and `content`. */}
### 3.3 HiCache (Hierarchical KV Caching)
{/* TODO: keep only if the model is large enough for hierarchical KV caching; link
the HiCache card in the Playground. Otherwise delete this subsection. */}