diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx index e1a624461..14afc85c9 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx @@ -2,7 +2,6 @@ title: GLM-5.1 metatags: description: "Deploy GLM-5.1 with SGLang on NVIDIA H100/H200/B300/GB300 and AMD MI300X/MI325X/MI355X." -tag: NEW --- ## 1. Model Introduction diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx new file mode 100644 index 000000000..0ef366ce8 --- /dev/null +++ b/docs_new/cookbook/autoregressive/GLM/GLM-5.2.mdx @@ -0,0 +1,195 @@ +--- +title: GLM-5.2 +description: "Deploy GLM-5.2 with SGLang — Z.ai's DeepSeek-Sparse-Attention (DSA) Mixture-of-Experts model with MTP speculative decoding and 1M context, on H200, B200, and GB300." +tag: NEW +--- + +## Deployment + + + + + +For all methods and hardware platforms, see the [official SGLang installation guide](../../../docs/get-started/install). The two paths below match the **Python / Docker** toggle in the command panel. + + + + + +```bash Command +pip install --upgrade pip +pip install uv +uv pip install sglang +``` + +Then run the **Python** output of the command panel below in that environment. + + + + + +```bash Command +docker pull lmsysorg/sglang:latest +``` + +For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces. + + + + + + + +Pick your hardware + recipe to generate the launch command. The three serving strategies cover the common operating points: + +- **Low-Latency** — fastest reply for a single user. Pick for chat. +- **Balanced** — good speed with several users at once. Use for typical multi-user serving. +- **High-Throughput** — most tokens per second across many users. Best for batch jobs. + +import { Deployment } from "/src/snippets/_deployment.jsx"; +import { config } from "/src/snippets/configs/zai-org/glm-5.2.jsx"; +import { benchmarks } from "/src/snippets/configs/zai-org/glm-5.2-benchmarks.jsx"; + + + +## Playground + +The Playground is where you experiment with **SGLang features beyond the verified matrix**. The Deploy panel above only emits combinations the SGLang team has signed off on; the Playground lets you turn on additional knobs on top of whichever cell the Deploy panel is currently showing. + +import { Playground } from "/src/snippets/_playground.jsx"; + + + +## 1. Model Introduction + +**GLM-5.2** is Z.ai's flagship Mixture-of-Experts model built on **DeepSeek Sparse Attention (DSA)**: a lightning indexer selects a sparse set of key tokens per query (top-2048), so attention cost stays near-constant as context grows. It ships in two precisions — **FP8** (`zai-org/GLM-5.2-FP8`) and full **BF16** (`zai-org/GLM-5.2`) — both with **78 transformer layers**, **256 routed experts** (8 active per token), a **1M-token context window**, and a single **MTP (Multi-Token Prediction)** layer for built-in EAGLE-style speculative decoding. FP8 is the recommended deployment; BF16 (~1.5 TB) needs an 8×B300 node or a multi-node setup. + + + + + + + + + + + + + + + + + + + + + +
ModelArchitectureContext
GLM-5.2-FP8MoE · DSA · 256 experts (top-8) · MTP · FP81,048,576
GLM-5.2MoE · DSA · 256 experts (top-8) · MTP · BF161,048,576
+ +**Recommended generation:** `temperature=1.0`, `top_p=0.95` (the checkpoint's `generation_config.json` defaults; informational — do not hardcode in client code). + +**Resources:** [GLM-5.2-FP8](https://huggingface.co/zai-org/GLM-5.2-FP8) · [GLM-5.2 (BF16)](https://huggingface.co/zai-org/GLM-5.2). + +## 2. Configuration Tips + +- **DeepSeek Sparse Attention (DSA).** GLM-5.2 uses the `glm_moe_dsa` architecture; SGLang auto-selects the DSA attention backends (`flashmla_sparse` prefill, `fa3` decode, `sgl-kernel` indexer topk). No attention-backend flag is needed on the supported hardware. +- **MTP / speculative decoding.** The checkpoint ships one nextn layer. Enable EAGLE MTP for lower latency (`--speculative-algorithm EAGLE --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4` for low-latency; `1-1-2` for balanced). The config's `index_share_for_mtp_iteration` reuses the DSA indexer's topk across draft steps (effective only at `--speculative-eagle-topk 1`). +- **Context Parallelism (CP) for long prefill.** DSA prefill CP splits the long-prefill attention across `--attn-cp-size` ranks. On **Hopper (H200)** this gives a large prefill-latency win at long context — e.g. round-robin CP (`--tp 8 --attn-cp-size 8 --enable-dsa-prefill-context-parallel --dsa-prefill-cp-mode round-robin-split`) cut 64K-token prefill TTFT roughly **2.5–2.8×** vs. plain TP8 in our testing. Trade-offs: CP partitions the KV pool (lower max context at the same `--mem-fraction-static`) and adds some decode-side overhead, so it pays off only for long sequences. **CP is currently verified on Hopper only** — the Blackwell (sm100) DSA-CP FP8 rope kernel is not yet adapted, so leave CP off on B200/GB300. +- **Memory.** The FP8 weights are large (MoE total, not active params). Start around `--mem-fraction-static 0.8` on H200 (TP8) and tune up; raise it for the 4-GPU GB300 single-node layout (TP4). +- **DP-Attention + DeepEP** for the balanced/high-throughput strategies spreads attention across data-parallel ranks and routes MoE through DeepEP. +- **BF16 weights need more GPUs (unverified).** The full-precision build (`zai-org/GLM-5.2`, ~1.5 TB) does not fit a single 8×H200 / 8×B200 / 4×GB300 node. It fits single-node on **8×B300** (TP8, ~2.1 TB HBM); on the smaller GPUs it needs a **multi-node** layout (e.g. 2×8×H200 or 2×8×B200 at TP16, 2×4×GB300 at TP8). The BF16 recipes in the panel are **proposed/inferred, not yet benchmarked** (`verified: false`) — FP8 is the recommended deployment. Use the same DSA / MTP / chunked-prefill guidance as FP8. +- **Chunked-prefill size is regime-dependent.** At long input (8K+) the default `--chunked-prefill-size 2048` is too small and leaves the balanced point prefill-bound (queueing dominates TTFT). Raising it to `--chunked-prefill-size 32768` on the balanced recipe gave roughly **+34–78% output throughput and −39–59% TTFT** on 8×H200 and 8×B200 (8K-in / 1K-out) in our testing. It is **neutral for high-throughput** (decode-bound there) — keep the default. `--max-running-requests` tracks KV capacity, not a tuning free-for-all: ~60–90 concurrent 8K+1K FP8 requests fit on a single 8-GPU node, so pin balanced near `--max-running-requests 80` and let high-throughput run wider. + +## 3. Advanced Usage + +### 3.1 Reasoning + +GLM-5.2 is a hybrid-reasoning model. Enable the `glm45` reasoning parser (toggle **Reasoning Parser** in the **Parsers** card of the [Playground above](#playground)) to separate thinking from the final answer — thinking lands in `message.reasoning_content`, the answer in `message.content`. Thinking is on by default; turn it off with `chat_template_kwargs: {"thinking": False}`. + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") +resp = client.chat.completions.create( + model="zai-org/GLM-5.2-FP8", + messages=[{"role": "user", "content": "What is 15% of 240?"}], + extra_body={"chat_template_kwargs": {"thinking": True}}, +) +msg = resp.choices[0].message +print("Reasoning:", getattr(msg, "reasoning_content", None)) +print("Answer:", msg.content) +``` + + + + + +```text Output +Reasoning: 1. **Identify the core question:** The user wants to find 15% of 240. +2. **Convert the percentage to a decimal:** 15% = 0.15 +3. **Multiply by the total:** 0.15 * 240 = 36 + (Quick mental math: 10% of 240 = 24; 5% = 12; 24 + 12 = 36.) + +Answer: 15% of 240 is **36**. + +Here is how you can calculate it: +0.15 × 240 = 36 +``` + + + +### 3.2 Tool Calling + +Enable the `glm47` tool-call parser (toggle **Tool Call Parser** in the **Parsers** card of the [Playground above](#playground)) to surface structured tool calls via `message.tool_calls`. GLM-5.2 emits the newer `…` format, so it needs the **`glm47`** parser — the older `glm45` parser does not parse it (the call would be left as raw text in `content`). On thinking mode the turn also fills `reasoning_content`, so print both fields. + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") +tools = [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, +}] +resp = client.chat.completions.create( + model="zai-org/GLM-5.2-FP8", + messages=[{"role": "user", "content": "What's the weather in Paris?"}], + tools=tools, +) +msg = resp.choices[0].message +print("Reasoning:", getattr(msg, "reasoning_content", None)) +print("Tool calls:", msg.tool_calls) +``` + + + + + +```text Output +Reasoning: The user wants to know the weather in Paris. I'll call the get_weather function with "Paris" as the city. + +Tool calls: [ + { + "id": "call_13fcd52146934b7781d06d4a", + "type": "function", + "function": {"name": "get_weather", "arguments": "{\"city\": \"Paris\"}"} + } +] +``` + + + +### 3.3 HiCache (Hierarchical KV Caching) + +For long-context, prefix-heavy workloads, enable hierarchical KV caching to spill cold KV blocks to host memory (toggle the **Hierarchical KV Cache** card in the [Playground above](#playground)). Useful given GLM-5.2's 1M-token window; pair `--hicache-ratio` with a write policy that matches your reuse pattern. diff --git a/docs_new/cookbook/autoregressive/intro.mdx b/docs_new/cookbook/autoregressive/intro.mdx index 221084ac7..88d487997 100644 --- a/docs_new/cookbook/autoregressive/intro.mdx +++ b/docs_new/cookbook/autoregressive/intro.mdx @@ -28,7 +28,7 @@ metatags: " }, + NODE_RANK: { target: "command", label: "This node rank", default: "" }, + HF_TOKEN: { target: "command", label: "HF token (Docker)", default: "" }, + CURL_HOST: { target: "curl", label: "Server host", default: "localhost" }, + CURL_PORT: { target: "curl", label: "Server port", default: "30000" }, + }, + + curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\ +-H 'Content-Type: application/json' \\ +-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`, + + // Reproduce commands for the Benchmark card's "⚡ Reproduce" modal. + benchmarkCommands: { + speed: +`python3 -m sglang.bench_serving \\ + --backend sglang \\ + --host {{CURL_HOST}} --port {{CURL_PORT}} \\ + --model {{MODEL_NAME}} \\ + --dataset-name {{DATASET}} \\ + --random-input-len {{ISL}} --random-output-len {{OSL}} \\ + --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\ + --warmup-requests 64 --flush-cache`, + accuracy: { + gsm8k_pct: +`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +sgl-eval run gsm8k \\ + --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ + --num-threads 32`, + aime25_pct: +`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +sgl-eval run aime25 \\ + --model {{MODEL_NAME}} --api-key \\ + --n-repeats 16 --max-tokens 64000 \\ + --temperature 1.0 --top-p 0.95 --thinking \\ + --out-dir /sgl-workspace/logs \\ + --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1`, + }, + numPromptsByConc: { 1: 8, 16: 64, 64: 128, 256: 512, 1024: 2048, 4096: 8192 }, + }, + + // Per-variant accuracy applied to every cell; per-cell `accuracy` overrides. + // Both measured via sgl-eval (thinking mode) on H200. aime25 = pass@1 avg-of-16 + // (n-repeats 16, max-tokens 64000, temp 1.0, top-p 0.95); pass@16 100%, majority@16 93.3%. + defaultAccuracy: { + default: { gsm8k_pct: 98.2, aime25_pct: 87.7 }, + }, + + accuracyLabels: [ + ["aime25_pct", "AIME25", "%"], + ["gsm8k_pct", "GSM8K (1-shot)", "%"], + ], + + dockerImages: { + h200: "lmsysorg/sglang:latest", + b200: "lmsysorg/sglang:latest", + gb300: "lmsysorg/sglang:latest", + b300: "lmsysorg/sglang:latest", + }, + + github: { + cookbookModel: "zai-org/glm-5.2", + }, + + playgroundFeatures: { + + // ----- Card 1: "Attention Parallelism" ----- + // DSA prefill Context Parallelism (CP) splits the long-prefill attention across + // `cp` ranks — verified on Hopper (H200). On Blackwell the DSA-CP FP8 rope kernel + // is not yet adapted, so keep CP off there for now. + attention: { + knobs: [ + { id: "tp", label: "TP", values: [null, 4, 8] }, + { id: "cp", label: "CP (DSA prefill)", values: [null, 1, 2, 4, 8], + disable: { hw: ["b200", "gb300", "b300"] }, + disableReason: "DSA prefill Context Parallel is verified on Hopper (H200); the Blackwell sm100 DSA-CP FP8 rope kernel is not yet adapted." }, + { id: "dpAttn", label: "DP-Attention", + values: [null, false, 4, 8], + labels: { "auto": "Auto", "false": "Off" } }, + ], + }, + + // ----- Card 2: "MoE Parallelism" ----- + moe: { + backend: { + options: [ + { id: null, label: "Inherited" }, + { id: "deepep", label: "DeepEP", flags: ["--moe-a2a-backend deepep"] }, + ], + }, + ep: { label: "EP", values: [null, 4, 8] }, + }, + + // ----- Card 3: "Parsers" ----- + parsers: { + items: [ + { id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser glm45" }, + { id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser glm47" }, + ], + }, + + // ----- Card 4: "Speculative Decoding" ----- + // GLM-5.2 ships a single MTP (nextn) layer; index_share_for_mtp_iteration reuses the + // DSA indexer topk across draft steps (topk==1 only). + speculative: { + options: [ + { id: "current", label: "Inherited from base" }, + { id: "off", label: "Off (greedy)" }, + { id: "mtp-314", label: "EAGLE / MTP 3-1-4", + flags: ["--speculative-algorithm EAGLE", "--speculative-num-steps 3", + "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 4"] }, + { id: "mtp-112", label: "EAGLE / MTP 1-1-2", + flags: ["--speculative-algorithm EAGLE", "--speculative-num-steps 1", + "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 2"] }, + ], + }, + + // ----- Card 5: "Hierarchical KV Cache" ----- + hicache: { + backends: [ + { id: null, label: "Auto" }, + { id: "file", label: "File" }, + { id: "mooncake", label: "Mooncake" }, + ], + writePolicies: [ + { id: "auto", label: "Auto" }, + { id: "write_through", label: "Write-through" }, + { id: "write_back", label: "Write-back" }, + ], + }, + }, + + cells: [ + // ==================================================================== + // H200 + FP8 (Hopper) — TP8. CP (DSA prefill) verified here. + // ==================================================================== + { + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--mem-fraction-static 0.8", + "--cuda-graph-max-bs 32", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--moe-a2a-backend deepep", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 1", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 2", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 128", + // Large chunked-prefill is the dominant balanced lever (prefill-bound at this + // concurrency); max-running tracks KV capacity (~60-80 for 8K+1K reqs on 8xH200). + "--chunked-prefill-size 32768", + "--max-running-requests 80", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--moe-a2a-backend deepep", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 256", + "--max-running-requests 256", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + + // ==================================================================== + // B200 + FP8 (Blackwell) — TP8. low-latency verified on b200-verda-k8s + // ==================================================================== + { + match: { hw: "b200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--mem-fraction-static 0.8", + "--cuda-graph-max-bs 32", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--moe-a2a-backend deepep", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 1", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 2", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 128", + // Large chunked-prefill is the dominant balanced lever (prefill-bound at this + // concurrency); max-running tracks KV capacity (~89 for 8K+1K reqs on 8xB200). + "--chunked-prefill-size 32768", + "--max-running-requests 80", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--moe-a2a-backend deepep", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 256", + "--max-running-requests 256", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + + // ==================================================================== + // GB300 + FP8 (Grace-Blackwell, 4-GPU single node) — TP4. + // Flags mirror the B200 (sm100) configs; all three strategies verified end-to-end on + // a single 4xGB300 node (v0.5.13.post1). GB300 leads B200 per-GPU in every regime. + // Stage the weights on node-local NVMe first — shared cluster-storage reads are slow. + // ==================================================================== + { + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 32", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--dp 4", + "--enable-dp-attention", + "--moe-a2a-backend deepep", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 1", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 2", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 128", + // Same prefill lever as H200/B200 balanced; max-running tracks the TP4 KV capacity. + "--chunked-prefill-size 32768", + "--max-running-requests 80", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 4", + "--dp 4", + "--enable-dp-attention", + "--moe-a2a-backend deepep", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 256", + "--max-running-requests 256", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + + // ==================================================================== + // B300 + FP8 (Blackwell Ultra, 8-GPU single node) — TP8. + // Inferred from the verified B200 (sm100) FP8 recipe; B300 is the same Blackwell + // family (sm103). Benchmarks pending → verified:false. + // ==================================================================== + { + match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--mem-fraction-static 0.8", + "--cuda-graph-max-bs 32", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--moe-a2a-backend deepep", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 1", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 2", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 128", + "--chunked-prefill-size 32768", + "--max-running-requests 80", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--dp 8", + "--enable-dp-attention", + "--moe-a2a-backend deepep", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 256", + "--max-running-requests 256", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + + // ==================================================================== + // B300 + BF16 (Blackwell Ultra, 8-GPU single node) — TP8. + // The unquantized GLM-5.2 (~700B, ~1.51 TB) only fits single-node on 8xB300 + // (~2.1 TB HBM); smaller GPUs need multi-node (e.g. 2x 8xH200). Recipes are + // proposed, single-node TP8; benchmarks pending → verified:false. + // ==================================================================== + { + match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--mem-fraction-static 0.9", + "--cuda-graph-max-bs 32", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 1", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 2", + "--mem-fraction-static 0.9", + "--cuda-graph-max-bs 128", + "--chunked-prefill-size 32768", + "--max-running-requests 80", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + verified: false, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--mem-fraction-static 0.9", + "--cuda-graph-max-bs 256", + "--max-running-requests 256", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + + // ==================================================================== + // BF16 multi-node (inferred) — the 1.51 TB checkpoint spread over 2 nodes. + // 2x 8xH200 / 2x 8xB200 at TP16, 2x 4xGB300 at TP8. The engine injects + // --nnodes / --node-rank / --dist-init-addr from the Multi-Nodes selector. + // Recipes inferred from the single-node B300 path; not benchmarked → verified:false. + // ==================================================================== + { + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "multi-2" }, + verified: false, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 16", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 32", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "multi-2" }, + verified: false, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 16", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 1", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 2", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 128", + "--chunked-prefill-size 32768", + "--max-running-requests 80", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "multi-2" }, + verified: false, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 16", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 256", + "--max-running-requests 256", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "multi-2" }, + verified: false, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 16", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 32", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "multi-2" }, + verified: false, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 16", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 1", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 2", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 128", + "--chunked-prefill-size 32768", + "--max-running-requests 80", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "multi-2" }, + verified: false, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 16", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 256", + "--max-running-requests 256", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "multi-2" }, + verified: false, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 3", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 4", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 32", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "multi-2" }, + verified: false, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--speculative-algorithm EAGLE", + "--speculative-num-steps 1", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 2", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 128", + "--chunked-prefill-size 32768", + "--max-running-requests 80", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "multi-2" }, + verified: false, + env: [], + flags: [ + "--trust-remote-code", + "--model-path {{MODEL_NAME}}", + "--tp 8", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs 256", + "--max-running-requests 256", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + ], +};