From dfc40e0efe10af66db91d234ae10715e9990e5ea Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Wed, 26 Aug 2026 22:00:16 +0800 Subject: [PATCH] Add GLM-5.3-Flash cookbook (#36440) --- docs/cookbook/autoregressive/GLM/GLM-5.2.mdx | 1 - .../autoregressive/GLM/GLM-5.3-Flash.mdx | 250 +++++++++ docs/docs.json | 1 + docs/src/snippets/_deployment.jsx | 20 +- .../zai-org/glm-5.3-flash-benchmarks.jsx | 80 +++ .../configs/zai-org/glm-5.3-flash.jsx | 502 ++++++++++++++++++ 6 files changed, 850 insertions(+), 4 deletions(-) create mode 100644 docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx create mode 100644 docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx create mode 100644 docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx index 3b19ce986..47e9ec0c3 100644 --- a/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx +++ b/docs/cookbook/autoregressive/GLM/GLM-5.2.mdx @@ -1,7 +1,6 @@ --- title: GLM-5.2 description: "Deploy GLM-5.2 with SGLang — Z.ai's DeepSeek-Sparse-Attention (DSA) Mixture-of-Experts model with MTP speculative decoding and 1M context, on H200, B200, B300, GB300, and AMD MI300X/MI325X/MI355X." -tag: NEW --- ## Deployment diff --git a/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx new file mode 100644 index 000000000..395056c7a --- /dev/null +++ b/docs/cookbook/autoregressive/GLM/GLM-5.3-Flash.mdx @@ -0,0 +1,250 @@ +--- +title: GLM-5.3-Flash +description: "Deploy GLM-5.3-Flash with SGLang using recipes for H100, H200, B200, B300, GB200, and GB300, with MTP and multimodal serving." +tag: NEW +--- + +## Deployment + + + + + +Use an SGLang build that includes GLM-5.3-Flash support. + +```bash Command +docker pull lmsysorg/sglang:glm-5.3-flash +``` + +The deployment panel can render a complete `docker run` command for the selected hardware and options. See [Install SGLang with Docker](/docs/get-started/install#method-3-using-docker) for host setup. + + + +Choose your hardware, then choose the operating point that matches your workload: + +- **Low Latency** starts with adaptive MTP 5/1/6 speculative decoding and tensor parallelism to shorten interactive responses. +- **High Throughput** starts with speculative decoding off, which avoids draft-and-verify overhead under sustained batches. + +Every listed hardware platform exposes both strategies. A **Verified** badge means that exact hardware and command were tested. **Not Verified** means the command is a supported starting point that still needs workload validation. A choice is disabled only when the underlying runtime combination is known to be unsupported. + +The recommended selection is only a starting point. The same panel also lets you override the KV/DSA pairing, multimodal feature transport, and HiCache tiers. Changing an option that was not part of the measured command changes the badge to **Not Verified** without hiding the option. + +import { Deployment } from "/src/snippets/_deployment.jsx"; +import { config } from "/src/snippets/configs/zai-org/glm-5.3-flash.jsx"; +import { benchmarks } from "/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx"; + + + +## Playground + +Use the Playground for lower-level tuning such as attention parallelism, MoE communication, and reasoning or tool parsers. It inherits every selection from the deployment panel and shows only the command-line diff. + +import { Playground } from "/src/snippets/_playground.jsx"; + + + +## 1. Model introduction + +GLM-5.3-Flash is a natively multimodal Mixture-of-Experts model built around a hybrid attention architecture — 320B total parameters with 18B active. Its 45 text layers combine MLA attention, DSA sparse attention, and KDA linear attention, and a 24-layer vision encoder handles image and video input. The checkpoint uses 288 routed experts with 8 active experts per token and includes a native MTP draft layer for speculative decoding. See the [GLM-5.3-Flash blog](https://z.ai/blog/glm-5.3-flash) and the [GLM-5 technical report](https://arxiv.org/abs/2602.15763) for training details. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
AttributeDescription
ArchitectureMoE with hybrid attention (MLA, DSA, KDA), mHC, and MTP — 320B total / 18B active parameters
PrecisionFP8 weights with BF16 KV cache by default
Context1M tokens
InputsText, image, and video
Generation defaultstemperature=1.0, top_p=0.95, thinking enabled
+ +The deployment recipes use the checkpoint's generation configuration. Override sampling only when your application has its own evaluated settings. + +## 2. Configuration tips + +### Choose a strategy + +Start with **Low Latency** for chat and agent workloads. Adaptive MTP changes the draft depth as acceptance changes, reducing unnecessary draft work when the server is busy. Measure **High Throughput** for heavily batched traffic where disabling speculative decoding can be more efficient. SGLang serves MTP through `--speculative-algorithm NEXTN`, so generated commands use that flag value. + +Strategy labels describe the workload goal, not a hardware restriction. Both strategies stay available when you switch hardware; only the verification badge changes. + +### Size both memory pools + +GLM-5.3-Flash maintains a paged KV pool for attention and a separate KDA state pool. The KDA state pool can limit concurrency before the KV pool is full. If startup reduces `max_running_requests` because of KDA state capacity, increase `--mamba-full-memory-ratio` or set `--max-mamba-cache-size` for the expected concurrency, then tune `--max-running-requests` to the workload. + +Keep the prefix cache enabled for every strategy. + +Keep the checkpoint's KDA lower-bound setting unchanged. In particular, do not override `linear_lower_bound` through `--json-model-override-args`. + +### Keep the KV and DSA backends paired + +The default recipe pairs a BF16 KV cache with TileLang DSA. On Blackwell GPUs, the deployment panel also offers FP8 KV with TRT-LLM DSA as a memory-saving option. Switch the dtype and both DSA backends together; TileLang DSA with FP8 KV is not a valid CUDA combination. FP8 KV with TRT-LLM DSA is disabled on H100 and H200. + +### Extend the cache hierarchy + +Keep **HiCache** off when GPU memory is sufficient. Select **L1 + L2** to spill reusable cache entries into host memory. Select **+ L3** only after configuring Mooncake on every serving node; the generated command exposes the required configuration path. These options remain selectable but are marked **Not Verified** until the resulting command is validated on the chosen hardware. + +### Multimodal memory + +All strategies enable multimodal serving. The processor samples video at 2 FPS and caps video input at 240,000 visual tokens. Install `torchcodec` in the serving environment before sending video requests. For very long videos on 4x GB300, use encoder disaggregation to isolate the vision encoder's memory spikes from language decoding. + +The default multimodal feature transport is automatic, and on a single CUDA node auto resolves to CPU transport. CUDA IPC is opt-in: pass `--mm-feature-transport cuda_ipc` when lower transfer latency matters more than the GPU memory the IPC pool reserves. CUDA VMM transport applies only to multi-node GB200/GB300 systems on the MNNVL fabric, where auto selects it. + +## 3. Advanced usage + +### 3.1 Reasoning + +Thinking is enabled by the checkpoint's generation configuration, and generated commands enable `--reasoning-parser glm45` by default. The OpenAI-compatible API then places thinking in `message.reasoning_content` and the final answer in `message.content`. You can disable **Reasoning Parser** in the Playground when an integration needs the raw response format. + +To disable thinking for a request, pass `chat_template_kwargs: {"thinking": false}` in the request body. + +### 3.2 Tool calling + +Generated commands enable `--tool-call-parser glm47` by default, so structured calls are returned in `message.tool_calls`. You can disable **Tool Call Parser** in the Playground when tool calling is not needed. On follow-up turns, read both `reasoning_content` and `content` because a thinking model can use either field around tool execution. + +### 3.3 Multimodal serving + +The base recipes accept image and video content through the OpenAI-compatible chat API. Keep the processor defaults unless you have measured a different sampling or resize policy. Inputs above the video token budget are clamped to the processor's limit rather than rejected. + +### 3.4 Encoder disaggregation + +Encoder disaggregation separates vision preprocessing from language inference. The verified topology uses one 4x GB300 node shared by an encoder-only TP4 process on port 30001 and a language-only TP4/EP4 process on port 30000. Start the encoder first. + + + +```bash Command +sglang serve \ + --model-path zai-org/GLM-5.3-Flash \ + --tp-size 4 \ + --encoder-only \ + --host 0.0.0.0 \ + --port 30001 +``` + + + + + +```bash Command +sglang serve \ + --model-path zai-org/GLM-5.3-Flash \ + --tp-size 4 \ + --ep-size 4 \ + --attention-backend dsa \ + --dsa-prefill-backend tilelang \ + --dsa-decode-backend tilelang \ + --linear-attn-backend triton \ + --kv-cache-dtype bfloat16 \ + --quantization fp8 \ + --moe-runner-backend deep_gemm \ + --max-running-requests 64 \ + --chunked-prefill-size 8192 \ + --max-prefill-tokens 8192 \ + --disable-shared-experts-fusion \ + --disable-prefill-cuda-graph \ + --speculative-algorithm NEXTN \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --speculative-adaptive \ + --language-only \ + --encoder-urls http://localhost:30001 \ + --mem-fraction-static 0.78 \ + --host 0.0.0.0 \ + --port 30000 +``` + + + +This topology served image requests and videos up to 238,080 visual tokens. In a concurrent long-video workload, the largest observed decode gap fell from 5.53 seconds in unified serving to 1.79 seconds with encoder disaggregation. Keep `--mem-fraction-static 0.78` on the language process so the encoder retains room for vision workspaces. + +Install `torchcodec` for video and see the [encoder disaggregation guide](/docs/advanced_features/epd_disaggregation) for the generic architecture and operational model. + +### 3.5 PD disaggregation (preview) + +PD splits prefill and decode into separate server groups behind a router. For this hybrid model, the transfer moves both the paged DSA KV and the KDA recurrent state. + + +PD serving is mechanically validated with dummy weights only — startup, bootstrap, state transfer, and request flow all work on 4x GB300. It has not been load- or accuracy-tested. Treat it as a preview until the real-weight gate completes. + + + + +```bash Prefill (GPU 0-1) +sglang serve \ + --model-path zai-org/GLM-5.3-Flash \ + --tp-size 2 \ + --dsa-prefill-backend tilelang \ + --dsa-decode-backend tilelang \ + --kv-cache-dtype bfloat16 \ + --moe-runner-backend triton \ + --disaggregation-mode prefill \ + --disaggregation-bootstrap-port 8998 \ + --disaggregation-transfer-backend nixl \ + --host 0.0.0.0 \ + --port 31000 +``` + +```bash Decode (GPU 2-3) +sglang serve \ + --model-path zai-org/GLM-5.3-Flash \ + --tp-size 2 \ + --base-gpu-id 2 \ + --dsa-prefill-backend tilelang \ + --dsa-decode-backend tilelang \ + --kv-cache-dtype bfloat16 \ + --moe-runner-backend triton \ + --disaggregation-mode decode \ + --disaggregation-transfer-backend nixl \ + --host 0.0.0.0 \ + --port 32000 +``` + +```bash Router +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --mini-lb \ + --prefill http://127.0.0.1:31000 8998 \ + --decode http://127.0.0.1:32000 \ + --host 0.0.0.0 \ + --port 30000 +``` + +The positional `8998` after `--prefill` must equal the prefill server's `--disaggregation-bootstrap-port`. + + + +Operational notes: + +- Give each role a distinct `--nccl-port` when both share one node. +- Single-node NIXL needs `UCX_NET_DEVICES=lo` and `UCX_TLS=tcp,cuda_copy,cuda_ipc,self,sm` in both server environments. +- The validated arm used the triton MoE runner; deep_gemm under PD is untested. + +Known limitations: + +- Speculative decoding does not start under PD at the current cut (draft-graph capture width assert on the prefill role; decode-side memory pressure at TP2). Run PD without speculative flags. +- Prefill and decode with different TP sizes transfer state through slice paths, but numeric correctness is unverified. Keep both roles at the same TP size. diff --git a/docs/docs.json b/docs/docs.json index 84c7201de..8553e9b7a 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1226,6 +1226,7 @@ { "group": "GLM", "pages": [ + "cookbook/autoregressive/GLM/GLM-5.3-Flash", "cookbook/autoregressive/GLM/GLM-5.2", "cookbook/autoregressive/GLM/GLM-5.1", "cookbook/autoregressive/GLM/GLM-5", diff --git a/docs/src/snippets/_deployment.jsx b/docs/src/snippets/_deployment.jsx index 23a82ebc1..7b0ccd88f 100644 --- a/docs/src/snippets/_deployment.jsx +++ b/docs/src/snippets/_deployment.jsx @@ -671,12 +671,19 @@ export const Deployment = ({ config, benchmarks }) => { // Overlay dims ride along: they never key cells, so snapping must not drop // them (it did — a strict-mode hash round-trip lost the spec default). // Keep the parsed value when it names a real option, else the row default. + // A hash can also name an option that showWhen hides (or a rule disables) + // for the composed selection; snap those like an interactive reseat would. for (const spec of overlayDimSpecs) { const want = parsed[spec.id]; const opts = spec.options || []; - valid[spec.id] = opts.some((o) => o.id === want) + const picked = opts.some((o) => o.id === want) ? want : spec.default ?? (opts[0] && opts[0].id) ?? ""; + const withPick = { ...valid, [spec.id]: picked }; + const usable = visibleOptions(spec, withPick).filter((o) => !optionDisabled(o, withPick)); + valid[spec.id] = usable.some((o) => o.id === picked) + ? picked + : (usable[0] && usable[0].id) ?? picked; } return valid; }; @@ -2488,8 +2495,15 @@ export const Deployment = ({ config, benchmarks }) => { {modal === "bench" && benchEntry && (() => { const bc = buildBenchCommands(benchEntry, sel); if (!bc) return null; - const selSummary = - `${sel.hw.toUpperCase()} · ${sel.variant} · ${sel.quant.toUpperCase()} · ${sel.strategy} · ${sel.nodes}`; + const selSummary = [ + sel.hw && sel.hw.toUpperCase(), + sel.variant, + sel.quant && sel.quant.toUpperCase(), + sel.strategy, + sel.nodes, + ] + .filter((part) => part !== undefined && part !== null && part !== "") + .join(" · "); let selConc = null; let speedCmd = null; if (bc.speed) { diff --git a/docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx b/docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx new file mode 100644 index 000000000..1a3281752 --- /dev/null +++ b/docs/src/snippets/configs/zai-org/glm-5.3-flash-benchmarks.jsx @@ -0,0 +1,80 @@ +export const benchmarks = [ + { + match: { hw: "gb300", strategy: "low-latency" }, + sglang_version: "f13cb6f6a7", + latencyPercentile: "Mean", + speed: [ + { + workload: { + dataset: "random", + isl: 1024, + osl: 256, + max_concurrency: 16, + num_prompts: 80, + }, + ttft_ms: 589.2, + tpot_ms: 6.48, + tokens_per_sec_per_gpu: 2277.46, + }, + ], + accuracy: { gsm8k_pct: 97.50 }, + notes: + "Measured on 4x GB300 (TP4/EP4) with the final weights (zai-org/GLM-5.3-Flash, c5b82b63e37b) at the rc2 cut (f13cb6f6a7), adaptive MTP 5/1/6 with SGLANG_SIMULATE_ACC_LEN=3 (accept length confirmed 3.00 in the bench summary and server log): 80 random requests at 1,024 input / 256 output tokens and concurrency 16 produced 1,821.97 aggregate output tok/s after two discarded warmups. Simulated accept length makes this a throughput-mechanism number. Accuracy is from the shared non-simulated full GSM8K gate: 97.50% with a 100% stop rate over all 1,319 problems.", + }, + { + match: { hw: "gb300", strategy: "high-throughput" }, + sglang_version: "f13cb6f6a7", + latencyPercentile: "Mean", + speed: [ + { + workload: { + dataset: "random", + isl: 1024, + osl: 256, + max_concurrency: 16, + num_prompts: 80, + }, + ttft_ms: 684.63, + tpot_ms: 11.53, + tokens_per_sec_per_gpu: 1410.4, + }, + { + workload: { + dataset: "random", + isl: 1024, + osl: 256, + max_concurrency: 64, + num_prompts: 320, + }, + ttft_ms: 1691.64, + tpot_ms: 19.88, + tokens_per_sec_per_gpu: 3023.31, + }, + { + workload: { + dataset: "random", + isl: 1024, + osl: 256, + max_concurrency: 256, + num_prompts: 1280, + }, + ttft_ms: 5192.23, + tpot_ms: 43.35, + tokens_per_sec_per_gpu: 4856.19, + }, + ], + accuracy: { gsm8k_pct: 97.50 }, + notes: + "Measured on 4x GB300 (TP4/EP4) with the final weights (zai-org/GLM-5.3-Flash, c5b82b63e37b) at the rc2 cut (f13cb6f6a7), speculative decoding off, after two discarded warmups per row: 1,128.32 / 2,418.65 / 3,884.95 aggregate output tok/s at concurrency 16 / 64 / 256 (80 / 320 / 1,280 random requests at 1,024 input / 256 output tokens; the high-concurrency rows with --max-running-requests 256 and decode graph batch 256). Throughput at 256 is still scaling but sublinear (prefill queueing). Accuracy is the full GSM8K gate on the same server: 97.50% with a 100% stop rate over all 1,319 problems.", + }, + { match: { hw: "h100", strategy: "low-latency" } }, + { match: { hw: "h100", strategy: "high-throughput" } }, + { match: { hw: "h200", strategy: "low-latency" } }, + { match: { hw: "h200", strategy: "high-throughput" } }, + { match: { hw: "b200", strategy: "low-latency" } }, + { match: { hw: "b200", strategy: "high-throughput" } }, + { match: { hw: "b300", strategy: "low-latency" } }, + { match: { hw: "b300", strategy: "high-throughput" } }, + { match: { hw: "gb200", strategy: "low-latency" } }, + { match: { hw: "gb200", strategy: "high-throughput" } }, +]; diff --git a/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx b/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx new file mode 100644 index 000000000..833561e45 --- /dev/null +++ b/docs/src/snippets/configs/zai-org/glm-5.3-flash.jsx @@ -0,0 +1,502 @@ +export const config = { + modelName: "GLM-5.3-Flash", + + supportedHardware: ["gb300", "h100", "h200", "b200", "b300", "gb200"], + + matchDims: [ + { + id: "strategy", + title: "Strategy", + options: [ + { id: "low-latency", label: "Low Latency", subtitle: "Adaptive MTP 5/1/6" }, + { id: "high-throughput", label: "High Throughput", subtitle: "Spec decode off" }, + ], + }, + ], + + isRecommendedSelection(s) { + return ( + s.kvDsaPair === "bf16-tilelang" && + s.mmTransport === "auto" && + s.hicache === "off" + ); + }, + + overlayDims: [ + { + id: "kvDsaPair", + title: "KV Cache + DSA Backend", + default: "bf16-tilelang", + options: [ + { + id: "bf16-tilelang", + label: "BF16 + TileLang", + stripPrefixes: ["--kv-cache-dtype", "--dsa-prefill-backend", "--dsa-decode-backend"], + flags: [ + "--kv-cache-dtype bfloat16", + "--dsa-prefill-backend tilelang", + "--dsa-decode-backend tilelang", + ], + }, + { + id: "fp8-trtllm", + label: "FP8 + TRT-LLM", + disabled: (s) => ["h100", "h200"].includes(s.hw), + disableReason: "FP8 KV cache with TRT-LLM DSA is not supported on Hopper GPUs.", + stripPrefixes: ["--kv-cache-dtype", "--dsa-prefill-backend", "--dsa-decode-backend"], + flags: [ + "--kv-cache-dtype fp8_e4m3", + "--dsa-prefill-backend trtllm", + "--dsa-decode-backend trtllm", + ], + hints: ["Reduces KV-cache memory. Validate accuracy and memory headroom for your workload."], + }, + ], + }, + { + id: "mmTransport", + title: "VLM Transport", + default: "auto", + options: [ + { id: "auto", label: "Auto", subtitle: "Topology-aware" }, + { + id: "cpu", + label: "CPU", + subtitle: "Save GPU memory", + flags: ["--mm-feature-transport cpu"], + }, + ], + }, + { + id: "hicache", + title: "HiCache", + default: "off", + options: [ + { id: "off", label: "Off" }, + { + id: "l2", + label: "L1 + L2", + subtitle: "Host memory", + flags: ["--enable-hierarchical-cache", "--hicache-size 32"], + hints: ["32 GB host tier; the default ratio can demand more host RAM than the node has free."], + }, + { + id: "l3", + label: "+ L3", + subtitle: "Mooncake", + flags: ["--enable-hierarchical-cache", "--hicache-size 32", "--hicache-storage-backend mooncake"], + env: ["SGLANG_HICACHE_MOONCAKE_CONFIG_PATH={{MOONCAKE_CONFIG}}"], + hints: ["Start Mooncake and place the configuration file on every serving node."], + }, + ], + }, + ], + + modelNames: { + default: "zai-org/GLM-5.3-Flash", + }, + + placeholders: { + HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" }, + PORT: { target: "command", label: "Bind port", default: "30000" }, + HF_TOKEN: { target: "command", label: "HF token (Docker)", default: "" }, + MOONCAKE_CONFIG: { target: "command", label: "Mooncake config", default: "" }, + CURL_HOST: { target: "curl", label: "Server host", default: "localhost" }, + CURL_PORT: { target: "curl", label: "Server port", default: "30000" }, + }, + + curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\ +-H 'Content-Type: application/json' \\ +-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`, + + benchmarkCommands: { + speed: +`# Low Latency speed runs serve with SGLANG_SIMULATE_ACC_LEN=3 to pin the accept +# length; that number is throughput evidence only. Never run accuracy against it. +python3 -m sglang.bench_serving \\ + --backend sglang \\ + --host {{CURL_HOST}} --port {{CURL_PORT}} \\ + --model {{MODEL_NAME}} \\ + --dataset-name {{DATASET}} \\ + --random-input-len {{ISL}} --random-output-len {{OSL}} --random-range-ratio 1.0 \\ + --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\ + --request-rate inf --temperature 0 --seed 42 \\ + --flush-cache`, + // num_prompts = 5 × concurrency (measured floor 16). + numPromptsByConc: { 1: 16, 16: 80, 64: 320, 256: 1280, 1024: 5120 }, + accuracy: { + gsm8k_pct: +`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +sgl-eval run gsm8k \\ + --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ + --model {{MODEL_NAME}} \\ + --num-threads 64 \\ + --max-tokens 32768 \\ + --temperature 1.0 \\ + --top-p 0.95 \\ + --thinking`, + }, + }, + + accuracyLabels: [ + ["gsm8k_pct", "GSM8K", "%"], + ], + + // Support is not in a public sglang release yet, so the nightly images do + // not work; every NVIDIA lane uses the purpose-built CUDA 13 image. + dockerImages: { + gb300: "lmsysorg/sglang:glm-5.3-flash", + h100: "lmsysorg/sglang:glm-5.3-flash", + h200: "lmsysorg/sglang:glm-5.3-flash", + b200: "lmsysorg/sglang:glm-5.3-flash", + b300: "lmsysorg/sglang:glm-5.3-flash", + gb200: "lmsysorg/sglang:glm-5.3-flash", + }, + + github: { + cookbookModel: "zai-org/glm-5.3-flash", + }, + + playgroundFeatures: { + attention: { + knobs: [ + { id: "tp", label: "TP", values: [ + null, 1, 2, 4, + { + value: 8, + disable: [ + { + when: { hw: ["gb300", "gb200"] }, + reason: "TP=8 needs 8 GPUs; the GB300 and GB200 recipes run on 4.", + }, + ], + }, + ]}, + { id: "cp", label: "CP", values: [null, 1, 2, 4] }, + { + id: "dpAttn", + label: "DP-Attention", + values: [ + null, false, 1, 2, 4, + { + value: 8, + disable: [ + { + when: { hw: ["gb300", "gb200"] }, + reason: "DP-Attention=8 needs 8 ranks; the GB300 and GB200 recipes run on 4.", + }, + ], + }, + ], + labels: { auto: "Auto", false: "Off" }, + disable: [ + { + when: { strategy: ["low-latency"] }, + reason: "Low Latency uses adaptive MTP, which does not support DP-Attention.", + }, + ], + disableReason: "Low Latency uses adaptive MTP, which does not support DP-Attention.", + }, + ], + }, + + moe: { + backend: { + options: [ + { id: null, label: "Inherited" }, + { + id: "deep_gemm", + label: "DeepGemm", + flags: ["--moe-runner-backend deep_gemm"], + }, + ], + }, + ep: { label: "EP", values: [ + null, 2, 4, + { + value: 8, + disable: [ + { + when: { hw: ["gb300", "gb200"] }, + reason: "EP=8 needs 8 GPUs; the GB300 and GB200 recipes run on 4.", + }, + ], + }, + ]}, + }, + + parsers: { + items: [ + { id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser glm45" }, + { id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser glm47" }, + ], + }, + + }, + + cells: [ + { + match: { hw: "gb300", strategy: "low-latency" }, + nnodes: 1, + verified: true, + verificationStatus: (s) => config.isRecommendedSelection(s) ? "verified" : "unverified", + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp-size 4", + "--ep-size 4", + "--dsa-prefill-backend tilelang", + "--dsa-decode-backend tilelang", + "--kv-cache-dtype bfloat16", + "--moe-runner-backend deep_gemm", + "--speculative-algorithm NEXTN", + "--speculative-num-steps 5", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 6", + "--speculative-adaptive", + "--reasoning-parser glm45", + "--tool-call-parser glm47", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb300", strategy: "high-throughput" }, + nnodes: 1, + verified: true, + verificationStatus: (s) => config.isRecommendedSelection(s) ? "verified" : "unverified", + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp-size 4", + "--ep-size 4", + "--dsa-prefill-backend tilelang", + "--dsa-decode-backend tilelang", + "--kv-cache-dtype bfloat16", + "--moe-runner-backend deep_gemm", + "--reasoning-parser glm45", + "--tool-call-parser glm47", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h100", strategy: "low-latency" }, + nnodes: 1, + verified: false, + verificationStatus: (s) => config.isRecommendedSelection(s) ? "in-progress" : "unverified", + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--ep-size 8", + "--mem-fraction-static 0.75", + "--dsa-prefill-backend tilelang", + "--dsa-decode-backend tilelang", + "--kv-cache-dtype bfloat16", + "--moe-runner-backend deep_gemm", + "--speculative-algorithm NEXTN", + "--speculative-num-steps 5", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 6", + "--speculative-adaptive", + "--reasoning-parser glm45", + "--tool-call-parser glm47", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h100", strategy: "high-throughput" }, + nnodes: 1, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--ep-size 8", + "--mem-fraction-static 0.75", + "--dsa-prefill-backend tilelang", + "--dsa-decode-backend tilelang", + "--kv-cache-dtype bfloat16", + "--moe-runner-backend deep_gemm", + "--reasoning-parser glm45", + "--tool-call-parser glm47", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h200", strategy: "low-latency" }, + nnodes: 1, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--ep-size 8", + "--dsa-prefill-backend tilelang", + "--dsa-decode-backend tilelang", + "--kv-cache-dtype bfloat16", + "--moe-runner-backend deep_gemm", + "--speculative-algorithm NEXTN", + "--speculative-num-steps 5", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 6", + "--speculative-adaptive", + "--reasoning-parser glm45", + "--tool-call-parser glm47", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h200", strategy: "high-throughput" }, + nnodes: 1, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--ep-size 8", + "--dsa-prefill-backend tilelang", + "--dsa-decode-backend tilelang", + "--kv-cache-dtype bfloat16", + "--moe-runner-backend deep_gemm", + "--reasoning-parser glm45", + "--tool-call-parser glm47", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b200", strategy: "low-latency" }, + nnodes: 1, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--ep-size 8", + "--dsa-prefill-backend tilelang", + "--dsa-decode-backend tilelang", + "--kv-cache-dtype bfloat16", + "--moe-runner-backend deep_gemm", + "--speculative-algorithm NEXTN", + "--speculative-num-steps 5", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 6", + "--speculative-adaptive", + "--reasoning-parser glm45", + "--tool-call-parser glm47", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b200", strategy: "high-throughput" }, + nnodes: 1, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--ep-size 8", + "--dsa-prefill-backend tilelang", + "--dsa-decode-backend tilelang", + "--kv-cache-dtype bfloat16", + "--moe-runner-backend deep_gemm", + "--reasoning-parser glm45", + "--tool-call-parser glm47", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", strategy: "low-latency" }, + nnodes: 1, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--ep-size 8", + "--dsa-prefill-backend tilelang", + "--dsa-decode-backend tilelang", + "--kv-cache-dtype bfloat16", + "--moe-runner-backend deep_gemm", + "--speculative-algorithm NEXTN", + "--speculative-num-steps 5", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 6", + "--speculative-adaptive", + "--reasoning-parser glm45", + "--tool-call-parser glm47", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", strategy: "high-throughput" }, + nnodes: 1, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp-size 8", + "--ep-size 8", + "--dsa-prefill-backend tilelang", + "--dsa-decode-backend tilelang", + "--kv-cache-dtype bfloat16", + "--moe-runner-backend deep_gemm", + "--reasoning-parser glm45", + "--tool-call-parser glm47", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb200", strategy: "low-latency" }, + nnodes: 1, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp-size 4", + "--ep-size 4", + "--dsa-prefill-backend tilelang", + "--dsa-decode-backend tilelang", + "--kv-cache-dtype bfloat16", + "--moe-runner-backend deep_gemm", + "--speculative-algorithm NEXTN", + "--speculative-num-steps 5", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 6", + "--speculative-adaptive", + "--reasoning-parser glm45", + "--tool-call-parser glm47", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "gb200", strategy: "high-throughput" }, + nnodes: 1, + verified: false, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp-size 4", + "--ep-size 4", + "--dsa-prefill-backend tilelang", + "--dsa-decode-backend tilelang", + "--kv-cache-dtype bfloat16", + "--moe-runner-backend deep_gemm", + "--reasoning-parser glm45", + "--tool-call-parser glm47", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + ], +};