cookbook: add a Speculative card to the GLM-5.3-Flash playground (#36740)
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
zijiexia
co-authored by
Claude Opus 5
Claude Opus 5
parent
dc10483592
commit
6ccfeb59bc
@@ -41,7 +41,7 @@ Generated commands leave `--mamba-full-memory-ratio` at its `0.9` default, which
|
||||
|
||||
## Playground
|
||||
|
||||
Use the Playground for lower-level tuning such as attention parallelism, MoE communication, and reasoning or tool parsers. It inherits every selection from the deployment panel and shows only the command-line diff.
|
||||
Use the Playground for lower-level tuning such as attention parallelism, MoE communication, speculative decoding, and reasoning or tool parsers. It inherits every selection from the deployment panel and shows only the command-line diff.
|
||||
|
||||
import { Playground } from "/src/snippets/_playground.jsx";
|
||||
|
||||
@@ -96,6 +96,16 @@ Start with **Low Latency** for chat and agent workloads. Adaptive MTP changes th
|
||||
|
||||
Strategy labels describe the workload goal. Both strategies stay available on NVIDIA GPUs; the AMD ROCm recipes expose only High Throughput until MTP speculative decoding is validated there.
|
||||
|
||||
### Change the speculative algorithm
|
||||
|
||||
The **Speculative** card in the Playground changes the algorithm without leaving the selected strategy:
|
||||
|
||||
- **EAGLE / Adaptive MTP 5-1-6** is exactly what Low Latency serves, so a Low Latency base starts on this chip. Pick it from a High Throughput base to keep that recipe's other settings and add the MTP head.
|
||||
- **Off (greedy)** strips the whole `--speculative-*` family, which is what High Throughput already starts from.
|
||||
- **DFlash2** swaps the in-checkpoint MTP head for the trained block-diffusion draft in [`incoai/GLM-5.3-Flash-DFlash2`](https://huggingface.co/incoai/GLM-5.3-Flash-DFlash2). The draft proposes a whole block per step and the target verifies it in one forward pass, so output quality stays the target's. Its block size comes from the draft checkpoint, and the draft runs on `fa4` rather than the target's DSA backends. It needs a build that carries the GLM-5.3-Flash hidden-state capture from [PR #36708](https://github.com/sgl-project/sglang/pull/36708), which is merged into the [PR #36507](https://github.com/sgl-project/sglang/pull/36507) support branch (`xinyuan/glm-5.3-flash-support`) rather than into `main`, so the image pinned above is not enough on its own — pull that branch at its current head, or add #36708's commit on top of an older checkout. The draft repository is also access-gated: request access on its model page, then download it alongside the target before serving. This combination is not yet measured on the cookbook hardware, so treat it as a starting point.
|
||||
|
||||
Neither algorithm runs with DP-Attention, and neither is available on the AMD ROCm recipes; the card disables the affected chips and names the reason.
|
||||
|
||||
### Size both memory pools
|
||||
|
||||
GLM-5.3-Flash maintains a paged KV pool for attention and a separate KDA state pool. The KDA state pool can limit concurrency before the KV pool is full. If startup reduces `max_running_requests` because of KDA state capacity, increase `--mamba-full-memory-ratio` or set `--max-mamba-cache-size` for the expected concurrency, then tune `--max-running-requests` to the workload.
|
||||
|
||||
@@ -9,7 +9,9 @@
|
||||
// attention — TP/CP/DP-Attention knobs
|
||||
// moe — backend (+ MegaMoE quantization sub-select) + EP
|
||||
// parsers — per-item toggle flags
|
||||
// speculative — single-select preset
|
||||
// speculative — single-select preset; an option may carry `note` (a
|
||||
// prerequisite line rendered under the chips while that option
|
||||
// is the one in effect)
|
||||
//
|
||||
// Axis-level `showWhen(base)` (any axis): the card is not rendered when the Deploy
|
||||
// panel has not switched that feature on. `base` carries the cell match dims plus
|
||||
@@ -842,6 +844,11 @@ export const Playground = ({ config }) => {
|
||||
|| head === "--speculative-num-steps"
|
||||
|| head === "--speculative-eagle-topk"
|
||||
|| head === "--speculative-num-draft-tokens"
|
||||
// Adaptive draft depth is part of an EAGLE preset, not a
|
||||
// separate knob: a base that carries it must strip it when
|
||||
// another algorithm is picked, or the flag survives and the
|
||||
// server warns it away (only EAGLE/EAGLE3 honor it).
|
||||
|| head === "--speculative-adaptive"
|
||||
|| head === "--speculative-dspark-block-size"
|
||||
|| head === "--enable-linear-replayssm-spec"
|
||||
|| head === "--linear-replayssm-cache-len"
|
||||
@@ -870,6 +877,7 @@ export const Playground = ({ config }) => {
|
||||
flags = h.stripFlagsByFirstToken(flags, [
|
||||
"--speculative-algorithm", "--speculative-num-steps",
|
||||
"--speculative-eagle-topk", "--speculative-num-draft-tokens",
|
||||
"--speculative-adaptive",
|
||||
"--speculative-dspark-block-size", "--enable-linear-replayssm-spec",
|
||||
"--linear-replayssm-cache-len",
|
||||
"--speculative-ngram-max-bfs-breadth",
|
||||
@@ -891,6 +899,12 @@ export const Playground = ({ config }) => {
|
||||
.map((opt) => h.evaluateChip(opt, base))
|
||||
.filter((c) => !c.hidden && !(hideCurrent && c.value === "current"));
|
||||
if (visible.length === 0) return null;
|
||||
// An option may carry a `note`: a prerequisite the reader must act on
|
||||
// before the composed command runs at all (an algorithm whose support
|
||||
// is not in the page's pinned image yet, a draft checkpoint to fetch).
|
||||
// Shown only for the option in effect, so the card stays a chip row
|
||||
// until the pick actually needs something.
|
||||
const note = (visible.find((c) => c.value === display) || {}).note;
|
||||
return (
|
||||
<div key={axisId} style={s.card}>
|
||||
<div style={s.compactRow}>
|
||||
@@ -903,6 +917,7 @@ export const Playground = ({ config }) => {
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
{note && <div style={s.axisNote}>{note}</div>}
|
||||
</div>
|
||||
);
|
||||
},
|
||||
@@ -1773,6 +1788,16 @@ export const Playground = ({ config }) => {
|
||||
color: isDark ? "#e5e7eb" : "#374151",
|
||||
whiteSpace: "pre-wrap", overflowX: "auto", margin: 0,
|
||||
},
|
||||
// Amber callout for a prerequisite an axis option carries — rendered
|
||||
// inside the axis card, so it reads as a condition on the pick rather
|
||||
// than on the composed command.
|
||||
axisNote: {
|
||||
margin: "6px 0 0", padding: "6px 10px", borderRadius: "6px",
|
||||
fontSize: "11px", lineHeight: "1.45",
|
||||
background: isDark ? "#78350f" : "#fef3c7",
|
||||
color: isDark ? "#fde68a" : "#92400e",
|
||||
border: `1px solid ${isDark ? "#92400e" : "#fcd34d"}`,
|
||||
},
|
||||
// Amber callout under the playground command when the effective (post-
|
||||
// override) command turns speculative decoding on without setting
|
||||
// --max-running-requests (SGLang then caps it at 48).
|
||||
|
||||
@@ -267,6 +267,72 @@ sgl-eval run gsm8k \\
|
||||
],
|
||||
},
|
||||
|
||||
// ----- Card: "Speculative" -----
|
||||
// The Deploy panel only picks speculation through the Strategy dim (Low
|
||||
// Latency = the checkpoint's adaptive MTP head, High Throughput = off).
|
||||
// This card is the finer control, and it adds the one algorithm no cell
|
||||
// ships: DFlash2, whose draft is a separate checkpoint.
|
||||
//
|
||||
// The EAGLE preset is byte-identical to what the Low Latency cells carry,
|
||||
// so a Low Latency base derives onto that chip instead of showing
|
||||
// "Inherited from base", and re-picking it is a no-op.
|
||||
speculative: {
|
||||
options: [
|
||||
{ id: "current", label: "Inherited from base" },
|
||||
{ id: "off", label: "Off (greedy)" },
|
||||
{
|
||||
id: "eagle",
|
||||
label: "EAGLE / Adaptive MTP 5-1-6",
|
||||
flags: [
|
||||
"--speculative-algorithm EAGLE",
|
||||
"--speculative-num-steps 5",
|
||||
"--speculative-eagle-topk 1",
|
||||
"--speculative-num-draft-tokens 6",
|
||||
"--speculative-adaptive",
|
||||
],
|
||||
disable: [
|
||||
{
|
||||
when: { dpAttnOn: [true] },
|
||||
reason: "Adaptive MTP does not support DP-Attention — the server falls back to a static draft depth and warns. Turn DP-Attention off in the Attention card above.",
|
||||
},
|
||||
{
|
||||
when: { hw: ["mi300x", "mi325x", "mi355x"] },
|
||||
reason: "MTP speculative decoding has not been validated for GLM-5.3-Flash on AMD ROCm; the Strategy row disables Low Latency there for the same reason.",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "dflash",
|
||||
label: "DFlash2",
|
||||
// Block-wise draft: the block size comes from the draft checkpoint,
|
||||
// so no --speculative-num-draft-tokens here. The draft is a dense
|
||||
// model and does not run on the target's DSA backends, hence the
|
||||
// explicit draft attention backend.
|
||||
flags: [
|
||||
"--speculative-algorithm DFLASH",
|
||||
"--speculative-draft-model-path incoai/GLM-5.3-Flash-DFlash2",
|
||||
"--speculative-draft-attention-backend fa4",
|
||||
],
|
||||
// DFLASH needs this model's hidden-state capture, which landed on the
|
||||
// GLM-5.3-Flash support branch (PR #36708 into #36507's
|
||||
// xinyuan/glm-5.3-flash-support), not on main — so it postdates the
|
||||
// image the Install accordion pins. Drop this note once #36507 merges
|
||||
// and a published image carries it.
|
||||
note: "⚠️ Needs the GLM-5.3-Flash hidden-state capture from PR #36708. It is merged into the PR #36507 support branch (xinyuan/glm-5.3-flash-support), not into main, so pull that branch at its current head — or add #36708's commit on top of an older checkout — before serving. The lmsysorg/sglang:glm-5.3-flash image alone is not enough.",
|
||||
disable: [
|
||||
{
|
||||
when: { dpAttnOn: [true] },
|
||||
reason: "DFLASH speculative decoding does not support DP-Attention — the server rejects the combination at startup. Turn DP-Attention off in the Attention card above.",
|
||||
},
|
||||
{
|
||||
when: { hw: ["mi300x", "mi325x", "mi355x"] },
|
||||
reason: "DFLASH speculative decoding only supports CUDA and NPU devices; the server rejects it on ROCm at startup.",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
|
||||
},
|
||||
|
||||
cells: [
|
||||
|
||||
Reference in New Issue
Block a user