diff --git a/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx b/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx index 14afc85c9..28e1b4e90 100644 --- a/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx +++ b/docs_new/cookbook/autoregressive/GLM/GLM-5.1.mdx @@ -43,6 +43,7 @@ import { GLM51Deployment } from '/src/snippets/autoregressive/glm-51-deployment. NVFP4 FP8 BF16 + MXFP4 @@ -51,43 +52,49 @@ import { GLM51Deployment } from '/src/snippets/autoregressive/glm-51-deployment. — tp=16 — + — H200 — tp=8 — + — B300 tp=8 — — + — GB300 tp=4 — — + — MI300X/MI325X — tp=8 tp=8 + — MI355X — tp=8 tp=8 + tp=4 - **H100 and H200**: FP8 is the recommended deployment path. - **B300 and GB300**: NVFP4 is the recommended deployment path. Use `nvidia/GLM-5.1-NVFP4` with `--quantization modelopt_fp4`. Use `tp=8` on B300 and `tp=4` on GB300. The CUDA 13 image variant is required for B300 and GB300. -- **AMD GPUs**: Both BF16 and FP8 checkpoints are supported on MI300X/MI325X/MI355X at tp=8. Use `--dsa-prefill-backend tilelang --dsa-decode-backend tilelang` for the DSA attention backend. Add `--chunked-prefill-size 131072` and `--watchdog-timeout 1200` (20 minutes for weight loading). FP8 uses approximately half the memory of BF16 (~89 GB/GPU vs ~175 GB/GPU). EAGLE speculative decoding is not currently supported on AMD for GLM-5.1. +- **AMD GPUs**: BF16 and FP8 checkpoints run on MI300X/MI325X/MI355X at tp=8. On MI355X (gfx950), the MXFP4 checkpoint `amd/GLM-5.1-MXFP4` is also supported at tp=4 with `--kv-cache-dtype fp8_e4m3`. All AMD paths pass `--dsa-prefill-backend tilelang --dsa-decode-backend tilelang`, `--chunked-prefill-size 131072`, and `--watchdog-timeout 1200` (20 minutes for weight loading). FP8 uses approximately half the memory of BF16 (~89 GB/GPU vs ~175 GB/GPU). EAGLE speculative decoding is supported on MI355X (gfx950) and unverified on MI300X/MI325X (gfx942). - For other configuration tips (MTP, DSA kernel, Context Parallel, HiSparse, NVFP4, Index Cache), see the [DeepSeek-V3.2 cookbook page](../DeepSeek/DeepSeek-V3_2). GLM-5.1 and DeepSeek-V3.2 share the same model structure, so the optimization techniques are common. - Use `--json-model-override-args '{"index_topk_pattern": "FFSFSSSFSSFFFSSSFFFSFSSSSSSFFSFFSFFSSFFFFFFSFFFFFSFFSSSSSSFSFFFSFSSSFSFFSFFSSS"}'` to enable the [IndexCache](https://github.com/THUDM/IndexCache) method for GLM-5.1. This can improve serving efficiency with only a small accuracy loss. If you are running rigorous accuracy evaluations, do not enable this feature. @@ -154,6 +161,32 @@ sglang serve \ The following ROCm commands are additional options for AMD GPUs and do not replace the NVIDIA instructions above. +#### MXFP4 (MI355X / gfx950) + +On MI355X (gfx950), set `SGLANG_DSA_TRITON_PREFILL=1` to enable a faster Triton attention kernel for the prefill phase (opt-in, off by default). Keep `--dsa-prefill-backend tilelang` as shown. The EAGLE speculative-decoding flags below are optional but recommended on gfx950. + +```shell Command +# SGLANG_DSA_TRITON_PREFILL=1 is optional; it enables a faster Triton prefill kernel on gfx950 +SGLANG_DSA_TRITON_PREFILL=1 sglang serve \ + --model-path amd/GLM-5.1-MXFP4 \ + --tp 4 \ + --trust-remote-code \ + --kv-cache-dtype fp8_e4m3 \ + --tool-call-parser glm47 \ + --reasoning-parser glm45 \ + --dsa-prefill-backend tilelang \ + --dsa-decode-backend tilelang \ + --chunked-prefill-size 131072 \ + --mem-fraction-static 0.85 \ + --watchdog-timeout 1200 \ + --speculative-algorithm EAGLE \ + --speculative-num-steps 3 \ + --speculative-eagle-topk 1 \ + --speculative-num-draft-tokens 4 \ + --host 0.0.0.0 \ + --port 30000 +``` + #### FP8 (Recommended) ```shell Command diff --git a/docs_new/src/snippets/autoregressive/glm-51-deployment.jsx b/docs_new/src/snippets/autoregressive/glm-51-deployment.jsx index a5cd90fc8..e2cfa33a5 100644 --- a/docs_new/src/snippets/autoregressive/glm-51-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/glm-51-deployment.jsx @@ -4,7 +4,9 @@ export const GLM51Deployment = () => { // Recommended quantization per hardware: // H100 / H200 → FP8 // B300 / GB300 → NVFP4 - // MI300X/MI325X/MI355X → BF16 (FP8 not verified on AMD) + // MI300X / MI325X → BF16 (FP8 not verified on AMD) + // MI355X (gfx950) → MXFP4 (amd/GLM-5.1-MXFP4); BF16 also supported. + // MI350X is identical to MI355X (cooling only) and is omitted here. const options = { hardware: { name: 'hardware', @@ -25,11 +27,13 @@ export const GLM51Deployment = () => { getDynamicItems: (values) => { const hw = values.hardware; const isAMD = ['mi300x', 'mi325x', 'mi355x'].includes(hw); + const isGfx950 = hw === 'mi355x'; // MI350X identical (cooling only) const supportsNVFP4 = ['b300', 'gb300'].includes(hw); const isB300 = hw === 'b300'; const isGB300 = hw === 'gb300'; return [ - { id: 'bf16', label: 'BF16', subtitle: 'Full Weights', default: isAMD, disabled: !isAMD, disabledReason: supportsNVFP4 ? 'NVFP4 is recommended for this hardware' : 'FP8 is recommended for this hardware' }, + { id: 'mxfp4', label: 'MXFP4', subtitle: 'gfx950', default: isGfx950, disabled: !isGfx950, disabledReason: !isGfx950 ? 'MXFP4 verified on MI355X (gfx950)' : '' }, + { id: 'bf16', label: 'BF16', subtitle: 'Full Weights', default: isAMD && !isGfx950, disabled: !isAMD, disabledReason: supportsNVFP4 ? 'NVFP4 is recommended for this hardware' : 'FP8 is recommended for this hardware' }, { id: 'fp8', label: 'FP8', subtitle: 'High Throughput', default: !isAMD && !supportsNVFP4, disabled: isAMD || supportsNVFP4, disabledReason: isAMD ? 'FP8 not verified on AMD' : (supportsNVFP4 ? 'NVFP4 is recommended for this hardware' : '') }, { id: 'nvfp4', label: 'NVFP4', subtitle: 'Blackwell FP4', default: isB300 || isGB300, disabled: !supportsNVFP4, disabledReason: !supportsNVFP4 ? 'NVFP4 only on B300/GB300' : '' } ]; @@ -58,15 +62,6 @@ export const GLM51Deployment = () => { { id: 'disabled', label: 'Disabled', subtitle: 'Low Latency', default: true }, { id: 'enabled', label: 'Enabled', subtitle: 'High Throughput', default: false } ] - }, - speculative: { - name: 'speculative', - title: 'Speculative Decoding', - condition: (values) => !['mi300x', 'mi325x', 'mi355x'].includes(values.hardware), - items: [ - { id: 'disabled', label: 'Disabled', default: false }, - { id: 'enabled', label: 'Enabled', default: true } - ] } }; @@ -77,7 +72,7 @@ export const GLM51Deployment = () => { gb300: { nvfp4: { tp: 4, mem: 0.80 }, fp8: { tp: 4, mem: 0.9 } }, mi300x: { bf16: { tp: 8, mem: 0.80 } }, mi325x: { bf16: { tp: 8, mem: 0.80 } }, - mi355x: { bf16: { tp: 8, mem: 0.80 } } + mi355x: { bf16: { tp: 8, mem: 0.80 }, mxfp4: { tp: 4, mem: 0.85 } } }; const resolveItems = (option, values) => { @@ -135,17 +130,22 @@ export const GLM51Deployment = () => { const generateCommand = () => { const { hardware, quantization } = values; const isAMD = ['mi300x', 'mi325x', 'mi355x'].includes(hardware); + const isGfx950 = hardware === 'mi355x'; // MI350X identical (cooling only) const recommendsNVFP4 = ['b300', 'gb300'].includes(hardware); - const effectiveQuant = isAMD ? 'bf16' : (recommendsNVFP4 ? 'nvfp4' : 'fp8'); + const effectiveQuant = isAMD + ? (isGfx950 && quantization === 'mxfp4' ? 'mxfp4' : 'bf16') + : (recommendsNVFP4 ? 'nvfp4' : 'fp8'); const suffix = effectiveQuant === 'fp8' ? '-FP8' : ''; - const modelName = effectiveQuant === 'nvfp4' ? 'nvidia/GLM-5.1-NVFP4' : `zai-org/GLM-5.1${suffix}`; + const modelName = + effectiveQuant === 'nvfp4' ? 'nvidia/GLM-5.1-NVFP4' + : effectiveQuant === 'mxfp4' ? 'amd/GLM-5.1-MXFP4' + : `zai-org/GLM-5.1${suffix}`; const hwConfig = modelConfigs[hardware][effectiveQuant]; if (!hwConfig) return '# Configuration not available for the selected hardware and quantization.'; const tpValue = hwConfig.tp; const memFraction = hwConfig.mem; - const enableSpec = values.speculative === 'enabled'; let cmd = 'sglang serve \\\n'; cmd += ` --model-path ${modelName}`; @@ -158,6 +158,7 @@ export const GLM51Deployment = () => { if (isAMD) { cmd += ' \\\n --trust-remote-code'; + if (effectiveQuant === 'mxfp4') cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; cmd += ' \\\n --dsa-prefill-backend tilelang'; cmd += ' \\\n --dsa-decode-backend tilelang'; cmd += ' \\\n --chunked-prefill-size 131072'; @@ -169,7 +170,9 @@ export const GLM51Deployment = () => { } if (values.reasoning === 'enabled') cmd += ' \\\n --reasoning-parser glm45'; if (values.toolcall === 'enabled') cmd += ' \\\n --tool-call-parser glm47'; - if (enableSpec) { + // EAGLE MTP speculative decoding: emitted by default (recommended). Excluded + // only on MI300X/MI325X (gfx942), where it is not yet verified. + if (!['mi300x', 'mi325x'].includes(hardware)) { cmd += ' \\\n --speculative-algorithm EAGLE'; cmd += ' \\\n --speculative-num-steps 3'; cmd += ' \\\n --speculative-eagle-topk 1';