diff --git a/docs_new/cookbook/autoregressive/Google/Gemma4.mdx b/docs_new/cookbook/autoregressive/Google/Gemma4.mdx index b8bd13879..20a039895 100644 --- a/docs_new/cookbook/autoregressive/Google/Gemma4.mdx +++ b/docs_new/cookbook/autoregressive/Google/Gemma4.mdx @@ -113,6 +113,7 @@ For other installation methods, please refer to the [official SGLang installatio - **Attention backend on Blackwell (B200/sm100)**: SGLang defaults to the `trtllm_mha` backend on sm100, which is fastest for text but applies *causal* attention to image tokens. For multimodal (image) workloads on B200, pass `--attention-backend triton` to restore bidirectional image-token attention and full vision quality. Text-only and audio workloads are unaffected by the default. - For the 26B-A4B MoE model, consider `--tp 2` for high-throughput workloads. - **Speculative Decoding (MTP)**: Each Gemma 4 variant ships with a paired `*-assistant` draft model that enables NEXTN multi-token prediction. Enable it via the selector above, or pass `--speculative-algorithm NEXTN --speculative-draft-model-path google/gemma-4--it-assistant --speculative-num-steps 5 --speculative-num-draft-tokens 6 --speculative-eagle-topk 1`. MTP can significantly reduce latency for interactive use cases. The 26B-A4B MoE model requires `--tp 2` when MTP is enabled. +- **QAT checkpoints**: Toggle **Checkpoint → QAT** in the selector to target the `qat-q4_0-unquantized` releases. These keep bf16 weights, so memory and TP requirements match the standard checkpoints, and each has a matching `*-qat-q4_0-unquantized-assistant` draft model for MTP. - Hardware requirements: diff --git a/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx b/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx index 36e60e5f9..167c49e18 100644 --- a/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx @@ -11,6 +11,14 @@ export const Gemma4Deployment = () => { { id: '26b-a4b', label: '26B-A4B (MoE)', default: false }, ] }, + checkpoint: { + name: 'checkpoint', + title: 'Checkpoint', + items: [ + { id: 'standard', label: 'Standard', subtitle: 'BF16', default: true }, + { id: 'qat', label: 'QAT', subtitle: 'q4_0-unquantized', default: false }, + ] + }, hardware: { name: 'hardware', title: 'Hardware Platform', @@ -90,12 +98,17 @@ export const Gemma4Deployment = () => { '26b-a4b': 'google/gemma-4-26B-A4B-it', }; + // QAT releases keep bf16 weights (q4_0-unquantized), so the only change is + // the model-path suffix; TP/memory requirements match the standard checkpoints. + const qatSuffix = values.checkpoint === 'qat' ? '-qat-q4_0-unquantized' : ''; + const modelPath = `${modelNames[modelSize]}${qatSuffix}`; + const mtpEnabled = values.speculative === 'enabled'; if (mtpEnabled && modelSize === '26b-a4b' && hardware !== 'mi300x') { tp = 2; } - let cmd = `sglang serve --model-path ${modelNames[modelSize]}`; + let cmd = `sglang serve --model-path ${modelPath}`; if (tp > 1) { cmd += ` \\\n --tp ${tp}`; } @@ -110,7 +123,7 @@ export const Gemma4Deployment = () => { if (mtpEnabled) { cmd += ` \\\n --speculative-algorithm NEXTN`; - cmd += ` \\\n --speculative-draft-model-path ${modelNames[modelSize]}-assistant`; + cmd += ` \\\n --speculative-draft-model-path ${modelPath}-assistant`; cmd += ` \\\n --speculative-num-steps 5`; cmd += ` \\\n --speculative-num-draft-tokens 6`; cmd += ` \\\n --speculative-eagle-topk 1`;