From efbe67d2378770a39f475565bdd81146276cd58b Mon Sep 17 00:00:00 2001 From: Xiaoyu Zhang <1182563586@qq.com> Date: Thu, 25 Jun 2026 11:31:25 +0800 Subject: [PATCH] Tune Gemma4 26B-A4B B200 memory recipe (#29252) --- docs_new/cookbook/autoregressive/Google/Gemma4.mdx | 1 + docs_new/src/snippets/autoregressive/gemma4-deployment.jsx | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs_new/cookbook/autoregressive/Google/Gemma4.mdx b/docs_new/cookbook/autoregressive/Google/Gemma4.mdx index 20a039895..2c799ca3b 100644 --- a/docs_new/cookbook/autoregressive/Google/Gemma4.mdx +++ b/docs_new/cookbook/autoregressive/Google/Gemma4.mdx @@ -111,6 +111,7 @@ For other installation methods, please refer to the [official SGLang installatio - SGLang automatically selects the Triton attention backend for Gemma 4 models (required for bidirectional image-token attention during prefill). - **Attention backend on Blackwell (B200/sm100)**: SGLang defaults to the `trtllm_mha` backend on sm100, which is fastest for text but applies *causal* attention to image tokens. For multimodal (image) workloads on B200, pass `--attention-backend triton` to restore bidirectional image-token attention and full vision quality. Text-only and audio workloads are unaffected by the default. +- **Gemma 4 26B-A4B on B200**: Use `--mem-fraction-static 0.75` to leave workspace headroom for the Triton MoE path. - For the 26B-A4B MoE model, consider `--tp 2` for high-throughput workloads. - **Speculative Decoding (MTP)**: Each Gemma 4 variant ships with a paired `*-assistant` draft model that enables NEXTN multi-token prediction. Enable it via the selector above, or pass `--speculative-algorithm NEXTN --speculative-draft-model-path google/gemma-4--it-assistant --speculative-num-steps 5 --speculative-num-draft-tokens 6 --speculative-eagle-topk 1`. MTP can significantly reduce latency for interactive use cases. The 26B-A4B MoE model requires `--tp 2` when MTP is enabled. - **QAT checkpoints**: Toggle **Checkpoint → QAT** in the selector to target the `qat-q4_0-unquantized` releases. These keep bf16 weights, so memory and TP requirements match the standard checkpoints, and each has a matching `*-qat-q4_0-unquantized-assistant` draft model for MTP. diff --git a/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx b/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx index 89b102e88..6dd7d3e84 100644 --- a/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx @@ -75,7 +75,7 @@ export const Gemma4Deployment = () => { e4b: { tp: 1, mem: 0.9 }, '12b': { tp: 1, mem: 0.9 }, '31b': { tp: 1, mem: 0.9 }, - '26b-a4b': { tp: 1, mem: 0.9 }, + '26b-a4b': { tp: 1, mem: 0.75 }, }, b300: { e2b: { tp: 1, mem: 0.9 },