diff --git a/docs_new/cookbook/autoregressive/Google/Gemma4.mdx b/docs_new/cookbook/autoregressive/Google/Gemma4.mdx index f0910da21..a462c2fa5 100644 --- a/docs_new/cookbook/autoregressive/Google/Gemma4.mdx +++ b/docs_new/cookbook/autoregressive/Google/Gemma4.mdx @@ -94,6 +94,7 @@ For the full Docker setup and other installation methods, please refer to the [o - SGLang automatically selects the Triton attention backend for Gemma 4 models (required for bidirectional image-token attention during prefill). - For the 26B-A4B MoE model, consider `--tp 2` for high-throughput workloads. +- **Speculative Decoding (MTP)**: Each Gemma 4 variant ships with a paired `*-assistant` draft model that enables NEXTN multi-token prediction. Enable it via the selector above, or pass `--speculative-algorithm NEXTN --speculative-draft-model-path google/gemma-4--it-assistant --speculative-num-steps 5 --speculative-num-draft-tokens 6 --speculative-eagle-topk 1`. MTP can significantly reduce latency for interactive use cases. The 26B-A4B MoE model requires `--tp 2` when MTP is enabled. - Hardware requirements: @@ -159,6 +160,60 @@ sglang serve --model-path google/gemma-4-26B-A4B-it \ --host 0.0.0.0 --port 30000 ``` +#### Speculative Decoding (MTP) Server Commands + +Each Gemma 4 variant ships with a paired `*-assistant` draft model for NEXTN multi-token prediction. Use the commands below to enable MTP for the corresponding target model. These match the configuration generated when you toggle **Speculative Decoding (MTP) → Enabled** in the [interactive selector](#31-basic-configuration). + +```bash Command +# Gemma 4 E2B + MTP +sglang serve \ + --model-path google/gemma-4-E2B-it \ + --speculative-algorithm NEXTN \ + --speculative-draft-model-path google/gemma-4-E2B-it-assistant \ + --speculative-num-steps 5 \ + --speculative-num-draft-tokens 6 \ + --speculative-eagle-topk 1 \ + --mem-fraction-static 0.85 +``` + +```bash Command +# Gemma 4 E4B + MTP +sglang serve \ + --model-path google/gemma-4-E4B-it \ + --speculative-algorithm NEXTN \ + --speculative-draft-model-path google/gemma-4-E4B-it-assistant \ + --speculative-num-steps 5 \ + --speculative-num-draft-tokens 6 \ + --speculative-eagle-topk 1 \ + --mem-fraction-static 0.85 +``` + +```bash Command +# Gemma 4 31B + MTP +sglang serve \ + --model-path google/gemma-4-31B-it \ + --tp-size 2 \ + --speculative-algorithm NEXTN \ + --speculative-draft-model-path google/gemma-4-31B-it-assistant \ + --speculative-num-steps 5 \ + --speculative-num-draft-tokens 6 \ + --speculative-eagle-topk 1 \ + --mem-fraction-static 0.85 +``` + +```bash Command +# Gemma 4 26B-A4B + MTP +sglang serve \ + --model-path google/gemma-4-26B-A4B-it \ + --tp-size 2 \ + --speculative-algorithm NEXTN \ + --speculative-draft-model-path google/gemma-4-26B-A4B-it-assistant \ + --speculative-num-steps 5 \ + --speculative-num-draft-tokens 6 \ + --speculative-eagle-topk 1 \ + --mem-fraction-static 0.85 +``` + ### 4.1 Basic Usage ```python Example @@ -1247,12 +1302,12 @@ Median ITL (ms): 29.31 **gemma-4-E2B-it** ```shell Command -$ python -m sglang.bench_one_batch --correct --model gg-hf-gg/gemma-4-E2B-it .... +$ python -m sglang.bench_one_batch --correct --model google/gemma-4-E2B-it .... prefill logits (final): tensor([[-25.3063, -2.5718, -10.3674, ..., -25.3779, -25.5181, -25.2337]], device='cuda:0') .... -$ python scripts/playground/reference_hf.py --model-path gg-hf-gg/gemma-4-E2B-it +$ python scripts/playground/reference_hf.py --model-path google/gemma-4-E2B-it .... prefill logits (final) tensor([-25.3281, -2.1367, -10.2266, ..., -25.4375, -25.5000, -25.2500], device='cuda:0', dtype=torch.float16) @@ -1262,12 +1317,12 @@ prefill logits (final) tensor([-25.3281, -2.1367, -10.2266, ..., -25.4375, -25 **gemma-4-E4B-it** ```shell Command -$ python -m sglang.bench_one_batch --correct --model gg-hf-gg/gemma-4-E4B-it .... +$ python -m sglang.bench_one_batch --correct --model google/gemma-4-E4B-it .... prefill logits (final): tensor([[-17.6478, 7.9901, -5.6505, ..., -17.5658, -17.6478, -17.7293]], device='cuda:0') .... -$ python scripts/playground/reference_hf.py --model-path gg-hf-gg/gemma-4-E4B-it +$ python scripts/playground/reference_hf.py --model-path google/gemma-4-E4B-it .... prefill logits (final) tensor([-17.5625, 8.0469, -5.5742, ..., -17.4688, -17.5625, -17.6719], device='cuda:0', dtype=torch.float16) @@ -1276,12 +1331,12 @@ prefill logits (final) tensor([-17.5625, 8.0469, -5.5742, ..., -17.4688, -17 **gemma-4-31B-it** ```shell Command -$ python -m sglang.bench_one_batch --correct --model gg-hf-gg/gemma-4-31B-it .... +$ python -m sglang.bench_one_batch --correct --model google/gemma-4-31B-it .... prefill logits (final): tensor([[-2.0748, 1.1245, -7.4356, ..., -2.1059, -2.1525, -2.2303]], device='cuda:0') .... -$ python scripts/playground/reference_hf.py --model-path gg-hf-gg/gemma-4-31B-it +$ python scripts/playground/reference_hf.py --model-path google/gemma-4-31B-it .... prefill logits (final) tensor([-2.1133, 1.2656, -7.4766, ..., -2.1523, -2.2012, -2.2695], device='cuda:0', dtype=torch.float16) diff --git a/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx b/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx index c78d4c6ad..47a907584 100644 --- a/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx @@ -41,6 +41,15 @@ export const Gemma4Deployment = () => { ], commandRule: (value) => value === 'enabled' ? '--tool-call-parser gemma4' : null }, + speculative: { + name: 'speculative', + title: 'Speculative Decoding (MTP)', + condition: (values) => !['mi300x'].includes(values.hardware), + items: [ + { id: 'disabled', label: 'Disabled', subtitle: 'Baseline', default: true }, + { id: 'enabled', label: 'Enabled', subtitle: 'Lower Latency', default: false } + ] + }, }; const modelConfigs = { @@ -68,7 +77,7 @@ export const Gemma4Deployment = () => { const hwConfig = modelConfigs[hardware]?.[modelSize]; if (!hwConfig) return `# Error: Unknown hardware/model combination`; - const { tp, mem } = hwConfig; + let { tp, mem } = hwConfig; const modelNames = { 'e2b': 'google/gemma-4-E2B-it', @@ -77,6 +86,11 @@ export const Gemma4Deployment = () => { '26b-a4b': 'google/gemma-4-26B-A4B-it', }; + const mtpEnabled = values.speculative === 'enabled'; + if (mtpEnabled && modelSize === '26b-a4b' && hardware !== 'mi300x') { + tp = 2; + } + let cmd = `sglang serve --model-path ${modelNames[modelSize]}`; if (tp > 1) { cmd += ` \\\n --tp ${tp}`; @@ -90,6 +104,14 @@ export const Gemma4Deployment = () => { } }); + if (mtpEnabled) { + cmd += ` \\\n --speculative-algorithm NEXTN`; + cmd += ` \\\n --speculative-draft-model-path ${modelNames[modelSize]}-assistant`; + cmd += ` \\\n --speculative-num-steps 5`; + cmd += ` \\\n --speculative-num-draft-tokens 6`; + cmd += ` \\\n --speculative-eagle-topk 1`; + } + cmd += ` \\\n --mem-fraction-static ${mem}`; cmd += ` \\\n --host 0.0.0.0 --port 30000`;