diff --git a/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx b/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx index ff5055dd9..ab5e76ab6 100644 --- a/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx +++ b/docs_new/cookbook/autoregressive/NVIDIA/Nemotron3-Ultra.mdx @@ -60,7 +60,11 @@ The generator only emits a runnable command for combinations that NVIDIA / SGLan - **Attention backend**: **H100/H200**: Use flash attention 3 backend by default. - **B200/GB200/B300/GB300**: Use flashinfer backend by default. + **B200/GB200/B300/GB300**: Append `--attention-backend trtllm_mha`. The flashinfer default breaks the overlap scheduler on Blackwell, so `trtllm_mha` is required there. + +- **Mamba scheduler strategy**: + + Always launch with `--mamba-scheduler-strategy extra_buffer`. This hybrid Transformer-Mamba model requires the `extra_buffer` strategy for correct scheduling of its Mamba state. - **TP support**: @@ -76,6 +80,14 @@ The generator only emits a runnable command for combinations that NVIDIA / SGLan By default the attention layers are tensor-parallel (sharded across all TP ranks). Enabling DP attention (the toggle above, or `--dp --enable-dp-attention`) instead runs attention as `N` data-parallel groups: each DP rank serves its own slice of the requests with its own KV cache. `--dp` must divide `--tp`. +- **Expert parallel (EP)**: + + This MoE only supports `ep_size == 1` (off) or `ep_size == tp_size`. To enable expert parallelism, append `--ep ` with the same value as `--tp`. + +- **Multi-token prediction (MTP)**: + + Enable MTP speculative decoding (the toggle above) for low latency. + - **FP8 KV cache**: To enable fp8 kv cache, please append `--kv-cache-dtype fp8_e4m3`. @@ -93,10 +105,10 @@ The generator only emits a runnable command for combinations that NVIDIA / SGLan ```shell Command python3 -m sglang.launch_server \ --model-path nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16 \ - --host 0.0.0.0 \ - --port 5000 \ --trust-remote-code \ --tp 8 \ + --mamba-scheduler-strategy extra_buffer \ + --attention-backend trtllm_mha \ --tool-call-parser qwen3_coder \ --reasoning-parser nemotron_3 ``` @@ -109,7 +121,7 @@ SGLang provides an OpenAI-compatible endpoint. Example with the OpenAI Python cl from openai import OpenAI client = OpenAI( - base_url="http://localhost:5000/v1", + base_url="http://localhost:30000/v1", api_key="EMPTY", ) @@ -140,7 +152,7 @@ Streaming chat completion: from openai import OpenAI client = OpenAI( - base_url="http://localhost:5000/v1", + base_url="http://localhost:30000/v1", api_key="EMPTY", ) @@ -181,7 +193,7 @@ The model supports two modes — Reasoning ON (default) vs OFF. This can be togg from openai import OpenAI client = OpenAI( - base_url="http://localhost:5000/v1", + base_url="http://localhost:30000/v1", api_key="EMPTY", ) @@ -238,7 +250,7 @@ Call functions using the OpenAI Tools schema and inspect returned `tool_calls`. from openai import OpenAI client = OpenAI( - base_url="http://localhost:5000/v1", + base_url="http://localhost:30000/v1", api_key="EMPTY", ) @@ -268,7 +280,7 @@ TOOLS = [ ] completion = client.chat.completions.create( - model="nemotron", + model="nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", messages=[ {"role": "system", "content": "You are a coding agent. Use tools to inspect the repo before answering."}, {"role": "user", "content": "Where is the `RadixCache` class defined?"} @@ -374,7 +386,7 @@ SERVED_MODEL_NAME = "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16" # Client client = ThinkingBudgetClient( - base_url="http://127.0.0.1:5000/v1", + base_url="http://127.0.0.1:30000/v1", api_key="null", tokenizer_name_or_path=SERVED_MODEL_NAME ) @@ -425,9 +437,9 @@ python3 -m sglang.launch_server \ --model-path nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 \ --trust-remote-code \ --tp 4 \ - --max-running-requests 1024 \ - --host 0.0.0.0 \ - --port 5000 + --mamba-scheduler-strategy extra_buffer \ + --attention-backend trtllm_mha \ + --max-running-requests 1024 ``` - Benchmark Command: @@ -435,8 +447,6 @@ python3 -m sglang.launch_server \ ```shell Command python3 -m sglang.bench_serving \ --backend sglang \ - --host 0.0.0.0 \ - --port 5000 \ --model nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 \ --dataset-name random \ --random-input-len 1024 \ @@ -503,12 +513,14 @@ python3 -m sglang.launch_server \ --model-path nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 \ --trust-remote-code \ --tp 4 \ + --mamba-scheduler-strategy extra_buffer \ + --attention-backend trtllm_mha \ --reasoning-parser nemotron_3 ``` **Run Benchmark** ```bash Command -python3 benchmark/gsm8k/bench_sglang.py --port 5000 +python3 benchmark/gsm8k/bench_sglang.py ``` **Test Results:** @@ -523,7 +535,7 @@ Output throughput: 745.333 token/s **Run Benchmark** ```bash Command -python3 benchmark/mmlu/bench_sglang.py --port 5000 +python3 benchmark/mmlu/bench_sglang.py ``` **Test Results:** diff --git a/docs_new/src/snippets/autoregressive/nemotron3-ultra-deployment.jsx b/docs_new/src/snippets/autoregressive/nemotron3-ultra-deployment.jsx index f03b57fb5..ed29d705f 100644 --- a/docs_new/src/snippets/autoregressive/nemotron3-ultra-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/nemotron3-ultra-deployment.jsx @@ -97,6 +97,16 @@ export const Nemotron3UltraDeployment = () => { }); } }, + ep: { + name: 'ep', + title: 'Expert Parallel (EP)', + items: [ + { id: 'enabled', label: 'Enabled', subtitle: 'EP = TP' }, + { id: 'disabled', label: 'Disabled', default: true } + ], + // This MoE only supports ep_size == 1 or ep_size == tp_size; when on, EP equals TP. + commandRule: (value, state) => value === 'enabled' ? `--ep ${state.tp}` : null + }, dpattention: { name: 'dpattention', title: 'DP Attention', @@ -126,16 +136,6 @@ export const Nemotron3UltraDeployment = () => { ? `--dp ${value} \\\n --enable-dp-attention` : null }, - ep: { - name: 'ep', - title: 'Expert Parallel (EP)', - items: [ - { id: 'enabled', label: 'Enabled', subtitle: 'EP = TP' }, - { id: 'disabled', label: 'Disabled', default: true } - ], - // This MoE only supports ep_size == 1 or ep_size == tp_size; when on, EP equals TP. - commandRule: (value, state) => value === 'enabled' ? `--ep ${state.tp}` : null - }, mtp: { name: 'mtp', title: 'Multi-token Prediction (MTP)', @@ -143,8 +143,7 @@ export const Nemotron3UltraDeployment = () => { { id: 'enabled', label: 'Enabled', default: true }, { id: 'disabled', label: 'Disabled', default: false } ], - // On Blackwell, the flashinfer default breaks the spec-v2 overlap scheduler, so override to trtllm_mha - commandRule: (value, state) => value === 'enabled' ? '--speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4 \\\n --mamba-scheduler-strategy extra_buffer' + (['b200', 'gb200', 'b300', 'gb300'].includes(state.hardware) ? ' \\\n --attention-backend trtllm_mha' : '') : null + commandRule: (value) => value === 'enabled' ? '--speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4' : null }, kvcache: { name: 'kvcache', @@ -208,12 +207,25 @@ export const Nemotron3UltraDeployment = () => { const modelPath = MODEL_PATHS[model] || MODEL_PATHS['bf16']; - const specV2Env = values.mtp === 'enabled' ? 'SGLANG_ENABLE_SPEC_V2=1 ' : ''; - let cmd = `${specV2Env}python3 -m sglang.launch_server \\\n`; + let cmd = `python3 -m sglang.launch_server \\\n`; cmd += ` --model-path ${modelPath} \\\n`; cmd += ` --trust-remote-code \\\n`; cmd += ` --tp ${tp} \\\n`; + for (const [key, option] of Object.entries(options)) { + if (option.commandRule) { + const rule = option.commandRule(values[key], values); + if (rule) { + cmd += ` ${rule} \\\n`; + } + } + } + + cmd += ` --mamba-scheduler-strategy extra_buffer \\\n`; + if (['b200', 'gb200', 'b300', 'gb300'].includes(hardware)) { + cmd += ` --attention-backend trtllm_mha \\\n`; + } + if (kvcache && kvcache !== 'none') { cmd += ` --kv-cache-dtype ${kvcache} \\\n`; } @@ -224,15 +236,6 @@ export const Nemotron3UltraDeployment = () => { cmd += ` --node-rank <0|1> \\\n`; } - for (const [key, option] of Object.entries(options)) { - if (option.commandRule) { - const rule = option.commandRule(values[key], values); - if (rule) { - cmd += ` ${rule} \\\n`; - } - } - } - cmd = cmd.trimEnd(); if (cmd.endsWith('\\')) { cmd = cmd.slice(0, -1).trimEnd();