diff --git a/docs/cookbook/autoregressive/NVIDIA/Nemotron3.5-Lightning.mdx b/docs/cookbook/autoregressive/NVIDIA/Nemotron3.5-Lightning.mdx new file mode 100644 index 000000000..9f6dc0b7d --- /dev/null +++ b/docs/cookbook/autoregressive/NVIDIA/Nemotron3.5-Lightning.mdx @@ -0,0 +1,162 @@ +--- +title: Nemotron3.5-Lightning +description: "Deploy NVIDIA Nemotron 3.5 Lightning with SGLang — NVFP4 serving with MTP, DFlash, and DSpark speculative decoding, reasoning, and tool calling." +tag: NEW +--- + +## Deployment + + + + + +For all methods and hardware platforms, see the [official SGLang installation guide](../../../docs/get-started/install). The two paths below match the **Python / Docker** toggle in the command panel. + + + + + +```bash Command +pip install --upgrade pip +pip install uv +SGLANG_BUILD_RUST_EXTS=none uv pip install 'git+https://github.com/sgl-project/sglang.git@refs/pull/33554/head#subdirectory=python' +``` + +Then run the **Python** output of the command panel below in that environment. + + + + + +```bash Command +docker pull lmsysorg/sglang:dev-nemotron3-5-lightning +``` + +For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces. + + + + + + + +Pick your hardware and recipe to generate the launch command. Every platform publishes four operating points: **Balanced** (no speculation) plus three speculative decoders — **MTP**, **DFlash**, and **DSpark**. Use the Playground below to explore knobs beyond them. + +import { Deployment } from "/src/snippets/_deployment.jsx"; +import { config } from "/src/snippets/configs/nvidia/nemotron-3.5-lightning.jsx"; +import { benchmarks } from "/src/snippets/configs/nvidia/nemotron-3.5-lightning-benchmarks.jsx"; + + + +## Playground + +The Playground is where you experiment with **SGLang features beyond the verified matrix**. The Deploy panel above only emits combinations the SGLang team has signed off on; the Playground lets you turn on additional knobs on top of whichever cell the Deploy panel is currently showing. + +import { Playground } from "/src/snippets/_playground.jsx"; + + + +## 1. Model Introduction + +**NVIDIA Nemotron 3.5 Lightning** is a 30B-A3B hybrid reasoning LLM. See the Hugging Face model cards below for architecture and evaluation details. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CheckpointPrecisionUse
NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4NVFP4Serving — the checkpoint this page deploys
NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16BF16Full-precision reference
NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DFlashW4A16DFlash speculative draft model
NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSparkW4A16DSpark speculative draft model
+ +MTP needs no separate download — the draft head is embedded in the target checkpoint. + +## 2. Usage + +The server speaks the OpenAI API. With `--reasoning-parser nemotron_3` enabled, the thinking trace lands in `message.reasoning_content` and the answer in `message.content`. + +```python Example +from openai import OpenAI + +client = OpenAI( + base_url="http://127.0.0.1:8000/v1", + api_key="null", +) + +response = client.chat.completions.create( + model="nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Briefly explain: what is SGLang?"}, + ], + temperature=1.0, + top_p=0.95, + max_tokens=1024, +) +choice = response.choices[0] +print("Reasoning:", choice.message.reasoning_content) +print("Content:", choice.message.content) +``` + +### 2.1 Tool Calling + +With `--tool-call-parser qwen3_coder` enabled, structured tool calls are returned in `message.tool_calls`. + +```python Example +from openai import OpenAI + +client = OpenAI( + base_url="http://127.0.0.1:8000/v1", + api_key="null", +) + +TOOLS = [ + { + "type": "function", + "function": { + "name": "calculate_tip", + "parameters": { + "type": "object", + "properties": { + "bill_total": {"type": "integer", "description": "The total amount of the bill"}, + "tip_percentage": {"type": "integer", "description": "The percentage of tip to be applied"}, + }, + "required": ["bill_total", "tip_percentage"], + }, + }, + } +] + +response = client.chat.completions.create( + model="nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", + messages=[{"role": "user", "content": "My bill is $50. What will be the amount for 15% tip?"}], + tools=TOOLS, + max_tokens=1024, +) +choice = response.choices[0] +print("Content:", choice.message.content) +print("Tool calls:", choice.message.tool_calls) +``` diff --git a/docs/cookbook/autoregressive/intro.mdx b/docs/cookbook/autoregressive/intro.mdx index a7de3cbba..43e8e81ed 100644 --- a/docs/cookbook/autoregressive/intro.mdx +++ b/docs/cookbook/autoregressive/intro.mdx @@ -82,7 +82,7 @@ metatags: " }, + CURL_HOST: { target: "curl", label: "Server host", default: "localhost" }, + CURL_PORT: { target: "curl", label: "Server port", default: "30000" }, + }, + + curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\ +-H 'Content-Type: application/json' \\ +-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`, + + benchmarkCommands: { + speed: +`python3 -m sglang.bench_serving \\ + --backend sglang \\ + --host {{CURL_HOST}} --port {{CURL_PORT}} \\ + --model {{MODEL_NAME}} \\ + --dataset-name {{DATASET}} \\ + --random-input-len {{ISL}} --random-output-len {{OSL}} \\ + --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\ + --flush-cache`, + accuracy: { + gsm8k_pct: +`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval +sgl-eval run gsm8k \\ + --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ + --num-threads 32`, + }, + numPromptsByConc: { 1: 8, 16: 32, 64: 128, 256: 512, 1024: 2048, 4096: 4096 }, + }, + + accuracyLabels: [["gsm8k_pct", "GSM8K", "%"]], + + dockerImages: { + // Multi-arch index (amd64 + arm64), so one tag covers H100, B200, and GB10. + // Equivalent to dev-cu13-nemotron3-5-lightning. + b200: "lmsysorg/sglang:dev-nemotron3-5-lightning", + h100: "lmsysorg/sglang:dev-nemotron3-5-lightning", + "dgx-spark": "lmsysorg/sglang:dev-nemotron3-5-lightning", + }, + + github: { + cookbookModel: "nvidia/nemotron-3.5-lightning", + }, + + playgroundFeatures: { + attention: { + knobs: [ + { id: "tp", label: "TP", values: [null, 1, 2, 4, 8] }, + ], + }, + + moe: { + backend: { + options: [ + { id: null, label: "Inherited" }, + { id: "marlin", label: "Marlin (W4A16)", flags: ["--moe-runner-backend marlin"] }, + { id: "deepep", label: "DeepEP", flags: ["--moe-a2a-backend deepep"] }, + ], + }, + ep: { label: "EP", values: [null, 1, 2, 4, 8] }, + }, + + parsers: { + items: [ + { id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser nemotron_3" }, + { id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser qwen3_coder" }, + ], + }, + + speculative: { + options: [ + { id: "current", label: "Inherited from base" }, + { id: "off", label: "Off (greedy)" }, + { id: "mtp", label: "EAGLE / MTP", + flags: ["--speculative-algorithm EAGLE", + "--speculative-draft-model-path {{MODEL_NAME}}", + "--speculative-num-steps 5", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 6"] }, + { id: "dflash", label: "DFlash", + flags: ["--speculative-algorithm DFLASH", + "--speculative-draft-model-path nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DFlash", + "--speculative-dflash-block-size 4"] }, + { id: "dspark", label: "DSpark", + flags: ["--speculative-algorithm DSPARK", + "--speculative-draft-model-path nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", + "--speculative-dspark-block-size 3"] }, + ], + }, + }, + + cells: [ + // ==== NVIDIA B200 (SM100) + NVFP4, single GPU ==== + // The Nemotron-H resolver selects FlashInfer target attention without + // speculation and TRT-LLM MHA target/eligible-draft attention with it. + { + match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--mamba-backend flashinfer", + "--mamba-ssm-dtype float16", + "--enable-mamba-cache-stochastic-rounding", + "--mamba-cache-philox-rounds 5", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs-decode 16", + "--reasoning-parser nemotron_3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "mtp", nodes: "single" }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--mamba-backend flashinfer", + "--mamba-ssm-dtype float16", + "--enable-mamba-cache-stochastic-rounding", + "--mamba-cache-philox-rounds 5", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs-decode 16", + "--speculative-algorithm EAGLE", + "--speculative-draft-model-path {{MODEL_NAME}}", + "--speculative-num-steps 5", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 6", + "--reasoning-parser nemotron_3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // DFlash uses depth five on B200; its full-attention draft resolves to + // FlashInfer while target verification remains TRT-LLM MHA. + { + match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "dflash", nodes: "single" }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--mamba-backend flashinfer", + "--mamba-ssm-dtype float16", + "--enable-mamba-cache-stochastic-rounding", + "--mamba-cache-philox-rounds 5", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs-decode 16", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DFlash", + "--speculative-dflash-block-size 6", + "--reasoning-parser nemotron_3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "dspark", nodes: "single" }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--mamba-backend flashinfer", + "--mamba-ssm-dtype float16", + "--enable-mamba-cache-stochastic-rounding", + "--mamba-cache-philox-rounds 5", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs-decode 16", + "--speculative-algorithm DSPARK", + "--speculative-draft-model-path nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", + "--speculative-dspark-block-size 3", + "--reasoning-parser nemotron_3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // ==== NVIDIA Hopper (SM90) + NVFP4, single GPU ==== + // FA3 target attention is selected by default and inherited by the draft. + { + match: { hw: "h100", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--mamba-ssm-dtype float16", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs-decode 16", + "--reasoning-parser nemotron_3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // MTP: the draft head is embedded in the target checkpoint, so SGLang's + // EAGLE path points --speculative-draft-model-path back at the target. + { + match: { hw: "h100", variant: "default", quant: "nvfp4", strategy: "mtp", nodes: "single" }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--mamba-ssm-dtype float16", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs-decode 16", + "--speculative-algorithm EAGLE", + "--speculative-draft-model-path {{MODEL_NAME}}", + "--speculative-num-steps 5", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 6", + "--reasoning-parser nemotron_3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // DFlash: separate draft model; depth three -> block/verify width four. + { + match: { hw: "h100", variant: "default", quant: "nvfp4", strategy: "dflash", nodes: "single" }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--mamba-ssm-dtype float16", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs-decode 16", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DFlash", + "--speculative-dflash-block-size 4", + "--reasoning-parser nemotron_3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // DSpark: separate draft model; gamma three. + { + match: { hw: "h100", variant: "default", quant: "nvfp4", strategy: "dspark", nodes: "single" }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--mamba-ssm-dtype float16", + "--mem-fraction-static 0.85", + "--cuda-graph-max-bs-decode 16", + "--speculative-algorithm DSPARK", + "--speculative-draft-model-path nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", + "--speculative-dspark-block-size 3", + "--reasoning-parser nemotron_3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // ==== NVIDIA DGX Spark (GB10 / SM121) + NVFP4, single GPU ==== + // The Nemotron-H resolver selects Triton target attention plus FlashInfer + // draft attention for speculative decoding on SM121. + { + match: { hw: "dgx-spark", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--mamba-ssm-dtype float16", + "--mem-fraction-static 0.78", + "--cuda-graph-max-bs-decode 4", + "--reasoning-parser nemotron_3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // MTP: the draft head is embedded in the target checkpoint, so SGLang's + // EAGLE path points --speculative-draft-model-path back at the target. + { + match: { hw: "dgx-spark", variant: "default", quant: "nvfp4", strategy: "mtp", nodes: "single" }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--mamba-ssm-dtype float16", + "--mem-fraction-static 0.78", + "--cuda-graph-max-bs-decode 4", + "--speculative-algorithm EAGLE", + "--speculative-draft-model-path {{MODEL_NAME}}", + "--speculative-num-steps 5", + "--speculative-eagle-topk 1", + "--speculative-num-draft-tokens 6", + "--reasoning-parser nemotron_3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // DFlash: separate draft model; depth three -> block/verify width four. + { + match: { hw: "dgx-spark", variant: "default", quant: "nvfp4", strategy: "dflash", nodes: "single" }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--mamba-ssm-dtype float16", + "--mem-fraction-static 0.78", + "--cuda-graph-max-bs-decode 4", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DFlash", + "--speculative-dflash-block-size 4", + "--reasoning-parser nemotron_3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // DSpark: separate draft model; gamma three. + { + match: { hw: "dgx-spark", variant: "default", quant: "nvfp4", strategy: "dspark", nodes: "single" }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--mamba-ssm-dtype float16", + "--mem-fraction-static 0.78", + "--cuda-graph-max-bs-decode 4", + "--speculative-algorithm DSPARK", + "--speculative-draft-model-path nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4-DSpark", + "--speculative-dspark-block-size 3", + "--reasoning-parser nemotron_3", + "--tool-call-parser qwen3_coder", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + ], +};