diff --git a/docs_new/cookbook/autoregressive/InternLM/Intern-S1.mdx b/docs_new/cookbook/autoregressive/InternLM/Intern-S1.mdx index ae68d7be0..e5047e6d6 100644 --- a/docs_new/cookbook/autoregressive/InternLM/Intern-S1.mdx +++ b/docs_new/cookbook/autoregressive/InternLM/Intern-S1.mdx @@ -4,25 +4,28 @@ metatags: description: "Deploy Intern-S1 with SGLang - community contribution guide for InternLM's Intern-S1 model deployment." --- -## πŸ“ Community Contribution Welcome +import { InternS1Deployment } from '/src/snippets/autoregressive/intern-s1-deployment.jsx'; -This guide is currently under development. We welcome community contributions! +## 1. Model Introduction -If you have experience deploying **Intern-S1** with SGLang, please help us complete this documentation. +Intern-S1 includes the large **Intern-S1** MoE model and the smaller **Intern-S1-mini** dense model. The command generator below covers BF16 and FP8 serving on NVIDIA H100/H200/B200/B300 platforms. -## πŸš€ How to Contribute +## 2. SGLang Installation -```shell Command -git clone https://github.com/YOUR_USERNAME/sglang-cookbook.git -cd sglang-cookbook -git checkout -b add-intern-s1-guide -# Edit this file and submit a PR +Refer to the [official SGLang installation guide](../../../docs/get-started/install), or install from source: + +```bash Command +uv pip install 'git+https://github.com/sgl-project/sglang.git#subdirectory=python' ``` -## πŸ“š Reference +## 3. Model Deployment -- [GLM-4.6V](../GLM/GLM-4.6V) +### 3.1 Basic Configuration ---- + -**Let's build this together!** 🌟 +### 3.2 Configuration Tips + +- FP8 checkpoints use the matching BF16 checkpoint as tokenizer path. +- B300 deployments use `--attention-backend flashinfer`. +- Enable `--reasoning-parser interns1` and `--tool-call-parser interns1` when your workload needs structured reasoning or tool-call parsing. diff --git a/docs_new/src/snippets/autoregressive/deepseek-math-v2-deployment.jsx b/docs_new/src/snippets/autoregressive/deepseek-math-v2-deployment.jsx index 5906d7877..95726df1e 100644 --- a/docs_new/src/snippets/autoregressive/deepseek-math-v2-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/deepseek-math-v2-deployment.jsx @@ -73,6 +73,13 @@ export const DeepSeekMathV2Deployment = () => { if (memFraction) { cmd += ` \\\n --mem-fraction-static ${memFraction}`; } + if (hardware === 'b300') { + cmd += ' \\\n --attention-backend flashinfer'; + if (values.dpattention !== 'enabled') { + cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion'; + cmd += ' \\\n --cuda-graph-backend-prefill disabled'; + } + } cmd += ' \\\n --host 0.0.0.0 \\\n --port 30000'; diff --git a/docs_new/src/snippets/autoregressive/deepseek-r1-advanced-deployment.jsx b/docs_new/src/snippets/autoregressive/deepseek-r1-advanced-deployment.jsx index b342d6b84..f24213932 100644 --- a/docs_new/src/snippets/autoregressive/deepseek-r1-advanced-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/deepseek-r1-advanced-deployment.jsx @@ -9,6 +9,11 @@ const lookupData = { "label": "B200", "default": true }, + { + "id": "b300", + "label": "B300", + "default": false + }, { "id": "h200", "label": "H200", @@ -182,6 +187,66 @@ const lookupData = { "fp8_gemm_backend": "flashinfer_trtllm" } }, + { + "hardware": "b300", + "quantization": "fp8", + "gpu_count": 8, + "scenario": "low-latency", + "parameters": { + "model_path": "deepseek-ai/DeepSeek-R1-0528", + "tensor_parallel_size": 8, + "kv_cache_dtype": "fp8_e4m3", + "attention_backend": "flashinfer", + "enforce_disable_flashinfer_allreduce_fusion": true, + "enable_symm_mem": true + } + }, + { + "hardware": "b300", + "quantization": "fp8", + "gpu_count": 8, + "scenario": "high-throughput", + "parameters": { + "model_path": "deepseek-ai/DeepSeek-R1-0528", + "tensor_parallel_size": 8, + "kv_cache_dtype": "fp8_e4m3", + "attention_backend": "flashinfer", + "enforce_disable_flashinfer_allreduce_fusion": true, + "enable_symm_mem": true + } + }, + { + "hardware": "b300", + "quantization": "fp4", + "gpu_count": 8, + "scenario": "low-latency", + "parameters": { + "model_path": "nvidia/DeepSeek-R1-0528-FP4-v2", + "tensor_parallel_size": 8, + "kv_cache_dtype": "fp8_e4m3", + "attention_backend": "flashinfer", + "enforce_disable_flashinfer_allreduce_fusion": true, + "moe_runner_backend": "flashinfer_cutlass", + "mem_fraction_static": 0.85, + "enable_symm_mem": true + } + }, + { + "hardware": "b300", + "quantization": "fp4", + "gpu_count": 8, + "scenario": "high-throughput", + "parameters": { + "model_path": "nvidia/DeepSeek-R1-0528-FP4-v2", + "tensor_parallel_size": 8, + "kv_cache_dtype": "fp8_e4m3", + "attention_backend": "flashinfer", + "enforce_disable_flashinfer_allreduce_fusion": true, + "moe_runner_backend": "flashinfer_cutlass", + "mem_fraction_static": 0.85, + "enable_symm_mem": true + } + }, { "hardware": "h200", "quantization": "fp8", @@ -387,7 +452,7 @@ const lookupData = { { "hardware": "h200", "quantization": "fp4", - "error": "FP4 is only available for B200 hardware. Please select FP8 quantization." + "error": "FP4 is only available for B200/B300 hardware. Please select FP8 quantization." } ] }; @@ -407,6 +472,7 @@ const fieldToFlag = { enable_flashinfer_allreduce_fusion: 'enable-flashinfer-allreduce-fusion', scheduler_recv_interval: 'scheduler-recv-interval', enable_symm_mem: 'enable-symm-mem', + enforce_disable_flashinfer_allreduce_fusion: 'enforce-disable-flashinfer-allreduce-fusion', disable_radix_cache: 'disable-radix-cache', attention_backend: 'attention-backend', moe_runner_backend: 'moe-runner-backend', @@ -497,7 +563,7 @@ const resolveItems = (option, values) => title: 'Hardware Platform', items: uiOptions.hardware .filter((option) => - ['b200', 'h200', 'mi300x', 'mi325x', 'mi355x'].includes(option.id) + ['b200', 'b300', 'h200', 'mi300x', 'mi325x', 'mi355x'].includes(option.id) ) .map((option) => ({ id: option.id, diff --git a/docs_new/src/snippets/autoregressive/deepseek-r1-basic-deployment.jsx b/docs_new/src/snippets/autoregressive/deepseek-r1-basic-deployment.jsx index 0f8cbd3b8..2893d7ba8 100644 --- a/docs_new/src/snippets/autoregressive/deepseek-r1-basic-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/deepseek-r1-basic-deployment.jsx @@ -7,6 +7,7 @@ export const DeepSeekR1BasicDeployment = () => { { id: 'h100', label: 'H100', default: false }, { id: 'h200', label: 'H200', default: false }, { id: 'b200', label: 'B200', default: true }, + { id: 'b300', label: 'B300', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, { id: 'mi355x', label: 'MI355X', default: false }, @@ -138,6 +139,18 @@ export const DeepSeekR1BasicDeployment = () => { ' \\\n --kv-cache-dtype fp8_e4m3 # Optional: enables fp8 kv cache and fp8 attention kernels to improve performance'; } + if (hardware === 'b300') { + command += ' \\\n --kv-cache-dtype fp8_e4m3'; + command += ' \\\n --attention-backend flashinfer'; + command += ' \\\n --enforce-disable-flashinfer-allreduce-fusion'; + if (quantization === 'fp4') { + command += ' \\\n --moe-runner-backend flashinfer_cutlass'; + } + if (quantization === 'fp4' || strategyValues.includes('mtp')) { + command += ' \\\n --mem-fraction-static 0.85'; + } + } + if (isXeon) { command += ' \\\n --device cpu \\\n --disable-overlap-schedule'; if (quantization === 'int8') { diff --git a/docs_new/src/snippets/autoregressive/deepseek-v32-deployment.jsx b/docs_new/src/snippets/autoregressive/deepseek-v32-deployment.jsx index ea687f5a7..2502e85d7 100644 --- a/docs_new/src/snippets/autoregressive/deepseek-v32-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/deepseek-v32-deployment.jsx @@ -3,7 +3,7 @@ export const DeepSeekV32Deployment = () => { // // Model variants: // DeepSeek-V3.2, V3.2-Exp, V3.2-Speciale β†’ deepseek-ai/ family, TP=8 - // DeepSeek-V3.2-NVFP4 β†’ nvidia/ family, B200 only, TP=4 + // DeepSeek-V3.2-NVFP4 β†’ nvidia/ family, B200/B300 only, TP=4 // DeepSeek-V3.2-MXFP4 β†’ amd/ family, MI300X/MI355X only, TP=8 const options = { hardware: { @@ -12,6 +12,7 @@ export const DeepSeekV32Deployment = () => { items: [ { id: 'h200', label: 'H200', default: true }, { id: 'b200', label: 'B200', default: false }, + { id: 'b300', label: 'B300', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi355x', label: 'MI355X', default: false } ] @@ -21,13 +22,13 @@ export const DeepSeekV32Deployment = () => { title: 'Model Name', getDynamicItems: (values) => { const hw = values.hardware; - const isB200 = hw === 'b200'; + const isBlackwell = hw === 'b200' || hw === 'b300'; const isAMD = hw === 'mi300x' || hw === 'mi355x'; return [ - { id: 'v32', label: 'DeepSeek-V3.2', default: !isB200 && !isAMD }, + { id: 'v32', label: 'DeepSeek-V3.2', default: !isBlackwell && !isAMD }, { id: 'v32speciale', label: 'DeepSeek-V3.2-Speciale', default: false }, { id: 'v32exp', label: 'DeepSeek-V3.2-Exp', default: false }, - { id: 'v32nvfp4', label: 'DeepSeek-V3.2-NVFP4', default: isB200, disabled: !isB200, disabledReason: 'NVFP4 requires B200 (Blackwell)' }, + { id: 'v32nvfp4', label: 'DeepSeek-V3.2-NVFP4', default: isBlackwell, disabled: !isBlackwell, disabledReason: 'NVFP4 requires B200/B300 (Blackwell)' }, { id: 'v32mxfp4', label: 'DeepSeek-V3.2-MXFP4', default: isAMD, disabled: !isAMD, disabledReason: 'MXFP4 requires AMD MI300X/MI355X' } ]; } @@ -101,7 +102,7 @@ export const DeepSeekV32Deployment = () => { return () => observer.disconnect(); }, []); - // When hardware changes, re-resolve model name defaults (NVFP4β†’B200, MXFP4β†’AMD). + // When hardware changes, re-resolve model name defaults (NVFP4β†’Blackwell, MXFP4β†’AMD). useEffect(() => { setValues(prev => { const next = { ...prev }; @@ -139,10 +140,12 @@ export const DeepSeekV32Deployment = () => { const isNvfp4 = modelname === 'v32nvfp4'; const isMxfp4 = modelname === 'v32mxfp4'; const isAMD = hardware === 'mi300x' || hardware === 'mi355x'; + const isB300 = hardware === 'b300'; + const isBlackwell = hardware === 'b200' || isB300; - // Validation: NVFP4 requires B200 - if (isNvfp4 && hardware !== 'b200') { - return `# Error: DeepSeek-V3.2-NVFP4 requires NVIDIA B200 (Blackwell) hardware\n# Please select "B200" for Hardware Platform or choose a different model`; + // Validation: NVFP4 requires Blackwell + if (isNvfp4 && !isBlackwell) { + return `# Error: DeepSeek-V3.2-NVFP4 requires NVIDIA B200/B300 (Blackwell) hardware\n# Please select "B200" or "B300" for Hardware Platform or choose a different model`; } // Validation: MXFP4 requires AMD MI300X/MI355X @@ -176,8 +179,16 @@ export const DeepSeekV32Deployment = () => { let cmd = 'sglang serve \\\n'; cmd += ` --model-path ${modelName}`; cmd += ' \\\n --tp 4'; - cmd += ' \\\n --quantization modelopt_fp4'; - cmd += ' \\\n --moe-runner-backend flashinfer_trtllm'; + if (isB300) { + cmd += ' \\\n --attention-backend flashinfer'; + cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion'; + cmd += ' \\\n --cuda-graph-backend-prefill disabled'; + cmd += ' \\\n --moe-runner-backend flashinfer_cutlass'; + cmd += ' \\\n --disable-flashinfer-autotune'; + } else { + cmd += ' \\\n --quantization modelopt_fp4'; + cmd += ' \\\n --moe-runner-backend flashinfer_trtllm'; + } return cmd; } @@ -222,6 +233,14 @@ export const DeepSeekV32Deployment = () => { cmd += ' \\\n --speculative-num-draft-tokens 4'; } + if (isB300) { + cmd += ' \\\n --attention-backend flashinfer'; + if (!strategyArray.includes('dp') || strategyArray.includes('ep') || strategyArray.includes('mtp')) { + cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion'; + cmd += ' \\\n --cuda-graph-backend-prefill disabled'; + } + } + // Add tool-call-parser if enabled (not supported for Speciale) if (toolcall === 'enabled' && modelname !== 'v32speciale') { if (modelname === 'v32exp') { diff --git a/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx b/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx index 167c49e18..89b102e88 100644 --- a/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/gemma4-deployment.jsx @@ -28,6 +28,7 @@ export const Gemma4Deployment = () => { return [ { id: 'h200', label: 'H200', default: true }, { id: 'b200', label: 'B200', default: false }, + { id: 'b300', label: 'B300', default: false }, { id: 'mi300x', label: 'MI300X', default: false, disabled: !showMI300X }, ]; } @@ -76,6 +77,13 @@ export const Gemma4Deployment = () => { '31b': { tp: 1, mem: 0.9 }, '26b-a4b': { tp: 1, mem: 0.9 }, }, + b300: { + e2b: { tp: 1, mem: 0.9 }, + e4b: { tp: 1, mem: 0.9 }, + '12b': { tp: 1, mem: 0.9 }, + '31b': { tp: 1, mem: 0.9 }, + '26b-a4b': { tp: 1, mem: 0.9 }, + }, mi300x: { '31b': { tp: 1, mem: 0.80 }, '26b-a4b': { tp: 1, mem: 0.80 }, @@ -129,6 +137,10 @@ export const Gemma4Deployment = () => { cmd += ` \\\n --speculative-eagle-topk 1`; } + if (hardware === 'b300') { + cmd += ` \\\n --attention-backend triton`; + } + cmd += ` \\\n --mem-fraction-static ${mem}`; cmd += ` \\\n --host 0.0.0.0 --port 30000`; diff --git a/docs_new/src/snippets/autoregressive/glm-46-deployment.jsx b/docs_new/src/snippets/autoregressive/glm-46-deployment.jsx index 2c55fdbba..558c5b073 100644 --- a/docs_new/src/snippets/autoregressive/glm-46-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/glm-46-deployment.jsx @@ -8,6 +8,7 @@ export const GLM46Deployment = () => { { id: 'h100', label: 'H100', default: true }, { id: 'h200', label: 'H200', default: false }, { id: 'b200', label: 'B200', default: false }, + { id: 'b300', label: 'B300', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, { id: 'mi355x', label: 'MI355X', default: false } @@ -132,6 +133,9 @@ export const GLM46Deployment = () => { // Strategy-specific parameters if (strategyArray.includes('dp')) { cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`; + if (hardware === 'b300') { + cmd += ` \\\n --cuda-graph-max-bs-decode 256`; + } } if (strategyArray.includes('ep')) { cmd += ` \\\n --ep 8`; @@ -151,6 +155,11 @@ export const GLM46Deployment = () => { cmd += ` \\\n --reasoning-parser glm45`; } + if (hardware === 'b300') { + cmd += ` \\\n --attention-backend flashinfer`; + cmd += ` \\\n --enforce-disable-flashinfer-allreduce-fusion`; + } + return cmd; }; diff --git a/docs_new/src/snippets/autoregressive/glm-46v-deployment.jsx b/docs_new/src/snippets/autoregressive/glm-46v-deployment.jsx index 19d48c973..da722ea99 100644 --- a/docs_new/src/snippets/autoregressive/glm-46v-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/glm-46v-deployment.jsx @@ -6,6 +6,7 @@ export const GLM46VDeployment = () => { title: 'Hardware Platform', items: [ { id: 'b200', label: 'B200', default: true }, + { id: 'b300', label: 'B300', default: false }, { id: 'h100', label: 'H100', default: false }, { id: 'h200', label: 'H200', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, @@ -94,6 +95,7 @@ export const GLM46VDeployment = () => { h100: { tp: 8 }, h200: { tp: 8 }, b200: { tp: 8 }, + b300: { tp: 8 }, mi300x: { tp: 8 }, mi325x: { tp: 8 }, mi355x: { tp: 8 } @@ -103,6 +105,7 @@ export const GLM46VDeployment = () => { h100: { tp: 1 }, h200: { tp: 1 }, b200: { tp: 1 }, + b300: { tp: 1 }, mi300x: { tp: 1 }, mi325x: { tp: 1 }, mi355x: { tp: 1 } @@ -122,6 +125,10 @@ export const GLM46VDeployment = () => { const quantSuffix = quantization === 'fp8' ? '-FP8' : ''; const modelName = `zai-org/${config.baseName}${quantSuffix}`; + if (hardware === 'b300' && modelsize === '9b' && quantization === 'fp8') { + return '# Error: GLM-4.6V-Flash-FP8 is not available on B300 in this cookbook configuration\n# Please use BF16 for GLM-4.6V-Flash or select the 106B model'; + } + let cmd = 'python -m sglang.launch_server \\\n'; cmd += ` --model ${modelName}`; @@ -142,6 +149,12 @@ export const GLM46VDeployment = () => { cmd += ` \\\n --tool-call-parser glm45`; } + if (hardware === 'b300') { + cmd += ` \\\n --attention-backend flashinfer`; + cmd += ` \\\n --enforce-disable-flashinfer-allreduce-fusion`; + cmd += ` \\\n --cuda-graph-backend-decode disabled`; + } + return cmd; }; diff --git a/docs_new/src/snippets/autoregressive/glm-5-deployment.jsx b/docs_new/src/snippets/autoregressive/glm-5-deployment.jsx index 16188cfba..1346c07c4 100644 --- a/docs_new/src/snippets/autoregressive/glm-5-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/glm-5-deployment.jsx @@ -4,6 +4,7 @@ export const GLM5Deployment = () => { // Supported quantization per hardware: // H100 / H200 / MI300X / MI325X / MI355X β†’ BF16 (AMD only) + FP8 (NV only) // B200 β†’ NVFP4 (default), FP8, BF16 + // B300 β†’ NVFP4 (default), FP8 // // BF16 always needs 2x GPUs compared to FP8. AMD only supports BF16. const options = { @@ -13,6 +14,7 @@ export const GLM5Deployment = () => { items: [ { id: 'h200', label: 'H200', default: true }, { id: 'b200', label: 'B200', default: false }, + { id: 'b300', label: 'B300', default: false }, { id: 'h100', label: 'H100', default: false }, { id: 'mi300x', label: 'MI300X/MI325X', default: false }, { id: 'mi355x', label: 'MI355X', default: false } @@ -24,11 +26,12 @@ export const GLM5Deployment = () => { getDynamicItems: (values) => { const hw = values.hardware; const isAMD = hw === 'mi300x' || hw === 'mi355x'; - const isB200 = hw === 'b200'; + const isB300 = hw === 'b300'; + const isBlackwell = hw === 'b200' || isB300; return [ - { id: 'bf16', label: 'BF16', subtitle: 'Full Weights', default: isAMD }, - { id: 'fp8', label: 'FP8', subtitle: 'High Throughput', default: !isAMD && !isB200, disabled: isAMD, disabledReason: 'FP8 not verified on AMD' }, - { id: 'nvfp4', label: 'NVFP4', subtitle: 'Highest Throughput', default: isB200, disabled: !isB200, disabledReason: 'NVFP4 only on B200' } + { id: 'bf16', label: 'BF16', subtitle: 'Full Weights', default: isAMD, disabled: isB300, disabledReason: isB300 ? 'BF16 requires more than the validated 8-GPU B300 node' : '' }, + { id: 'fp8', label: 'FP8', subtitle: 'High Throughput', default: !isAMD && !isBlackwell, disabled: isAMD, disabledReason: 'FP8 not verified on AMD' }, + { id: 'nvfp4', label: 'NVFP4', subtitle: 'Highest Throughput', default: isBlackwell, disabled: !isBlackwell, disabledReason: 'NVFP4 only on B200/B300' } ]; } }, @@ -75,6 +78,7 @@ export const GLM5Deployment = () => { h100: { fp8: { tp: 16, mem: 0.85 }, bf16: { tp: 32, mem: 0.85 } }, h200: { fp8: { tp: 8, mem: 0.85 }, bf16: { tp: 16, mem: 0.85 } }, b200: { nvfp4: { tp: 4, mem: 0.9 }, fp8: { tp: 8, mem: 0.9 }, bf16: { tp: 16, mem: 0.9 } }, + b300: { nvfp4: { tp: 4, mem: 0.9 }, fp8: { tp: 8, mem: 0.9 } }, mi300x: { bf16: { tp: 8, mem: 0.80 } }, mi355x: { bf16: { tp: 8, mem: 0.80 } } }; @@ -147,7 +151,10 @@ export const GLM5Deployment = () => { modelName = `zai-org/GLM-5${suffix}`; } - const hwConfig = modelConfigs[hardware][effectiveQuant]; + const hwConfig = modelConfigs[hardware]?.[effectiveQuant]; + if (!hwConfig) { + return '# Please select a valid hardware and quantization combination'; + } const tpValue = hwConfig.tp; const memFraction = hwConfig.mem; @@ -155,19 +162,27 @@ export const GLM5Deployment = () => { cmd += ` --model-path ${modelName}`; cmd += ` \\\n --tp ${tpValue}`; - // NVFP4 B200: trtllm DSA backends, flashinfer fusion, FP8 KV cache. if (isNVFP4) { cmd += ' \\\n --trust-remote-code'; - cmd += ' \\\n --quantization modelopt_fp4'; - cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; - cmd += ' \\\n --dsa-decode-backend trtllm'; - cmd += ' \\\n --dsa-prefill-backend trtllm'; - cmd += ' \\\n --moe-runner-backend flashinfer_trtllm'; - cmd += ' \\\n --enable-flashinfer-allreduce-fusion'; - cmd += ' \\\n --enable-dp-lm-head'; - cmd += ' \\\n --disable-radix-cache'; - cmd += ' \\\n --max-prefill-tokens 32768'; - cmd += ' \\\n --chunked-prefill-size 32768'; + if (hardware === 'b300') { + cmd += ' \\\n --attention-backend flashinfer'; + cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion'; + cmd += ' \\\n --cuda-graph-backend-prefill disabled'; + cmd += ' \\\n --moe-runner-backend flashinfer_cutlass'; + cmd += ' \\\n --cuda-graph-backend-decode disabled'; + cmd += ' \\\n --disable-flashinfer-autotune'; + } else { + cmd += ' \\\n --quantization modelopt_fp4'; + cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; + cmd += ' \\\n --dsa-decode-backend trtllm'; + cmd += ' \\\n --dsa-prefill-backend trtllm'; + cmd += ' \\\n --moe-runner-backend flashinfer_trtllm'; + cmd += ' \\\n --enable-flashinfer-allreduce-fusion'; + cmd += ' \\\n --enable-dp-lm-head'; + cmd += ' \\\n --disable-radix-cache'; + cmd += ' \\\n --max-prefill-tokens 32768'; + cmd += ' \\\n --chunked-prefill-size 32768'; + } cmd += ` \\\n --mem-fraction-static ${memFraction}`; cmd += ' \\\n --scheduler-recv-interval 10'; cmd += ' \\\n --tokenizer-worker-num 6'; @@ -185,6 +200,9 @@ export const GLM5Deployment = () => { if (values.dpattention === 'enabled') { cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`; + if (hardware === 'b300') { + cmd += ' \\\n --cuda-graph-max-bs-decode 256'; + } } if (values.reasoning === 'enabled') cmd += ' \\\n --reasoning-parser glm45'; if (values.toolcall === 'enabled') cmd += ' \\\n --tool-call-parser glm47'; @@ -206,6 +224,12 @@ export const GLM5Deployment = () => { cmd += ' \\\n --enable-flashinfer-allreduce-fusion'; } + if (hardware === 'b300' && effectiveQuant === 'fp8') { + cmd += ' \\\n --attention-backend flashinfer'; + cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion'; + cmd += ' \\\n --cuda-graph-backend-prefill disabled'; + } + // H200 FP8: flashinfer allreduce fusion. if (hardware === 'h200' && effectiveQuant === 'fp8') { cmd += ' \\\n --enable-flashinfer-allreduce-fusion'; diff --git a/docs_new/src/snippets/autoregressive/gpt-oss-deployment.jsx b/docs_new/src/snippets/autoregressive/gpt-oss-deployment.jsx index 6740554b9..7f2c743a1 100644 --- a/docs_new/src/snippets/autoregressive/gpt-oss-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/gpt-oss-deployment.jsx @@ -6,6 +6,7 @@ export const GPTOSSDeployment = () => { title: 'Hardware Platform', items: [ { id: 'b200', label: 'B200', default: true }, + { id: 'b300', label: 'B300', default: false }, { id: 'h200', label: 'H200', default: false }, { id: 'h100', label: 'H100', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, @@ -102,6 +103,7 @@ export const GPTOSSDeployment = () => { h100: { tp: 8 }, h200: { tp: 8 }, b200: { tp: 8 }, + b300: { tp: 8 }, mi300x: { tp: 8 }, mi325x: { tp: 8 }, mi355x: { tp: 8 } @@ -111,6 +113,7 @@ export const GPTOSSDeployment = () => { h100: { tp: 1 }, h200: { tp: 1 }, b200: { tp: 1 }, + b300: { tp: 1 }, mi300x: { tp: 1 }, mi325x: { tp: 1 }, mi355x: { tp: 1 } @@ -170,6 +173,12 @@ export const GPTOSSDeployment = () => { cmd += ` \\\n --tool-call-parser gpt-oss`; } + if (hardware === 'b300') { + cmd += ` \\\n --attention-backend triton`; + cmd += ` \\\n --moe-runner-backend triton`; + cmd += ` \\\n --enforce-disable-flashinfer-allreduce-fusion`; + } + // Add speculative decoding if enabled (MI30x handled above) if (speculative === 'enabled') { cmd += ` \\\n --speculative-algorithm EAGLE3 \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`; diff --git a/docs_new/src/snippets/autoregressive/intern-s1-deployment.jsx b/docs_new/src/snippets/autoregressive/intern-s1-deployment.jsx new file mode 100644 index 000000000..5ddf97ff8 --- /dev/null +++ b/docs_new/src/snippets/autoregressive/intern-s1-deployment.jsx @@ -0,0 +1,167 @@ +export const InternS1Deployment = () => { + const options = { + hardware: { + name: 'hardware', + title: 'Hardware Platform', + items: [ + { id: 'b200', label: 'B200', default: true }, + { id: 'b300', label: 'B300', default: false }, + { id: 'h100', label: 'H100', default: false }, + { id: 'h200', label: 'H200', default: false }, + ], + }, + modelsize: { + name: 'modelsize', + title: 'Model Size', + items: [ + { id: 'S1', label: '235B', subtitle: 'MoE', default: true }, + { id: 'S1-mini', label: '8B', subtitle: 'Dense', default: false }, + ], + }, + quantization: { + name: 'quantization', + title: 'Quantization', + items: [ + { id: 'bf16', label: 'BF16', default: true }, + { id: 'fp8', label: 'FP8', default: false }, + ], + }, + reasoning: { + name: 'reasoning', + title: 'Reasoning Parser', + items: [ + { id: 'disabled', label: 'Disabled', default: true }, + { id: 'enabled', label: 'Enabled', default: false }, + ], + }, + toolcall: { + name: 'toolcall', + title: 'Tool Call Parser', + items: [ + { id: 'disabled', label: 'Disabled', default: true }, + { id: 'enabled', label: 'Enabled', default: false }, + ], + }, + }; + + const modelConfigs = { + S1: { + baseName: 'S1', + h100: { bf16: { tp: 8 }, fp8: { tp: 8, ep: 2 } }, + h200: { bf16: { tp: 8 }, fp8: { tp: 8, ep: 2 } }, + b200: { bf16: { tp: 8 }, fp8: { tp: 8, ep: 2 } }, + b300: { bf16: { tp: 8 }, fp8: { tp: 8, ep: 2 } }, + }, + 'S1-mini': { + baseName: 'S1-mini', + h100: { bf16: { tp: 1 }, fp8: { tp: 1 } }, + h200: { bf16: { tp: 1 }, fp8: { tp: 1 } }, + b200: { bf16: { tp: 1 }, fp8: { tp: 1 } }, + b300: { bf16: { tp: 1 }, fp8: { tp: 1 } }, + }, + }; + + const getInitialState = () => { + const initialState = {}; + Object.entries(options).forEach(([key, option]) => { + const defaultItem = option.items.find((item) => item.default); + initialState[key] = defaultItem ? defaultItem.id : option.items[0].id; + }); + return initialState; + }; + + const [values, setValues] = useState(getInitialState); + const [isDark, setIsDark] = useState(false); + + useEffect(() => { + const checkDarkMode = () => { + const html = document.documentElement; + const isDarkMode = + html.classList.contains('dark') || + html.getAttribute('data-theme') === 'dark' || + html.style.colorScheme === 'dark'; + setIsDark(isDarkMode); + }; + checkDarkMode(); + const observer = new MutationObserver(checkDarkMode); + observer.observe(document.documentElement, { + attributes: true, + attributeFilter: ['class', 'data-theme', 'style'], + }); + return () => observer.disconnect(); + }, []); + + const handleRadioChange = (optionName, value) => { + setValues((prev) => ({ ...prev, [optionName]: value })); + }; + + const generateCommand = () => { + const { hardware, modelsize, quantization, reasoning, toolcall } = values; + const modelConfig = modelConfigs[modelsize]; + const hwConfig = modelConfig?.[hardware]?.[quantization]; + if (!hwConfig) { + return '# Please select a valid hardware and quantization combination'; + } + + const quantSuffix = quantization === 'fp8' ? '-FP8' : ''; + const modelName = `internlm/Intern-${modelConfig.baseName}${quantSuffix}`; + + const flags = []; + flags.push(` --model ${modelName}`); + if (hwConfig.tp > 1) flags.push(` --tp ${hwConfig.tp}`); + if (hwConfig.ep) flags.push(` --ep ${hwConfig.ep}`); + if (quantization === 'fp8') flags.push(` --tokenizer-path internlm/Intern-${modelConfig.baseName}`); + if (reasoning === 'enabled') flags.push(' --reasoning-parser interns1'); + if (toolcall === 'enabled') flags.push(' --tool-call-parser interns1'); + flags.push(' --trust-remote-code'); + if (hardware === 'b300') flags.push(' --attention-backend flashinfer'); + + return `python -m sglang.launch_server \\\n${flags.join(' \\\n')}`; + }; + + const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' }; + const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' }; + const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' }; + const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 }; + const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' }; + const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' }; + const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 }; + const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` }; + + return ( +
+ {Object.entries(options).map(([key, option]) => ( +
+
{option.title}
+
+ {option.items.map((item) => { + const isChecked = values[option.name] === item.id; + return ( + + ); + })} +
+
+ ))} +
+
Run this Command:
+ 
{generateCommand()}
+
+
+ ); +}; diff --git a/docs_new/src/snippets/autoregressive/kimi-k2-deployment.jsx b/docs_new/src/snippets/autoregressive/kimi-k2-deployment.jsx index 425ae3a97..04ccff38f 100644 --- a/docs_new/src/snippets/autoregressive/kimi-k2-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/kimi-k2-deployment.jsx @@ -8,6 +8,7 @@ export const KimiK2Deployment = () => { items: [ { id: 'h200', label: 'H200', default: true }, { id: 'b200', label: 'B200', default: false }, + { id: 'b300', label: 'B300', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, { id: 'mi355x', label: 'MI355X', default: false } @@ -82,6 +83,15 @@ export const KimiK2Deployment = () => { cmd += ` \\\n --trust-remote-code`; + if (hardware === 'b300') { + cmd += ` \\\n --attention-backend flashinfer`; + if (strategyArray.includes('dp')) { + cmd += ` \\\n --prefill-attention-backend triton`; + } + cmd += ` \\\n --enforce-disable-flashinfer-allreduce-fusion`; + cmd += ` \\\n --mem-fraction-static 0.85`; + } + if (toolcall === 'enabled') { cmd += ` \\\n --tool-call-parser kimi_k2`; } diff --git a/docs_new/src/snippets/autoregressive/llada-21-deployment.jsx b/docs_new/src/snippets/autoregressive/llada-21-deployment.jsx index 025bf8247..5d239c302 100644 --- a/docs_new/src/snippets/autoregressive/llada-21-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/llada-21-deployment.jsx @@ -9,6 +9,7 @@ export const LLaDA21Deployment = () => { { id: 'h100', label: 'H100', default: true }, { id: 'h200', label: 'H200', default: false }, { id: 'b200', label: 'B200', default: false }, + { id: 'b300', label: 'B300', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, { id: 'mi355x', label: 'MI355X', default: false } @@ -34,7 +35,7 @@ export const LLaDA21Deployment = () => { if (modelsize === 'mini') { tpSize = 1; } else { - if (hardware === 'b200') { + if (hardware === 'b200' || hardware === 'b300') { tpSize = 2; } else { tpSize = 4; @@ -48,7 +49,7 @@ export const LLaDA21Deployment = () => { args.push(`--trust-remote-code`); args.push(`--mem-fraction-static 0.8`); args.push(`--max-running-requests 1`); - if (hardware === 'h100' || hardware === 'h200' || hardware === 'b200') { + if (hardware === 'h100' || hardware === 'h200' || hardware === 'b200' || hardware === 'b300') { args.push(`--attention-backend flashinfer`); } diff --git a/docs_new/src/snippets/autoregressive/minimax-m27-deployment.jsx b/docs_new/src/snippets/autoregressive/minimax-m27-deployment.jsx index 2254773b2..8a326e2e8 100644 --- a/docs_new/src/snippets/autoregressive/minimax-m27-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/minimax-m27-deployment.jsx @@ -8,6 +8,7 @@ export const MiniMaxM27Deployment = () => { items: [ { id: 'h200', label: 'H200', default: true }, { id: 'b200', label: 'B200', default: false }, + { id: 'b300', label: 'B300', default: false }, { id: 'gb300', label: 'GB300', default: false }, { id: 'a100', label: 'A100', default: false }, { id: 'h100', label: 'H100', default: false }, @@ -23,6 +24,7 @@ export const MiniMaxM27Deployment = () => { getDynamicItems: (values) => { const hw = values.hardware; const isAMD = hw === 'mi300x' || hw === 'mi325x' || hw === 'mi355x'; + const isB300 = hw === 'b300'; const isGB300 = hw === 'gb300'; const isXeon = hw === 'xeon'; if (isXeon) { @@ -33,8 +35,8 @@ export const MiniMaxM27Deployment = () => { const canUse2GPU = isAMD || isGB300; return [ { id: '2gpu', label: '2', default: canUse2GPU, disabled: !canUse2GPU }, - { id: '4gpu', label: '4', default: !canUse2GPU, disabled: false }, - { id: '8gpu', label: '8', default: false, disabled: isGB300 } + { id: '4gpu', label: '4', default: !canUse2GPU || isB300, disabled: false }, + { id: '8gpu', label: '8', default: false, disabled: isGB300 || isB300 } ]; } }, @@ -43,11 +45,11 @@ export const MiniMaxM27Deployment = () => { title: 'Precision', getDynamicItems: (values) => { const hw = values.hardware; - const isBlackwell = hw === 'b200' || hw === 'gb300'; + const isBlackwell = hw === 'b200' || hw === 'b300' || hw === 'gb300'; return [ { id: 'fp8', label: 'FP8', default: true, disabled: false }, { id: 'fp4', label: 'FP4', default: false, disabled: !isBlackwell, - disabledReason: 'NVFP4 requires Blackwell (B200/GB300)' } + disabledReason: 'NVFP4 requires Blackwell (B200/B300/GB300)' } ]; } }, @@ -131,6 +133,7 @@ export const MiniMaxM27Deployment = () => { const { hardware, gpuCount, precision, thinking, toolcall } = values; const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x'; + const isB300 = hardware === 'b300'; const isGB300 = hardware === 'gb300'; const isXeon = hardware === 'xeon'; const canUse2GPU = isAMD || isGB300; @@ -139,11 +142,11 @@ export const MiniMaxM27Deployment = () => { return '# Please select compatible hardware\n# 2-GPU requires AMD MI300X/MI325X/MI355X or GB300'; } - const isBlackwell = hardware === 'b200' || hardware === 'gb300'; + const isBlackwell = hardware === 'b200' || hardware === 'b300' || hardware === 'gb300'; const isFp4 = precision === 'fp4'; if (isFp4 && !isBlackwell) { - return '# NVFP4 requires Blackwell hardware (B200 or GB300)'; + return '# NVFP4 requires Blackwell hardware (B200, B300, or GB300)'; } const modelName = isFp4 ? 'nvidia/MiniMax-M2.7-NVFP4' : 'MiniMaxAI/MiniMax-M2.7'; @@ -184,6 +187,9 @@ export const MiniMaxM27Deployment = () => { cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; cmd += ' \\\n --attention-backend triton'; } + if (isB300) { + cmd += ' \\\n --attention-backend flashinfer'; + } if (isBlackwell) { cmd += ' \\\n --moe-runner-backend flashinfer_trtllm_routed'; diff --git a/docs_new/src/snippets/autoregressive/mistral-small-4-deployment.jsx b/docs_new/src/snippets/autoregressive/mistral-small-4-deployment.jsx index d70f47b58..67ed4f709 100644 --- a/docs_new/src/snippets/autoregressive/mistral-small-4-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/mistral-small-4-deployment.jsx @@ -82,6 +82,10 @@ export const MistralSmall4Deployment = () => { } }); + if (hardware === 'b300') { + cmd += ` \\\n --attention-backend flashinfer`; + } + return cmd; }; diff --git a/docs_new/src/snippets/autoregressive/nemotron3-nano-deployment.jsx b/docs_new/src/snippets/autoregressive/nemotron3-nano-deployment.jsx index 421bcb46e..e7063a567 100644 --- a/docs_new/src/snippets/autoregressive/nemotron3-nano-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/nemotron3-nano-deployment.jsx @@ -7,7 +7,8 @@ export const Nemotron3NanoDeployment = () => { title: 'Hardware Platform', items: [ { id: 'h200', label: 'H200', default: false }, - { id: 'b200', label: 'B200', default: true } + { id: 'b200', label: 'B200', default: true }, + { id: 'b300', label: 'B300', default: false } ] }, modelVariant: { @@ -71,6 +72,9 @@ export const Nemotron3NanoDeployment = () => { cmd += ` --trust-remote-code \\\n`; cmd += ` --tp ${tp} \\\n`; cmd += ` --kv-cache-dtype ${kvcache} \\\n`; + if (hardware === 'b300') { + cmd += ` --attention-backend flashinfer \\\n`; + } // Add thinking parser and tool call parser if enabled for (const [key, option] of Object.entries(options)) { diff --git a/docs_new/src/snippets/autoregressive/nemotron3-super-deployment.jsx b/docs_new/src/snippets/autoregressive/nemotron3-super-deployment.jsx index 9afd87862..7e7ddf8d0 100644 --- a/docs_new/src/snippets/autoregressive/nemotron3-super-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/nemotron3-super-deployment.jsx @@ -20,7 +20,8 @@ export const Nemotron3SuperDeployment = () => { title: 'Hardware Platform', items: [ { id: 'h200', label: 'H200', default: false }, - { id: 'b200', label: 'B200', default: true } + { id: 'b200', label: 'B200', default: true }, + { id: 'b300', label: 'B300', default: false } ] }, tp: { @@ -87,6 +88,9 @@ export const Nemotron3SuperDeployment = () => { if (kvcache && kvcache !== 'none') { cmd += ` --kv-cache-dtype ${kvcache} \\\n`; } + if (values.hardware === 'b300') { + cmd += ` --attention-backend flashinfer \\\n`; + } for (const [key, option] of Object.entries(options)) { if (option.commandRule) { diff --git a/docs_new/src/snippets/autoregressive/qwen3-coder-next-deployment.jsx b/docs_new/src/snippets/autoregressive/qwen3-coder-next-deployment.jsx index 2d4a0f144..f36c588fd 100644 --- a/docs_new/src/snippets/autoregressive/qwen3-coder-next-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/qwen3-coder-next-deployment.jsx @@ -7,6 +7,7 @@ export const Qwen3CoderNextDeployment = () => { { id: 'h200', label: 'H200', default: true }, { id: 'h100', label: 'H100', default: false }, { id: 'b200', label: 'B200', default: false }, + { id: 'b300', label: 'B300', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, { id: 'mi355x', label: 'MI355X', default: false }, @@ -48,6 +49,7 @@ export const Qwen3CoderNextDeployment = () => { h100: { bf16: { tp: 4 }, fp8: { tp: 2 } }, h200: { bf16: { tp: 2 }, fp8: { tp: 1 } }, b200: { bf16: { tp: 2 }, fp8: { tp: 1 } }, + b300: { bf16: { tp: 2 }, fp8: { tp: 1 } }, mi300x: { bf16: { tp: 2 }, fp8: { tp: 1 } }, mi325x: { bf16: { tp: 2 }, fp8: { tp: 1 } }, mi355x: { bf16: { tp: 2 }, fp8: { tp: 1 } }, @@ -99,6 +101,9 @@ export const Qwen3CoderNextDeployment = () => { if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') { cmd += ` \\\n --attention-backend triton`; } + if (hardware === 'b300') { + cmd += ` \\\n --attention-backend flashinfer`; + } return cmd; }; diff --git a/docs_new/src/snippets/autoregressive/qwen3-deployment.jsx b/docs_new/src/snippets/autoregressive/qwen3-deployment.jsx index 0d4209c3e..08876fc72 100644 --- a/docs_new/src/snippets/autoregressive/qwen3-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/qwen3-deployment.jsx @@ -7,6 +7,7 @@ export const Qwen3Deployment = () => { h100: { tp: 8, ep: 0, bf16: true, fp8: true }, h200: { tp: 8, ep: 0, bf16: true, fp8: true }, b200: { tp: 8, ep: 0, bf16: true, fp8: true }, + b300: { tp: 8, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 4, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 4, ep: 0, bf16: true, fp8: true }, mi355x: { tp: 4, ep: 0, bf16: true, fp8: true }, @@ -18,6 +19,7 @@ export const Qwen3Deployment = () => { h100: { tp: 1, ep: 0, bf16: true, fp8: true }, h200: { tp: 1, ep: 0, bf16: true, fp8: true }, b200: { tp: 1, ep: 0, bf16: true, fp8: true }, + b300: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, @@ -29,6 +31,7 @@ export const Qwen3Deployment = () => { h100: { tp: 1, ep: 0, bf16: true, fp8: true }, h200: { tp: 1, ep: 0, bf16: true, fp8: true }, b200: { tp: 1, ep: 0, bf16: true, fp8: true }, + b300: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, @@ -40,6 +43,7 @@ export const Qwen3Deployment = () => { h100: { tp: 1, ep: 0, bf16: true, fp8: true }, h200: { tp: 1, ep: 0, bf16: true, fp8: true }, b200: { tp: 1, ep: 0, bf16: true, fp8: true }, + b300: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, @@ -51,6 +55,7 @@ export const Qwen3Deployment = () => { h100: { tp: 1, ep: 0, bf16: true, fp8: true }, h200: { tp: 1, ep: 0, bf16: true, fp8: true }, b200: { tp: 1, ep: 0, bf16: true, fp8: true }, + b300: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, @@ -62,6 +67,7 @@ export const Qwen3Deployment = () => { h100: { tp: 1, ep: 0, bf16: true, fp8: true }, h200: { tp: 1, ep: 0, bf16: true, fp8: true }, b200: { tp: 1, ep: 0, bf16: true, fp8: true }, + b300: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, @@ -73,6 +79,7 @@ export const Qwen3Deployment = () => { h100: { tp: 1, ep: 0, bf16: true, fp8: true }, h200: { tp: 1, ep: 0, bf16: true, fp8: true }, b200: { tp: 1, ep: 0, bf16: true, fp8: true }, + b300: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, @@ -84,6 +91,7 @@ export const Qwen3Deployment = () => { h100: { tp: 1, ep: 0, bf16: true, fp8: true }, h200: { tp: 1, ep: 0, bf16: true, fp8: true }, b200: { tp: 1, ep: 0, bf16: true, fp8: true }, + b300: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, @@ -98,6 +106,7 @@ export const Qwen3Deployment = () => { title: 'Hardware Platform', items: [ { id: 'b200', label: 'B200', default: true }, + { id: 'b300', label: 'B300', default: false }, { id: 'h100', label: 'H100', default: false }, { id: 'h200', label: 'H200', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, @@ -297,6 +306,11 @@ export const Qwen3Deployment = () => { cmd += ' \\\n --tool-call-parser qwen25'; } + if (hardware === 'b300') { + cmd += ' \\\n --attention-backend flashinfer'; + cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion'; + } + return cmd; }; diff --git a/docs_new/src/snippets/autoregressive/qwen3-next-deployment.jsx b/docs_new/src/snippets/autoregressive/qwen3-next-deployment.jsx index 2ca887cd8..dd886e4cf 100644 --- a/docs_new/src/snippets/autoregressive/qwen3-next-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/qwen3-next-deployment.jsx @@ -5,6 +5,7 @@ export const Qwen3NextDeployment = () => { title: 'Hardware Platform', items: [ { id: 'b200', label: 'B200', default: true }, + { id: 'b300', label: 'B300', default: false }, { id: 'h200', label: 'H200', default: false }, { id: 'h100', label: 'H100', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, @@ -75,6 +76,7 @@ export const Qwen3NextDeployment = () => { h100: { tp: 4, ep: 0, bf16: true, fp8: true }, h200: { tp: 2, ep: 0, bf16: true, fp8: true }, b200: { tp: 2, ep: 0, bf16: true, fp8: true }, + b300: { tp: 2, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 2, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 2, ep: 0, bf16: true, fp8: true }, mi355x: { tp: 2, ep: 0, bf16: true, fp8: true }, @@ -136,6 +138,10 @@ export const Qwen3NextDeployment = () => { if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') { cmd += ` \\\n --attention-backend triton`; } + if (hardware === 'b300') { + cmd += ` \\\n --attention-backend flashinfer`; + cmd += ` \\\n --enforce-disable-flashinfer-allreduce-fusion`; + } return cmd; }; diff --git a/docs_new/src/snippets/autoregressive/qwen3-vl-deployment.jsx b/docs_new/src/snippets/autoregressive/qwen3-vl-deployment.jsx index 3374fc7e1..90abf23b3 100644 --- a/docs_new/src/snippets/autoregressive/qwen3-vl-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/qwen3-vl-deployment.jsx @@ -6,6 +6,7 @@ export const Qwen3VLDeployment = () => { title: 'Hardware Platform', items: [ { id: 'b200', label: 'B200', default: true }, + { id: 'b300', label: 'B300', default: false }, { id: 'h100', label: 'H100', default: false }, { id: 'h200', label: 'H200', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, @@ -60,6 +61,7 @@ export const Qwen3VLDeployment = () => { h100: { tp: 8, ep: 0, bf16: true, fp8: true }, h200: { tp: 8, ep: 0, bf16: true, fp8: true }, b200: { tp: 8, ep: 0, bf16: true, fp8: true }, + b300: { tp: 8, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 8, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 8, ep: 0, bf16: true, fp8: true }, mi355x: { tp: 8, ep: 0, bf16: true, fp8: true }, @@ -71,6 +73,7 @@ export const Qwen3VLDeployment = () => { h100: { tp: 1, ep: 0, bf16: true, fp8: true }, h200: { tp: 1, ep: 0, bf16: true, fp8: true }, b200: { tp: 1, ep: 0, bf16: true, fp8: true }, + b300: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, @@ -82,6 +85,7 @@ export const Qwen3VLDeployment = () => { h100: { tp: 1, ep: 0, bf16: true, fp8: true }, h200: { tp: 1, ep: 0, bf16: true, fp8: true }, b200: { tp: 1, ep: 0, bf16: true, fp8: true }, + b300: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, @@ -93,6 +97,7 @@ export const Qwen3VLDeployment = () => { h100: { tp: 1, ep: 0, bf16: true, fp8: true }, h200: { tp: 1, ep: 0, bf16: true, fp8: true }, b200: { tp: 1, ep: 0, bf16: true, fp8: true }, + b300: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, @@ -104,6 +109,7 @@ export const Qwen3VLDeployment = () => { h100: { tp: 1, ep: 0, bf16: true, fp8: true }, h200: { tp: 1, ep: 0, bf16: true, fp8: true }, b200: { tp: 1, ep: 0, bf16: true, fp8: true }, + b300: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, @@ -115,6 +121,7 @@ export const Qwen3VLDeployment = () => { h100: { tp: 1, ep: 0, bf16: true, fp8: true }, h200: { tp: 1, ep: 0, bf16: true, fp8: true }, b200: { tp: 1, ep: 0, bf16: true, fp8: true }, + b300: { tp: 1, ep: 0, bf16: true, fp8: true }, mi300x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi325x: { tp: 1, ep: 0, bf16: true, fp8: true }, mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }, @@ -213,6 +220,11 @@ export const Qwen3VLDeployment = () => { cmd += ' \\\n --tool-call-parser qwen'; } + if (hardware === 'b300') { + cmd += ' \\\n --attention-backend flashinfer'; + cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion'; + } + return cmd; }; diff --git a/docs_new/src/snippets/autoregressive/qwen35-deployment.jsx b/docs_new/src/snippets/autoregressive/qwen35-deployment.jsx index a86ffd9d2..7c0f3d70d 100644 --- a/docs_new/src/snippets/autoregressive/qwen35-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/qwen35-deployment.jsx @@ -8,19 +8,19 @@ export const Qwen35Deployment = () => { // 27B, 9B, 4B, 2B, 0.8B // // GPU requirements (BF16): - // 397B-A17B: H100 tp=16 (2 nodes), H200 tp=8, B200 tp=8, B300 tp=4, MI300X tp=8, MI325X tp=4, MI355X tp=4 + // 397B-A17B: H100 tp=16 (2 nodes), H200 tp=8, B200 tp=8, B300 tp=8, MI300X tp=8, MI325X tp=4, MI355X tp=4 // 122B-A10B: H100 tp=4, H200 tp=4, B200 tp=2, B300 tp=2, MI300X tp=2, MI325X tp=1, MI355X tp=1 // 35B-A3B: H100 tp=1 (tp=2 w/ MTP), H200 tp=1, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1 // 27B: H100 tp=1 (tp=2 w/ MTP); tp=1 on all other hardware // 9B/4B/2B/0.8B: tp=1 on all hardware (including MI300X, MI325X, MI355X) // // GPU requirements (FP8, where available): - // 397B-A17B: H100 tp=8, H200 tp=8 ep=8, B200 tp=4, B300 tp=2, MI300X tp=4, MI325X tp=2, MI355X tp=2 + // 397B-A17B: H100 tp=8, H200 tp=8 ep=8, B200 tp=4, B300 tp=4, MI300X tp=4, MI325X tp=2, MI355X tp=2 // 122B-A10B: H100 tp=2 (tp=4 w/ MTP), H200 tp=2, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1 // 35B-A3B: H100 tp=1, H200 tp=1, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1 // 27B: tp=1 on all hardware (including MI300X, MI325X, MI355X) // - // FP4 (397B only): NVFP4 on Blackwell B200 tp=4, B300 tp=2; AMD MXFP4 on MI355X tp=2 + // FP4 (397B only): NVFP4 on Blackwell B200/B300 tp=4; AMD MXFP4 on MI355X tp=2 const MOE_MODELS = new Set(['397b', '122b', '35b']); const FP8_MODELS = new Set(['397b', '122b', '35b', '27b']); @@ -149,7 +149,7 @@ export const Qwen35Deployment = () => { h100: { bf16: { tp: 16, mem: 0.8, multinode: true, nnodes: 2 }, fp8: { tp: 8, mem: 0.8 } }, h200: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 8, ep: 8, mem: 0.8 } }, b200: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 4, mem: 0.8 }, fp4: { tp: 4, mem: 0.85 } }, - b300: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 }, fp4: { tp: 2, mem: 0.8 } }, + b300: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 4, mem: 0.8 }, fp4: { tp: 4, mem: 0.8 } }, mi300x: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 4, mem: 0.8 } }, mi325x: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 } }, mi355x: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 }, fp4: { tp: 2, mem: 0.8 } }, @@ -372,7 +372,7 @@ export const Qwen35Deployment = () => { const commandRules = { reasoning: (value) => value === 'enabled' ? '--reasoning-parser qwen3' : null, toolcall: (value) => value === 'enabled' ? '--tool-call-parser qwen3_coder' : null, - speculative: (value) => value === 'enabled' ? '--speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4' : null, + speculative: (value) => value === 'enabled' ? '--speculative-algorithm NEXTN \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4' : null, mambaCache: (value) => value === 'v2' ? '--mamba-scheduler-strategy extra_buffer' : null, }; @@ -423,9 +423,12 @@ export const Qwen35Deployment = () => { } // Append backend configurations - if (hardware === 'b200' || hardware === 'b300') { + if (hardware === 'b200' || (hardware === 'b300' && quantization === 'fp4')) { cmd += ` \\\n --attention-backend trtllm_mha`; } + if (hardware === 'b300' && quantization !== 'fp4') { + cmd += ` \\\n --attention-backend flashinfer`; + } // Append AMD GPU-specific backend configurations. // All AMD MI GPUs use the AITER unified-attention backend (pair with @@ -466,15 +469,20 @@ export const Qwen35Deployment = () => { } } else { // NVIDIA NVFP4 on Blackwell (B200 / B300). - cmd += ' \\\n --quantization modelopt_fp4'; - cmd += ' \\\n --fp4-gemm-backend flashinfer_cutlass'; - cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; - cmd += ' \\\n --moe-runner-backend flashinfer_trtllm'; - cmd += ' \\\n --chunked-prefill-size 32768'; - cmd += ' \\\n --max-prefill-tokens 32768'; - cmd += ' \\\n --max-running-requests 128'; - cmd += ' \\\n --stream-interval 30'; - cmd += ' \\\n --disable-radix-cache'; + if (hardware === 'b300') { + cmd += ' \\\n --moe-runner-backend flashinfer_trtllm'; + cmd += ' \\\n --fp4-gemm-backend flashinfer_cutlass'; + } else { + cmd += ' \\\n --quantization modelopt_fp4'; + cmd += ' \\\n --fp4-gemm-backend flashinfer_cutlass'; + cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; + cmd += ' \\\n --moe-runner-backend flashinfer_trtllm'; + cmd += ' \\\n --chunked-prefill-size 32768'; + cmd += ' \\\n --max-prefill-tokens 32768'; + cmd += ' \\\n --max-running-requests 128'; + cmd += ' \\\n --stream-interval 30'; + cmd += ' \\\n --disable-radix-cache'; + } } } diff --git a/docs_new/src/snippets/autoregressive/qwen36-deployment.jsx b/docs_new/src/snippets/autoregressive/qwen36-deployment.jsx index 76b1cf2a2..d30008291 100644 --- a/docs_new/src/snippets/autoregressive/qwen36-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/qwen36-deployment.jsx @@ -8,6 +8,7 @@ export const Qwen36Deployment = () => { { id: 'h100', label: 'H100', default: true }, { id: 'h200', label: 'H200', default: false }, { id: 'b200', label: 'B200', default: false }, + { id: 'b300', label: 'B300', default: false }, { id: 'xeon', label: 'XEON', default: false }, ], }, @@ -85,6 +86,7 @@ export const Qwen36Deployment = () => { h100: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } }, h200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } }, b200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } }, + b300: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } }, xeon: { bf16: { tp: 3 }, fp8: { tp: 3 } }, }, '27b': { @@ -92,6 +94,7 @@ export const Qwen36Deployment = () => { h100: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } }, h200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } }, b200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } }, + b300: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } }, xeon: { bf16: { tp: 6 }, fp8: { tp: 6 } }, }, }; @@ -192,6 +195,9 @@ export const Qwen36Deployment = () => { if (hardware === 'b200') { cmd += ` \\\n --attention-backend trtllm_mha`; } + if (hardware === 'b300') { + cmd += ` \\\n --attention-backend flashinfer`; + } if (hwConfig.mem !== undefined) { cmd += ` \\\n --mem-fraction-static ${hwConfig.mem}`; } diff --git a/docs_new/src/snippets/autoregressive/ring-25-1t-deployment.jsx b/docs_new/src/snippets/autoregressive/ring-25-1t-deployment.jsx index 6ffa5169e..6e6cb03e2 100644 --- a/docs_new/src/snippets/autoregressive/ring-25-1t-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/ring-25-1t-deployment.jsx @@ -2,7 +2,7 @@ export const Ring251TDeployment = () => { // Config mirrors sgl-cookbook src/components/autoregressive/Ring25ConfigGenerator/index.js. // // GPU requirements: - // H200 / B200 / GB200 / GB300 / MI355X: single-node (tp per platform) + // H200 / B200 / B300 / GB200 / GB300 / MI355X: single-node (tp per platform) // MI300X / MI325X: two nodes, tp-size 8, pp-size 2 (multi-node scripts) const options = { hardware: { @@ -11,6 +11,7 @@ export const Ring251TDeployment = () => { items: [ { id: 'h200', label: 'H200', default: true }, { id: 'b200', label: 'B200', default: false }, + { id: 'b300', label: 'B300', default: false }, { id: 'gb200', label: 'GB200', default: false }, { id: 'gb300', label: 'GB300', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, @@ -39,6 +40,7 @@ export const Ring251TDeployment = () => { const modelConfigs = { h200: { fp8: { tp: 8 } }, b200: { fp8: { tp: 8 } }, + b300: { fp8: { tp: 8 } }, gb200: { fp8: { tp: 4 } }, gb300: { fp8: { tp: 4 } }, mi300x: { fp8: { tp: 8, pp: 2, nnodes: 2 } }, @@ -156,6 +158,9 @@ export const Ring251TDeployment = () => { cmd += ` --model-path ${modelName}`; cmd += ` \\\n --tp ${tpValue}`; cmd += ' \\\n --trust-remote-code'; + if (hardware === 'b300') { + cmd += ' \\\n --attention-backend flashinfer'; + } extraFlags.forEach((flag) => { cmd += ` \\\n ${flag}`; diff --git a/docs_new/src/snippets/diffusion/flux-deployment.jsx b/docs_new/src/snippets/diffusion/flux-deployment.jsx index 2a004865c..f3a830215 100644 --- a/docs_new/src/snippets/diffusion/flux-deployment.jsx +++ b/docs_new/src/snippets/diffusion/flux-deployment.jsx @@ -8,6 +8,7 @@ export const FluxDeployment = () => { title: 'Hardware Platform', items: [ { id: 'b200', label: 'B200', default: true }, + { id: 'b300', label: 'B300', default: false }, { id: 'h200', label: 'H200', default: false }, { id: 'h100', label: 'H100', default: false }, { id: 'mi355x', label: 'MI355X', default: false }, diff --git a/docs_new/src/snippets/diffusion/qwen-image-edit-deployment.jsx b/docs_new/src/snippets/diffusion/qwen-image-edit-deployment.jsx index 866cbd70e..460c6e483 100644 --- a/docs_new/src/snippets/diffusion/qwen-image-edit-deployment.jsx +++ b/docs_new/src/snippets/diffusion/qwen-image-edit-deployment.jsx @@ -8,6 +8,7 @@ export const QwenImageEditDeployment = () => { title: 'Hardware Platform', items: [ { id: 'b200', label: 'B200', default: true }, + { id: 'b300', label: 'B300', default: false }, { id: 'h200', label: 'H200', default: false }, { id: 'h100', label: 'H100', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, diff --git a/docs_new/src/snippets/diffusion/wan22-deployment.jsx b/docs_new/src/snippets/diffusion/wan22-deployment.jsx index fe749d531..ddda4ffc4 100644 --- a/docs_new/src/snippets/diffusion/wan22-deployment.jsx +++ b/docs_new/src/snippets/diffusion/wan22-deployment.jsx @@ -6,6 +6,7 @@ title: 'Hardware Platform', items: [ { id: 'b200', label: 'B200', default: true }, + { id: 'b300', label: 'B300', default: false }, { id: 'h200', label: 'H200', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, @@ -109,7 +110,7 @@ }; const generateCommand = () => { - const { task, modelsize, selectedLoraPath, bestPractice } = values; + const { hardware, task, modelsize, selectedLoraPath, bestPractice } = values; const configKey = `${task}-${modelsize}`; const config = modelConfigs[configKey]; if (!config) { @@ -118,7 +119,11 @@ let command = `sglang serve \\\n --model-path ${config.repoId} \\\n --dit-layerwise-offload true`; if (bestPractice === 'on') { - command += ` \\\n --num-gpus 4 \\\n --ulysses-degree 2 \\\n --enable-cfg-parallel`; + if (hardware === 'b300') { + command += ` \\\n --tp-size 2 \\\n --num-gpus 8 \\\n --sp-degree 2 \\\n --ulysses-degree 2 \\\n --enable-cfg-parallel`; + } else { + command += ` \\\n --num-gpus 4 \\\n --ulysses-degree 2 \\\n --enable-cfg-parallel`; + } } if (selectedLoraPath && selectedLoraPath !== 'none') { command += ` \\\n --lora-path ${selectedLoraPath}`;