minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[docs] Add B300 cookbook deployment options (#28697)

Browse Source
This commit is contained in:
Xiaoyu Zhang
2026-06-18 21:52:45 -07:00
committed by GitHub
parent 24d15dd92e
commit 7e20e25848
27 changed files with 503 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs_new/cookbook/autoregressive/InternLM/Intern-S1.mdx
+16 -13
View File
@@ -4,25 +4,28 @@ metatags:
description: "Deploy Intern-S1 with SGLang - community contribution guide for InternLM's Intern-S1 model deployment."
---
## 📝 Community Contribution Welcome
import { InternS1Deployment } from '/src/snippets/autoregressive/intern-s1-deployment.jsx';
This guide is currently under development. We welcome community contributions!
## 1. Model Introduction
If you have experience deploying **Intern-S1** with SGLang, please help us complete this documentation.
Intern-S1 includes the large **Intern-S1** MoE model and the smaller **Intern-S1-mini** dense model. The command generator below covers BF16 and FP8 serving on NVIDIA H100/H200/B200/B300 platforms.
## 🚀 How to Contribute
## 2. SGLang Installation
```shell Command
git clone https://github.com/YOUR_USERNAME/sglang-cookbook.git
cd sglang-cookbook
git checkout -b add-intern-s1-guide
# Edit this file and submit a PR
Refer to the [official SGLang installation guide](../../../docs/get-started/install), or install from source:
```bash Command
uv pip install 'git+https://github.com/sgl-project/sglang.git#subdirectory=python'
```
## 📚 Reference
## 3. Model Deployment
- [GLM-4.6V](../GLM/GLM-4.6V)
### 3.1 Basic Configuration
---
<InternS1Deployment />
**Let's build this together!** 🌟
### 3.2 Configuration Tips
- FP8 checkpoints use the matching BF16 checkpoint as tokenizer path.
- B300 deployments use `--attention-backend flashinfer`.
- Enable `--reasoning-parser interns1` and `--tool-call-parser interns1` when your workload needs structured reasoning or tool-call parsing.
docs_new/src/snippets/autoregressive/deepseek-math-v2-deployment.jsx
+7
View File
@@ -73,6 +73,13 @@ export const DeepSeekMathV2Deployment = () => {
if (memFraction) {
cmd += ` \\\n --mem-fraction-static ${memFraction}`;
}
if (hardware === 'b300') {
cmd += ' \\\n --attention-backend flashinfer';
if (values.dpattention !== 'enabled') {
cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
cmd += ' \\\n --cuda-graph-backend-prefill disabled';
}
}
cmd += ' \\\n --host 0.0.0.0 \\\n --port 30000';
docs_new/src/snippets/autoregressive/deepseek-r1-advanced-deployment.jsx
+68 -2
View File
@@ -9,6 +9,11 @@ const lookupData = {
"label": "B200",
"default": true
},
{
"id": "b300",
"label": "B300",
"default": false
},
{
"id": "h200",
"label": "H200",
@@ -182,6 +187,66 @@ const lookupData = {
"fp8_gemm_backend": "flashinfer_trtllm"
}
},
{
"hardware": "b300",
"quantization": "fp8",
"gpu_count": 8,
"scenario": "low-latency",
"parameters": {
"model_path": "deepseek-ai/DeepSeek-R1-0528",
"tensor_parallel_size": 8,
"kv_cache_dtype": "fp8_e4m3",
"attention_backend": "flashinfer",
"enforce_disable_flashinfer_allreduce_fusion": true,
"enable_symm_mem": true
}
},
{
"hardware": "b300",
"quantization": "fp8",
"gpu_count": 8,
"scenario": "high-throughput",
"parameters": {
"model_path": "deepseek-ai/DeepSeek-R1-0528",
"tensor_parallel_size": 8,
"kv_cache_dtype": "fp8_e4m3",
"attention_backend": "flashinfer",
"enforce_disable_flashinfer_allreduce_fusion": true,
"enable_symm_mem": true
}
},
{
"hardware": "b300",
"quantization": "fp4",
"gpu_count": 8,
"scenario": "low-latency",
"parameters": {
"model_path": "nvidia/DeepSeek-R1-0528-FP4-v2",
"tensor_parallel_size": 8,
"kv_cache_dtype": "fp8_e4m3",
"attention_backend": "flashinfer",
"enforce_disable_flashinfer_allreduce_fusion": true,
"moe_runner_backend": "flashinfer_cutlass",
"mem_fraction_static": 0.85,
"enable_symm_mem": true
}
},
{
"hardware": "b300",
"quantization": "fp4",
"gpu_count": 8,
"scenario": "high-throughput",
"parameters": {
"model_path": "nvidia/DeepSeek-R1-0528-FP4-v2",
"tensor_parallel_size": 8,
"kv_cache_dtype": "fp8_e4m3",
"attention_backend": "flashinfer",
"enforce_disable_flashinfer_allreduce_fusion": true,
"moe_runner_backend": "flashinfer_cutlass",
"mem_fraction_static": 0.85,
"enable_symm_mem": true
}
},
{
"hardware": "h200",
"quantization": "fp8",
@@ -387,7 +452,7 @@ const lookupData = {
{
"hardware": "h200",
"quantization": "fp4",
"error": "FP4 is only available for B200 hardware. Please select FP8 quantization."
"error": "FP4 is only available for B200/B300 hardware. Please select FP8 quantization."
}
]
};
@@ -407,6 +472,7 @@ const fieldToFlag = {
enable_flashinfer_allreduce_fusion: 'enable-flashinfer-allreduce-fusion',
scheduler_recv_interval: 'scheduler-recv-interval',
enable_symm_mem: 'enable-symm-mem',
enforce_disable_flashinfer_allreduce_fusion: 'enforce-disable-flashinfer-allreduce-fusion',
disable_radix_cache: 'disable-radix-cache',
attention_backend: 'attention-backend',
moe_runner_backend: 'moe-runner-backend',
@@ -497,7 +563,7 @@ const resolveItems = (option, values) =>
title: 'Hardware Platform',
items: uiOptions.hardware
.filter((option) =>
['b200', 'h200', 'mi300x', 'mi325x', 'mi355x'].includes(option.id)
['b200', 'b300', 'h200', 'mi300x', 'mi325x', 'mi355x'].includes(option.id)
)
.map((option) => ({
id: option.id,
docs_new/src/snippets/autoregressive/deepseek-r1-basic-deployment.jsx
+13
View File
@@ -7,6 +7,7 @@ export const DeepSeekR1BasicDeployment = () => {
{ id: 'h100', label: 'H100', default: false },
{ id: 'h200', label: 'H200', default: false },
{ id: 'b200', label: 'B200', default: true },
{ id: 'b300', label: 'B300', default: false },
{ id: 'mi300x', label: 'MI300X', default: false },
{ id: 'mi325x', label: 'MI325X', default: false },
{ id: 'mi355x', label: 'MI355X', default: false },
@@ -138,6 +139,18 @@ export const DeepSeekR1BasicDeployment = () => {
' \\\n --kv-cache-dtype fp8_e4m3 # Optional: enables fp8 kv cache and fp8 attention kernels to improve performance';
}
if (hardware === 'b300') {
command += ' \\\n --kv-cache-dtype fp8_e4m3';
command += ' \\\n --attention-backend flashinfer';
command += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
if (quantization === 'fp4') {
command += ' \\\n --moe-runner-backend flashinfer_cutlass';
}
if (quantization === 'fp4' || strategyValues.includes('mtp')) {
command += ' \\\n --mem-fraction-static 0.85';
}
}
if (isXeon) {
command += ' \\\n --device cpu \\\n --disable-overlap-schedule';
if (quantization === 'int8') {
docs_new/src/snippets/autoregressive/deepseek-v32-deployment.jsx
+29 -10
View File
@@ -3,7 +3,7 @@ export const DeepSeekV32Deployment = () => {
//
// Model variants:
// DeepSeek-V3.2, V3.2-Exp, V3.2-Speciale → deepseek-ai/ family, TP=8
// DeepSeek-V3.2-NVFP4 → nvidia/ family, B200 only, TP=4
// DeepSeek-V3.2-NVFP4 → nvidia/ family, B200/B300 only, TP=4
// DeepSeek-V3.2-MXFP4 → amd/ family, MI300X/MI355X only, TP=8
const options = {
hardware: {
@@ -12,6 +12,7 @@ export const DeepSeekV32Deployment = () => {
items: [
{ id: 'h200', label: 'H200', default: true },
{ id: 'b200', label: 'B200', default: false },
{ id: 'b300', label: 'B300', default: false },
{ id: 'mi300x', label: 'MI300X', default: false },
{ id: 'mi355x', label: 'MI355X', default: false }
]
@@ -21,13 +22,13 @@ export const DeepSeekV32Deployment = () => {
title: 'Model Name',
getDynamicItems: (values) => {
const hw = values.hardware;
const isB200 = hw === 'b200';
const isBlackwell = hw === 'b200' || hw === 'b300';
const isAMD = hw === 'mi300x' || hw === 'mi355x';
return [
{ id: 'v32', label: 'DeepSeek-V3.2', default: !isB200 && !isAMD },
{ id: 'v32', label: 'DeepSeek-V3.2', default: !isBlackwell && !isAMD },
{ id: 'v32speciale', label: 'DeepSeek-V3.2-Speciale', default: false },
{ id: 'v32exp', label: 'DeepSeek-V3.2-Exp', default: false },
{ id: 'v32nvfp4', label: 'DeepSeek-V3.2-NVFP4', default: isB200, disabled: !isB200, disabledReason: 'NVFP4 requires B200 (Blackwell)' },
{ id: 'v32nvfp4', label: 'DeepSeek-V3.2-NVFP4', default: isBlackwell, disabled: !isBlackwell, disabledReason: 'NVFP4 requires B200/B300 (Blackwell)' },
{ id: 'v32mxfp4', label: 'DeepSeek-V3.2-MXFP4', default: isAMD, disabled: !isAMD, disabledReason: 'MXFP4 requires AMD MI300X/MI355X' }
];
}
@@ -101,7 +102,7 @@ export const DeepSeekV32Deployment = () => {
return () => observer.disconnect();
}, []);
// When hardware changes, re-resolve model name defaults (NVFP4→B200, MXFP4→AMD).
// When hardware changes, re-resolve model name defaults (NVFP4→Blackwell, MXFP4→AMD).
useEffect(() => {
setValues(prev => {
const next = { ...prev };
@@ -139,10 +140,12 @@ export const DeepSeekV32Deployment = () => {
const isNvfp4 = modelname === 'v32nvfp4';
const isMxfp4 = modelname === 'v32mxfp4';
const isAMD = hardware === 'mi300x' || hardware === 'mi355x';
const isB300 = hardware === 'b300';
const isBlackwell = hardware === 'b200' || isB300;
// Validation: NVFP4 requires B200
if (isNvfp4 && hardware !== 'b200') {
return `# Error: DeepSeek-V3.2-NVFP4 requires NVIDIA B200 (Blackwell) hardware\n# Please select "B200" for Hardware Platform or choose a different model`;
// Validation: NVFP4 requires Blackwell
if (isNvfp4 && !isBlackwell) {
return `# Error: DeepSeek-V3.2-NVFP4 requires NVIDIA B200/B300 (Blackwell) hardware\n# Please select "B200" or "B300" for Hardware Platform or choose a different model`;
}
// Validation: MXFP4 requires AMD MI300X/MI355X
@@ -176,8 +179,16 @@ export const DeepSeekV32Deployment = () => {
let cmd = 'sglang serve \\\n';
cmd += ` --model-path ${modelName}`;
cmd += ' \\\n --tp 4';
cmd += ' \\\n --quantization modelopt_fp4';
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm';
if (isB300) {
cmd += ' \\\n --attention-backend flashinfer';
cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
cmd += ' \\\n --cuda-graph-backend-prefill disabled';
cmd += ' \\\n --moe-runner-backend flashinfer_cutlass';
cmd += ' \\\n --disable-flashinfer-autotune';
} else {
cmd += ' \\\n --quantization modelopt_fp4';
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm';
}
return cmd;
}
@@ -222,6 +233,14 @@ export const DeepSeekV32Deployment = () => {
cmd += ' \\\n --speculative-num-draft-tokens 4';
}
if (isB300) {
cmd += ' \\\n --attention-backend flashinfer';
if (!strategyArray.includes('dp') || strategyArray.includes('ep') || strategyArray.includes('mtp')) {
cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
cmd += ' \\\n --cuda-graph-backend-prefill disabled';
}
}
// Add tool-call-parser if enabled (not supported for Speciale)
if (toolcall === 'enabled' && modelname !== 'v32speciale') {
if (modelname === 'v32exp') {
docs_new/src/snippets/autoregressive/gemma4-deployment.jsx
+12
View File
@@ -28,6 +28,7 @@ export const Gemma4Deployment = () => {
return [
{ id: 'h200', label: 'H200', default: true },
{ id: 'b200', label: 'B200', default: false },
{ id: 'b300', label: 'B300', default: false },
{ id: 'mi300x', label: 'MI300X', default: false, disabled: !showMI300X },
];
}
@@ -76,6 +77,13 @@ export const Gemma4Deployment = () => {
'31b': { tp: 1, mem: 0.9 },
'26b-a4b': { tp: 1, mem: 0.9 },
},
b300: {
e2b: { tp: 1, mem: 0.9 },
e4b: { tp: 1, mem: 0.9 },
'12b': { tp: 1, mem: 0.9 },
'31b': { tp: 1, mem: 0.9 },
'26b-a4b': { tp: 1, mem: 0.9 },
},
mi300x: {
'31b': { tp: 1, mem: 0.80 },
'26b-a4b': { tp: 1, mem: 0.80 },
@@ -129,6 +137,10 @@ export const Gemma4Deployment = () => {
cmd += ` \\\n --speculative-eagle-topk 1`;
}
if (hardware === 'b300') {
cmd += ` \\\n --attention-backend triton`;
}
cmd += ` \\\n --mem-fraction-static ${mem}`;
cmd += ` \\\n --host 0.0.0.0 --port 30000`;
docs_new/src/snippets/autoregressive/glm-46-deployment.jsx
+9
View File
@@ -8,6 +8,7 @@ export const GLM46Deployment = () => {
{ id: 'h100', label: 'H100', default: true },
{ id: 'h200', label: 'H200', default: false },
{ id: 'b200', label: 'B200', default: false },
{ id: 'b300', label: 'B300', default: false },
{ id: 'mi300x', label: 'MI300X', default: false },
{ id: 'mi325x', label: 'MI325X', default: false },
{ id: 'mi355x', label: 'MI355X', default: false }
@@ -132,6 +133,9 @@ export const GLM46Deployment = () => {
// Strategy-specific parameters
if (strategyArray.includes('dp')) {
cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`;
if (hardware === 'b300') {
cmd += ` \\\n --cuda-graph-max-bs-decode 256`;
}
}
if (strategyArray.includes('ep')) {
cmd += ` \\\n --ep 8`;
@@ -151,6 +155,11 @@ export const GLM46Deployment = () => {
cmd += ` \\\n --reasoning-parser glm45`;
}
if (hardware === 'b300') {
cmd += ` \\\n --attention-backend flashinfer`;
cmd += ` \\\n --enforce-disable-flashinfer-allreduce-fusion`;
}
return cmd;
};
docs_new/src/snippets/autoregressive/glm-46v-deployment.jsx
+13
View File
@@ -6,6 +6,7 @@ export const GLM46VDeployment = () => {
title: 'Hardware Platform',
items: [
{ id: 'b200', label: 'B200', default: true },
{ id: 'b300', label: 'B300', default: false },
{ id: 'h100', label: 'H100', default: false },
{ id: 'h200', label: 'H200', default: false },
{ id: 'mi300x', label: 'MI300X', default: false },
@@ -94,6 +95,7 @@ export const GLM46VDeployment = () => {
h100: { tp: 8 },
h200: { tp: 8 },
b200: { tp: 8 },
b300: { tp: 8 },
mi300x: { tp: 8 },
mi325x: { tp: 8 },
mi355x: { tp: 8 }
@@ -103,6 +105,7 @@ export const GLM46VDeployment = () => {
h100: { tp: 1 },
h200: { tp: 1 },
b200: { tp: 1 },
b300: { tp: 1 },
mi300x: { tp: 1 },
mi325x: { tp: 1 },
mi355x: { tp: 1 }
@@ -122,6 +125,10 @@ export const GLM46VDeployment = () => {
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
const modelName = `zai-org/${config.baseName}${quantSuffix}`;
if (hardware === 'b300' && modelsize === '9b' && quantization === 'fp8') {
return '# Error: GLM-4.6V-Flash-FP8 is not available on B300 in this cookbook configuration\n# Please use BF16 for GLM-4.6V-Flash or select the 106B model';
}
let cmd = 'python -m sglang.launch_server \\\n';
cmd += ` --model ${modelName}`;
@@ -142,6 +149,12 @@ export const GLM46VDeployment = () => {
cmd += ` \\\n --tool-call-parser glm45`;
}
if (hardware === 'b300') {
cmd += ` \\\n --attention-backend flashinfer`;
cmd += ` \\\n --enforce-disable-flashinfer-allreduce-fusion`;
cmd += ` \\\n --cuda-graph-backend-decode disabled`;
}
return cmd;
};
docs_new/src/snippets/autoregressive/glm-5-deployment.jsx
+40 -16
View File
@@ -4,6 +4,7 @@ export const GLM5Deployment = () => {
// Supported quantization per hardware:
// H100 / H200 / MI300X / MI325X / MI355X → BF16 (AMD only) + FP8 (NV only)
// B200 → NVFP4 (default), FP8, BF16
// B300 → NVFP4 (default), FP8
//
// BF16 always needs 2x GPUs compared to FP8. AMD only supports BF16.
const options = {
@@ -13,6 +14,7 @@ export const GLM5Deployment = () => {
items: [
{ id: 'h200', label: 'H200', default: true },
{ id: 'b200', label: 'B200', default: false },
{ id: 'b300', label: 'B300', default: false },
{ id: 'h100', label: 'H100', default: false },
{ id: 'mi300x', label: 'MI300X/MI325X', default: false },
{ id: 'mi355x', label: 'MI355X', default: false }
@@ -24,11 +26,12 @@ export const GLM5Deployment = () => {
getDynamicItems: (values) => {
const hw = values.hardware;
const isAMD = hw === 'mi300x' || hw === 'mi355x';
const isB200 = hw === 'b200';
const isB300 = hw === 'b300';
const isBlackwell = hw === 'b200' || isB300;
return [
{ id: 'bf16', label: 'BF16', subtitle: 'Full Weights', default: isAMD },
{ id: 'fp8', label: 'FP8', subtitle: 'High Throughput', default: !isAMD && !isB200, disabled: isAMD, disabledReason: 'FP8 not verified on AMD' },
{ id: 'nvfp4', label: 'NVFP4', subtitle: 'Highest Throughput', default: isB200, disabled: !isB200, disabledReason: 'NVFP4 only on B200' }
{ id: 'bf16', label: 'BF16', subtitle: 'Full Weights', default: isAMD, disabled: isB300, disabledReason: isB300 ? 'BF16 requires more than the validated 8-GPU B300 node' : '' },
{ id: 'fp8', label: 'FP8', subtitle: 'High Throughput', default: !isAMD && !isBlackwell, disabled: isAMD, disabledReason: 'FP8 not verified on AMD' },
{ id: 'nvfp4', label: 'NVFP4', subtitle: 'Highest Throughput', default: isBlackwell, disabled: !isBlackwell, disabledReason: 'NVFP4 only on B200/B300' }
];
}
},
@@ -75,6 +78,7 @@ export const GLM5Deployment = () => {
h100: { fp8: { tp: 16, mem: 0.85 }, bf16: { tp: 32, mem: 0.85 } },
h200: { fp8: { tp: 8, mem: 0.85 }, bf16: { tp: 16, mem: 0.85 } },
b200: { nvfp4: { tp: 4, mem: 0.9 }, fp8: { tp: 8, mem: 0.9 }, bf16: { tp: 16, mem: 0.9 } },
b300: { nvfp4: { tp: 4, mem: 0.9 }, fp8: { tp: 8, mem: 0.9 } },
mi300x: { bf16: { tp: 8, mem: 0.80 } },
mi355x: { bf16: { tp: 8, mem: 0.80 } }
};
@@ -147,7 +151,10 @@ export const GLM5Deployment = () => {
modelName = `zai-org/GLM-5${suffix}`;
}
const hwConfig = modelConfigs[hardware][effectiveQuant];
const hwConfig = modelConfigs[hardware]?.[effectiveQuant];
if (!hwConfig) {
return '# Please select a valid hardware and quantization combination';
}
const tpValue = hwConfig.tp;
const memFraction = hwConfig.mem;
@@ -155,19 +162,27 @@ export const GLM5Deployment = () => {
cmd += ` --model-path ${modelName}`;
cmd += ` \\\n --tp ${tpValue}`;
// NVFP4 B200: trtllm DSA backends, flashinfer fusion, FP8 KV cache.
if (isNVFP4) {
cmd += ' \\\n --trust-remote-code';
cmd += ' \\\n --quantization modelopt_fp4';
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
cmd += ' \\\n --dsa-decode-backend trtllm';
cmd += ' \\\n --dsa-prefill-backend trtllm';
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm';
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
cmd += ' \\\n --enable-dp-lm-head';
cmd += ' \\\n --disable-radix-cache';
cmd += ' \\\n --max-prefill-tokens 32768';
cmd += ' \\\n --chunked-prefill-size 32768';
if (hardware === 'b300') {
cmd += ' \\\n --attention-backend flashinfer';
cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
cmd += ' \\\n --cuda-graph-backend-prefill disabled';
cmd += ' \\\n --moe-runner-backend flashinfer_cutlass';
cmd += ' \\\n --cuda-graph-backend-decode disabled';
cmd += ' \\\n --disable-flashinfer-autotune';
} else {
cmd += ' \\\n --quantization modelopt_fp4';
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
cmd += ' \\\n --dsa-decode-backend trtllm';
cmd += ' \\\n --dsa-prefill-backend trtllm';
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm';
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
cmd += ' \\\n --enable-dp-lm-head';
cmd += ' \\\n --disable-radix-cache';
cmd += ' \\\n --max-prefill-tokens 32768';
cmd += ' \\\n --chunked-prefill-size 32768';
}
cmd += ` \\\n --mem-fraction-static ${memFraction}`;
cmd += ' \\\n --scheduler-recv-interval 10';
cmd += ' \\\n --tokenizer-worker-num 6';
@@ -185,6 +200,9 @@ export const GLM5Deployment = () => {
if (values.dpattention === 'enabled') {
cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`;
if (hardware === 'b300') {
cmd += ' \\\n --cuda-graph-max-bs-decode 256';
}
}
if (values.reasoning === 'enabled') cmd += ' \\\n --reasoning-parser glm45';
if (values.toolcall === 'enabled') cmd += ' \\\n --tool-call-parser glm47';
@@ -206,6 +224,12 @@ export const GLM5Deployment = () => {
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
}
if (hardware === 'b300' && effectiveQuant === 'fp8') {
cmd += ' \\\n --attention-backend flashinfer';
cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
cmd += ' \\\n --cuda-graph-backend-prefill disabled';
}
// H200 FP8: flashinfer allreduce fusion.
if (hardware === 'h200' && effectiveQuant === 'fp8') {
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
docs_new/src/snippets/autoregressive/gpt-oss-deployment.jsx
+9
View File
@@ -6,6 +6,7 @@ export const GPTOSSDeployment = () => {
title: 'Hardware Platform',
items: [
{ id: 'b200', label: 'B200', default: true },
{ id: 'b300', label: 'B300', default: false },
{ id: 'h200', label: 'H200', default: false },
{ id: 'h100', label: 'H100', default: false },
{ id: 'mi300x', label: 'MI300X', default: false },
@@ -102,6 +103,7 @@ export const GPTOSSDeployment = () => {
h100: { tp: 8 },
h200: { tp: 8 },
b200: { tp: 8 },
b300: { tp: 8 },
mi300x: { tp: 8 },
mi325x: { tp: 8 },
mi355x: { tp: 8 }
@@ -111,6 +113,7 @@ export const GPTOSSDeployment = () => {
h100: { tp: 1 },
h200: { tp: 1 },
b200: { tp: 1 },
b300: { tp: 1 },
mi300x: { tp: 1 },
mi325x: { tp: 1 },
mi355x: { tp: 1 }
@@ -170,6 +173,12 @@ export const GPTOSSDeployment = () => {
cmd += ` \\\n --tool-call-parser gpt-oss`;
}
if (hardware === 'b300') {
cmd += ` \\\n --attention-backend triton`;
cmd += ` \\\n --moe-runner-backend triton`;
cmd += ` \\\n --enforce-disable-flashinfer-allreduce-fusion`;
}
// Add speculative decoding if enabled (MI30x handled above)
if (speculative === 'enabled') {
cmd += ` \\\n --speculative-algorithm EAGLE3 \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
docs_new/src/snippets/autoregressive/intern-s1-deployment.jsx
+167
View File
@@ -0,0 +1,167 @@
export const InternS1Deployment = () => {
const options = {
hardware: {
name: 'hardware',
title: 'Hardware Platform',
items: [
{ id: 'b200', label: 'B200', default: true },
{ id: 'b300', label: 'B300', default: false },
{ id: 'h100', label: 'H100', default: false },
{ id: 'h200', label: 'H200', default: false },
],
},
modelsize: {
name: 'modelsize',
title: 'Model Size',
items: [
{ id: 'S1', label: '235B', subtitle: 'MoE', default: true },
{ id: 'S1-mini', label: '8B', subtitle: 'Dense', default: false },
],
},
quantization: {
name: 'quantization',
title: 'Quantization',
items: [
{ id: 'bf16', label: 'BF16', default: true },
{ id: 'fp8', label: 'FP8', default: false },
],
},
reasoning: {
name: 'reasoning',
title: 'Reasoning Parser',
items: [
{ id: 'disabled', label: 'Disabled', default: true },
{ id: 'enabled', label: 'Enabled', default: false },
],
},
toolcall: {
name: 'toolcall',
title: 'Tool Call Parser',
items: [
{ id: 'disabled', label: 'Disabled', default: true },
{ id: 'enabled', label: 'Enabled', default: false },
],
},
};
const modelConfigs = {
S1: {
baseName: 'S1',
h100: { bf16: { tp: 8 }, fp8: { tp: 8, ep: 2 } },
h200: { bf16: { tp: 8 }, fp8: { tp: 8, ep: 2 } },
b200: { bf16: { tp: 8 }, fp8: { tp: 8, ep: 2 } },
b300: { bf16: { tp: 8 }, fp8: { tp: 8, ep: 2 } },
},
'S1-mini': {
baseName: 'S1-mini',
h100: { bf16: { tp: 1 }, fp8: { tp: 1 } },
h200: { bf16: { tp: 1 }, fp8: { tp: 1 } },
b200: { bf16: { tp: 1 }, fp8: { tp: 1 } },
b300: { bf16: { tp: 1 }, fp8: { tp: 1 } },
},
};
const getInitialState = () => {
const initialState = {};
Object.entries(options).forEach(([key, option]) => {
const defaultItem = option.items.find((item) => item.default);
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
});
return initialState;
};
const [values, setValues] = useState(getInitialState);
const [isDark, setIsDark] = useState(false);
useEffect(() => {
const checkDarkMode = () => {
const html = document.documentElement;
const isDarkMode =
html.classList.contains('dark') ||
html.getAttribute('data-theme') === 'dark' ||
html.style.colorScheme === 'dark';
setIsDark(isDarkMode);
};
checkDarkMode();
const observer = new MutationObserver(checkDarkMode);
observer.observe(document.documentElement, {
attributes: true,
attributeFilter: ['class', 'data-theme', 'style'],
});
return () => observer.disconnect();
}, []);
const handleRadioChange = (optionName, value) => {
setValues((prev) => ({ ...prev, [optionName]: value }));
};
const generateCommand = () => {
const { hardware, modelsize, quantization, reasoning, toolcall } = values;
const modelConfig = modelConfigs[modelsize];
const hwConfig = modelConfig?.[hardware]?.[quantization];
if (!hwConfig) {
return '# Please select a valid hardware and quantization combination';
}
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
const modelName = `internlm/Intern-${modelConfig.baseName}${quantSuffix}`;
const flags = [];
flags.push(` --model ${modelName}`);
if (hwConfig.tp > 1) flags.push(` --tp ${hwConfig.tp}`);
if (hwConfig.ep) flags.push(` --ep ${hwConfig.ep}`);
if (quantization === 'fp8') flags.push(` --tokenizer-path internlm/Intern-${modelConfig.baseName}`);
if (reasoning === 'enabled') flags.push(' --reasoning-parser interns1');
if (toolcall === 'enabled') flags.push(' --tool-call-parser interns1');
flags.push(' --trust-remote-code');
if (hardware === 'b300') flags.push(' --attention-backend flashinfer');
return `python -m sglang.launch_server \\\n${flags.join(' \\\n')}`;
};
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
return (
<div style={containerStyle} className="not-prose">
{Object.entries(options).map(([key, option]) => (
<div key={key} style={cardStyle}>
<div style={titleStyle}>{option.title}</div>
<div style={itemsStyle}>
{option.items.map((item) => {
const isChecked = values[option.name] === item.id;
return (
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
<input
type="radio"
name={option.name}
value={item.id}
checked={isChecked}
onChange={() => handleRadioChange(option.name, item.id)}
style={{ display: 'none' }}
/>
{item.label}
{item.subtitle && (
<small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>
{item.subtitle}
</small>
)}
</label>
);
})}
</div>
</div>
))}
<div style={cardStyle}>
<div style={titleStyle}>Run this Command:</div>
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
</div>
</div>
);
};
docs_new/src/snippets/autoregressive/kimi-k2-deployment.jsx
+10
View File
@@ -8,6 +8,7 @@ export const KimiK2Deployment = () => {
items: [
{ id: 'h200', label: 'H200', default: true },
{ id: 'b200', label: 'B200', default: false },
{ id: 'b300', label: 'B300', default: false },
{ id: 'mi300x', label: 'MI300X', default: false },
{ id: 'mi325x', label: 'MI325X', default: false },
{ id: 'mi355x', label: 'MI355X', default: false }
@@ -82,6 +83,15 @@ export const KimiK2Deployment = () => {
cmd += ` \\\n --trust-remote-code`;
if (hardware === 'b300') {
cmd += ` \\\n --attention-backend flashinfer`;
if (strategyArray.includes('dp')) {
cmd += ` \\\n --prefill-attention-backend triton`;
}
cmd += ` \\\n --enforce-disable-flashinfer-allreduce-fusion`;
cmd += ` \\\n --mem-fraction-static 0.85`;
}
if (toolcall === 'enabled') {
cmd += ` \\\n --tool-call-parser kimi_k2`;
}
docs_new/src/snippets/autoregressive/llada-21-deployment.jsx
+3 -2
View File
@@ -9,6 +9,7 @@ export const LLaDA21Deployment = () => {
{ id: 'h100', label: 'H100', default: true },
{ id: 'h200', label: 'H200', default: false },
{ id: 'b200', label: 'B200', default: false },
{ id: 'b300', label: 'B300', default: false },
{ id: 'mi300x', label: 'MI300X', default: false },
{ id: 'mi325x', label: 'MI325X', default: false },
{ id: 'mi355x', label: 'MI355X', default: false }
@@ -34,7 +35,7 @@ export const LLaDA21Deployment = () => {
if (modelsize === 'mini') {
tpSize = 1;
} else {
if (hardware === 'b200') {
if (hardware === 'b200' || hardware === 'b300') {
tpSize = 2;
} else {
tpSize = 4;
@@ -48,7 +49,7 @@ export const LLaDA21Deployment = () => {
args.push(`--trust-remote-code`);
args.push(`--mem-fraction-static 0.8`);
args.push(`--max-running-requests 1`);
if (hardware === 'h100' || hardware === 'h200' || hardware === 'b200') {
if (hardware === 'h100' || hardware === 'h200' || hardware === 'b200' || hardware === 'b300') {
args.push(`--attention-backend flashinfer`);
}
docs_new/src/snippets/autoregressive/minimax-m27-deployment.jsx
+12 -6
View File
@@ -8,6 +8,7 @@ export const MiniMaxM27Deployment = () => {
items: [
{ id: 'h200', label: 'H200', default: true },
{ id: 'b200', label: 'B200', default: false },
{ id: 'b300', label: 'B300', default: false },
{ id: 'gb300', label: 'GB300', default: false },
{ id: 'a100', label: 'A100', default: false },
{ id: 'h100', label: 'H100', default: false },
@@ -23,6 +24,7 @@ export const MiniMaxM27Deployment = () => {
getDynamicItems: (values) => {
const hw = values.hardware;
const isAMD = hw === 'mi300x' || hw === 'mi325x' || hw === 'mi355x';
const isB300 = hw === 'b300';
const isGB300 = hw === 'gb300';
const isXeon = hw === 'xeon';
if (isXeon) {
@@ -33,8 +35,8 @@ export const MiniMaxM27Deployment = () => {
const canUse2GPU = isAMD || isGB300;
return [
{ id: '2gpu', label: '2', default: canUse2GPU, disabled: !canUse2GPU },
{ id: '4gpu', label: '4', default: !canUse2GPU, disabled: false },
{ id: '8gpu', label: '8', default: false, disabled: isGB300 }
{ id: '4gpu', label: '4', default: !canUse2GPU || isB300, disabled: false },
{ id: '8gpu', label: '8', default: false, disabled: isGB300 || isB300 }
];
}
},
@@ -43,11 +45,11 @@ export const MiniMaxM27Deployment = () => {
title: 'Precision',
getDynamicItems: (values) => {
const hw = values.hardware;
const isBlackwell = hw === 'b200' || hw === 'gb300';
const isBlackwell = hw === 'b200' || hw === 'b300' || hw === 'gb300';
return [
{ id: 'fp8', label: 'FP8', default: true, disabled: false },
{ id: 'fp4', label: 'FP4', default: false, disabled: !isBlackwell,
disabledReason: 'NVFP4 requires Blackwell (B200/GB300)' }
disabledReason: 'NVFP4 requires Blackwell (B200/B300/GB300)' }
];
}
},
@@ -131,6 +133,7 @@ export const MiniMaxM27Deployment = () => {
const { hardware, gpuCount, precision, thinking, toolcall } = values;
const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x';
const isB300 = hardware === 'b300';
const isGB300 = hardware === 'gb300';
const isXeon = hardware === 'xeon';
const canUse2GPU = isAMD || isGB300;
@@ -139,11 +142,11 @@ export const MiniMaxM27Deployment = () => {
return '# Please select compatible hardware\n# 2-GPU requires AMD MI300X/MI325X/MI355X or GB300';
}
const isBlackwell = hardware === 'b200' || hardware === 'gb300';
const isBlackwell = hardware === 'b200' || hardware === 'b300' || hardware === 'gb300';
const isFp4 = precision === 'fp4';
if (isFp4 && !isBlackwell) {
return '# NVFP4 requires Blackwell hardware (B200 or GB300)';
return '# NVFP4 requires Blackwell hardware (B200, B300, or GB300)';
}
const modelName = isFp4 ? 'nvidia/MiniMax-M2.7-NVFP4' : 'MiniMaxAI/MiniMax-M2.7';
@@ -184,6 +187,9 @@ export const MiniMaxM27Deployment = () => {
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
cmd += ' \\\n --attention-backend triton';
}
if (isB300) {
cmd += ' \\\n --attention-backend flashinfer';
}
if (isBlackwell) {
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm_routed';
docs_new/src/snippets/autoregressive/mistral-small-4-deployment.jsx
+4
View File
@@ -82,6 +82,10 @@ export const MistralSmall4Deployment = () => {
}
});
if (hardware === 'b300') {
cmd += ` \\\n --attention-backend flashinfer`;
}
return cmd;
};
docs_new/src/snippets/autoregressive/nemotron3-nano-deployment.jsx
+5 -1
View File
@@ -7,7 +7,8 @@ export const Nemotron3NanoDeployment = () => {
title: 'Hardware Platform',
items: [
{ id: 'h200', label: 'H200', default: false },
{ id: 'b200', label: 'B200', default: true }
{ id: 'b200', label: 'B200', default: true },
{ id: 'b300', label: 'B300', default: false }
]
},
modelVariant: {
@@ -71,6 +72,9 @@ export const Nemotron3NanoDeployment = () => {
cmd += ` --trust-remote-code \\\n`;
cmd += ` --tp ${tp} \\\n`;
cmd += ` --kv-cache-dtype ${kvcache} \\\n`;
if (hardware === 'b300') {
cmd += ` --attention-backend flashinfer \\\n`;
}
// Add thinking parser and tool call parser if enabled
for (const [key, option] of Object.entries(options)) {
docs_new/src/snippets/autoregressive/nemotron3-super-deployment.jsx
+5 -1
View File
@@ -20,7 +20,8 @@ export const Nemotron3SuperDeployment = () => {
title: 'Hardware Platform',
items: [
{ id: 'h200', label: 'H200', default: false },
{ id: 'b200', label: 'B200', default: true }
{ id: 'b200', label: 'B200', default: true },
{ id: 'b300', label: 'B300', default: false }
]
},
tp: {
@@ -87,6 +88,9 @@ export const Nemotron3SuperDeployment = () => {
if (kvcache && kvcache !== 'none') {
cmd += ` --kv-cache-dtype ${kvcache} \\\n`;
}
if (values.hardware === 'b300') {
cmd += ` --attention-backend flashinfer \\\n`;
}
for (const [key, option] of Object.entries(options)) {
if (option.commandRule) {
docs_new/src/snippets/autoregressive/qwen3-coder-next-deployment.jsx
+5
View File
@@ -7,6 +7,7 @@ export const Qwen3CoderNextDeployment = () => {
{ id: 'h200', label: 'H200', default: true },
{ id: 'h100', label: 'H100', default: false },
{ id: 'b200', label: 'B200', default: false },
{ id: 'b300', label: 'B300', default: false },
{ id: 'mi300x', label: 'MI300X', default: false },
{ id: 'mi325x', label: 'MI325X', default: false },
{ id: 'mi355x', label: 'MI355X', default: false },
@@ -48,6 +49,7 @@ export const Qwen3CoderNextDeployment = () => {
h100: { bf16: { tp: 4 }, fp8: { tp: 2 } },
h200: { bf16: { tp: 2 }, fp8: { tp: 1 } },
b200: { bf16: { tp: 2 }, fp8: { tp: 1 } },
b300: { bf16: { tp: 2 }, fp8: { tp: 1 } },
mi300x: { bf16: { tp: 2 }, fp8: { tp: 1 } },
mi325x: { bf16: { tp: 2 }, fp8: { tp: 1 } },
mi355x: { bf16: { tp: 2 }, fp8: { tp: 1 } },
@@ -99,6 +101,9 @@ export const Qwen3CoderNextDeployment = () => {
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
cmd += ` \\\n --attention-backend triton`;
}
if (hardware === 'b300') {
cmd += ` \\\n --attention-backend flashinfer`;
}
return cmd;
};
docs_new/src/snippets/autoregressive/qwen3-deployment.jsx
+14
View File
@@ -7,6 +7,7 @@ export const Qwen3Deployment = () => {
h100: { tp: 8, ep: 0, bf16: true, fp8: true },
h200: { tp: 8, ep: 0, bf16: true, fp8: true },
b200: { tp: 8, ep: 0, bf16: true, fp8: true },
b300: { tp: 8, ep: 0, bf16: true, fp8: true },
mi300x: { tp: 4, ep: 0, bf16: true, fp8: true },
mi325x: { tp: 4, ep: 0, bf16: true, fp8: true },
mi355x: { tp: 4, ep: 0, bf16: true, fp8: true },
@@ -18,6 +19,7 @@ export const Qwen3Deployment = () => {
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
@@ -29,6 +31,7 @@ export const Qwen3Deployment = () => {
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
@@ -40,6 +43,7 @@ export const Qwen3Deployment = () => {
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
@@ -51,6 +55,7 @@ export const Qwen3Deployment = () => {
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
@@ -62,6 +67,7 @@ export const Qwen3Deployment = () => {
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
@@ -73,6 +79,7 @@ export const Qwen3Deployment = () => {
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
@@ -84,6 +91,7 @@ export const Qwen3Deployment = () => {
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
@@ -98,6 +106,7 @@ export const Qwen3Deployment = () => {
title: 'Hardware Platform',
items: [
{ id: 'b200', label: 'B200', default: true },
{ id: 'b300', label: 'B300', default: false },
{ id: 'h100', label: 'H100', default: false },
{ id: 'h200', label: 'H200', default: false },
{ id: 'mi300x', label: 'MI300X', default: false },
@@ -297,6 +306,11 @@ export const Qwen3Deployment = () => {
cmd += ' \\\n --tool-call-parser qwen25';
}
if (hardware === 'b300') {
cmd += ' \\\n --attention-backend flashinfer';
cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
}
return cmd;
};
docs_new/src/snippets/autoregressive/qwen3-next-deployment.jsx
+6
View File
@@ -5,6 +5,7 @@ export const Qwen3NextDeployment = () => {
title: 'Hardware Platform',
items: [
{ id: 'b200', label: 'B200', default: true },
{ id: 'b300', label: 'B300', default: false },
{ id: 'h200', label: 'H200', default: false },
{ id: 'h100', label: 'H100', default: false },
{ id: 'mi300x', label: 'MI300X', default: false },
@@ -75,6 +76,7 @@ export const Qwen3NextDeployment = () => {
h100: { tp: 4, ep: 0, bf16: true, fp8: true },
h200: { tp: 2, ep: 0, bf16: true, fp8: true },
b200: { tp: 2, ep: 0, bf16: true, fp8: true },
b300: { tp: 2, ep: 0, bf16: true, fp8: true },
mi300x: { tp: 2, ep: 0, bf16: true, fp8: true },
mi325x: { tp: 2, ep: 0, bf16: true, fp8: true },
mi355x: { tp: 2, ep: 0, bf16: true, fp8: true },
@@ -136,6 +138,10 @@ export const Qwen3NextDeployment = () => {
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
cmd += ` \\\n --attention-backend triton`;
}
if (hardware === 'b300') {
cmd += ` \\\n --attention-backend flashinfer`;
cmd += ` \\\n --enforce-disable-flashinfer-allreduce-fusion`;
}
return cmd;
};
docs_new/src/snippets/autoregressive/qwen3-vl-deployment.jsx
+12
View File
@@ -6,6 +6,7 @@ export const Qwen3VLDeployment = () => {
title: 'Hardware Platform',
items: [
{ id: 'b200', label: 'B200', default: true },
{ id: 'b300', label: 'B300', default: false },
{ id: 'h100', label: 'H100', default: false },
{ id: 'h200', label: 'H200', default: false },
{ id: 'mi300x', label: 'MI300X', default: false },
@@ -60,6 +61,7 @@ export const Qwen3VLDeployment = () => {
h100: { tp: 8, ep: 0, bf16: true, fp8: true },
h200: { tp: 8, ep: 0, bf16: true, fp8: true },
b200: { tp: 8, ep: 0, bf16: true, fp8: true },
b300: { tp: 8, ep: 0, bf16: true, fp8: true },
mi300x: { tp: 8, ep: 0, bf16: true, fp8: true },
mi325x: { tp: 8, ep: 0, bf16: true, fp8: true },
mi355x: { tp: 8, ep: 0, bf16: true, fp8: true },
@@ -71,6 +73,7 @@ export const Qwen3VLDeployment = () => {
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
@@ -82,6 +85,7 @@ export const Qwen3VLDeployment = () => {
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
@@ -93,6 +97,7 @@ export const Qwen3VLDeployment = () => {
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
@@ -104,6 +109,7 @@ export const Qwen3VLDeployment = () => {
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
@@ -115,6 +121,7 @@ export const Qwen3VLDeployment = () => {
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
@@ -213,6 +220,11 @@ export const Qwen3VLDeployment = () => {
cmd += ' \\\n --tool-call-parser qwen';
}
if (hardware === 'b300') {
cmd += ' \\\n --attention-backend flashinfer';
cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
}
return cmd;
};
docs_new/src/snippets/autoregressive/qwen35-deployment.jsx
+23 -15
View File
@@ -8,19 +8,19 @@ export const Qwen35Deployment = () => {
// 27B, 9B, 4B, 2B, 0.8B
//
// GPU requirements (BF16):
// 397B-A17B: H100 tp=16 (2 nodes), H200 tp=8, B200 tp=8, B300 tp=4, MI300X tp=8, MI325X tp=4, MI355X tp=4
// 397B-A17B: H100 tp=16 (2 nodes), H200 tp=8, B200 tp=8, B300 tp=8, MI300X tp=8, MI325X tp=4, MI355X tp=4
// 122B-A10B: H100 tp=4, H200 tp=4, B200 tp=2, B300 tp=2, MI300X tp=2, MI325X tp=1, MI355X tp=1
// 35B-A3B: H100 tp=1 (tp=2 w/ MTP), H200 tp=1, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1
// 27B: H100 tp=1 (tp=2 w/ MTP); tp=1 on all other hardware
// 9B/4B/2B/0.8B: tp=1 on all hardware (including MI300X, MI325X, MI355X)
//
// GPU requirements (FP8, where available):
// 397B-A17B: H100 tp=8, H200 tp=8 ep=8, B200 tp=4, B300 tp=2, MI300X tp=4, MI325X tp=2, MI355X tp=2
// 397B-A17B: H100 tp=8, H200 tp=8 ep=8, B200 tp=4, B300 tp=4, MI300X tp=4, MI325X tp=2, MI355X tp=2
// 122B-A10B: H100 tp=2 (tp=4 w/ MTP), H200 tp=2, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1
// 35B-A3B: H100 tp=1, H200 tp=1, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1
// 27B: tp=1 on all hardware (including MI300X, MI325X, MI355X)
//
// FP4 (397B only): NVFP4 on Blackwell B200 tp=4, B300 tp=2; AMD MXFP4 on MI355X tp=2
// FP4 (397B only): NVFP4 on Blackwell B200/B300 tp=4; AMD MXFP4 on MI355X tp=2
const MOE_MODELS = new Set(['397b', '122b', '35b']);
const FP8_MODELS = new Set(['397b', '122b', '35b', '27b']);
@@ -149,7 +149,7 @@ export const Qwen35Deployment = () => {
h100: { bf16: { tp: 16, mem: 0.8, multinode: true, nnodes: 2 }, fp8: { tp: 8, mem: 0.8 } },
h200: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 8, ep: 8, mem: 0.8 } },
b200: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 4, mem: 0.8 }, fp4: { tp: 4, mem: 0.85 } },
b300: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 }, fp4: { tp: 2, mem: 0.8 } },
b300: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 4, mem: 0.8 }, fp4: { tp: 4, mem: 0.8 } },
mi300x: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 4, mem: 0.8 } },
mi325x: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 } },
mi355x: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 }, fp4: { tp: 2, mem: 0.8 } },
@@ -372,7 +372,7 @@ export const Qwen35Deployment = () => {
const commandRules = {
reasoning: (value) => value === 'enabled' ? '--reasoning-parser qwen3' : null,
toolcall: (value) => value === 'enabled' ? '--tool-call-parser qwen3_coder' : null,
speculative: (value) => value === 'enabled' ? '--speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4' : null,
speculative: (value) => value === 'enabled' ? '--speculative-algorithm NEXTN \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4' : null,
mambaCache: (value) => value === 'v2' ? '--mamba-scheduler-strategy extra_buffer' : null,
};
@@ -423,9 +423,12 @@ export const Qwen35Deployment = () => {
}
// Append backend configurations
if (hardware === 'b200' || hardware === 'b300') {
if (hardware === 'b200' || (hardware === 'b300' && quantization === 'fp4')) {
cmd += ` \\\n --attention-backend trtllm_mha`;
}
if (hardware === 'b300' && quantization !== 'fp4') {
cmd += ` \\\n --attention-backend flashinfer`;
}
// Append AMD GPU-specific backend configurations.
// All AMD MI GPUs use the AITER unified-attention backend (pair with
@@ -466,15 +469,20 @@ export const Qwen35Deployment = () => {
}
} else {
// NVIDIA NVFP4 on Blackwell (B200 / B300).
cmd += ' \\\n --quantization modelopt_fp4';
cmd += ' \\\n --fp4-gemm-backend flashinfer_cutlass';
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm';
cmd += ' \\\n --chunked-prefill-size 32768';
cmd += ' \\\n --max-prefill-tokens 32768';
cmd += ' \\\n --max-running-requests 128';
cmd += ' \\\n --stream-interval 30';
cmd += ' \\\n --disable-radix-cache';
if (hardware === 'b300') {
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm';
cmd += ' \\\n --fp4-gemm-backend flashinfer_cutlass';
} else {
cmd += ' \\\n --quantization modelopt_fp4';
cmd += ' \\\n --fp4-gemm-backend flashinfer_cutlass';
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm';
cmd += ' \\\n --chunked-prefill-size 32768';
cmd += ' \\\n --max-prefill-tokens 32768';
cmd += ' \\\n --max-running-requests 128';
cmd += ' \\\n --stream-interval 30';
cmd += ' \\\n --disable-radix-cache';
}
}
}
docs_new/src/snippets/autoregressive/qwen36-deployment.jsx
+6
View File
@@ -8,6 +8,7 @@ export const Qwen36Deployment = () => {
{ id: 'h100', label: 'H100', default: true },
{ id: 'h200', label: 'H200', default: false },
{ id: 'b200', label: 'B200', default: false },
{ id: 'b300', label: 'B300', default: false },
{ id: 'xeon', label: 'XEON', default: false },
],
},
@@ -85,6 +86,7 @@ export const Qwen36Deployment = () => {
h100: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
h200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
b200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
b300: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
xeon: { bf16: { tp: 3 }, fp8: { tp: 3 } },
},
'27b': {
@@ -92,6 +94,7 @@ export const Qwen36Deployment = () => {
h100: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
h200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
b200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
b300: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
xeon: { bf16: { tp: 6 }, fp8: { tp: 6 } },
},
};
@@ -192,6 +195,9 @@ export const Qwen36Deployment = () => {
if (hardware === 'b200') {
cmd += ` \\\n --attention-backend trtllm_mha`;
}
if (hardware === 'b300') {
cmd += ` \\\n --attention-backend flashinfer`;
}
if (hwConfig.mem !== undefined) {
cmd += ` \\\n --mem-fraction-static ${hwConfig.mem}`;
}
docs_new/src/snippets/autoregressive/ring-25-1t-deployment.jsx
+6 -1
View File
@@ -2,7 +2,7 @@ export const Ring251TDeployment = () => {
// Config mirrors sgl-cookbook src/components/autoregressive/Ring25ConfigGenerator/index.js.
//
// GPU requirements:
// H200 / B200 / GB200 / GB300 / MI355X: single-node (tp per platform)
// H200 / B200 / B300 / GB200 / GB300 / MI355X: single-node (tp per platform)
// MI300X / MI325X: two nodes, tp-size 8, pp-size 2 (multi-node scripts)
const options = {
hardware: {
@@ -11,6 +11,7 @@ export const Ring251TDeployment = () => {
items: [
{ id: 'h200', label: 'H200', default: true },
{ id: 'b200', label: 'B200', default: false },
{ id: 'b300', label: 'B300', default: false },
{ id: 'gb200', label: 'GB200', default: false },
{ id: 'gb300', label: 'GB300', default: false },
{ id: 'mi300x', label: 'MI300X', default: false },
@@ -39,6 +40,7 @@ export const Ring251TDeployment = () => {
const modelConfigs = {
h200: { fp8: { tp: 8 } },
b200: { fp8: { tp: 8 } },
b300: { fp8: { tp: 8 } },
gb200: { fp8: { tp: 4 } },
gb300: { fp8: { tp: 4 } },
mi300x: { fp8: { tp: 8, pp: 2, nnodes: 2 } },
@@ -156,6 +158,9 @@ export const Ring251TDeployment = () => {
cmd += ` --model-path ${modelName}`;
cmd += ` \\\n --tp ${tpValue}`;
cmd += ' \\\n --trust-remote-code';
if (hardware === 'b300') {
cmd += ' \\\n --attention-backend flashinfer';
}
extraFlags.forEach((flag) => {
cmd += ` \\\n ${flag}`;
docs_new/src/snippets/diffusion/flux-deployment.jsx
+1
View File
@@ -8,6 +8,7 @@ export const FluxDeployment = () => {
title: 'Hardware Platform',
items: [
{ id: 'b200', label: 'B200', default: true },
{ id: 'b300', label: 'B300', default: false },
{ id: 'h200', label: 'H200', default: false },
{ id: 'h100', label: 'H100', default: false },
{ id: 'mi355x', label: 'MI355X', default: false },
docs_new/src/snippets/diffusion/qwen-image-edit-deployment.jsx
+1
View File
@@ -8,6 +8,7 @@ export const QwenImageEditDeployment = () => {
title: 'Hardware Platform',
items: [
{ id: 'b200', label: 'B200', default: true },
{ id: 'b300', label: 'B300', default: false },
{ id: 'h200', label: 'H200', default: false },
{ id: 'h100', label: 'H100', default: false },
{ id: 'mi300x', label: 'MI300X', default: false },
docs_new/src/snippets/diffusion/wan22-deployment.jsx
+7 -2
View File
@@ -6,6 +6,7 @@
title: 'Hardware Platform',
items: [
{ id: 'b200', label: 'B200', default: true },
{ id: 'b300', label: 'B300', default: false },
{ id: 'h200', label: 'H200', default: false },
{ id: 'mi300x', label: 'MI300X', default: false },
{ id: 'mi325x', label: 'MI325X', default: false },
@@ -109,7 +110,7 @@
};
const generateCommand = () => {
const { task, modelsize, selectedLoraPath, bestPractice } = values;
const { hardware, task, modelsize, selectedLoraPath, bestPractice } = values;
const configKey = `${task}-${modelsize}`;
const config = modelConfigs[configKey];
if (!config) {
@@ -118,7 +119,11 @@
let command = `sglang serve \\\n --model-path ${config.repoId} \\\n --dit-layerwise-offload true`;
if (bestPractice === 'on') {
command += ` \\\n --num-gpus 4 \\\n --ulysses-degree 2 \\\n --enable-cfg-parallel`;
if (hardware === 'b300') {
command += ` \\\n --tp-size 2 \\\n --num-gpus 8 \\\n --sp-degree 2 \\\n --ulysses-degree 2 \\\n --enable-cfg-parallel`;
} else {
command += ` \\\n --num-gpus 4 \\\n --ulysses-degree 2 \\\n --enable-cfg-parallel`;
}
}
if (selectedLoraPath && selectedLoraPath !== 'none') {
command += ` \\\n --lora-path ${selectedLoraPath}`;