[docs] Add B300 cookbook deployment options (#28697)
This commit is contained in:
@@ -4,25 +4,28 @@ metatags:
|
|||||||
description: "Deploy Intern-S1 with SGLang - community contribution guide for InternLM's Intern-S1 model deployment."
|
description: "Deploy Intern-S1 with SGLang - community contribution guide for InternLM's Intern-S1 model deployment."
|
||||||
---
|
---
|
||||||
|
|
||||||
## 📝 Community Contribution Welcome
|
import { InternS1Deployment } from '/src/snippets/autoregressive/intern-s1-deployment.jsx';
|
||||||
|
|
||||||
This guide is currently under development. We welcome community contributions!
|
## 1. Model Introduction
|
||||||
|
|
||||||
If you have experience deploying **Intern-S1** with SGLang, please help us complete this documentation.
|
Intern-S1 includes the large **Intern-S1** MoE model and the smaller **Intern-S1-mini** dense model. The command generator below covers BF16 and FP8 serving on NVIDIA H100/H200/B200/B300 platforms.
|
||||||
|
|
||||||
## 🚀 How to Contribute
|
## 2. SGLang Installation
|
||||||
|
|
||||||
```shell Command
|
Refer to the [official SGLang installation guide](../../../docs/get-started/install), or install from source:
|
||||||
git clone https://github.com/YOUR_USERNAME/sglang-cookbook.git
|
|
||||||
cd sglang-cookbook
|
```bash Command
|
||||||
git checkout -b add-intern-s1-guide
|
uv pip install 'git+https://github.com/sgl-project/sglang.git#subdirectory=python'
|
||||||
# Edit this file and submit a PR
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## 📚 Reference
|
## 3. Model Deployment
|
||||||
|
|
||||||
- [GLM-4.6V](../GLM/GLM-4.6V)
|
### 3.1 Basic Configuration
|
||||||
|
|
||||||
---
|
<InternS1Deployment />
|
||||||
|
|
||||||
**Let's build this together!** 🌟
|
### 3.2 Configuration Tips
|
||||||
|
|
||||||
|
- FP8 checkpoints use the matching BF16 checkpoint as tokenizer path.
|
||||||
|
- B300 deployments use `--attention-backend flashinfer`.
|
||||||
|
- Enable `--reasoning-parser interns1` and `--tool-call-parser interns1` when your workload needs structured reasoning or tool-call parsing.
|
||||||
|
|||||||
@@ -73,6 +73,13 @@ export const DeepSeekMathV2Deployment = () => {
|
|||||||
if (memFraction) {
|
if (memFraction) {
|
||||||
cmd += ` \\\n --mem-fraction-static ${memFraction}`;
|
cmd += ` \\\n --mem-fraction-static ${memFraction}`;
|
||||||
}
|
}
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ' \\\n --attention-backend flashinfer';
|
||||||
|
if (values.dpattention !== 'enabled') {
|
||||||
|
cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
|
||||||
|
cmd += ' \\\n --cuda-graph-backend-prefill disabled';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
cmd += ' \\\n --host 0.0.0.0 \\\n --port 30000';
|
cmd += ' \\\n --host 0.0.0.0 \\\n --port 30000';
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,11 @@ const lookupData = {
|
|||||||
"label": "B200",
|
"label": "B200",
|
||||||
"default": true
|
"default": true
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"id": "b300",
|
||||||
|
"label": "B300",
|
||||||
|
"default": false
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"id": "h200",
|
"id": "h200",
|
||||||
"label": "H200",
|
"label": "H200",
|
||||||
@@ -182,6 +187,66 @@ const lookupData = {
|
|||||||
"fp8_gemm_backend": "flashinfer_trtllm"
|
"fp8_gemm_backend": "flashinfer_trtllm"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"hardware": "b300",
|
||||||
|
"quantization": "fp8",
|
||||||
|
"gpu_count": 8,
|
||||||
|
"scenario": "low-latency",
|
||||||
|
"parameters": {
|
||||||
|
"model_path": "deepseek-ai/DeepSeek-R1-0528",
|
||||||
|
"tensor_parallel_size": 8,
|
||||||
|
"kv_cache_dtype": "fp8_e4m3",
|
||||||
|
"attention_backend": "flashinfer",
|
||||||
|
"enforce_disable_flashinfer_allreduce_fusion": true,
|
||||||
|
"enable_symm_mem": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"hardware": "b300",
|
||||||
|
"quantization": "fp8",
|
||||||
|
"gpu_count": 8,
|
||||||
|
"scenario": "high-throughput",
|
||||||
|
"parameters": {
|
||||||
|
"model_path": "deepseek-ai/DeepSeek-R1-0528",
|
||||||
|
"tensor_parallel_size": 8,
|
||||||
|
"kv_cache_dtype": "fp8_e4m3",
|
||||||
|
"attention_backend": "flashinfer",
|
||||||
|
"enforce_disable_flashinfer_allreduce_fusion": true,
|
||||||
|
"enable_symm_mem": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"hardware": "b300",
|
||||||
|
"quantization": "fp4",
|
||||||
|
"gpu_count": 8,
|
||||||
|
"scenario": "low-latency",
|
||||||
|
"parameters": {
|
||||||
|
"model_path": "nvidia/DeepSeek-R1-0528-FP4-v2",
|
||||||
|
"tensor_parallel_size": 8,
|
||||||
|
"kv_cache_dtype": "fp8_e4m3",
|
||||||
|
"attention_backend": "flashinfer",
|
||||||
|
"enforce_disable_flashinfer_allreduce_fusion": true,
|
||||||
|
"moe_runner_backend": "flashinfer_cutlass",
|
||||||
|
"mem_fraction_static": 0.85,
|
||||||
|
"enable_symm_mem": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"hardware": "b300",
|
||||||
|
"quantization": "fp4",
|
||||||
|
"gpu_count": 8,
|
||||||
|
"scenario": "high-throughput",
|
||||||
|
"parameters": {
|
||||||
|
"model_path": "nvidia/DeepSeek-R1-0528-FP4-v2",
|
||||||
|
"tensor_parallel_size": 8,
|
||||||
|
"kv_cache_dtype": "fp8_e4m3",
|
||||||
|
"attention_backend": "flashinfer",
|
||||||
|
"enforce_disable_flashinfer_allreduce_fusion": true,
|
||||||
|
"moe_runner_backend": "flashinfer_cutlass",
|
||||||
|
"mem_fraction_static": 0.85,
|
||||||
|
"enable_symm_mem": true
|
||||||
|
}
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"hardware": "h200",
|
"hardware": "h200",
|
||||||
"quantization": "fp8",
|
"quantization": "fp8",
|
||||||
@@ -387,7 +452,7 @@ const lookupData = {
|
|||||||
{
|
{
|
||||||
"hardware": "h200",
|
"hardware": "h200",
|
||||||
"quantization": "fp4",
|
"quantization": "fp4",
|
||||||
"error": "FP4 is only available for B200 hardware. Please select FP8 quantization."
|
"error": "FP4 is only available for B200/B300 hardware. Please select FP8 quantization."
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
};
|
};
|
||||||
@@ -407,6 +472,7 @@ const fieldToFlag = {
|
|||||||
enable_flashinfer_allreduce_fusion: 'enable-flashinfer-allreduce-fusion',
|
enable_flashinfer_allreduce_fusion: 'enable-flashinfer-allreduce-fusion',
|
||||||
scheduler_recv_interval: 'scheduler-recv-interval',
|
scheduler_recv_interval: 'scheduler-recv-interval',
|
||||||
enable_symm_mem: 'enable-symm-mem',
|
enable_symm_mem: 'enable-symm-mem',
|
||||||
|
enforce_disable_flashinfer_allreduce_fusion: 'enforce-disable-flashinfer-allreduce-fusion',
|
||||||
disable_radix_cache: 'disable-radix-cache',
|
disable_radix_cache: 'disable-radix-cache',
|
||||||
attention_backend: 'attention-backend',
|
attention_backend: 'attention-backend',
|
||||||
moe_runner_backend: 'moe-runner-backend',
|
moe_runner_backend: 'moe-runner-backend',
|
||||||
@@ -497,7 +563,7 @@ const resolveItems = (option, values) =>
|
|||||||
title: 'Hardware Platform',
|
title: 'Hardware Platform',
|
||||||
items: uiOptions.hardware
|
items: uiOptions.hardware
|
||||||
.filter((option) =>
|
.filter((option) =>
|
||||||
['b200', 'h200', 'mi300x', 'mi325x', 'mi355x'].includes(option.id)
|
['b200', 'b300', 'h200', 'mi300x', 'mi325x', 'mi355x'].includes(option.id)
|
||||||
)
|
)
|
||||||
.map((option) => ({
|
.map((option) => ({
|
||||||
id: option.id,
|
id: option.id,
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ export const DeepSeekR1BasicDeployment = () => {
|
|||||||
{ id: 'h100', label: 'H100', default: false },
|
{ id: 'h100', label: 'H100', default: false },
|
||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'b200', label: 'B200', default: true },
|
{ id: 'b200', label: 'B200', default: true },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||||
{ id: 'mi355x', label: 'MI355X', default: false },
|
{ id: 'mi355x', label: 'MI355X', default: false },
|
||||||
@@ -138,6 +139,18 @@ export const DeepSeekR1BasicDeployment = () => {
|
|||||||
' \\\n --kv-cache-dtype fp8_e4m3 # Optional: enables fp8 kv cache and fp8 attention kernels to improve performance';
|
' \\\n --kv-cache-dtype fp8_e4m3 # Optional: enables fp8 kv cache and fp8 attention kernels to improve performance';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
command += ' \\\n --kv-cache-dtype fp8_e4m3';
|
||||||
|
command += ' \\\n --attention-backend flashinfer';
|
||||||
|
command += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
|
||||||
|
if (quantization === 'fp4') {
|
||||||
|
command += ' \\\n --moe-runner-backend flashinfer_cutlass';
|
||||||
|
}
|
||||||
|
if (quantization === 'fp4' || strategyValues.includes('mtp')) {
|
||||||
|
command += ' \\\n --mem-fraction-static 0.85';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (isXeon) {
|
if (isXeon) {
|
||||||
command += ' \\\n --device cpu \\\n --disable-overlap-schedule';
|
command += ' \\\n --device cpu \\\n --disable-overlap-schedule';
|
||||||
if (quantization === 'int8') {
|
if (quantization === 'int8') {
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ export const DeepSeekV32Deployment = () => {
|
|||||||
//
|
//
|
||||||
// Model variants:
|
// Model variants:
|
||||||
// DeepSeek-V3.2, V3.2-Exp, V3.2-Speciale → deepseek-ai/ family, TP=8
|
// DeepSeek-V3.2, V3.2-Exp, V3.2-Speciale → deepseek-ai/ family, TP=8
|
||||||
// DeepSeek-V3.2-NVFP4 → nvidia/ family, B200 only, TP=4
|
// DeepSeek-V3.2-NVFP4 → nvidia/ family, B200/B300 only, TP=4
|
||||||
// DeepSeek-V3.2-MXFP4 → amd/ family, MI300X/MI355X only, TP=8
|
// DeepSeek-V3.2-MXFP4 → amd/ family, MI300X/MI355X only, TP=8
|
||||||
const options = {
|
const options = {
|
||||||
hardware: {
|
hardware: {
|
||||||
@@ -12,6 +12,7 @@ export const DeepSeekV32Deployment = () => {
|
|||||||
items: [
|
items: [
|
||||||
{ id: 'h200', label: 'H200', default: true },
|
{ id: 'h200', label: 'H200', default: true },
|
||||||
{ id: 'b200', label: 'B200', default: false },
|
{ id: 'b200', label: 'B200', default: false },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||||
]
|
]
|
||||||
@@ -21,13 +22,13 @@ export const DeepSeekV32Deployment = () => {
|
|||||||
title: 'Model Name',
|
title: 'Model Name',
|
||||||
getDynamicItems: (values) => {
|
getDynamicItems: (values) => {
|
||||||
const hw = values.hardware;
|
const hw = values.hardware;
|
||||||
const isB200 = hw === 'b200';
|
const isBlackwell = hw === 'b200' || hw === 'b300';
|
||||||
const isAMD = hw === 'mi300x' || hw === 'mi355x';
|
const isAMD = hw === 'mi300x' || hw === 'mi355x';
|
||||||
return [
|
return [
|
||||||
{ id: 'v32', label: 'DeepSeek-V3.2', default: !isB200 && !isAMD },
|
{ id: 'v32', label: 'DeepSeek-V3.2', default: !isBlackwell && !isAMD },
|
||||||
{ id: 'v32speciale', label: 'DeepSeek-V3.2-Speciale', default: false },
|
{ id: 'v32speciale', label: 'DeepSeek-V3.2-Speciale', default: false },
|
||||||
{ id: 'v32exp', label: 'DeepSeek-V3.2-Exp', default: false },
|
{ id: 'v32exp', label: 'DeepSeek-V3.2-Exp', default: false },
|
||||||
{ id: 'v32nvfp4', label: 'DeepSeek-V3.2-NVFP4', default: isB200, disabled: !isB200, disabledReason: 'NVFP4 requires B200 (Blackwell)' },
|
{ id: 'v32nvfp4', label: 'DeepSeek-V3.2-NVFP4', default: isBlackwell, disabled: !isBlackwell, disabledReason: 'NVFP4 requires B200/B300 (Blackwell)' },
|
||||||
{ id: 'v32mxfp4', label: 'DeepSeek-V3.2-MXFP4', default: isAMD, disabled: !isAMD, disabledReason: 'MXFP4 requires AMD MI300X/MI355X' }
|
{ id: 'v32mxfp4', label: 'DeepSeek-V3.2-MXFP4', default: isAMD, disabled: !isAMD, disabledReason: 'MXFP4 requires AMD MI300X/MI355X' }
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
@@ -101,7 +102,7 @@ export const DeepSeekV32Deployment = () => {
|
|||||||
return () => observer.disconnect();
|
return () => observer.disconnect();
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
// When hardware changes, re-resolve model name defaults (NVFP4→B200, MXFP4→AMD).
|
// When hardware changes, re-resolve model name defaults (NVFP4→Blackwell, MXFP4→AMD).
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setValues(prev => {
|
setValues(prev => {
|
||||||
const next = { ...prev };
|
const next = { ...prev };
|
||||||
@@ -139,10 +140,12 @@ export const DeepSeekV32Deployment = () => {
|
|||||||
const isNvfp4 = modelname === 'v32nvfp4';
|
const isNvfp4 = modelname === 'v32nvfp4';
|
||||||
const isMxfp4 = modelname === 'v32mxfp4';
|
const isMxfp4 = modelname === 'v32mxfp4';
|
||||||
const isAMD = hardware === 'mi300x' || hardware === 'mi355x';
|
const isAMD = hardware === 'mi300x' || hardware === 'mi355x';
|
||||||
|
const isB300 = hardware === 'b300';
|
||||||
|
const isBlackwell = hardware === 'b200' || isB300;
|
||||||
|
|
||||||
// Validation: NVFP4 requires B200
|
// Validation: NVFP4 requires Blackwell
|
||||||
if (isNvfp4 && hardware !== 'b200') {
|
if (isNvfp4 && !isBlackwell) {
|
||||||
return `# Error: DeepSeek-V3.2-NVFP4 requires NVIDIA B200 (Blackwell) hardware\n# Please select "B200" for Hardware Platform or choose a different model`;
|
return `# Error: DeepSeek-V3.2-NVFP4 requires NVIDIA B200/B300 (Blackwell) hardware\n# Please select "B200" or "B300" for Hardware Platform or choose a different model`;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validation: MXFP4 requires AMD MI300X/MI355X
|
// Validation: MXFP4 requires AMD MI300X/MI355X
|
||||||
@@ -176,8 +179,16 @@ export const DeepSeekV32Deployment = () => {
|
|||||||
let cmd = 'sglang serve \\\n';
|
let cmd = 'sglang serve \\\n';
|
||||||
cmd += ` --model-path ${modelName}`;
|
cmd += ` --model-path ${modelName}`;
|
||||||
cmd += ' \\\n --tp 4';
|
cmd += ' \\\n --tp 4';
|
||||||
|
if (isB300) {
|
||||||
|
cmd += ' \\\n --attention-backend flashinfer';
|
||||||
|
cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
|
||||||
|
cmd += ' \\\n --cuda-graph-backend-prefill disabled';
|
||||||
|
cmd += ' \\\n --moe-runner-backend flashinfer_cutlass';
|
||||||
|
cmd += ' \\\n --disable-flashinfer-autotune';
|
||||||
|
} else {
|
||||||
cmd += ' \\\n --quantization modelopt_fp4';
|
cmd += ' \\\n --quantization modelopt_fp4';
|
||||||
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm';
|
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm';
|
||||||
|
}
|
||||||
return cmd;
|
return cmd;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -222,6 +233,14 @@ export const DeepSeekV32Deployment = () => {
|
|||||||
cmd += ' \\\n --speculative-num-draft-tokens 4';
|
cmd += ' \\\n --speculative-num-draft-tokens 4';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (isB300) {
|
||||||
|
cmd += ' \\\n --attention-backend flashinfer';
|
||||||
|
if (!strategyArray.includes('dp') || strategyArray.includes('ep') || strategyArray.includes('mtp')) {
|
||||||
|
cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
|
||||||
|
cmd += ' \\\n --cuda-graph-backend-prefill disabled';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Add tool-call-parser if enabled (not supported for Speciale)
|
// Add tool-call-parser if enabled (not supported for Speciale)
|
||||||
if (toolcall === 'enabled' && modelname !== 'v32speciale') {
|
if (toolcall === 'enabled' && modelname !== 'v32speciale') {
|
||||||
if (modelname === 'v32exp') {
|
if (modelname === 'v32exp') {
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ export const Gemma4Deployment = () => {
|
|||||||
return [
|
return [
|
||||||
{ id: 'h200', label: 'H200', default: true },
|
{ id: 'h200', label: 'H200', default: true },
|
||||||
{ id: 'b200', label: 'B200', default: false },
|
{ id: 'b200', label: 'B200', default: false },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false, disabled: !showMI300X },
|
{ id: 'mi300x', label: 'MI300X', default: false, disabled: !showMI300X },
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
@@ -76,6 +77,13 @@ export const Gemma4Deployment = () => {
|
|||||||
'31b': { tp: 1, mem: 0.9 },
|
'31b': { tp: 1, mem: 0.9 },
|
||||||
'26b-a4b': { tp: 1, mem: 0.9 },
|
'26b-a4b': { tp: 1, mem: 0.9 },
|
||||||
},
|
},
|
||||||
|
b300: {
|
||||||
|
e2b: { tp: 1, mem: 0.9 },
|
||||||
|
e4b: { tp: 1, mem: 0.9 },
|
||||||
|
'12b': { tp: 1, mem: 0.9 },
|
||||||
|
'31b': { tp: 1, mem: 0.9 },
|
||||||
|
'26b-a4b': { tp: 1, mem: 0.9 },
|
||||||
|
},
|
||||||
mi300x: {
|
mi300x: {
|
||||||
'31b': { tp: 1, mem: 0.80 },
|
'31b': { tp: 1, mem: 0.80 },
|
||||||
'26b-a4b': { tp: 1, mem: 0.80 },
|
'26b-a4b': { tp: 1, mem: 0.80 },
|
||||||
@@ -129,6 +137,10 @@ export const Gemma4Deployment = () => {
|
|||||||
cmd += ` \\\n --speculative-eagle-topk 1`;
|
cmd += ` \\\n --speculative-eagle-topk 1`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ` \\\n --attention-backend triton`;
|
||||||
|
}
|
||||||
|
|
||||||
cmd += ` \\\n --mem-fraction-static ${mem}`;
|
cmd += ` \\\n --mem-fraction-static ${mem}`;
|
||||||
cmd += ` \\\n --host 0.0.0.0 --port 30000`;
|
cmd += ` \\\n --host 0.0.0.0 --port 30000`;
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ export const GLM46Deployment = () => {
|
|||||||
{ id: 'h100', label: 'H100', default: true },
|
{ id: 'h100', label: 'H100', default: true },
|
||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'b200', label: 'B200', default: false },
|
{ id: 'b200', label: 'B200', default: false },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||||
@@ -132,6 +133,9 @@ export const GLM46Deployment = () => {
|
|||||||
// Strategy-specific parameters
|
// Strategy-specific parameters
|
||||||
if (strategyArray.includes('dp')) {
|
if (strategyArray.includes('dp')) {
|
||||||
cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`;
|
cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`;
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ` \\\n --cuda-graph-max-bs-decode 256`;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (strategyArray.includes('ep')) {
|
if (strategyArray.includes('ep')) {
|
||||||
cmd += ` \\\n --ep 8`;
|
cmd += ` \\\n --ep 8`;
|
||||||
@@ -151,6 +155,11 @@ export const GLM46Deployment = () => {
|
|||||||
cmd += ` \\\n --reasoning-parser glm45`;
|
cmd += ` \\\n --reasoning-parser glm45`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ` \\\n --attention-backend flashinfer`;
|
||||||
|
cmd += ` \\\n --enforce-disable-flashinfer-allreduce-fusion`;
|
||||||
|
}
|
||||||
|
|
||||||
return cmd;
|
return cmd;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ export const GLM46VDeployment = () => {
|
|||||||
title: 'Hardware Platform',
|
title: 'Hardware Platform',
|
||||||
items: [
|
items: [
|
||||||
{ id: 'b200', label: 'B200', default: true },
|
{ id: 'b200', label: 'B200', default: true },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'h100', label: 'H100', default: false },
|
{ id: 'h100', label: 'H100', default: false },
|
||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
@@ -94,6 +95,7 @@ export const GLM46VDeployment = () => {
|
|||||||
h100: { tp: 8 },
|
h100: { tp: 8 },
|
||||||
h200: { tp: 8 },
|
h200: { tp: 8 },
|
||||||
b200: { tp: 8 },
|
b200: { tp: 8 },
|
||||||
|
b300: { tp: 8 },
|
||||||
mi300x: { tp: 8 },
|
mi300x: { tp: 8 },
|
||||||
mi325x: { tp: 8 },
|
mi325x: { tp: 8 },
|
||||||
mi355x: { tp: 8 }
|
mi355x: { tp: 8 }
|
||||||
@@ -103,6 +105,7 @@ export const GLM46VDeployment = () => {
|
|||||||
h100: { tp: 1 },
|
h100: { tp: 1 },
|
||||||
h200: { tp: 1 },
|
h200: { tp: 1 },
|
||||||
b200: { tp: 1 },
|
b200: { tp: 1 },
|
||||||
|
b300: { tp: 1 },
|
||||||
mi300x: { tp: 1 },
|
mi300x: { tp: 1 },
|
||||||
mi325x: { tp: 1 },
|
mi325x: { tp: 1 },
|
||||||
mi355x: { tp: 1 }
|
mi355x: { tp: 1 }
|
||||||
@@ -122,6 +125,10 @@ export const GLM46VDeployment = () => {
|
|||||||
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||||
const modelName = `zai-org/${config.baseName}${quantSuffix}`;
|
const modelName = `zai-org/${config.baseName}${quantSuffix}`;
|
||||||
|
|
||||||
|
if (hardware === 'b300' && modelsize === '9b' && quantization === 'fp8') {
|
||||||
|
return '# Error: GLM-4.6V-Flash-FP8 is not available on B300 in this cookbook configuration\n# Please use BF16 for GLM-4.6V-Flash or select the 106B model';
|
||||||
|
}
|
||||||
|
|
||||||
let cmd = 'python -m sglang.launch_server \\\n';
|
let cmd = 'python -m sglang.launch_server \\\n';
|
||||||
cmd += ` --model ${modelName}`;
|
cmd += ` --model ${modelName}`;
|
||||||
|
|
||||||
@@ -142,6 +149,12 @@ export const GLM46VDeployment = () => {
|
|||||||
cmd += ` \\\n --tool-call-parser glm45`;
|
cmd += ` \\\n --tool-call-parser glm45`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ` \\\n --attention-backend flashinfer`;
|
||||||
|
cmd += ` \\\n --enforce-disable-flashinfer-allreduce-fusion`;
|
||||||
|
cmd += ` \\\n --cuda-graph-backend-decode disabled`;
|
||||||
|
}
|
||||||
|
|
||||||
return cmd;
|
return cmd;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ export const GLM5Deployment = () => {
|
|||||||
// Supported quantization per hardware:
|
// Supported quantization per hardware:
|
||||||
// H100 / H200 / MI300X / MI325X / MI355X → BF16 (AMD only) + FP8 (NV only)
|
// H100 / H200 / MI300X / MI325X / MI355X → BF16 (AMD only) + FP8 (NV only)
|
||||||
// B200 → NVFP4 (default), FP8, BF16
|
// B200 → NVFP4 (default), FP8, BF16
|
||||||
|
// B300 → NVFP4 (default), FP8
|
||||||
//
|
//
|
||||||
// BF16 always needs 2x GPUs compared to FP8. AMD only supports BF16.
|
// BF16 always needs 2x GPUs compared to FP8. AMD only supports BF16.
|
||||||
const options = {
|
const options = {
|
||||||
@@ -13,6 +14,7 @@ export const GLM5Deployment = () => {
|
|||||||
items: [
|
items: [
|
||||||
{ id: 'h200', label: 'H200', default: true },
|
{ id: 'h200', label: 'H200', default: true },
|
||||||
{ id: 'b200', label: 'B200', default: false },
|
{ id: 'b200', label: 'B200', default: false },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'h100', label: 'H100', default: false },
|
{ id: 'h100', label: 'H100', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X/MI325X', default: false },
|
{ id: 'mi300x', label: 'MI300X/MI325X', default: false },
|
||||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||||
@@ -24,11 +26,12 @@ export const GLM5Deployment = () => {
|
|||||||
getDynamicItems: (values) => {
|
getDynamicItems: (values) => {
|
||||||
const hw = values.hardware;
|
const hw = values.hardware;
|
||||||
const isAMD = hw === 'mi300x' || hw === 'mi355x';
|
const isAMD = hw === 'mi300x' || hw === 'mi355x';
|
||||||
const isB200 = hw === 'b200';
|
const isB300 = hw === 'b300';
|
||||||
|
const isBlackwell = hw === 'b200' || isB300;
|
||||||
return [
|
return [
|
||||||
{ id: 'bf16', label: 'BF16', subtitle: 'Full Weights', default: isAMD },
|
{ id: 'bf16', label: 'BF16', subtitle: 'Full Weights', default: isAMD, disabled: isB300, disabledReason: isB300 ? 'BF16 requires more than the validated 8-GPU B300 node' : '' },
|
||||||
{ id: 'fp8', label: 'FP8', subtitle: 'High Throughput', default: !isAMD && !isB200, disabled: isAMD, disabledReason: 'FP8 not verified on AMD' },
|
{ id: 'fp8', label: 'FP8', subtitle: 'High Throughput', default: !isAMD && !isBlackwell, disabled: isAMD, disabledReason: 'FP8 not verified on AMD' },
|
||||||
{ id: 'nvfp4', label: 'NVFP4', subtitle: 'Highest Throughput', default: isB200, disabled: !isB200, disabledReason: 'NVFP4 only on B200' }
|
{ id: 'nvfp4', label: 'NVFP4', subtitle: 'Highest Throughput', default: isBlackwell, disabled: !isBlackwell, disabledReason: 'NVFP4 only on B200/B300' }
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -75,6 +78,7 @@ export const GLM5Deployment = () => {
|
|||||||
h100: { fp8: { tp: 16, mem: 0.85 }, bf16: { tp: 32, mem: 0.85 } },
|
h100: { fp8: { tp: 16, mem: 0.85 }, bf16: { tp: 32, mem: 0.85 } },
|
||||||
h200: { fp8: { tp: 8, mem: 0.85 }, bf16: { tp: 16, mem: 0.85 } },
|
h200: { fp8: { tp: 8, mem: 0.85 }, bf16: { tp: 16, mem: 0.85 } },
|
||||||
b200: { nvfp4: { tp: 4, mem: 0.9 }, fp8: { tp: 8, mem: 0.9 }, bf16: { tp: 16, mem: 0.9 } },
|
b200: { nvfp4: { tp: 4, mem: 0.9 }, fp8: { tp: 8, mem: 0.9 }, bf16: { tp: 16, mem: 0.9 } },
|
||||||
|
b300: { nvfp4: { tp: 4, mem: 0.9 }, fp8: { tp: 8, mem: 0.9 } },
|
||||||
mi300x: { bf16: { tp: 8, mem: 0.80 } },
|
mi300x: { bf16: { tp: 8, mem: 0.80 } },
|
||||||
mi355x: { bf16: { tp: 8, mem: 0.80 } }
|
mi355x: { bf16: { tp: 8, mem: 0.80 } }
|
||||||
};
|
};
|
||||||
@@ -147,7 +151,10 @@ export const GLM5Deployment = () => {
|
|||||||
modelName = `zai-org/GLM-5${suffix}`;
|
modelName = `zai-org/GLM-5${suffix}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
const hwConfig = modelConfigs[hardware][effectiveQuant];
|
const hwConfig = modelConfigs[hardware]?.[effectiveQuant];
|
||||||
|
if (!hwConfig) {
|
||||||
|
return '# Please select a valid hardware and quantization combination';
|
||||||
|
}
|
||||||
const tpValue = hwConfig.tp;
|
const tpValue = hwConfig.tp;
|
||||||
const memFraction = hwConfig.mem;
|
const memFraction = hwConfig.mem;
|
||||||
|
|
||||||
@@ -155,9 +162,16 @@ export const GLM5Deployment = () => {
|
|||||||
cmd += ` --model-path ${modelName}`;
|
cmd += ` --model-path ${modelName}`;
|
||||||
cmd += ` \\\n --tp ${tpValue}`;
|
cmd += ` \\\n --tp ${tpValue}`;
|
||||||
|
|
||||||
// NVFP4 B200: trtllm DSA backends, flashinfer fusion, FP8 KV cache.
|
|
||||||
if (isNVFP4) {
|
if (isNVFP4) {
|
||||||
cmd += ' \\\n --trust-remote-code';
|
cmd += ' \\\n --trust-remote-code';
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ' \\\n --attention-backend flashinfer';
|
||||||
|
cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
|
||||||
|
cmd += ' \\\n --cuda-graph-backend-prefill disabled';
|
||||||
|
cmd += ' \\\n --moe-runner-backend flashinfer_cutlass';
|
||||||
|
cmd += ' \\\n --cuda-graph-backend-decode disabled';
|
||||||
|
cmd += ' \\\n --disable-flashinfer-autotune';
|
||||||
|
} else {
|
||||||
cmd += ' \\\n --quantization modelopt_fp4';
|
cmd += ' \\\n --quantization modelopt_fp4';
|
||||||
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
|
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
|
||||||
cmd += ' \\\n --dsa-decode-backend trtllm';
|
cmd += ' \\\n --dsa-decode-backend trtllm';
|
||||||
@@ -168,6 +182,7 @@ export const GLM5Deployment = () => {
|
|||||||
cmd += ' \\\n --disable-radix-cache';
|
cmd += ' \\\n --disable-radix-cache';
|
||||||
cmd += ' \\\n --max-prefill-tokens 32768';
|
cmd += ' \\\n --max-prefill-tokens 32768';
|
||||||
cmd += ' \\\n --chunked-prefill-size 32768';
|
cmd += ' \\\n --chunked-prefill-size 32768';
|
||||||
|
}
|
||||||
cmd += ` \\\n --mem-fraction-static ${memFraction}`;
|
cmd += ` \\\n --mem-fraction-static ${memFraction}`;
|
||||||
cmd += ' \\\n --scheduler-recv-interval 10';
|
cmd += ' \\\n --scheduler-recv-interval 10';
|
||||||
cmd += ' \\\n --tokenizer-worker-num 6';
|
cmd += ' \\\n --tokenizer-worker-num 6';
|
||||||
@@ -185,6 +200,9 @@ export const GLM5Deployment = () => {
|
|||||||
|
|
||||||
if (values.dpattention === 'enabled') {
|
if (values.dpattention === 'enabled') {
|
||||||
cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`;
|
cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`;
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ' \\\n --cuda-graph-max-bs-decode 256';
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (values.reasoning === 'enabled') cmd += ' \\\n --reasoning-parser glm45';
|
if (values.reasoning === 'enabled') cmd += ' \\\n --reasoning-parser glm45';
|
||||||
if (values.toolcall === 'enabled') cmd += ' \\\n --tool-call-parser glm47';
|
if (values.toolcall === 'enabled') cmd += ' \\\n --tool-call-parser glm47';
|
||||||
@@ -206,6 +224,12 @@ export const GLM5Deployment = () => {
|
|||||||
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
|
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (hardware === 'b300' && effectiveQuant === 'fp8') {
|
||||||
|
cmd += ' \\\n --attention-backend flashinfer';
|
||||||
|
cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
|
||||||
|
cmd += ' \\\n --cuda-graph-backend-prefill disabled';
|
||||||
|
}
|
||||||
|
|
||||||
// H200 FP8: flashinfer allreduce fusion.
|
// H200 FP8: flashinfer allreduce fusion.
|
||||||
if (hardware === 'h200' && effectiveQuant === 'fp8') {
|
if (hardware === 'h200' && effectiveQuant === 'fp8') {
|
||||||
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
|
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ export const GPTOSSDeployment = () => {
|
|||||||
title: 'Hardware Platform',
|
title: 'Hardware Platform',
|
||||||
items: [
|
items: [
|
||||||
{ id: 'b200', label: 'B200', default: true },
|
{ id: 'b200', label: 'B200', default: true },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'h100', label: 'H100', default: false },
|
{ id: 'h100', label: 'H100', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
@@ -102,6 +103,7 @@ export const GPTOSSDeployment = () => {
|
|||||||
h100: { tp: 8 },
|
h100: { tp: 8 },
|
||||||
h200: { tp: 8 },
|
h200: { tp: 8 },
|
||||||
b200: { tp: 8 },
|
b200: { tp: 8 },
|
||||||
|
b300: { tp: 8 },
|
||||||
mi300x: { tp: 8 },
|
mi300x: { tp: 8 },
|
||||||
mi325x: { tp: 8 },
|
mi325x: { tp: 8 },
|
||||||
mi355x: { tp: 8 }
|
mi355x: { tp: 8 }
|
||||||
@@ -111,6 +113,7 @@ export const GPTOSSDeployment = () => {
|
|||||||
h100: { tp: 1 },
|
h100: { tp: 1 },
|
||||||
h200: { tp: 1 },
|
h200: { tp: 1 },
|
||||||
b200: { tp: 1 },
|
b200: { tp: 1 },
|
||||||
|
b300: { tp: 1 },
|
||||||
mi300x: { tp: 1 },
|
mi300x: { tp: 1 },
|
||||||
mi325x: { tp: 1 },
|
mi325x: { tp: 1 },
|
||||||
mi355x: { tp: 1 }
|
mi355x: { tp: 1 }
|
||||||
@@ -170,6 +173,12 @@ export const GPTOSSDeployment = () => {
|
|||||||
cmd += ` \\\n --tool-call-parser gpt-oss`;
|
cmd += ` \\\n --tool-call-parser gpt-oss`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ` \\\n --attention-backend triton`;
|
||||||
|
cmd += ` \\\n --moe-runner-backend triton`;
|
||||||
|
cmd += ` \\\n --enforce-disable-flashinfer-allreduce-fusion`;
|
||||||
|
}
|
||||||
|
|
||||||
// Add speculative decoding if enabled (MI30x handled above)
|
// Add speculative decoding if enabled (MI30x handled above)
|
||||||
if (speculative === 'enabled') {
|
if (speculative === 'enabled') {
|
||||||
cmd += ` \\\n --speculative-algorithm EAGLE3 \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
|
cmd += ` \\\n --speculative-algorithm EAGLE3 \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
|
||||||
|
|||||||
@@ -0,0 +1,167 @@
|
|||||||
|
export const InternS1Deployment = () => {
|
||||||
|
const options = {
|
||||||
|
hardware: {
|
||||||
|
name: 'hardware',
|
||||||
|
title: 'Hardware Platform',
|
||||||
|
items: [
|
||||||
|
{ id: 'b200', label: 'B200', default: true },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
|
{ id: 'h100', label: 'H100', default: false },
|
||||||
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
modelsize: {
|
||||||
|
name: 'modelsize',
|
||||||
|
title: 'Model Size',
|
||||||
|
items: [
|
||||||
|
{ id: 'S1', label: '235B', subtitle: 'MoE', default: true },
|
||||||
|
{ id: 'S1-mini', label: '8B', subtitle: 'Dense', default: false },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
quantization: {
|
||||||
|
name: 'quantization',
|
||||||
|
title: 'Quantization',
|
||||||
|
items: [
|
||||||
|
{ id: 'bf16', label: 'BF16', default: true },
|
||||||
|
{ id: 'fp8', label: 'FP8', default: false },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
reasoning: {
|
||||||
|
name: 'reasoning',
|
||||||
|
title: 'Reasoning Parser',
|
||||||
|
items: [
|
||||||
|
{ id: 'disabled', label: 'Disabled', default: true },
|
||||||
|
{ id: 'enabled', label: 'Enabled', default: false },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
toolcall: {
|
||||||
|
name: 'toolcall',
|
||||||
|
title: 'Tool Call Parser',
|
||||||
|
items: [
|
||||||
|
{ id: 'disabled', label: 'Disabled', default: true },
|
||||||
|
{ id: 'enabled', label: 'Enabled', default: false },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const modelConfigs = {
|
||||||
|
S1: {
|
||||||
|
baseName: 'S1',
|
||||||
|
h100: { bf16: { tp: 8 }, fp8: { tp: 8, ep: 2 } },
|
||||||
|
h200: { bf16: { tp: 8 }, fp8: { tp: 8, ep: 2 } },
|
||||||
|
b200: { bf16: { tp: 8 }, fp8: { tp: 8, ep: 2 } },
|
||||||
|
b300: { bf16: { tp: 8 }, fp8: { tp: 8, ep: 2 } },
|
||||||
|
},
|
||||||
|
'S1-mini': {
|
||||||
|
baseName: 'S1-mini',
|
||||||
|
h100: { bf16: { tp: 1 }, fp8: { tp: 1 } },
|
||||||
|
h200: { bf16: { tp: 1 }, fp8: { tp: 1 } },
|
||||||
|
b200: { bf16: { tp: 1 }, fp8: { tp: 1 } },
|
||||||
|
b300: { bf16: { tp: 1 }, fp8: { tp: 1 } },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const getInitialState = () => {
|
||||||
|
const initialState = {};
|
||||||
|
Object.entries(options).forEach(([key, option]) => {
|
||||||
|
const defaultItem = option.items.find((item) => item.default);
|
||||||
|
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||||
|
});
|
||||||
|
return initialState;
|
||||||
|
};
|
||||||
|
|
||||||
|
const [values, setValues] = useState(getInitialState);
|
||||||
|
const [isDark, setIsDark] = useState(false);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const checkDarkMode = () => {
|
||||||
|
const html = document.documentElement;
|
||||||
|
const isDarkMode =
|
||||||
|
html.classList.contains('dark') ||
|
||||||
|
html.getAttribute('data-theme') === 'dark' ||
|
||||||
|
html.style.colorScheme === 'dark';
|
||||||
|
setIsDark(isDarkMode);
|
||||||
|
};
|
||||||
|
checkDarkMode();
|
||||||
|
const observer = new MutationObserver(checkDarkMode);
|
||||||
|
observer.observe(document.documentElement, {
|
||||||
|
attributes: true,
|
||||||
|
attributeFilter: ['class', 'data-theme', 'style'],
|
||||||
|
});
|
||||||
|
return () => observer.disconnect();
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const handleRadioChange = (optionName, value) => {
|
||||||
|
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||||
|
};
|
||||||
|
|
||||||
|
const generateCommand = () => {
|
||||||
|
const { hardware, modelsize, quantization, reasoning, toolcall } = values;
|
||||||
|
const modelConfig = modelConfigs[modelsize];
|
||||||
|
const hwConfig = modelConfig?.[hardware]?.[quantization];
|
||||||
|
if (!hwConfig) {
|
||||||
|
return '# Please select a valid hardware and quantization combination';
|
||||||
|
}
|
||||||
|
|
||||||
|
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||||
|
const modelName = `internlm/Intern-${modelConfig.baseName}${quantSuffix}`;
|
||||||
|
|
||||||
|
const flags = [];
|
||||||
|
flags.push(` --model ${modelName}`);
|
||||||
|
if (hwConfig.tp > 1) flags.push(` --tp ${hwConfig.tp}`);
|
||||||
|
if (hwConfig.ep) flags.push(` --ep ${hwConfig.ep}`);
|
||||||
|
if (quantization === 'fp8') flags.push(` --tokenizer-path internlm/Intern-${modelConfig.baseName}`);
|
||||||
|
if (reasoning === 'enabled') flags.push(' --reasoning-parser interns1');
|
||||||
|
if (toolcall === 'enabled') flags.push(' --tool-call-parser interns1');
|
||||||
|
flags.push(' --trust-remote-code');
|
||||||
|
if (hardware === 'b300') flags.push(' --attention-backend flashinfer');
|
||||||
|
|
||||||
|
return `python -m sglang.launch_server \\\n${flags.join(' \\\n')}`;
|
||||||
|
};
|
||||||
|
|
||||||
|
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||||
|
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||||
|
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||||
|
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||||
|
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||||
|
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||||
|
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||||
|
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div style={containerStyle} className="not-prose">
|
||||||
|
{Object.entries(options).map(([key, option]) => (
|
||||||
|
<div key={key} style={cardStyle}>
|
||||||
|
<div style={titleStyle}>{option.title}</div>
|
||||||
|
<div style={itemsStyle}>
|
||||||
|
{option.items.map((item) => {
|
||||||
|
const isChecked = values[option.name] === item.id;
|
||||||
|
return (
|
||||||
|
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||||
|
<input
|
||||||
|
type="radio"
|
||||||
|
name={option.name}
|
||||||
|
value={item.id}
|
||||||
|
checked={isChecked}
|
||||||
|
onChange={() => handleRadioChange(option.name, item.id)}
|
||||||
|
style={{ display: 'none' }}
|
||||||
|
/>
|
||||||
|
{item.label}
|
||||||
|
{item.subtitle && (
|
||||||
|
<small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>
|
||||||
|
{item.subtitle}
|
||||||
|
</small>
|
||||||
|
)}
|
||||||
|
</label>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
<div style={cardStyle}>
|
||||||
|
<div style={titleStyle}>Run this Command:</div>
|
||||||
|
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -8,6 +8,7 @@ export const KimiK2Deployment = () => {
|
|||||||
items: [
|
items: [
|
||||||
{ id: 'h200', label: 'H200', default: true },
|
{ id: 'h200', label: 'H200', default: true },
|
||||||
{ id: 'b200', label: 'B200', default: false },
|
{ id: 'b200', label: 'B200', default: false },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||||
@@ -82,6 +83,15 @@ export const KimiK2Deployment = () => {
|
|||||||
|
|
||||||
cmd += ` \\\n --trust-remote-code`;
|
cmd += ` \\\n --trust-remote-code`;
|
||||||
|
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ` \\\n --attention-backend flashinfer`;
|
||||||
|
if (strategyArray.includes('dp')) {
|
||||||
|
cmd += ` \\\n --prefill-attention-backend triton`;
|
||||||
|
}
|
||||||
|
cmd += ` \\\n --enforce-disable-flashinfer-allreduce-fusion`;
|
||||||
|
cmd += ` \\\n --mem-fraction-static 0.85`;
|
||||||
|
}
|
||||||
|
|
||||||
if (toolcall === 'enabled') {
|
if (toolcall === 'enabled') {
|
||||||
cmd += ` \\\n --tool-call-parser kimi_k2`;
|
cmd += ` \\\n --tool-call-parser kimi_k2`;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ export const LLaDA21Deployment = () => {
|
|||||||
{ id: 'h100', label: 'H100', default: true },
|
{ id: 'h100', label: 'H100', default: true },
|
||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'b200', label: 'B200', default: false },
|
{ id: 'b200', label: 'B200', default: false },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||||
@@ -34,7 +35,7 @@ export const LLaDA21Deployment = () => {
|
|||||||
if (modelsize === 'mini') {
|
if (modelsize === 'mini') {
|
||||||
tpSize = 1;
|
tpSize = 1;
|
||||||
} else {
|
} else {
|
||||||
if (hardware === 'b200') {
|
if (hardware === 'b200' || hardware === 'b300') {
|
||||||
tpSize = 2;
|
tpSize = 2;
|
||||||
} else {
|
} else {
|
||||||
tpSize = 4;
|
tpSize = 4;
|
||||||
@@ -48,7 +49,7 @@ export const LLaDA21Deployment = () => {
|
|||||||
args.push(`--trust-remote-code`);
|
args.push(`--trust-remote-code`);
|
||||||
args.push(`--mem-fraction-static 0.8`);
|
args.push(`--mem-fraction-static 0.8`);
|
||||||
args.push(`--max-running-requests 1`);
|
args.push(`--max-running-requests 1`);
|
||||||
if (hardware === 'h100' || hardware === 'h200' || hardware === 'b200') {
|
if (hardware === 'h100' || hardware === 'h200' || hardware === 'b200' || hardware === 'b300') {
|
||||||
args.push(`--attention-backend flashinfer`);
|
args.push(`--attention-backend flashinfer`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ export const MiniMaxM27Deployment = () => {
|
|||||||
items: [
|
items: [
|
||||||
{ id: 'h200', label: 'H200', default: true },
|
{ id: 'h200', label: 'H200', default: true },
|
||||||
{ id: 'b200', label: 'B200', default: false },
|
{ id: 'b200', label: 'B200', default: false },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'gb300', label: 'GB300', default: false },
|
{ id: 'gb300', label: 'GB300', default: false },
|
||||||
{ id: 'a100', label: 'A100', default: false },
|
{ id: 'a100', label: 'A100', default: false },
|
||||||
{ id: 'h100', label: 'H100', default: false },
|
{ id: 'h100', label: 'H100', default: false },
|
||||||
@@ -23,6 +24,7 @@ export const MiniMaxM27Deployment = () => {
|
|||||||
getDynamicItems: (values) => {
|
getDynamicItems: (values) => {
|
||||||
const hw = values.hardware;
|
const hw = values.hardware;
|
||||||
const isAMD = hw === 'mi300x' || hw === 'mi325x' || hw === 'mi355x';
|
const isAMD = hw === 'mi300x' || hw === 'mi325x' || hw === 'mi355x';
|
||||||
|
const isB300 = hw === 'b300';
|
||||||
const isGB300 = hw === 'gb300';
|
const isGB300 = hw === 'gb300';
|
||||||
const isXeon = hw === 'xeon';
|
const isXeon = hw === 'xeon';
|
||||||
if (isXeon) {
|
if (isXeon) {
|
||||||
@@ -33,8 +35,8 @@ export const MiniMaxM27Deployment = () => {
|
|||||||
const canUse2GPU = isAMD || isGB300;
|
const canUse2GPU = isAMD || isGB300;
|
||||||
return [
|
return [
|
||||||
{ id: '2gpu', label: '2', default: canUse2GPU, disabled: !canUse2GPU },
|
{ id: '2gpu', label: '2', default: canUse2GPU, disabled: !canUse2GPU },
|
||||||
{ id: '4gpu', label: '4', default: !canUse2GPU, disabled: false },
|
{ id: '4gpu', label: '4', default: !canUse2GPU || isB300, disabled: false },
|
||||||
{ id: '8gpu', label: '8', default: false, disabled: isGB300 }
|
{ id: '8gpu', label: '8', default: false, disabled: isGB300 || isB300 }
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -43,11 +45,11 @@ export const MiniMaxM27Deployment = () => {
|
|||||||
title: 'Precision',
|
title: 'Precision',
|
||||||
getDynamicItems: (values) => {
|
getDynamicItems: (values) => {
|
||||||
const hw = values.hardware;
|
const hw = values.hardware;
|
||||||
const isBlackwell = hw === 'b200' || hw === 'gb300';
|
const isBlackwell = hw === 'b200' || hw === 'b300' || hw === 'gb300';
|
||||||
return [
|
return [
|
||||||
{ id: 'fp8', label: 'FP8', default: true, disabled: false },
|
{ id: 'fp8', label: 'FP8', default: true, disabled: false },
|
||||||
{ id: 'fp4', label: 'FP4', default: false, disabled: !isBlackwell,
|
{ id: 'fp4', label: 'FP4', default: false, disabled: !isBlackwell,
|
||||||
disabledReason: 'NVFP4 requires Blackwell (B200/GB300)' }
|
disabledReason: 'NVFP4 requires Blackwell (B200/B300/GB300)' }
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -131,6 +133,7 @@ export const MiniMaxM27Deployment = () => {
|
|||||||
const { hardware, gpuCount, precision, thinking, toolcall } = values;
|
const { hardware, gpuCount, precision, thinking, toolcall } = values;
|
||||||
|
|
||||||
const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x';
|
const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x';
|
||||||
|
const isB300 = hardware === 'b300';
|
||||||
const isGB300 = hardware === 'gb300';
|
const isGB300 = hardware === 'gb300';
|
||||||
const isXeon = hardware === 'xeon';
|
const isXeon = hardware === 'xeon';
|
||||||
const canUse2GPU = isAMD || isGB300;
|
const canUse2GPU = isAMD || isGB300;
|
||||||
@@ -139,11 +142,11 @@ export const MiniMaxM27Deployment = () => {
|
|||||||
return '# Please select compatible hardware\n# 2-GPU requires AMD MI300X/MI325X/MI355X or GB300';
|
return '# Please select compatible hardware\n# 2-GPU requires AMD MI300X/MI325X/MI355X or GB300';
|
||||||
}
|
}
|
||||||
|
|
||||||
const isBlackwell = hardware === 'b200' || hardware === 'gb300';
|
const isBlackwell = hardware === 'b200' || hardware === 'b300' || hardware === 'gb300';
|
||||||
const isFp4 = precision === 'fp4';
|
const isFp4 = precision === 'fp4';
|
||||||
|
|
||||||
if (isFp4 && !isBlackwell) {
|
if (isFp4 && !isBlackwell) {
|
||||||
return '# NVFP4 requires Blackwell hardware (B200 or GB300)';
|
return '# NVFP4 requires Blackwell hardware (B200, B300, or GB300)';
|
||||||
}
|
}
|
||||||
|
|
||||||
const modelName = isFp4 ? 'nvidia/MiniMax-M2.7-NVFP4' : 'MiniMaxAI/MiniMax-M2.7';
|
const modelName = isFp4 ? 'nvidia/MiniMax-M2.7-NVFP4' : 'MiniMaxAI/MiniMax-M2.7';
|
||||||
@@ -184,6 +187,9 @@ export const MiniMaxM27Deployment = () => {
|
|||||||
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
|
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
|
||||||
cmd += ' \\\n --attention-backend triton';
|
cmd += ' \\\n --attention-backend triton';
|
||||||
}
|
}
|
||||||
|
if (isB300) {
|
||||||
|
cmd += ' \\\n --attention-backend flashinfer';
|
||||||
|
}
|
||||||
|
|
||||||
if (isBlackwell) {
|
if (isBlackwell) {
|
||||||
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm_routed';
|
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm_routed';
|
||||||
|
|||||||
@@ -82,6 +82,10 @@ export const MistralSmall4Deployment = () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ` \\\n --attention-backend flashinfer`;
|
||||||
|
}
|
||||||
|
|
||||||
return cmd;
|
return cmd;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ export const Nemotron3NanoDeployment = () => {
|
|||||||
title: 'Hardware Platform',
|
title: 'Hardware Platform',
|
||||||
items: [
|
items: [
|
||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'b200', label: 'B200', default: true }
|
{ id: 'b200', label: 'B200', default: true },
|
||||||
|
{ id: 'b300', label: 'B300', default: false }
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
modelVariant: {
|
modelVariant: {
|
||||||
@@ -71,6 +72,9 @@ export const Nemotron3NanoDeployment = () => {
|
|||||||
cmd += ` --trust-remote-code \\\n`;
|
cmd += ` --trust-remote-code \\\n`;
|
||||||
cmd += ` --tp ${tp} \\\n`;
|
cmd += ` --tp ${tp} \\\n`;
|
||||||
cmd += ` --kv-cache-dtype ${kvcache} \\\n`;
|
cmd += ` --kv-cache-dtype ${kvcache} \\\n`;
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ` --attention-backend flashinfer \\\n`;
|
||||||
|
}
|
||||||
|
|
||||||
// Add thinking parser and tool call parser if enabled
|
// Add thinking parser and tool call parser if enabled
|
||||||
for (const [key, option] of Object.entries(options)) {
|
for (const [key, option] of Object.entries(options)) {
|
||||||
|
|||||||
@@ -20,7 +20,8 @@ export const Nemotron3SuperDeployment = () => {
|
|||||||
title: 'Hardware Platform',
|
title: 'Hardware Platform',
|
||||||
items: [
|
items: [
|
||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'b200', label: 'B200', default: true }
|
{ id: 'b200', label: 'B200', default: true },
|
||||||
|
{ id: 'b300', label: 'B300', default: false }
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
tp: {
|
tp: {
|
||||||
@@ -87,6 +88,9 @@ export const Nemotron3SuperDeployment = () => {
|
|||||||
if (kvcache && kvcache !== 'none') {
|
if (kvcache && kvcache !== 'none') {
|
||||||
cmd += ` --kv-cache-dtype ${kvcache} \\\n`;
|
cmd += ` --kv-cache-dtype ${kvcache} \\\n`;
|
||||||
}
|
}
|
||||||
|
if (values.hardware === 'b300') {
|
||||||
|
cmd += ` --attention-backend flashinfer \\\n`;
|
||||||
|
}
|
||||||
|
|
||||||
for (const [key, option] of Object.entries(options)) {
|
for (const [key, option] of Object.entries(options)) {
|
||||||
if (option.commandRule) {
|
if (option.commandRule) {
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ export const Qwen3CoderNextDeployment = () => {
|
|||||||
{ id: 'h200', label: 'H200', default: true },
|
{ id: 'h200', label: 'H200', default: true },
|
||||||
{ id: 'h100', label: 'H100', default: false },
|
{ id: 'h100', label: 'H100', default: false },
|
||||||
{ id: 'b200', label: 'B200', default: false },
|
{ id: 'b200', label: 'B200', default: false },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||||
{ id: 'mi355x', label: 'MI355X', default: false },
|
{ id: 'mi355x', label: 'MI355X', default: false },
|
||||||
@@ -48,6 +49,7 @@ export const Qwen3CoderNextDeployment = () => {
|
|||||||
h100: { bf16: { tp: 4 }, fp8: { tp: 2 } },
|
h100: { bf16: { tp: 4 }, fp8: { tp: 2 } },
|
||||||
h200: { bf16: { tp: 2 }, fp8: { tp: 1 } },
|
h200: { bf16: { tp: 2 }, fp8: { tp: 1 } },
|
||||||
b200: { bf16: { tp: 2 }, fp8: { tp: 1 } },
|
b200: { bf16: { tp: 2 }, fp8: { tp: 1 } },
|
||||||
|
b300: { bf16: { tp: 2 }, fp8: { tp: 1 } },
|
||||||
mi300x: { bf16: { tp: 2 }, fp8: { tp: 1 } },
|
mi300x: { bf16: { tp: 2 }, fp8: { tp: 1 } },
|
||||||
mi325x: { bf16: { tp: 2 }, fp8: { tp: 1 } },
|
mi325x: { bf16: { tp: 2 }, fp8: { tp: 1 } },
|
||||||
mi355x: { bf16: { tp: 2 }, fp8: { tp: 1 } },
|
mi355x: { bf16: { tp: 2 }, fp8: { tp: 1 } },
|
||||||
@@ -99,6 +101,9 @@ export const Qwen3CoderNextDeployment = () => {
|
|||||||
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||||
cmd += ` \\\n --attention-backend triton`;
|
cmd += ` \\\n --attention-backend triton`;
|
||||||
}
|
}
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ` \\\n --attention-backend flashinfer`;
|
||||||
|
}
|
||||||
|
|
||||||
return cmd;
|
return cmd;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ export const Qwen3Deployment = () => {
|
|||||||
h100: { tp: 8, ep: 0, bf16: true, fp8: true },
|
h100: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||||
h200: { tp: 8, ep: 0, bf16: true, fp8: true },
|
h200: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||||
b200: { tp: 8, ep: 0, bf16: true, fp8: true },
|
b200: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||||
|
b300: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||||
mi300x: { tp: 4, ep: 0, bf16: true, fp8: true },
|
mi300x: { tp: 4, ep: 0, bf16: true, fp8: true },
|
||||||
mi325x: { tp: 4, ep: 0, bf16: true, fp8: true },
|
mi325x: { tp: 4, ep: 0, bf16: true, fp8: true },
|
||||||
mi355x: { tp: 4, ep: 0, bf16: true, fp8: true },
|
mi355x: { tp: 4, ep: 0, bf16: true, fp8: true },
|
||||||
@@ -18,6 +19,7 @@ export const Qwen3Deployment = () => {
|
|||||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
|
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
@@ -29,6 +31,7 @@ export const Qwen3Deployment = () => {
|
|||||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
|
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
@@ -40,6 +43,7 @@ export const Qwen3Deployment = () => {
|
|||||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
|
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
@@ -51,6 +55,7 @@ export const Qwen3Deployment = () => {
|
|||||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
|
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
@@ -62,6 +67,7 @@ export const Qwen3Deployment = () => {
|
|||||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
|
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
@@ -73,6 +79,7 @@ export const Qwen3Deployment = () => {
|
|||||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
|
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
@@ -84,6 +91,7 @@ export const Qwen3Deployment = () => {
|
|||||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
|
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
@@ -98,6 +106,7 @@ export const Qwen3Deployment = () => {
|
|||||||
title: 'Hardware Platform',
|
title: 'Hardware Platform',
|
||||||
items: [
|
items: [
|
||||||
{ id: 'b200', label: 'B200', default: true },
|
{ id: 'b200', label: 'B200', default: true },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'h100', label: 'H100', default: false },
|
{ id: 'h100', label: 'H100', default: false },
|
||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
@@ -297,6 +306,11 @@ export const Qwen3Deployment = () => {
|
|||||||
cmd += ' \\\n --tool-call-parser qwen25';
|
cmd += ' \\\n --tool-call-parser qwen25';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ' \\\n --attention-backend flashinfer';
|
||||||
|
cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
|
||||||
|
}
|
||||||
|
|
||||||
return cmd;
|
return cmd;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ export const Qwen3NextDeployment = () => {
|
|||||||
title: 'Hardware Platform',
|
title: 'Hardware Platform',
|
||||||
items: [
|
items: [
|
||||||
{ id: 'b200', label: 'B200', default: true },
|
{ id: 'b200', label: 'B200', default: true },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'h100', label: 'H100', default: false },
|
{ id: 'h100', label: 'H100', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
@@ -75,6 +76,7 @@ export const Qwen3NextDeployment = () => {
|
|||||||
h100: { tp: 4, ep: 0, bf16: true, fp8: true },
|
h100: { tp: 4, ep: 0, bf16: true, fp8: true },
|
||||||
h200: { tp: 2, ep: 0, bf16: true, fp8: true },
|
h200: { tp: 2, ep: 0, bf16: true, fp8: true },
|
||||||
b200: { tp: 2, ep: 0, bf16: true, fp8: true },
|
b200: { tp: 2, ep: 0, bf16: true, fp8: true },
|
||||||
|
b300: { tp: 2, ep: 0, bf16: true, fp8: true },
|
||||||
mi300x: { tp: 2, ep: 0, bf16: true, fp8: true },
|
mi300x: { tp: 2, ep: 0, bf16: true, fp8: true },
|
||||||
mi325x: { tp: 2, ep: 0, bf16: true, fp8: true },
|
mi325x: { tp: 2, ep: 0, bf16: true, fp8: true },
|
||||||
mi355x: { tp: 2, ep: 0, bf16: true, fp8: true },
|
mi355x: { tp: 2, ep: 0, bf16: true, fp8: true },
|
||||||
@@ -136,6 +138,10 @@ export const Qwen3NextDeployment = () => {
|
|||||||
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||||
cmd += ` \\\n --attention-backend triton`;
|
cmd += ` \\\n --attention-backend triton`;
|
||||||
}
|
}
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ` \\\n --attention-backend flashinfer`;
|
||||||
|
cmd += ` \\\n --enforce-disable-flashinfer-allreduce-fusion`;
|
||||||
|
}
|
||||||
|
|
||||||
return cmd;
|
return cmd;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ export const Qwen3VLDeployment = () => {
|
|||||||
title: 'Hardware Platform',
|
title: 'Hardware Platform',
|
||||||
items: [
|
items: [
|
||||||
{ id: 'b200', label: 'B200', default: true },
|
{ id: 'b200', label: 'B200', default: true },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'h100', label: 'H100', default: false },
|
{ id: 'h100', label: 'H100', default: false },
|
||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
@@ -60,6 +61,7 @@ export const Qwen3VLDeployment = () => {
|
|||||||
h100: { tp: 8, ep: 0, bf16: true, fp8: true },
|
h100: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||||
h200: { tp: 8, ep: 0, bf16: true, fp8: true },
|
h200: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||||
b200: { tp: 8, ep: 0, bf16: true, fp8: true },
|
b200: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||||
|
b300: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||||
mi300x: { tp: 8, ep: 0, bf16: true, fp8: true },
|
mi300x: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||||
mi325x: { tp: 8, ep: 0, bf16: true, fp8: true },
|
mi325x: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||||
mi355x: { tp: 8, ep: 0, bf16: true, fp8: true },
|
mi355x: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||||
@@ -71,6 +73,7 @@ export const Qwen3VLDeployment = () => {
|
|||||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
|
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
@@ -82,6 +85,7 @@ export const Qwen3VLDeployment = () => {
|
|||||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
|
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
@@ -93,6 +97,7 @@ export const Qwen3VLDeployment = () => {
|
|||||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
|
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
@@ -104,6 +109,7 @@ export const Qwen3VLDeployment = () => {
|
|||||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
|
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
@@ -115,6 +121,7 @@ export const Qwen3VLDeployment = () => {
|
|||||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
|
b300: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||||
@@ -213,6 +220,11 @@ export const Qwen3VLDeployment = () => {
|
|||||||
cmd += ' \\\n --tool-call-parser qwen';
|
cmd += ' \\\n --tool-call-parser qwen';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ' \\\n --attention-backend flashinfer';
|
||||||
|
cmd += ' \\\n --enforce-disable-flashinfer-allreduce-fusion';
|
||||||
|
}
|
||||||
|
|
||||||
return cmd;
|
return cmd;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -8,19 +8,19 @@ export const Qwen35Deployment = () => {
|
|||||||
// 27B, 9B, 4B, 2B, 0.8B
|
// 27B, 9B, 4B, 2B, 0.8B
|
||||||
//
|
//
|
||||||
// GPU requirements (BF16):
|
// GPU requirements (BF16):
|
||||||
// 397B-A17B: H100 tp=16 (2 nodes), H200 tp=8, B200 tp=8, B300 tp=4, MI300X tp=8, MI325X tp=4, MI355X tp=4
|
// 397B-A17B: H100 tp=16 (2 nodes), H200 tp=8, B200 tp=8, B300 tp=8, MI300X tp=8, MI325X tp=4, MI355X tp=4
|
||||||
// 122B-A10B: H100 tp=4, H200 tp=4, B200 tp=2, B300 tp=2, MI300X tp=2, MI325X tp=1, MI355X tp=1
|
// 122B-A10B: H100 tp=4, H200 tp=4, B200 tp=2, B300 tp=2, MI300X tp=2, MI325X tp=1, MI355X tp=1
|
||||||
// 35B-A3B: H100 tp=1 (tp=2 w/ MTP), H200 tp=1, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1
|
// 35B-A3B: H100 tp=1 (tp=2 w/ MTP), H200 tp=1, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1
|
||||||
// 27B: H100 tp=1 (tp=2 w/ MTP); tp=1 on all other hardware
|
// 27B: H100 tp=1 (tp=2 w/ MTP); tp=1 on all other hardware
|
||||||
// 9B/4B/2B/0.8B: tp=1 on all hardware (including MI300X, MI325X, MI355X)
|
// 9B/4B/2B/0.8B: tp=1 on all hardware (including MI300X, MI325X, MI355X)
|
||||||
//
|
//
|
||||||
// GPU requirements (FP8, where available):
|
// GPU requirements (FP8, where available):
|
||||||
// 397B-A17B: H100 tp=8, H200 tp=8 ep=8, B200 tp=4, B300 tp=2, MI300X tp=4, MI325X tp=2, MI355X tp=2
|
// 397B-A17B: H100 tp=8, H200 tp=8 ep=8, B200 tp=4, B300 tp=4, MI300X tp=4, MI325X tp=2, MI355X tp=2
|
||||||
// 122B-A10B: H100 tp=2 (tp=4 w/ MTP), H200 tp=2, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1
|
// 122B-A10B: H100 tp=2 (tp=4 w/ MTP), H200 tp=2, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1
|
||||||
// 35B-A3B: H100 tp=1, H200 tp=1, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1
|
// 35B-A3B: H100 tp=1, H200 tp=1, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1
|
||||||
// 27B: tp=1 on all hardware (including MI300X, MI325X, MI355X)
|
// 27B: tp=1 on all hardware (including MI300X, MI325X, MI355X)
|
||||||
//
|
//
|
||||||
// FP4 (397B only): NVFP4 on Blackwell B200 tp=4, B300 tp=2; AMD MXFP4 on MI355X tp=2
|
// FP4 (397B only): NVFP4 on Blackwell B200/B300 tp=4; AMD MXFP4 on MI355X tp=2
|
||||||
|
|
||||||
const MOE_MODELS = new Set(['397b', '122b', '35b']);
|
const MOE_MODELS = new Set(['397b', '122b', '35b']);
|
||||||
const FP8_MODELS = new Set(['397b', '122b', '35b', '27b']);
|
const FP8_MODELS = new Set(['397b', '122b', '35b', '27b']);
|
||||||
@@ -149,7 +149,7 @@ export const Qwen35Deployment = () => {
|
|||||||
h100: { bf16: { tp: 16, mem: 0.8, multinode: true, nnodes: 2 }, fp8: { tp: 8, mem: 0.8 } },
|
h100: { bf16: { tp: 16, mem: 0.8, multinode: true, nnodes: 2 }, fp8: { tp: 8, mem: 0.8 } },
|
||||||
h200: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 8, ep: 8, mem: 0.8 } },
|
h200: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 8, ep: 8, mem: 0.8 } },
|
||||||
b200: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 4, mem: 0.8 }, fp4: { tp: 4, mem: 0.85 } },
|
b200: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 4, mem: 0.8 }, fp4: { tp: 4, mem: 0.85 } },
|
||||||
b300: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 }, fp4: { tp: 2, mem: 0.8 } },
|
b300: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 4, mem: 0.8 }, fp4: { tp: 4, mem: 0.8 } },
|
||||||
mi300x: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 4, mem: 0.8 } },
|
mi300x: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 4, mem: 0.8 } },
|
||||||
mi325x: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 } },
|
mi325x: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 } },
|
||||||
mi355x: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 }, fp4: { tp: 2, mem: 0.8 } },
|
mi355x: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 }, fp4: { tp: 2, mem: 0.8 } },
|
||||||
@@ -372,7 +372,7 @@ export const Qwen35Deployment = () => {
|
|||||||
const commandRules = {
|
const commandRules = {
|
||||||
reasoning: (value) => value === 'enabled' ? '--reasoning-parser qwen3' : null,
|
reasoning: (value) => value === 'enabled' ? '--reasoning-parser qwen3' : null,
|
||||||
toolcall: (value) => value === 'enabled' ? '--tool-call-parser qwen3_coder' : null,
|
toolcall: (value) => value === 'enabled' ? '--tool-call-parser qwen3_coder' : null,
|
||||||
speculative: (value) => value === 'enabled' ? '--speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4' : null,
|
speculative: (value) => value === 'enabled' ? '--speculative-algorithm NEXTN \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4' : null,
|
||||||
mambaCache: (value) => value === 'v2' ? '--mamba-scheduler-strategy extra_buffer' : null,
|
mambaCache: (value) => value === 'v2' ? '--mamba-scheduler-strategy extra_buffer' : null,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -423,9 +423,12 @@ export const Qwen35Deployment = () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Append backend configurations
|
// Append backend configurations
|
||||||
if (hardware === 'b200' || hardware === 'b300') {
|
if (hardware === 'b200' || (hardware === 'b300' && quantization === 'fp4')) {
|
||||||
cmd += ` \\\n --attention-backend trtllm_mha`;
|
cmd += ` \\\n --attention-backend trtllm_mha`;
|
||||||
}
|
}
|
||||||
|
if (hardware === 'b300' && quantization !== 'fp4') {
|
||||||
|
cmd += ` \\\n --attention-backend flashinfer`;
|
||||||
|
}
|
||||||
|
|
||||||
// Append AMD GPU-specific backend configurations.
|
// Append AMD GPU-specific backend configurations.
|
||||||
// All AMD MI GPUs use the AITER unified-attention backend (pair with
|
// All AMD MI GPUs use the AITER unified-attention backend (pair with
|
||||||
@@ -466,6 +469,10 @@ export const Qwen35Deployment = () => {
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// NVIDIA NVFP4 on Blackwell (B200 / B300).
|
// NVIDIA NVFP4 on Blackwell (B200 / B300).
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm';
|
||||||
|
cmd += ' \\\n --fp4-gemm-backend flashinfer_cutlass';
|
||||||
|
} else {
|
||||||
cmd += ' \\\n --quantization modelopt_fp4';
|
cmd += ' \\\n --quantization modelopt_fp4';
|
||||||
cmd += ' \\\n --fp4-gemm-backend flashinfer_cutlass';
|
cmd += ' \\\n --fp4-gemm-backend flashinfer_cutlass';
|
||||||
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
|
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
|
||||||
@@ -477,6 +484,7 @@ export const Qwen35Deployment = () => {
|
|||||||
cmd += ' \\\n --disable-radix-cache';
|
cmd += ' \\\n --disable-radix-cache';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Add memory fraction last
|
// Add memory fraction last
|
||||||
if (memFraction !== undefined) {
|
if (memFraction !== undefined) {
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ export const Qwen36Deployment = () => {
|
|||||||
{ id: 'h100', label: 'H100', default: true },
|
{ id: 'h100', label: 'H100', default: true },
|
||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'b200', label: 'B200', default: false },
|
{ id: 'b200', label: 'B200', default: false },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'xeon', label: 'XEON', default: false },
|
{ id: 'xeon', label: 'XEON', default: false },
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
@@ -85,6 +86,7 @@ export const Qwen36Deployment = () => {
|
|||||||
h100: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
h100: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||||
h200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
h200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||||
b200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
b200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||||
|
b300: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||||
xeon: { bf16: { tp: 3 }, fp8: { tp: 3 } },
|
xeon: { bf16: { tp: 3 }, fp8: { tp: 3 } },
|
||||||
},
|
},
|
||||||
'27b': {
|
'27b': {
|
||||||
@@ -92,6 +94,7 @@ export const Qwen36Deployment = () => {
|
|||||||
h100: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
h100: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||||
h200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
h200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||||
b200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
b200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||||
|
b300: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||||
xeon: { bf16: { tp: 6 }, fp8: { tp: 6 } },
|
xeon: { bf16: { tp: 6 }, fp8: { tp: 6 } },
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
@@ -192,6 +195,9 @@ export const Qwen36Deployment = () => {
|
|||||||
if (hardware === 'b200') {
|
if (hardware === 'b200') {
|
||||||
cmd += ` \\\n --attention-backend trtllm_mha`;
|
cmd += ` \\\n --attention-backend trtllm_mha`;
|
||||||
}
|
}
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ` \\\n --attention-backend flashinfer`;
|
||||||
|
}
|
||||||
if (hwConfig.mem !== undefined) {
|
if (hwConfig.mem !== undefined) {
|
||||||
cmd += ` \\\n --mem-fraction-static ${hwConfig.mem}`;
|
cmd += ` \\\n --mem-fraction-static ${hwConfig.mem}`;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ export const Ring251TDeployment = () => {
|
|||||||
// Config mirrors sgl-cookbook src/components/autoregressive/Ring25ConfigGenerator/index.js.
|
// Config mirrors sgl-cookbook src/components/autoregressive/Ring25ConfigGenerator/index.js.
|
||||||
//
|
//
|
||||||
// GPU requirements:
|
// GPU requirements:
|
||||||
// H200 / B200 / GB200 / GB300 / MI355X: single-node (tp per platform)
|
// H200 / B200 / B300 / GB200 / GB300 / MI355X: single-node (tp per platform)
|
||||||
// MI300X / MI325X: two nodes, tp-size 8, pp-size 2 (multi-node scripts)
|
// MI300X / MI325X: two nodes, tp-size 8, pp-size 2 (multi-node scripts)
|
||||||
const options = {
|
const options = {
|
||||||
hardware: {
|
hardware: {
|
||||||
@@ -11,6 +11,7 @@ export const Ring251TDeployment = () => {
|
|||||||
items: [
|
items: [
|
||||||
{ id: 'h200', label: 'H200', default: true },
|
{ id: 'h200', label: 'H200', default: true },
|
||||||
{ id: 'b200', label: 'B200', default: false },
|
{ id: 'b200', label: 'B200', default: false },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'gb200', label: 'GB200', default: false },
|
{ id: 'gb200', label: 'GB200', default: false },
|
||||||
{ id: 'gb300', label: 'GB300', default: false },
|
{ id: 'gb300', label: 'GB300', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
@@ -39,6 +40,7 @@ export const Ring251TDeployment = () => {
|
|||||||
const modelConfigs = {
|
const modelConfigs = {
|
||||||
h200: { fp8: { tp: 8 } },
|
h200: { fp8: { tp: 8 } },
|
||||||
b200: { fp8: { tp: 8 } },
|
b200: { fp8: { tp: 8 } },
|
||||||
|
b300: { fp8: { tp: 8 } },
|
||||||
gb200: { fp8: { tp: 4 } },
|
gb200: { fp8: { tp: 4 } },
|
||||||
gb300: { fp8: { tp: 4 } },
|
gb300: { fp8: { tp: 4 } },
|
||||||
mi300x: { fp8: { tp: 8, pp: 2, nnodes: 2 } },
|
mi300x: { fp8: { tp: 8, pp: 2, nnodes: 2 } },
|
||||||
@@ -156,6 +158,9 @@ export const Ring251TDeployment = () => {
|
|||||||
cmd += ` --model-path ${modelName}`;
|
cmd += ` --model-path ${modelName}`;
|
||||||
cmd += ` \\\n --tp ${tpValue}`;
|
cmd += ` \\\n --tp ${tpValue}`;
|
||||||
cmd += ' \\\n --trust-remote-code';
|
cmd += ' \\\n --trust-remote-code';
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
cmd += ' \\\n --attention-backend flashinfer';
|
||||||
|
}
|
||||||
|
|
||||||
extraFlags.forEach((flag) => {
|
extraFlags.forEach((flag) => {
|
||||||
cmd += ` \\\n ${flag}`;
|
cmd += ` \\\n ${flag}`;
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ export const FluxDeployment = () => {
|
|||||||
title: 'Hardware Platform',
|
title: 'Hardware Platform',
|
||||||
items: [
|
items: [
|
||||||
{ id: 'b200', label: 'B200', default: true },
|
{ id: 'b200', label: 'B200', default: true },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'h100', label: 'H100', default: false },
|
{ id: 'h100', label: 'H100', default: false },
|
||||||
{ id: 'mi355x', label: 'MI355X', default: false },
|
{ id: 'mi355x', label: 'MI355X', default: false },
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ export const QwenImageEditDeployment = () => {
|
|||||||
title: 'Hardware Platform',
|
title: 'Hardware Platform',
|
||||||
items: [
|
items: [
|
||||||
{ id: 'b200', label: 'B200', default: true },
|
{ id: 'b200', label: 'B200', default: true },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'h100', label: 'H100', default: false },
|
{ id: 'h100', label: 'H100', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
title: 'Hardware Platform',
|
title: 'Hardware Platform',
|
||||||
items: [
|
items: [
|
||||||
{ id: 'b200', label: 'B200', default: true },
|
{ id: 'b200', label: 'B200', default: true },
|
||||||
|
{ id: 'b300', label: 'B300', default: false },
|
||||||
{ id: 'h200', label: 'H200', default: false },
|
{ id: 'h200', label: 'H200', default: false },
|
||||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||||
@@ -109,7 +110,7 @@
|
|||||||
};
|
};
|
||||||
|
|
||||||
const generateCommand = () => {
|
const generateCommand = () => {
|
||||||
const { task, modelsize, selectedLoraPath, bestPractice } = values;
|
const { hardware, task, modelsize, selectedLoraPath, bestPractice } = values;
|
||||||
const configKey = `${task}-${modelsize}`;
|
const configKey = `${task}-${modelsize}`;
|
||||||
const config = modelConfigs[configKey];
|
const config = modelConfigs[configKey];
|
||||||
if (!config) {
|
if (!config) {
|
||||||
@@ -118,8 +119,12 @@
|
|||||||
|
|
||||||
let command = `sglang serve \\\n --model-path ${config.repoId} \\\n --dit-layerwise-offload true`;
|
let command = `sglang serve \\\n --model-path ${config.repoId} \\\n --dit-layerwise-offload true`;
|
||||||
if (bestPractice === 'on') {
|
if (bestPractice === 'on') {
|
||||||
|
if (hardware === 'b300') {
|
||||||
|
command += ` \\\n --tp-size 2 \\\n --num-gpus 8 \\\n --sp-degree 2 \\\n --ulysses-degree 2 \\\n --enable-cfg-parallel`;
|
||||||
|
} else {
|
||||||
command += ` \\\n --num-gpus 4 \\\n --ulysses-degree 2 \\\n --enable-cfg-parallel`;
|
command += ` \\\n --num-gpus 4 \\\n --ulysses-degree 2 \\\n --enable-cfg-parallel`;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
if (selectedLoraPath && selectedLoraPath !== 'none') {
|
if (selectedLoraPath && selectedLoraPath !== 'none') {
|
||||||
command += ` \\\n --lora-path ${selectedLoraPath}`;
|
command += ` \\\n --lora-path ${selectedLoraPath}`;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user