Add new Mintlify documentation site (docs_new/) (#23001)
Co-authored-by: AdityaVKochar <adityavardhankochar@gmail.com> Co-authored-by: mintlify[bot] <109931778+mintlify[bot]@users.noreply.github.com> Co-authored-by: adhyan-jain <adhyanjain2006@gmail.com> Co-authored-by: Adhyan Jain <71976554+adhyan-jain@users.noreply.github.com> Co-authored-by: Maitri-shah29 <maitrirajivshah@gmail.com> Co-authored-by: Adarsh Shirawalmath <114558126+adarshxs@users.noreply.github.com> Co-authored-by: Maitri Shah <shah29maitri@gmail.com> Co-authored-by: Aditya Vardhan Kochar <80113212+AdityaVKochar@users.noreply.github.com> Co-authored-by: Rishit Shivam <164783543+pokymono@users.noreply.github.com> Co-authored-by: Rishitshivam <164783543+Rishitshivam@users.noreply.github.com> Co-authored-by: IshhanKheria <ishhankheria06@gmail.com> Co-authored-by: Ishita Joshi <ishitata.joshi@gmail.com> Co-authored-by: Richard Chen <104477092+Richardczl98@users.noreply.github.com> Co-authored-by: longGGGGGG <553746008@qq.com> Co-authored-by: Richard <richardchen@radixark.ai> Co-authored-by: Nakul Sinha <nakul.new4socials@gmail.com> Co-authored-by: Divyam Agrawal <ludicrouslytrue@gmail.com> Co-authored-by: Richardczl98 <Zhenlinc@stanford.edu> Co-authored-by: Krishang Zinzuwadia <krishangzinzuwadia@gmail.com> Co-authored-by: nimeshas <nimesha.s106@gmail.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Jignas Paturu <86356085+JignasP@users.noreply.github.com> Co-authored-by: zijiexia <37504505+zijiexia@users.noreply.github.com>
This commit is contained in:
co-authored by
AdityaVKochar
mintlify[bot]
adhyan-jain
Adhyan Jain
Maitri-shah29
Adarsh Shirawalmath
Maitri Shah
Aditya Vardhan Kochar
Rishit Shivam
Rishitshivam
IshhanKheria
Ishita Joshi
Richard Chen
longGGGGGG
Richard
Nakul Sinha
Divyam Agrawal
Richardczl98
Krishang Zinzuwadia
nimeshas
Claude Opus 4.6
github-actions[bot]
Jignas Paturu
zijiexia
parent
575fdc2c4c
commit
a3291b5654
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"generatedAt": "2026-04-07T00:14:40.777335Z",
|
||||
"sourceRepo": "https://github.com/lm-sys/lm-sys.github.io/tree/main/blog",
|
||||
"keywords": [
|
||||
"sglang",
|
||||
"sgl-project/sglang",
|
||||
"sgl-kernel",
|
||||
"sglang-jax",
|
||||
"sgl diffusion",
|
||||
"sglang diffusion"
|
||||
],
|
||||
"maxCards": 6,
|
||||
"totalBlogFilesScanned": 80,
|
||||
"cardsPublished": 6,
|
||||
"posts": [
|
||||
{
|
||||
"slug": "2026-03-25-gtc2026",
|
||||
"title": "Highlights of SGLang at NVIDIA GTC 2026",
|
||||
"url": "https://lmsys.org/blog/2026-03-25-gtc2026/",
|
||||
"image": "https://lmsys.org/images/blog/gtc2026/happyhour-crowd.jpg",
|
||||
"date": "March 31, 2026"
|
||||
},
|
||||
{
|
||||
"slug": "2026-03-25-eep-partial-failure-tolerance",
|
||||
"title": "Elastic EP in SGLang: Achieving Partial Failure Tolerance for DeepSeek MoE Deployments",
|
||||
"url": "https://lmsys.org/blog/2026-03-25-eep-partial-failure-tolerance/",
|
||||
"image": "https://lmsys.org/images/blog/eep-partial-failure-tolerance/figure.png",
|
||||
"date": "March 25, 2026"
|
||||
},
|
||||
{
|
||||
"slug": "2026-03-17-rocm-miles-rl-amd",
|
||||
"title": "ROCm Support for Miles: Large-Scale RL Post-Training on AMD Instinct\u2122 GPUs",
|
||||
"url": "https://lmsys.org/blog/2026-03-17-rocm-miles-rl-amd/",
|
||||
"image": "https://lmsys.org/images/blog/rocm_miles_rl/fig_1.png",
|
||||
"date": "March 17, 2026"
|
||||
},
|
||||
{
|
||||
"slug": "2026-03-11-run-nvidia-nemotron-3-super",
|
||||
"title": "SGLang Adds Day-0 Support for NVIDIA Nemotron 3 Super for building High-Efficiency Multi-Agent Systems",
|
||||
"url": "https://lmsys.org/blog/2026-03-11-run-nvidia-nemotron-3-super/",
|
||||
"image": "https://lmsys.org/images/blog/nemotron-3-super/figure_1.svg",
|
||||
"date": "March 11, 2026"
|
||||
},
|
||||
{
|
||||
"slug": "2026-02-20-gb300-inferencex",
|
||||
"title": "Unlocking 25x Inference Performance with SGLang on NVIDIA GB300 NVL72",
|
||||
"url": "https://lmsys.org/blog/2026-02-20-gb300-inferencex/",
|
||||
"image": "https://lmsys.org/images/blog/gb300_inferencex/img-1.png",
|
||||
"date": "February 20, 2026"
|
||||
},
|
||||
{
|
||||
"slug": "2026-02-19-gb300-longctx",
|
||||
"title": "Deploying DeepSeek on GB300 NVL72: Big Wins in Long-Context Inference",
|
||||
"url": "https://lmsys.org/blog/2026-02-19-gb300-longctx/",
|
||||
"image": "https://lmsys.org/images/blog/gb300_longctx/cover.png",
|
||||
"date": "February 19, 2026"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,359 @@
|
||||
export const DeepSeekMathV2Deployment = () => {
|
||||
const modelFamily = 'deepseek-ai';
|
||||
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'b200', label: 'B200', subtitle: '183GB', default: true },
|
||||
{ id: 'b300', label: 'B300', subtitle: '275GB', default: false }
|
||||
]
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--reasoning-parser deepseek-r1' : null
|
||||
},
|
||||
dpattention: {
|
||||
name: 'dpattention',
|
||||
title: 'DP Attention',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', subtitle: 'Low Latency', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', subtitle: 'High Throughput', default: false }
|
||||
],
|
||||
commandRule: null
|
||||
}
|
||||
};
|
||||
|
||||
// BF16 only, B200/B300 tp=8
|
||||
const modelConfigs = {
|
||||
b200: { bf16: { tp: 8, mem: null } },
|
||||
b300: { bf16: { tp: 8, mem: null } }
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware } = values;
|
||||
|
||||
const modelName = `${modelFamily}/DeepSeek-Math-V2`;
|
||||
|
||||
const hwConfig = modelConfigs[hardware].bf16;
|
||||
const tpValue = hwConfig.tp;
|
||||
const memFraction = hwConfig.mem;
|
||||
|
||||
let cmd = 'sglang serve --model-path';
|
||||
cmd += ` ${modelName}`;
|
||||
|
||||
// TP setting
|
||||
cmd += ` \\\n --tp ${tpValue}`;
|
||||
|
||||
// DP Attention: --dp matches --tp
|
||||
if (values.dpattention === 'enabled') {
|
||||
cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`;
|
||||
}
|
||||
|
||||
// EP setting (commonly matches tp for MoE models)
|
||||
cmd += ` \\\n --ep ${tpValue}`;
|
||||
|
||||
// Apply commandRule from all options
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.commandRule) {
|
||||
const rule = option.commandRule(values[key]);
|
||||
if (rule) {
|
||||
cmd += ` \\\n ${rule}`;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Memory fraction based on hardware and quantization (skip for 8-card configs)
|
||||
if (memFraction) {
|
||||
cmd += ` \\\n --mem-fraction-static ${memFraction}`;
|
||||
}
|
||||
|
||||
cmd += ' \\\n --host 0.0.0.0 \\\n --port 30000';
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,168 @@
|
||||
export const DeepSeekOCRDeployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'mi300x', label: 'MI300X', default: true },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'fp16', label: 'FP16', default: true }
|
||||
]
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true },
|
||||
{ id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false },
|
||||
{ id: 'ep', label: 'EP', subtitle: 'Expert Parallel', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = option.items.filter(item => item.default).map(item => item.id);
|
||||
} else {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
}
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode - prioritize page theme over system preference
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
// Check Mintlify's theme class on html element
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues(prev => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
} else {
|
||||
return { ...prev, [optionName]: currentValues.filter(id => id !== itemId) };
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, quantization, strategy } = values;
|
||||
const strategyArray = Array.isArray(strategy) ? strategy : [];
|
||||
|
||||
// Validation checks
|
||||
// Check MI300X compatibility - MI300X + DeepSeek-OCR only supports FP16 quantization
|
||||
if ((hardware === 'mi300x') && quantization !== 'fp16') {
|
||||
return '# Error: MI300X + DeepSeek-OCR only supports FP16 quantization\n# Please select FP16 quantization';
|
||||
}
|
||||
|
||||
// Model path
|
||||
let modelPath = 'deepseek-ai/DeepSeek-OCR';
|
||||
|
||||
let cmd = 'python3 -m sglang.launch_server \\\n';
|
||||
cmd += ` --model-path ${modelPath}`;
|
||||
cmd += ` \\\n --dtype float16`;
|
||||
|
||||
// TP strategy
|
||||
if (strategyArray.includes('tp')) {
|
||||
cmd += ` \\\n --tp 1`;
|
||||
}
|
||||
|
||||
// DP strategy
|
||||
if (strategyArray.includes('dp')) {
|
||||
cmd += ` \\\n --dp 1 \\\n --enable-dp-attention`;
|
||||
}
|
||||
|
||||
// EP strategy
|
||||
if (strategyArray.includes('ep')) {
|
||||
cmd += ` \\\n --ep 1`;
|
||||
}
|
||||
|
||||
cmd += ` \\\n --enable-symm-mem # Optional: improves performance, but may be unstable`;
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,342 @@
|
||||
export const DeepSeekOCR2Deployment = () => {
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false },
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'fp16', label: 'FP16', default: true },
|
||||
]
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true },
|
||||
{ id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false },
|
||||
{ id: 'ep', label: 'EP', subtitle: 'Expert Parallel', default: false }
|
||||
]
|
||||
},
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, strategy } = values;
|
||||
|
||||
const strategyArray = Array.isArray(strategy) ? strategy : [];
|
||||
|
||||
let modelPath = 'deepseek-ai/DeepSeek-OCR-2';
|
||||
|
||||
let cmd = 'sglang serve \\\n';
|
||||
cmd += ` --model-path ${modelPath}`;
|
||||
cmd += ` \\\n --enable-multimodal`;
|
||||
|
||||
if (strategyArray.includes('tp')) {
|
||||
cmd += ` \\\n --tp 1`;
|
||||
}
|
||||
|
||||
if (strategyArray.includes('dp')) {
|
||||
cmd += ` \\\n --dp 1 \\\n --enable-dp-attention`;
|
||||
}
|
||||
|
||||
if (strategyArray.includes('ep')) {
|
||||
cmd += ` \\\n --ep 1`;
|
||||
}
|
||||
|
||||
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||
cmd += ` \\\n --attention-backend triton` + ` \\\n --trust-remote-code`;
|
||||
}
|
||||
|
||||
cmd += ` \\\n --host 0.0.0.0 \\\n --port 30000`;
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,856 @@
|
||||
export const DeepSeekR1AdvancedDeployment = () => {
|
||||
const lookupData = {
|
||||
"model": "deepseek-r1",
|
||||
"version": "v0.5.6",
|
||||
"ui_options": {
|
||||
"hardware": [
|
||||
{
|
||||
"id": "b200",
|
||||
"label": "B200",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"id": "h200",
|
||||
"label": "H200",
|
||||
"default": false
|
||||
},
|
||||
{
|
||||
"id": "mi300x",
|
||||
"label": "MI300X",
|
||||
"default": false
|
||||
},
|
||||
{
|
||||
"id": "mi325x",
|
||||
"label": "MI325X",
|
||||
"default": false
|
||||
},
|
||||
{
|
||||
"id": "mi355x",
|
||||
"label": "MI355X",
|
||||
"default": false
|
||||
}
|
||||
],
|
||||
"quantization": [
|
||||
{
|
||||
"id": "fp8",
|
||||
"label": "FP8",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"id": "fp4",
|
||||
"label": "FP4",
|
||||
"default": false
|
||||
}
|
||||
],
|
||||
"scenario": [
|
||||
{
|
||||
"id": "low-latency",
|
||||
"label": "Low Latency",
|
||||
"subtitle": "Concurrency 4-8",
|
||||
"default": true
|
||||
},
|
||||
{
|
||||
"id": "high-throughput",
|
||||
"label": "High Throughput",
|
||||
"subtitle": "Concurrency 16-128",
|
||||
"default": false
|
||||
}
|
||||
],
|
||||
"gpu_count": [
|
||||
{
|
||||
"id": 4,
|
||||
"label": "4 GPUs",
|
||||
"default": false
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"label": "8 GPUs",
|
||||
"default": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"configs": [
|
||||
{
|
||||
"hardware": "b200",
|
||||
"quantization": "fp4",
|
||||
"gpu_count": 4,
|
||||
"scenario": "low-latency",
|
||||
"parameters": {
|
||||
"model_path": "nvidia/DeepSeek-R1-0528-FP4-v2",
|
||||
"tensor_parallel_size": 4,
|
||||
"cuda_graph_max_bs": 256,
|
||||
"max_running_requests": 256,
|
||||
"mem_fraction_static": 0.85,
|
||||
"ep_size": 4,
|
||||
"scheduler_recv_interval": 10,
|
||||
"enable_symm_mem": true,
|
||||
"stream_interval": 10
|
||||
}
|
||||
},
|
||||
{
|
||||
"hardware": "b200",
|
||||
"quantization": "fp4",
|
||||
"gpu_count": 4,
|
||||
"scenario": "high-throughput",
|
||||
"parameters": {
|
||||
"model_path": "nvidia/DeepSeek-R1-0528-FP4-v2",
|
||||
"tensor_parallel_size": 4,
|
||||
"cuda_graph_max_bs": 256,
|
||||
"max_running_requests": 256,
|
||||
"mem_fraction_static": 0.85,
|
||||
"ep_size": 4,
|
||||
"scheduler_recv_interval": 30,
|
||||
"enable_symm_mem": true,
|
||||
"stream_interval": 10
|
||||
}
|
||||
},
|
||||
{
|
||||
"hardware": "b200",
|
||||
"quantization": "fp4",
|
||||
"gpu_count": 8,
|
||||
"scenario": "low-latency",
|
||||
"parameters": {
|
||||
"model_path": "nvidia/DeepSeek-R1-0528-FP4-v2",
|
||||
"tensor_parallel_size": 8,
|
||||
"cuda_graph_max_bs": 256,
|
||||
"max_running_requests": 256,
|
||||
"mem_fraction_static": 0.85,
|
||||
"kv_cache_dtype": "fp8_e4m3",
|
||||
"chunked_prefill_size": 16384,
|
||||
"ep_size": 8,
|
||||
"scheduler_recv_interval": 10,
|
||||
"enable_symm_mem": true,
|
||||
"stream_interval": 10
|
||||
}
|
||||
},
|
||||
{
|
||||
"hardware": "b200",
|
||||
"quantization": "fp4",
|
||||
"gpu_count": 8,
|
||||
"scenario": "high-throughput",
|
||||
"parameters": {
|
||||
"model_path": "nvidia/DeepSeek-R1-0528-FP4-v2",
|
||||
"tensor_parallel_size": 8,
|
||||
"cuda_graph_max_bs": 256,
|
||||
"max_running_requests": 256,
|
||||
"mem_fraction_static": 0.85,
|
||||
"kv_cache_dtype": "fp8_e4m3",
|
||||
"chunked_prefill_size": 16384,
|
||||
"ep_size": 8,
|
||||
"scheduler_recv_interval": 30,
|
||||
"enable_symm_mem": true,
|
||||
"stream_interval": 10
|
||||
}
|
||||
},
|
||||
{
|
||||
"hardware": "b200",
|
||||
"quantization": "fp8",
|
||||
"gpu_count": 8,
|
||||
"scenario": "low-latency",
|
||||
"parameters": {
|
||||
"env_vars": "SGLANG_ENABLE_JIT_DEEPGEMM=false",
|
||||
"model_path": "deepseek-ai/DeepSeek-R1-0528",
|
||||
"tensor_parallel_size": 8,
|
||||
"cuda_graph_max_bs": 128,
|
||||
"max_running_requests": 128,
|
||||
"mem_fraction_static": 0.82,
|
||||
"kv_cache_dtype": "fp8_e4m3",
|
||||
"chunked_prefill_size": 32768,
|
||||
"max_prefill_tokens": 32768,
|
||||
"scheduler_recv_interval": 10,
|
||||
"stream_interval": 30,
|
||||
"fp8_gemm_backend": "flashinfer_trtllm"
|
||||
}
|
||||
},
|
||||
{
|
||||
"hardware": "b200",
|
||||
"quantization": "fp8",
|
||||
"gpu_count": 8,
|
||||
"scenario": "high-throughput",
|
||||
"parameters": {
|
||||
"env_vars": "SGLANG_ENABLE_JIT_DEEPGEMM=false",
|
||||
"model_path": "deepseek-ai/DeepSeek-R1-0528",
|
||||
"tensor_parallel_size": 8,
|
||||
"cuda_graph_max_bs": 128,
|
||||
"max_running_requests": 128,
|
||||
"mem_fraction_static": 0.82,
|
||||
"kv_cache_dtype": "fp8_e4m3",
|
||||
"chunked_prefill_size": 32768,
|
||||
"max_prefill_tokens": 32768,
|
||||
"scheduler_recv_interval": 30,
|
||||
"stream_interval": 30,
|
||||
"fp8_gemm_backend": "flashinfer_trtllm"
|
||||
}
|
||||
},
|
||||
{
|
||||
"hardware": "h200",
|
||||
"quantization": "fp8",
|
||||
"gpu_count": 8,
|
||||
"scenario": "low-latency",
|
||||
"parameters": {
|
||||
"model_path": "deepseek-ai/DeepSeek-R1-0528",
|
||||
"trust_remote_code": true,
|
||||
"tensor_parallel_size": 8,
|
||||
"disable_radix_cache": true,
|
||||
"max_running_requests": 256,
|
||||
"cuda_graph_max_bs": 256,
|
||||
"chunked_prefill_size": 32768,
|
||||
"max_prefill_tokens": 32768,
|
||||
"mem_fraction_static": 0.82,
|
||||
"attention_backend": "flashinfer",
|
||||
"stream_interval": 10,
|
||||
"decode_log_interval": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"hardware": "h200",
|
||||
"quantization": "fp8",
|
||||
"gpu_count": 8,
|
||||
"scenario": "high-throughput",
|
||||
"parameters": {
|
||||
"model_path": "deepseek-ai/DeepSeek-R1-0528",
|
||||
"trust_remote_code": true,
|
||||
"tensor_parallel_size": 8,
|
||||
"disable_radix_cache": true,
|
||||
"max_running_requests": 512,
|
||||
"cuda_graph_max_bs": 512,
|
||||
"chunked_prefill_size": 32768,
|
||||
"max_prefill_tokens": 32768,
|
||||
"mem_fraction_static": 0.82,
|
||||
"attention_backend": "flashinfer",
|
||||
"stream_interval": 10,
|
||||
"decode_log_interval": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"hardware": "mi300x",
|
||||
"quantization": "fp8",
|
||||
"gpu_count": 8,
|
||||
"scenario": "low-latency",
|
||||
"parameters": {
|
||||
"env_vars": "SGLANG_USE_AITER=1 SGLANG_AITER_MLA_PERSIST=1",
|
||||
"model_path": "deepseek-ai/DeepSeek-R1-0528",
|
||||
"trust_remote_code": true,
|
||||
"tensor_parallel_size": 8,
|
||||
"mem_fraction_static": 0.8,
|
||||
"cuda_graph_max_bs": 128,
|
||||
"chunked_prefill_size": 131072,
|
||||
"num_continuous_decode_steps": 4,
|
||||
"max_prefill_tokens": 131072,
|
||||
"kv_cache_dtype": "fp8_e4m3",
|
||||
"attention_backend": "aiter",
|
||||
"disable_radix_cache": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"hardware": "mi300x",
|
||||
"quantization": "fp8",
|
||||
"gpu_count": 8,
|
||||
"scenario": "high-throughput",
|
||||
"parameters": {
|
||||
"env_vars": "SGLANG_USE_AITER=1 SGLANG_AITER_MLA_PERSIST=1",
|
||||
"model_path": "deepseek-ai/DeepSeek-R1-0528",
|
||||
"trust_remote_code": true,
|
||||
"tensor_parallel_size": 8,
|
||||
"mem_fraction_static": 0.8,
|
||||
"cuda_graph_max_bs": 512,
|
||||
"chunked_prefill_size": 131072,
|
||||
"num_continuous_decode_steps": 4,
|
||||
"max_prefill_tokens": 131072,
|
||||
"kv_cache_dtype": "fp8_e4m3",
|
||||
"attention_backend": "aiter",
|
||||
"disable_radix_cache": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"hardware": "mi325x",
|
||||
"quantization": "fp8",
|
||||
"gpu_count": 8,
|
||||
"scenario": "low-latency",
|
||||
"parameters": {
|
||||
"env_vars": "SGLANG_USE_AITER=1 SGLANG_AITER_MLA_PERSIST=1",
|
||||
"model_path": "deepseek-ai/DeepSeek-R1-0528",
|
||||
"trust_remote_code": true,
|
||||
"tensor_parallel_size": 8,
|
||||
"mem_fraction_static": 0.8,
|
||||
"cuda_graph_max_bs": 128,
|
||||
"chunked_prefill_size": 131072,
|
||||
"num_continuous_decode_steps": 4,
|
||||
"max_prefill_tokens": 131072,
|
||||
"kv_cache_dtype": "fp8_e4m3",
|
||||
"attention_backend": "aiter",
|
||||
"disable_radix_cache": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"hardware": "mi325x",
|
||||
"quantization": "fp8",
|
||||
"gpu_count": 8,
|
||||
"scenario": "high-throughput",
|
||||
"parameters": {
|
||||
"env_vars": "SGLANG_USE_AITER=1 SGLANG_AITER_MLA_PERSIST=1",
|
||||
"model_path": "deepseek-ai/DeepSeek-R1-0528",
|
||||
"trust_remote_code": true,
|
||||
"tensor_parallel_size": 8,
|
||||
"mem_fraction_static": 0.8,
|
||||
"cuda_graph_max_bs": 512,
|
||||
"chunked_prefill_size": 131072,
|
||||
"num_continuous_decode_steps": 4,
|
||||
"max_prefill_tokens": 131072,
|
||||
"kv_cache_dtype": "fp8_e4m3",
|
||||
"attention_backend": "aiter",
|
||||
"disable_radix_cache": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"hardware": "mi355x",
|
||||
"quantization": "fp8",
|
||||
"gpu_count": 8,
|
||||
"scenario": "low-latency",
|
||||
"parameters": {
|
||||
"env_vars": "SGLANG_USE_AITER=1 RCCL_MSCCL_ENABLE=0 ROCM_QUICK_REDUCE_QUANTIZATION=INT4",
|
||||
"model_path": "deepseek-ai/DeepSeek-R1-0528",
|
||||
"trust_remote_code": true,
|
||||
"tensor_parallel_size": 8,
|
||||
"mem_fraction_static": 0.8,
|
||||
"disable_radix_cache": true,
|
||||
"chunked_prefill_size": 196608,
|
||||
"num_continuous_decode_steps": 4,
|
||||
"max_prefill_tokens": 196608,
|
||||
"cuda_graph_max_bs": 128,
|
||||
"attention_backend": "aiter",
|
||||
"kv_cache_dtype": "fp8_e4m3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"hardware": "mi355x",
|
||||
"quantization": "fp8",
|
||||
"gpu_count": 8,
|
||||
"scenario": "high-throughput",
|
||||
"parameters": {
|
||||
"env_vars": "SGLANG_USE_AITER=1 RCCL_MSCCL_ENABLE=0 ROCM_QUICK_REDUCE_QUANTIZATION=INT4",
|
||||
"model_path": "deepseek-ai/DeepSeek-R1-0528",
|
||||
"trust_remote_code": true,
|
||||
"tensor_parallel_size": 8,
|
||||
"mem_fraction_static": 0.8,
|
||||
"disable_radix_cache": true,
|
||||
"chunked_prefill_size": 196608,
|
||||
"num_continuous_decode_steps": 4,
|
||||
"max_prefill_tokens": 196608,
|
||||
"cuda_graph_max_bs": 512,
|
||||
"attention_backend": "aiter",
|
||||
"kv_cache_dtype": "fp8_e4m3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"hardware": "mi355x",
|
||||
"quantization": "fp4",
|
||||
"gpu_count": 8,
|
||||
"scenario": "low-latency",
|
||||
"parameters": {
|
||||
"env_vars": "SGLANG_USE_AITER=1 ROCM_QUICK_REDUCE_QUANTIZATION=INT4",
|
||||
"model_path": "deepseek-ai/DeepSeek-R1-0528",
|
||||
"trust_remote_code": true,
|
||||
"tensor_parallel_size": 8,
|
||||
"mem_fraction_static": 0.8,
|
||||
"disable_radix_cache": true,
|
||||
"chunked_prefill_size": 196608,
|
||||
"num_continuous_decode_steps": 4,
|
||||
"max_prefill_tokens": 196608,
|
||||
"cuda_graph_max_bs": 128,
|
||||
"attention_backend": "aiter",
|
||||
"kv_cache_dtype": "fp8_e4m3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"hardware": "mi355x",
|
||||
"quantization": "fp4",
|
||||
"gpu_count": 8,
|
||||
"scenario": "high-throughput",
|
||||
"parameters": {
|
||||
"env_vars": "SGLANG_USE_AITER=1 ROCM_QUICK_REDUCE_QUANTIZATION=INT4",
|
||||
"model_path": "deepseek-ai/DeepSeek-R1-0528",
|
||||
"trust_remote_code": true,
|
||||
"tensor_parallel_size": 8,
|
||||
"mem_fraction_static": 0.8,
|
||||
"disable_radix_cache": true,
|
||||
"chunked_prefill_size": 196608,
|
||||
"num_continuous_decode_steps": 4,
|
||||
"max_prefill_tokens": 196608,
|
||||
"cuda_graph_max_bs": 512,
|
||||
"attention_backend": "aiter",
|
||||
"kv_cache_dtype": "fp8_e4m3"
|
||||
}
|
||||
}
|
||||
],
|
||||
"validation": [
|
||||
{
|
||||
"hardware": "h200",
|
||||
"quantization": "fp4",
|
||||
"error": "FP4 is only available for B200 hardware. Please select FP8 quantization."
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
const fieldToFlag = {
|
||||
model_path: 'model-path',
|
||||
trust_remote_code: 'trust-remote-code',
|
||||
tensor_parallel_size: 'tp',
|
||||
data_parallel_size: 'dp',
|
||||
ep_size: 'ep-size',
|
||||
cuda_graph_max_bs: 'cuda-graph-max-bs',
|
||||
max_running_requests: 'max-running-requests',
|
||||
mem_fraction_static: 'mem-fraction-static',
|
||||
kv_cache_dtype: 'kv-cache-dtype',
|
||||
chunked_prefill_size: 'chunked-prefill-size',
|
||||
max_prefill_tokens: 'max-prefill-tokens',
|
||||
enable_flashinfer_allreduce_fusion: 'enable-flashinfer-allreduce-fusion',
|
||||
scheduler_recv_interval: 'scheduler-recv-interval',
|
||||
enable_symm_mem: 'enable-symm-mem',
|
||||
disable_radix_cache: 'disable-radix-cache',
|
||||
attention_backend: 'attention-backend',
|
||||
moe_runner_backend: 'moe-runner-backend',
|
||||
stream_interval: 'stream-interval',
|
||||
quantization: 'quantization',
|
||||
decode_log_interval: 'decode-log-interval',
|
||||
fp8_gemm_backend: 'fp8-gemm-backend',
|
||||
num_continuous_decode_steps: 'num-continuous-decode-steps',
|
||||
};
|
||||
|
||||
const findConfig = (hardware, quantization, gpuCount, scenario) => {
|
||||
const match = lookupData.configs.find((entry) => {
|
||||
const hardwareMatch = entry.hardware === hardware;
|
||||
const quantizationMatch = entry.quantization === quantization;
|
||||
const gpuCountMatch = !entry.gpu_count || entry.gpu_count === Number.parseInt(gpuCount, 10);
|
||||
const scenarioMatch = entry.scenario === scenario;
|
||||
return hardwareMatch && quantizationMatch && gpuCountMatch && scenarioMatch;
|
||||
});
|
||||
return match ? match.parameters : null;
|
||||
};
|
||||
|
||||
const getAvailableGpuCounts = (hardware, quantization) => {
|
||||
const entries = lookupData.configs.filter(
|
||||
(entry) => entry.hardware === hardware && entry.quantization === quantization
|
||||
);
|
||||
const gpuCounts = [...new Set(entries.map((entry) => entry.gpu_count))].filter(Boolean);
|
||||
return gpuCounts.length > 0 ? gpuCounts.sort((a, b) => a - b) : [8];
|
||||
};
|
||||
|
||||
const generateCommandFromConfig = (config) => {
|
||||
if (!config) {
|
||||
return '# Error: Configuration not found';
|
||||
}
|
||||
|
||||
let command = '';
|
||||
if (config.env_vars) {
|
||||
command = `${config.env_vars} `;
|
||||
}
|
||||
|
||||
command += 'python3 -m sglang.launch_server \\\n';
|
||||
command += ` --model-path ${config.model_path}`;
|
||||
|
||||
for (const [key, value] of Object.entries(config)) {
|
||||
if (key === 'model_path' || key === 'env_vars') {
|
||||
continue;
|
||||
}
|
||||
|
||||
const flagName = fieldToFlag[key];
|
||||
if (!flagName) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (typeof value === 'boolean') {
|
||||
if (value) {
|
||||
command += ` \\\n --${flagName}`;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
command += ` \\\n --${flagName} ${value}`;
|
||||
}
|
||||
|
||||
return command;
|
||||
};
|
||||
|
||||
const validateSelection = (hardware, quantization) => {
|
||||
for (const rule of lookupData.validation || []) {
|
||||
const hardwareMatch = Array.isArray(rule.hardware)
|
||||
? rule.hardware.includes(hardware)
|
||||
: rule.hardware === hardware;
|
||||
const quantizationMatch = Array.isArray(rule.quantization)
|
||||
? rule.quantization.includes(quantization)
|
||||
: rule.quantization === quantization;
|
||||
if (hardwareMatch && quantizationMatch) {
|
||||
return rule.error;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const resolveItems = (option, values) =>
|
||||
typeof option.getDynamicItems === 'function' ? option.getDynamicItems(values) : option.items;
|
||||
|
||||
const uiOptions = lookupData.ui_options;
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: uiOptions.hardware
|
||||
.filter((option) =>
|
||||
['b200', 'h200', 'mi300x', 'mi325x', 'mi355x'].includes(option.id)
|
||||
)
|
||||
.map((option) => ({
|
||||
id: option.id,
|
||||
label: option.label,
|
||||
default: option.id === 'b200',
|
||||
})),
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
getDynamicItems: (values) =>
|
||||
uiOptions.quantization.map((option) => {
|
||||
const fp4Disabled = ['h200', 'mi300x', 'mi325x'].includes(values.hardware) && option.id === 'fp4';
|
||||
return {
|
||||
id: option.id,
|
||||
label: option.label,
|
||||
default:
|
||||
['h200', 'mi300x', 'mi325x'].includes(values.hardware)
|
||||
? option.id === 'fp8'
|
||||
: option.default,
|
||||
disabled: fp4Disabled,
|
||||
disabledReason: fp4Disabled ? 'FP4 not supported on H200, MI300X, MI325X' : '',
|
||||
};
|
||||
}),
|
||||
},
|
||||
gpuCount: {
|
||||
name: 'gpuCount',
|
||||
title: 'GPU Count',
|
||||
getDynamicItems: (values) => {
|
||||
const availableGpuCounts = getAvailableGpuCounts(values.hardware, values.quantization);
|
||||
const allGpuCounts = uiOptions.gpu_count.map((option) =>
|
||||
typeof option.id === 'number' ? option.id : Number.parseInt(option.id, 10)
|
||||
);
|
||||
const defaultGpuCount = Math.max(...availableGpuCounts);
|
||||
|
||||
return allGpuCounts.map((count) => ({
|
||||
id: String(count),
|
||||
label: `${count} GPUs`,
|
||||
default: count === defaultGpuCount,
|
||||
disabled: !availableGpuCounts.includes(count),
|
||||
disabledReason: availableGpuCounts.includes(count)
|
||||
? ''
|
||||
: `${count} GPUs not available for ${values.hardware.toUpperCase()} ${values.quantization.toUpperCase()}`,
|
||||
}));
|
||||
},
|
||||
},
|
||||
scenario: {
|
||||
name: 'scenario',
|
||||
title: 'Scenario',
|
||||
items: uiOptions.scenario.map((option) => ({
|
||||
id: option.id,
|
||||
label: option.label,
|
||||
subtitle: option.subtitle,
|
||||
default: option.default,
|
||||
})),
|
||||
},
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
const items = resolveItems(option, initialState) || [];
|
||||
const fallback =
|
||||
items.find((item) => item.default && !item.disabled) ||
|
||||
items.find((item) => !item.disabled) ||
|
||||
items[0];
|
||||
initialState[key] = fallback ? fallback.id : '';
|
||||
}
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => {
|
||||
const next = { ...prev, [optionName]: value };
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (typeof option.getDynamicItems !== 'function') {
|
||||
continue;
|
||||
}
|
||||
const items = option.getDynamicItems(next);
|
||||
const current = items.find((item) => item.id === next[key]);
|
||||
if (!current || current.disabled) {
|
||||
const fallback =
|
||||
items.find((item) => item.default && !item.disabled) ||
|
||||
items.find((item) => !item.disabled);
|
||||
if (fallback) {
|
||||
next[key] = fallback.id;
|
||||
}
|
||||
}
|
||||
}
|
||||
return next;
|
||||
});
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const generateCommand = (vals) => {
|
||||
const validationError = validateSelection(vals.hardware, vals.quantization);
|
||||
if (validationError) {
|
||||
return `# Error: ${validationError}`;
|
||||
}
|
||||
|
||||
const config = findConfig(
|
||||
vals.hardware,
|
||||
vals.quantization,
|
||||
vals.gpuCount || '8',
|
||||
vals.scenario
|
||||
);
|
||||
if (!config) {
|
||||
return `# Error: No configuration found for:
|
||||
# Hardware: ${vals.hardware}
|
||||
# Quantization: ${vals.quantization}
|
||||
# GPU Count: ${vals.gpuCount}
|
||||
# Scenario: ${vals.scenario}
|
||||
# This combination is not yet supported.`;
|
||||
}
|
||||
|
||||
return generateCommandFromConfig(config);
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,394 @@
|
||||
export const DeepSeekR1BasicDeployment = () => {
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false },
|
||||
],
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
getDynamicItems: (values) => {
|
||||
const fp4Disabled = values.hardware === 'h100' || values.hardware === 'mi300x';
|
||||
return [
|
||||
{ id: 'fp8', label: 'FP8', default: true },
|
||||
{
|
||||
id: 'fp4',
|
||||
label: 'FP4',
|
||||
default: false,
|
||||
disabled: fp4Disabled,
|
||||
disabledReason: 'H100 and MI300X only support FP8 quantization',
|
||||
},
|
||||
];
|
||||
},
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true },
|
||||
{ id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false },
|
||||
{ id: 'ep', label: 'EP', subtitle: 'Expert Parallel', default: false },
|
||||
{ id: 'mtp', label: 'MTP', subtitle: 'Multi-token Prediction', default: false },
|
||||
],
|
||||
},
|
||||
thinking: {
|
||||
name: 'thinking',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false },
|
||||
],
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
const resolveItems = (option, values) =>
|
||||
typeof option.getDynamicItems === 'function' ? option.getDynamicItems(values) : option.items;
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
continue;
|
||||
}
|
||||
|
||||
const items = resolveItems(option, initialState) || [];
|
||||
const fallback =
|
||||
items.find((item) => item.default && !item.disabled) ||
|
||||
items.find((item) => !item.disabled) ||
|
||||
items[0];
|
||||
initialState[key] = fallback ? fallback.id : '';
|
||||
}
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, quantization, strategy, thinking, toolcall } = values;
|
||||
const strategyValues = Array.isArray(strategy) ? strategy : [];
|
||||
|
||||
if ((hardware === 'h100' || hardware === 'mi300x') && quantization === 'fp4') {
|
||||
return '# Error: H100 and MI300X only support FP8 quantization';
|
||||
}
|
||||
|
||||
const modelPath =
|
||||
quantization === 'fp4'
|
||||
? 'nvidia/DeepSeek-R1-0528-FP4-v2'
|
||||
: 'deepseek-ai/DeepSeek-R1-0528';
|
||||
|
||||
let command = 'python3 -m sglang.launch_server \\\n';
|
||||
command += ` --model-path ${modelPath}`;
|
||||
|
||||
if (strategyValues.includes('tp')) {
|
||||
command += ' \\\n --tp 8';
|
||||
}
|
||||
if (strategyValues.includes('dp')) {
|
||||
command += ' \\\n --dp 8 \\\n --enable-dp-attention';
|
||||
}
|
||||
if (strategyValues.includes('ep')) {
|
||||
command += ' \\\n --ep 8';
|
||||
}
|
||||
if (strategyValues.includes('mtp')) {
|
||||
command = 'SGLANG_ENABLE_SPEC_V2=1 ' + command;
|
||||
command +=
|
||||
' \\\n --speculative-algorithm EAGLE' +
|
||||
' \\\n --speculative-num-steps 3' +
|
||||
' \\\n --speculative-eagle-topk 1' +
|
||||
' \\\n --speculative-num-draft-tokens 4';
|
||||
}
|
||||
|
||||
command += ' \\\n --enable-symm-mem # Optional: improves performance, but may be unstable';
|
||||
|
||||
if (hardware === 'b200' || (hardware === 'mi355x' && quantization === 'fp8')) {
|
||||
command +=
|
||||
' \\\n --kv-cache-dtype fp8_e4m3 # Optional: enables fp8 kv cache and fp8 attention kernels to improve performance';
|
||||
}
|
||||
|
||||
if (thinking === 'enabled') {
|
||||
command += ' \\\n --reasoning-parser deepseek-r1';
|
||||
}
|
||||
if (toolcall === 'enabled') {
|
||||
command +=
|
||||
' \\\n --tool-call-parser deepseekv3' +
|
||||
' \\\n --chat-template examples/chat_template/tool_chat_template_deepseekr1.jinja';
|
||||
}
|
||||
|
||||
return command;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => {
|
||||
const next = { ...prev, [optionName]: value };
|
||||
if (optionName === 'hardware') {
|
||||
const quantizationItems = resolveItems(options.quantization, next);
|
||||
const current = quantizationItems.find((item) => item.id === next.quantization);
|
||||
if (!current || current.disabled) {
|
||||
const fallback =
|
||||
quantizationItems.find((item) => item.default && !item.disabled) ||
|
||||
quantizationItems.find((item) => !item.disabled);
|
||||
if (fallback) {
|
||||
next.quantization = fallback.id;
|
||||
}
|
||||
}
|
||||
}
|
||||
return next;
|
||||
});
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,186 @@
|
||||
export const DeepSeekV3Deployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'fp8', label: 'FP8', default: true },
|
||||
{ id: 'fp4', label: 'FP4', default: false }
|
||||
]
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true },
|
||||
{ id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false },
|
||||
{ id: 'ep', label: 'EP', subtitle: 'Expert Parallel', default: false },
|
||||
{ id: 'mtp', label: 'MTP', subtitle: 'Multi-token Prediction', default: false }
|
||||
]
|
||||
},
|
||||
thinking: {
|
||||
name: 'thinking',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = option.items.filter(item => item.default).map(item => item.id);
|
||||
} else {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
}
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode - prioritize page theme over system preference
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
// Check Mintlify's theme class on html element
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues(prev => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
} else {
|
||||
return { ...prev, [optionName]: currentValues.filter(id => id !== itemId) };
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, quantization, strategy, thinking, toolcall } = values;
|
||||
const strategyArray = Array.isArray(strategy) ? strategy : [];
|
||||
|
||||
// Validation - H100/H200/MI300X/MI325X only supports FP8
|
||||
if (['h100', 'h200', 'mi300x', 'mi325x'].includes(hardware) && quantization === 'fp4') {
|
||||
return '# Error: This hardware only supports FP8 quantization\n# Please select FP8 quantization or use B200/MI355X hardware';
|
||||
}
|
||||
|
||||
const modelPath = quantization === 'fp4' ? 'nvidia/DeepSeek-V3-0324-NVFP4' : 'deepseek-ai/DeepSeek-V3';
|
||||
|
||||
let cmd = 'python3 -m sglang.launch_server \\\n';
|
||||
cmd += ` --model-path ${modelPath}`;
|
||||
|
||||
if (strategyArray.includes('tp')) cmd += ' \\\n --tp 8';
|
||||
if (strategyArray.includes('dp')) cmd += ' \\\n --dp 8 \\\n --enable-dp-attention';
|
||||
if (strategyArray.includes('ep')) cmd += ' \\\n --ep 8';
|
||||
if (strategyArray.includes('mtp')) {
|
||||
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
|
||||
cmd += ' \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4';
|
||||
}
|
||||
|
||||
cmd += ' \\\n --enable-symm-mem # Optional: improves performance, but may be unstable';
|
||||
|
||||
if (hardware === 'b200') {
|
||||
cmd += ' \\\n --kv-cache-dtype fp8_e4m3 # Optional: enables fp8 kv cache and fp8 attention kernels';
|
||||
}
|
||||
|
||||
if (thinking === 'enabled') cmd += ' \\\n --reasoning-parser deepseek-v3';
|
||||
if (toolcall === 'enabled') cmd += ' \\\n --tool-call-parser deepseekv3 \\\n --chat-template examples/chat_template/tool_chat_template_deepseekv3.jinja';
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,197 @@
|
||||
export const DeepSeekV31Deployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
modelname: {
|
||||
name: 'modelname',
|
||||
title: 'Model Name',
|
||||
items: [
|
||||
{ id: 'v31', label: 'DeepSeek-V3.1', default: true },
|
||||
{ id: 'v31terminus', label: 'DeepSeek-V3.1-Terminus', default: false }
|
||||
]
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'tp', label: 'TP', default: true, required: true },
|
||||
{ id: 'dp', label: 'DP attention', default: false },
|
||||
{ id: 'ep', label: 'EP', default: false },
|
||||
{ id: 'mtp', label: 'Multi-token Prediction', default: false }
|
||||
]
|
||||
},
|
||||
reasoningParser: {
|
||||
name: 'reasoningParser',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = option.items.filter(item => item.default).map(item => item.id);
|
||||
} else {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
}
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode - prioritize page theme over system preference
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
// Check Mintlify's theme class on html element
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues(prev => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
} else {
|
||||
return { ...prev, [optionName]: currentValues.filter(id => id !== itemId) };
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, modelname, strategy, reasoningParser, toolcall } = values;
|
||||
const strategyArray = Array.isArray(strategy) ? strategy : [];
|
||||
|
||||
// Model name mapping
|
||||
const modelMap = {
|
||||
'v31': 'DeepSeek-V3.1',
|
||||
'v31terminus': 'DeepSeek-V3.1-Terminus'
|
||||
};
|
||||
|
||||
const modelName = `deepseek-ai/${modelMap[modelname]}`;
|
||||
|
||||
let cmd = 'python3 -m sglang.launch_server \\\n';
|
||||
cmd += ` --model-path ${modelName}`;
|
||||
|
||||
// TP is mandatory
|
||||
cmd += ` \\\n --tp 8`;
|
||||
if (strategyArray.includes('dp')) {
|
||||
cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`;
|
||||
}
|
||||
if (strategyArray.includes('ep')) {
|
||||
cmd += ` \\\n --ep 8`;
|
||||
}
|
||||
// Multi-token prediction (MTP) configuration
|
||||
if (strategyArray.includes('mtp')) {
|
||||
cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
|
||||
}
|
||||
|
||||
// Add tool-call-parser if enabled
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --tool-call-parser deepseekv31`;
|
||||
}
|
||||
|
||||
// Add reasoning-parser when enabled
|
||||
if (reasoningParser === 'enabled') {
|
||||
cmd += ` \\\n --reasoning-parser deepseek-v3`;
|
||||
}
|
||||
|
||||
// Add chat-template if tool calling is enabled
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --chat-template ./examples/chat_template/tool_chat_template_deepseekv31.jinja`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,318 @@
|
||||
export const DeepSeekV32Deployment = () => {
|
||||
// Config mirrors sgl-cookbook src/components/autoregressive/DeepSeekConfigGenerator/index.js.
|
||||
//
|
||||
// Model variants:
|
||||
// DeepSeek-V3.2, V3.2-Exp, V3.2-Speciale → deepseek-ai/ family, TP=8
|
||||
// DeepSeek-V3.2-NVFP4 → nvidia/ family, B200 only, TP=4
|
||||
// DeepSeek-V3.2-MXFP4 → amd/ family, MI300X/MI355X only, TP=8
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
modelname: {
|
||||
name: 'modelname',
|
||||
title: 'Model Name',
|
||||
getDynamicItems: (values) => {
|
||||
const hw = values.hardware;
|
||||
const isB200 = hw === 'b200';
|
||||
const isAMD = hw === 'mi300x' || hw === 'mi355x';
|
||||
return [
|
||||
{ id: 'v32', label: 'DeepSeek-V3.2', default: !isB200 && !isAMD },
|
||||
{ id: 'v32speciale', label: 'DeepSeek-V3.2-Speciale', default: false },
|
||||
{ id: 'v32exp', label: 'DeepSeek-V3.2-Exp', default: false },
|
||||
{ id: 'v32nvfp4', label: 'DeepSeek-V3.2-NVFP4', default: isB200, disabled: !isB200, disabledReason: 'NVFP4 requires B200 (Blackwell)' },
|
||||
{ id: 'v32mxfp4', label: 'DeepSeek-V3.2-MXFP4', default: isAMD, disabled: !isAMD, disabledReason: 'MXFP4 requires AMD MI300X/MI355X' }
|
||||
];
|
||||
}
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
condition: (values) => values.modelname !== 'v32nvfp4' && values.modelname !== 'v32mxfp4',
|
||||
items: [
|
||||
{ id: 'tp', label: 'TP', default: true, required: true },
|
||||
{ id: 'dp', label: 'DP attention', default: false },
|
||||
{ id: 'ep', label: 'EP', default: false },
|
||||
{ id: 'mtp', label: 'Multi-token Prediction', default: false }
|
||||
]
|
||||
},
|
||||
reasoningParser: {
|
||||
name: 'reasoningParser',
|
||||
title: 'Reasoning Parser',
|
||||
condition: (values) => values.modelname !== 'v32nvfp4' && values.modelname !== 'v32mxfp4',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
condition: (values) => values.modelname !== 'v32nvfp4' && values.modelname !== 'v32mxfp4' && values.modelname !== 'v32speciale',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
const resolveItems = (option, vals) => {
|
||||
if (typeof option.getDynamicItems === 'function') return option.getDynamicItems(vals);
|
||||
return option.items;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (option.type === 'checkbox') {
|
||||
const items = resolveItems(option, initialState);
|
||||
initialState[key] = items.filter(i => i.default).map(i => i.id);
|
||||
} else {
|
||||
const items = resolveItems(option, initialState);
|
||||
const def = items.find(i => i.default && !i.disabled) || items.find(i => !i.disabled) || items[0];
|
||||
initialState[key] = def.id;
|
||||
}
|
||||
}
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
// When hardware changes, re-resolve model name defaults (NVFP4→B200, MXFP4→AMD).
|
||||
useEffect(() => {
|
||||
setValues(prev => {
|
||||
const next = { ...prev };
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (typeof option.getDynamicItems !== 'function') continue;
|
||||
const items = option.getDynamicItems(next);
|
||||
const current = items.find(i => i.id === next[key]);
|
||||
if (!current || current.disabled) {
|
||||
const fallback = items.find(i => i.default && !i.disabled) || items.find(i => !i.disabled);
|
||||
if (fallback) next[key] = fallback.id;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}, [values.hardware]);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues(prev => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
} else {
|
||||
return { ...prev, [optionName]: currentValues.filter(id => id !== itemId) };
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const generateCommand = () => {
|
||||
const { hardware, modelname, strategy, reasoningParser, toolcall } = values;
|
||||
|
||||
const isNvfp4 = modelname === 'v32nvfp4';
|
||||
const isMxfp4 = modelname === 'v32mxfp4';
|
||||
const isAMD = hardware === 'mi300x' || hardware === 'mi355x';
|
||||
|
||||
// Validation: NVFP4 requires B200
|
||||
if (isNvfp4 && hardware !== 'b200') {
|
||||
return `# Error: DeepSeek-V3.2-NVFP4 requires NVIDIA B200 (Blackwell) hardware\n# Please select "B200" for Hardware Platform or choose a different model`;
|
||||
}
|
||||
|
||||
// Validation: MXFP4 requires AMD MI300X/MI355X
|
||||
if (isMxfp4 && !isAMD) {
|
||||
return `# Error: DeepSeek-V3.2-MXFP4 requires AMD MI300X/MI355X hardware\n# Please select "MI300X" or "MI355X" for Hardware Platform or choose a different model`;
|
||||
}
|
||||
|
||||
// Validation: Speciale doesn't support tool calling
|
||||
if (modelname === 'v32speciale' && toolcall === 'enabled') {
|
||||
return `# Error: DeepSeek-V3.2-Speciale doesn't support tool calling\n# Please select "Disabled" for Tool Call Parser or choose a different model`;
|
||||
}
|
||||
|
||||
// Model name mapping
|
||||
const modelMap = {
|
||||
'v32': 'DeepSeek-V3.2',
|
||||
'v32exp': 'DeepSeek-V3.2-Exp',
|
||||
'v32speciale': 'DeepSeek-V3.2-Speciale',
|
||||
'v32nvfp4': 'DeepSeek-V3.2-NVFP4',
|
||||
'v32mxfp4': 'DeepSeek-V3.2-mxfp4'
|
||||
};
|
||||
|
||||
let modelFamily;
|
||||
if (isNvfp4) modelFamily = 'nvidia';
|
||||
else if (isMxfp4) modelFamily = 'amd';
|
||||
else modelFamily = 'deepseek-ai';
|
||||
|
||||
const modelName = `${modelFamily}/${modelMap[modelname]}`;
|
||||
|
||||
// NVFP4: fixed config
|
||||
if (isNvfp4) {
|
||||
let cmd = 'sglang serve \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
cmd += ' \\\n --tp 4';
|
||||
cmd += ' \\\n --quantization modelopt_fp4';
|
||||
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm';
|
||||
return cmd;
|
||||
}
|
||||
|
||||
// MXFP4: fixed config for AMD
|
||||
if (isMxfp4) {
|
||||
let cmd = 'sglang serve \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
cmd += ' \\\n --tp 8';
|
||||
cmd += ' \\\n --trust-remote-code';
|
||||
return cmd;
|
||||
}
|
||||
|
||||
let cmd = 'sglang serve \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
// Hardware platform specific parameters
|
||||
if (isAMD) {
|
||||
cmd += ' \\\n --trust-remote-code';
|
||||
cmd += ' \\\n --nsa-prefill-backend tilelang';
|
||||
cmd += ' \\\n --nsa-decode-backend tilelang';
|
||||
cmd += ' \\\n --cuda-graph-max-bs 64';
|
||||
}
|
||||
|
||||
// Strategy configurations
|
||||
const strategyArray = Array.isArray(strategy) ? strategy : [];
|
||||
const tpSize = 8;
|
||||
const dpSize = 8;
|
||||
const epSize = 8;
|
||||
cmd += ` \\\n --tp ${tpSize}`;
|
||||
if (strategyArray.includes('dp')) {
|
||||
cmd += ` \\\n --dp ${dpSize} \\\n --enable-dp-attention`;
|
||||
}
|
||||
if (strategyArray.includes('ep')) {
|
||||
cmd += ` \\\n --ep ${epSize}`;
|
||||
}
|
||||
|
||||
// Multi-token prediction (MTP) configuration
|
||||
if (strategyArray.includes('mtp')) {
|
||||
cmd += ' \\\n --speculative-algorithm EAGLE';
|
||||
cmd += ' \\\n --speculative-num-steps 3';
|
||||
cmd += ' \\\n --speculative-eagle-topk 1';
|
||||
cmd += ' \\\n --speculative-num-draft-tokens 4';
|
||||
}
|
||||
|
||||
// Add tool-call-parser if enabled (not supported for Speciale)
|
||||
if (toolcall === 'enabled' && modelname !== 'v32speciale') {
|
||||
if (modelname === 'v32exp') {
|
||||
cmd += ' \\\n --tool-call-parser deepseekv31';
|
||||
} else if (modelname === 'v32') {
|
||||
cmd += ' \\\n --tool-call-parser deepseekv32';
|
||||
}
|
||||
}
|
||||
|
||||
// Add reasoning-parser when enabled
|
||||
if (reasoningParser === 'enabled') {
|
||||
cmd += ' \\\n --reasoning-parser deepseek-v3';
|
||||
}
|
||||
|
||||
// Add chat-template if tool calling is enabled (only for v32exp)
|
||||
if (toolcall === 'enabled' && modelname === 'v32exp') {
|
||||
cmd += ' \\\n --chat-template ./examples/chat_template/tool_chat_template_deepseekv32.jinja';
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.4 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (typeof option.condition === 'function' && !option.condition(values)) return null;
|
||||
const items = resolveItems(option, values);
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.required || !!item.disabled;
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? { ...disabledStyle, ...(item.required ? {} : {}) } : {}) }}
|
||||
title={item.disabledReason || ''}
|
||||
>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = !!item.disabled;
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}
|
||||
title={item.disabledReason || ''}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,182 @@
|
||||
export const Devstral2Deployment = () => {
|
||||
// Config options based on Devstral2ConfigGenerator
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
model: {
|
||||
name: 'model',
|
||||
title: 'Model',
|
||||
items: [
|
||||
{ id: 'small', label: 'Devstral Small 2 (24B)', default: true },
|
||||
{ id: 'large', label: 'Devstral 2 (123B)', default: false }
|
||||
]
|
||||
},
|
||||
weights: {
|
||||
name: 'weights',
|
||||
title: 'Weights / Precision',
|
||||
items: [
|
||||
{ id: 'fp8', label: 'FP8', default: true }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Model configurations
|
||||
const modelConfigs = {
|
||||
small: {
|
||||
modelId: 'mistralai/Devstral-Small-2-24B-Instruct-2512',
|
||||
tpByHardware: { h100: 1, h200: 1, b200: 1, mi300x: 1, mi325x: 1, mi355x: 1 },
|
||||
allowedWeights: ['fp8']
|
||||
},
|
||||
large: {
|
||||
modelId: 'mistralai/Devstral-2-123B-Instruct-2512',
|
||||
tpByHardware: { h100: 4, h200: 2, b200: 2, mi300x: 2, mi325x: 2, mi355x: 2 },
|
||||
allowedWeights: ['fp8']
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = option.items.filter(item => item.default).map(item => item.id);
|
||||
} else {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
}
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues(prev => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
} else {
|
||||
return { ...prev, [optionName]: currentValues.filter(id => id !== itemId) };
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, model, weights, toolcall } = values;
|
||||
|
||||
const modelCfg = modelConfigs[model];
|
||||
if (!modelCfg) return `# Error: Unknown model selection: ${model}`;
|
||||
|
||||
if (!modelCfg.allowedWeights.includes(weights)) {
|
||||
const allowed = modelCfg.allowedWeights.map(w => w.toUpperCase()).join(', ');
|
||||
return `# Error: ${modelCfg.modelId} only supports: ${allowed}\n# Please change "Weights / Precision" to a supported value.`;
|
||||
}
|
||||
|
||||
const tp = modelCfg.tpByHardware[hardware];
|
||||
if (!tp) return `# Error: Unknown hardware platform: ${hardware}`;
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelCfg.modelId}`;
|
||||
|
||||
if (tp > 1) {
|
||||
cmd += ` \\\n --tp ${tp}`;
|
||||
}
|
||||
|
||||
// Add tool-call-parser if enabled
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --tool-call-parser mistral`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,345 @@
|
||||
export const Ernie45Deployment = () => {
|
||||
const options = {
|
||||
modelsize: {
|
||||
name: 'modelsize',
|
||||
title: 'Model Size',
|
||||
items: [
|
||||
{ id: '21b', label: '21B', subtitle: 'A3B', default: true },
|
||||
{ id: '300b', label: '300B', subtitle: 'A47B', default: false }
|
||||
]
|
||||
},
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'mi300x', label: 'MI300X', default: true },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true },
|
||||
{ id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false, disabledWhen: (values) => values.modelsize === '21b' },
|
||||
{ id: 'ep', label: 'EP', subtitle: 'Expert Parallel', default: false, disabledWhen: (values) => values.modelsize === '21b' }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { modelsize, hardware, strategy } = values;
|
||||
|
||||
const strategyArray = Array.isArray(strategy) ? strategy : [];
|
||||
|
||||
let modelPath;
|
||||
if (modelsize === '21b') {
|
||||
modelPath = 'baidu/ERNIE-4.5-21B-A3B-PT';
|
||||
} else if (modelsize === '300b') {
|
||||
modelPath = 'baidu/ERNIE-4.5-300B-A47B-PT';
|
||||
} else {
|
||||
modelPath = 'baidu/ERNIE-4.5-21B-A3B-PT';
|
||||
}
|
||||
|
||||
let cmd = 'python3 -m sglang.launch_server \\\n';
|
||||
cmd += ` --model-path ${modelPath}`;
|
||||
|
||||
const tpValue = modelsize === '300b' ? 8 : 1;
|
||||
const dpValue = modelsize === '300b' ? 8 : null;
|
||||
const epValue = modelsize === '300b' ? 8 : null;
|
||||
|
||||
if (strategyArray.includes('tp')) {
|
||||
cmd += ` \\\n --tp ${tpValue}`;
|
||||
}
|
||||
|
||||
if (strategyArray.includes('dp') && modelsize === '300b') {
|
||||
cmd += ` \\\n --dp ${dpValue} \\\n --enable-dp-attention`;
|
||||
}
|
||||
|
||||
if (strategyArray.includes('ep') && modelsize === '300b') {
|
||||
cmd += ` \\\n --ep ${epValue}`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,376 @@
|
||||
export const Gemma4Deployment = () => {
|
||||
const options = {
|
||||
modelSize: {
|
||||
name: 'modelSize',
|
||||
title: 'Model Variant',
|
||||
items: [
|
||||
{ id: 'e2b', label: 'E2B (~2B)', default: false },
|
||||
{ id: 'e4b', label: 'E4B (~4B)', default: true },
|
||||
{ id: '31b', label: '31B (Dense)', default: false },
|
||||
{ id: '26b-a4b', label: '26B-A4B (MoE)', default: false },
|
||||
]
|
||||
},
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
getDynamicItems: (values) => {
|
||||
const size = values.modelSize;
|
||||
const showMI300X = size === '31b' || size === '26b-a4b';
|
||||
return [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false, disabled: !showMI300X },
|
||||
];
|
||||
}
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--reasoning-parser gemma4' : null
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--tool-call-parser gemma4' : null
|
||||
},
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
h200: {
|
||||
e2b: { tp: 1, mem: 0.85 },
|
||||
e4b: { tp: 1, mem: 0.85 },
|
||||
'31b': { tp: 2, mem: 0.85 },
|
||||
'26b-a4b': { tp: 1, mem: 0.85 },
|
||||
},
|
||||
b200: {
|
||||
e2b: { tp: 1, mem: 0.9 },
|
||||
e4b: { tp: 1, mem: 0.9 },
|
||||
'31b': { tp: 1, mem: 0.9 },
|
||||
'26b-a4b': { tp: 1, mem: 0.9 },
|
||||
},
|
||||
mi300x: {
|
||||
'31b': { tp: 1, mem: 0.80 },
|
||||
'26b-a4b': { tp: 1, mem: 0.80 },
|
||||
},
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, modelSize } = values;
|
||||
|
||||
const hwConfig = modelConfigs[hardware]?.[modelSize];
|
||||
if (!hwConfig) return `# Error: Unknown hardware/model combination`;
|
||||
|
||||
const { tp, mem } = hwConfig;
|
||||
|
||||
const modelNames = {
|
||||
'e2b': 'google/gemma-4-E2B-it',
|
||||
'e4b': 'google/gemma-4-E4B-it',
|
||||
'31b': 'google/gemma-4-31B-it',
|
||||
'26b-a4b': 'google/gemma-4-26B-A4B-it',
|
||||
};
|
||||
|
||||
let cmd = `sglang serve --model-path ${modelNames[modelSize]}`;
|
||||
if (tp > 1) {
|
||||
cmd += ` \\\n --tp ${tp}`;
|
||||
}
|
||||
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (key === 'modelSize' || key === 'hardware') return;
|
||||
if (option.commandRule) {
|
||||
const rule = option.commandRule(values[key]);
|
||||
if (rule) cmd += ` \\\n ${rule}`;
|
||||
}
|
||||
});
|
||||
|
||||
cmd += ` \\\n --mem-fraction-static ${mem}`;
|
||||
cmd += ` \\\n --host 0.0.0.0 --port 30000`;
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,197 @@
|
||||
export const GLM45Deployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'mi300x', label: 'MI300X', default: true },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false }
|
||||
]
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true },
|
||||
{ id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false },
|
||||
{ id: 'ep', label: 'EP', subtitle: 'Expert Parallel', default: false },
|
||||
{ id: 'mtp', label: 'MTP', subtitle: 'Multi-token Prediction', default: false }
|
||||
]
|
||||
},
|
||||
thinking: {
|
||||
name: 'thinking',
|
||||
title: 'Thinking Capabilities',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = option.items.filter(item => item.default).map(item => item.id);
|
||||
} else {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
}
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues(prev => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
} else {
|
||||
return { ...prev, [optionName]: currentValues.filter(id => id !== itemId) };
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, quantization, strategy, thinking, toolcall } = values;
|
||||
const strategyArray = Array.isArray(strategy) ? strategy : [];
|
||||
|
||||
const modelSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
const modelName = `zai-org/GLM-4.5${modelSuffix}`;
|
||||
|
||||
// Determine TP value based on hardware and quantization
|
||||
let tpValue = 4; // Default for MI300X/MI325X
|
||||
if (hardware === 'mi355x') {
|
||||
tpValue = quantization === 'fp8' ? 2 : 4; // MI355X: TP=2 for FP8, TP=4 for BF16
|
||||
}
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
// TP is mandatory
|
||||
cmd += ` \\\n --tp ${tpValue}`;
|
||||
|
||||
// MI300X/MI325X BF16 requires extra flags
|
||||
if ((hardware === 'mi300x' || hardware === 'mi325x') && quantization === 'bf16') {
|
||||
cmd += ` \\\n --max-context-length 8192 \\\n --mem-fraction-static 0.9`;
|
||||
}
|
||||
|
||||
// Strategy-specific parameters
|
||||
if (strategyArray.includes('dp')) {
|
||||
cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`;
|
||||
}
|
||||
if (strategyArray.includes('ep')) {
|
||||
cmd += ` \\\n --ep 8`;
|
||||
}
|
||||
if (strategyArray.includes('mtp')) {
|
||||
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
|
||||
cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
|
||||
}
|
||||
|
||||
// Add tool call parser if enabled
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --tool-call-parser glm45`;
|
||||
}
|
||||
|
||||
// Add thinking parser if enabled
|
||||
if (thinking === 'enabled') {
|
||||
cmd += ` \\\n --reasoning-parser glm45`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,175 @@
|
||||
export const GLM45VDeployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false }
|
||||
]
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
{ id: 'disabled', label: 'Disabled', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
{ id: 'disabled', label: 'Disabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, quantization, reasoning, toolcall } = values;
|
||||
|
||||
// Model configuration
|
||||
const config = {
|
||||
baseName: 'GLM-4.5V',
|
||||
b200: { tp: 4 },
|
||||
h100: { tp: 4 },
|
||||
h200: { tp: 4 },
|
||||
mi300x: { tp: 4 },
|
||||
mi355x: { tp: 4 }
|
||||
};
|
||||
|
||||
const hwConfig = config[hardware];
|
||||
if (!hwConfig) {
|
||||
return `# Error: Unknown hardware platform: ${hardware}`;
|
||||
}
|
||||
|
||||
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
const modelName = `zai-org/${config.baseName}${quantSuffix}`;
|
||||
|
||||
// Check if AMD hardware
|
||||
const isAMD = ['mi300x', 'mi325x', 'mi355x'].includes(hardware);
|
||||
|
||||
let cmd = '';
|
||||
if (isAMD) {
|
||||
cmd = 'SGLANG_USE_AITER=0 python3 -m sglang.launch_server \\\n';
|
||||
cmd += ` --model-path ${modelName}`;
|
||||
cmd += ` \\\n --tp-size ${hwConfig.tp}`;
|
||||
} else {
|
||||
cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
if (hwConfig.tp > 1) {
|
||||
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
||||
}
|
||||
}
|
||||
|
||||
// Add reasoning parser
|
||||
if (reasoning === 'enabled') {
|
||||
cmd += ' \\\n --reasoning-parser glm45';
|
||||
}
|
||||
|
||||
// Add tool call parser
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ' \\\n --tool-call-parser glm45';
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,207 @@
|
||||
export const GLM46Deployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h100', label: 'H100', default: true },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false }
|
||||
]
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true },
|
||||
{ id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false },
|
||||
{ id: 'ep', label: 'EP', subtitle: 'Expert Parallel', default: false },
|
||||
{ id: 'mtp', label: 'MTP', subtitle: 'Multi-token Prediction', default: false }
|
||||
]
|
||||
},
|
||||
thinking: {
|
||||
name: 'thinking',
|
||||
title: 'Thinking Capabilities',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = option.items.filter(item => item.default).map(item => item.id);
|
||||
} else {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
}
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues(prev => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
} else {
|
||||
return { ...prev, [optionName]: currentValues.filter(id => id !== itemId) };
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, quantization, strategy, thinking, toolcall } = values;
|
||||
const strategyArray = Array.isArray(strategy) ? strategy : [];
|
||||
|
||||
// Check for H100 + BF16 error
|
||||
if (hardware === 'h100' && quantization === 'bf16') {
|
||||
return '# Error: GLM-4.6 in BF16 precision requires more VRAM than 8*H100\n# Please use H200/B200 or select FP8 quantization';
|
||||
}
|
||||
|
||||
const modelSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
const modelName = `zai-org/GLM-4.6${modelSuffix}`;
|
||||
|
||||
// Determine TP value based on hardware and quantization
|
||||
let tpValue = 8; // Default for NVIDIA GPUs
|
||||
if (hardware === 'mi300x' || hardware === 'mi325x') {
|
||||
tpValue = 4; // MI300X/MI325X: TP=4 for both BF16 and FP8
|
||||
} else if (hardware === 'mi355x') {
|
||||
tpValue = quantization === 'fp8' ? 2 : 4; // MI355X: TP=2 for FP8, TP=4 for BF16
|
||||
}
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
// TP is mandatory
|
||||
cmd += ` \\\n --tp ${tpValue}`;
|
||||
|
||||
// MI300X/MI325X BF16 requires extra flags
|
||||
if ((hardware === 'mi300x' || hardware === 'mi325x') && quantization === 'bf16') {
|
||||
cmd += ` \\\n --max-context-length 8192 \\\n --mem-fraction-static 0.9`;
|
||||
}
|
||||
|
||||
// Strategy-specific parameters
|
||||
if (strategyArray.includes('dp')) {
|
||||
cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`;
|
||||
}
|
||||
if (strategyArray.includes('ep')) {
|
||||
cmd += ` \\\n --ep 8`;
|
||||
}
|
||||
if (strategyArray.includes('mtp')) {
|
||||
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
|
||||
cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
|
||||
}
|
||||
|
||||
// Add tool call parser if enabled
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --tool-call-parser glm45`;
|
||||
}
|
||||
|
||||
// Add thinking parser if enabled
|
||||
if (thinking === 'enabled') {
|
||||
cmd += ` \\\n --reasoning-parser glm45`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,196 @@
|
||||
export const GLM46VDeployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
modelsize: {
|
||||
name: 'modelsize',
|
||||
title: 'Model Size',
|
||||
items: [
|
||||
{ id: '106b', label: '106B', subtitle: 'GLM-4.6V', default: true },
|
||||
{ id: '9b', label: '9B', subtitle: 'GLM-4.6V-Flash', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false }
|
||||
]
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
{ id: 'disabled', label: 'Disabled', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
{ id: 'disabled', label: 'Disabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = option.items.filter(item => item.default).map(item => item.id);
|
||||
} else {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
}
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, modelsize, quantization, reasoning, toolcall } = values;
|
||||
|
||||
// Model configurations
|
||||
const modelConfigs = {
|
||||
'106b': {
|
||||
baseName: 'GLM-4.6V',
|
||||
h100: { tp: 8 },
|
||||
h200: { tp: 8 },
|
||||
b200: { tp: 8 },
|
||||
mi300x: { tp: 8 },
|
||||
mi355x: { tp: 8 }
|
||||
},
|
||||
'9b': {
|
||||
baseName: 'GLM-4.6V-Flash',
|
||||
h100: { tp: 1 },
|
||||
h200: { tp: 1 },
|
||||
b200: { tp: 1 },
|
||||
mi300x: { tp: 1 },
|
||||
mi355x: { tp: 1 }
|
||||
}
|
||||
};
|
||||
|
||||
const config = modelConfigs[modelsize];
|
||||
if (!config) {
|
||||
return `# Error: Unknown model size: ${modelsize}`;
|
||||
}
|
||||
|
||||
const hwConfig = config[hardware];
|
||||
if (!hwConfig) {
|
||||
return `# Error: Unknown hardware platform: ${hardware}`;
|
||||
}
|
||||
|
||||
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
const modelName = `zai-org/${config.baseName}${quantSuffix}`;
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
if (hwConfig.tp > 1) {
|
||||
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
||||
if (hwConfig.tp === 8) {
|
||||
cmd += ` \\\n --mm-enable-dp-encoder`;
|
||||
}
|
||||
}
|
||||
|
||||
// Add reasoning parser if enabled
|
||||
if (reasoning === 'enabled') {
|
||||
cmd += ` \\\n --reasoning-parser glm45`;
|
||||
}
|
||||
|
||||
// Add tool call parser if enabled
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --tool-call-parser glm45`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,197 @@
|
||||
export const GLM47Deployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'mi300x', label: 'MI300X', default: true },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false }
|
||||
]
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true },
|
||||
{ id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false },
|
||||
{ id: 'ep', label: 'EP', subtitle: 'Expert Parallel', default: false },
|
||||
{ id: 'mtp', label: 'MTP', subtitle: 'Multi-token Prediction', default: false }
|
||||
]
|
||||
},
|
||||
thinking: {
|
||||
name: 'thinking',
|
||||
title: 'Thinking Capabilities',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = option.items.filter(item => item.default).map(item => item.id);
|
||||
} else {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
}
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues(prev => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
} else {
|
||||
return { ...prev, [optionName]: currentValues.filter(id => id !== itemId) };
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, quantization, strategy, thinking, toolcall } = values;
|
||||
const strategyArray = Array.isArray(strategy) ? strategy : [];
|
||||
|
||||
const modelSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
const modelName = `zai-org/GLM-4.7${modelSuffix}`;
|
||||
|
||||
// Determine TP value based on hardware and quantization
|
||||
let tpValue = 4; // Default for MI300X and MI325X
|
||||
if (hardware === 'mi355x') {
|
||||
tpValue = quantization === 'fp8' ? 2 : 4; // MI355X: TP=2 for FP8, TP=4 for BF16
|
||||
}
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
// TP is mandatory
|
||||
cmd += ` \\\n --tp ${tpValue}`;
|
||||
|
||||
// MI300X/MI325X BF16 requires extra flags
|
||||
if ((hardware === 'mi300x' || hardware === 'mi325x') && quantization === 'bf16') {
|
||||
cmd += ` \\\n --max-context-length 8192 \\\n --mem-fraction-static 0.9`;
|
||||
}
|
||||
|
||||
// Strategy-specific parameters
|
||||
if (strategyArray.includes('dp')) {
|
||||
cmd += ` \\\n --dp 8 \\\n --enable-dp-attention`;
|
||||
}
|
||||
if (strategyArray.includes('ep')) {
|
||||
cmd += ` \\\n --ep 8`;
|
||||
}
|
||||
if (strategyArray.includes('mtp')) {
|
||||
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
|
||||
cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
|
||||
}
|
||||
|
||||
// Add tool call parser if enabled
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --tool-call-parser glm47`;
|
||||
}
|
||||
|
||||
// Add thinking parser if enabled
|
||||
if (thinking === 'enabled') {
|
||||
cmd += ` \\\n --reasoning-parser glm47`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,191 @@
|
||||
export const GLM47FlashDeployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h100', label: 'H100', default: true },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'b200', label: 'B200', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true }
|
||||
]
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'tp', label: 'TP', subtitle: 'Tensor Parallel', default: true, required: true },
|
||||
{ id: 'dp', label: 'DP', subtitle: 'Data Parallel', default: false },
|
||||
{ id: 'mtp', label: 'MTP', subtitle: 'Multi-token Prediction', default: false }
|
||||
]
|
||||
},
|
||||
thinking: {
|
||||
name: 'thinking',
|
||||
title: 'Thinking Capabilities',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = option.items.filter(item => item.default).map(item => item.id);
|
||||
} else {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
}
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues(prev => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
} else {
|
||||
return { ...prev, [optionName]: currentValues.filter(id => id !== itemId) };
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, quantization, strategy, thinking, toolcall } = values;
|
||||
const strategyArray = Array.isArray(strategy) ? strategy : [];
|
||||
|
||||
const modelName = `zai-org/GLM-4.7-Flash`;
|
||||
|
||||
// GLM-4.7-Flash is a 30B-A3B MoE model, lighter than GLM-4.7
|
||||
const tpValue = 1; // Default for single GPU
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n ';
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
// TP is mandatory
|
||||
cmd += ` \\\n --tp ${tpValue}`;
|
||||
|
||||
if (hardware === 'b200') {
|
||||
cmd += ` \\\n --attention-backend triton`;
|
||||
}
|
||||
|
||||
// Strategy-specific parameters
|
||||
if (strategyArray.includes('dp')) {
|
||||
cmd += ` \\\n --dp 1 \\\n --enable-dp-attention`;
|
||||
}
|
||||
if (strategyArray.includes('mtp')) {
|
||||
cmd = 'SGLANG_ENABLE_SPEC_V2=1 ' + cmd;
|
||||
|
||||
if (hardware === 'b200') {
|
||||
cmd += ` \\\n --speculative-draft-attention-backend triton`;
|
||||
}
|
||||
cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
|
||||
}
|
||||
|
||||
// Add tool call parser if enabled
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --tool-call-parser glm47`;
|
||||
}
|
||||
|
||||
// Add thinking parser if enabled
|
||||
if (thinking === 'enabled') {
|
||||
cmd += ` \\\n --reasoning-parser glm45`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,266 @@
|
||||
export const GLM5Deployment = () => {
|
||||
// Config mirrors sgl-cookbook src/components/autoregressive/GLM5ConfigGenerator/index.js.
|
||||
//
|
||||
// Supported quantization per hardware:
|
||||
// H100 / H200 / MI300X / MI325X / MI355X → BF16 (AMD only) + FP8 (NV only)
|
||||
// B200 → NVFP4 (default), FP8, BF16
|
||||
//
|
||||
// BF16 always needs 2x GPUs compared to FP8. AMD only supports BF16.
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X/MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
getDynamicItems: (values) => {
|
||||
const hw = values.hardware;
|
||||
const isAMD = hw === 'mi300x' || hw === 'mi355x';
|
||||
const isB200 = hw === 'b200';
|
||||
return [
|
||||
{ id: 'bf16', label: 'BF16', subtitle: 'Full Weights', default: isAMD },
|
||||
{ id: 'fp8', label: 'FP8', subtitle: 'High Throughput', default: !isAMD && !isB200, disabled: isAMD, disabledReason: 'FP8 not verified on AMD' },
|
||||
{ id: 'nvfp4', label: 'NVFP4', subtitle: 'Highest Throughput', default: isB200, disabled: !isB200, disabledReason: 'NVFP4 only on B200' }
|
||||
];
|
||||
}
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
condition: (values) => values.quantization !== 'nvfp4',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
condition: (values) => values.quantization !== 'nvfp4',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
]
|
||||
},
|
||||
dpattention: {
|
||||
name: 'dpattention',
|
||||
title: 'DP Attention',
|
||||
condition: (values) => values.quantization !== 'nvfp4',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', subtitle: 'Low Latency', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', subtitle: 'High Throughput', default: false }
|
||||
]
|
||||
},
|
||||
speculative: {
|
||||
name: 'speculative',
|
||||
title: 'Speculative Decoding',
|
||||
condition: (values) => values.hardware !== 'mi300x' && values.hardware !== 'mi355x' && values.quantization !== 'nvfp4',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// BF16 always 2× the GPUs of FP8.
|
||||
const modelConfigs = {
|
||||
h100: { fp8: { tp: 16, mem: 0.85 }, bf16: { tp: 32, mem: 0.85 } },
|
||||
h200: { fp8: { tp: 8, mem: 0.85 }, bf16: { tp: 16, mem: 0.85 } },
|
||||
b200: { nvfp4: { tp: 4, mem: 0.9 }, fp8: { tp: 8, mem: 0.9 }, bf16: { tp: 16, mem: 0.9 } },
|
||||
mi300x: { bf16: { tp: 8, mem: 0.80 } },
|
||||
mi355x: { bf16: { tp: 8, mem: 0.80 } }
|
||||
};
|
||||
|
||||
const resolveItems = (option, values) => {
|
||||
if (typeof option.getDynamicItems === 'function') return option.getDynamicItems(values);
|
||||
return option.items;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
const items = resolveItems(option, initialState);
|
||||
const def = items.find(i => i.default && !i.disabled) || items.find(i => !i.disabled) || items[0];
|
||||
initialState[key] = def.id;
|
||||
}
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
// When hardware changes, re-resolve quantization (and downstream) defaults to
|
||||
// stay consistent (AMD→BF16, B200→NVFP4, etc.).
|
||||
useEffect(() => {
|
||||
setValues(prev => {
|
||||
const next = { ...prev };
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (typeof option.getDynamicItems !== 'function') continue;
|
||||
const items = option.getDynamicItems(next);
|
||||
const current = items.find(i => i.id === next[key]);
|
||||
if (!current || current.disabled) {
|
||||
const fallback = items.find(i => i.default && !i.disabled) || items.find(i => !i.disabled);
|
||||
if (fallback) next[key] = fallback.id;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}, [values.hardware]);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const generateCommand = () => {
|
||||
const { hardware, quantization } = values;
|
||||
const isAMD = hardware === 'mi300x' || hardware === 'mi355x';
|
||||
const isNVFP4 = quantization === 'nvfp4';
|
||||
const effectiveQuant = isAMD ? 'bf16' : quantization;
|
||||
|
||||
let modelName;
|
||||
if (isNVFP4) {
|
||||
modelName = 'nvidia/GLM-5-NVFP4';
|
||||
} else {
|
||||
const suffix = effectiveQuant === 'fp8' ? '-FP8' : '';
|
||||
modelName = `zai-org/GLM-5${suffix}`;
|
||||
}
|
||||
|
||||
const hwConfig = modelConfigs[hardware][effectiveQuant];
|
||||
const tpValue = hwConfig.tp;
|
||||
const memFraction = hwConfig.mem;
|
||||
|
||||
let cmd = 'sglang serve \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
cmd += ` \\\n --tp ${tpValue}`;
|
||||
|
||||
// NVFP4 B200: trtllm NSA backends, flashinfer fusion, FP8 KV cache.
|
||||
if (isNVFP4) {
|
||||
cmd += ' \\\n --trust-remote-code';
|
||||
cmd += ' \\\n --quantization modelopt_fp4';
|
||||
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
|
||||
cmd += ' \\\n --nsa-decode-backend trtllm';
|
||||
cmd += ' \\\n --nsa-prefill-backend trtllm';
|
||||
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm';
|
||||
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
|
||||
cmd += ' \\\n --enable-dp-lm-head';
|
||||
cmd += ' \\\n --disable-radix-cache';
|
||||
cmd += ' \\\n --max-prefill-tokens 32768';
|
||||
cmd += ' \\\n --chunked-prefill-size 32768';
|
||||
cmd += ` \\\n --mem-fraction-static ${memFraction}`;
|
||||
cmd += ' \\\n --scheduler-recv-interval 10';
|
||||
cmd += ' \\\n --tokenizer-worker-num 6';
|
||||
return cmd;
|
||||
}
|
||||
|
||||
// AMD-specific: NSA tilelang backend.
|
||||
if (isAMD) {
|
||||
cmd += ' \\\n --trust-remote-code';
|
||||
cmd += ' \\\n --nsa-prefill-backend tilelang';
|
||||
cmd += ' \\\n --nsa-decode-backend tilelang';
|
||||
cmd += ' \\\n --chunked-prefill-size 131072';
|
||||
cmd += ' \\\n --watchdog-timeout 1200';
|
||||
}
|
||||
|
||||
if (values.dpattention === 'enabled') {
|
||||
cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`;
|
||||
}
|
||||
if (values.reasoning === 'enabled') cmd += ' \\\n --reasoning-parser glm45';
|
||||
if (values.toolcall === 'enabled') cmd += ' \\\n --tool-call-parser glm47';
|
||||
if (values.speculative === 'enabled') {
|
||||
cmd += ' \\\n --speculative-algorithm EAGLE';
|
||||
cmd += ' \\\n --speculative-num-steps 3';
|
||||
cmd += ' \\\n --speculative-eagle-topk 1';
|
||||
cmd += ' \\\n --speculative-num-draft-tokens 4';
|
||||
}
|
||||
|
||||
// B200 FP8: consolidated optimized flags.
|
||||
if (hardware === 'b200' && effectiveQuant === 'fp8') {
|
||||
cmd += ' \\\n --ep 1';
|
||||
cmd += ' \\\n --quantization fp8';
|
||||
cmd += ' \\\n --attention-backend nsa';
|
||||
cmd += ' \\\n --nsa-decode-backend trtllm';
|
||||
cmd += ' \\\n --nsa-prefill-backend trtllm';
|
||||
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm';
|
||||
cmd += ' \\\n --enable-flashinfer-allreduce-fusion';
|
||||
}
|
||||
|
||||
cmd += ` \\\n --mem-fraction-static ${memFraction}`;
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.4 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (typeof option.condition === 'function' && !option.condition(values)) return null;
|
||||
const items = resolveItems(option, values);
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = !!item.disabled;
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}
|
||||
title={item.disabledReason || ''}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,228 @@
|
||||
export const GLM51Deployment = () => {
|
||||
// Config mirrors sgl-cookbook src/components/autoregressive/GLM51ConfigGenerator/index.js.
|
||||
//
|
||||
// Supported quantization per hardware:
|
||||
// H100 / H200 / B200 → BF16 + FP8
|
||||
// GB300 → FP8 only
|
||||
// MI300X/MI325X/MI355X → BF16 (FP8 not verified on AMD)
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'gb300', label: 'GB300', default: false },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X/MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
getDynamicItems: (values) => {
|
||||
const hw = values.hardware;
|
||||
const isAMD = hw === 'mi300x' || hw === 'mi355x';
|
||||
const isGB300 = hw === 'gb300';
|
||||
return [
|
||||
{ id: 'bf16', label: 'BF16', subtitle: 'Full Weights', default: isAMD, disabled: isGB300, disabledReason: isGB300 ? 'BF16 is not recommended on GB300 for GLM-5.1' : '' },
|
||||
{ id: 'fp8', label: 'FP8', subtitle: 'High Throughput', default: !isAMD, disabled: isAMD, disabledReason: isAMD ? 'FP8 not verified on AMD' : '' }
|
||||
];
|
||||
}
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
]
|
||||
},
|
||||
dpattention: {
|
||||
name: 'dpattention',
|
||||
title: 'DP Attention',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', subtitle: 'Low Latency', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', subtitle: 'High Throughput', default: false }
|
||||
]
|
||||
},
|
||||
speculative: {
|
||||
name: 'speculative',
|
||||
title: 'Speculative Decoding',
|
||||
condition: (values) => values.hardware !== 'mi300x' && values.hardware !== 'mi355x',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
h100: { fp8: { tp: 16, mem: 0.85 }, bf16: { tp: 32, mem: 0.85 } },
|
||||
h200: { fp8: { tp: 8, mem: 0.85 }, bf16: { tp: 16, mem: 0.85 } },
|
||||
b200: { fp8: { tp: 8, mem: 0.9 }, bf16: { tp: 16, mem: 0.9 } },
|
||||
gb300: { fp8: { tp: 4, mem: 0.9 } },
|
||||
mi300x: { bf16: { tp: 8, mem: 0.80 } },
|
||||
mi355x: { bf16: { tp: 8, mem: 0.80 } }
|
||||
};
|
||||
|
||||
const resolveItems = (option, values) => {
|
||||
if (typeof option.getDynamicItems === 'function') return option.getDynamicItems(values);
|
||||
return option.items;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
const items = resolveItems(option, initialState);
|
||||
const def = items.find(i => i.default && !i.disabled) || items.find(i => !i.disabled) || items[0];
|
||||
initialState[key] = def.id;
|
||||
}
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
setValues(prev => {
|
||||
const next = { ...prev };
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (typeof option.getDynamicItems !== 'function') continue;
|
||||
const items = option.getDynamicItems(next);
|
||||
const current = items.find(i => i.id === next[key]);
|
||||
if (!current || current.disabled) {
|
||||
const fallback = items.find(i => i.default && !i.disabled) || items.find(i => !i.disabled);
|
||||
if (fallback) next[key] = fallback.id;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}, [values.hardware]);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const generateCommand = () => {
|
||||
const { hardware, quantization } = values;
|
||||
const isAMD = hardware === 'mi300x' || hardware === 'mi355x';
|
||||
const isGB300 = hardware === 'gb300';
|
||||
const effectiveQuant = isAMD ? 'bf16' : (isGB300 && quantization === 'bf16' ? 'fp8' : quantization);
|
||||
const suffix = effectiveQuant === 'fp8' ? '-FP8' : '';
|
||||
const modelName = `zai-org/GLM-5.1${suffix}`;
|
||||
|
||||
const hwConfig = modelConfigs[hardware][effectiveQuant];
|
||||
if (!hwConfig) return '# Configuration not available for the selected hardware and quantization.';
|
||||
|
||||
const tpValue = hwConfig.tp;
|
||||
const memFraction = hwConfig.mem;
|
||||
const enableSpec = values.speculative === 'enabled';
|
||||
|
||||
let cmd = '';
|
||||
if (enableSpec) cmd += 'SGLANG_ENABLE_SPEC_V2=1 ';
|
||||
cmd += 'sglang serve \\\n';
|
||||
cmd += ` --model-path ${modelName}`;
|
||||
cmd += ` \\\n --tp ${tpValue}`;
|
||||
|
||||
if (isAMD) {
|
||||
cmd += ' \\\n --trust-remote-code';
|
||||
cmd += ' \\\n --nsa-prefill-backend tilelang';
|
||||
cmd += ' \\\n --nsa-decode-backend tilelang';
|
||||
cmd += ' \\\n --chunked-prefill-size 131072';
|
||||
cmd += ' \\\n --watchdog-timeout 1200';
|
||||
}
|
||||
|
||||
if (values.dpattention === 'enabled') {
|
||||
cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`;
|
||||
}
|
||||
if (values.reasoning === 'enabled') cmd += ' \\\n --reasoning-parser glm45';
|
||||
if (values.toolcall === 'enabled') cmd += ' \\\n --tool-call-parser glm47';
|
||||
if (enableSpec) {
|
||||
cmd += ' \\\n --speculative-algorithm EAGLE';
|
||||
cmd += ' \\\n --speculative-num-steps 3';
|
||||
cmd += ' \\\n --speculative-eagle-topk 1';
|
||||
cmd += ' \\\n --speculative-num-draft-tokens 4';
|
||||
}
|
||||
|
||||
cmd += ` \\\n --mem-fraction-static ${memFraction}`;
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.4 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (typeof option.condition === 'function' && !option.condition(values)) return null;
|
||||
const items = resolveItems(option, values);
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = !!item.disabled;
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}
|
||||
title={item.disabledReason || ''}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,372 @@
|
||||
export const GLMGlyphDeployment = () => {
|
||||
const modelFamily = 'zai-org';
|
||||
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false }
|
||||
]
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
{ id: 'disabled', label: 'Disabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--reasoning-parser glm45' : null
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
{ id: 'disabled', label: 'Disabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--tool-call-parser glm45' : null
|
||||
}
|
||||
};
|
||||
|
||||
const modelConfig = {
|
||||
baseName: 'Glyph',
|
||||
b200: { tp: 4, bf16: true, fp8: true },
|
||||
h100: { tp: 4, bf16: true, fp8: true },
|
||||
h200: { tp: 4, bf16: true, fp8: true },
|
||||
mi300x: { tp: 4, bf16: true, fp8: true },
|
||||
mi325x: { tp: 4, bf16: true, fp8: true },
|
||||
mi355x: { tp: 2, bf16: true, fp8: true }
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, quantization } = values;
|
||||
|
||||
const hwConfig = modelConfig[hardware];
|
||||
if (!hwConfig) {
|
||||
return `# Error: Unknown hardware platform: ${hardware}`;
|
||||
}
|
||||
|
||||
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
const modelName = `${modelFamily}/${modelConfig.baseName}${quantSuffix}`;
|
||||
|
||||
const isAMD = ['mi300x', 'mi325x', 'mi355x'].includes(hardware);
|
||||
|
||||
let cmd = '';
|
||||
if (isAMD) {
|
||||
cmd = 'python3 -m sglang.launch_server \\\n';
|
||||
cmd += ` --model-path ${modelName}`;
|
||||
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
||||
} else {
|
||||
cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
if (hwConfig.tp > 1) {
|
||||
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
||||
}
|
||||
}
|
||||
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (key === 'hardware' || key === 'quantization') continue;
|
||||
|
||||
if (option.commandRule) {
|
||||
const rule = option.commandRule(values[key]);
|
||||
if (rule) {
|
||||
cmd += ` \\\n ${rule}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,140 @@
|
||||
export const GLMOCRDeployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h100', label: 'H100', default: true },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'b200', label: 'B200', default: false }
|
||||
]
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'mtp', label: 'MTP', subtitle: 'Multi-token Prediction', default: true }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = option.items.filter(item => item.default).map(item => item.id);
|
||||
} else {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
}
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode - prioritize page theme over system preference
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues(prev => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
} else {
|
||||
return { ...prev, [optionName]: currentValues.filter(id => id !== itemId) };
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { strategy } = values;
|
||||
const strategyArray = Array.isArray(strategy) ? strategy : [];
|
||||
|
||||
const modelName = 'zai-org/GLM-OCR';
|
||||
|
||||
let cmd = 'SGLANG_USE_CUDA_IPC_TRANSPORT=1 python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
if (strategyArray.includes('mtp')) {
|
||||
cmd += ` \\\n --speculative-algorithm EAGLE`;
|
||||
cmd += ` \\\n --speculative-num-steps 3`;
|
||||
cmd += ` \\\n --speculative-eagle-topk 1`;
|
||||
cmd += ` \\\n --speculative-num-draft-tokens 4`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,237 @@
|
||||
export const GPTOSSDeployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
modelsize: {
|
||||
name: 'modelsize',
|
||||
title: 'Model Size',
|
||||
items: [
|
||||
{ id: '120b', label: '120B', subtitle: 'MOE', default: true },
|
||||
{ id: '20b', label: '20B', subtitle: 'MOE', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'mxfp4', label: 'MXFP4', default: true },
|
||||
{ id: 'bf16', label: 'BF16', default: false }
|
||||
]
|
||||
},
|
||||
reasoningParser: {
|
||||
name: 'reasoningParser',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
speculative: {
|
||||
name: 'speculative',
|
||||
title: 'Speculative Decoding',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = option.items.filter(item => item.default).map(item => item.id);
|
||||
} else {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
}
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, modelsize, quantization, reasoningParser, toolcall, speculative } = values;
|
||||
|
||||
// Model configurations
|
||||
const modelConfigs = {
|
||||
'120b': {
|
||||
baseName: '120b',
|
||||
h100: { tp: 8 },
|
||||
h200: { tp: 8 },
|
||||
b200: { tp: 8 },
|
||||
mi300x: { tp: 8 },
|
||||
mi325x: { tp: 8 },
|
||||
mi355x: { tp: 8 }
|
||||
},
|
||||
'20b': {
|
||||
baseName: '20b',
|
||||
h100: { tp: 1 },
|
||||
h200: { tp: 1 },
|
||||
b200: { tp: 1 },
|
||||
mi300x: { tp: 1 },
|
||||
mi325x: { tp: 1 },
|
||||
mi355x: { tp: 1 }
|
||||
}
|
||||
};
|
||||
|
||||
const config = modelConfigs[modelsize];
|
||||
if (!config) {
|
||||
return `# Error: Unknown model size: ${modelsize}`;
|
||||
}
|
||||
|
||||
const hwConfig = config[hardware];
|
||||
if (!hwConfig) {
|
||||
return `# Error: Unknown hardware platform: ${hardware}`;
|
||||
}
|
||||
|
||||
const quantSuffix = quantization === 'bf16' ? '-bf16' : '';
|
||||
const orgPrefix = quantization === 'bf16' ? 'lmsys' : 'openai';
|
||||
const modelName = `${orgPrefix}/gpt-oss-${config.baseName}${quantSuffix}`;
|
||||
|
||||
let cmd = '';
|
||||
|
||||
// MI30x GPUs with speculative decoding: Work In Progress
|
||||
if ((hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') && speculative === 'enabled') {
|
||||
return '# MI30x GPUs Speculative Decoding: Work In Progress';
|
||||
}
|
||||
|
||||
// MI300X/MI325X MXFP4: Work In Progress (only MI355X with gfx950 supports MXFP4)
|
||||
if ((hardware === 'mi300x' || hardware === 'mi325x') && quantization === 'mxfp4') {
|
||||
return '# MI300X/MI325X GPUs with MXFP4 quantization: Work In Progress';
|
||||
}
|
||||
|
||||
// AMD MI30x requires SGLANG_USE_AITER=0 due to YaRN RoPE precision issues
|
||||
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||
cmd += 'SGLANG_USE_AITER=0 ';
|
||||
}
|
||||
|
||||
if (speculative === 'enabled') {
|
||||
cmd += 'SGLANG_ENABLE_SPEC_V2=1 SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 ';
|
||||
}
|
||||
|
||||
cmd += 'python -m sglang.launch_server \\\n';
|
||||
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
if (hwConfig.tp > 1) {
|
||||
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
||||
}
|
||||
|
||||
// Add reasoning parser if enabled
|
||||
if (reasoningParser === 'enabled') {
|
||||
cmd += ` \\\n --reasoning-parser gpt-oss`;
|
||||
}
|
||||
|
||||
// Add tool call parser if enabled
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --tool-call-parser gpt-oss`;
|
||||
}
|
||||
|
||||
// Add speculative decoding if enabled (MI30x handled above)
|
||||
if (speculative === 'enabled') {
|
||||
cmd += ` \\\n --speculative-algorithm EAGLE3 \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4`;
|
||||
|
||||
if (modelsize === '120b') {
|
||||
cmd += ` \\\n --speculative-draft-model-path nvidia/gpt-oss-120b-Eagle3`;
|
||||
} else if (modelsize === '20b') {
|
||||
cmd += ` \\\n --speculative-draft-model-path zhuyksir/EAGLE3-gpt-oss-20b-bf16`;
|
||||
}
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,373 @@
|
||||
export const KimiK2Deployment = () => {
|
||||
const modelFamily = 'moonshotai';
|
||||
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
modelname: {
|
||||
name: 'modelname',
|
||||
title: 'Model Name',
|
||||
items: [
|
||||
{ id: 'instruct', label: 'Kimi-K2-Instruct', default: true },
|
||||
{ id: 'thinking', label: 'Kimi-K2-Thinking', default: false }
|
||||
]
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'tp', label: 'TP', default: true, required: true },
|
||||
{ id: 'dp', label: 'DP attention', default: false },
|
||||
{ id: 'ep', label: 'EP', default: false }
|
||||
]
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, modelname, strategy, reasoning, toolcall } = values;
|
||||
|
||||
if (modelname === 'instruct' && reasoning === 'enabled') {
|
||||
return `# Error: Kimi-K2-Instruct doesn't support reasoning parser\n# Please select "Disabled" for Reasoning Parser or choose Kimi-K2-Thinking model`;
|
||||
}
|
||||
|
||||
const modelMap = {
|
||||
'instruct': 'Kimi-K2-Instruct',
|
||||
'thinking': 'Kimi-K2-Thinking'
|
||||
};
|
||||
|
||||
const modelName = `${modelFamily}/${modelMap[modelname]}`;
|
||||
|
||||
let cmd = 'python3 -m sglang.launch_server \\\n';
|
||||
|
||||
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||
cmd = 'SGLANG_ROCM_FUSED_DECODE_MLA=0 ' + cmd;
|
||||
}
|
||||
|
||||
cmd += ` --model-path ${modelName}`;
|
||||
|
||||
const strategyArray = Array.isArray(strategy) ? strategy : [];
|
||||
cmd += ` \\\n --tp 8`;
|
||||
if (strategyArray.includes('dp')) {
|
||||
cmd += ` \\\n --dp 4 \\\n --enable-dp-attention`;
|
||||
}
|
||||
if (strategyArray.includes('ep')) {
|
||||
cmd += ` \\\n --ep 4`;
|
||||
}
|
||||
|
||||
cmd += ` \\\n --trust-remote-code`;
|
||||
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --tool-call-parser kimi_k2`;
|
||||
}
|
||||
|
||||
if (reasoning === 'enabled') {
|
||||
cmd += ` \\\n --reasoning-parser kimi_k2`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,264 @@
|
||||
export const KimiK25Deployment = () => {
|
||||
// Config mirrors sgl-cookbook src/components/autoregressive/KimiK25ConfigGenerator/index.js.
|
||||
//
|
||||
// GPU requirements:
|
||||
// H200: tp=8
|
||||
// B300: tp=8
|
||||
// MI300X: tp=4 (64 heads / 4 = 16 heads per GPU, AITER MLA requires heads_per_gpu % 16 == 0)
|
||||
// MI325X: tp=4 (same constraint as MI300X)
|
||||
// MI350X: tp=4 (same constraint as MI300X)
|
||||
// MI355X: tp=4 (same constraint as MI300X)
|
||||
//
|
||||
// NVFP4 quantization is only supported on NVIDIA Blackwell (B300).
|
||||
// Speculative decoding is only supported on H200 and B300.
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'b300', label: 'B300', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi350x', label: 'MI350X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
getDynamicItems: (values) => {
|
||||
const hw = values.hardware;
|
||||
const isB300 = hw === 'b300';
|
||||
return [
|
||||
{ id: 'int4', label: 'INT4', subtitle: 'initial model', default: true },
|
||||
{ id: 'nvfp4', label: 'NVFP4', subtitle: 'Blackwell only', default: false, disabled: !isB300, disabledReason: 'NVFP4 only on B300' }
|
||||
];
|
||||
}
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
]
|
||||
},
|
||||
dpattention: {
|
||||
name: 'dpattention',
|
||||
title: 'DP Attention',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', subtitle: 'Low Latency', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', subtitle: 'High Throughput', default: false }
|
||||
]
|
||||
},
|
||||
speculative: {
|
||||
name: 'speculative',
|
||||
title: 'Speculative Decoding',
|
||||
condition: (values) => values.hardware === 'h200' || values.hardware === 'b300',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
h200: { tp: 8 },
|
||||
b300: { tp: 8 },
|
||||
mi300x: { tp: 4 },
|
||||
mi325x: { tp: 4 },
|
||||
mi350x: { tp: 4 },
|
||||
mi355x: { tp: 4 }
|
||||
};
|
||||
|
||||
const resolveItems = (option, values) => {
|
||||
if (typeof option.getDynamicItems === 'function') return option.getDynamicItems(values);
|
||||
return option.items;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
const items = resolveItems(option, initialState);
|
||||
const def = items.find(i => i.default && !i.disabled) || items.find(i => !i.disabled) || items[0];
|
||||
initialState[key] = def.id;
|
||||
}
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
// When hardware changes, re-resolve quantization defaults (NVFP4 only on B300).
|
||||
useEffect(() => {
|
||||
setValues(prev => {
|
||||
const next = { ...prev };
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (typeof option.getDynamicItems !== 'function') continue;
|
||||
const items = option.getDynamicItems(next);
|
||||
const current = items.find(i => i.id === next[key]);
|
||||
if (!current || current.disabled) {
|
||||
const fallback = items.find(i => i.default && !i.disabled) || items.find(i => !i.disabled);
|
||||
if (fallback) next[key] = fallback.id;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}, [values.hardware]);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
// Generate command - mirrors sgl-cookbook's config.generateCommand(values) exactly.
|
||||
const generateCommand = () => {
|
||||
const { hardware, quantization, speculative } = values;
|
||||
const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi350x' || hardware === 'mi355x';
|
||||
|
||||
// NVFP4 is only supported on NVIDIA Blackwell (B300)
|
||||
if (quantization === 'nvfp4' && hardware !== 'b300') {
|
||||
return '# NVFP4 quantization is only supported on NVIDIA Blackwell GPUs (B300)';
|
||||
}
|
||||
|
||||
// Speculative decoding only supported on H200 and B300
|
||||
if (speculative === 'enabled' && hardware !== 'h200' && hardware !== 'b300') {
|
||||
return '# Speculative Decoding for Kimi-K2.5 is only supported on H200 and B300';
|
||||
}
|
||||
|
||||
// Model path depends on quantization
|
||||
const modelName = quantization === 'nvfp4'
|
||||
? 'nvidia/Kimi-K2.5-NVFP4'
|
||||
: 'moonshotai/Kimi-K2.5';
|
||||
|
||||
const hwConfig = modelConfigs[hardware];
|
||||
const tpValue = hwConfig.tp;
|
||||
|
||||
let cmd = '';
|
||||
|
||||
// AMD ROCm environment variables
|
||||
if (isAMD) {
|
||||
cmd += 'SGLANG_USE_AITER=1 SGLANG_ROCM_FUSED_DECODE_MLA=0 ';
|
||||
}
|
||||
|
||||
// Speculative decoding env var
|
||||
if (speculative === 'enabled') {
|
||||
cmd += 'SGLANG_ENABLE_SPEC_V2=1 ';
|
||||
}
|
||||
|
||||
// If we added any env vars above, break to a new line for readability
|
||||
if (isAMD || speculative === 'enabled') {
|
||||
cmd += '\\\n';
|
||||
}
|
||||
|
||||
cmd += 'sglang serve \\\n';
|
||||
cmd += ` --model-path ${modelName}`;
|
||||
cmd += ` \\\n --tp ${tpValue}`;
|
||||
cmd += ' \\\n --trust-remote-code';
|
||||
|
||||
// DP Attention: --dp matches --tp
|
||||
if (values.dpattention === 'enabled') {
|
||||
cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`;
|
||||
}
|
||||
|
||||
// Reasoning parser
|
||||
if (values.reasoning === 'enabled') {
|
||||
cmd += ' \\\n --reasoning-parser kimi_k2';
|
||||
}
|
||||
|
||||
// Tool call parser
|
||||
if (values.toolcall === 'enabled') {
|
||||
cmd += ' \\\n --tool-call-parser kimi_k2';
|
||||
}
|
||||
|
||||
// Speculative decoding (EAGLE3)
|
||||
if (speculative === 'enabled') {
|
||||
cmd += ' \\\n --speculative-algorithm EAGLE3 \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4 \\\n --speculative-draft-model-path lightseekorg/kimi-k2.5-eagle3';
|
||||
}
|
||||
|
||||
// AMD: FP8 KV cache for memory efficiency
|
||||
if (isAMD) {
|
||||
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
|
||||
}
|
||||
|
||||
cmd += ' \\\n --host 0.0.0.0 \\\n --port 30000';
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.4 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (typeof option.condition === 'function' && !option.condition(values)) return null;
|
||||
const items = resolveItems(option, values);
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = !!item.disabled;
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}
|
||||
title={item.disabledReason || ''}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,181 @@
|
||||
export const KimiK26Deployment = () => {
|
||||
// Config mirrors sgl-cookbook src/components/autoregressive/KimiK26ConfigGenerator/index.js.
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'b300', label: 'B300', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi350x', label: 'MI350X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false },
|
||||
],
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
],
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
],
|
||||
},
|
||||
dpattention: {
|
||||
name: 'dpattention',
|
||||
title: 'DP Attention',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', subtitle: 'Low Latency', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', subtitle: 'High Throughput', default: false },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
h200: { tp: 8 },
|
||||
b300: { tp: 8 },
|
||||
mi300x: { tp: 4 },
|
||||
mi325x: { tp: 4 },
|
||||
mi350x: { tp: 4 },
|
||||
mi355x: { tp: 4 },
|
||||
};
|
||||
|
||||
const resolveItems = (option, values) =>
|
||||
typeof option.getDynamicItems === 'function' ? option.getDynamicItems(values) : option.items || [];
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
const items = resolveItems(option, initialState);
|
||||
const def = items.find((item) => item.default && !item.disabled) || items.find((item) => !item.disabled) || items[0];
|
||||
initialState[key] = def.id;
|
||||
}
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const generateCommand = () => {
|
||||
const { hardware, reasoning, toolcall, dpattention } = values;
|
||||
const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi350x' || hardware === 'mi355x';
|
||||
const hwConfig = modelConfigs[hardware];
|
||||
const tpValue = hwConfig.tp;
|
||||
|
||||
let cmd = '';
|
||||
|
||||
if (isAMD) {
|
||||
cmd += 'SGLANG_USE_AITER=1 SGLANG_ROCM_FUSED_DECODE_MLA=0 \\\n';
|
||||
}
|
||||
|
||||
cmd += 'sglang serve \\\n';
|
||||
cmd += ' --model-path moonshotai/Kimi-K2.6';
|
||||
cmd += ` \\\n --tp ${tpValue}`;
|
||||
if (isAMD) {
|
||||
cmd += ' \\\n --mem-fraction-static 0.8';
|
||||
}
|
||||
cmd += ' \\\n --trust-remote-code';
|
||||
|
||||
if (dpattention === 'enabled') {
|
||||
cmd += ` \\\n --dp ${tpValue} \\\n --enable-dp-attention`;
|
||||
}
|
||||
|
||||
if (reasoning === 'enabled') {
|
||||
cmd += ' \\\n --reasoning-parser kimi_k2';
|
||||
}
|
||||
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ' \\\n --tool-call-parser kimi_k2';
|
||||
}
|
||||
|
||||
if (isAMD) {
|
||||
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
|
||||
}
|
||||
|
||||
cmd += ' \\\n --host 0.0.0.0 \\\n --port 30000';
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.4 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
const items = resolveItems(option, values);
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = !!item.disabled;
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}
|
||||
title={item.disabledReason || ''}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,358 @@
|
||||
export const KimiLinearDeployment = () => {
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'mi300x', label: 'MI300x', default: false },
|
||||
{ id: 'mi325x', label: 'MI325x', default: false },
|
||||
{ id: 'mi355x', label: 'MI355x', default: false }
|
||||
]
|
||||
},
|
||||
modelname: {
|
||||
name: 'modelname',
|
||||
title: 'Model Name',
|
||||
items: [
|
||||
{ id: 'instruct', label: 'Kimi-Linear-48B-A3B-Instruct', default: true },
|
||||
]
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'tp', label: 'TP', default: true, required: true },
|
||||
]
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, modelname, strategy, reasoning, toolcall } = values;
|
||||
|
||||
if (modelname === 'instruct' && reasoning === 'enabled') {
|
||||
return `# Error: Kimi-Linear doesn't support reasoning parser\n# Please select "Disabled" for Reasoning Parser or choose Kimi-Linear-Thinking model`;
|
||||
}
|
||||
|
||||
const modelMap = {
|
||||
'instruct': 'moonshotai/Kimi-Linear-48B-A3B-Instruct',
|
||||
};
|
||||
|
||||
const modelName = modelMap[modelname];
|
||||
|
||||
let cmd = 'python3 -m sglang.launch_server \\\n';
|
||||
|
||||
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||
cmd = 'SGLANG_ROCM_FUSED_DECODE_MLA=0 ' + cmd;
|
||||
}
|
||||
|
||||
cmd += ` --model-path ${modelName}`;
|
||||
|
||||
cmd += ` \\\n --tp 4`;
|
||||
|
||||
cmd += ` \\\n --trust-remote-code`;
|
||||
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --tool-call-parser kimi_k2`;
|
||||
}
|
||||
|
||||
if (reasoning === 'enabled') {
|
||||
cmd += ` \\\n --reasoning-parser kimi_k2`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,189 @@
|
||||
export const Ling251TDeployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'gb200', label: 'GB200', default: false },
|
||||
{ id: 'gb300', label: 'GB300', default: false }
|
||||
]
|
||||
},
|
||||
parallelism: {
|
||||
name: 'parallelism',
|
||||
title: 'Parallelism Strategy',
|
||||
items: [
|
||||
{ id: 'tp4pp2', label: 'TP4 + PP2', default: true },
|
||||
{ id: 'tp8', label: 'TP8', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
{ id: 'disabled', label: 'Disabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = option.items.filter(item => item.default).map(item => item.id);
|
||||
} else {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
}
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues(prev => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
} else {
|
||||
return { ...prev, [optionName]: currentValues.filter(id => id !== itemId) };
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, parallelism, toolcall } = values;
|
||||
|
||||
const isGB = hardware === 'gb200' || hardware === 'gb300';
|
||||
const envPrefix = isGB ? 'NCCL_IB_DISABLE=1 ' : '';
|
||||
|
||||
let tp, pp;
|
||||
if (isGB && parallelism === 'tp8') {
|
||||
tp = 8;
|
||||
pp = null;
|
||||
} else if (isGB) {
|
||||
tp = 4;
|
||||
pp = 2;
|
||||
} else {
|
||||
tp = 8;
|
||||
pp = 2;
|
||||
}
|
||||
|
||||
const needMemFrac = hardware === 'h200' || (isGB && parallelism !== 'tp8');
|
||||
|
||||
const generateNodeCmd = (rank) => {
|
||||
let cmd = `${envPrefix}python3 -m sglang.launch_server \\\n`;
|
||||
cmd += ` --model-path inclusionAI/Ling-2.5-1T \\\n`;
|
||||
cmd += ` --trust-remote-code \\\n`;
|
||||
cmd += ` --tp-size ${tp} \\\n`;
|
||||
if (pp) {
|
||||
cmd += ` --pp-size ${pp} \\\n`;
|
||||
}
|
||||
cmd += ` --nnodes 2 \\\n`;
|
||||
cmd += ` --node-rank ${rank} \\\n`;
|
||||
if (rank === 0) {
|
||||
cmd += ` --host 0.0.0.0 \\\n`;
|
||||
cmd += ` --port \${PORT} \\\n`;
|
||||
}
|
||||
cmd += ` --dist-init-addr \${MASTER_IP}:\${DIST_PORT}`;
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --tool-call-parser qwen`;
|
||||
}
|
||||
if (needMemFrac) {
|
||||
cmd += ` \\\n --mem-frac 0.95`;
|
||||
}
|
||||
return cmd;
|
||||
};
|
||||
|
||||
let output = `# MASTER_IP is Node 0 IP. PORT and DIST_PORT can be assigned by yourself.\n\n`;
|
||||
output += `# Node 0:\n`;
|
||||
output += generateNodeCmd(0);
|
||||
output += `\n\n\n# Node 1:\n`;
|
||||
output += generateNodeCmd(1);
|
||||
|
||||
return output;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
const isGB = values.hardware === 'gb200' || values.hardware === 'gb300';
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
// Only show parallelism for GB200/GB300
|
||||
if (key === 'parallelism' && !isGB) return null;
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isItemDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isItemDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isItemDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,338 @@
|
||||
export const LLaDA21Deployment = () => {
|
||||
const modelFamily = 'inclusionAI';
|
||||
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h100', label: 'H100', default: true },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
modelsize: {
|
||||
name: 'modelsize',
|
||||
title: 'Model Size',
|
||||
items: [
|
||||
{ id: 'mini', label: 'Mini (16B)', subtitle: 'MoE', default: true },
|
||||
{ id: 'flash', label: 'Flash (100B)', subtitle: 'MoE', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, modelsize } = values;
|
||||
|
||||
const modelName = modelsize === 'mini' ? 'LLaDA2.1-mini' : 'LLaDA2.1-flash';
|
||||
const modelPath = `${modelFamily}/${modelName}`;
|
||||
|
||||
let tpSize;
|
||||
if (modelsize === 'mini') {
|
||||
tpSize = 1;
|
||||
} else {
|
||||
if (hardware === 'b200') {
|
||||
tpSize = 2;
|
||||
} else {
|
||||
tpSize = 4;
|
||||
}
|
||||
}
|
||||
|
||||
const args = [];
|
||||
args.push(`--model-path ${modelPath}`);
|
||||
args.push(`--dllm-algorithm JointThreshold`);
|
||||
args.push(`--tp ${tpSize}`);
|
||||
args.push(`--trust-remote-code`);
|
||||
args.push(`--mem-fraction-static 0.8`);
|
||||
args.push(`--max-running-requests 1`);
|
||||
if (hardware === 'h100' || hardware === 'h200' || hardware === 'b200') {
|
||||
args.push(`--attention-backend flashinfer`);
|
||||
}
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` ${args.join(' \\\n ')}`;
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,252 @@
|
||||
export const Llama31Deployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h100', label: 'H100', default: true },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
modelsize: {
|
||||
name: 'modelsize',
|
||||
title: 'Model Size',
|
||||
items: [
|
||||
{ id: '8b', label: '8B', default: false },
|
||||
{ id: '70b', label: '70B', default: true },
|
||||
{ id: '405b', label: '405B', default: false }
|
||||
]
|
||||
},
|
||||
category: {
|
||||
name: 'category',
|
||||
title: 'Category',
|
||||
items: [
|
||||
{ id: 'base', label: 'Base', default: false },
|
||||
{ id: 'instruct', label: 'Instruct', default: true }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
optimization: {
|
||||
name: 'optimization',
|
||||
title: 'Optimization Mode',
|
||||
items: [
|
||||
{ id: 'basic', label: 'Basic', default: true },
|
||||
{ id: 'throughput', label: 'Throughput Optimized', default: false },
|
||||
{ id: 'latency', label: 'Latency Optimized', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, optimization, modelsize, category, toolcall, quantization } = values;
|
||||
|
||||
const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x';
|
||||
|
||||
// Model size mapping
|
||||
const sizeMap = {
|
||||
'8b': '8B',
|
||||
'70b': '70B',
|
||||
'405b': '405B'
|
||||
};
|
||||
const sizeToken = sizeMap[modelsize] || '70B';
|
||||
const categorySuffix = category === 'instruct' ? '-Instruct' : '';
|
||||
|
||||
// Determine model path
|
||||
let modelPath;
|
||||
if (quantization === 'fp8' && category === 'instruct') {
|
||||
if (modelsize === '405b') {
|
||||
// Meta official FP8 for 405B
|
||||
modelPath = `meta-llama/Llama-3.1-${sizeToken}${categorySuffix}-FP8`;
|
||||
} else if (isAMD) {
|
||||
// AMD FP8-KV variants for 70B/8B on AMD GPUs
|
||||
modelPath = `amd/Llama-3.1-${sizeToken}${categorySuffix}-FP8-KV`;
|
||||
} else {
|
||||
modelPath = `meta-llama/Llama-3.1-${sizeToken}${categorySuffix}`;
|
||||
}
|
||||
} else {
|
||||
modelPath = `meta-llama/Llama-3.1-${sizeToken}${categorySuffix}`;
|
||||
}
|
||||
|
||||
// Determine TP size
|
||||
let tpSize;
|
||||
if (isAMD) {
|
||||
// AMD GPU TP configuration
|
||||
const amdTpConfig = {
|
||||
'mi300x': {
|
||||
'405b': { bf16: 8, fp8: 4 },
|
||||
'70b': { bf16: 1, fp8: 1 },
|
||||
'8b': { bf16: 1, fp8: 1 }
|
||||
},
|
||||
'mi325x': {
|
||||
'405b': { bf16: 8, fp8: 4 },
|
||||
'70b': { bf16: 1, fp8: 1 },
|
||||
'8b': { bf16: 1, fp8: 1 }
|
||||
},
|
||||
'mi355x': {
|
||||
'405b': { bf16: 4, fp8: 2 },
|
||||
'70b': { bf16: 1, fp8: 1 },
|
||||
'8b': { bf16: 1, fp8: 1 }
|
||||
}
|
||||
};
|
||||
tpSize = quantization === 'fp8'
|
||||
? amdTpConfig[hardware][modelsize].fp8
|
||||
: amdTpConfig[hardware][modelsize].bf16;
|
||||
} else {
|
||||
// NVIDIA GPU TP configuration
|
||||
if (modelsize === '405b') {
|
||||
tpSize = 8;
|
||||
} else if (modelsize === '70b' && (hardware === 'h100' || hardware === 'h200')) {
|
||||
tpSize = 2;
|
||||
}
|
||||
}
|
||||
|
||||
// Build command args
|
||||
const args = [];
|
||||
args.push(`--model-path ${modelPath}`);
|
||||
|
||||
if (tpSize) {
|
||||
args.push(`--tp ${tpSize}`);
|
||||
}
|
||||
|
||||
// Add quantization flag only if not using FP8 variant model
|
||||
if (quantization === 'fp8' && category !== 'instruct') {
|
||||
args.push(`--quantization fp8`);
|
||||
}
|
||||
|
||||
// NVIDIA-specific optimizations
|
||||
if (!isAMD) {
|
||||
if (optimization === 'throughput') {
|
||||
args.push(`--enable-dp-attention`);
|
||||
args.push(`--mem-fraction-static 0.85`);
|
||||
} else if (optimization === 'latency') {
|
||||
args.push(`--speculative-algorithm EAGLE3`);
|
||||
args.push(`--speculative-num-steps 3`);
|
||||
args.push(`--speculative-eagle-topk 1`);
|
||||
args.push(`--speculative-num-draft-tokens 4`);
|
||||
if (modelsize === '8b' && category === 'instruct') {
|
||||
args.push(`--speculative-draft-model-path yuhuili/EAGLE3-LLaMA3.1-Instruct-8B`);
|
||||
} else {
|
||||
args.push(`--speculative-draft-model-path \${EAGLE3_MODEL_PATH}`);
|
||||
}
|
||||
args.push(`--disable-shared-experts-fusion`);
|
||||
args.push(`--max-running-requests 64`);
|
||||
args.push(`--mem-fraction-static 0.85`);
|
||||
args.push(`--kv-cache-dtype fp8_e4m3`);
|
||||
args.push(`--context-length 32768`);
|
||||
}
|
||||
}
|
||||
|
||||
if (toolcall === 'enabled') {
|
||||
args.push(`--tool-call-parser llama3`);
|
||||
}
|
||||
|
||||
let cmd = 'sglang serve \\\n';
|
||||
cmd += ` ${args.join(' \\\n ')}`;
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,138 @@
|
||||
export const Llama33Deployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'mi300x', label: 'MI300X', default: true },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Calling',
|
||||
items: [
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
{ id: 'disabled', label: 'Disabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, quantization, toolcall } = values;
|
||||
|
||||
// Select model based on quantization
|
||||
const modelPath = quantization === 'fp8'
|
||||
? 'amd/Llama-3.3-70B-Instruct-FP8-KV'
|
||||
: 'meta-llama/Llama-3.3-70B-Instruct';
|
||||
|
||||
// Build command
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model-path ${modelPath} \\\n`;
|
||||
cmd += ` --tp 1`;
|
||||
|
||||
// Add tool calling parser
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ' \\\n --tool-call-parser llama3';
|
||||
}
|
||||
|
||||
cmd += ' \\\n --host 0.0.0.0 \\\n';
|
||||
cmd += ' --port 30000';
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,347 @@
|
||||
export const Llama4MaverickDeployment = () => {
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'mi300x', label: 'MI300x', default: true },
|
||||
{ id: 'mi325x', label: 'MI325x', default: false },
|
||||
{ id: 'mi355x', label: 'MI355x', default: false }
|
||||
]
|
||||
},
|
||||
host: {
|
||||
name: 'host',
|
||||
title: 'Host',
|
||||
type: 'text',
|
||||
default: '0.0.0.0',
|
||||
placeholder: '0.0.0.0'
|
||||
},
|
||||
port: {
|
||||
name: 'port',
|
||||
title: 'Port',
|
||||
type: 'text',
|
||||
default: '8000',
|
||||
placeholder: '8000'
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, quantization, toolcall, speculative, host, port } = values;
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model-path meta-llama/Llama-4-Scout-17B-16E-Instruct`;
|
||||
|
||||
if (hardware === 'h100' || hardware === 'h200') {
|
||||
cmd += ` \\\n --tp 8`;
|
||||
} else if (hardware === 'b200') {
|
||||
cmd += ` \\\n --tp 8`;
|
||||
} else if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||
cmd += ` \\\n --tp 8`;
|
||||
}
|
||||
|
||||
if (quantization === 'fp8') {
|
||||
cmd += ` \\\n --quantization fp8`;
|
||||
}
|
||||
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --tool-call-parser pythonic`;
|
||||
}
|
||||
|
||||
if (speculative === 'enabled') {
|
||||
cmd += ` \\\n --speculative-algorithm EAGLE3 \\\n`;
|
||||
cmd += ` --speculative-draft-model-path lmsys/sglang-EAGLE3-Llama-4-Scout-17B-16E-Instruct-v1 \\\n`;
|
||||
cmd += ` --speculative-num-steps 3 \\\n`;
|
||||
cmd += ` --speculative-eagle-topk 1 \\\n`;
|
||||
cmd += ` --speculative-num-draft-tokens 4 \\\n`;
|
||||
cmd += ` --mem-fraction-static 0.75 \\\n`;
|
||||
cmd += ` --cuda-graph-max-bs 2`;
|
||||
}
|
||||
|
||||
cmd += ` \\\n --enable-multimodal`;
|
||||
cmd += ` \\\n --context-length 65536`;
|
||||
cmd += ` \\\n --dtype bfloat16`;
|
||||
cmd += ` \\\n --trust-remote-code`;
|
||||
cmd += ` \\\n --host ${host || '0.0.0.0'}`;
|
||||
cmd += ` \\\n --port ${port || '8000'}`;
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,374 @@
|
||||
export const Llama4ScoutDeployment = () => {
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'h100', label: 'H100', default: true },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300x', default: false },
|
||||
{ id: 'mi325x', label: 'MI325x', default: false },
|
||||
{ id: 'mi355x', label: 'MI355x', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
speculative: {
|
||||
name: 'speculative',
|
||||
title: 'Speculative Decoding (EAGLE3)',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enable EAGLE3', default: false }
|
||||
]
|
||||
},
|
||||
host: {
|
||||
name: 'host',
|
||||
title: 'Host',
|
||||
type: 'text',
|
||||
default: '0.0.0.0',
|
||||
placeholder: '0.0.0.0'
|
||||
},
|
||||
port: {
|
||||
name: 'port',
|
||||
title: 'Port',
|
||||
type: 'text',
|
||||
default: '8000',
|
||||
placeholder: '8000'
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, quantization, toolcall, speculative, host, port } = values;
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model-path meta-llama/Llama-4-Scout-17B-16E-Instruct`;
|
||||
|
||||
if (hardware === 'h100' || hardware === 'h200') {
|
||||
cmd += ` \\\n --tp 8`;
|
||||
} else if (hardware === 'b200') {
|
||||
cmd += ` \\\n --tp 8`;
|
||||
} else if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||
cmd += ` \\\n --tp 8`;
|
||||
}
|
||||
|
||||
if (quantization === 'fp8') {
|
||||
cmd += ` \\\n --quantization fp8`;
|
||||
}
|
||||
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --tool-call-parser pythonic`;
|
||||
}
|
||||
|
||||
if (speculative === 'enabled') {
|
||||
cmd += ` \\\n --speculative-algorithm EAGLE3 \\\n`;
|
||||
cmd += ` --speculative-draft-model-path lmsys/sglang-EAGLE3-Llama-4-Scout-17B-16E-Instruct-v1 \\\n`;
|
||||
cmd += ` --speculative-num-steps 3 \\\n`;
|
||||
cmd += ` --speculative-eagle-topk 1 \\\n`;
|
||||
cmd += ` --speculative-num-draft-tokens 4 \\\n`;
|
||||
cmd += ` --mem-fraction-static 0.75 \\\n`;
|
||||
cmd += ` --cuda-graph-max-bs 2`;
|
||||
}
|
||||
|
||||
cmd += ` \\\n --enable-multimodal`;
|
||||
cmd += ` \\\n --context-length 65536`;
|
||||
cmd += ` \\\n --dtype bfloat16`;
|
||||
cmd += ` \\\n --trust-remote-code`;
|
||||
cmd += ` \\\n --host ${host || '0.0.0.0'}`;
|
||||
cmd += ` \\\n --port ${port || '8000'}`;
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,194 @@
|
||||
export const MiMoV2FlashDeployment = () => {
|
||||
// Config mirrors sgl-cookbook src/components/autoregressive/MiMoConfigGenerator/index.js.
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
modelname: {
|
||||
name: 'modelname',
|
||||
title: 'Model Name',
|
||||
items: [
|
||||
{ id: 'mimo-v2-flash', label: 'MiMo-V2-Flash', default: true }
|
||||
]
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'tp', label: 'TP 8 (Required)', default: true, disabled: true },
|
||||
{ id: 'dp', label: 'DP Attention (DP 2)', default: true },
|
||||
{ id: 'mtp', label: 'Multi-token Prediction (MTP)', default: true },
|
||||
{ id: 'optimization', label: 'Performance Optimizations', default: true }
|
||||
]
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning & Tools',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'reasoning', label: 'Reasoning Parser (Qwen3)', default: true },
|
||||
{ id: 'toolcall', label: 'Tool Call Parser', default: true }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = option.items.filter(item => item.default).map(item => item.id);
|
||||
} else {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
}
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues(prev => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
} else {
|
||||
return { ...prev, [optionName]: currentValues.filter(id => id !== itemId) };
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Generate command — mirrors sgl-cookbook's config.generateCommand(values) exactly
|
||||
const generateCommand = () => {
|
||||
const { hardware, strategy, reasoning } = values;
|
||||
const isMI355X = hardware === 'mi355x';
|
||||
|
||||
const modelPath = 'XiaomiMiMo/MiMo-V2-Flash';
|
||||
const strategyArray = Array.isArray(strategy) ? strategy : [];
|
||||
const reasoningArray = Array.isArray(reasoning) ? reasoning : [];
|
||||
|
||||
if (isMI355X && strategyArray.includes('mtp')) {
|
||||
return '# MI355X Speculative Decoding (EAGLE): Work In Progress\n'
|
||||
+ '# Uncheck "Multi-token Prediction (MTP)" to view the validated non-speculative MI355X command.';
|
||||
}
|
||||
|
||||
const commandPrefix = isMI355X
|
||||
? 'PYTHONPATH=/sgl-workspace/aiter SGLANG_USE_AITER=0 USE_ROCM_AITER_ROPE_BACKEND=0'
|
||||
: 'SGLANG_ENABLE_SPEC_V2=1';
|
||||
const tpSize = isMI355X ? 4 : 8;
|
||||
|
||||
let cmd = `${commandPrefix} sglang serve \\\n`;
|
||||
cmd += ` --model-path ${modelPath} \\\n`;
|
||||
cmd += ` --trust-remote-code \\\n`;
|
||||
cmd += ` --tp-size ${tpSize}`;
|
||||
|
||||
// DP settings
|
||||
if (!isMI355X && strategyArray.includes('dp')) {
|
||||
cmd += ` \\\n --dp-size 2 \\\n --enable-dp-attention`;
|
||||
}
|
||||
|
||||
// Performance Optimizations
|
||||
if (strategyArray.includes('optimization')) {
|
||||
cmd += ` \\\n --mem-fraction-static 0.75 \\\n --max-running-requests 128 \\\n --chunked-prefill-size 16384 \\\n --model-loader-extra-config '{"enable_multithread_load": "true","num_threads": 64}'`;
|
||||
cmd += isMI355X
|
||||
? ` \\\n --attention-backend triton \\\n --prefill-attention-backend triton \\\n --decode-attention-backend triton \\\n --disable-custom-all-reduce`
|
||||
: ` \\\n --attention-backend fa3`;
|
||||
}
|
||||
|
||||
// MTP/Speculative settings
|
||||
if (!isMI355X && strategyArray.includes('mtp')) {
|
||||
cmd += ` \\\n --speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4 \\\n --enable-multi-layer-eagle`;
|
||||
}
|
||||
|
||||
// Reasoning Parser
|
||||
if (reasoningArray.includes('reasoning')) {
|
||||
cmd += ` \\\n --reasoning-parser qwen3`;
|
||||
}
|
||||
|
||||
// Tool Call Parser
|
||||
if (reasoningArray.includes('toolcall')) {
|
||||
cmd += ` \\\n --tool-call-parser mimo`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled = item.disabled;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,353 @@
|
||||
export const MiniMaxM2Deployment = () => {
|
||||
const modelFamily = 'MiniMaxAI';
|
||||
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'mi300x', label: 'MI300X', default: true },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
modelname: {
|
||||
name: 'modelname',
|
||||
title: 'Model Name',
|
||||
items: [
|
||||
{ id: 'M2.1', label: 'MiniMax-M2.1', default: true },
|
||||
{ id: 'M2', label: 'MiniMax-M2', default: false }
|
||||
]
|
||||
},
|
||||
strategy: {
|
||||
name: 'strategy',
|
||||
title: 'Deployment Strategy',
|
||||
type: 'checkbox',
|
||||
items: [
|
||||
{ id: 'tp', label: 'TP', default: true, required: true },
|
||||
]
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, modelname, strategy, reasoning, toolcall } = values;
|
||||
|
||||
const modelMap = {
|
||||
'M2.1': 'MiniMax-M2.1',
|
||||
'M2': 'MiniMax-M2'
|
||||
};
|
||||
|
||||
const modelName = `${modelFamily}/${modelMap[modelname]}`;
|
||||
|
||||
let cmd = 'sglang serve \\\n';
|
||||
cmd += ` --model-path ${modelName}`;
|
||||
|
||||
cmd += ` \\\n --tp 4`;
|
||||
|
||||
cmd += ` \\\n --trust-remote-code`;
|
||||
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --tool-call-parser minimax-m2`;
|
||||
}
|
||||
|
||||
if (reasoning === 'enabled') {
|
||||
cmd += ` \\\n --reasoning-parser minimax-append-think`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,390 @@
|
||||
export const MiniMaxM25Deployment = () => {
|
||||
const modelFamily = 'MiniMaxAI';
|
||||
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'a100', label: 'A100', default: false },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
gpuCount: {
|
||||
name: 'gpuCount',
|
||||
title: 'GPU Count',
|
||||
getDynamicItems: (values) => {
|
||||
const isAMD = values.hardware === 'mi300x' || values.hardware === 'mi325x' || values.hardware === 'mi355x';
|
||||
return [
|
||||
{
|
||||
id: '2gpu',
|
||||
label: '2',
|
||||
default: isAMD,
|
||||
disabled: !isAMD
|
||||
},
|
||||
{
|
||||
id: '4gpu',
|
||||
label: '4',
|
||||
default: !isAMD,
|
||||
disabled: false
|
||||
},
|
||||
{
|
||||
id: '8gpu',
|
||||
label: '8',
|
||||
default: false,
|
||||
disabled: false
|
||||
}
|
||||
];
|
||||
}
|
||||
},
|
||||
thinking: {
|
||||
name: 'thinking',
|
||||
title: 'Thinking Capabilities',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--reasoning-parser minimax-append-think' : null
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--tool-call-parser minimax-m2' : null
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, gpuCount, thinking, toolcall } = values;
|
||||
|
||||
const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x';
|
||||
if (gpuCount === '2gpu' && !isAMD) {
|
||||
return '# Please select compatible hardware\n# 2-GPU requires AMD MI300X/MI325X/MI355X';
|
||||
}
|
||||
|
||||
const modelName = `${modelFamily}/MiniMax-M2.5`;
|
||||
|
||||
let cmd = '';
|
||||
cmd += 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model-path ${modelName}`;
|
||||
|
||||
if (gpuCount === '8gpu') {
|
||||
cmd += ` \\\n --tp 8`;
|
||||
cmd += ` \\\n --ep 8`;
|
||||
} else if (gpuCount === '4gpu') {
|
||||
cmd += ` \\\n --tp 4`;
|
||||
if (isAMD) {
|
||||
cmd += ` \\\n --ep 4`;
|
||||
}
|
||||
} else if (gpuCount === '2gpu') {
|
||||
cmd += ` \\\n --tp 2`;
|
||||
if (isAMD) {
|
||||
cmd += ` \\\n --ep 2`;
|
||||
}
|
||||
}
|
||||
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ` \\\n --tool-call-parser minimax-m2`;
|
||||
}
|
||||
|
||||
if (thinking === 'enabled') {
|
||||
cmd += ` \\\n --reasoning-parser minimax-append-think`;
|
||||
}
|
||||
|
||||
cmd += ` \\\n --trust-remote-code`;
|
||||
cmd += ` \\\n --mem-fraction-static 0.85`;
|
||||
|
||||
if (isAMD) {
|
||||
cmd += ` \\\n --kv-cache-dtype fp8_e4m3`;
|
||||
cmd += ` \\\n --attention-backend triton`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,203 @@
|
||||
export const MiniMaxM27Deployment = () => {
|
||||
// Config options. `getDynamicItems(values)` is evaluated at render time so that
|
||||
// e.g. the 2-GPU option is only enabled on AMD or GB300 hardware.
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'gb300', label: 'GB300', default: false },
|
||||
{ id: 'a100', label: 'A100', default: false },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
gpuCount: {
|
||||
name: 'gpuCount',
|
||||
title: 'GPU Count',
|
||||
getDynamicItems: (values) => {
|
||||
const hw = values.hardware;
|
||||
const isAMD = hw === 'mi300x' || hw === 'mi325x' || hw === 'mi355x';
|
||||
const isGB300 = hw === 'gb300';
|
||||
const canUse2GPU = isAMD || isGB300;
|
||||
return [
|
||||
{ id: '2gpu', label: '2', default: canUse2GPU, disabled: !canUse2GPU },
|
||||
{ id: '4gpu', label: '4', default: !canUse2GPU, disabled: false },
|
||||
{ id: '8gpu', label: '8', default: false, disabled: isGB300 }
|
||||
];
|
||||
}
|
||||
},
|
||||
thinking: {
|
||||
name: 'thinking',
|
||||
title: 'Thinking Capabilities',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Helper: resolve an option's items (static or dynamic) given current values
|
||||
const resolveItems = (option, values) => {
|
||||
if (typeof option.getDynamicItems === 'function') {
|
||||
return option.getDynamicItems(values);
|
||||
}
|
||||
return option.items;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
// Resolve hardware first so gpuCount's dynamic items can see it
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
const items = resolveItems(option, initialState);
|
||||
const defaultItem = items.find(i => i.default && !i.disabled) || items.find(i => !i.disabled) || items[0];
|
||||
initialState[key] = defaultItem.id;
|
||||
}
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
// When hardware changes, re-evaluate gpuCount so disabled/default shifts apply
|
||||
useEffect(() => {
|
||||
setValues(prev => {
|
||||
const next = { ...prev };
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (typeof option.getDynamicItems !== 'function') continue;
|
||||
const items = option.getDynamicItems(next);
|
||||
const current = items.find(i => i.id === next[key]);
|
||||
if (!current || current.disabled) {
|
||||
const fallback = items.find(i => i.default && !i.disabled) || items.find(i => !i.disabled);
|
||||
if (fallback) next[key] = fallback.id;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}, [values.hardware]);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
// Generate command mirrors sgl-cookbook src/components/autoregressive/MiniMaxM27ConfigGenerator/index.js
|
||||
const generateCommand = () => {
|
||||
const { hardware, gpuCount, thinking, toolcall } = values;
|
||||
|
||||
const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x';
|
||||
const isGB300 = hardware === 'gb300';
|
||||
const canUse2GPU = isAMD || isGB300;
|
||||
|
||||
if (gpuCount === '2gpu' && !canUse2GPU) {
|
||||
return '# Please select compatible hardware\n# 2-GPU requires AMD MI300X/MI325X/MI355X or GB300';
|
||||
}
|
||||
|
||||
const modelName = 'MiniMaxAI/MiniMax-M2.7';
|
||||
|
||||
let cmd = 'sglang serve \\\n';
|
||||
cmd += ` --model-path ${modelName}`;
|
||||
|
||||
if (gpuCount === '8gpu') {
|
||||
cmd += ' \\\n --tp 8';
|
||||
cmd += ' \\\n --ep 8';
|
||||
} else if (gpuCount === '4gpu') {
|
||||
cmd += ' \\\n --tp 4';
|
||||
if (isAMD) cmd += ' \\\n --ep 4';
|
||||
} else if (gpuCount === '2gpu') {
|
||||
cmd += ' \\\n --tp 2';
|
||||
if (isAMD) cmd += ' \\\n --ep 2';
|
||||
}
|
||||
|
||||
if (toolcall === 'enabled') cmd += ' \\\n --tool-call-parser minimax-m2';
|
||||
if (thinking === 'enabled') cmd += ' \\\n --reasoning-parser minimax-append-think';
|
||||
|
||||
cmd += ' \\\n --trust-remote-code';
|
||||
cmd += ' \\\n --mem-fraction-static 0.85';
|
||||
|
||||
if (isAMD) {
|
||||
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
|
||||
cmd += ' \\\n --attention-backend triton';
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.4 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
const items = resolveItems(option, values);
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = !!item.disabled;
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}
|
||||
title={item.disabledReason || ''}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,348 @@
|
||||
export const Ministral3Deployment = () => {
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'mi300x', label: 'MI300x', default: true },
|
||||
{ id: 'mi325x', label: 'MI325x', default: false },
|
||||
{ id: 'mi355x', label: 'MI355x', default: false }
|
||||
]
|
||||
},
|
||||
model: {
|
||||
name: 'model',
|
||||
title: 'Model',
|
||||
items: [
|
||||
{ id: 'small', label: 'Ministral-3-8B-Instruct-2512', default: true },
|
||||
{ id: 'large', label: 'Ministral-3-14B-Instruct-2512', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'enabled', label: 'enabled', default: true },
|
||||
{ id: 'disabled', label: 'disabled', default: false }
|
||||
],
|
||||
commandRule: (value) => (value === 'enabled' ? '--tool-call-parser mistral' : null)
|
||||
}
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
small: {
|
||||
modelId: 'mistralai/Ministral-3-8B-Instruct-2512',
|
||||
tpByHardware: { mi300x: 1, mi325x: 1, mi355x: 1 }
|
||||
},
|
||||
large: {
|
||||
modelId: 'mistralai/Ministral-3-14B-Instruct-2512',
|
||||
tpByHardware: { mi300x: 1, mi325x: 1, mi355x: 1 }
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, model } = values;
|
||||
|
||||
const modelCfg = modelConfigs[model];
|
||||
if (!modelCfg) return `# Error: Unknown model selection: ${model}`;
|
||||
|
||||
const tp = modelCfg.tpByHardware[hardware];
|
||||
if (!tp) return `# Error: Unknown hardware platform: ${hardware}`;
|
||||
|
||||
let cmd = 'sglang serve \\\n';
|
||||
|
||||
cmd += ` --model ${modelCfg.modelId}`;
|
||||
|
||||
if (tp > 1) {
|
||||
cmd += ` \\\n --tp ${tp}`;
|
||||
}
|
||||
|
||||
cmd += ` \\\n --trust-remote-code`;
|
||||
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (option.commandRule) {
|
||||
const rule = option.commandRule(values[key]);
|
||||
if (rule) cmd += ` \\\n ${rule}`;
|
||||
}
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,365 @@
|
||||
export const MistralSmall4Deployment = () => {
|
||||
const modelId = 'mistralai/Mistral-Small-4-119B-2603';
|
||||
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
getDynamicItems: (values) => {
|
||||
const isNvfp4 = values.quantization === 'fp4';
|
||||
return [
|
||||
{ id: 'h100', label: 'H100', default: !isNvfp4, disabled: isNvfp4 },
|
||||
{ id: 'h200', label: 'H200', default: false, disabled: isNvfp4 },
|
||||
{ id: 'b200', label: 'B200', default: isNvfp4, disabled: false },
|
||||
{ id: 'b300', label: 'B300', default: false, disabled: false },
|
||||
];
|
||||
}
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'fp8', label: 'FP8', default: true },
|
||||
{ id: 'fp4', label: 'NVFP4', subtitle: 'Blackwell only', default: false },
|
||||
]
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--reasoning-parser mistral' : null
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--tool-call-parser mistral' : null
|
||||
},
|
||||
speculative: {
|
||||
name: 'speculative',
|
||||
title: 'Speculative Decoding (EAGLE)',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--speculative-algorithm EAGLE \\\n --speculative-draft-model-path mistralai/Mistral-Small-4-119B-2603-eagle \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4' : null
|
||||
},
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
h100: { fp8: { tp: 2 } },
|
||||
h200: { fp8: { tp: 2 } },
|
||||
b200: { fp8: { tp: 1 }, fp4: { tp: 1 } },
|
||||
b300: { fp8: { tp: 1 }, fp4: { tp: 1 } },
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, quantization } = values;
|
||||
|
||||
const hwConfig = modelConfigs[hardware]?.[quantization];
|
||||
if (!hwConfig) return `# Error: Unknown hardware/quantization combination`;
|
||||
|
||||
const { tp } = hwConfig;
|
||||
|
||||
const modelName = quantization === 'fp4'
|
||||
? 'mistralai/Mistral-Small-4-119B-2603-NVFP4'
|
||||
: modelId;
|
||||
|
||||
let cmd = `sglang serve --model-path ${modelName}`;
|
||||
cmd += ` \\\n --tp ${tp}`;
|
||||
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (key === 'quantization' || key === 'hardware') return;
|
||||
if (option.commandRule) {
|
||||
const rule = option.commandRule(values[key]);
|
||||
if (rule) cmd += ` \\\n ${rule}`;
|
||||
}
|
||||
});
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,371 @@
|
||||
export const Nemotron3NanoDeployment = () => {
|
||||
const modelFamily = 'nvidia';
|
||||
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'b200', label: 'B200', default: true }
|
||||
]
|
||||
},
|
||||
modelVariant: {
|
||||
name: 'modelVariant',
|
||||
title: 'Model Variant',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false },
|
||||
{ id: 'nvfp4', label: 'NVFP4', default: false }
|
||||
]
|
||||
},
|
||||
tp: {
|
||||
name: 'tp',
|
||||
title: 'Tensor Parallel (TP)',
|
||||
items: [
|
||||
{ id: '1', label: 'TP=1', default: true },
|
||||
{ id: '2', label: 'TP=2', default: false },
|
||||
{ id: '4', label: 'TP=4', default: false },
|
||||
{ id: '8', label: 'TP=8', default: false }
|
||||
]
|
||||
},
|
||||
kvcache: {
|
||||
name: 'kvcache',
|
||||
title: 'KV Cache DType',
|
||||
items: [
|
||||
{ id: 'fp8_e4m3', label: 'fp8_e4m3', default: true },
|
||||
{ id: 'bf16', label: 'bf16', default: false }
|
||||
]
|
||||
},
|
||||
thinking: {
|
||||
name: 'thinking',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--reasoning-parser nemotron_3' : null
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--tool-call-parser qwen3_coder' : null
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, modelVariant, tp, kvcache, thinking, toolcall } = values;
|
||||
|
||||
// Default to FP8 if not selected
|
||||
const variant = modelVariant || 'fp8';
|
||||
const baseName = 'NVIDIA-Nemotron-3-Nano-30B-A3B';
|
||||
|
||||
const modelName = `${modelFamily}/${baseName}-${variant.toUpperCase()}`;
|
||||
|
||||
let cmd = 'python3 -m sglang.launch_server \\\n';
|
||||
cmd += ` --model-path ${modelName} \\\n`;
|
||||
cmd += ` --trust-remote-code \\\n`;
|
||||
cmd += ` --tp ${tp} \\\n`;
|
||||
cmd += ` --kv-cache-dtype ${kvcache} \\\n`;
|
||||
|
||||
// Add thinking parser and tool call parser if enabled
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (option.commandRule) {
|
||||
const rule = option.commandRule(values[key]);
|
||||
if (rule) {
|
||||
cmd += ` ${rule} \\\n`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove trailing backslash from last option
|
||||
cmd = cmd.trimEnd();
|
||||
if (cmd.endsWith('\\')) {
|
||||
cmd = cmd.slice(0, -1).trimEnd();
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,381 @@
|
||||
export const Nemotron3SuperDeployment = () => {
|
||||
const MODEL_PATHS = {
|
||||
bf16: 'nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16',
|
||||
fp8: 'nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8',
|
||||
nvfp4: 'nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4',
|
||||
};
|
||||
|
||||
const options = {
|
||||
model: {
|
||||
name: 'model',
|
||||
title: 'Model',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false },
|
||||
{ id: 'nvfp4', label: 'NVFP4', default: false },
|
||||
]
|
||||
},
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'b200', label: 'B200', default: true }
|
||||
]
|
||||
},
|
||||
tp: {
|
||||
name: 'tp',
|
||||
title: 'Tensor Parallel (TP)',
|
||||
items: [
|
||||
{ id: '2', label: 'TP=2', default: false },
|
||||
{ id: '4', label: 'TP=4', default: true },
|
||||
{ id: '8', label: 'TP=8', default: false }
|
||||
]
|
||||
},
|
||||
mtp: {
|
||||
name: 'mtp',
|
||||
title: 'Multi-token Prediction (MTP)',
|
||||
items: [
|
||||
{ id: 'enabled', label: 'Enabled', default: false },
|
||||
{ id: 'disabled', label: 'Disabled', default: true }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4 \\\n --disable-radix-cache' : null
|
||||
},
|
||||
kvcache: {
|
||||
name: 'kvcache',
|
||||
title: 'KV Cache DType',
|
||||
items: [
|
||||
{ id: 'none', label: 'None', default: true },
|
||||
{ id: 'fp8_e4m3', label: 'fp8_e4m3', default: false },
|
||||
{ id: 'bf16', label: 'bf16', default: false }
|
||||
]
|
||||
},
|
||||
thinking: {
|
||||
name: 'thinking',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
{ id: 'disabled', label: 'Disabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--reasoning-parser nemotron_3' : null
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
{ id: 'disabled', label: 'Disabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--tool-call-parser qwen3_coder' : null
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { tp, kvcache, model } = values;
|
||||
|
||||
const modelPath = MODEL_PATHS[model] || MODEL_PATHS['bf16'];
|
||||
|
||||
let cmd = 'python3 -m sglang.launch_server \\\n';
|
||||
cmd += ` --model-path ${modelPath} \\\n`;
|
||||
cmd += ` --trust-remote-code \\\n`;
|
||||
cmd += ` --tp ${tp} \\\n`;
|
||||
|
||||
if (kvcache && kvcache !== 'none') {
|
||||
cmd += ` --kv-cache-dtype ${kvcache} \\\n`;
|
||||
}
|
||||
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (option.commandRule) {
|
||||
const rule = option.commandRule(values[key]);
|
||||
if (rule) {
|
||||
cmd += ` ${rule} \\\n`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cmd = cmd.trimEnd();
|
||||
if (cmd.endsWith('\\')) {
|
||||
cmd = cmd.slice(0, -1).trimEnd();
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,364 @@
|
||||
export const Qwen25VLDeployment = () => {
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'mi300x', label: 'MI300X', default: true },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
modelsize: {
|
||||
name: 'modelsize',
|
||||
title: 'Model Size',
|
||||
items: [
|
||||
{ id: '72b', label: '72B', subtitle: 'Dense', default: true },
|
||||
{ id: '32b', label: '32B', subtitle: 'Dense', default: false },
|
||||
{ id: '7b', label: '7B', subtitle: 'Dense', default: false },
|
||||
{ id: '3b', label: '3B', subtitle: 'Dense', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
'72b': {
|
||||
baseName: '72B',
|
||||
mi300x: { tp: 8, ep: 0 },
|
||||
mi325x: { tp: 8, ep: 0 },
|
||||
mi355x: { tp: 8, ep: 0 }
|
||||
},
|
||||
'32b': {
|
||||
baseName: '32B',
|
||||
mi300x: { tp: 2, ep: 0 },
|
||||
mi325x: { tp: 2, ep: 0 },
|
||||
mi355x: { tp: 2, ep: 0 }
|
||||
},
|
||||
'7b': {
|
||||
baseName: '7B',
|
||||
mi300x: { tp: 1, ep: 0 },
|
||||
mi325x: { tp: 1, ep: 0 },
|
||||
mi355x: { tp: 1, ep: 0 }
|
||||
},
|
||||
'3b': {
|
||||
baseName: '3B',
|
||||
mi300x: { tp: 1, ep: 0 },
|
||||
mi325x: { tp: 1, ep: 0 },
|
||||
mi355x: { tp: 1, ep: 0 }
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, modelsize: modelSize } = values;
|
||||
|
||||
const modelSizeConfig = modelConfigs[modelSize];
|
||||
if (!modelSizeConfig) {
|
||||
return `# Error: Unknown model size: ${modelSize}`;
|
||||
}
|
||||
|
||||
const hwConfig = modelSizeConfig[hardware];
|
||||
if (!hwConfig) {
|
||||
return `# Error: Unknown hardware platform: ${hardware}`;
|
||||
}
|
||||
|
||||
const modelName = `Qwen/Qwen2.5-VL-${modelSizeConfig.baseName}-Instruct`;
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
if (hwConfig.tp > 1) {
|
||||
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
||||
}
|
||||
|
||||
if ((hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') && modelSize === '72b') {
|
||||
cmd += ` \\\n --context-length 128000`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,139 @@
|
||||
export const Qwen3CoderDeployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'mi300x', label: 'MI300X', default: true }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Model configurations
|
||||
const modelConfigs = {
|
||||
'480b': {
|
||||
baseName: '480B-A35B',
|
||||
mi300x: { tp: 8, ep: 0 }
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, quantization } = values;
|
||||
|
||||
const config = modelConfigs['480b'];
|
||||
const hwConfig = config[hardware];
|
||||
|
||||
if (!hwConfig) {
|
||||
return `# Error: Unknown hardware platform: ${hardware}`;
|
||||
}
|
||||
|
||||
// Build model name
|
||||
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
const modelName = `Qwen/Qwen3-Coder-${config.baseName}-Instruct${quantSuffix}`;
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
// TP is always 8 for this model
|
||||
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
||||
|
||||
// FP8 requires EP=2 for MoE dimension alignment
|
||||
if (quantization === 'fp8') {
|
||||
cmd += ` \\\n --ep 2`;
|
||||
}
|
||||
|
||||
// Context length verified on MI300X
|
||||
cmd += ` \\\n --context-length 8192`;
|
||||
|
||||
// Page size for MoE models
|
||||
cmd += ` \\\n --page-size 32`;
|
||||
|
||||
// FP8 requires trust-remote-code
|
||||
if (quantization === 'fp8') {
|
||||
cmd += ` \\\n --trust-remote-code`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = item.disabled;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} disabled={isDisabled} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,426 @@
|
||||
export const Qwen3CoderDeployment = () => {
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'mi300x', label: 'MI300X', default: true },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'gb200', label: 'GB200', default: false }
|
||||
]
|
||||
},
|
||||
modelSize: {
|
||||
name: 'modelSize',
|
||||
title: 'Model Size',
|
||||
items: [
|
||||
{ id: '480b', label: '480B', subtitle: 'MOE', default: true },
|
||||
{ id: '30b', label: '30B', subtitle: 'MOE', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false },
|
||||
{ id: 'nvfp4', label: 'NVFP4', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--tool-call-parser qwen3_coder' : null
|
||||
}
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
'480b': {
|
||||
baseName: '480B-A35B',
|
||||
mi300x: { tp: 8 },
|
||||
mi325x: { tp: 8 },
|
||||
mi355x: { tp: 8 },
|
||||
b200: { tp: 8 },
|
||||
gb200: { tp: 8 }
|
||||
},
|
||||
'30b': {
|
||||
baseName: '30B-A3B',
|
||||
mi300x: { tp: 1 },
|
||||
mi325x: { tp: 1 },
|
||||
mi355x: { tp: 1 }
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, modelSize, quantization } = values;
|
||||
|
||||
const isNvidia = hardware === 'b200' || hardware === 'gb200';
|
||||
|
||||
const modelConfig = modelConfigs[modelSize];
|
||||
const hwConfig = modelConfig[hardware];
|
||||
|
||||
if (!hwConfig) {
|
||||
return `# Configuration not available: ${modelSize.toUpperCase()} model has not been verified on ${hardware.toUpperCase()}.`;
|
||||
}
|
||||
|
||||
// NVFP4 is only available on NVIDIA hardware
|
||||
if (quantization === 'nvfp4' && !isNvidia) {
|
||||
return `# NVFP4 quantization is only available on NVIDIA B200/GB200 hardware.`;
|
||||
}
|
||||
|
||||
// BF16 not verified on NVIDIA
|
||||
if (quantization === 'bf16' && isNvidia) {
|
||||
return `# BF16 deployment on ${hardware.toUpperCase()} has not been verified yet. Please use FP8 or NVFP4.`;
|
||||
}
|
||||
|
||||
// Build model name
|
||||
let modelName;
|
||||
if (quantization === 'nvfp4') {
|
||||
modelName = `nvidia/Qwen3-Coder-${modelConfig.baseName}-Instruct-NVFP`;
|
||||
} else {
|
||||
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
modelName = `Qwen/Qwen3-Coder-${modelConfig.baseName}-Instruct${quantSuffix}`;
|
||||
}
|
||||
|
||||
let cmd = '';
|
||||
if (!isNvidia) {
|
||||
cmd += 'SGLANG_USE_AITER=0 ';
|
||||
}
|
||||
cmd += 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
// TP setting
|
||||
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
||||
|
||||
// EP and DP attention settings
|
||||
if (quantization === 'nvfp4') {
|
||||
cmd += ` \\\n --ep 1`;
|
||||
cmd += ` \\\n --enable-dp-attention`;
|
||||
} else if (modelSize === '480b' && quantization === 'fp8') {
|
||||
// FP8 requires EP=2 for 480B model due to MoE dimension alignment
|
||||
// moe_intermediate_size=2560, with tp=8 ep=1: 2560/8=320, 320%128!=0
|
||||
// with tp=8 ep=2: 2560/4=640, 640%128=0 ✓
|
||||
cmd += ` \\\n --ep 2`;
|
||||
}
|
||||
|
||||
// MOE runner backend for NVIDIA
|
||||
if (isNvidia) {
|
||||
if (quantization === 'nvfp4') {
|
||||
cmd += ` \\\n --moe-runner-backend flashinfer_cutlass`;
|
||||
cmd += ` \\\n --quantization modelopt_fp4`;
|
||||
} else if (quantization === 'fp8') {
|
||||
cmd += ` \\\n --moe-runner-backend triton`;
|
||||
}
|
||||
}
|
||||
|
||||
// Apply commandRule from all options
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.commandRule && values[key]) {
|
||||
// Pass the full values object so commandRule can access other option values
|
||||
const additionalCmd = option.commandRule(values[key], values);
|
||||
if (additionalCmd) {
|
||||
cmd += ` \\\n ${additionalCmd}`;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// AMD-specific flags
|
||||
if (!isNvidia) {
|
||||
// Context length verified on MI300X/MI325X/MI355X
|
||||
cmd += ` \\\n --context-length 8192`;
|
||||
|
||||
// Page size for MoE models
|
||||
cmd += ` \\\n --page-size 32`;
|
||||
|
||||
// FP8 requires trust-remote-code
|
||||
if (quantization === 'fp8') {
|
||||
cmd += ` \\\n --trust-remote-code`;
|
||||
}
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,370 @@
|
||||
export const Qwen3CoderNextDeployment = () => {
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
{ id: 'disabled', label: 'Disabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--tool-call-parser qwen3_coder' : null
|
||||
},
|
||||
mambaCache: {
|
||||
name: 'mambaCache',
|
||||
title: 'Mamba Radix Cache',
|
||||
items: [
|
||||
{ id: 'v1', label: 'V1', default: true },
|
||||
{ id: 'v2', label: 'V2', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'v2' ? '--mamba-scheduler-strategy extra_buffer \\\n --page-size 64' : null
|
||||
}
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
default: {
|
||||
baseName: 'Qwen3-Coder-Next',
|
||||
h100: { bf16: { tp: 4 }, fp8: { tp: 2 } },
|
||||
h200: { bf16: { tp: 2 }, fp8: { tp: 1 } },
|
||||
b200: { bf16: { tp: 2 }, fp8: { tp: 1 } },
|
||||
mi300x: { bf16: { tp: 2 }, fp8: { tp: 1 } },
|
||||
mi325x: { bf16: { tp: 2 }, fp8: { tp: 1 } },
|
||||
mi355x: { bf16: { tp: 2 }, fp8: { tp: 1 } }
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, quantization } = values;
|
||||
|
||||
const hwConfig = modelConfigs.default[hardware];
|
||||
if (!hwConfig) {
|
||||
return `# Error: Unknown hardware platform: ${hardware}`;
|
||||
}
|
||||
|
||||
const quantConfig = hwConfig[quantization];
|
||||
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
const modelName = `Qwen/${modelConfigs.default.baseName}${quantSuffix}`;
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
// TP setting
|
||||
if (quantConfig.tp > 1) {
|
||||
cmd += ` \\\n --tp ${quantConfig.tp}`;
|
||||
}
|
||||
|
||||
// Apply commandRule from all options
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.commandRule && values[key]) {
|
||||
const additionalCmd = option.commandRule(values[key], values);
|
||||
if (additionalCmd) {
|
||||
cmd += ` \\\n ${additionalCmd}`;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// AMD GPUs require triton attention backend
|
||||
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||
cmd += ` \\\n --attention-backend triton`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,330 @@
|
||||
export const Qwen3Deployment = () => {
|
||||
// Model configurations
|
||||
const modelConfigs = {
|
||||
'235b': {
|
||||
baseName: '235B-A22B',
|
||||
hasThinkingVariants: true,
|
||||
h100: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||
h200: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||
b200: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||
mi300x: { tp: 4, ep: 0, bf16: true, fp8: true },
|
||||
mi325x: { tp: 4, ep: 0, bf16: true, fp8: true },
|
||||
mi355x: { tp: 4, ep: 0, bf16: true, fp8: true }
|
||||
},
|
||||
'30b': {
|
||||
baseName: '30B-A3B',
|
||||
hasThinkingVariants: true,
|
||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }
|
||||
},
|
||||
'32b': {
|
||||
baseName: '32B',
|
||||
hasThinkingVariants: false,
|
||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }
|
||||
},
|
||||
'14b': {
|
||||
baseName: '14B',
|
||||
hasThinkingVariants: false,
|
||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }
|
||||
},
|
||||
'8b': {
|
||||
baseName: '8B',
|
||||
hasThinkingVariants: false,
|
||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }
|
||||
},
|
||||
'4b': {
|
||||
baseName: '4B',
|
||||
hasThinkingVariants: true,
|
||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }
|
||||
},
|
||||
'1.7b': {
|
||||
baseName: '1.7B',
|
||||
hasThinkingVariants: false,
|
||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }
|
||||
},
|
||||
'0.6b': {
|
||||
baseName: '0.6B',
|
||||
hasThinkingVariants: false,
|
||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }
|
||||
}
|
||||
};
|
||||
|
||||
// Base options
|
||||
const baseOptions = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
modelsize: {
|
||||
name: 'modelsize',
|
||||
title: 'Model Size',
|
||||
items: [
|
||||
{ id: '235b', label: '235B', subtitle: 'MOE', default: true },
|
||||
{ id: '30b', label: '30B', subtitle: 'MOE', default: false },
|
||||
{ id: '32b', label: '32B', subtitle: 'Dense', default: false },
|
||||
{ id: '14b', label: '14B', subtitle: 'Dense', default: false },
|
||||
{ id: '8b', label: '8B', subtitle: 'Dense', default: false },
|
||||
{ id: '4b', label: '4B', subtitle: 'Dense', default: false },
|
||||
{ id: '1.7b', label: '1.7B', subtitle: 'Dense', default: false },
|
||||
{ id: '0.6b', label: '0.6B', subtitle: 'Dense', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false }
|
||||
]
|
||||
},
|
||||
category: {
|
||||
name: 'category',
|
||||
title: 'Categories',
|
||||
items: [
|
||||
{ id: 'base', label: 'Base', default: true },
|
||||
{ id: 'instruct', label: 'Instruct', default: false },
|
||||
{ id: 'thinking', label: 'Thinking', default: false }
|
||||
]
|
||||
},
|
||||
reasoningParser: {
|
||||
name: 'reasoningParser',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Get dynamic options based on current values
|
||||
const getDisplayOptions = (values) => {
|
||||
const options = { ...baseOptions };
|
||||
const currentModelConfig = modelConfigs[values.modelsize];
|
||||
|
||||
// If model doesn't have thinking variants, disable non-base category options
|
||||
if (currentModelConfig && !currentModelConfig.hasThinkingVariants) {
|
||||
options.category = {
|
||||
...baseOptions.category,
|
||||
items: baseOptions.category.items.map(item => ({
|
||||
...item,
|
||||
disabled: item.id !== 'base'
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
// Only show reasoningParser when category is not 'instruct'
|
||||
if (values.category === 'instruct') {
|
||||
delete options.reasoningParser;
|
||||
}
|
||||
|
||||
return options;
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(baseOptions).forEach(([key, option]) => {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => {
|
||||
const newValues = { ...prev, [optionName]: value };
|
||||
|
||||
// Auto-switch to 'base' category for models without thinking variants
|
||||
if (optionName === 'modelsize') {
|
||||
const modelConfig = modelConfigs[value];
|
||||
if (modelConfig && !modelConfig.hasThinkingVariants) {
|
||||
if (newValues.category !== 'base') {
|
||||
newValues.category = 'base';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reset reasoningParser when switching to 'instruct' category
|
||||
if (optionName === 'category' && value === 'instruct') {
|
||||
newValues.reasoningParser = 'disabled';
|
||||
}
|
||||
|
||||
return newValues;
|
||||
});
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, modelsize, quantization, category, reasoningParser, toolcall } = values;
|
||||
const displayOptions = getDisplayOptions(values);
|
||||
|
||||
// Special error handling
|
||||
const commandKey = `${hardware}-${modelsize}-${quantization}-${category}`;
|
||||
if (commandKey === 'h100-235b-bf16-instruct' || commandKey === 'h100-235b-bf16-thinking') {
|
||||
return '# Error: Model is too large, cannot fit into 8*H100\n# Please use H200 (141GB) or select FP8 quantization';
|
||||
}
|
||||
|
||||
const config = modelConfigs[modelsize];
|
||||
if (!config) {
|
||||
return `# Error: Unknown model size: ${modelsize}`;
|
||||
}
|
||||
|
||||
const hwConfig = config[hardware];
|
||||
if (!hwConfig) {
|
||||
return `# Error: Unknown hardware platform: ${hardware}`;
|
||||
}
|
||||
|
||||
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
|
||||
// Build model name based on model category
|
||||
let modelName;
|
||||
if (config.hasThinkingVariants) {
|
||||
if (category === 'base') {
|
||||
modelName = `Qwen/Qwen3-${config.baseName}${quantSuffix}`;
|
||||
} else {
|
||||
const thinkingSuffix = category === 'thinking' ? '-Thinking' : '-Instruct';
|
||||
const dateSuffix = '-2507';
|
||||
modelName = `Qwen/Qwen3-${config.baseName}${thinkingSuffix}${dateSuffix}${quantSuffix}`;
|
||||
}
|
||||
} else {
|
||||
modelName = `Qwen/Qwen3-${config.baseName}${quantSuffix}`;
|
||||
}
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
if (hwConfig.tp > 1) {
|
||||
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
||||
}
|
||||
|
||||
let ep = hwConfig.ep;
|
||||
if (quantization === 'fp8' && hwConfig.tp === 8) {
|
||||
ep = 2;
|
||||
}
|
||||
|
||||
if (ep > 0) {
|
||||
cmd += ` \\\n --ep ${ep}`;
|
||||
}
|
||||
|
||||
// Add reasoning parser
|
||||
if (reasoningParser === 'enabled' && category !== 'instruct') {
|
||||
cmd += ' \\\n --reasoning-parser qwen3';
|
||||
}
|
||||
|
||||
// Add tool call parser
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ' \\\n --tool-call-parser qwen25';
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Get current display options
|
||||
const displayOptions = getDisplayOptions(values);
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(displayOptions).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = item.disabled;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} disabled={isDisabled} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,409 @@
|
||||
export const Qwen3NextDeployment = () => {
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
modelsize: {
|
||||
name: 'modelsize',
|
||||
title: 'Model Size',
|
||||
items: [
|
||||
{ id: '80b', label: '80B', subtitle: 'MOE', default: true },
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', subtitle: 'Full Weights', default: true },
|
||||
{ id: 'fp8', label: 'FP8', subtitle: 'High Throughput', default: false }
|
||||
]
|
||||
},
|
||||
thinking: {
|
||||
name: 'thinking',
|
||||
title: 'Thinking Capabilities',
|
||||
items: [
|
||||
{ id: 'instruct', label: 'Instruct', subtitle: 'General Purpose', default: true },
|
||||
{ id: 'thinking', label: 'Thinking', subtitle: 'Reasoning / CoT', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'thinking' ? '--reasoning-parser qwen3' : null
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--tool-call-parser qwen' : null
|
||||
},
|
||||
speculative: {
|
||||
name: 'speculative',
|
||||
title: 'Speculative Decoding',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4' : null
|
||||
},
|
||||
mambaCache: {
|
||||
name: 'mambaCache',
|
||||
title: 'Mamba Radix Cache',
|
||||
items: [
|
||||
{ id: 'v1', label: 'V1', default: true },
|
||||
{ id: 'v2', label: 'V2', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'v2' ? '--mamba-scheduler-strategy extra_buffer \\\n --page-size 64' : null
|
||||
}
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
'80b': {
|
||||
baseName: '80B-A3B',
|
||||
isMOE: true,
|
||||
h100: { tp: 4, ep: 0, bf16: true, fp8: true },
|
||||
h200: { tp: 2, ep: 0, bf16: true, fp8: true },
|
||||
b200: { tp: 2, ep: 0, bf16: true, fp8: true },
|
||||
mi300x: { tp: 2, ep: 0, bf16: true, fp8: true },
|
||||
mi325x: { tp: 2, ep: 0, bf16: true, fp8: true },
|
||||
mi355x: { tp: 2, ep: 0, bf16: true, fp8: true }
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, modelsize: modelSize, quantization, thinking } = values;
|
||||
const commandKey = `${hardware}-${modelSize}-${quantization}-${thinking}`;
|
||||
|
||||
const modelSizeConfig = modelConfigs[modelSize];
|
||||
if (!modelSizeConfig) {
|
||||
return `# Error: Unknown model size: ${modelSize}`;
|
||||
}
|
||||
|
||||
const hwConfig = modelSizeConfig[hardware];
|
||||
if (!hwConfig) {
|
||||
return `# Error: Unknown hardware platform: ${hardware}`;
|
||||
}
|
||||
|
||||
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
const thinkingSuffix = thinking === 'thinking' ? '-Thinking' : '-Instruct';
|
||||
const modelName = `Qwen/Qwen3-Next-${modelSizeConfig.baseName}${thinkingSuffix}${quantSuffix}`;
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
if (hwConfig.tp > 1) {
|
||||
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
||||
}
|
||||
|
||||
let ep = hwConfig.ep;
|
||||
if (quantization === 'fp8' && hwConfig.tp === 8) {
|
||||
ep = 2;
|
||||
}
|
||||
|
||||
if (ep > 0) {
|
||||
cmd += ` \\\n --ep ${ep}`;
|
||||
}
|
||||
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (option.commandRule) {
|
||||
const rule = option.commandRule(values[key]);
|
||||
if (rule) {
|
||||
cmd += ` \\\n ${rule}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AMD GPUs require triton attention backend
|
||||
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||
cmd += ` \\\n --attention-backend triton`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,245 @@
|
||||
export const Qwen3VLDeployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
modelsize: {
|
||||
name: 'modelsize',
|
||||
title: 'Model Size',
|
||||
items: [
|
||||
{ id: '235b', label: '235B', subtitle: 'MOE', default: true },
|
||||
{ id: '30b', label: '30B', subtitle: 'MOE', default: false },
|
||||
{ id: '32b', label: '32B', subtitle: 'Dense', default: false },
|
||||
{ id: '8b', label: '8B', subtitle: 'Dense', default: false },
|
||||
{ id: '4b', label: '4B', subtitle: 'Dense', default: false },
|
||||
{ id: '2b', label: '2B', subtitle: 'Dense', default: false }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false }
|
||||
]
|
||||
},
|
||||
thinking: {
|
||||
name: 'thinking',
|
||||
title: 'Thinking Capabilities',
|
||||
items: [
|
||||
{ id: 'instruct', label: 'Instruct', default: true },
|
||||
{ id: 'thinking', label: 'Thinking', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Model configurations
|
||||
const modelConfigs = {
|
||||
'235b': {
|
||||
baseName: '235B-A22B',
|
||||
isMOE: true,
|
||||
h100: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||
h200: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||
b200: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||
mi300x: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||
mi325x: { tp: 8, ep: 0, bf16: true, fp8: true },
|
||||
mi355x: { tp: 8, ep: 0, bf16: true, fp8: true }
|
||||
},
|
||||
'30b': {
|
||||
baseName: '30B-A3B',
|
||||
isMOE: true,
|
||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }
|
||||
},
|
||||
'32b': {
|
||||
baseName: '32B',
|
||||
isMOE: false,
|
||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }
|
||||
},
|
||||
'8b': {
|
||||
baseName: '8B',
|
||||
isMOE: false,
|
||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }
|
||||
},
|
||||
'4b': {
|
||||
baseName: '4B',
|
||||
isMOE: false,
|
||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }
|
||||
},
|
||||
'2b': {
|
||||
baseName: '2B',
|
||||
isMOE: false,
|
||||
h100: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
h200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
b200: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi300x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi325x: { tp: 1, ep: 0, bf16: true, fp8: true },
|
||||
mi355x: { tp: 1, ep: 0, bf16: true, fp8: true }
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { hardware, modelsize, quantization, thinking, toolcall } = values;
|
||||
const commandKey = `${hardware}-${modelsize}-${quantization}-${thinking}`;
|
||||
|
||||
// Special error handling
|
||||
if (commandKey === 'h100-235b-bf16-instruct' || commandKey === 'h100-235b-bf16-thinking') {
|
||||
return '# Error: Model is too large, cannot fit into 8*H100\n# Please use H200 (141GB) or select FP8 quantization';
|
||||
}
|
||||
|
||||
const config = modelConfigs[modelsize];
|
||||
if (!config) {
|
||||
return `# Error: Unknown model size: ${modelsize}`;
|
||||
}
|
||||
|
||||
const hwConfig = config[hardware];
|
||||
if (!hwConfig) {
|
||||
return `# Error: Unknown hardware platform: ${hardware}`;
|
||||
}
|
||||
|
||||
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
const thinkingSuffix = thinking === 'thinking' ? '-Thinking' : '-Instruct';
|
||||
const modelName = `Qwen/Qwen3-VL-${config.baseName}${thinkingSuffix}${quantSuffix}`;
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
if (hwConfig.tp > 1) {
|
||||
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
||||
}
|
||||
|
||||
let ep = hwConfig.ep;
|
||||
if (quantization === 'fp8' && hwConfig.tp === 8) {
|
||||
ep = 2;
|
||||
}
|
||||
|
||||
if (ep > 0) {
|
||||
cmd += ` \\\n --ep ${ep}`;
|
||||
}
|
||||
|
||||
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||
if (modelsize === '32b' && quantization === 'bf16') {
|
||||
cmd += ` \\\n --context-length 65536`;
|
||||
}
|
||||
}
|
||||
|
||||
if (thinking === 'thinking') {
|
||||
cmd += ' \\\n --reasoning-parser qwen3';
|
||||
}
|
||||
|
||||
if (toolcall === 'enabled') {
|
||||
cmd += ' \\\n --tool-call-parser qwen';
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = item.disabled;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} disabled={isDisabled} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,442 @@
|
||||
export const Qwen35Deployment = () => {
|
||||
// Qwen3.5 Configuration Generator
|
||||
//
|
||||
// MoE models (Gated Delta Networks + sparse MoE, hybrid architecture):
|
||||
// 397B-A17B, 122B-A10B, 35B-A3B
|
||||
//
|
||||
// Dense models (standard transformer):
|
||||
// 27B, 9B, 4B, 2B, 0.8B
|
||||
//
|
||||
// GPU requirements (BF16):
|
||||
// 397B-A17B: H100 tp=16, H200 tp=8, B200 tp=8, B300 tp=4, MI300X tp=8, MI325X tp=4, MI355X tp=4
|
||||
// 122B-A10B: H100 tp=4, H200 tp=2, B200 tp=2, B300 tp=1, MI300X tp=2, MI325X tp=1, MI355X tp=1
|
||||
// 35B-A3B: H100 tp=1, H200 tp=1, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1
|
||||
// 27B/9B/4B/2B/0.8B: tp=1 on all hardware (including MI300X, MI325X, MI355X)
|
||||
//
|
||||
// GPU requirements (FP8, where available):
|
||||
// 397B-A17B: H100 tp=8, H200 tp=8 ep=8, B200 tp=4, B300 tp=2, MI300X tp=4, MI325X tp=2, MI355X tp=2
|
||||
// 122B-A10B: H100 tp=2, H200 tp=1, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1
|
||||
// 35B-A3B: H100 tp=1, H200 tp=1, B200 tp=1, B300 tp=1, MI300X tp=1, MI325X tp=1, MI355X tp=1
|
||||
// 27B: tp=1 on all hardware (including MI300X, MI325X, MI355X)
|
||||
//
|
||||
// FP4 (397B only, Blackwell required): B200 tp=4, B300 tp=2
|
||||
|
||||
const MOE_MODELS = new Set(['397b', '122b', '35b']);
|
||||
const FP8_MODELS = new Set(['397b', '122b', '35b', '27b']);
|
||||
|
||||
// Maps model id -> HuggingFace model name suffix
|
||||
const MODEL_SUFFIX = {
|
||||
'397b': '397B-A17B',
|
||||
'122b': '122B-A10B',
|
||||
'35b': '35B-A3B',
|
||||
'27b': '27B',
|
||||
'9b': '9B',
|
||||
'4b': '4B',
|
||||
'2b': '2B',
|
||||
'0.8b': '0.8B',
|
||||
};
|
||||
|
||||
const options = {
|
||||
model: {
|
||||
name: 'model',
|
||||
title: 'Model Variant',
|
||||
items: [
|
||||
{ id: '397b', label: '397B', subtitle: 'MoE', default: true },
|
||||
{ id: '122b', label: '122B', subtitle: 'MoE', default: false },
|
||||
{ id: '35b', label: '35B', subtitle: 'MoE', default: false },
|
||||
{ id: '27b', label: '27B', subtitle: 'Dense', default: false },
|
||||
{ id: '9b', label: '9B', subtitle: 'Dense', default: false },
|
||||
{ id: '4b', label: '4B', subtitle: 'Dense', default: false },
|
||||
{ id: '2b', label: '2B', subtitle: 'Dense', default: false },
|
||||
{ id: '0.8b', label: '0.8B', subtitle: 'Dense', default: false },
|
||||
]
|
||||
},
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
getDynamicItems: (values) => {
|
||||
const isNvfp4 = values.quantization === 'fp4';
|
||||
return [
|
||||
{ id: 'h100', label: 'H100', default: !isNvfp4, disabled: isNvfp4 },
|
||||
{ id: 'h200', label: 'H200', default: false, disabled: isNvfp4 },
|
||||
{ id: 'b200', label: 'B200', default: false, disabled: false },
|
||||
{ id: 'b300', label: 'B300', default: isNvfp4, disabled: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false, disabled: isNvfp4 },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false, disabled: isNvfp4 },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false, disabled: isNvfp4 }
|
||||
];
|
||||
}
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
getDynamicItems: (values) => {
|
||||
const hasFp8 = FP8_MODELS.has(values.model);
|
||||
const hasFp4 = values.model === '397b';
|
||||
return [
|
||||
{ id: 'bf16', label: 'BF16', default: !hasFp8 },
|
||||
{ id: 'fp8', label: 'FP8', default: hasFp8, disabled: !hasFp8,
|
||||
disabledReason: 'No FP8 variant available for this model' },
|
||||
{ id: 'fp4', label: 'FP4', default: false, disabled: !hasFp4,
|
||||
disabledReason: 'FP4 is only available for Qwen3.5-397B-A17B' }
|
||||
];
|
||||
}
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
]
|
||||
},
|
||||
speculative: {
|
||||
name: 'speculative',
|
||||
title: 'Speculative Decoding (MTP)',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true }
|
||||
]
|
||||
},
|
||||
mambaCache: {
|
||||
name: 'mambaCache',
|
||||
title: 'Mamba Radix Cache',
|
||||
condition: (values) => MOE_MODELS.has(values.model),
|
||||
getDynamicItems: (currentValues) => {
|
||||
const amdGpus = ['mi300x', 'mi325x', 'mi355x'];
|
||||
const isAmdGpu = amdGpus.includes(currentValues.hardware);
|
||||
const mtpEnabled = currentValues.speculative === 'enabled';
|
||||
|
||||
// MTP requires V2 mamba radix cache
|
||||
if (mtpEnabled && !isAmdGpu) {
|
||||
return [
|
||||
{ id: 'v1', label: 'V1', default: false, disabled: true },
|
||||
{ id: 'v2', label: 'V2', default: true }
|
||||
];
|
||||
}
|
||||
|
||||
// Show V2 as disabled for AMD GPUs (V2 requires FLA backend, NVIDIA only)
|
||||
if (isAmdGpu) {
|
||||
return [
|
||||
{ id: 'v1', label: 'V1', default: true },
|
||||
{ id: 'v2', label: 'V2', default: false, disabled: true }
|
||||
];
|
||||
}
|
||||
|
||||
// Show both V1 and V2 enabled for NVIDIA GPUs
|
||||
return [
|
||||
{ id: 'v1', label: 'V1', default: true },
|
||||
{ id: 'v2', label: 'V2', default: false }
|
||||
];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
'397b': {
|
||||
h100: { bf16: { tp: 16, mem: 0.8 }, fp8: { tp: 8, mem: 0.8 } },
|
||||
h200: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 8, ep: 8, mem: 0.8 } },
|
||||
b200: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 4, mem: 0.8 }, fp4: { tp: 4, mem: 0.85 } },
|
||||
b300: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 }, fp4: { tp: 2, mem: 0.8 } },
|
||||
mi300x: { bf16: { tp: 8, mem: 0.8 }, fp8: { tp: 4, mem: 0.8 } },
|
||||
mi325x: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 } },
|
||||
mi355x: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 } }
|
||||
},
|
||||
'122b': {
|
||||
h100: { bf16: { tp: 4, mem: 0.8 }, fp8: { tp: 2, mem: 0.8 } },
|
||||
h200: { bf16: { tp: 2, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
b200: { bf16: { tp: 2, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
b300: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
mi300x: { bf16: { tp: 2, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
mi325x: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
mi355x: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } }
|
||||
},
|
||||
'35b': {
|
||||
h100: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
h200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
b200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
b300: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
mi300x: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
mi325x: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
mi355x: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } }
|
||||
},
|
||||
'27b': {
|
||||
h100: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
h200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
b200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
b300: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
mi300x: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
mi325x: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
mi355x: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } }
|
||||
},
|
||||
'9b': {
|
||||
h100: { bf16: { tp: 1, mem: 0.8 } },
|
||||
h200: { bf16: { tp: 1, mem: 0.8 } },
|
||||
b200: { bf16: { tp: 1, mem: 0.8 } },
|
||||
b300: { bf16: { tp: 1, mem: 0.8 } },
|
||||
mi300x: { bf16: { tp: 1, mem: 0.8 } },
|
||||
mi325x: { bf16: { tp: 1, mem: 0.8 } },
|
||||
mi355x: { bf16: { tp: 1, mem: 0.8 } }
|
||||
},
|
||||
'4b': {
|
||||
h100: { bf16: { tp: 1, mem: 0.8 } },
|
||||
h200: { bf16: { tp: 1, mem: 0.8 } },
|
||||
b200: { bf16: { tp: 1, mem: 0.8 } },
|
||||
b300: { bf16: { tp: 1, mem: 0.8 } },
|
||||
mi300x: { bf16: { tp: 1, mem: 0.8 } },
|
||||
mi325x: { bf16: { tp: 1, mem: 0.8 } },
|
||||
mi355x: { bf16: { tp: 1, mem: 0.8 } }
|
||||
},
|
||||
'2b': {
|
||||
h100: { bf16: { tp: 1, mem: 0.8 } },
|
||||
h200: { bf16: { tp: 1, mem: 0.8 } },
|
||||
b200: { bf16: { tp: 1, mem: 0.8 } },
|
||||
b300: { bf16: { tp: 1, mem: 0.8 } },
|
||||
mi300x: { bf16: { tp: 1, mem: 0.8 } },
|
||||
mi325x: { bf16: { tp: 1, mem: 0.8 } },
|
||||
mi355x: { bf16: { tp: 1, mem: 0.8 } }
|
||||
},
|
||||
'0.8b': {
|
||||
h100: { bf16: { tp: 1, mem: 0.8 } },
|
||||
h200: { bf16: { tp: 1, mem: 0.8 } },
|
||||
b200: { bf16: { tp: 1, mem: 0.8 } },
|
||||
b300: { bf16: { tp: 1, mem: 0.8 } },
|
||||
mi300x: { bf16: { tp: 1, mem: 0.8 } },
|
||||
mi325x: { bf16: { tp: 1, mem: 0.8 } },
|
||||
mi355x: { bf16: { tp: 1, mem: 0.8 } }
|
||||
}
|
||||
};
|
||||
|
||||
const resolveItems = (option, vals) =>
|
||||
typeof option.getDynamicItems === 'function' ? option.getDynamicItems(vals) : option.items;
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
const items = resolveItems(option, initialState);
|
||||
const def = items.find(i => i.default && !i.disabled) || items.find(i => !i.disabled) || items[0];
|
||||
initialState[key] = def.id;
|
||||
}
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
// When hardware or model changes, re-resolve dynamic selections to stay consistent.
|
||||
useEffect(() => {
|
||||
setValues(prev => {
|
||||
const next = { ...prev };
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (typeof option.getDynamicItems !== 'function') continue;
|
||||
const items = option.getDynamicItems(next);
|
||||
const current = items.find(i => i.id === next[key]);
|
||||
if (!current || current.disabled) {
|
||||
const fallback = items.find(i => i.default && !i.disabled) || items.find(i => !i.disabled);
|
||||
if (fallback) next[key] = fallback.id;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}, [values.hardware, values.model]);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
// Generate command — must produce byte-identical output to sgl-cookbook's
|
||||
// config.generateCommand(values) for every valid combination.
|
||||
const generateCommand = () => {
|
||||
const { model, hardware, quantization, speculative, mambaCache } = values;
|
||||
|
||||
const hwConfig = modelConfigs[model]?.[hardware]?.[quantization];
|
||||
if (!hwConfig) {
|
||||
if (quantization === 'fp4') {
|
||||
return '# FP4 requires B200/B300 (Blackwell) and is only available for Qwen3.5-397B-A17B';
|
||||
}
|
||||
return '# Please select a valid hardware and quantization combination';
|
||||
}
|
||||
|
||||
let modelName;
|
||||
if (quantization === 'fp4') {
|
||||
modelName = 'nvidia/Qwen3.5-397B-A17B-NVFP4';
|
||||
} else {
|
||||
const suffix = MODEL_SUFFIX[model];
|
||||
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
modelName = `Qwen/Qwen3.5-${suffix}${quantSuffix}`;
|
||||
}
|
||||
|
||||
const tpValue = hwConfig.tp;
|
||||
const epValue = hwConfig.ep;
|
||||
const memFraction = hwConfig.mem;
|
||||
|
||||
// Initialize the base command
|
||||
let cmd = `sglang serve --model-path ${modelName}`;
|
||||
if (tpValue > 1) {
|
||||
cmd += ` \\\n --tp ${tpValue}`;
|
||||
}
|
||||
if (epValue) {
|
||||
cmd += ` \\\n --expert-parallel-size ${epValue}`;
|
||||
}
|
||||
|
||||
// Force Mamba V1 for AMD GPUs (V2 requires FLA backend)
|
||||
// Force Mamba V2 when MTP is enabled
|
||||
const amdGpus = ['mi300x', 'mi325x', 'mi355x'];
|
||||
const actualMambaCache = amdGpus.includes(hardware) ? 'v1' : (speculative === 'enabled' ? 'v2' : mambaCache);
|
||||
|
||||
// Apply commandRules from options (reasoning, toolcall, speculative, mambaCache)
|
||||
// Skip quantization and model (handled via model name)
|
||||
const commandRules = {
|
||||
reasoning: (value) => value === 'enabled' ? '--reasoning-parser qwen3' : null,
|
||||
toolcall: (value) => value === 'enabled' ? '--tool-call-parser qwen3_coder' : null,
|
||||
speculative: (value) => value === 'enabled' ? '--speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4' : null,
|
||||
mambaCache: (value) => value === 'v2' ? '--mamba-scheduler-strategy extra_buffer' : null,
|
||||
};
|
||||
|
||||
// Iterate options in order, applying commandRules
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (key === 'quantization' || key === 'model') continue;
|
||||
// Skip options that don't pass their condition
|
||||
if (option.condition && !option.condition(values)) continue;
|
||||
const rule = commandRules[key];
|
||||
if (rule) {
|
||||
const adjustedValue = key === 'mambaCache' ? actualMambaCache : values[key];
|
||||
const result = rule(adjustedValue);
|
||||
if (result) {
|
||||
cmd += ` \\\n ${result}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Chunked prefill tuning for H200 FP8 + MTP (validated on H200 only)
|
||||
if (hardware === 'h200' && quantization === 'fp8' && speculative === 'enabled') {
|
||||
cmd += ` \\\n --max-running-requests 128`;
|
||||
cmd += ` \\\n --chunked-prefill-size 16384`;
|
||||
cmd += ` \\\n --tokenizer-worker-num 6`;
|
||||
}
|
||||
|
||||
// Enable allreduce fusion for all Qwen3.5 configs (skip for FP4: benchmark only enables this for TP>=8).
|
||||
if (quantization !== 'fp4') {
|
||||
cmd += ` \\\n --enable-flashinfer-allreduce-fusion`;
|
||||
}
|
||||
|
||||
// H200 FP8-specific optimizations
|
||||
if (hardware === 'h200' && quantization === 'fp8') {
|
||||
cmd += ` \\\n --attention-backend flashinfer`;
|
||||
if (MOE_MODELS.has(model)) {
|
||||
cmd += ` \\\n --mamba-ssm-dtype bfloat16`;
|
||||
}
|
||||
}
|
||||
|
||||
// Append backend configurations
|
||||
if (hardware === 'b200' || hardware === 'b300') {
|
||||
cmd += ` \\\n --attention-backend trtllm_mha`;
|
||||
}
|
||||
|
||||
// Append AMD GPU-specific backend configurations
|
||||
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||
cmd += ` \\\n --attention-backend triton`;
|
||||
}
|
||||
|
||||
// Tokenizer workers for H200 and B200/B300
|
||||
if (hardware === 'h200' || hardware === 'b200' || hardware === 'b300') {
|
||||
if (speculative === 'disabled') {
|
||||
cmd += ` \\\n --tokenizer-worker-num 6`;
|
||||
}
|
||||
}
|
||||
|
||||
// FP4-specific backend settings
|
||||
if (quantization === 'fp4') {
|
||||
cmd += ' \\\n --quantization modelopt_fp4';
|
||||
cmd += ' \\\n --fp4-gemm-backend flashinfer_cutlass';
|
||||
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
|
||||
cmd += ' \\\n --moe-runner-backend flashinfer_trtllm';
|
||||
cmd += ' \\\n --chunked-prefill-size 32768';
|
||||
cmd += ' \\\n --max-prefill-tokens 32768';
|
||||
cmd += ' \\\n --max-running-requests 128';
|
||||
cmd += ' \\\n --stream-interval 30';
|
||||
cmd += ' \\\n --disable-radix-cache';
|
||||
}
|
||||
|
||||
// Add memory fraction last
|
||||
cmd += ` \\\n --mem-fraction-static ${memFraction}`;
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.4 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (typeof option.condition === 'function' && !option.condition(values)) return null;
|
||||
const items = resolveItems(option, values);
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = !!item.disabled;
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}
|
||||
title={item.disabledReason || ''}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,219 @@
|
||||
export const Qwen36Deployment = () => {
|
||||
// Config mirrors sgl-cookbook src/components/autoregressive/Qwen36ConfigGenerator/index.js.
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h100', label: 'H100', default: true },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
],
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'fp8', label: 'FP8', default: true },
|
||||
{ id: 'bf16', label: 'BF16', default: false },
|
||||
],
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--reasoning-parser qwen3' : null,
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--tool-call-parser qwen3_coder' : null,
|
||||
},
|
||||
speculative: {
|
||||
name: 'speculative',
|
||||
title: 'Speculative Decoding (MTP)',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: false },
|
||||
{ id: 'enabled', label: 'Enabled', default: true },
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4' : null,
|
||||
},
|
||||
mambaCache: {
|
||||
name: 'mambaCache',
|
||||
title: 'Mamba Radix Cache',
|
||||
getDynamicItems: (values) => {
|
||||
const mtpEnabled = values.speculative === 'enabled';
|
||||
if (mtpEnabled) {
|
||||
return [
|
||||
{ id: 'v1', label: 'V1', default: false, disabled: true },
|
||||
{ id: 'v2', label: 'V2', default: true },
|
||||
];
|
||||
}
|
||||
return [
|
||||
{ id: 'v1', label: 'V1', default: true },
|
||||
{ id: 'v2', label: 'V2', default: false },
|
||||
];
|
||||
},
|
||||
commandRule: (value) => value === 'v2' ? '--mamba-scheduler-strategy extra_buffer' : null,
|
||||
},
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
h100: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
h200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
b200: { bf16: { tp: 1, mem: 0.8 }, fp8: { tp: 1, mem: 0.8 } },
|
||||
};
|
||||
|
||||
const resolveItems = (option, vals) =>
|
||||
typeof option.getDynamicItems === 'function' ? option.getDynamicItems(vals) : option.items;
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
const items = resolveItems(option, initialState);
|
||||
const def = items.find((item) => item.default && !item.disabled) || items.find((item) => !item.disabled) || items[0];
|
||||
initialState[key] = def.id;
|
||||
}
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
setValues((prev) => {
|
||||
const next = { ...prev };
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (typeof option.getDynamicItems !== 'function') continue;
|
||||
const items = option.getDynamicItems(next);
|
||||
const current = items.find((item) => item.id === next[key]);
|
||||
if (!current || current.disabled) {
|
||||
const fallback = items.find((item) => item.default && !item.disabled) || items.find((item) => !item.disabled);
|
||||
if (fallback) next[key] = fallback.id;
|
||||
}
|
||||
}
|
||||
return next;
|
||||
});
|
||||
}, [values.speculative]);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const generateCommand = () => {
|
||||
const { hardware, quantization, speculative } = values;
|
||||
const hwConfig = modelConfigs[hardware]?.[quantization];
|
||||
if (!hwConfig) {
|
||||
return '# Please select a valid hardware and quantization combination';
|
||||
}
|
||||
|
||||
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
const modelName = `Qwen/Qwen3.6-35B-A3B${quantSuffix}`;
|
||||
|
||||
let cmd = '';
|
||||
if (speculative === 'enabled') {
|
||||
cmd += 'SGLANG_ENABLE_SPEC_V2=1 ';
|
||||
}
|
||||
|
||||
cmd += `sglang serve --model-path ${modelName}`;
|
||||
if (hwConfig.tp > 1) {
|
||||
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
||||
}
|
||||
|
||||
const adjustedValues = {
|
||||
...values,
|
||||
mambaCache: speculative === 'enabled' ? 'v2' : values.mambaCache,
|
||||
};
|
||||
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (key === 'quantization' || key === 'hardware') continue;
|
||||
if (!option.commandRule) continue;
|
||||
const rule = option.commandRule(adjustedValues[key]);
|
||||
if (rule) {
|
||||
cmd += ` \\\n ${rule}`;
|
||||
}
|
||||
}
|
||||
|
||||
if (hardware === 'b200') {
|
||||
cmd += ` \\\n --attention-backend trtllm_mha`;
|
||||
}
|
||||
|
||||
cmd += ` \\\n --mem-fraction-static ${hwConfig.mem}`;
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.4 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
const items = resolveItems(option, values);
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = !!item.disabled;
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isDisabled ? disabledStyle : {}) }}
|
||||
title={item.disabledReason || ''}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,217 @@
|
||||
export const Ring251TDeployment = () => {
|
||||
// Config mirrors sgl-cookbook src/components/autoregressive/Ring25ConfigGenerator/index.js.
|
||||
//
|
||||
// GPU requirements:
|
||||
// H200 / B200 / GB200 / GB300 / MI355X: single-node (tp per platform)
|
||||
// MI300X / MI325X: two nodes, tp-size 8, pp-size 2 (multi-node scripts)
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'b200', label: 'B200', default: false },
|
||||
{ id: 'gb200', label: 'GB200', default: false },
|
||||
{ id: 'gb300', label: 'GB300', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
h200: { fp8: { tp: 8 } },
|
||||
b200: { fp8: { tp: 8 } },
|
||||
gb200: { fp8: { tp: 4 } },
|
||||
gb300: { fp8: { tp: 4 } },
|
||||
mi300x: { fp8: { tp: 8, pp: 2, nnodes: 2 } },
|
||||
mi325x: { fp8: { tp: 8, pp: 2, nnodes: 2 } },
|
||||
mi355x: { fp8: { tp: 8 } }
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = option.items.filter(item => item.default).map(item => item.id);
|
||||
} else {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
}
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues(prev => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
} else {
|
||||
return { ...prev, [optionName]: currentValues.filter(id => id !== itemId) };
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Generate command — byte-identical to sgl-cookbook Ring25ConfigGenerator
|
||||
const generateCommand = () => {
|
||||
const { hardware, reasoning, toolcall } = values;
|
||||
const modelName = 'inclusionAI/Ring-2.5-1T';
|
||||
const amdMultiNode = hardware === 'mi300x' || hardware === 'mi325x';
|
||||
|
||||
// Extra flags from reasoning / toolcall
|
||||
const extraFlags = [];
|
||||
if (reasoning === 'enabled') extraFlags.push('--reasoning-parser deepseek-r1');
|
||||
if (toolcall === 'enabled') extraFlags.push('--tool-call-parser qwen');
|
||||
|
||||
if (amdMultiNode) {
|
||||
const hwConfig = modelConfigs[hardware].fp8;
|
||||
const tpSize = hwConfig.tp;
|
||||
const ppSize = hwConfig.pp;
|
||||
|
||||
const buildAmdNodeCmd = (nodeRank) => {
|
||||
let cmd = 'sglang serve \\\n';
|
||||
cmd += `--model-path ${modelName} \\\n`;
|
||||
cmd += '--trust-remote-code \\\n';
|
||||
cmd += `--tp-size ${tpSize} \\\n`;
|
||||
cmd += `--pp-size ${ppSize} \\\n`;
|
||||
cmd += `--nnodes ${hwConfig.nnodes} \\\n`;
|
||||
cmd += `--node-rank ${nodeRank} \\\n`;
|
||||
if (nodeRank === 0) {
|
||||
cmd += '--host 0.0.0.0 \\\n';
|
||||
cmd += '--port 30000 \\\n';
|
||||
}
|
||||
cmd += '--dist-init-addr ${MASTER_IP}:${DIST_PORT} \\\n';
|
||||
cmd += '--attention-backend triton \\\n';
|
||||
cmd += '--model-loader-extra-config \'{"enable_multithread_load": "true","num_threads": 64}\' \\\n';
|
||||
cmd += '--mem-frac 0.95';
|
||||
extraFlags.forEach((flag) => {
|
||||
cmd += ` \\\n${flag}`;
|
||||
});
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const envBlock =
|
||||
'export MASTER_IP=<your-node0-ip> # Replace with the IP of Node 0\n' +
|
||||
'export PORT=30000\n' +
|
||||
'export DIST_PORT=20000\n' +
|
||||
'# Replace <nic-ifname> with your actual NIC interface name\n' +
|
||||
'export GLOO_SOCKET_IFNAME=<nic-ifname>\n' +
|
||||
'export TP_SOCKET_IFNAME=<nic-ifname>\n';
|
||||
|
||||
let out = envBlock + '\n';
|
||||
|
||||
out += '\n# Node 0:\n';
|
||||
out += buildAmdNodeCmd(0);
|
||||
|
||||
out += '\n\n\n# Node 1:\n';
|
||||
out += buildAmdNodeCmd(1);
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// Single-node path (H200, B200, GB200, GB300, MI355X)
|
||||
const hwConfig = modelConfigs[hardware].fp8;
|
||||
const tpValue = hwConfig.tp;
|
||||
|
||||
let cmd = 'sglang serve \\\n';
|
||||
cmd += ` --model-path ${modelName}`;
|
||||
cmd += ` \\\n --tp ${tpValue}`;
|
||||
cmd += ' \\\n --trust-remote-code';
|
||||
|
||||
extraFlags.forEach((flag) => {
|
||||
cmd += ` \\\n ${flag}`;
|
||||
});
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const disabledStyle = { cursor: 'not-allowed', opacity: 0.5 };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'checkbox' ? (
|
||||
option.items.map(item => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isItemDisabled = item.required;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}), ...(isItemDisabled ? disabledStyle : {}) }}>
|
||||
<input type="checkbox" checked={isChecked} disabled={isItemDisabled} onChange={(e) => handleCheckboxChange(option.name, item.id, e.target.checked)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,393 @@
|
||||
export const Step35Deployment = () => {
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'h200', label: 'H200', default: true },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi350x', label: 'MI350X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
modelsize: {
|
||||
name: 'modelsize',
|
||||
title: 'Model Size',
|
||||
items: [
|
||||
{ id: '196b', label: '196B', subtitle: 'MOE', default: true },
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false }
|
||||
]
|
||||
},
|
||||
reasoningParser: {
|
||||
name: 'reasoningParser',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--reasoning-parser step3p5' : null
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--tool-call-parser step3p5' : null
|
||||
},
|
||||
speculative: {
|
||||
name: 'speculative',
|
||||
title: 'Speculative Decoding',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
],
|
||||
commandRule: (value) => {
|
||||
if (value !== 'enabled') return null;
|
||||
|
||||
let cmd = '--speculative-algorithm EAGLE \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4 \\\n --enable-multi-layer-eagle ';
|
||||
|
||||
return cmd;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
'196b': {
|
||||
baseName: '196b',
|
||||
isMOE: true,
|
||||
h200: { tp: 4, bf16: true },
|
||||
mi300x: { tp: 4, bf16: true },
|
||||
mi325x: { tp: 4, bf16: true },
|
||||
mi350x: { tp: 4, bf16: true },
|
||||
mi355x: { tp: 4, bf16: true },
|
||||
},
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, modelsize: modelSize, quantization, reasoningParser } = values;
|
||||
const isAMD = hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi350x' || hardware === 'mi355x';
|
||||
|
||||
const modelSizeConfig = modelConfigs[modelSize];
|
||||
const hwConfig = modelSizeConfig[hardware];
|
||||
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
const modelName = `stepfun-ai/Step-3.5-Flash${quantSuffix}`;
|
||||
|
||||
let tpValue = hwConfig.tp;
|
||||
|
||||
let cmd = '';
|
||||
|
||||
cmd += 'sglang serve \\\n';
|
||||
cmd += ` --model-path ${modelName}`;
|
||||
|
||||
if (tpValue > 1) {
|
||||
cmd += ` \\\n --tp ${tpValue}`;
|
||||
}
|
||||
// EP required for FP8, and for AMD BF16 (AITER CK GEMM N=320 crash without EP)
|
||||
if (quantSuffix === '-FP8' || isAMD) {
|
||||
cmd += ` \\\n --ep ${tpValue}`;
|
||||
}
|
||||
|
||||
// Trust remote code for custom architecture
|
||||
cmd += ' \\\n --trust-remote-code';
|
||||
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (option.commandRule) {
|
||||
const rule = option.commandRule(values[key], values);
|
||||
|
||||
if (rule) {
|
||||
cmd += ` \\\n ${rule}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,383 @@
|
||||
export const Step3VL10BDeployment = () => {
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'a100', label: 'A100', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
},
|
||||
modelsize: {
|
||||
name: 'modelsize',
|
||||
title: 'Model Size',
|
||||
items: [
|
||||
{ id: '10b', label: '10B', subtitle: 'Dense', default: true }
|
||||
]
|
||||
},
|
||||
quantization: {
|
||||
name: 'quantization',
|
||||
title: 'Quantization',
|
||||
items: [
|
||||
{ id: 'bf16', label: 'BF16', default: true },
|
||||
{ id: 'fp8', label: 'FP8', default: false }
|
||||
]
|
||||
},
|
||||
reasoning: {
|
||||
name: 'reasoning',
|
||||
title: 'Reasoning Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--reasoning-parser deepseek-r1' : null
|
||||
},
|
||||
toolcall: {
|
||||
name: 'toolcall',
|
||||
title: 'Tool Call Parser',
|
||||
items: [
|
||||
{ id: 'disabled', label: 'Disabled', default: true },
|
||||
{ id: 'enabled', label: 'Enabled', default: false }
|
||||
],
|
||||
commandRule: (value) => value === 'enabled' ? '--tool-call-parser hermes' : null
|
||||
}
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
'10b': {
|
||||
baseName: '10B',
|
||||
isMOE: false,
|
||||
b200: { tp: 1, bf16: true, fp8: true },
|
||||
h100: { tp: 1, bf16: true, fp8: true },
|
||||
h200: { tp: 1, bf16: true, fp8: true },
|
||||
a100: { tp: 1, bf16: true, fp8: true },
|
||||
mi300x: { tp: 1, bf16: true, fp8: true },
|
||||
mi325x: { tp: 1, bf16: true, fp8: true },
|
||||
mi355x: { tp: 1, bf16: true, fp8: true }
|
||||
}
|
||||
};
|
||||
|
||||
const generateCommand = (values) => {
|
||||
const { hardware, modelsize: modelSize, quantization } = values;
|
||||
|
||||
const modelSizeConfig = modelConfigs[modelSize];
|
||||
if (!modelSizeConfig) {
|
||||
return `# Error: Unknown model size: ${modelSize}`;
|
||||
}
|
||||
|
||||
const hwConfig = modelSizeConfig[hardware];
|
||||
if (!hwConfig) {
|
||||
return `# Error: Unknown hardware platform: ${hardware}`;
|
||||
}
|
||||
|
||||
const quantSuffix = quantization === 'fp8' ? '-FP8' : '';
|
||||
const modelName = `stepfun-ai/Step3-VL-10B${quantSuffix}`;
|
||||
|
||||
let cmd = 'python -m sglang.launch_server \\\n';
|
||||
cmd += ` --model ${modelName}`;
|
||||
|
||||
if (hwConfig.tp > 1) {
|
||||
cmd += ` \\\n --tp ${hwConfig.tp}`;
|
||||
}
|
||||
|
||||
cmd += ' \\\n --host 0.0.0.0 \\\n --port 30000';
|
||||
if (hardware === 'mi300x' || hardware === 'mi325x' || hardware === 'mi355x') {
|
||||
cmd += ' \\\n --attention-backend triton';
|
||||
}
|
||||
cmd += ' \\\n --trust-remote-code';
|
||||
|
||||
for (const [key, option] of Object.entries(options)) {
|
||||
if (option.commandRule) {
|
||||
const rule = option.commandRule(values[key]);
|
||||
if (rule) {
|
||||
cmd += ` \\\n ${rule}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = generateCommand(values);
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,335 @@
|
||||
export const FluxDeployment = () => {
|
||||
const config = {
|
||||
modelFamily: 'FLUX',
|
||||
|
||||
options: {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
]
|
||||
},
|
||||
version: {
|
||||
name: 'version',
|
||||
title: 'Model Version',
|
||||
items: [
|
||||
{ id: 'flux1-dev', label: 'FLUX.1-dev', subtitle: '12B', default: true },
|
||||
{ id: 'flux2-dev', label: 'FLUX.2-dev', subtitle: '32B', default: false }
|
||||
]
|
||||
}
|
||||
},
|
||||
|
||||
modelConfigs: {
|
||||
'flux1-dev': { repoId: 'black-forest-labs/FLUX.1-dev' },
|
||||
'flux2-dev': { repoId: 'black-forest-labs/FLUX.2-dev' }
|
||||
},
|
||||
|
||||
generateCommand: function(values) {
|
||||
const { version } = values;
|
||||
const config = this.modelConfigs[version];
|
||||
|
||||
return `sglang serve \\
|
||||
--model-path ${config.repoId} \\
|
||||
--ulysses-degree=1 \\
|
||||
--ring-degree=1`;
|
||||
}
|
||||
};
|
||||
|
||||
if (!config || !config.options) {
|
||||
return <div>Error: Invalid configuration provided</div>;
|
||||
}
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(config.options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(config.options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = config.generateCommand ? config.generateCommand.call(config, values) : '';
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(config.options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,115 @@
|
||||
export const MOVADeployment = () => {
|
||||
// Config options
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'a100', label: 'A100', default: false }
|
||||
]
|
||||
},
|
||||
resolution: {
|
||||
name: 'resolution',
|
||||
title: 'Resolution',
|
||||
items: [
|
||||
{ id: '360p', label: '360p', subtitle: 'Fast inference, lower VRAM', default: true },
|
||||
{ id: '720p', label: '720p', subtitle: 'Higher resolution', default: false }
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(options).forEach(([key, option]) => {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
// Generate command
|
||||
const generateCommand = () => {
|
||||
const { resolution } = values;
|
||||
const modelPath = resolution === '720p'
|
||||
? 'OpenMOSS-Team/MOVA-720p'
|
||||
: 'OpenMOSS-Team/MOVA-360p';
|
||||
|
||||
return `export SG_OUTPUT_DIR=/root/output_mova
|
||||
mkdir -p "$SG_OUTPUT_DIR"
|
||||
|
||||
sglang serve \\
|
||||
--model-path ${modelPath} \\
|
||||
--host 0.0.0.0 \\
|
||||
--port 30002 \\
|
||||
--adjust-frames false \\
|
||||
--num-gpus 8 \\
|
||||
--ring-degree 2 \\
|
||||
--ulysses-degree 4 \\
|
||||
--tp 1 \\
|
||||
--enable-torch-compile \\
|
||||
--save-output \\
|
||||
--output-dir "$SG_OUTPUT_DIR"`;
|
||||
};
|
||||
|
||||
// Styles - with dark mode support
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,316 @@
|
||||
export const QwenImageDeployment = () => {
|
||||
const config = {
|
||||
modelFamily: 'Qwen-Image',
|
||||
|
||||
options: {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'mi300x', label: 'MI300X', default: true },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
}
|
||||
},
|
||||
|
||||
generateCommand: function(values) {
|
||||
return `sglang serve \\
|
||||
--model-path Qwen/Qwen-Image \\
|
||||
--ulysses-degree=1 \\
|
||||
--ring-degree=1`;
|
||||
}
|
||||
};
|
||||
|
||||
if (!config || !config.options) {
|
||||
return <div>Error: Invalid configuration provided</div>;
|
||||
}
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(config.options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(config.options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = config.generateCommand ? config.generateCommand.call(config, values) : '';
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(config.options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,319 @@
|
||||
export const QwenImageEditDeployment = () => {
|
||||
const config = {
|
||||
modelFamily: 'Qwen-Image-Edit',
|
||||
|
||||
options: {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'h100', label: 'H100', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false }
|
||||
]
|
||||
}
|
||||
},
|
||||
|
||||
generateCommand: function(values) {
|
||||
return `sglang serve \\
|
||||
--model-path Qwen/Qwen-Image-Edit-2511 \\
|
||||
--ulysses-degree=1 \\
|
||||
--ring-degree=1`;
|
||||
}
|
||||
};
|
||||
|
||||
if (!config || !config.options) {
|
||||
return <div>Error: Invalid configuration provided</div>;
|
||||
}
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(config.options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(config.options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = config.generateCommand ? config.generateCommand.call(config, values) : '';
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(config.options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,337 @@
|
||||
export const Wan21Deployment = () => {
|
||||
const MODELSIZE_DEFS = [
|
||||
{
|
||||
id: '14b',
|
||||
label: '14B',
|
||||
subtitle: 'High-quality, 480P/720P',
|
||||
default: true,
|
||||
validTasks: ['t2v', 'i2v'],
|
||||
},
|
||||
{
|
||||
id: '1_3b',
|
||||
label: '1.3B',
|
||||
subtitle: 'Lightweight, 480P',
|
||||
default: false,
|
||||
validTasks: ['t2v'],
|
||||
},
|
||||
];
|
||||
|
||||
const modelConfigs = {
|
||||
't2v-14b': {
|
||||
repoId: 'Wan-AI/Wan2.1-T2V-14B-Diffusers',
|
||||
supportedLoras: [
|
||||
{ id: 'general', label: 'General Wan2.1 LoRA', path: 'NIVEDAN/wan2.1-lora' },
|
||||
],
|
||||
},
|
||||
't2v-1_3b': {
|
||||
repoId: 'Wan-AI/Wan2.1-T2V-1.3B-Diffusers',
|
||||
supportedLoras: [],
|
||||
},
|
||||
'i2v-14b': {
|
||||
repoId: 'Wan-AI/Wan2.1-I2V-14B-720P-Diffusers',
|
||||
supportedLoras: [
|
||||
{ id: 'fight', label: 'Fight Style LoRA', path: 'valiantcat/Wan2.1-Fight-LoRA' },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [{ id: 'mi300x', label: 'MI300X/MI325X/MI355X', default: true }],
|
||||
},
|
||||
task: {
|
||||
name: 'task',
|
||||
title: 'Task Type',
|
||||
items: [
|
||||
{ id: 't2v', label: 'Text-to-Video (T2V)', default: true },
|
||||
{ id: 'i2v', label: 'Image-to-Video (I2V)', default: false },
|
||||
],
|
||||
},
|
||||
modelsize: {
|
||||
name: 'modelsize',
|
||||
title: 'Model Variant',
|
||||
items: MODELSIZE_DEFS.map(({ validTasks, ...rest }) => rest),
|
||||
},
|
||||
bestPractice: {
|
||||
name: 'bestPractice',
|
||||
title: 'Sequence Parallelism',
|
||||
items: [
|
||||
{ id: 'off', label: 'Standard', default: true },
|
||||
{ id: 'on', label: 'Best Practice (4 GPUs)', default: false },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
function modelSizeItemsForTask(task) {
|
||||
return MODELSIZE_DEFS.filter((item) => item.validTasks.includes(task)).map(
|
||||
({ validTasks, ...rest }) => rest
|
||||
);
|
||||
}
|
||||
|
||||
const getInitialState = () => {
|
||||
const task = 't2v';
|
||||
const sizes = modelSizeItemsForTask(task);
|
||||
const modelsize = sizes.find((size) => size.default)?.id || sizes[0].id;
|
||||
const configKey = `${task}-${modelsize}`;
|
||||
const supported = modelConfigs[configKey]?.supportedLoras || [];
|
||||
return {
|
||||
hardware: 'mi300x',
|
||||
task,
|
||||
modelsize,
|
||||
bestPractice: 'off',
|
||||
selectedLoraPath: supported[0]?.path ?? '',
|
||||
};
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, itemId) => {
|
||||
setValues((prev) => {
|
||||
let next = { ...prev, [optionName]: itemId };
|
||||
|
||||
if (optionName === 'task') {
|
||||
const sizes = modelSizeItemsForTask(itemId);
|
||||
if (!sizes.some((size) => size.id === next.modelsize)) {
|
||||
next.modelsize = sizes.find((size) => size.default)?.id || sizes[0].id;
|
||||
}
|
||||
}
|
||||
|
||||
if (optionName === 'task' || optionName === 'modelsize') {
|
||||
const configKey = `${next.task}-${next.modelsize}`;
|
||||
const supported = modelConfigs[configKey]?.supportedLoras || [];
|
||||
if (supported.length === 0) {
|
||||
next.selectedLoraPath = '';
|
||||
} else if (
|
||||
next.selectedLoraPath &&
|
||||
!supported.some((lora) => lora.path === next.selectedLoraPath)
|
||||
) {
|
||||
next.selectedLoraPath = supported[0].path;
|
||||
}
|
||||
}
|
||||
|
||||
return next;
|
||||
});
|
||||
};
|
||||
|
||||
const handleLoraToggle = (path) => {
|
||||
setValues((prev) => ({
|
||||
...prev,
|
||||
selectedLoraPath: prev.selectedLoraPath === path ? '' : path,
|
||||
}));
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const generateCommand = () => {
|
||||
const { task, modelsize, selectedLoraPath, bestPractice } = values;
|
||||
const configKey = `${task}-${modelsize}`;
|
||||
const config = modelConfigs[configKey];
|
||||
|
||||
if (!config) {
|
||||
return '# Error: Invalid configuration';
|
||||
}
|
||||
|
||||
let command = `sglang serve \\\n --model-path ${config.repoId} \\\n --dit-layerwise-offload true`;
|
||||
|
||||
if (bestPractice === 'on') {
|
||||
command += ` \\\n --num-gpus 4 \\\n --ulysses-degree 2 \\\n --enable-cfg-parallel`;
|
||||
}
|
||||
|
||||
if (selectedLoraPath) {
|
||||
command += ` \\\n --lora-path ${selectedLoraPath}`;
|
||||
}
|
||||
|
||||
return command;
|
||||
};
|
||||
|
||||
const modelSizeItems = modelSizeItemsForTask(values.task);
|
||||
const loraConfigKey = `${values.task}-${values.modelsize}`;
|
||||
const availableLoras = modelConfigs[loraConfigKey]?.supportedLoras || [];
|
||||
const command = generateCommand();
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{(key === 'modelsize' ? modelSizeItems : option.items).map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
onChange={() => handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
|
||||
{availableLoras.length > 0 && (
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Select LoRA Model (Only some of the supported LoRAs are listed here)</div>
|
||||
<div style={itemsStyle}>
|
||||
{availableLoras.map((lora) => {
|
||||
const isChecked = values.selectedLoraPath === lora.path;
|
||||
return (
|
||||
<label
|
||||
key={lora.id}
|
||||
style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}
|
||||
onClick={(event) => {
|
||||
event.preventDefault();
|
||||
handleLoraToggle(lora.path);
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name="selectedLoraPath"
|
||||
value={lora.path}
|
||||
checked={isChecked}
|
||||
readOnly
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{lora.label}
|
||||
<small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>
|
||||
{lora.path}
|
||||
</small>
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,216 @@
|
||||
|
||||
export const Wan22Deployment = () => {
|
||||
const options = {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'mi300x', label: 'MI300X', default: false },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false },
|
||||
],
|
||||
},
|
||||
task: {
|
||||
name: 'task',
|
||||
title: 'Task Type',
|
||||
items: [
|
||||
{ id: 'i2v', label: 'Image-to-Video (I2V)', default: false },
|
||||
{ id: 't2v', label: 'Text-to-Video (T2V)', default: true },
|
||||
{ id: 'ti2v', label: 'Text/Image-to-Video (TI2V)', default: false },
|
||||
],
|
||||
},
|
||||
modelsize: {
|
||||
name: 'modelsize',
|
||||
title: 'Model Size',
|
||||
items: [
|
||||
{ id: '14b', label: 'A14B', subtitle: 'Diffusers (A14B)', default: true, validTasks: ['i2v', 't2v'] },
|
||||
{ id: '5b', label: '5B', subtitle: 'Diffusers', default: false, validTasks: ['ti2v'] },
|
||||
],
|
||||
},
|
||||
bestPractice: {
|
||||
name: 'bestPractice',
|
||||
title: 'Sequence Parallelism',
|
||||
items: [
|
||||
{ id: 'off', label: 'Standard', default: true },
|
||||
{ id: 'on', label: 'Best Practice (4 GPUs)', default: false },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
const modelConfigs = {
|
||||
'i2v-14b': {
|
||||
repoId: 'Wan-AI/Wan2.2-I2V-A14B-Diffusers',
|
||||
supportedLoras: [{ id: 'distill', path: 'lightx2v/Wan2.2-Distill-Loras' }],
|
||||
},
|
||||
't2v-14b': {
|
||||
repoId: 'Wan-AI/Wan2.2-T2V-A14B-Diffusers',
|
||||
supportedLoras: [{ id: 'arcane', path: 'Cseti/wan2.2-14B-Arcane_Jinx-lora-v1' }],
|
||||
},
|
||||
'ti2v-5b': {
|
||||
repoId: 'Wan-AI/Wan2.2-TI2V-5B-Diffusers',
|
||||
supportedLoras: [],
|
||||
},
|
||||
};
|
||||
|
||||
const getInitialState = () => ({
|
||||
hardware: 'b200',
|
||||
task: 't2v',
|
||||
modelsize: '14b',
|
||||
bestPractice: 'off',
|
||||
selectedLoraPath: 'none',
|
||||
});
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const availableLoras = (() => {
|
||||
const configKey = `${values.task}-${values.modelsize}`;
|
||||
return modelConfigs[configKey]?.supportedLoras || [];
|
||||
})();
|
||||
|
||||
const handleRadioChange = (optionName, itemId) => {
|
||||
setValues((prev) => {
|
||||
const next = { ...prev, [optionName]: itemId };
|
||||
if (optionName === 'task') {
|
||||
next.modelsize = itemId === 'ti2v' ? '5b' : '14b';
|
||||
}
|
||||
|
||||
const configKey = `${next.task}-${next.modelsize}`;
|
||||
const nextSupported = modelConfigs[configKey]?.supportedLoras || [];
|
||||
const isValid = nextSupported.some((lora) => lora.path === prev.selectedLoraPath);
|
||||
if (!isValid) {
|
||||
next.selectedLoraPath = 'none';
|
||||
}
|
||||
return next;
|
||||
});
|
||||
};
|
||||
|
||||
const handleLoraToggle = (path) => {
|
||||
setValues((prev) => ({
|
||||
...prev,
|
||||
selectedLoraPath: prev.selectedLoraPath === path ? 'none' : path,
|
||||
}));
|
||||
};
|
||||
|
||||
const generateCommand = () => {
|
||||
const { task, modelsize, selectedLoraPath, bestPractice } = values;
|
||||
const configKey = `${task}-${modelsize}`;
|
||||
const config = modelConfigs[configKey];
|
||||
if (!config) {
|
||||
return '# Error: Invalid configuration';
|
||||
}
|
||||
|
||||
let command = `sglang serve \\\n --model-path ${config.repoId} \\\n --dit-layerwise-offload true`;
|
||||
if (bestPractice === 'on') {
|
||||
command += ` \\\n --num-gpus 4 \\\n --ulysses-degree 2 \\\n --enable-cfg-parallel`;
|
||||
}
|
||||
if (selectedLoraPath && selectedLoraPath !== 'none') {
|
||||
command += ` \\\n --lora-path ${selectedLoraPath}`;
|
||||
}
|
||||
return command;
|
||||
};
|
||||
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'center', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '140px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const itemsStyle = { display: 'flex', rowGap: '2px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center', flex: 1 };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', flex: 1, background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(options).map(([key, option]) => {
|
||||
const itemsToDisplay = key === 'modelsize'
|
||||
? option.items.filter((item) => item.validTasks.includes(values.task))
|
||||
: option.items;
|
||||
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{itemsToDisplay.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
checked={isChecked}
|
||||
onChange={() => handleRadioChange(key, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Select LoRA Model (Only some of the supported LoRAs are listed here)</div>
|
||||
<div style={itemsStyle}>
|
||||
{availableLoras.length === 0 && (
|
||||
<div style={{ color: isDark ? '#999' : '#666', fontSize: '12px', padding: '8px' }}>
|
||||
No LoRA models available for this model.
|
||||
</div>
|
||||
)}
|
||||
{availableLoras.map((lora) => {
|
||||
const isSelected = values.selectedLoraPath === lora.path;
|
||||
return (
|
||||
<label
|
||||
key={lora.id}
|
||||
style={{ ...labelBaseStyle, ...(isSelected ? checkedStyle : {}) }}
|
||||
onClick={(event) => {
|
||||
event.preventDefault();
|
||||
handleLoraToggle(lora.path);
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name="loraModelSelection"
|
||||
checked={isSelected}
|
||||
readOnly
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{lora.id}
|
||||
<small style={{ ...subtitleStyle, color: isSelected ? 'rgba(255,255,255,0.85)' : 'inherit' }}>
|
||||
{lora.path}
|
||||
</small>
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,319 @@
|
||||
export const ZImageTurboDeployment = () => {
|
||||
const config = {
|
||||
modelFamily: 'Z-Image-Turbo',
|
||||
|
||||
options: {
|
||||
hardware: {
|
||||
name: 'hardware',
|
||||
title: 'Hardware Platform',
|
||||
items: [
|
||||
{ id: 'mi300x', label: 'MI300X', default: true },
|
||||
{ id: 'mi325x', label: 'MI325X', default: false },
|
||||
{ id: 'mi355x', label: 'MI355X', default: false },
|
||||
{ id: 'b200', label: 'B200', default: true },
|
||||
{ id: 'h200', label: 'H200', default: false },
|
||||
{ id: 'h100', label: 'H100', default: false }
|
||||
]
|
||||
}
|
||||
},
|
||||
|
||||
generateCommand: function(values) {
|
||||
return `sglang serve \\
|
||||
--model-path Tongyi-MAI/Z-Image-Turbo \\
|
||||
--ulysses-degree=1 \\
|
||||
--ring-degree=1`;
|
||||
}
|
||||
};
|
||||
|
||||
if (!config || !config.options) {
|
||||
return <div>Error: Invalid configuration provided</div>;
|
||||
}
|
||||
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.entries(config.options).forEach(([key, option]) => {
|
||||
if (option.type === 'checkbox') {
|
||||
initialState[key] = (option.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
return;
|
||||
}
|
||||
|
||||
if (option.type === 'text') {
|
||||
initialState[key] = option.default || '';
|
||||
return;
|
||||
}
|
||||
|
||||
let items = option.items || [];
|
||||
if (option.getDynamicItems) {
|
||||
const defaultValues = {};
|
||||
Object.entries(config.options).forEach(([innerKey, innerOption]) => {
|
||||
if (innerOption.type === 'checkbox') {
|
||||
defaultValues[innerKey] = (innerOption.items || [])
|
||||
.filter((item) => item.default)
|
||||
.map((item) => item.id);
|
||||
} else if (innerOption.type === 'text') {
|
||||
defaultValues[innerKey] = innerOption.default || '';
|
||||
} else if (innerOption.items && innerOption.items.length > 0) {
|
||||
const defaultItem = innerOption.items.find((item) => item.default);
|
||||
defaultValues[innerKey] = defaultItem ? defaultItem.id : innerOption.items[0].id;
|
||||
}
|
||||
});
|
||||
items = option.getDynamicItems(defaultValues);
|
||||
}
|
||||
|
||||
const defaultItem = items && items.find((item) => item.default);
|
||||
initialState[key] = defaultItem ? defaultItem.id : items && items[0] ? items[0].id : '';
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode =
|
||||
html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class', 'data-theme', 'style'],
|
||||
});
|
||||
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
const handleRadioChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (optionName, itemId, isChecked) => {
|
||||
setValues((prev) => {
|
||||
const currentValues = prev[optionName] || [];
|
||||
if (isChecked) {
|
||||
return { ...prev, [optionName]: [...currentValues, itemId] };
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
[optionName]: currentValues.filter((id) => id !== itemId),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const handleTextChange = (optionName, value) => {
|
||||
setValues((prev) => ({ ...prev, [optionName]: value }));
|
||||
};
|
||||
|
||||
const command = config.generateCommand ? config.generateCommand.call(config, values) : '';
|
||||
|
||||
const containerStyle = {
|
||||
maxWidth: '900px',
|
||||
margin: '0 auto',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '4px',
|
||||
};
|
||||
const cardStyle = {
|
||||
padding: '8px 12px',
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`,
|
||||
borderRadius: '4px',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: '12px',
|
||||
background: isDark ? '#1f2937' : '#fff',
|
||||
};
|
||||
const titleStyle = {
|
||||
fontSize: '13px',
|
||||
fontWeight: '600',
|
||||
minWidth: '140px',
|
||||
flexShrink: 0,
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const itemsStyle = {
|
||||
display: 'flex',
|
||||
rowGap: '2px',
|
||||
columnGap: '6px',
|
||||
flexWrap: 'wrap',
|
||||
alignItems: 'center',
|
||||
flex: 1,
|
||||
};
|
||||
const labelBaseStyle = {
|
||||
padding: '4px 10px',
|
||||
border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`,
|
||||
borderRadius: '3px',
|
||||
cursor: 'pointer',
|
||||
display: 'inline-flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
fontWeight: '500',
|
||||
fontSize: '13px',
|
||||
transition: 'all 0.2s',
|
||||
userSelect: 'none',
|
||||
minWidth: '45px',
|
||||
textAlign: 'center',
|
||||
flex: 1,
|
||||
background: isDark ? '#374151' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : 'inherit',
|
||||
};
|
||||
const checkedStyle = {
|
||||
background: '#D45D44',
|
||||
color: 'white',
|
||||
borderColor: '#D45D44',
|
||||
};
|
||||
const disabledStyle = {
|
||||
cursor: 'not-allowed',
|
||||
opacity: 0.5,
|
||||
};
|
||||
const subtitleStyle = {
|
||||
display: 'block',
|
||||
fontSize: '9px',
|
||||
marginTop: '1px',
|
||||
lineHeight: '1.1',
|
||||
opacity: 0.7,
|
||||
};
|
||||
const textInputStyle = {
|
||||
flex: 1,
|
||||
padding: '8px 10px',
|
||||
borderRadius: '4px',
|
||||
border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`,
|
||||
background: isDark ? '#111827' : '#fff',
|
||||
color: isDark ? '#e5e7eb' : '#111827',
|
||||
fontSize: '13px',
|
||||
};
|
||||
const commandDisplayStyle = {
|
||||
flex: 1,
|
||||
padding: '12px 16px',
|
||||
background: isDark ? '#111827' : '#f5f5f5',
|
||||
borderRadius: '6px',
|
||||
fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace",
|
||||
fontSize: '12px',
|
||||
lineHeight: '1.5',
|
||||
color: isDark ? '#e5e7eb' : '#374151',
|
||||
whiteSpace: 'pre-wrap',
|
||||
overflowX: 'auto',
|
||||
margin: 0,
|
||||
border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`,
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(config.options).map(([key, option]) => {
|
||||
if (option.condition && !option.condition(values)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const items = option.getDynamicItems ? option.getDynamicItems(values) : option.items || [];
|
||||
|
||||
return (
|
||||
<div key={key} style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={itemsStyle}>
|
||||
{option.type === 'text' ? (
|
||||
<input
|
||||
type="text"
|
||||
value={values[option.name] || ''}
|
||||
placeholder={option.placeholder || ''}
|
||||
onChange={(event) => handleTextChange(option.name, event.target.value)}
|
||||
style={textInputStyle}
|
||||
/>
|
||||
) : option.type === 'checkbox' ? (
|
||||
(option.items || []).map((item) => {
|
||||
const isChecked = (values[option.name] || []).includes(item.id);
|
||||
const isDisabled =
|
||||
item.required ||
|
||||
(typeof item.disabledWhen === 'function' && item.disabledWhen(values));
|
||||
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={(event) =>
|
||||
handleCheckboxChange(option.name, item.id, event.target.checked)
|
||||
}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
) : (
|
||||
items.map((item) => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
const isDisabled = Boolean(item.disabled);
|
||||
|
||||
return (
|
||||
<label
|
||||
key={item.id}
|
||||
title={item.disabledReason || ''}
|
||||
style={{
|
||||
...labelBaseStyle,
|
||||
...(isChecked ? checkedStyle : {}),
|
||||
...(isDisabled ? disabledStyle : {}),
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="radio"
|
||||
name={option.name}
|
||||
value={item.id}
|
||||
checked={isChecked}
|
||||
disabled={isDisabled}
|
||||
onChange={() => !isDisabled && handleRadioChange(option.name, item.id)}
|
||||
style={{ display: 'none' }}
|
||||
/>
|
||||
{item.label}
|
||||
{item.subtitle && (
|
||||
<small
|
||||
style={{
|
||||
...subtitleStyle,
|
||||
color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit',
|
||||
}}
|
||||
>
|
||||
{item.subtitle}
|
||||
</small>
|
||||
)}
|
||||
</label>
|
||||
);
|
||||
})
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Run this Command:</div>
|
||||
<pre style={commandDisplayStyle}>{command}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,214 @@
|
||||
export const SpecBundleDeployment = () => {
|
||||
// Config options based on SpecBundleConfigGenerator - matching original structure exactly
|
||||
const baseConfig = {
|
||||
options: {
|
||||
mode: {
|
||||
name: 'mode',
|
||||
title: 'Launch Mode',
|
||||
renderType: 'radio',
|
||||
items: [
|
||||
{ id: 'with-server', label: 'With Server', subtitle: 'Launch SGLang server & Benchmark concurrently', default: true },
|
||||
{ id: 'without-server', label: 'Without Server', subtitle: 'Connect to an existing server (--skip-launch-server)', default: false }
|
||||
]
|
||||
},
|
||||
common: {
|
||||
name: 'common',
|
||||
title: 'Common Configuration',
|
||||
renderType: 'inputs',
|
||||
items: [
|
||||
{ id: 'modelPath', label: 'Model Path', type: 'text', placeholder: 'e.g., meta-llama/Llama-3.1-8B-Instruct', default: 'meta-llama/Llama-3.1-8B-Instruct', description: 'Path to the target model.' },
|
||||
{ id: 'port', label: 'Port', type: 'number', default: 30000, description: 'Port to launch/connect the SGLang server.' },
|
||||
{ id: 'configList', label: 'Config List', type: 'text', default: '1,3,1,4', description: 'Format: <batch-size>,<num-steps>,<topk>,<num-draft-tokens>' },
|
||||
{ id: 'benchmarkList', label: 'Benchmark List', type: 'textarea', default: 'mtbench:5 ceval:5:accountant', description: 'Format: <benchmark-name>:<num-prompts>:<subset>. Supported: aime, ceval, financeqa, gpqa, gsm8k, humaneval, livecodebench, math500, mmlu, mmstar, mtbench, simpleqa' }
|
||||
]
|
||||
},
|
||||
server: {
|
||||
name: 'server',
|
||||
title: 'Server Configuration',
|
||||
renderType: 'inputs',
|
||||
requiredMode: 'with-server',
|
||||
items: [
|
||||
{ id: 'draftModelPath', label: 'Draft Model Path', type: 'text', placeholder: 'Path to draft model', default: '', description: 'Path to the speculative draft model.' },
|
||||
{ id: 'tpSize', label: 'TP Size', type: 'number', default: 1, description: 'Number of GPUs for Tensor Parallelism.' },
|
||||
{ id: 'memFraction', label: 'Memory Fraction Static', type: 'number', step: '0.1', default: 0.9, description: 'The memory fraction for the static memory.' },
|
||||
{ id: 'attentionBackend', label: 'Attention Backend', type: 'text', default: '', description: 'The attention backend used in sglang' },
|
||||
{ id: 'trustRemoteCode', label: 'Trust Remote Code', type: 'checkbox', default: true, description: 'Whether to trust remote code.' }
|
||||
]
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Initialize state - matching original logic
|
||||
const getInitialState = () => {
|
||||
const initialState = {};
|
||||
Object.values(baseConfig.options).forEach(option => {
|
||||
if (option.renderType === 'radio') {
|
||||
const defaultItem = option.items.find(item => item.default);
|
||||
initialState[option.name] = defaultItem ? defaultItem.id : option.items[0].id;
|
||||
} else if (option.renderType === 'inputs') {
|
||||
option.items.forEach(item => {
|
||||
initialState[item.id] = item.default;
|
||||
});
|
||||
}
|
||||
});
|
||||
return initialState;
|
||||
};
|
||||
|
||||
const [values, setValues] = useState(getInitialState);
|
||||
const [isDark, setIsDark] = useState(false);
|
||||
|
||||
// Detect dark mode
|
||||
useEffect(() => {
|
||||
const checkDarkMode = () => {
|
||||
const html = document.documentElement;
|
||||
const isDarkMode = html.classList.contains('dark') ||
|
||||
html.getAttribute('data-theme') === 'dark' ||
|
||||
html.style.colorScheme === 'dark';
|
||||
setIsDark(isDarkMode);
|
||||
};
|
||||
checkDarkMode();
|
||||
const observer = new MutationObserver(checkDarkMode);
|
||||
observer.observe(document.documentElement, { attributes: true, attributeFilter: ['class', 'data-theme', 'style'] });
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
// Get display options based on current mode
|
||||
const getDisplayOptions = () => {
|
||||
const options = {};
|
||||
const currentMode = values.mode;
|
||||
Object.entries(baseConfig.options).forEach(([key, option]) => {
|
||||
if (option.requiredMode && option.requiredMode !== currentMode) {
|
||||
return;
|
||||
}
|
||||
options[key] = option;
|
||||
});
|
||||
return options;
|
||||
};
|
||||
|
||||
const handleRadioChange = (optionName, itemId) => {
|
||||
setValues(prev => ({ ...prev, [optionName]: itemId }));
|
||||
};
|
||||
|
||||
const handleInputChange = (itemId, value) => {
|
||||
setValues(prev => ({ ...prev, [itemId]: value }));
|
||||
};
|
||||
|
||||
const handleCheckboxChange = (itemId, checked) => {
|
||||
setValues(prev => ({ ...prev, [itemId]: checked }));
|
||||
};
|
||||
|
||||
// Generate command - matching original logic
|
||||
const generateCommand = () => {
|
||||
const { mode, modelPath, port, configList, benchmarkList, draftModelPath, tpSize, memFraction, attentionBackend, trustRemoteCode } = values;
|
||||
|
||||
let cmd = 'python bench_eagle3.py';
|
||||
if (modelPath) cmd += ` \\\n --model-path ${modelPath}`;
|
||||
if (port) cmd += ` \\\n --port ${port}`;
|
||||
if (configList) cmd += ` \\\n --config-list ${configList}`;
|
||||
if (benchmarkList) cmd += ` \\\n --benchmark-list ${benchmarkList.replace(/\n/g, ' ')}`;
|
||||
|
||||
if (mode === 'without-server') {
|
||||
cmd += ' \\\n --skip-launch-server';
|
||||
} else {
|
||||
if (draftModelPath) cmd += ` \\\n --speculative-draft-model-path ${draftModelPath}`;
|
||||
if (tpSize) cmd += ` \\\n --tp-size ${tpSize}`;
|
||||
if (memFraction) cmd += ` \\\n --mem-fraction-static ${memFraction}`;
|
||||
if (attentionBackend) cmd += ` \\\n --attention-backend ${attentionBackend}`;
|
||||
if (trustRemoteCode) cmd += ` \\\n --trust-remote-code`;
|
||||
}
|
||||
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Styles
|
||||
const containerStyle = { maxWidth: '900px', margin: '0 auto', display: 'flex', flexDirection: 'column', gap: '4px' };
|
||||
const cardStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}`, borderLeft: `3px solid ${isDark ? '#E85D4D' : '#D45D44'}`, borderRadius: '4px', display: 'flex', alignItems: 'flex-start', gap: '12px', background: isDark ? '#1f2937' : '#fff' };
|
||||
const titleStyle = { fontSize: '13px', fontWeight: '600', minWidth: '180px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit', paddingTop: '4px' };
|
||||
const contentStyle = { flex: 1 };
|
||||
const itemsStyle = { display: 'flex', rowGap: '4px', columnGap: '6px', flexWrap: 'wrap', alignItems: 'center' };
|
||||
const labelBaseStyle = { padding: '4px 10px', border: `1px solid ${isDark ? '#9ca3af' : '#d1d5db'}`, borderRadius: '3px', cursor: 'pointer', display: 'inline-flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center', fontWeight: '500', fontSize: '13px', transition: 'all 0.2s', userSelect: 'none', minWidth: '45px', textAlign: 'center', background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit' };
|
||||
const checkedStyle = { background: '#D45D44', color: 'white', borderColor: '#D45D44' };
|
||||
const subtitleStyle = { display: 'block', fontSize: '9px', marginTop: '1px', lineHeight: '1.1', opacity: 0.7 };
|
||||
const inputGroupStyle = { display: 'flex', flexDirection: 'column', gap: '8px' };
|
||||
const inputRowStyle = { display: 'flex', alignItems: 'flex-start', gap: '12px' };
|
||||
const inputLabelStyle = { fontSize: '13px', fontWeight: '500', minWidth: '180px', flexShrink: 0, color: isDark ? '#e5e7eb' : 'inherit', paddingTop: '8px' };
|
||||
const inputContentStyle = { flex: 1, display: 'flex', flexDirection: 'column' };
|
||||
const inputStyle = { padding: '8px 12px', border: `1px solid ${isDark ? '#4b5563' : '#d1d5db'}`, borderRadius: '4px', fontSize: '13px', background: isDark ? '#374151' : '#fff', color: isDark ? '#e5e7eb' : 'inherit', width: '100%', boxSizing: 'border-box' };
|
||||
const textareaStyle = { ...inputStyle, minHeight: '60px', resize: 'vertical' };
|
||||
const descStyle = { color: isDark ? '#9ca3af' : '#666', marginTop: '4px', fontSize: '11px' };
|
||||
const commandDisplayStyle = { flex: 1, padding: '12px 16px', background: isDark ? '#111827' : '#f5f5f5', borderRadius: '6px', fontFamily: "'Menlo', 'Monaco', 'Courier New', monospace", fontSize: '12px', lineHeight: '1.5', color: isDark ? '#e5e7eb' : '#374151', whiteSpace: 'pre-wrap', overflowX: 'auto', margin: 0, border: `1px solid ${isDark ? '#374151' : '#e5e7eb'}` };
|
||||
|
||||
const displayOptions = getDisplayOptions();
|
||||
|
||||
return (
|
||||
<div style={containerStyle} className="not-prose">
|
||||
{Object.entries(displayOptions).map(([key, option]) => (
|
||||
<div key={key}>
|
||||
{/* Render Radio Group - title on left */}
|
||||
{option.renderType === 'radio' && (
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>{option.title}</div>
|
||||
<div style={{ ...contentStyle, ...itemsStyle }}>
|
||||
{option.items.map(item => {
|
||||
const isChecked = values[option.name] === item.id;
|
||||
return (
|
||||
<label key={item.id} style={{ ...labelBaseStyle, ...(isChecked ? checkedStyle : {}) }}>
|
||||
<input type="radio" name={option.name} value={item.id} checked={isChecked} onChange={() => handleRadioChange(option.name, item.id)} style={{ display: 'none' }} />
|
||||
{item.label}
|
||||
{item.subtitle && <small style={{ ...subtitleStyle, color: isChecked ? 'rgba(255,255,255,0.85)' : 'inherit' }}>{item.subtitle}</small>}
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Render Input Group - each input has label on left */}
|
||||
{option.renderType === 'inputs' && (
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: '4px' }}>
|
||||
{option.items.map(item => (
|
||||
<div key={item.id} style={cardStyle}>
|
||||
<div style={inputLabelStyle}>{item.label}</div>
|
||||
<div style={inputContentStyle}>
|
||||
{item.type === 'textarea' ? (
|
||||
<textarea
|
||||
value={values[item.id]}
|
||||
onChange={(e) => handleInputChange(item.id, e.target.value)}
|
||||
style={textareaStyle}
|
||||
/>
|
||||
) : item.type === 'checkbox' ? (
|
||||
<label style={{ display: 'flex', alignItems: 'center', cursor: 'pointer', gap: '8px', padding: '4px 0' }}>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={values[item.id]}
|
||||
onChange={(e) => handleCheckboxChange(item.id, e.target.checked)}
|
||||
style={{ width: '16px', height: '16px', cursor: 'pointer' }}
|
||||
/>
|
||||
<span style={{ fontSize: '13px', color: isDark ? '#e5e7eb' : 'inherit' }}>Enabled</span>
|
||||
</label>
|
||||
) : (
|
||||
<input
|
||||
type={item.type}
|
||||
value={values[item.id]}
|
||||
placeholder={item.placeholder}
|
||||
step={item.step}
|
||||
onChange={(e) => handleInputChange(item.id, e.target.value)}
|
||||
style={inputStyle}
|
||||
/>
|
||||
)}
|
||||
{item.description && <small style={descStyle}>{item.description}</small>}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
|
||||
<div style={cardStyle}>
|
||||
<div style={titleStyle}>Generated Command</div>
|
||||
<pre style={commandDisplayStyle}>{generateCommand()}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
Reference in New Issue
Block a user