docs(cookbook): mark every Kimi-K3 cell in-progress; land Playground "Switch base" on the configurator (#32586)
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
356c11d5d9
commit
b79388f338
@@ -101,9 +101,9 @@ are scheduled to release by July 27, 2026**. The recipes on this page were valid
|
|||||||
repository (`moonshotai/Kimi-K3`) and a public `lmsysorg/sglang` image with K3 support will be
|
repository (`moonshotai/Kimi-K3`) and a public `lmsysorg/sglang` image with K3 support will be
|
||||||
available at launch.
|
available at launch.
|
||||||
|
|
||||||
Every cell on this page is currently marked **Not Verified**: the recipes run, but none has a
|
Every cell in the Deploy panel above is currently marked **Final Verification In Progress**: the
|
||||||
serving round on the final weights and current code behind it. Re-measure throughput and accuracy
|
recipe runs, but its serving round on the final weights and current code is still open. Re-measure
|
||||||
before you rely on any of them.
|
throughput and accuracy before you rely on any of them.
|
||||||
</Note>
|
</Note>
|
||||||
|
|
||||||
**Recommended generation:** `temperature=1.0`, `top_p=0.95`, `presence_penalty=0`, `frequency_penalty=0` (fixed by the model; informational — do not hardcode in sample code).
|
**Recommended generation:** `temperature=1.0`, `top_p=0.95`, `presence_penalty=0`, `frequency_penalty=0` (fixed by the model; informational — do not hardcode in sample code).
|
||||||
|
|||||||
@@ -23,7 +23,8 @@
|
|||||||
// `flags` / `env` / `hints` (each a literal array or a function
|
// `flags` / `env` / `hints` (each a literal array or a function
|
||||||
// of the whole selection), and a row-level `default` / `showWhen`.
|
// of the whole selection), and a row-level `default` / `showWhen`.
|
||||||
// `hints` render as `# ...` lines above the command.
|
// `hints` render as `# ...` lines above the command.
|
||||||
// cells {match, verified?, nnodes?, warn?, redirect?, env, flags}[] — one per
|
// cells {match, verified?, verificationStatus?, nnodes?, warn?, redirect?,
|
||||||
|
// env, flags}[] — one per
|
||||||
// (hw × match dims); env/flags are flat literals, only
|
// (hw × match dims); env/flags are flat literals, only
|
||||||
// {{PLACEHOLDER}} subst applied. `nnodes` supplies the node
|
// {{PLACEHOLDER}} subst applied. `nnodes` supplies the node
|
||||||
// count for configs with no `nodes` dim (default 1). `warn`
|
// count for configs with no `nodes` dim (default 1). `warn`
|
||||||
@@ -31,6 +32,10 @@
|
|||||||
// embed [label](#anchor) links. `redirect: true` renders the
|
// embed [label](#anchor) links. `redirect: true` renders the
|
||||||
// banner ALONE — no command, header, or copy buttons — for
|
// banner ALONE — no command, header, or copy buttons — for
|
||||||
// cells that only point somewhere else.
|
// cells that only point somewhere else.
|
||||||
|
// `verified` is the boolean badge baseline.
|
||||||
|
// `verificationStatus` overrides it with a third state —
|
||||||
|
// "verified" | "in-progress" | "unverified" — for a recipe
|
||||||
|
// whose verification round is open rather than absent.
|
||||||
// modelNames HF slug lookup, `hw|variant|quant` then `variant|quant`
|
// modelNames HF slug lookup, `hw|variant|quant` then `variant|quant`
|
||||||
// placeholders {{KEY}} → {target: 'command'|'curl', label, default?}
|
// placeholders {{KEY}} → {target: 'command'|'curl', label, default?}
|
||||||
// curl cURL template (uses {{MODEL_NAME}} + placeholders)
|
// curl cURL template (uses {{MODEL_NAME}} + placeholders)
|
||||||
@@ -166,18 +171,30 @@ export const Deployment = ({ config, benchmarks }) => {
|
|||||||
color: isDark ? "#fde68a" : "#92400e",
|
color: isDark ? "#fde68a" : "#92400e",
|
||||||
border: `1px solid ${isDark ? "#92400e" : "#fcd34d"}`,
|
border: `1px solid ${isDark ? "#92400e" : "#fcd34d"}`,
|
||||||
},
|
},
|
||||||
badge: (verified) => ({
|
// Takes either a boolean (legacy `cell.verified`) or a status id — see
|
||||||
|
// VERIFY_LABEL / verifyStatusOf in section 3.
|
||||||
|
badge: (status) => ({
|
||||||
display: "inline-flex", alignItems: "center", gap: "6px",
|
display: "inline-flex", alignItems: "center", gap: "6px",
|
||||||
padding: "2px 8px", borderRadius: "10px",
|
padding: "2px 8px", borderRadius: "10px",
|
||||||
background: verified ? (isDark ? "#064e3b" : "#d1fae5")
|
background: {
|
||||||
: (isDark ? "#78350f" : "#fef3c7"),
|
verified: isDark ? "#064e3b" : "#d1fae5",
|
||||||
color: verified ? (isDark ? "#a7f3d0" : "#065f46")
|
"in-progress": isDark ? "#1e3a8a" : "#dbeafe",
|
||||||
: (isDark ? "#fde68a" : "#92400e"),
|
unverified: isDark ? "#78350f" : "#fef3c7",
|
||||||
fontSize: "11px", fontWeight: 600,
|
}[verifyStatusOf(status)],
|
||||||
|
color: {
|
||||||
|
verified: isDark ? "#a7f3d0" : "#065f46",
|
||||||
|
"in-progress": isDark ? "#bfdbfe" : "#1e40af",
|
||||||
|
unverified: isDark ? "#fde68a" : "#92400e",
|
||||||
|
}[verifyStatusOf(status)],
|
||||||
|
// The in-progress label is long; keep the pill on one line and let the
|
||||||
|
// header row wrap around it instead of breaking the text mid-badge.
|
||||||
|
fontSize: "11px", fontWeight: 600, whiteSpace: "nowrap",
|
||||||
}),
|
}),
|
||||||
badgeDot: (verified) => ({
|
badgeDot: (status) => ({
|
||||||
width: "8px", height: "8px", borderRadius: "50%",
|
width: "8px", height: "8px", borderRadius: "50%",
|
||||||
background: verified ? "#10b981" : "#f59e0b",
|
background: { verified: "#10b981", "in-progress": "#3b82f6", unverified: "#f59e0b" }[
|
||||||
|
verifyStatusOf(status)
|
||||||
|
],
|
||||||
}),
|
}),
|
||||||
iconButton: {
|
iconButton: {
|
||||||
padding: "4px 10px",
|
padding: "4px 10px",
|
||||||
@@ -388,6 +405,25 @@ export const Deployment = ({ config, benchmarks }) => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// ==== 3. Pure helpers (no React state) ====
|
// ==== 3. Pure helpers (no React state) ====
|
||||||
|
// Verification badge state. A cell's boolean `verified` is the baseline;
|
||||||
|
// `cell.verificationStatus` overrides it, which is how a recipe whose
|
||||||
|
// verification round is open reports that instead of collapsing into the flat
|
||||||
|
// Verified / Not Verified pair.
|
||||||
|
const VERIFY_LABEL = {
|
||||||
|
verified: "Verified",
|
||||||
|
"in-progress": "Final Verification In Progress",
|
||||||
|
unverified: "Not Verified",
|
||||||
|
};
|
||||||
|
// Booleans keep their historical meaning; an unrecognized status id falls
|
||||||
|
// back to "unverified" rather than to truthiness (a typo must never read as
|
||||||
|
// a green Verified badge).
|
||||||
|
const verifyStatusOf = (v) =>
|
||||||
|
typeof v === "string"
|
||||||
|
? (VERIFY_LABEL[v] ? v : "unverified")
|
||||||
|
: (v ? "verified" : "unverified");
|
||||||
|
const cellVerifyStatus = (c) =>
|
||||||
|
c ? verifyStatusOf(c.verificationStatus ?? c.verified) : "unverified";
|
||||||
|
|
||||||
// Two kinds of selector row:
|
// Two kinds of selector row:
|
||||||
// match dims participate in cell lookup (cell.match[dim] === sel[dim])
|
// match dims participate in cell lookup (cell.match[dim] === sel[dim])
|
||||||
// overlay dims never touch cell lookup; the picked option contributes flags
|
// overlay dims never touch cell lookup; the picked option contributes flags
|
||||||
@@ -1122,6 +1158,7 @@ export const Deployment = ({ config, benchmarks }) => {
|
|||||||
// ==== 5. Derived values ====
|
// ==== 5. Derived values ====
|
||||||
const s = makeStyles(isDark);
|
const s = makeStyles(isDark);
|
||||||
const cell = findCell(config.cells, sel);
|
const cell = findCell(config.cells, sel);
|
||||||
|
const verifyStatus = cellVerifyStatus(cell);
|
||||||
// Pin the calculator-computed ratio into the rendered command (before the
|
// Pin the calculator-computed ratio into the rendered command (before the
|
||||||
// host/port tail); cells themselves stay ratio-free.
|
// host/port tail); cells themselves stay ratio-free.
|
||||||
const cellWithRatio = (() => {
|
const cellWithRatio = (() => {
|
||||||
@@ -1337,9 +1374,9 @@ export const Deployment = ({ config, benchmarks }) => {
|
|||||||
) : (<>
|
) : (<>
|
||||||
<div style={s.commandHeader}>
|
<div style={s.commandHeader}>
|
||||||
<div style={s.headerLeft}>
|
<div style={s.headerLeft}>
|
||||||
<div style={s.badge(Boolean(cell && cell.verified))}>
|
<div style={s.badge(verifyStatus)}>
|
||||||
<span style={s.badgeDot(Boolean(cell && cell.verified))} />
|
<span style={s.badgeDot(verifyStatus)} />
|
||||||
{cell && cell.verified ? "Verified" : "Not Verified"}
|
{VERIFY_LABEL[verifyStatus]}
|
||||||
</div>
|
</div>
|
||||||
<div style={s.runModeWrap} role="tablist" aria-label="Output format">
|
<div style={s.runModeWrap} role="tablist" aria-label="Output format">
|
||||||
{runModes.map((mode, index) => (
|
{runModes.map((mode, index) => (
|
||||||
|
|||||||
@@ -101,6 +101,11 @@ export const Playground = ({ config }) => {
|
|||||||
}) || cell;
|
}) || cell;
|
||||||
// Shared with `_deployment.jsx` (HOST/PORT/etc. unified across the page).
|
// Shared with `_deployment.jsx` (HOST/PORT/etc. unified across the page).
|
||||||
const STORAGE_KEY = "sglang-deploy-env";
|
const STORAGE_KEY = "sglang-deploy-env";
|
||||||
|
// ==== MIRROR of DEPLOYMENT_COMPONENT_ID in _deployment.jsx — keep identical ====
|
||||||
|
// Snippets cannot import each other. The Deploy panel puts this id on its
|
||||||
|
// root (carrying its own scrollMarginTop), which is what "jump to the base"
|
||||||
|
// should land on.
|
||||||
|
const DEPLOYMENT_COMPONENT_ID = "deployment-configurator";
|
||||||
|
|
||||||
const pgFeatures = config.playgroundFeatures || {};
|
const pgFeatures = config.playgroundFeatures || {};
|
||||||
// Single-host PD runs prefill + decode as two engines on one box. Each derives
|
// Single-host PD runs prefill + decode as two engines on one box. Each derives
|
||||||
@@ -2218,12 +2223,16 @@ export const Playground = ({ config }) => {
|
|||||||
<span style={{ fontWeight: 600 }}>Inherited base from Deployment:</span>
|
<span style={{ fontWeight: 600 }}>Inherited base from Deployment:</span>
|
||||||
<code style={{ fontFamily: "Menlo, monospace" }}>{baseSummary}</code>
|
<code style={{ fontFamily: "Menlo, monospace" }}>{baseSummary}</code>
|
||||||
{/* scrollIntoView (not hash nav) so the base-cell hash survives.
|
{/* scrollIntoView (not hash nav) so the base-cell hash survives.
|
||||||
Deploy heading slugs to "deployment" or "deploy". */}
|
Target the configurator ITSELF: the "## Deployment" heading sits
|
||||||
|
above the install accordion and a screenful of prose, so landing on
|
||||||
|
the heading leaves the panel you came for off-screen. The heading
|
||||||
|
slugs stay as fallbacks for a page that renders no configurator. */}
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
style={s.switchBaseBtn}
|
style={s.switchBaseBtn}
|
||||||
onClick={() => {
|
onClick={() => {
|
||||||
const el = document.getElementById("deployment")
|
const el = document.getElementById(DEPLOYMENT_COMPONENT_ID)
|
||||||
|
|| document.getElementById("deployment")
|
||||||
|| document.getElementById("deploy");
|
|| document.getElementById("deploy");
|
||||||
if (el) el.scrollIntoView({ behavior: "smooth", block: "start" });
|
if (el) el.scrollIntoView({ behavior: "smooth", block: "start" });
|
||||||
}}
|
}}
|
||||||
|
|||||||
@@ -647,11 +647,16 @@ export const config = {
|
|||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
|
||||||
|
// Verification marks: every cell carries "Final Verification In Progress" —
|
||||||
|
// the recipe runs, but its serving round on the final weights and current
|
||||||
|
// code is still open. Flip a cell to `verified: true` (and drop its
|
||||||
|
// `verificationStatus`) once that round lands.
|
||||||
cells: [
|
cells: [
|
||||||
{
|
{
|
||||||
match: { hw: "b300", pdMode: "unified", strategy: "low-latency" },
|
match: { hw: "b300", pdMode: "unified", strategy: "low-latency" },
|
||||||
nnodes: 1,
|
nnodes: 1,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [],
|
env: [],
|
||||||
flags: [
|
flags: [
|
||||||
"--trust-remote-code",
|
"--trust-remote-code",
|
||||||
@@ -670,6 +675,7 @@ export const config = {
|
|||||||
match: { hw: "b300", pdMode: "unified", strategy: "balanced" },
|
match: { hw: "b300", pdMode: "unified", strategy: "balanced" },
|
||||||
nnodes: 1,
|
nnodes: 1,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [],
|
env: [],
|
||||||
flags: [
|
flags: [
|
||||||
"--trust-remote-code",
|
"--trust-remote-code",
|
||||||
@@ -688,6 +694,7 @@ export const config = {
|
|||||||
match: { hw: "b300", pdMode: "unified", strategy: "high-throughput" },
|
match: { hw: "b300", pdMode: "unified", strategy: "high-throughput" },
|
||||||
nnodes: 1,
|
nnodes: 1,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
redirect: true,
|
redirect: true,
|
||||||
warn: "High-Throughput is the large-scale lane: pick a Cluster Size and a Large-Scale Preset in the [Playground](#playground) to compose the DP x EP command on top of this hardware's Balanced recipe.",
|
warn: "High-Throughput is the large-scale lane: pick a Cluster Size and a Large-Scale Preset in the [Playground](#playground) to compose the DP x EP command on top of this hardware's Balanced recipe.",
|
||||||
env: [],
|
env: [],
|
||||||
@@ -709,6 +716,7 @@ export const config = {
|
|||||||
match: { hw: "b200", pdMode: "unified", strategy: "low-latency" },
|
match: { hw: "b200", pdMode: "unified", strategy: "low-latency" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [],
|
env: [],
|
||||||
flags: [
|
flags: [
|
||||||
"--trust-remote-code",
|
"--trust-remote-code",
|
||||||
@@ -729,6 +737,7 @@ export const config = {
|
|||||||
match: { hw: "b200", pdMode: "unified", strategy: "balanced" },
|
match: { hw: "b200", pdMode: "unified", strategy: "balanced" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [],
|
env: [],
|
||||||
flags: [
|
flags: [
|
||||||
"--trust-remote-code",
|
"--trust-remote-code",
|
||||||
@@ -750,6 +759,7 @@ export const config = {
|
|||||||
match: { hw: "b200", pdMode: "unified", strategy: "high-throughput" },
|
match: { hw: "b200", pdMode: "unified", strategy: "high-throughput" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
redirect: true,
|
redirect: true,
|
||||||
warn: "High-Throughput is the large-scale lane: pick a Cluster Size and a Large-Scale Preset in the [Playground](#playground) to compose the DP x EP command on top of this hardware's Balanced recipe.",
|
warn: "High-Throughput is the large-scale lane: pick a Cluster Size and a Large-Scale Preset in the [Playground](#playground) to compose the DP x EP command on top of this hardware's Balanced recipe.",
|
||||||
env: [],
|
env: [],
|
||||||
@@ -774,6 +784,7 @@ export const config = {
|
|||||||
match: { hw: "b200", pdMode: "unified", strategy: "long-context" },
|
match: { hw: "b200", pdMode: "unified", strategy: "long-context" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [],
|
env: [],
|
||||||
flags: [
|
flags: [
|
||||||
"--trust-remote-code",
|
"--trust-remote-code",
|
||||||
@@ -798,6 +809,7 @@ export const config = {
|
|||||||
match: { hw: "mi350x", pdMode: "unified", strategy: "balanced" },
|
match: { hw: "mi350x", pdMode: "unified", strategy: "balanced" },
|
||||||
nnodes: 1,
|
nnodes: 1,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_USE_AITER=1",
|
"SGLANG_USE_AITER=1",
|
||||||
"SGLANG_AITER_K3_OPT=1",
|
"SGLANG_AITER_K3_OPT=1",
|
||||||
@@ -823,6 +835,7 @@ export const config = {
|
|||||||
match: { hw: "mi355x", pdMode: "unified", strategy: "balanced" },
|
match: { hw: "mi355x", pdMode: "unified", strategy: "balanced" },
|
||||||
nnodes: 1,
|
nnodes: 1,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_USE_AITER=1",
|
"SGLANG_USE_AITER=1",
|
||||||
"SGLANG_AITER_K3_OPT=1",
|
"SGLANG_AITER_K3_OPT=1",
|
||||||
@@ -849,6 +862,7 @@ export const config = {
|
|||||||
match: { hw: "h100", pdMode: "unified", strategy: "low-latency" },
|
match: { hw: "h100", pdMode: "unified", strategy: "low-latency" },
|
||||||
nnodes: 4,
|
nnodes: 4,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"NCCL_CUMEM_ENABLE=1",
|
"NCCL_CUMEM_ENABLE=1",
|
||||||
"PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
|
"PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
|
||||||
@@ -880,6 +894,7 @@ export const config = {
|
|||||||
match: { hw: "h100", pdMode: "unified", strategy: "balanced" },
|
match: { hw: "h100", pdMode: "unified", strategy: "balanced" },
|
||||||
nnodes: 4,
|
nnodes: 4,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"NCCL_CUMEM_ENABLE=1",
|
"NCCL_CUMEM_ENABLE=1",
|
||||||
"PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
|
"PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
|
||||||
@@ -911,6 +926,7 @@ export const config = {
|
|||||||
match: { hw: "h100", pdMode: "unified", strategy: "high-throughput" },
|
match: { hw: "h100", pdMode: "unified", strategy: "high-throughput" },
|
||||||
nnodes: 4,
|
nnodes: 4,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"NCCL_CUMEM_ENABLE=1",
|
"NCCL_CUMEM_ENABLE=1",
|
||||||
"PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
|
"PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
|
||||||
@@ -941,6 +957,7 @@ export const config = {
|
|||||||
match: { hw: "h200", pdMode: "unified", strategy: "low-latency" },
|
match: { hw: "h200", pdMode: "unified", strategy: "low-latency" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"NCCL_MNNVL_ENABLE=1",
|
"NCCL_MNNVL_ENABLE=1",
|
||||||
"NCCL_CUMEM_ENABLE=1",
|
"NCCL_CUMEM_ENABLE=1",
|
||||||
@@ -966,6 +983,7 @@ export const config = {
|
|||||||
match: { hw: "h200", pdMode: "unified", strategy: "balanced" },
|
match: { hw: "h200", pdMode: "unified", strategy: "balanced" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"NCCL_MNNVL_ENABLE=1",
|
"NCCL_MNNVL_ENABLE=1",
|
||||||
"NCCL_CUMEM_ENABLE=1",
|
"NCCL_CUMEM_ENABLE=1",
|
||||||
@@ -991,6 +1009,7 @@ export const config = {
|
|||||||
match: { hw: "h200", pdMode: "unified", strategy: "high-throughput" },
|
match: { hw: "h200", pdMode: "unified", strategy: "high-throughput" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"NCCL_MNNVL_ENABLE=1",
|
"NCCL_MNNVL_ENABLE=1",
|
||||||
"NCCL_CUMEM_ENABLE=1",
|
"NCCL_CUMEM_ENABLE=1",
|
||||||
@@ -1016,6 +1035,7 @@ export const config = {
|
|||||||
match: { hw: "gb300", pdMode: "unified", strategy: "low-latency" },
|
match: { hw: "gb300", pdMode: "unified", strategy: "low-latency" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
||||||
],
|
],
|
||||||
@@ -1037,6 +1057,7 @@ export const config = {
|
|||||||
match: { hw: "gb300", pdMode: "unified", strategy: "balanced" },
|
match: { hw: "gb300", pdMode: "unified", strategy: "balanced" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
||||||
],
|
],
|
||||||
@@ -1056,6 +1077,7 @@ export const config = {
|
|||||||
match: { hw: "gb300", pdMode: "unified", strategy: "high-throughput" },
|
match: { hw: "gb300", pdMode: "unified", strategy: "high-throughput" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
redirect: true,
|
redirect: true,
|
||||||
warn: "High-Throughput is the large-scale lane: pick a Cluster Size and a Large-Scale Preset in the [Playground](#playground) to compose the DP x EP command on top of this hardware's Balanced recipe.",
|
warn: "High-Throughput is the large-scale lane: pick a Cluster Size and a Large-Scale Preset in the [Playground](#playground) to compose the DP x EP command on top of this hardware's Balanced recipe.",
|
||||||
env: [
|
env: [
|
||||||
@@ -1078,6 +1100,7 @@ export const config = {
|
|||||||
match: { hw: "gb200", pdMode: "unified", strategy: "low-latency" },
|
match: { hw: "gb200", pdMode: "unified", strategy: "low-latency" },
|
||||||
nnodes: 4,
|
nnodes: 4,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
||||||
],
|
],
|
||||||
@@ -1098,6 +1121,7 @@ export const config = {
|
|||||||
match: { hw: "gb200", pdMode: "unified", strategy: "balanced" },
|
match: { hw: "gb200", pdMode: "unified", strategy: "balanced" },
|
||||||
nnodes: 4,
|
nnodes: 4,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
||||||
],
|
],
|
||||||
@@ -1118,6 +1142,7 @@ export const config = {
|
|||||||
match: { hw: "gb200", pdMode: "unified", strategy: "high-throughput" },
|
match: { hw: "gb200", pdMode: "unified", strategy: "high-throughput" },
|
||||||
nnodes: 4,
|
nnodes: 4,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
redirect: true,
|
redirect: true,
|
||||||
warn: "High-Throughput is the large-scale lane: pick a Cluster Size and a Large-Scale Preset in the [Playground](#playground) to compose the DP x EP command on top of this hardware's Balanced recipe.",
|
warn: "High-Throughput is the large-scale lane: pick a Cluster Size and a Large-Scale Preset in the [Playground](#playground) to compose the DP x EP command on top of this hardware's Balanced recipe.",
|
||||||
env: [
|
env: [
|
||||||
@@ -1147,6 +1172,7 @@ export const config = {
|
|||||||
match: { hw: "b300", pdMode: "prefill", strategy: "default" },
|
match: { hw: "b300", pdMode: "prefill", strategy: "default" },
|
||||||
nnodes: 1,
|
nnodes: 1,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [],
|
env: [],
|
||||||
flags: [
|
flags: [
|
||||||
"--trust-remote-code",
|
"--trust-remote-code",
|
||||||
@@ -1174,6 +1200,7 @@ export const config = {
|
|||||||
match: { hw: "b300", pdMode: "prefill", strategy: "long-context" },
|
match: { hw: "b300", pdMode: "prefill", strategy: "long-context" },
|
||||||
nnodes: 1,
|
nnodes: 1,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [],
|
env: [],
|
||||||
flags: [
|
flags: [
|
||||||
"--trust-remote-code",
|
"--trust-remote-code",
|
||||||
@@ -1199,6 +1226,7 @@ export const config = {
|
|||||||
match: { hw: "gb300", pdMode: "prefill", strategy: "default" },
|
match: { hw: "gb300", pdMode: "prefill", strategy: "default" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
||||||
],
|
],
|
||||||
@@ -1223,6 +1251,7 @@ export const config = {
|
|||||||
match: { hw: "gb300", pdMode: "prefill", strategy: "long-context" },
|
match: { hw: "gb300", pdMode: "prefill", strategy: "long-context" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
||||||
],
|
],
|
||||||
@@ -1260,6 +1289,7 @@ export const config = {
|
|||||||
match: { hw: "b300", pdMode: "decode", strategy: "balanced" },
|
match: { hw: "b300", pdMode: "decode", strategy: "balanced" },
|
||||||
nnodes: 1,
|
nnodes: 1,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [],
|
env: [],
|
||||||
flags: [
|
flags: [
|
||||||
"--trust-remote-code",
|
"--trust-remote-code",
|
||||||
@@ -1281,6 +1311,7 @@ export const config = {
|
|||||||
match: { hw: "b300", pdMode: "decode", strategy: "low-latency" },
|
match: { hw: "b300", pdMode: "decode", strategy: "low-latency" },
|
||||||
nnodes: 1,
|
nnodes: 1,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [],
|
env: [],
|
||||||
flags: [
|
flags: [
|
||||||
"--trust-remote-code",
|
"--trust-remote-code",
|
||||||
@@ -1302,6 +1333,7 @@ export const config = {
|
|||||||
match: { hw: "b300", pdMode: "decode", strategy: "high-throughput" },
|
match: { hw: "b300", pdMode: "decode", strategy: "high-throughput" },
|
||||||
nnodes: 1,
|
nnodes: 1,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [],
|
env: [],
|
||||||
flags: [
|
flags: [
|
||||||
"--trust-remote-code",
|
"--trust-remote-code",
|
||||||
@@ -1323,6 +1355,7 @@ export const config = {
|
|||||||
match: { hw: "b200", pdMode: "decode", strategy: "low-latency" },
|
match: { hw: "b200", pdMode: "decode", strategy: "low-latency" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [],
|
env: [],
|
||||||
flags: [
|
flags: [
|
||||||
"--trust-remote-code",
|
"--trust-remote-code",
|
||||||
@@ -1345,6 +1378,7 @@ export const config = {
|
|||||||
match: { hw: "b200", pdMode: "decode", strategy: "balanced" },
|
match: { hw: "b200", pdMode: "decode", strategy: "balanced" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [],
|
env: [],
|
||||||
flags: [
|
flags: [
|
||||||
"--trust-remote-code",
|
"--trust-remote-code",
|
||||||
@@ -1368,6 +1402,7 @@ export const config = {
|
|||||||
match: { hw: "b200", pdMode: "decode", strategy: "high-throughput" },
|
match: { hw: "b200", pdMode: "decode", strategy: "high-throughput" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [],
|
env: [],
|
||||||
flags: [
|
flags: [
|
||||||
"--trust-remote-code",
|
"--trust-remote-code",
|
||||||
@@ -1391,6 +1426,7 @@ export const config = {
|
|||||||
match: { hw: "mi350x", pdMode: "decode", strategy: "balanced" },
|
match: { hw: "mi350x", pdMode: "decode", strategy: "balanced" },
|
||||||
nnodes: 1,
|
nnodes: 1,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_USE_AITER=1",
|
"SGLANG_USE_AITER=1",
|
||||||
"SGLANG_AITER_K3_OPT=1",
|
"SGLANG_AITER_K3_OPT=1",
|
||||||
@@ -1417,6 +1453,7 @@ export const config = {
|
|||||||
match: { hw: "mi355x", pdMode: "decode", strategy: "balanced" },
|
match: { hw: "mi355x", pdMode: "decode", strategy: "balanced" },
|
||||||
nnodes: 1,
|
nnodes: 1,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_USE_AITER=1",
|
"SGLANG_USE_AITER=1",
|
||||||
"SGLANG_AITER_K3_OPT=1",
|
"SGLANG_AITER_K3_OPT=1",
|
||||||
@@ -1443,6 +1480,7 @@ export const config = {
|
|||||||
match: { hw: "h100", pdMode: "decode", strategy: "low-latency" },
|
match: { hw: "h100", pdMode: "decode", strategy: "low-latency" },
|
||||||
nnodes: 4,
|
nnodes: 4,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"NCCL_CUMEM_ENABLE=1",
|
"NCCL_CUMEM_ENABLE=1",
|
||||||
"PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
|
"PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
|
||||||
@@ -1474,6 +1512,7 @@ export const config = {
|
|||||||
match: { hw: "h100", pdMode: "decode", strategy: "balanced" },
|
match: { hw: "h100", pdMode: "decode", strategy: "balanced" },
|
||||||
nnodes: 4,
|
nnodes: 4,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"NCCL_CUMEM_ENABLE=1",
|
"NCCL_CUMEM_ENABLE=1",
|
||||||
"PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
|
"PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
|
||||||
@@ -1505,6 +1544,7 @@ export const config = {
|
|||||||
match: { hw: "h100", pdMode: "decode", strategy: "high-throughput" },
|
match: { hw: "h100", pdMode: "decode", strategy: "high-throughput" },
|
||||||
nnodes: 4,
|
nnodes: 4,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"NCCL_CUMEM_ENABLE=1",
|
"NCCL_CUMEM_ENABLE=1",
|
||||||
"PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
|
"PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
|
||||||
@@ -1536,6 +1576,7 @@ export const config = {
|
|||||||
match: { hw: "h200", pdMode: "decode", strategy: "low-latency" },
|
match: { hw: "h200", pdMode: "decode", strategy: "low-latency" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"NCCL_MNNVL_ENABLE=1",
|
"NCCL_MNNVL_ENABLE=1",
|
||||||
"NCCL_CUMEM_ENABLE=1",
|
"NCCL_CUMEM_ENABLE=1",
|
||||||
@@ -1562,6 +1603,7 @@ export const config = {
|
|||||||
match: { hw: "h200", pdMode: "decode", strategy: "balanced" },
|
match: { hw: "h200", pdMode: "decode", strategy: "balanced" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"NCCL_MNNVL_ENABLE=1",
|
"NCCL_MNNVL_ENABLE=1",
|
||||||
"NCCL_CUMEM_ENABLE=1",
|
"NCCL_CUMEM_ENABLE=1",
|
||||||
@@ -1588,6 +1630,7 @@ export const config = {
|
|||||||
match: { hw: "h200", pdMode: "decode", strategy: "high-throughput" },
|
match: { hw: "h200", pdMode: "decode", strategy: "high-throughput" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"NCCL_MNNVL_ENABLE=1",
|
"NCCL_MNNVL_ENABLE=1",
|
||||||
"NCCL_CUMEM_ENABLE=1",
|
"NCCL_CUMEM_ENABLE=1",
|
||||||
@@ -1614,6 +1657,7 @@ export const config = {
|
|||||||
match: { hw: "gb300", pdMode: "decode", strategy: "low-latency" },
|
match: { hw: "gb300", pdMode: "decode", strategy: "low-latency" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
||||||
],
|
],
|
||||||
@@ -1636,6 +1680,7 @@ export const config = {
|
|||||||
match: { hw: "gb300", pdMode: "decode", strategy: "balanced" },
|
match: { hw: "gb300", pdMode: "decode", strategy: "balanced" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
||||||
],
|
],
|
||||||
@@ -1658,6 +1703,7 @@ export const config = {
|
|||||||
match: { hw: "gb300", pdMode: "decode", strategy: "high-throughput" },
|
match: { hw: "gb300", pdMode: "decode", strategy: "high-throughput" },
|
||||||
nnodes: 2,
|
nnodes: 2,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
||||||
],
|
],
|
||||||
@@ -1680,6 +1726,7 @@ export const config = {
|
|||||||
match: { hw: "gb200", pdMode: "decode", strategy: "low-latency" },
|
match: { hw: "gb200", pdMode: "decode", strategy: "low-latency" },
|
||||||
nnodes: 4,
|
nnodes: 4,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
||||||
],
|
],
|
||||||
@@ -1702,6 +1749,7 @@ export const config = {
|
|||||||
match: { hw: "gb200", pdMode: "decode", strategy: "balanced" },
|
match: { hw: "gb200", pdMode: "decode", strategy: "balanced" },
|
||||||
nnodes: 4,
|
nnodes: 4,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
||||||
],
|
],
|
||||||
@@ -1724,6 +1772,7 @@ export const config = {
|
|||||||
match: { hw: "gb200", pdMode: "decode", strategy: "high-throughput" },
|
match: { hw: "gb200", pdMode: "decode", strategy: "high-throughput" },
|
||||||
nnodes: 4,
|
nnodes: 4,
|
||||||
verified: false,
|
verified: false,
|
||||||
|
verificationStatus: "in-progress",
|
||||||
env: [
|
env: [
|
||||||
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
"SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0",
|
||||||
],
|
],
|
||||||
|
|||||||
Reference in New Issue
Block a user