diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index d30e4d384..c524dab44 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -52,11 +52,11 @@ docker run --gpus all \ AMD uses the daily-updated `lmsysorg/sglang-rocm` images. You can find the latest images on [Docker Hub](https://hub.docker.com/r/lmsysorg/sglang-rocm/tags). We recommend the ROCm 7.2 version. For example: -- **MI355X** → `lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623` +- **MI355X** → `lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710` - **MI300X** → `lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi30x-20260623` ```bash Command -docker pull lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623 +docker pull lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 docker run \ --device=/dev/kfd --device=/dev/dri \ @@ -66,7 +66,7 @@ docker run \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=" \ - lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623 \ + lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 \ sglang serve ``` diff --git a/docs_new/src/snippets/_playground.jsx b/docs_new/src/snippets/_playground.jsx index 0cfb1a578..be8a0a28a 100644 --- a/docs_new/src/snippets/_playground.jsx +++ b/docs_new/src/snippets/_playground.jsx @@ -1015,8 +1015,16 @@ export const Playground = ({ config }) => { ]); if (value.enable) { const isAmd = sel && /^mi\d/.test(sel.hw); - const ratio = (isAmd && fc.amdIo && fc.amdIo.ratio) || 2; - const useAmdIo = isAmd && fc.amdIo; + const pdMode = h.findFlagArg(flags, "--disaggregation-mode") || "off"; + const pdBackend = h.findFlagArg(flags, "--disaggregation-transfer-backend"); + const roleOverride = (fc.roleOverrides || []).find((item) => { + if (!item || item.mode !== pdMode) return false; + if (item.transferBackend && item.transferBackend !== pdBackend) return false; + return !item.when || h.matchConstraint(sel, item.when); + }); + const amdIo = roleOverride || (isAmd && fc.amdIo); + const ratio = (amdIo && amdIo.ratio) || 2; + const useAmdIo = isAmd && amdIo; const adds = [ "--enable-hierarchical-cache", `--hicache-ratio ${ratio}`, @@ -1028,19 +1036,21 @@ export const Playground = ({ config }) => { // for AMD ROCm overrides (e.g. page_first_direct + direct on MI355X). // Default (NVIDIA): page_first_direct + direct, ratio 2. if (useAmdIo) { - adds.push(`--hicache-mem-layout ${fc.amdIo.memLayout}`, - `--hicache-io-backend ${fc.amdIo.ioBackend}`); + adds.push(`--hicache-mem-layout ${amdIo.memLayout}`, + `--hicache-io-backend ${amdIo.ioBackend}`); } else if (value.backend) { adds.push("--hicache-mem-layout page_first_direct", "--hicache-io-backend direct"); } const writePolicy = (value.writePolicy && value.writePolicy !== "auto") - ? value.writePolicy : "write_through"; + ? value.writePolicy : ((amdIo && amdIo.writePolicy) || "write_through"); adds.push(`--hicache-write-policy ${writePolicy}`); // When amdStorageFileOnly is set, AMD emits storage flags only for "file". if ((isAmd && fc.amdStorageFileOnly) ? value.backend === "file" : !!value.backend) { adds.push(`--hicache-storage-backend ${value.backend}`, - "--hicache-storage-prefetch-policy wait_complete"); + `--hicache-storage-prefetch-policy ${(amdIo && amdIo.prefetchPolicy) || "wait_complete"}`); + } else if (amdIo && amdIo.prefetchPolicy) { + adds.push(`--hicache-storage-prefetch-policy ${amdIo.prefetchPolicy}`); } flags = h.insertBeforeTail(flags, adds); } @@ -1283,8 +1293,22 @@ export const Playground = ({ config }) => { ? `# then front BOTH with the Router (SGLang Model Gateway) shown below.\n` + `# Client traffic (cURL) targets the router (:${routerPort}), not this role server.` : `# then front BOTH with a router; client traffic targets the router, not this role server.`; + const hicacheCfg = config.playgroundFeatures + && config.playgroundFeatures.hicache; + const pdBackend = findFlagArg(f, "--disaggregation-transfer-backend"); + const hicacheEnabled = f.some((x) => x === "--enable-hierarchical-cache"); + const hicacheNotice = hicacheEnabled && hicacheCfg + ? (hicacheCfg.notices || []).find((item) => { + if (!item || item.mode !== pdMode) return false; + if (item.transferBackend && item.transferBackend !== pdBackend) return false; + return !item.when || matchConstraint(sel, item.when); + }) + : null; + const noticeLine = hicacheNotice && hicacheNotice.text + ? `# Note: ${hicacheNotice.text}\n` : ""; const banner = `# === PD Disaggregation: ${pdMode.toUpperCase()} role ===\n` + + noticeLine + `# Runs the ${pdMode} server. Also run the ${sibling} role on its peer host,\n` + routerLine; cmd = `${banner}\n${cmd}`; diff --git a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index bf790807e..1f56cdfb0 100644 --- a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -161,7 +161,7 @@ sgl-eval run aime25 \\ // AMD daily-updated lmsysorg/sglang-rocm images. Bump the dated tag when you // re-verify on a newer build. mi300x: "lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi30x-20260623", - mi355x: "lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708", + mi355x: "lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710", }, // Pre-selects the issue template's `model` dropdown on "Submit verified cell". @@ -325,6 +325,32 @@ sgl-eval run aime25 \\ excludesHw: ["rtx6000"], // AMD ROCm (MI300X/MI325X/MI350X/MI355X): page_first_direct + direct io. amdIo: { memLayout: "page_first_direct", ioBackend: "direct", ratio: 4 }, + roleOverrides: [ + { + when: { + hw: ["mi355x"], variant: ["pro"], quant: ["fp4"], + strategy: ["low-latency"], nodes: ["single"], + }, + mode: "prefill", + transferBackend: "mori", + memLayout: "page_first", + ioBackend: "direct", + ratio: 5, + writePolicy: "write_through", + prefetchPolicy: "best_effort", + }, + ], + notices: [ + { + when: { + hw: ["mi355x"], variant: ["pro"], quant: ["fp4"], + strategy: ["low-latency"], nodes: ["single"], + }, + mode: "decode", + transferBackend: "mori", + text: "HiCache is not recommended on the decode role with MORI.", + }, + ], amdStorageFileOnly: true, backends: [ { id: null, label: "Auto" },