minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Docs] Tune DeepSeek-V4 HiCache for MI355X PD (#31452)

Browse Source 
Signed-off-by: Duyi-Wang <duyi.wang@amd.com>
This commit is contained in:
Duyi-Wang
2026-07-17 11:50:58 +08:00
committed by GitHub
parent 40a3bd7659
commit 27a52d2530
3 changed files with 60 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx
+3 -3
View File
@@ -52,11 +52,11 @@ docker run --gpus all \
AMD uses the daily-updated `lmsysorg/sglang-rocm` images. You can find the latest images on [Docker Hub](https://hub.docker.com/r/lmsysorg/sglang-rocm/tags). We recommend the ROCm 7.2 version.
For example:
- **MI355X** → `lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623`
- **MI355X** → `lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710`
- **MI300X** → `lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi30x-20260623`
```bash Command
docker pull lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623
docker pull lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710
docker run \
--device=/dev/kfd --device=/dev/dri \
@@ -66,7 +66,7 @@ docker run \
-p 30000:30000 \
-v ~/.cache/huggingface:/root/.cache/huggingface \
--env "HF_TOKEN=<your-hf-token>" \
lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260623 \
lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710 \
sglang serve <use args below>
```
docs_new/src/snippets/_playground.jsx
+30 -6
View File
@@ -1015,8 +1015,16 @@ export const Playground = ({ config }) => {
]);
if (value.enable) {
const isAmd = sel && /^mi\d/.test(sel.hw);
const ratio = (isAmd && fc.amdIo && fc.amdIo.ratio) || 2;
const useAmdIo = isAmd && fc.amdIo;
const pdMode = h.findFlagArg(flags, "--disaggregation-mode") || "off";
const pdBackend = h.findFlagArg(flags, "--disaggregation-transfer-backend");
const roleOverride = (fc.roleOverrides || []).find((item) => {
if (!item || item.mode !== pdMode) return false;
if (item.transferBackend && item.transferBackend !== pdBackend) return false;
return !item.when || h.matchConstraint(sel, item.when);
});
const amdIo = roleOverride || (isAmd && fc.amdIo);
const ratio = (amdIo && amdIo.ratio) || 2;
const useAmdIo = isAmd && amdIo;
const adds = [
"--enable-hierarchical-cache",
`--hicache-ratio ${ratio}`,
@@ -1028,19 +1036,21 @@ export const Playground = ({ config }) => {
// for AMD ROCm overrides (e.g. page_first_direct + direct on MI355X).
// Default (NVIDIA): page_first_direct + direct, ratio 2.
if (useAmdIo) {
adds.push(`--hicache-mem-layout ${fc.amdIo.memLayout}`,
`--hicache-io-backend ${fc.amdIo.ioBackend}`);
adds.push(`--hicache-mem-layout ${amdIo.memLayout}`,
`--hicache-io-backend ${amdIo.ioBackend}`);
} else if (value.backend) {
adds.push("--hicache-mem-layout page_first_direct",
"--hicache-io-backend direct");
}
const writePolicy = (value.writePolicy && value.writePolicy !== "auto")
? value.writePolicy : "write_through";
? value.writePolicy : ((amdIo && amdIo.writePolicy) || "write_through");
adds.push(`--hicache-write-policy ${writePolicy}`);
// When amdStorageFileOnly is set, AMD emits storage flags only for "file".
if ((isAmd && fc.amdStorageFileOnly) ? value.backend === "file" : !!value.backend) {
adds.push(`--hicache-storage-backend ${value.backend}`,
"--hicache-storage-prefetch-policy wait_complete");
`--hicache-storage-prefetch-policy ${(amdIo && amdIo.prefetchPolicy) || "wait_complete"}`);
} else if (amdIo && amdIo.prefetchPolicy) {
adds.push(`--hicache-storage-prefetch-policy ${amdIo.prefetchPolicy}`);
}
flags = h.insertBeforeTail(flags, adds);
}
@@ -1283,8 +1293,22 @@ export const Playground = ({ config }) => {
? `# then front BOTH with the Router (SGLang Model Gateway) shown below.\n`
+ `# Client traffic (cURL) targets the router (:${routerPort}), not this role server.`
: `# then front BOTH with a router; client traffic targets the router, not this role server.`;
const hicacheCfg = config.playgroundFeatures
&& config.playgroundFeatures.hicache;
const pdBackend = findFlagArg(f, "--disaggregation-transfer-backend");
const hicacheEnabled = f.some((x) => x === "--enable-hierarchical-cache");
const hicacheNotice = hicacheEnabled && hicacheCfg
? (hicacheCfg.notices || []).find((item) => {
if (!item || item.mode !== pdMode) return false;
if (item.transferBackend && item.transferBackend !== pdBackend) return false;
return !item.when || matchConstraint(sel, item.when);
})
: null;
const noticeLine = hicacheNotice && hicacheNotice.text
? `# Note: ${hicacheNotice.text}\n` : "";
const banner =
`# === PD Disaggregation: ${pdMode.toUpperCase()} role ===\n` +
noticeLine +
`# Runs the ${pdMode} server. Also run the ${sibling} role on its peer host,\n` +
routerLine;
cmd = `${banner}\n${cmd}`;
docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx
+27 -1
View File
@@ -161,7 +161,7 @@ sgl-eval run aime25 \\
// AMD daily-updated lmsysorg/sglang-rocm images. Bump the dated tag when you
// re-verify on a newer build.
mi300x: "lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi30x-20260623",
mi355x: "lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260708",
mi355x: "lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260710",
},
// Pre-selects the issue template's `model` dropdown on "Submit verified cell".
@@ -325,6 +325,32 @@ sgl-eval run aime25 \\
excludesHw: ["rtx6000"],
// AMD ROCm (MI300X/MI325X/MI350X/MI355X): page_first_direct + direct io.
amdIo: { memLayout: "page_first_direct", ioBackend: "direct", ratio: 4 },
roleOverrides: [
{
when: {
hw: ["mi355x"], variant: ["pro"], quant: ["fp4"],
strategy: ["low-latency"], nodes: ["single"],
},
mode: "prefill",
transferBackend: "mori",
memLayout: "page_first",
ioBackend: "direct",
ratio: 5,
writePolicy: "write_through",
prefetchPolicy: "best_effort",
},
],
notices: [
{
when: {
hw: ["mi355x"], variant: ["pro"], quant: ["fp4"],
strategy: ["low-latency"], nodes: ["single"],
},
mode: "decode",
transferBackend: "mori",
text: "HiCache is not recommended on the decode role with MORI.",
},
],
amdStorageFileOnly: true,
backends: [
{ id: null, label: "Auto" },