diff --git a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx index d81060b59..d30e4d384 100644 --- a/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx +++ b/docs_new/cookbook/autoregressive/DeepSeek/DeepSeek-V4.mdx @@ -514,6 +514,10 @@ To enable HiCache, open the **HiCache** card in the [Playground above](#playgrou - **L2 (GPU + CPU)** — leave Storage on `auto` (default). Cold KV pages spill to CPU pinned memory only. - **L3 (GPU + CPU + Storage)** — pick a Storage backend (`file` / `mooncake` / `hf3fs` / `nixl`); the Playground emits the canonical `page_first_direct` mem-layout + `direct` IO backend + `wait_complete` prefetch policy, matching the [HiCache best-practices recipe](../../../docs/advanced_features/hicache_best_practices). +For AMD devices, +- **L2 (GPU + CPU)** — leave Storage on `auto` (default). Cold KV pages spill to CPU pinned memory only. Use `direct` IO backend + `page_first_direct` or `layer-first` mem-layout. +- **L3 (GPU + CPU + Storage)** — pick a Storage backend (`file`); the Playground emits the canonical `page_first_direct` mem-layout + `direct` IO backend + `wait_complete` prefetch policy, matching the [HiCache best-practices recipe](../../../docs/advanced_features/hicache_best_practices). + The Write policy knob defaults to `write_through` (the upstream default); switch to `write_back` / `write_through_selective` to trade durability for write speed when the storage tier is slow. For more details, see the [HiCache documentation](../../../docs/advanced_features/hicache). diff --git a/docs_new/src/snippets/_playground.jsx b/docs_new/src/snippets/_playground.jsx index 098b09d5b..0cfb1a578 100644 --- a/docs_new/src/snippets/_playground.jsx +++ b/docs_new/src/snippets/_playground.jsx @@ -1014,19 +1014,31 @@ export const Playground = ({ config }) => { "--hicache-storage-backend", "--hicache-storage-prefetch-policy", ]); if (value.enable) { + const isAmd = sel && /^mi\d/.test(sel.hw); + const ratio = (isAmd && fc.amdIo && fc.amdIo.ratio) || 2; + const useAmdIo = isAmd && fc.amdIo; const adds = [ "--enable-hierarchical-cache", - "--hicache-ratio 2", - "--hicache-size 0", + `--hicache-ratio ${ratio}`, ]; - if (value.backend) { + if (!useAmdIo) { + adds.push("--hicache-size 0"); + } + // Per-model configs can declare fc.amdIo = { memLayout, ioBackend, ratio } + // for AMD ROCm overrides (e.g. page_first_direct + direct on MI355X). + // Default (NVIDIA): page_first_direct + direct, ratio 2. + if (useAmdIo) { + adds.push(`--hicache-mem-layout ${fc.amdIo.memLayout}`, + `--hicache-io-backend ${fc.amdIo.ioBackend}`); + } else if (value.backend) { adds.push("--hicache-mem-layout page_first_direct", "--hicache-io-backend direct"); } const writePolicy = (value.writePolicy && value.writePolicy !== "auto") ? value.writePolicy : "write_through"; adds.push(`--hicache-write-policy ${writePolicy}`); - if (value.backend) { + // When amdStorageFileOnly is set, AMD emits storage flags only for "file". + if ((isAmd && fc.amdStorageFileOnly) ? value.backend === "file" : !!value.backend) { adds.push(`--hicache-storage-backend ${value.backend}`, "--hicache-storage-prefetch-policy wait_complete"); } diff --git a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index 0a44f154e..bf790807e 100644 --- a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -323,12 +323,18 @@ sgl-eval run aime25 \\ // ----- Card 6: "Hierarchical KV Cache" ----- hicache: { excludesHw: ["rtx6000"], + // AMD ROCm (MI300X/MI325X/MI350X/MI355X): page_first_direct + direct io. + amdIo: { memLayout: "page_first_direct", ioBackend: "direct", ratio: 4 }, + amdStorageFileOnly: true, backends: [ { id: null, label: "Auto" }, { id: "file", label: "File" }, - { id: "mooncake", label: "Mooncake" }, - { id: "hf3fs", label: "HF3FS" }, - { id: "nixl", label: "NiXL" }, + { id: "mooncake", label: "Mooncake", + hide: { hw: ["mi300x", "mi355x"] } }, + { id: "hf3fs", label: "HF3FS", + hide: { hw: ["mi300x", "mi355x"] } }, + { id: "nixl", label: "NiXL", + hide: { hw: ["mi300x", "mi355x"] } }, ], writePolicies: [ { id: "auto", label: "Auto" },