diff --git a/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx b/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx index 19fd93924..99ca67f4f 100644 --- a/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx +++ b/docs_new/cookbook/autoregressive/Moonshotai/Kimi-K2.6.mdx @@ -86,7 +86,7 @@ import { KimiK26Deployment } from '/src/snippets/autoregressive/kimi-k26-deploym ### 3.2 Configuration Tips -- **Memory**: Requires GPUs with ≥140GB each. Supported platforms: H200 (8×, TP=8), B300 (8×, TP=8), MI300X/MI325X (4×, TP=4), MI350X/MI355X (4×, TP=4). Use `--context-length 128000` to conserve memory. +- **Memory**: Requires GPUs with ≥140GB each. Supported platforms: H200 (8×, TP=8), B200 (8×, TP=8), B300 (8×, TP=8), GB200 (4×, TP=4), GB300 (4×, TP=4), MI300X/MI325X (4×, TP=4), MI350X/MI355X (4×, TP=4). Use `--context-length 128000` to conserve memory. - **AMD GPU TP Constraint**: On AMD GPUs, TP must be ≤ 4 (not 8). Kimi-K2.6 has 64 attention heads; the AITER MLA kernel requires `heads_per_gpu % 16 == 0`. With TP=4, each GPU gets 16 heads (valid). With TP=8, each GPU gets 8 heads (invalid). - **AMD Docker Image**: Use `lmsysorg/sglang:v0.5.9-rocm700-mi35x` for MI350X/MI355X and `lmsysorg/sglang:v0.5.9-rocm700-mi30x` for MI300X/MI325X. - **DP Attention**: Enable with `--dp --enable-dp-attention` for production throughput. A common choice is to set `--dp` equal to `--tp`, but this is not required. diff --git a/docs_new/src/snippets/autoregressive/kimi-k26-deployment.jsx b/docs_new/src/snippets/autoregressive/kimi-k26-deployment.jsx index be2ccb6b4..11a1b68d9 100644 --- a/docs_new/src/snippets/autoregressive/kimi-k26-deployment.jsx +++ b/docs_new/src/snippets/autoregressive/kimi-k26-deployment.jsx @@ -6,7 +6,10 @@ export const KimiK26Deployment = () => { title: 'Hardware Platform', items: [ { id: 'h200', label: 'H200', default: true }, + { id: 'b200', label: 'B200', default: false }, { id: 'b300', label: 'B300', default: false }, + { id: 'gb200', label: 'GB200', default: false }, + { id: 'gb300', label: 'GB300', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, { id: 'mi350x', label: 'MI350X', default: false }, @@ -41,7 +44,10 @@ export const KimiK26Deployment = () => { const modelConfigs = { h200: { tp: 8 }, + b200: { tp: 8 }, b300: { tp: 8 }, + gb200: { tp: 4 }, + gb300: { tp: 4 }, mi300x: { tp: 4 }, mi325x: { tp: 4 }, mi350x: { tp: 4 },