[Docs] Add fp8 kv cache for tokenspeed mla docs (#28201)

This commit is contained in:
Mohammad Miadh Angkad
2026-06-17 21:42:00 -07:00
committed by GitHub
parent cf93712937
commit 8d4a22c5af
3 changed files with 13 additions and 7 deletions
@@ -196,13 +196,15 @@ export const KimiK25Deployment = () => {
cmd += ' \\\n --speculative-algorithm EAGLE3 \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4 \\\n --speculative-draft-model-path lightseekorg/kimi-k2.5-eagle3-mla';
}
const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300';
// Blackwell (B300/GB300): tokenspeed MLA attention backend
if (hardware === 'b300' || hardware === 'gb300') {
if (usesTokenspeedMla) {
cmd += ' \\\n --attention-backend tokenspeed_mla';
}
// AMD: FP8 KV cache for memory efficiency
if (isAMD) {
// FP8 KV cache for AMD memory efficiency and tokenspeed MLA compatibility
if (isAMD || usesTokenspeedMla) {
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
}
@@ -194,11 +194,13 @@ export const KimiK26Deployment = () => {
cmd += ' \\\n --speculative-draft-model-path lightseekorg/kimi-k2.6-eagle3.1-mla';
}
if (hardware === 'b300' || hardware === 'gb300') {
const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300';
if (usesTokenspeedMla) {
cmd += ' \\\n --attention-backend tokenspeed_mla';
}
if (isAMD) {
if (isAMD || usesTokenspeedMla) {
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
}
@@ -144,11 +144,13 @@ export const KimiK27CodeDeployment = () => {
cmd += ' \\\n --tool-call-parser kimi_k2';
}
if (hardware === 'b300' || hardware === 'gb300') {
const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300';
if (usesTokenspeedMla) {
cmd += ' \\\n --attention-backend tokenspeed_mla';
}
if (isAMD) {
if (isAMD || usesTokenspeedMla) {
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
}