[Docs] Add fp8 kv cache for tokenspeed mla docs (#28201)
This commit is contained in:
@@ -196,13 +196,15 @@ export const KimiK25Deployment = () => {
|
||||
cmd += ' \\\n --speculative-algorithm EAGLE3 \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4 \\\n --speculative-draft-model-path lightseekorg/kimi-k2.5-eagle3-mla';
|
||||
}
|
||||
|
||||
const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300';
|
||||
|
||||
// Blackwell (B300/GB300): tokenspeed MLA attention backend
|
||||
if (hardware === 'b300' || hardware === 'gb300') {
|
||||
if (usesTokenspeedMla) {
|
||||
cmd += ' \\\n --attention-backend tokenspeed_mla';
|
||||
}
|
||||
|
||||
// AMD: FP8 KV cache for memory efficiency
|
||||
if (isAMD) {
|
||||
// FP8 KV cache for AMD memory efficiency and tokenspeed MLA compatibility
|
||||
if (isAMD || usesTokenspeedMla) {
|
||||
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
|
||||
}
|
||||
|
||||
|
||||
@@ -194,11 +194,13 @@ export const KimiK26Deployment = () => {
|
||||
cmd += ' \\\n --speculative-draft-model-path lightseekorg/kimi-k2.6-eagle3.1-mla';
|
||||
}
|
||||
|
||||
if (hardware === 'b300' || hardware === 'gb300') {
|
||||
const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300';
|
||||
|
||||
if (usesTokenspeedMla) {
|
||||
cmd += ' \\\n --attention-backend tokenspeed_mla';
|
||||
}
|
||||
|
||||
if (isAMD) {
|
||||
if (isAMD || usesTokenspeedMla) {
|
||||
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
|
||||
}
|
||||
|
||||
|
||||
@@ -144,11 +144,13 @@ export const KimiK27CodeDeployment = () => {
|
||||
cmd += ' \\\n --tool-call-parser kimi_k2';
|
||||
}
|
||||
|
||||
if (hardware === 'b300' || hardware === 'gb300') {
|
||||
const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300';
|
||||
|
||||
if (usesTokenspeedMla) {
|
||||
cmd += ' \\\n --attention-backend tokenspeed_mla';
|
||||
}
|
||||
|
||||
if (isAMD) {
|
||||
if (isAMD || usesTokenspeedMla) {
|
||||
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user