minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Docs] Add fp8 kv cache for tokenspeed mla docs (#28201)

Browse Source
This commit is contained in:
Mohammad Miadh Angkad
2026-06-17 21:42:00 -07:00
committed by GitHub
parent cf93712937
commit 8d4a22c5af
3 changed files with 13 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs_new/src/snippets/autoregressive/kimi-k25-deployment.jsx
+5 -3
View File
@@ -196,13 +196,15 @@ export const KimiK25Deployment = () => {
cmd += ' \\\n --speculative-algorithm EAGLE3 \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4 \\\n --speculative-draft-model-path lightseekorg/kimi-k2.5-eagle3-mla'; cmd += ' \\\n --speculative-algorithm EAGLE3 \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4 \\\n --speculative-draft-model-path lightseekorg/kimi-k2.5-eagle3-mla';
} }
const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300';
// Blackwell (B300/GB300): tokenspeed MLA attention backend // Blackwell (B300/GB300): tokenspeed MLA attention backend
if (hardware === 'b300' || hardware === 'gb300') { if (usesTokenspeedMla) {
cmd += ' \\\n --attention-backend tokenspeed_mla'; cmd += ' \\\n --attention-backend tokenspeed_mla';
} }
// AMD: FP8 KV cache for memory efficiency // FP8 KV cache for AMD memory efficiency and tokenspeed MLA compatibility
if (isAMD) { if (isAMD || usesTokenspeedMla) {
cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
} }
docs_new/src/snippets/autoregressive/kimi-k26-deployment.jsx
+4 -2
View File
@@ -194,11 +194,13 @@ export const KimiK26Deployment = () => {
cmd += ' \\\n --speculative-draft-model-path lightseekorg/kimi-k2.6-eagle3.1-mla'; cmd += ' \\\n --speculative-draft-model-path lightseekorg/kimi-k2.6-eagle3.1-mla';
} }
if (hardware === 'b300' || hardware === 'gb300') { const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300';
if (usesTokenspeedMla) {
cmd += ' \\\n --attention-backend tokenspeed_mla'; cmd += ' \\\n --attention-backend tokenspeed_mla';
} }
if (isAMD) { if (isAMD || usesTokenspeedMla) {
cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
} }
docs_new/src/snippets/autoregressive/kimi-k27-code-deployment.jsx
+4 -2
View File
@@ -144,11 +144,13 @@ export const KimiK27CodeDeployment = () => {
cmd += ' \\\n --tool-call-parser kimi_k2'; cmd += ' \\\n --tool-call-parser kimi_k2';
} }
if (hardware === 'b300' || hardware === 'gb300') { const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300';
if (usesTokenspeedMla) {
cmd += ' \\\n --attention-backend tokenspeed_mla'; cmd += ' \\\n --attention-backend tokenspeed_mla';
} }
if (isAMD) { if (isAMD || usesTokenspeedMla) {
cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
} }