MiniMax-M3: allocate the lightning-indexer K cache in fp8 on gfx95 (#36549)

Co-authored-by: Kevin Mi <mikevin920@yahoo.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Kevin Mi <mikevin920@gmail.com>
Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com>
Co-authored-by: Kevin Mi <kevin.mi@radixark.ai>
Co-authored-by: YC Yen-Ching Tseng <yctseng@amd.com>
This commit is contained in:
Chunan Zeng
2026-09-23 17:52:02 -07:00
committed by GitHub
co-authored by Kevin Mi Claude Fable 5 Kevin Mi Kevin Mi Kevin Mi YC Yen-Ching Tseng
parent ffac53d779
commit ec75d3d30f
5 changed files with 46 additions and 11 deletions
@@ -556,7 +556,12 @@ def _sparse_qk_index_gemma_rmsnorm_rope_cache_kernel(
idx_cache_base = (
idx_k_cache_ptr + loc * idx_k_cache_stride_s + cols * idx_k_cache_stride_d
)
tl.store(idx_cache_base, out_typed, mask=mask & is_idx_k)
# cast fp32 straight to the cache dtype (fp8 on gfx95) to avoid rounding twice via bf16
tl.store(
idx_cache_base,
out.to(idx_k_cache_ptr.dtype.element_ty),
mask=mask & is_idx_k,
)
def sparse_qk_index_gemma_rmsnorm_rope_cache(
+3
View File
@@ -1698,6 +1698,9 @@ class Envs:
# 2 is the accuracy-safe default: higher values reuse staler selections
# in the skip layers.
SGLANG_MINIMAX_M3_INDEX_TOPK_FREQ = EnvInt(2)
# gfx95: lightning-indexer K cache in fp8_e4m3fn (bf16 q x fp8 k in the scorers);
# main attention K/V keep kv_cache_dtype.
SGLANG_OPT_MINIMAX_M3_FP8_INDEX_CACHE = EnvBool(True)
# Run the sparse prefill main attention through AITER's Gluon paged attention
# instead of the Triton kernel. Unsupported cases fall back to Triton.
SGLANG_OPT_USE_MINIMAX_GLUON_PREFILL = EnvBool(True)
@@ -74,6 +74,7 @@ from sglang.srt.mem_cache.memory_pool import (
NoOpMHATokenToKVPool,
PageMajorMHATokenToKVPool,
ReqToTokenPool,
get_minimax_sparse_index_dtype,
)
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
from sglang.srt.platforms import current_platform
@@ -1855,14 +1856,12 @@ class KVCacheConfigurator:
size=max_total_num_tokens,
page_size=self.pool_page_size,
dtype=self.kv_cache_dtype,
# fp8 attn-GEMM mode opts the lightning-indexer cache into
# fp8 too (fp8 indexer GEMMs); fp8 KV without the mode
# (e5m2 or non-trtllm_mha backend) keeps the indexer bf16
# with the widening-dequant contract.
index_dtype=(
self.kv_cache_dtype
if m3_fp8_attn_gemm_enabled(resolving_view(self.server_args))
else self.model_dtype
index_dtype=get_minimax_sparse_index_dtype(
fp8_attn_gemm=m3_fp8_attn_gemm_enabled(
resolving_view(self.server_args)
),
kv_cache_dtype=self.kv_cache_dtype,
model_dtype=self.model_dtype,
),
head_num=self.model_config.get_num_kv_heads(
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
@@ -79,6 +79,7 @@ from sglang.srt.utils import (
is_cpu,
is_cuda,
is_float4_e2m1fn_x2,
is_gfx95_supported,
is_hip,
is_npu,
is_xpu,
@@ -110,6 +111,7 @@ _is_cpu = is_cpu()
_is_xpu = is_xpu()
_cpu_has_amx_support = cpu_has_amx_support()
_is_hip = is_hip()
_is_gfx95_supported = is_gfx95_supported()
_is_fp8_fnuz = is_fp8_fnuz()
# `SGLANG_AITER_KV_CACHE_LAYOUT` is only meaningful on the ROCm AITER backend
# (HIP + --enable-aiter / SGLANG_USE_AITER=1). On any other platform / backend
@@ -5429,6 +5431,18 @@ class MHATokenToKOnlyPool(KVCache):
return k_size_bytes, 0
def get_minimax_sparse_index_dtype(
*, fp8_attn_gemm: bool, kv_cache_dtype: torch.dtype, model_dtype: torch.dtype
) -> torch.dtype:
"""Return the index-K cache dtype; the pool's cell-size estimate must match it."""
# fp8 attn-GEMM mode runs the indexer GEMMs in fp8 too; plain fp8 KV keeps bf16.
if fp8_attn_gemm:
return kv_cache_dtype
if _is_gfx95_supported and envs.SGLANG_OPT_MINIMAX_M3_FP8_INDEX_CACHE.get():
return torch.float8_e4m3fn
return model_dtype
class MiniMaxSparseKVPool(KVCache):
def __init__(
self,
@@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Optional
import torch
from sglang.srt.arg_groups.overrides import resolving_view
from sglang.srt.configs.hybrid_arch import mambaish_config
from sglang.srt.configs.model_config import (
AttentionArch,
@@ -40,7 +41,10 @@ from sglang.srt.mem_cache.deepseek_v4_memory_pool import (
get_dsv4_indexer_bytes_per_token,
get_swa_ring_size,
)
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
from sglang.srt.mem_cache.memory_pool import (
DSATokenToKVPool,
get_minimax_sparse_index_dtype,
)
from sglang.srt.runtime_context import (
get_disagg,
get_exec,
@@ -359,6 +363,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
num_layers=num_layers,
)
elif is_minimax_sparse(model_config.hf_config):
from sglang.srt.server_args import m3_fp8_attn_gemm_enabled
# Mirrors MiniMaxSparseKVPool: main pool (K+V all layers) + indexer pool
# (sparse-only, single-head; kv layers store K+V, k-only layers store K).
sparse_cfg = get_minimax_sparse_attention_config(model_config.hf_config)
@@ -387,7 +393,15 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
kv_heads = model_config.get_num_kv_heads(get_parallel().attn_tp_size)
head_dim = model_config.head_dim
indexer_head_dim = sparse_cfg["sparse_index_dim"]
indexer_dtype_size = torch._utils._element_size(kvc.model_dtype)
indexer_dtype_size = torch._utils._element_size(
get_minimax_sparse_index_dtype(
fp8_attn_gemm=m3_fp8_attn_gemm_enabled(
resolving_view(kvc.server_args)
),
kv_cache_dtype=kvc.kv_cache_dtype,
model_dtype=kvc.model_dtype,
)
)
full_pool_ratio = 1
if get_memory().enable_hisparse: