MiniMax-M3: allocate the lightning-indexer K cache in fp8 on gfx95 (#36549)
Co-authored-by: Kevin Mi <mikevin920@yahoo.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Kevin Mi <mikevin920@gmail.com> Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com> Co-authored-by: Kevin Mi <kevin.mi@radixark.ai> Co-authored-by: YC Yen-Ching Tseng <yctseng@amd.com>
This commit is contained in:
co-authored by
Kevin Mi
Claude Fable 5
Kevin Mi
Kevin Mi
Kevin Mi
YC Yen-Ching Tseng
parent
ffac53d779
commit
ec75d3d30f
@@ -556,7 +556,12 @@ def _sparse_qk_index_gemma_rmsnorm_rope_cache_kernel(
|
||||
idx_cache_base = (
|
||||
idx_k_cache_ptr + loc * idx_k_cache_stride_s + cols * idx_k_cache_stride_d
|
||||
)
|
||||
tl.store(idx_cache_base, out_typed, mask=mask & is_idx_k)
|
||||
# cast fp32 straight to the cache dtype (fp8 on gfx95) to avoid rounding twice via bf16
|
||||
tl.store(
|
||||
idx_cache_base,
|
||||
out.to(idx_k_cache_ptr.dtype.element_ty),
|
||||
mask=mask & is_idx_k,
|
||||
)
|
||||
|
||||
|
||||
def sparse_qk_index_gemma_rmsnorm_rope_cache(
|
||||
|
||||
@@ -1698,6 +1698,9 @@ class Envs:
|
||||
# 2 is the accuracy-safe default: higher values reuse staler selections
|
||||
# in the skip layers.
|
||||
SGLANG_MINIMAX_M3_INDEX_TOPK_FREQ = EnvInt(2)
|
||||
# gfx95: lightning-indexer K cache in fp8_e4m3fn (bf16 q x fp8 k in the scorers);
|
||||
# main attention K/V keep kv_cache_dtype.
|
||||
SGLANG_OPT_MINIMAX_M3_FP8_INDEX_CACHE = EnvBool(True)
|
||||
# Run the sparse prefill main attention through AITER's Gluon paged attention
|
||||
# instead of the Triton kernel. Unsupported cases fall back to Triton.
|
||||
SGLANG_OPT_USE_MINIMAX_GLUON_PREFILL = EnvBool(True)
|
||||
|
||||
@@ -74,6 +74,7 @@ from sglang.srt.mem_cache.memory_pool import (
|
||||
NoOpMHATokenToKVPool,
|
||||
PageMajorMHATokenToKVPool,
|
||||
ReqToTokenPool,
|
||||
get_minimax_sparse_index_dtype,
|
||||
)
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
from sglang.srt.platforms import current_platform
|
||||
@@ -1855,14 +1856,12 @@ class KVCacheConfigurator:
|
||||
size=max_total_num_tokens,
|
||||
page_size=self.pool_page_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
# fp8 attn-GEMM mode opts the lightning-indexer cache into
|
||||
# fp8 too (fp8 indexer GEMMs); fp8 KV without the mode
|
||||
# (e5m2 or non-trtllm_mha backend) keeps the indexer bf16
|
||||
# with the widening-dequant contract.
|
||||
index_dtype=(
|
||||
self.kv_cache_dtype
|
||||
if m3_fp8_attn_gemm_enabled(resolving_view(self.server_args))
|
||||
else self.model_dtype
|
||||
index_dtype=get_minimax_sparse_index_dtype(
|
||||
fp8_attn_gemm=m3_fp8_attn_gemm_enabled(
|
||||
resolving_view(self.server_args)
|
||||
),
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
model_dtype=self.model_dtype,
|
||||
),
|
||||
head_num=self.model_config.get_num_kv_heads(
|
||||
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
|
||||
|
||||
@@ -79,6 +79,7 @@ from sglang.srt.utils import (
|
||||
is_cpu,
|
||||
is_cuda,
|
||||
is_float4_e2m1fn_x2,
|
||||
is_gfx95_supported,
|
||||
is_hip,
|
||||
is_npu,
|
||||
is_xpu,
|
||||
@@ -110,6 +111,7 @@ _is_cpu = is_cpu()
|
||||
_is_xpu = is_xpu()
|
||||
_cpu_has_amx_support = cpu_has_amx_support()
|
||||
_is_hip = is_hip()
|
||||
_is_gfx95_supported = is_gfx95_supported()
|
||||
_is_fp8_fnuz = is_fp8_fnuz()
|
||||
# `SGLANG_AITER_KV_CACHE_LAYOUT` is only meaningful on the ROCm AITER backend
|
||||
# (HIP + --enable-aiter / SGLANG_USE_AITER=1). On any other platform / backend
|
||||
@@ -5429,6 +5431,18 @@ class MHATokenToKOnlyPool(KVCache):
|
||||
return k_size_bytes, 0
|
||||
|
||||
|
||||
def get_minimax_sparse_index_dtype(
|
||||
*, fp8_attn_gemm: bool, kv_cache_dtype: torch.dtype, model_dtype: torch.dtype
|
||||
) -> torch.dtype:
|
||||
"""Return the index-K cache dtype; the pool's cell-size estimate must match it."""
|
||||
# fp8 attn-GEMM mode runs the indexer GEMMs in fp8 too; plain fp8 KV keeps bf16.
|
||||
if fp8_attn_gemm:
|
||||
return kv_cache_dtype
|
||||
if _is_gfx95_supported and envs.SGLANG_OPT_MINIMAX_M3_FP8_INDEX_CACHE.get():
|
||||
return torch.float8_e4m3fn
|
||||
return model_dtype
|
||||
|
||||
|
||||
class MiniMaxSparseKVPool(KVCache):
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
from sglang.srt.configs.model_config import (
|
||||
AttentionArch,
|
||||
@@ -40,7 +41,10 @@ from sglang.srt.mem_cache.deepseek_v4_memory_pool import (
|
||||
get_dsv4_indexer_bytes_per_token,
|
||||
get_swa_ring_size,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
DSATokenToKVPool,
|
||||
get_minimax_sparse_index_dtype,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_disagg,
|
||||
get_exec,
|
||||
@@ -359,6 +363,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
num_layers=num_layers,
|
||||
)
|
||||
elif is_minimax_sparse(model_config.hf_config):
|
||||
from sglang.srt.server_args import m3_fp8_attn_gemm_enabled
|
||||
|
||||
# Mirrors MiniMaxSparseKVPool: main pool (K+V all layers) + indexer pool
|
||||
# (sparse-only, single-head; kv layers store K+V, k-only layers store K).
|
||||
sparse_cfg = get_minimax_sparse_attention_config(model_config.hf_config)
|
||||
@@ -387,7 +393,15 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
kv_heads = model_config.get_num_kv_heads(get_parallel().attn_tp_size)
|
||||
head_dim = model_config.head_dim
|
||||
indexer_head_dim = sparse_cfg["sparse_index_dim"]
|
||||
indexer_dtype_size = torch._utils._element_size(kvc.model_dtype)
|
||||
indexer_dtype_size = torch._utils._element_size(
|
||||
get_minimax_sparse_index_dtype(
|
||||
fp8_attn_gemm=m3_fp8_attn_gemm_enabled(
|
||||
resolving_view(kvc.server_args)
|
||||
),
|
||||
kv_cache_dtype=kvc.kv_cache_dtype,
|
||||
model_dtype=kvc.model_dtype,
|
||||
)
|
||||
)
|
||||
|
||||
full_pool_ratio = 1
|
||||
if get_memory().enable_hisparse:
|
||||
|
||||
Reference in New Issue
Block a user