diff --git a/python/sglang/kernels/ops/attention/minimax_m3_qk_norm_rope.py b/python/sglang/kernels/ops/attention/minimax_m3_qk_norm_rope.py index 9136521a90..1a1807a645 100644 --- a/python/sglang/kernels/ops/attention/minimax_m3_qk_norm_rope.py +++ b/python/sglang/kernels/ops/attention/minimax_m3_qk_norm_rope.py @@ -556,7 +556,12 @@ def _sparse_qk_index_gemma_rmsnorm_rope_cache_kernel( idx_cache_base = ( idx_k_cache_ptr + loc * idx_k_cache_stride_s + cols * idx_k_cache_stride_d ) - tl.store(idx_cache_base, out_typed, mask=mask & is_idx_k) + # cast fp32 straight to the cache dtype (fp8 on gfx95) to avoid rounding twice via bf16 + tl.store( + idx_cache_base, + out.to(idx_k_cache_ptr.dtype.element_ty), + mask=mask & is_idx_k, + ) def sparse_qk_index_gemma_rmsnorm_rope_cache( diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 1012624d16..4e502e48b9 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1698,6 +1698,9 @@ class Envs: # 2 is the accuracy-safe default: higher values reuse staler selections # in the skip layers. SGLANG_MINIMAX_M3_INDEX_TOPK_FREQ = EnvInt(2) + # gfx95: lightning-indexer K cache in fp8_e4m3fn (bf16 q x fp8 k in the scorers); + # main attention K/V keep kv_cache_dtype. + SGLANG_OPT_MINIMAX_M3_FP8_INDEX_CACHE = EnvBool(True) # Run the sparse prefill main attention through AITER's Gluon paged attention # instead of the Triton kernel. Unsupported cases fall back to Triton. SGLANG_OPT_USE_MINIMAX_GLUON_PREFILL = EnvBool(True) diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 2157a39fcc..fed1680a2b 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -74,6 +74,7 @@ from sglang.srt.mem_cache.memory_pool import ( NoOpMHATokenToKVPool, PageMajorMHATokenToKVPool, ReqToTokenPool, + get_minimax_sparse_index_dtype, ) from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.platforms import current_platform @@ -1855,14 +1856,12 @@ class KVCacheConfigurator: size=max_total_num_tokens, page_size=self.pool_page_size, dtype=self.kv_cache_dtype, - # fp8 attn-GEMM mode opts the lightning-indexer cache into - # fp8 too (fp8 indexer GEMMs); fp8 KV without the mode - # (e5m2 or non-trtllm_mha backend) keeps the indexer bf16 - # with the widening-dequant contract. - index_dtype=( - self.kv_cache_dtype - if m3_fp8_attn_gemm_enabled(resolving_view(self.server_args)) - else self.model_dtype + index_dtype=get_minimax_sparse_index_dtype( + fp8_attn_gemm=m3_fp8_attn_gemm_enabled( + resolving_view(self.server_args) + ), + kv_cache_dtype=self.kv_cache_dtype, + model_dtype=self.model_dtype, ), head_num=self.model_config.get_num_kv_heads( get_parallel().attn_tp_size, get_parallel().attn_dcp_size diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 1f1c762bac..3e0e2e5798 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -79,6 +79,7 @@ from sglang.srt.utils import ( is_cpu, is_cuda, is_float4_e2m1fn_x2, + is_gfx95_supported, is_hip, is_npu, is_xpu, @@ -110,6 +111,7 @@ _is_cpu = is_cpu() _is_xpu = is_xpu() _cpu_has_amx_support = cpu_has_amx_support() _is_hip = is_hip() +_is_gfx95_supported = is_gfx95_supported() _is_fp8_fnuz = is_fp8_fnuz() # `SGLANG_AITER_KV_CACHE_LAYOUT` is only meaningful on the ROCm AITER backend # (HIP + --enable-aiter / SGLANG_USE_AITER=1). On any other platform / backend @@ -5429,6 +5431,18 @@ class MHATokenToKOnlyPool(KVCache): return k_size_bytes, 0 +def get_minimax_sparse_index_dtype( + *, fp8_attn_gemm: bool, kv_cache_dtype: torch.dtype, model_dtype: torch.dtype +) -> torch.dtype: + """Return the index-K cache dtype; the pool's cell-size estimate must match it.""" + # fp8 attn-GEMM mode runs the indexer GEMMs in fp8 too; plain fp8 KV keeps bf16. + if fp8_attn_gemm: + return kv_cache_dtype + if _is_gfx95_supported and envs.SGLANG_OPT_MINIMAX_M3_FP8_INDEX_CACHE.get(): + return torch.float8_e4m3fn + return model_dtype + + class MiniMaxSparseKVPool(KVCache): def __init__( self, diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index c1bd0f9dda..bbb41d5635 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Optional import torch +from sglang.srt.arg_groups.overrides import resolving_view from sglang.srt.configs.hybrid_arch import mambaish_config from sglang.srt.configs.model_config import ( AttentionArch, @@ -40,7 +41,10 @@ from sglang.srt.mem_cache.deepseek_v4_memory_pool import ( get_dsv4_indexer_bytes_per_token, get_swa_ring_size, ) -from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool +from sglang.srt.mem_cache.memory_pool import ( + DSATokenToKVPool, + get_minimax_sparse_index_dtype, +) from sglang.srt.runtime_context import ( get_disagg, get_exec, @@ -359,6 +363,8 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): num_layers=num_layers, ) elif is_minimax_sparse(model_config.hf_config): + from sglang.srt.server_args import m3_fp8_attn_gemm_enabled + # Mirrors MiniMaxSparseKVPool: main pool (K+V all layers) + indexer pool # (sparse-only, single-head; kv layers store K+V, k-only layers store K). sparse_cfg = get_minimax_sparse_attention_config(model_config.hf_config) @@ -387,7 +393,15 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): kv_heads = model_config.get_num_kv_heads(get_parallel().attn_tp_size) head_dim = model_config.head_dim indexer_head_dim = sparse_cfg["sparse_index_dim"] - indexer_dtype_size = torch._utils._element_size(kvc.model_dtype) + indexer_dtype_size = torch._utils._element_size( + get_minimax_sparse_index_dtype( + fp8_attn_gemm=m3_fp8_attn_gemm_enabled( + resolving_view(kvc.server_args) + ), + kv_cache_dtype=kvc.kv_cache_dtype, + model_dtype=kvc.model_dtype, + ) + ) full_pool_ratio = 1 if get_memory().enable_hisparse: