[DSA] Support TileLang FP8 KV cache on CUDA (#42957)

Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
Jzz1943
2026-10-11 09:12:18 +08:00
committed by GitHub
co-authored by Xiaoyu Zhang
parent 33d2ef1756
commit 9336681310
6 changed files with 92 additions and 27 deletions
@@ -1371,12 +1371,15 @@ def tilelang_sparse_fwd(
topk = indices.shape[-1]
assert topk % 64 == 0, "topk must be padded to a multiple of 64"
if _is_hip:
is_fp8_kv = kv.dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz)
is_fp8_kv = kv.dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz)
if _is_hip or is_fp8_kv:
if is_fp8_kv:
if q.dtype != kv.dtype:
q = q.to(kv.dtype)
if _is_gfx95_supported:
if not _is_hip:
block_I, threads, block_per_cu = 32, 128, 1
cu = torch.cuda.get_device_properties(q.device).multi_processor_count
elif _is_gfx95_supported:
block_I, threads, block_per_cu, cu = 64, 256, 2, 256
else:
block_I, threads, block_per_cu, cu = 64, 256, 1, 304
+32 -10
View File
@@ -529,6 +529,7 @@ def _check_dsa_backend_constraints(
decode_backend: Optional[str],
*,
hip: bool,
dcp_size: int = 1,
) -> None:
"""Validate DSA backend / platform / kv-cache-dtype constraints."""
chosen = {prefill_backend, decode_backend}
@@ -541,15 +542,21 @@ def _check_dsa_backend_constraints(
"(flashmla_kv on Hopper, trtllm on Blackwell)."
)
cuda_fp8_unsupported = {"tilelang"} & chosen
if not hip and kv_cache_dtype == "fp8_e4m3" and cuda_fp8_unsupported:
if hip or kv_cache_dtype != "fp8_e4m3" or "tilelang" not in chosen:
return
if prefill_backend != "tilelang" or decode_backend != "tilelang":
raise ValueError(
f"The {'/'.join(sorted(cuda_fp8_unsupported))} DSA prefill/decode kernels "
"only support an fp8_e4m3 KV cache on ROCm/HIP; on CUDA they require "
"a bfloat16 KV cache. Use --kv-cache-dtype bfloat16, or keep "
"--kv-cache-dtype fp8_e4m3 and pick an fp8-capable DSA backend "
"(flashmla_kv on Hopper, trtllm on Blackwell)."
"CUDA TileLang FP8 KV requires both DSA backends to be tilelang; "
"raw and scaled FP8 KV layouts cannot be mixed."
)
if dcp_size > 1:
raise ValueError("CUDA TileLang FP8 KV does not support --dcp-size > 1.")
import torch
major, minor = torch.cuda.get_device_capability()
if (major, minor) < (8, 9):
raise ValueError("CUDA TileLang FP8 KV requires SM89 or newer.")
def _check_tilelang_dsa_fp8_kv(
@@ -558,10 +565,11 @@ def _check_tilelang_dsa_fp8_kv(
decode_backend: Optional[str],
*,
hip: bool,
dcp_size: int = 1,
) -> None:
"""Backward-compatible entry point for the TileLang DSA validation."""
_check_dsa_backend_constraints(
kv_cache_dtype, prefill_backend, decode_backend, hip=hip
kv_cache_dtype, prefill_backend, decode_backend, hip=hip, dcp_size=dcp_size
)
@@ -652,11 +660,21 @@ def _dsa_split_backend_resolution(view: Any) -> dict:
declared["dsa_decode_backend"] = backend
prefill = declared.get("dsa_prefill_backend", view.dsa_prefill_backend)
decode = declared.get("dsa_decode_backend", view.dsa_decode_backend)
if (
not get_platform().is_hip
and kv_cache_dtype == "fp8_e4m3"
and "tilelang" in (prefill, decode)
):
raise ValueError("CUDA TileLang FP8 KV does not support HiSparse.")
# The hisparse allow-list in hisparse_hook is platform- but not
# dtype-aware, so an explicitly requested backend still has to clear the
# shared backend/kv-cache-dtype rules before this arm returns early.
_check_dsa_backend_constraints(
kv_cache_dtype, prefill, decode, hip=get_platform().is_hip
kv_cache_dtype,
prefill,
decode,
hip=get_platform().is_hip,
dcp_size=getattr(view, "dcp_size", 1) or 1,
)
logger.warning(
f"HiSparse enabled ({kv_cache_dtype}): using DSA backends "
@@ -686,7 +704,11 @@ def _dsa_split_backend_resolution(view: Any) -> dict:
prefill = declared.get("dsa_prefill_backend", view.dsa_prefill_backend)
decode = declared.get("dsa_decode_backend", view.dsa_decode_backend)
_check_dsa_backend_constraints(
kv_cache_dtype, prefill, decode, hip=get_platform().is_hip
kv_cache_dtype,
prefill,
decode,
hip=get_platform().is_hip,
dcp_size=getattr(view, "dcp_size", 1) or 1,
)
logger.warning(
f"Set DSA backends for {kv_cache_dtype} KV Cache: "
@@ -473,6 +473,15 @@ class DeepseekSparseAttnBackend(
self.supports_mha_one_shot: bool = True
self.dsa_prefill_impl: _DSA_IMPL_T = get_exec().kernel.dsa_prefill_backend
self.dsa_decode_impl: _DSA_IMPL_T = get_exec().kernel.dsa_decode_backend
if (
not _is_hip
and self.token_to_kv_pool.dtype == torch.float8_e4m3fn
and not self.dsa_kv_cache_store_fp8
and "tilelang" in (self.dsa_prefill_impl, self.dsa_decode_impl)
):
# The one-shot FP8 dequantizers require scaled KV rows; TileLang
# consumes raw FP8 KV without per-block scales.
self.supports_mha_one_shot = False
# Opt-in exact fast path for the Triton sparse-MLA prefill kernel.
self.dsa_triton_union: int = _resolve_dsa_triton_union(
union=get_exec().kernel.dsa_triton_union,
@@ -3100,6 +3100,14 @@ def calculate_mla_kv_cache_dim(
):
return kv_cache_dim
# CUDA TileLang uses raw FP8 rows without per-block scales. Argument
# validation requires both consumers to use the same TileLang layout.
if (
get_exec().kernel.dsa_prefill_backend == "tilelang"
and get_exec().kernel.dsa_decode_backend == "tilelang"
):
return kv_cache_dim
quant_block_size = DSATokenToKVPool.quant_block_size
rope_storage_dtype = DSATokenToKVPool.rope_storage_dtype
# Calculate override_kv_cache_dim for FP8 storage in backends that use scaled KV layout