[DSA] Support TileLang FP8 KV cache on CUDA (#42957)
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
@@ -1371,12 +1371,15 @@ def tilelang_sparse_fwd(
|
||||
topk = indices.shape[-1]
|
||||
assert topk % 64 == 0, "topk must be padded to a multiple of 64"
|
||||
|
||||
if _is_hip:
|
||||
is_fp8_kv = kv.dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz)
|
||||
is_fp8_kv = kv.dtype in (torch.float8_e4m3fn, torch.float8_e4m3fnuz)
|
||||
if _is_hip or is_fp8_kv:
|
||||
if is_fp8_kv:
|
||||
if q.dtype != kv.dtype:
|
||||
q = q.to(kv.dtype)
|
||||
if _is_gfx95_supported:
|
||||
if not _is_hip:
|
||||
block_I, threads, block_per_cu = 32, 128, 1
|
||||
cu = torch.cuda.get_device_properties(q.device).multi_processor_count
|
||||
elif _is_gfx95_supported:
|
||||
block_I, threads, block_per_cu, cu = 64, 256, 2, 256
|
||||
else:
|
||||
block_I, threads, block_per_cu, cu = 64, 256, 1, 304
|
||||
|
||||
@@ -529,6 +529,7 @@ def _check_dsa_backend_constraints(
|
||||
decode_backend: Optional[str],
|
||||
*,
|
||||
hip: bool,
|
||||
dcp_size: int = 1,
|
||||
) -> None:
|
||||
"""Validate DSA backend / platform / kv-cache-dtype constraints."""
|
||||
chosen = {prefill_backend, decode_backend}
|
||||
@@ -541,15 +542,21 @@ def _check_dsa_backend_constraints(
|
||||
"(flashmla_kv on Hopper, trtllm on Blackwell)."
|
||||
)
|
||||
|
||||
cuda_fp8_unsupported = {"tilelang"} & chosen
|
||||
if not hip and kv_cache_dtype == "fp8_e4m3" and cuda_fp8_unsupported:
|
||||
if hip or kv_cache_dtype != "fp8_e4m3" or "tilelang" not in chosen:
|
||||
return
|
||||
if prefill_backend != "tilelang" or decode_backend != "tilelang":
|
||||
raise ValueError(
|
||||
f"The {'/'.join(sorted(cuda_fp8_unsupported))} DSA prefill/decode kernels "
|
||||
"only support an fp8_e4m3 KV cache on ROCm/HIP; on CUDA they require "
|
||||
"a bfloat16 KV cache. Use --kv-cache-dtype bfloat16, or keep "
|
||||
"--kv-cache-dtype fp8_e4m3 and pick an fp8-capable DSA backend "
|
||||
"(flashmla_kv on Hopper, trtllm on Blackwell)."
|
||||
"CUDA TileLang FP8 KV requires both DSA backends to be tilelang; "
|
||||
"raw and scaled FP8 KV layouts cannot be mixed."
|
||||
)
|
||||
if dcp_size > 1:
|
||||
raise ValueError("CUDA TileLang FP8 KV does not support --dcp-size > 1.")
|
||||
|
||||
import torch
|
||||
|
||||
major, minor = torch.cuda.get_device_capability()
|
||||
if (major, minor) < (8, 9):
|
||||
raise ValueError("CUDA TileLang FP8 KV requires SM89 or newer.")
|
||||
|
||||
|
||||
def _check_tilelang_dsa_fp8_kv(
|
||||
@@ -558,10 +565,11 @@ def _check_tilelang_dsa_fp8_kv(
|
||||
decode_backend: Optional[str],
|
||||
*,
|
||||
hip: bool,
|
||||
dcp_size: int = 1,
|
||||
) -> None:
|
||||
"""Backward-compatible entry point for the TileLang DSA validation."""
|
||||
_check_dsa_backend_constraints(
|
||||
kv_cache_dtype, prefill_backend, decode_backend, hip=hip
|
||||
kv_cache_dtype, prefill_backend, decode_backend, hip=hip, dcp_size=dcp_size
|
||||
)
|
||||
|
||||
|
||||
@@ -652,11 +660,21 @@ def _dsa_split_backend_resolution(view: Any) -> dict:
|
||||
declared["dsa_decode_backend"] = backend
|
||||
prefill = declared.get("dsa_prefill_backend", view.dsa_prefill_backend)
|
||||
decode = declared.get("dsa_decode_backend", view.dsa_decode_backend)
|
||||
if (
|
||||
not get_platform().is_hip
|
||||
and kv_cache_dtype == "fp8_e4m3"
|
||||
and "tilelang" in (prefill, decode)
|
||||
):
|
||||
raise ValueError("CUDA TileLang FP8 KV does not support HiSparse.")
|
||||
# The hisparse allow-list in hisparse_hook is platform- but not
|
||||
# dtype-aware, so an explicitly requested backend still has to clear the
|
||||
# shared backend/kv-cache-dtype rules before this arm returns early.
|
||||
_check_dsa_backend_constraints(
|
||||
kv_cache_dtype, prefill, decode, hip=get_platform().is_hip
|
||||
kv_cache_dtype,
|
||||
prefill,
|
||||
decode,
|
||||
hip=get_platform().is_hip,
|
||||
dcp_size=getattr(view, "dcp_size", 1) or 1,
|
||||
)
|
||||
logger.warning(
|
||||
f"HiSparse enabled ({kv_cache_dtype}): using DSA backends "
|
||||
@@ -686,7 +704,11 @@ def _dsa_split_backend_resolution(view: Any) -> dict:
|
||||
prefill = declared.get("dsa_prefill_backend", view.dsa_prefill_backend)
|
||||
decode = declared.get("dsa_decode_backend", view.dsa_decode_backend)
|
||||
_check_dsa_backend_constraints(
|
||||
kv_cache_dtype, prefill, decode, hip=get_platform().is_hip
|
||||
kv_cache_dtype,
|
||||
prefill,
|
||||
decode,
|
||||
hip=get_platform().is_hip,
|
||||
dcp_size=getattr(view, "dcp_size", 1) or 1,
|
||||
)
|
||||
logger.warning(
|
||||
f"Set DSA backends for {kv_cache_dtype} KV Cache: "
|
||||
|
||||
@@ -473,6 +473,15 @@ class DeepseekSparseAttnBackend(
|
||||
self.supports_mha_one_shot: bool = True
|
||||
self.dsa_prefill_impl: _DSA_IMPL_T = get_exec().kernel.dsa_prefill_backend
|
||||
self.dsa_decode_impl: _DSA_IMPL_T = get_exec().kernel.dsa_decode_backend
|
||||
if (
|
||||
not _is_hip
|
||||
and self.token_to_kv_pool.dtype == torch.float8_e4m3fn
|
||||
and not self.dsa_kv_cache_store_fp8
|
||||
and "tilelang" in (self.dsa_prefill_impl, self.dsa_decode_impl)
|
||||
):
|
||||
# The one-shot FP8 dequantizers require scaled KV rows; TileLang
|
||||
# consumes raw FP8 KV without per-block scales.
|
||||
self.supports_mha_one_shot = False
|
||||
# Opt-in exact fast path for the Triton sparse-MLA prefill kernel.
|
||||
self.dsa_triton_union: int = _resolve_dsa_triton_union(
|
||||
union=get_exec().kernel.dsa_triton_union,
|
||||
|
||||
@@ -3100,6 +3100,14 @@ def calculate_mla_kv_cache_dim(
|
||||
):
|
||||
return kv_cache_dim
|
||||
|
||||
# CUDA TileLang uses raw FP8 rows without per-block scales. Argument
|
||||
# validation requires both consumers to use the same TileLang layout.
|
||||
if (
|
||||
get_exec().kernel.dsa_prefill_backend == "tilelang"
|
||||
and get_exec().kernel.dsa_decode_backend == "tilelang"
|
||||
):
|
||||
return kv_cache_dim
|
||||
|
||||
quant_block_size = DSATokenToKVPool.quant_block_size
|
||||
rope_storage_dtype = DSATokenToKVPool.rope_storage_dtype
|
||||
# Calculate override_kv_cache_dim for FP8 storage in backends that use scaled KV layout
|
||||
|
||||
Reference in New Issue
Block a user