[KDA] Fix ptx_kda prefill NaN without a gate lower bound and workspace growth (#41572)

Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
Yuan Luo
2026-09-30 17:23:54 +08:00
committed by GitHub
co-authored by luoyuan.luo
parent 8055ccd2cd
commit c0296186b7
4 changed files with 106 additions and 2 deletions
@@ -3179,7 +3179,7 @@ static CUtensorMap enc2dgb(void* ptr, uint64_t rows, uint64_t cols) {
// Cached workspace, keyed by (device, T, nc_tot, H, npieces, N). The factor
// tensors are torch allocations (caching allocator -> stream-safe reuse,
// freed with the process). The CUDA tensor maps encode over these tensors'
// freed when the cache is cleared). The CUDA tensor maps encode over these tensors'
// data pointers, which are stable for the cache entry's lifetime — so the
// maps are encoded ONCE here rather than per call (this IS the
// (pointer, shape) tensor-map cache; a fresh encode is only host-cheap ~us,
@@ -3188,6 +3188,9 @@ static CUtensorMap enc2dgb(void* ptr, uint64_t rows, uint64_t cols) {
// lower-triangular cells, the TMA-read upper cells must stay zero.
// The gate map is the exception: g is a CALLER tensor, so its pointer is only
// stable while the allocator hands back the same block. One-entry memo.
// Serving sees a new (T, N) on almost every batch and an entry is ~400 MiB at
// 16K tokens and 16 heads, so the cache is cleared at kMaxWorkspaces entries.
constexpr size_t kMaxWorkspaces = 4;
struct Workspace {
torch::Tensor P, u0, kdec, qdec, aqk_h, aqk_l, gC, pieceL, piecec, pflags;
torch::Tensor h0z; // zeros initial state (lazy)
@@ -3320,11 +3323,12 @@ static Route pick_route(const std::vector<int64_t>& cu, int64_t T, int64_t H, bo
static Workspace&
get_workspace(const torch::Device& dev, int64_t T, int64_t nc, int64_t H, int64_t npieces, int64_t N) {
// guarded by the GIL (single writer); entries live for the process
// guarded by the GIL (single writer)
static std::map<std::array<int64_t, 6>, Workspace> cache;
const std::array<int64_t, 6> key{dev.index(), T, nc, H, npieces, N};
auto it = cache.find(key);
if (it != cache.end()) return it->second;
if (cache.size() >= kMaxWorkspaces) cache.clear();
Workspace ws;
const auto ob = torch::TensorOptions().dtype(torch::kBFloat16).device(dev);
const auto of = torch::TensorOptions().dtype(torch::kFloat).device(dev);
@@ -68,6 +68,7 @@ class PtxKDAKernel(LinearAttnKernelBase):
# source tensor: this kernel instance is shared by every KDA layer.
self._param_flat = {}
self._unsupported_logged = False
self._no_lower_bound_logged = False
# (bucket, H, K, V, device) -> staging dict for ragged token counts.
self._staging = {}
@@ -225,6 +226,9 @@ class PtxKDAKernel(LinearAttnKernelBase):
)
eligible = (
not kwargs.get("is_spec_decode")
# Only the safe gate is supported: without a lower bound the
# per-chunk decay overflows fp32 exp and the kernel returns NaN.
and lower_bound is not None
# The native kernel cannot write the fp32 track snapshot buffer;
# a batch carrying one must take the Triton fallback, which
# forwards the snapshot arguments (see _triton_extend). Leaving
@@ -235,6 +239,12 @@ class PtxKDAKernel(LinearAttnKernelBase):
and supported_shape
)
if not eligible:
if lower_bound is None and not self._no_lower_bound_logged:
self._no_lower_bound_logged = True
logger.warning(
"PTX KDA prefill needs a gate lower bound (safe gate), and "
"this model has none. Falling back to Triton."
)
if shape_known and not supported_shape and not self._unsupported_logged:
self._unsupported_logged = True
logger.warning(