[KDA] Fix ptx_kda prefill NaN without a gate lower bound and workspace growth (#41572)
Co-authored-by: luoyuan.luo <luoyuan.luo@antgroup.com>
This commit is contained in:
@@ -3179,7 +3179,7 @@ static CUtensorMap enc2dgb(void* ptr, uint64_t rows, uint64_t cols) {
|
||||
|
||||
// Cached workspace, keyed by (device, T, nc_tot, H, npieces, N). The factor
|
||||
// tensors are torch allocations (caching allocator -> stream-safe reuse,
|
||||
// freed with the process). The CUDA tensor maps encode over these tensors'
|
||||
// freed when the cache is cleared). The CUDA tensor maps encode over these tensors'
|
||||
// data pointers, which are stable for the cache entry's lifetime — so the
|
||||
// maps are encoded ONCE here rather than per call (this IS the
|
||||
// (pointer, shape) tensor-map cache; a fresh encode is only host-cheap ~us,
|
||||
@@ -3188,6 +3188,9 @@ static CUtensorMap enc2dgb(void* ptr, uint64_t rows, uint64_t cols) {
|
||||
// lower-triangular cells, the TMA-read upper cells must stay zero.
|
||||
// The gate map is the exception: g is a CALLER tensor, so its pointer is only
|
||||
// stable while the allocator hands back the same block. One-entry memo.
|
||||
// Serving sees a new (T, N) on almost every batch and an entry is ~400 MiB at
|
||||
// 16K tokens and 16 heads, so the cache is cleared at kMaxWorkspaces entries.
|
||||
constexpr size_t kMaxWorkspaces = 4;
|
||||
struct Workspace {
|
||||
torch::Tensor P, u0, kdec, qdec, aqk_h, aqk_l, gC, pieceL, piecec, pflags;
|
||||
torch::Tensor h0z; // zeros initial state (lazy)
|
||||
@@ -3320,11 +3323,12 @@ static Route pick_route(const std::vector<int64_t>& cu, int64_t T, int64_t H, bo
|
||||
|
||||
static Workspace&
|
||||
get_workspace(const torch::Device& dev, int64_t T, int64_t nc, int64_t H, int64_t npieces, int64_t N) {
|
||||
// guarded by the GIL (single writer); entries live for the process
|
||||
// guarded by the GIL (single writer)
|
||||
static std::map<std::array<int64_t, 6>, Workspace> cache;
|
||||
const std::array<int64_t, 6> key{dev.index(), T, nc, H, npieces, N};
|
||||
auto it = cache.find(key);
|
||||
if (it != cache.end()) return it->second;
|
||||
if (cache.size() >= kMaxWorkspaces) cache.clear();
|
||||
Workspace ws;
|
||||
const auto ob = torch::TensorOptions().dtype(torch::kBFloat16).device(dev);
|
||||
const auto of = torch::TensorOptions().dtype(torch::kFloat).device(dev);
|
||||
|
||||
@@ -68,6 +68,7 @@ class PtxKDAKernel(LinearAttnKernelBase):
|
||||
# source tensor: this kernel instance is shared by every KDA layer.
|
||||
self._param_flat = {}
|
||||
self._unsupported_logged = False
|
||||
self._no_lower_bound_logged = False
|
||||
# (bucket, H, K, V, device) -> staging dict for ragged token counts.
|
||||
self._staging = {}
|
||||
|
||||
@@ -225,6 +226,9 @@ class PtxKDAKernel(LinearAttnKernelBase):
|
||||
)
|
||||
eligible = (
|
||||
not kwargs.get("is_spec_decode")
|
||||
# Only the safe gate is supported: without a lower bound the
|
||||
# per-chunk decay overflows fp32 exp and the kernel returns NaN.
|
||||
and lower_bound is not None
|
||||
# The native kernel cannot write the fp32 track snapshot buffer;
|
||||
# a batch carrying one must take the Triton fallback, which
|
||||
# forwards the snapshot arguments (see _triton_extend). Leaving
|
||||
@@ -235,6 +239,12 @@ class PtxKDAKernel(LinearAttnKernelBase):
|
||||
and supported_shape
|
||||
)
|
||||
if not eligible:
|
||||
if lower_bound is None and not self._no_lower_bound_logged:
|
||||
self._no_lower_bound_logged = True
|
||||
logger.warning(
|
||||
"PTX KDA prefill needs a gate lower bound (safe gate), and "
|
||||
"this model has none. Falling back to Triton."
|
||||
)
|
||||
if shape_known and not supported_shape and not self._unsupported_logged:
|
||||
self._unsupported_logged = True
|
||||
logger.warning(
|
||||
|
||||
Reference in New Issue
Block a user