[SM120] flash_mla: allocate the page-split buffer outside inference mode (#35116)

This commit is contained in:
eeecho
2026-08-24 17:42:23 -07:00
committed by GitHub
parent 0f7ba3d115
commit 91e7e84ee5
@@ -400,6 +400,10 @@ def _split_kv_pages_to_64(
key = f"flash_mla_sm120_split:{dev}" key = f"flash_mla_sm120_split:{dev}"
buf = buffers.get(key) buf = buffers.get(key)
if buf is None or buf.shape[0] < num_dst_pages: if buf is None or buf.shape[0] < num_dst_pages:
# The first allocation can happen under inference mode (autotune), but
# the buffer is written again during CUDA graph capture outside
# inference mode, where an inference tensor cannot be mutated.
with torch.inference_mode(False):
buf = torch.empty( buf = torch.empty(
num_dst_pages, num_dst_pages,
_BYTES_PER_DST_PAGE_PADDED, _BYTES_PER_DST_PAGE_PADDED,