[SM120] flash_mla: allocate the page-split buffer outside inference mode (#35116)
This commit is contained in:
@@ -400,6 +400,10 @@ def _split_kv_pages_to_64(
|
|||||||
key = f"flash_mla_sm120_split:{dev}"
|
key = f"flash_mla_sm120_split:{dev}"
|
||||||
buf = buffers.get(key)
|
buf = buffers.get(key)
|
||||||
if buf is None or buf.shape[0] < num_dst_pages:
|
if buf is None or buf.shape[0] < num_dst_pages:
|
||||||
|
# The first allocation can happen under inference mode (autotune), but
|
||||||
|
# the buffer is written again during CUDA graph capture outside
|
||||||
|
# inference mode, where an inference tensor cannot be mutated.
|
||||||
|
with torch.inference_mode(False):
|
||||||
buf = torch.empty(
|
buf = torch.empty(
|
||||||
num_dst_pages,
|
num_dst_pages,
|
||||||
_BYTES_PER_DST_PAGE_PADDED,
|
_BYTES_PER_DST_PAGE_PADDED,
|
||||||
|
|||||||
Reference in New Issue
Block a user