[Fix] engram: Avoid blocking history copies during PD admission (#44787)

Co-authored-by: Trevor Morris <tmorris@nvidia.com>
This commit is contained in:
Khoa Pham
2026-10-16 14:29:05 -07:00
committed by GitHub
co-authored by Trevor Morris
parent 2948662026
commit 71b60fa46c
@@ -181,16 +181,17 @@ class NgramEmbeddingManager:
for req in batch.reqs:
# full_untruncated_fill_ids is only refreshed on the extend and
# decode paths, which a PD decode request has not run yet.
fill_ids = req.origin_input_ids + req.output_ids
end = len(fill_ids) - 1
ids = fill_ids[max(0, end - n1) : end]
# Slice the tail first: origin_input_ids can be ~100k tokens.
tail = (req.origin_input_ids[-(n1 + 1) :] + req.output_ids)[-(n1 + 1) :]
ids = tail[:-1]
rows.append([0] * (n1 - len(ids)) + list(ids))
slots = torch.tensor(
# Pinned copies keep the admission fence from stalling the CPU.
slots = async_h2d(
[req.kv.req_pool_idx for req in batch.reqs],
dtype=torch.int64,
device=history.device,
)
history[slots] = torch.tensor(
history[slots] = async_h2d(
rows, dtype=history.dtype, device=history.device
).view(len(rows), n1)
return