[Fix] engram: Avoid blocking history copies during PD admission (#44787)
Co-authored-by: Trevor Morris <tmorris@nvidia.com>
This commit is contained in:
co-authored by
Trevor Morris
parent
2948662026
commit
71b60fa46c
@@ -181,16 +181,17 @@ class NgramEmbeddingManager:
|
||||
for req in batch.reqs:
|
||||
# full_untruncated_fill_ids is only refreshed on the extend and
|
||||
# decode paths, which a PD decode request has not run yet.
|
||||
fill_ids = req.origin_input_ids + req.output_ids
|
||||
end = len(fill_ids) - 1
|
||||
ids = fill_ids[max(0, end - n1) : end]
|
||||
# Slice the tail first: origin_input_ids can be ~100k tokens.
|
||||
tail = (req.origin_input_ids[-(n1 + 1) :] + req.output_ids)[-(n1 + 1) :]
|
||||
ids = tail[:-1]
|
||||
rows.append([0] * (n1 - len(ids)) + list(ids))
|
||||
slots = torch.tensor(
|
||||
# Pinned copies keep the admission fence from stalling the CPU.
|
||||
slots = async_h2d(
|
||||
[req.kv.req_pool_idx for req in batch.reqs],
|
||||
dtype=torch.int64,
|
||||
device=history.device,
|
||||
)
|
||||
history[slots] = torch.tensor(
|
||||
history[slots] = async_h2d(
|
||||
rows, dtype=history.dtype, device=history.device
|
||||
).view(len(rows), n1)
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user