From 71b60fa46c9da596f29c6182cdae8201c8c1596e Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Fri, 16 Oct 2026 14:29:05 -0700 Subject: [PATCH] [Fix] engram: Avoid blocking history copies during PD admission (#44787) Co-authored-by: Trevor Morris --- .../ngram_embedding_manager.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py b/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py index cd4055faf8..c76f6b2091 100644 --- a/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py +++ b/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py @@ -181,16 +181,17 @@ class NgramEmbeddingManager: for req in batch.reqs: # full_untruncated_fill_ids is only refreshed on the extend and # decode paths, which a PD decode request has not run yet. - fill_ids = req.origin_input_ids + req.output_ids - end = len(fill_ids) - 1 - ids = fill_ids[max(0, end - n1) : end] + # Slice the tail first: origin_input_ids can be ~100k tokens. + tail = (req.origin_input_ids[-(n1 + 1) :] + req.output_ids)[-(n1 + 1) :] + ids = tail[:-1] rows.append([0] * (n1 - len(ids)) + list(ids)) - slots = torch.tensor( + # Pinned copies keep the admission fence from stalling the CPU. + slots = async_h2d( [req.kv.req_pool_idx for req in batch.reqs], dtype=torch.int64, device=history.device, ) - history[slots] = torch.tensor( + history[slots] = async_h2d( rows, dtype=history.dtype, device=history.device ).view(len(rows), n1) return