diff --git a/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py b/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py index cd4055faf8..c76f6b2091 100644 --- a/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py +++ b/python/sglang/srt/model_executor/model_runner_components/ngram_embedding_manager.py @@ -181,16 +181,17 @@ class NgramEmbeddingManager: for req in batch.reqs: # full_untruncated_fill_ids is only refreshed on the extend and # decode paths, which a PD decode request has not run yet. - fill_ids = req.origin_input_ids + req.output_ids - end = len(fill_ids) - 1 - ids = fill_ids[max(0, end - n1) : end] + # Slice the tail first: origin_input_ids can be ~100k tokens. + tail = (req.origin_input_ids[-(n1 + 1) :] + req.output_ids)[-(n1 + 1) :] + ids = tail[:-1] rows.append([0] * (n1 - len(ids)) + list(ids)) - slots = torch.tensor( + # Pinned copies keep the admission fence from stalling the CPU. + slots = async_h2d( [req.kv.req_pool_idx for req in batch.reqs], dtype=torch.int64, device=history.device, ) - history[slots] = torch.tensor( + history[slots] = async_h2d( rows, dtype=history.dtype, device=history.device ).view(len(rows), n1) return