[Fix] engram: Clamp multimodal pad ids in host-built history rows (#45038)

This commit is contained in:
Liangsheng Yin
2026-10-18 12:24:05 -07:00
committed by GitHub
parent 0f4aedfecc
commit da414adda7
4 changed files with 100 additions and 5 deletions
+7
View File
@@ -273,6 +273,13 @@ class EngramHasher(nn.Module):
)
self.pad_row = num_req_slots
def history_row(self, ids: list[int]) -> list[int]:
"""Left-pad up to n - 1 predecessors into a history row. The hash reads every
id at or above MM_PAD_SHIFT_VALUE as an image token, so multimodal pad ids
clamp to it and fit the int32 history."""
n1 = self.max_ngram_size - 1
return [0] * (n1 - len(ids)) + [min(t, MM_PAD_SHIFT_VALUE) for t in ids]
@classmethod
def from_config(
cls, config, layout: EngramLayout, *, image_token_id: Optional[int] = None
@@ -173,8 +173,8 @@ def _engram_history(*, reqs, starts, runner):
n = hasher.max_ngram_size - 1
hist = []
for r, s in zip(reqs, starts):
ids = list(r.full_untruncated_fill_ids[max(0, s - n) : s])
hist.append([0] * (n - len(ids)) + ids)
ids = r.full_untruncated_fill_ids[max(0, s - n) : s]
hist.append(hasher.history_row(ids))
return torch.tensor(hist, dtype=torch.int32, device=runner.device)
@@ -183,8 +183,7 @@ class NgramEmbeddingManager:
# decode paths, which a PD decode request has not run yet.
# Slice the tail first: origin_input_ids can be ~100k tokens.
tail = (req.origin_input_ids[-(n1 + 1) :] + req.output_ids)[-(n1 + 1) :]
ids = tail[:-1]
rows.append([0] * (n1 - len(ids)) + list(ids))
rows.append(self.engram_hasher.history_row(tail[:-1]))
# Pinned copies keep the admission fence from stalling the CPU.
slots = async_h2d(
[req.kv.req_pool_idx for req in batch.reqs],
@@ -201,7 +200,7 @@ class NgramEmbeddingManager:
for req, start in zip(batch.reqs, batch.prefix_lens, strict=True):
lo = max(0, start - n1)
ids = req.full_untruncated_fill_ids[lo:start]
rows.append([0] * (n1 - len(ids)) + list(ids))
rows.append(self.engram_hasher.history_row(ids))
# The forward stream waits on the schedule stream, which orders this copy.
batch.engram_history = async_h2d(
rows, dtype=torch.int32, device=self.engram_hasher.history.device