[Fix] engram: Clamp multimodal pad ids in host-built history rows (#45038)
This commit is contained in:
@@ -273,6 +273,13 @@ class EngramHasher(nn.Module):
|
||||
)
|
||||
self.pad_row = num_req_slots
|
||||
|
||||
def history_row(self, ids: list[int]) -> list[int]:
|
||||
"""Left-pad up to n - 1 predecessors into a history row. The hash reads every
|
||||
id at or above MM_PAD_SHIFT_VALUE as an image token, so multimodal pad ids
|
||||
clamp to it and fit the int32 history."""
|
||||
n1 = self.max_ngram_size - 1
|
||||
return [0] * (n1 - len(ids)) + [min(t, MM_PAD_SHIFT_VALUE) for t in ids]
|
||||
|
||||
@classmethod
|
||||
def from_config(
|
||||
cls, config, layout: EngramLayout, *, image_token_id: Optional[int] = None
|
||||
|
||||
@@ -173,8 +173,8 @@ def _engram_history(*, reqs, starts, runner):
|
||||
n = hasher.max_ngram_size - 1
|
||||
hist = []
|
||||
for r, s in zip(reqs, starts):
|
||||
ids = list(r.full_untruncated_fill_ids[max(0, s - n) : s])
|
||||
hist.append([0] * (n - len(ids)) + ids)
|
||||
ids = r.full_untruncated_fill_ids[max(0, s - n) : s]
|
||||
hist.append(hasher.history_row(ids))
|
||||
return torch.tensor(hist, dtype=torch.int32, device=runner.device)
|
||||
|
||||
|
||||
|
||||
@@ -183,8 +183,7 @@ class NgramEmbeddingManager:
|
||||
# decode paths, which a PD decode request has not run yet.
|
||||
# Slice the tail first: origin_input_ids can be ~100k tokens.
|
||||
tail = (req.origin_input_ids[-(n1 + 1) :] + req.output_ids)[-(n1 + 1) :]
|
||||
ids = tail[:-1]
|
||||
rows.append([0] * (n1 - len(ids)) + list(ids))
|
||||
rows.append(self.engram_hasher.history_row(tail[:-1]))
|
||||
# Pinned copies keep the admission fence from stalling the CPU.
|
||||
slots = async_h2d(
|
||||
[req.kv.req_pool_idx for req in batch.reqs],
|
||||
@@ -201,7 +200,7 @@ class NgramEmbeddingManager:
|
||||
for req, start in zip(batch.reqs, batch.prefix_lens, strict=True):
|
||||
lo = max(0, start - n1)
|
||||
ids = req.full_untruncated_fill_ids[lo:start]
|
||||
rows.append([0] * (n1 - len(ids)) + list(ids))
|
||||
rows.append(self.engram_hasher.history_row(ids))
|
||||
# The forward stream waits on the schedule stream, which orders this copy.
|
||||
batch.engram_history = async_h2d(
|
||||
rows, dtype=torch.int32, device=self.engram_hasher.history.device
|
||||
|
||||
Reference in New Issue
Block a user