[Spec] Make the overlap bonus-token relay unconditional (#29122)
This commit is contained in:
@@ -147,7 +147,6 @@ class FutureMap:
|
||||
def _lazy_init_forward_buf(self, draft_input: EagleDraftInput):
|
||||
self._forward_buf_initialized = True
|
||||
|
||||
self.need_bonus_tokens = getattr(draft_input, "bonus_tokens", None) is not None
|
||||
self.need_topk = self.spec_algo.need_topk()
|
||||
self.need_hidden_states = (
|
||||
spec_need_hidden_states()
|
||||
@@ -218,18 +217,16 @@ class FutureMap:
|
||||
self.output_tokens_buf,
|
||||
hidden_states_buf,
|
||||
)
|
||||
if self.need_bonus_tokens:
|
||||
draft_input.bonus_tokens = bonus_tokens
|
||||
if hidden_states is not None:
|
||||
draft_input.hidden_states = hidden_states
|
||||
if self.draft_probs_buf is not None and draft_input.draft_probs is not None:
|
||||
draft_input.draft_probs = self.draft_probs_buf[indices]
|
||||
elif self.need_bonus_tokens:
|
||||
else:
|
||||
draft_input.bonus_tokens = self.output_tokens_buf[indices]
|
||||
if self.need_hidden_states and not self.need_topk:
|
||||
draft_input.hidden_states = self.hidden_states_buf[indices]
|
||||
if _DEBUG_ASSERT:
|
||||
if self.need_bonus_tokens:
|
||||
_assert_nonneg_and_invalidate(
|
||||
draft_input.bonus_tokens, self.output_tokens_buf, indices
|
||||
)
|
||||
@@ -322,7 +319,6 @@ class FutureMap:
|
||||
draft_input: EagleDraftInput = payload
|
||||
if not self._forward_buf_initialized:
|
||||
self._lazy_init_forward_buf(draft_input)
|
||||
if self.need_bonus_tokens:
|
||||
self.output_tokens_buf[indices] = draft_input.bonus_tokens.to(
|
||||
self.output_tokens_buf.dtype
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user