[Spec] Restore index_share_for_mtp_iteration in EAGLE V2 draft worker (#28192)
This commit is contained in:
@@ -188,6 +188,16 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
|||||||
self.eagle_use_aux_hidden_state = eagle_config.get(
|
self.eagle_use_aux_hidden_state = eagle_config.get(
|
||||||
"use_aux_hidden_state", True
|
"use_aux_hidden_state", True
|
||||||
)
|
)
|
||||||
|
# Reuse the first draft step's NSA/DSA indexer topk across the rest;
|
||||||
|
# topk == 1 only (select_top_k_tokens reorders rows, desyncing indices).
|
||||||
|
self.index_share_for_mtp_iteration = (
|
||||||
|
getattr(
|
||||||
|
self.draft_runner.model_config.hf_config,
|
||||||
|
"index_share_for_mtp_iteration",
|
||||||
|
False,
|
||||||
|
)
|
||||||
|
and self.topk == 1
|
||||||
|
)
|
||||||
self.draft_tp_context = (
|
self.draft_tp_context = (
|
||||||
draft_tp_context if server_args.enable_dp_attention else empty_context
|
draft_tp_context if server_args.enable_dp_attention else empty_context
|
||||||
)
|
)
|
||||||
@@ -542,6 +552,9 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
|||||||
|
|
||||||
# Forward multiple steps
|
# Forward multiple steps
|
||||||
scores = None
|
scores = None
|
||||||
|
if self.index_share_for_mtp_iteration:
|
||||||
|
forward_batch.reuse_mtp_topk_indices = True
|
||||||
|
forward_batch.topk_indices = None
|
||||||
for i in range(self.speculative_num_steps):
|
for i in range(self.speculative_num_steps):
|
||||||
input_ids, hidden_states, scores, tree_info = select_top_k_tokens(
|
input_ids, hidden_states, scores, tree_info = select_top_k_tokens(
|
||||||
i, topk_p, topk_index, hidden_states, scores, self.topk
|
i, topk_p, topk_index, hidden_states, scores, self.topk
|
||||||
@@ -609,6 +622,10 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
|||||||
hidden_states = logits_output.hidden_states
|
hidden_states = logits_output.hidden_states
|
||||||
forward_batch.positions.add_(1)
|
forward_batch.positions.add_(1)
|
||||||
|
|
||||||
|
if self.index_share_for_mtp_iteration:
|
||||||
|
forward_batch.topk_indices = None
|
||||||
|
forward_batch.reuse_mtp_topk_indices = False
|
||||||
|
|
||||||
# Organize the results
|
# Organize the results
|
||||||
if (
|
if (
|
||||||
self.topk == 1
|
self.topk == 1
|
||||||
|
|||||||
@@ -104,6 +104,15 @@ class StandaloneDraftWorker(EagleDraftWorker):
|
|||||||
)
|
)
|
||||||
self.tree_mask_mode = TreeMaskMode.FULL_MASK
|
self.tree_mask_mode = TreeMaskMode.FULL_MASK
|
||||||
self.plan_stream, self.plan_stream_ctx = _get_plan_stream(self.device)
|
self.plan_stream, self.plan_stream_ctx = _get_plan_stream(self.device)
|
||||||
|
# draft_forward reads this (set in EagleDraftWorker.__init__, skipped here).
|
||||||
|
self.index_share_for_mtp_iteration = (
|
||||||
|
getattr(
|
||||||
|
self.draft_runner.model_config.hf_config,
|
||||||
|
"index_share_for_mtp_iteration",
|
||||||
|
False,
|
||||||
|
)
|
||||||
|
and self.topk == 1
|
||||||
|
)
|
||||||
|
|
||||||
def alloc_memory_pool(
|
def alloc_memory_pool(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -187,6 +187,15 @@ class _EagleDraftWorkerHarness:
|
|||||||
self._topk1_parents_prealloc = None
|
self._topk1_parents_prealloc = None
|
||||||
self._topk1_score_indices_prealloc = None
|
self._topk1_score_indices_prealloc = None
|
||||||
EagleDraftWorker._rebuild_topk1_chain_buffers(self)
|
EagleDraftWorker._rebuild_topk1_chain_buffers(self)
|
||||||
|
# draft_forward reads this (set in EagleDraftWorker.__init__, skipped here).
|
||||||
|
self.index_share_for_mtp_iteration = (
|
||||||
|
getattr(
|
||||||
|
self.model_config.hf_config,
|
||||||
|
"index_share_for_mtp_iteration",
|
||||||
|
False,
|
||||||
|
)
|
||||||
|
and self.topk == 1
|
||||||
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def draft_model_runner(self):
|
def draft_model_runner(self):
|
||||||
|
|||||||
Reference in New Issue
Block a user