[SPEC] fix: use effective max draft tokens for adaptive spec initiali… (#26354)
Co-authored-by: maodoudou168 <maodoudou168@users.noreply.github.com>
This commit is contained in:
@@ -297,7 +297,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
# so we need to reserve the space for the draft tokens.
|
||||
self.num_reserved_tokens = max(
|
||||
server_args.speculative_eagle_topk * server_args.speculative_num_steps,
|
||||
server_args.speculative_num_draft_tokens,
|
||||
server_args.max_speculative_num_draft_tokens,
|
||||
)
|
||||
else:
|
||||
self.num_reserved_tokens = 0
|
||||
|
||||
@@ -240,7 +240,7 @@ def get_alloc_len_per_decode(server_args: Optional[ServerArgs] = None) -> int:
|
||||
|
||||
spec_steps = server_args.speculative_num_steps or 1
|
||||
spec_topk = server_args.speculative_eagle_topk or 1
|
||||
spec_tokens = server_args.speculative_num_draft_tokens
|
||||
spec_tokens = server_args.max_speculative_num_draft_tokens
|
||||
page_size = server_args.page_size
|
||||
|
||||
if page_size == 1 or spec_topk == 1:
|
||||
|
||||
Reference in New Issue
Block a user