minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Try to remove wrong logic about max total token in spec decoding (#14167)

Browse Source
This commit is contained in:
fzyzcjy
2025-12-01 15:29:58 +08:00
committed by GitHub
parent 0f8e53947d
commit f4a0c5c76b
1 changed files with 0 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/model_executor/model_runner.py
-17
View File
@@ -1689,25 +1689,8 @@ class ModelRunner:
if self.spec_algorithm.is_eagle() or self.spec_algorithm.is_standalone(): if self.spec_algorithm.is_eagle() or self.spec_algorithm.is_standalone():
if self.is_draft_worker: if self.is_draft_worker:
self.max_total_num_tokens = self.server_args.draft_runner_cache_size
max_num_reqs = self.server_args.max_num_reqs max_num_reqs = self.server_args.max_num_reqs
else: else:
# We are sharing the `token_to_kv_pool`, and both verify and draft tokens
# can be concurrently allocated, so we should give a headroom for it.
self.server_args.draft_runner_cache_size = (
self.max_total_num_tokens
# draft
+ max_num_reqs
* self.server_args.speculative_num_steps
* self.server_args.speculative_eagle_topk
# verify
+ max_num_reqs * self.server_args.speculative_num_draft_tokens
# buffer
+ 100
)
# Target worker and draft worker shares the same indices for the
# token_to_kv_pool, so we should make sure to match max_total_num_tokens.
self.max_total_num_tokens = self.server_args.draft_runner_cache_size
self.server_args.max_num_reqs = max_num_reqs self.server_args.max_num_reqs = max_num_reqs
if max_total_tokens is not None: if max_total_tokens is not None: