minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Spec] Single-source num_tokens_per_req derivation and access (#31013)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-07-14 18:41:08 -07:00
committed by GitHub
parent ca0ee3f1a8
commit 76dc427806
23 changed files with 116 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/model_executor/runner/eager_runner.py
+3 -9
View File
@@ -83,10 +83,8 @@ class EagerRunner(BaseRunner):
),
)
else:
num_tokens_per_req = (
mr.spec_algorithm.get_num_tokens_per_req_for_target_verify(
num_draft_tokens, mr.is_draft_worker
)
num_tokens_per_req = mr.decode_num_tokens_per_req(
num_draft_tokens=num_draft_tokens
)
else:
dllm_config = DllmConfig.from_server_args(sa)
@@ -150,11 +148,7 @@ class EagerRunner(BaseRunner):
mr = self.model_runner
num_tokens_per_req = 1
if mr.spec_algorithm.is_speculative():
num_tokens_per_req = (
mr.spec_algorithm.get_num_tokens_per_req_for_target_verify(
mr.server_args.speculative_num_draft_tokens, mr.is_draft_worker
)
)
num_tokens_per_req = mr.decode_num_tokens_per_req()
return (
self._alloc_dummy_decode_buffers(
self._eager_max_bs, num_tokens_per_req=num_tokens_per_req