minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[cache] add conservative estimation (#19482)

Browse Source
This commit is contained in:
Yilong Zhao
2026-02-27 18:14:46 +08:00
committed by GitHub
parent 9496bbd7b1
commit 07da4bed7b
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/managers/schedule_policy.py
+4 -2
View File
@@ -513,8 +513,10 @@ class PrefillAdder:
# TODO(lsyin): check this workaround logic, which only ensures the prefill will not out of memory, and may be too conservative # TODO(lsyin): check this workaround logic, which only ensures the prefill will not out of memory, and may be too conservative
extend_input_len = self.ceil_paged_tokens(extend_input_len) extend_input_len = self.ceil_paged_tokens(extend_input_len)
self.rem_total_token_offset += extend_input_len + max_new_tokens # alloc_extend reserves an extra page_size per request to make sure the budget doesn't over-commit
self.cur_rem_token_offset += extend_input_len page_overhead = self.page_size
self.rem_total_token_offset += extend_input_len + max_new_tokens + page_overhead
self.cur_rem_token_offset += extend_input_len + page_overhead
self.rem_input_tokens -= extend_input_len self.rem_input_tokens -= extend_input_len
if self.dllm_config is not None: if self.dllm_config is not None: