[mem_cache] Skip the release-time insert on optimistic prefill requeue; make refresh_fill_ids public (#42194)
build-sglang-image / build (push) Successful in 25m53s

(cherry picked from commit 07e1a4f082)
This commit is contained in:
Liangsheng Yin
2026-10-09 11:18:17 +08:00
committed by minke.yu
parent 6ba34507f1
commit 9ef65b51c2
9 changed files with 37 additions and 18 deletions
+5 -4
View File
@@ -1638,10 +1638,11 @@ class SchedulerDisaggregationPrefillMixin:
)
)
self._release_aborted_request(req)
# Mamba insertion donates the checkpoint and clears its sequence marker.
release_kv_cache(
req, self.tree_cache, is_insert=not self.tree_cache.supports_mamba()
)
# The checkpoint above (maybe_cache_unfinished_req) already handed the
# prefill KV to the tree; the request is not finished, so the release
# only frees the rest. (#42194 semantics; our tree predates #41520's
# mamba-donation rework, mamba models are not run on this branch.)
release_kv_cache(req, self.tree_cache, is_insert=False)
req.reset_for_retract()
req.output_ids = array("q")
req.start_send_idx = 0
+6 -6
View File
@@ -1034,12 +1034,12 @@ class Req(ReqDllmMixin):
else self.origin_input_ids
) # Before image padding
# Each decode stage's output ids. Append-only by contract:
# _refresh_fill_ids infers how many output tokens are already in
# refresh_fill_ids infers how many output tokens are already in
# full_untruncated_fill_ids from lengths alone, so in-place rewrites
# that preserve length would silently corrupt fill_ids.
self.output_ids = array("q")
# Full untruncated sequence: origin + output (+ DLLM mask block).
# Kept in sync by _refresh_fill_ids; admission only updates
# Kept in sync by refresh_fill_ids; admission only updates
# extend_range, never mutates this array's length.
self.full_untruncated_fill_ids = array("q")
self.extend_range: Optional[Range] = None
@@ -1533,7 +1533,7 @@ class Req(ReqDllmMixin):
def get_fill_ids(self) -> array:
return self.full_untruncated_fill_ids[: self.extend_range.end]
def _refresh_fill_ids(self) -> None:
def refresh_fill_ids(self) -> None:
"""Keep full_untruncated_fill_ids == origin_input_ids + output_ids by
appending only the new output tokens.
@@ -1562,7 +1562,7 @@ class Req(ReqDllmMixin):
self._init_fill_ids_for_dllm()
self.determine_dllm_phase()
else:
self._refresh_fill_ids()
self.refresh_fill_ids()
input_len = len(self.full_untruncated_fill_ids)
@@ -3110,7 +3110,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
for req, prefix_len in zip(
running_batch.reqs, running_prefix_lens, strict=True
):
req._refresh_fill_ids()
req.refresh_fill_ids()
req.set_extend_range(prefix_len, prefix_len + 1)
# Decode tokens of the running portion live in future_map.output_tokens_buf.
@@ -3188,7 +3188,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# cannot stand in for it. Rows past bs are a beam tail, not requests.
seq_lens = self.seq_lens_cpu[:bs].tolist()
for req, seq_len in zip(self.reqs, seq_lens, strict=True):
req._refresh_fill_ids()
req.refresh_fill_ids()
# end runs one past full_untruncated_fill_ids while output_ids
# trails; safe only while decoding_reqs suppresses cache_unfinished_req.
req.set_extend_range(seq_len - 1, seq_len)
@@ -157,7 +157,7 @@ class Session:
or carry_fill is input_ids_unpadded
):
# Unexpected type or aliased with an origin array (extending it
# below would double-append): let _refresh_fill_ids rebuild.
# below would double-append): let refresh_fill_ids rebuild.
carry_fill = None
else:
del carry_fill[self.committed_fill_len :]