From 07e1a4f0821ef09a1bf87e2fcb7f326183658295 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Fri, 2 Oct 2026 13:46:32 -0700 Subject: [PATCH] [mem_cache] Skip the release-time insert on optimistic prefill requeue; make `refresh_fill_ids` public (#42194) --- python/sglang/srt/disaggregation/prefill.py | 8 +++----- python/sglang/srt/dllm/mixin/req.py | 2 +- python/sglang/srt/managers/schedule_batch.py | 12 ++++++------ python/sglang/srt/mem_cache/common.py | 2 +- python/sglang/srt/session/session_controller.py | 2 +- python/sglang/test/mem_cache_utils.py | 3 +-- .../test_schedule_batch_convert_decode_to_extend.py | 2 +- .../managers/test_schedule_batch_out_of_place.py | 2 +- .../unit/mem_cache/test_decode_radix_lock_ref.py | 2 +- .../unit/mem_cache/test_page_interleave_shard.py | 3 +++ .../unit/mem_cache/test_prefill_memory_budget.py | 2 +- .../unit/mem_cache/test_session_token_share_unit.py | 8 ++++---- 12 files changed, 24 insertions(+), 24 deletions(-) diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index c0fb2d82e9..3f16ffcb52 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -1568,11 +1568,9 @@ class SchedulerDisaggregationPrefillMixin: ) ) self._release_aborted_request(req) - # Mamba insertion donates the checkpoint and clears its sequence marker. - is_insert = ( - not uses_write_through_cache and not self.tree_cache.supports_mamba() - ) - release_kv_cache(req, self.tree_cache, is_insert=is_insert) + # The checkpoint above already handed the prefill KV to the tree; the + # request is not finished, so the release only frees the rest. + release_kv_cache(req, self.tree_cache, is_insert=False) req.reset_for_retract() req.output_ids = array("q") req.start_send_idx = 0 diff --git a/python/sglang/srt/dllm/mixin/req.py b/python/sglang/srt/dllm/mixin/req.py index 0f4af4adeb..e6501e55cc 100644 --- a/python/sglang/srt/dllm/mixin/req.py +++ b/python/sglang/srt/dllm/mixin/req.py @@ -80,7 +80,7 @@ class ReqDllmMixin: return if self.dllm_config.requires_separate_context_encoding: - self._refresh_fill_ids() + self.refresh_fill_ids() self.dllm_block_offset = self.seqlen if self.dllm_initialized: self.full_untruncated_fill_ids.extend([0] * self.dllm_config.block_size) diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index acfe19a3e5..da97552647 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1065,12 +1065,12 @@ class Req(ReqDllmMixin): else self.origin_input_ids ) # Before image padding # Each decode stage's output ids. Append-only by contract: - # _refresh_fill_ids infers how many output tokens are already in + # refresh_fill_ids infers how many output tokens are already in # full_untruncated_fill_ids from lengths alone, so in-place rewrites # that preserve length would silently corrupt fill_ids. self.output_ids = array("q") # Full untruncated sequence: origin + output (+ DLLM mask block). - # Kept in sync by _refresh_fill_ids; admission only updates + # Kept in sync by refresh_fill_ids; admission only updates # extend_range, never mutates this array's length. self.full_untruncated_fill_ids = array("q") self.extend_range: Optional[Range] = None @@ -1571,7 +1571,7 @@ class Req(ReqDllmMixin): def get_fill_ids(self) -> array: return self.full_untruncated_fill_ids[: self.extend_range.end] - def _refresh_fill_ids(self) -> None: + def refresh_fill_ids(self) -> None: """Keep full_untruncated_fill_ids == origin_input_ids + output_ids by appending only the new output tokens. @@ -1600,7 +1600,7 @@ class Req(ReqDllmMixin): self._init_fill_ids_for_dllm() self.determine_dllm_phase() else: - self._refresh_fill_ids() + self.refresh_fill_ids() input_len = len(self.full_untruncated_fill_ids) @@ -3181,7 +3181,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): for req, prefix_len in zip( running_batch.reqs, running_prefix_lens, strict=True ): - req._refresh_fill_ids() + req.refresh_fill_ids() req.set_extend_range(prefix_len, prefix_len + 1) # Decode tokens of the running portion live in future_map.output_tokens_buf. @@ -3259,7 +3259,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): # cannot stand in for it. Rows past bs are a beam tail, not requests. seq_lens = self.seq_lens_cpu[:bs].tolist() for req, seq_len in zip(self.reqs, seq_lens, strict=True): - req._refresh_fill_ids() + req.refresh_fill_ids() # end runs one past full_untruncated_fill_ids while output_ids # trails; safe only while decoding_reqs suppresses the checkpoint insert. req.set_extend_range(seq_len - 1, seq_len) diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 01fde5cf6a..7950c66e48 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -324,7 +324,7 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr f"releasing unfinished request {req.rid} into a mamba tree" ) # The fill-id array lags output_ids until the next prepare_for_decode. - req._refresh_fill_ids() + req.refresh_fill_ids() tree_cache.insert_req(req, up_to=owned_kv_len) # The protected prefix is not this req's to free. tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)]) diff --git a/python/sglang/srt/session/session_controller.py b/python/sglang/srt/session/session_controller.py index 979b887fa5..8ff2044bc0 100644 --- a/python/sglang/srt/session/session_controller.py +++ b/python/sglang/srt/session/session_controller.py @@ -157,7 +157,7 @@ class Session: or carry_fill is input_ids_unpadded ): # Unexpected type or aliased with an origin array (extending it - # below would double-append): let _refresh_fill_ids rebuild. + # below would double-append): let refresh_fill_ids rebuild. carry_fill = None else: del carry_fill[self.committed_fill_len :] diff --git a/python/sglang/test/mem_cache_utils.py b/python/sglang/test/mem_cache_utils.py index 69362e56fc..ce155a7413 100644 --- a/python/sglang/test/mem_cache_utils.py +++ b/python/sglang/test/mem_cache_utils.py @@ -8,8 +8,7 @@ def finish_req(cache, req, up_to): finished, insert_req, free the rest, drop the lock.""" if getattr(req, "finished_reason", None) is None: req.finished_reason = FINISH_LENGTH(length=0) - if hasattr(req, "_refresh_fill_ids"): - req._refresh_fill_ids() + req.refresh_fill_ids() cache.insert_req(req, up_to=up_to) cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)]) cache.unpin(req) diff --git a/test/registered/unit/managers/test_schedule_batch_convert_decode_to_extend.py b/test/registered/unit/managers/test_schedule_batch_convert_decode_to_extend.py index 176cb4cef8..4e082a5307 100644 --- a/test/registered/unit/managers/test_schedule_batch_convert_decode_to_extend.py +++ b/test/registered/unit/managers/test_schedule_batch_convert_decode_to_extend.py @@ -28,7 +28,7 @@ class _FakeReq: """Carries the fill-id state convert_decode_to_extend touches, with the real Req methods so the array bookkeeping is not re-implemented here.""" - _refresh_fill_ids = Req._refresh_fill_ids + refresh_fill_ids = Req.refresh_fill_ids set_extend_range = Req.set_extend_range def __init__(self, *, num_prompt_tokens: int, num_output_tokens: int): diff --git a/test/registered/unit/managers/test_schedule_batch_out_of_place.py b/test/registered/unit/managers/test_schedule_batch_out_of_place.py index 867ab1ee47..0da8845ea2 100644 --- a/test/registered/unit/managers/test_schedule_batch_out_of_place.py +++ b/test/registered/unit/managers/test_schedule_batch_out_of_place.py @@ -132,7 +132,7 @@ class _FakeReq: self.full_untruncated_fill_ids = list(range(origin_len + output_len)) self.extend_range = None - def _refresh_fill_ids(self): + def refresh_fill_ids(self): self.full_untruncated_fill_ids = self.origin_input_ids + self.output_ids @property diff --git a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py index df2321507b..dcb46201ee 100644 --- a/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py +++ b/test/registered/unit/mem_cache/test_decode_radix_lock_ref.py @@ -116,7 +116,7 @@ class MockReq: def finished(self): return self.finished_reason is not None - def _refresh_fill_ids(self): + def refresh_fill_ids(self): pass def owned_kv_len(self): diff --git a/test/registered/unit/mem_cache/test_page_interleave_shard.py b/test/registered/unit/mem_cache/test_page_interleave_shard.py index a616f5b577..80997876d3 100644 --- a/test/registered/unit/mem_cache/test_page_interleave_shard.py +++ b/test/registered/unit/mem_cache/test_page_interleave_shard.py @@ -798,6 +798,9 @@ class _GraftReq: def get_fill_ids(self): return array("q", self.fill_ids) + def refresh_fill_ids(self): + pass # fill ids are fixed for the stand-in + def finished(self): return self.finished_reason is not None diff --git a/test/registered/unit/mem_cache/test_prefill_memory_budget.py b/test/registered/unit/mem_cache/test_prefill_memory_budget.py index b1dbad1abf..44f7c919ed 100644 --- a/test/registered/unit/mem_cache/test_prefill_memory_budget.py +++ b/test/registered/unit/mem_cache/test_prefill_memory_budget.py @@ -179,7 +179,7 @@ class TestSharedPrefillAdmission(unittest.TestCase): with get_parallel().override(attn_dcp_size=1): scheduler.init_req_max_new_tokens(req) self.assertEqual(req.sampling_params.max_new_tokens, 1) - req._refresh_fill_ids() + req.refresh_fill_ids() cache = SimpleNamespace( sliding_window_size=page_size, diff --git a/test/registered/unit/mem_cache/test_session_token_share_unit.py b/test/registered/unit/mem_cache/test_session_token_share_unit.py index b762f5026a..7b70670724 100644 --- a/test/registered/unit/mem_cache/test_session_token_share_unit.py +++ b/test/registered/unit/mem_cache/test_session_token_share_unit.py @@ -79,7 +79,7 @@ class TestSessionTokenShare(CustomTestCase): if baked is None: baked = len(output) req.output_ids.extend(output[:baked]) - req._refresh_fill_ids() + req.refresh_fill_ids() req.output_ids.extend(output[baked:]) self.session.finish_req(req) @@ -115,7 +115,7 @@ class TestSessionTokenShare(CustomTestCase): r2 = self._create("r2", [50, 51]) self.assertEqual(list(r2.origin_input_ids), in1 + out1 + [50, 51]) r2.output_ids.extend([6, 7]) - r2._refresh_fill_ids() + r2.refresh_fill_ids() self.session.abort_req() self.assertEqual(self.session.committed_origin_len, len(in1)) @@ -150,7 +150,7 @@ class TestSessionTokenShare(CustomTestCase): # array, then output trimmed to finished_len (like _trim_overshoot) # before finish. r1.output_ids.extend([1, 2, 3, 4, 5, 6]) - r1._refresh_fill_ids() + r1.refresh_fill_ids() del r1.output_ids[4:] self.session.finish_req(r1) self.assertEqual( @@ -162,7 +162,7 @@ class TestSessionTokenShare(CustomTestCase): r2 = self._create("r2", [50]) self.assertEqual(list(r2.origin_input_ids), in1 + [1, 2, 3, 4] + [50]) self.assertEqual(len(r2.full_untruncated_fill_ids), 0) # carry skipped - r2._refresh_fill_ids() + r2.refresh_fill_ids() self.assertEqual(list(r2.full_untruncated_fill_ids), list(r2.origin_input_ids))