[mem_cache] Skip the release-time insert on optimistic prefill requeue; make refresh_fill_ids public (#42194)
build-sglang-image / build (push) Successful in 25m53s
build-sglang-image / build (push) Successful in 25m53s
(cherry picked from commit 07e1a4f082)
This commit is contained in:
@@ -1638,10 +1638,11 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
)
|
||||
)
|
||||
self._release_aborted_request(req)
|
||||
# Mamba insertion donates the checkpoint and clears its sequence marker.
|
||||
release_kv_cache(
|
||||
req, self.tree_cache, is_insert=not self.tree_cache.supports_mamba()
|
||||
)
|
||||
# The checkpoint above (maybe_cache_unfinished_req) already handed the
|
||||
# prefill KV to the tree; the request is not finished, so the release
|
||||
# only frees the rest. (#42194 semantics; our tree predates #41520's
|
||||
# mamba-donation rework, mamba models are not run on this branch.)
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
req.reset_for_retract()
|
||||
req.output_ids = array("q")
|
||||
req.start_send_idx = 0
|
||||
|
||||
@@ -1034,12 +1034,12 @@ class Req(ReqDllmMixin):
|
||||
else self.origin_input_ids
|
||||
) # Before image padding
|
||||
# Each decode stage's output ids. Append-only by contract:
|
||||
# _refresh_fill_ids infers how many output tokens are already in
|
||||
# refresh_fill_ids infers how many output tokens are already in
|
||||
# full_untruncated_fill_ids from lengths alone, so in-place rewrites
|
||||
# that preserve length would silently corrupt fill_ids.
|
||||
self.output_ids = array("q")
|
||||
# Full untruncated sequence: origin + output (+ DLLM mask block).
|
||||
# Kept in sync by _refresh_fill_ids; admission only updates
|
||||
# Kept in sync by refresh_fill_ids; admission only updates
|
||||
# extend_range, never mutates this array's length.
|
||||
self.full_untruncated_fill_ids = array("q")
|
||||
self.extend_range: Optional[Range] = None
|
||||
@@ -1533,7 +1533,7 @@ class Req(ReqDllmMixin):
|
||||
def get_fill_ids(self) -> array:
|
||||
return self.full_untruncated_fill_ids[: self.extend_range.end]
|
||||
|
||||
def _refresh_fill_ids(self) -> None:
|
||||
def refresh_fill_ids(self) -> None:
|
||||
"""Keep full_untruncated_fill_ids == origin_input_ids + output_ids by
|
||||
appending only the new output tokens.
|
||||
|
||||
@@ -1562,7 +1562,7 @@ class Req(ReqDllmMixin):
|
||||
self._init_fill_ids_for_dllm()
|
||||
self.determine_dllm_phase()
|
||||
else:
|
||||
self._refresh_fill_ids()
|
||||
self.refresh_fill_ids()
|
||||
|
||||
input_len = len(self.full_untruncated_fill_ids)
|
||||
|
||||
@@ -3110,7 +3110,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
for req, prefix_len in zip(
|
||||
running_batch.reqs, running_prefix_lens, strict=True
|
||||
):
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
req.set_extend_range(prefix_len, prefix_len + 1)
|
||||
|
||||
# Decode tokens of the running portion live in future_map.output_tokens_buf.
|
||||
@@ -3188,7 +3188,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# cannot stand in for it. Rows past bs are a beam tail, not requests.
|
||||
seq_lens = self.seq_lens_cpu[:bs].tolist()
|
||||
for req, seq_len in zip(self.reqs, seq_lens, strict=True):
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
# end runs one past full_untruncated_fill_ids while output_ids
|
||||
# trails; safe only while decoding_reqs suppresses cache_unfinished_req.
|
||||
req.set_extend_range(seq_len - 1, seq_len)
|
||||
|
||||
@@ -157,7 +157,7 @@ class Session:
|
||||
or carry_fill is input_ids_unpadded
|
||||
):
|
||||
# Unexpected type or aliased with an origin array (extending it
|
||||
# below would double-append): let _refresh_fill_ids rebuild.
|
||||
# below would double-append): let refresh_fill_ids rebuild.
|
||||
carry_fill = None
|
||||
else:
|
||||
del carry_fill[self.committed_fill_len :]
|
||||
|
||||
@@ -28,7 +28,7 @@ class _FakeReq:
|
||||
"""Carries the fill-id state convert_decode_to_extend touches, with the
|
||||
real Req methods so the array bookkeeping is not re-implemented here."""
|
||||
|
||||
_refresh_fill_ids = Req._refresh_fill_ids
|
||||
refresh_fill_ids = Req.refresh_fill_ids
|
||||
set_extend_range = Req.set_extend_range
|
||||
|
||||
def __init__(self, *, num_prompt_tokens: int, num_output_tokens: int):
|
||||
|
||||
@@ -132,7 +132,7 @@ class _FakeReq:
|
||||
self.full_untruncated_fill_ids = list(range(origin_len + output_len))
|
||||
self.extend_range = None
|
||||
|
||||
def _refresh_fill_ids(self):
|
||||
def refresh_fill_ids(self):
|
||||
self.full_untruncated_fill_ids = self.origin_input_ids + self.output_ids
|
||||
|
||||
@property
|
||||
|
||||
@@ -103,6 +103,19 @@ class MockReq:
|
||||
def get_fill_ids(self):
|
||||
return self.full_untruncated_fill_ids[: self.extend_range.end]
|
||||
|
||||
rid = "mock-req"
|
||||
skip_radix_cache_insert = False
|
||||
finished_reason = None
|
||||
|
||||
def finished(self):
|
||||
return self.finished_reason is not None
|
||||
|
||||
def refresh_fill_ids(self):
|
||||
pass
|
||||
|
||||
def owned_kv_len(self):
|
||||
return self.kv.kv_committed_len
|
||||
|
||||
|
||||
def _make_req(fill_ids, req_pool_idx=0, cache_protected_len=0, last_node=None):
|
||||
return MockReq(fill_ids, req_pool_idx, cache_protected_len, last_node)
|
||||
|
||||
@@ -641,6 +641,11 @@ class _GraftReq:
|
||||
def get_fill_ids(self):
|
||||
return array("q", self.fill_ids)
|
||||
|
||||
def refresh_fill_ids(self):
|
||||
pass # fill ids are fixed for the stand-in
|
||||
|
||||
def finished(self):
|
||||
return self.finished_reason is not None
|
||||
|
||||
class TestRotationGraftDecline(CustomTestCase):
|
||||
"""The overlap disagg-prefill loop plans batch t+1 before batch t's radix
|
||||
|
||||
@@ -167,7 +167,7 @@ class TestSharedPrefillAdmission(unittest.TestCase):
|
||||
with get_parallel().override(attn_dcp_size=1):
|
||||
scheduler.init_req_max_new_tokens(req)
|
||||
self.assertEqual(req.sampling_params.max_new_tokens, 1)
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
|
||||
cache = SimpleNamespace(
|
||||
sliding_window_size=page_size,
|
||||
|
||||
@@ -78,7 +78,7 @@ class TestSessionTokenShare(CustomTestCase):
|
||||
if baked is None:
|
||||
baked = len(output)
|
||||
req.output_ids.extend(output[:baked])
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
req.output_ids.extend(output[baked:])
|
||||
self.session.finish_req(req)
|
||||
|
||||
@@ -114,7 +114,7 @@ class TestSessionTokenShare(CustomTestCase):
|
||||
r2 = self._create("r2", [50, 51])
|
||||
self.assertEqual(list(r2.origin_input_ids), in1 + out1 + [50, 51])
|
||||
r2.output_ids.extend([6, 7])
|
||||
r2._refresh_fill_ids()
|
||||
r2.refresh_fill_ids()
|
||||
self.session.abort_req()
|
||||
self.assertEqual(self.session.committed_origin_len, len(in1))
|
||||
|
||||
@@ -149,7 +149,7 @@ class TestSessionTokenShare(CustomTestCase):
|
||||
# array, then output trimmed to finished_len (like _trim_overshoot)
|
||||
# before finish.
|
||||
r1.output_ids.extend([1, 2, 3, 4, 5, 6])
|
||||
r1._refresh_fill_ids()
|
||||
r1.refresh_fill_ids()
|
||||
del r1.output_ids[4:]
|
||||
self.session.finish_req(r1)
|
||||
self.assertEqual(
|
||||
@@ -161,7 +161,7 @@ class TestSessionTokenShare(CustomTestCase):
|
||||
r2 = self._create("r2", [50])
|
||||
self.assertEqual(list(r2.origin_input_ids), in1 + [1, 2, 3, 4] + [50])
|
||||
self.assertEqual(len(r2.full_untruncated_fill_ids), 0) # carry skipped
|
||||
r2._refresh_fill_ids()
|
||||
r2.refresh_fill_ids()
|
||||
self.assertEqual(list(r2.full_untruncated_fill_ids), list(r2.origin_input_ids))
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user