[mem_cache] Skip the release-time insert on optimistic prefill requeue; make refresh_fill_ids public (#42194)
This commit is contained in:
@@ -1568,11 +1568,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
)
|
||||
)
|
||||
self._release_aborted_request(req)
|
||||
# Mamba insertion donates the checkpoint and clears its sequence marker.
|
||||
is_insert = (
|
||||
not uses_write_through_cache and not self.tree_cache.supports_mamba()
|
||||
)
|
||||
release_kv_cache(req, self.tree_cache, is_insert=is_insert)
|
||||
# The checkpoint above already handed the prefill KV to the tree; the
|
||||
# request is not finished, so the release only frees the rest.
|
||||
release_kv_cache(req, self.tree_cache, is_insert=False)
|
||||
req.reset_for_retract()
|
||||
req.output_ids = array("q")
|
||||
req.start_send_idx = 0
|
||||
|
||||
@@ -80,7 +80,7 @@ class ReqDllmMixin:
|
||||
return
|
||||
|
||||
if self.dllm_config.requires_separate_context_encoding:
|
||||
self._refresh_fill_ids()
|
||||
self.refresh_fill_ids()
|
||||
self.dllm_block_offset = self.seqlen
|
||||
if self.dllm_initialized:
|
||||
self.full_untruncated_fill_ids.extend([0] * self.dllm_config.block_size)
|
||||
|
||||
@@ -1065,12 +1065,12 @@ class Req(ReqDllmMixin):
|
||||
else self.origin_input_ids
|
||||
) # Before image padding
|
||||
# Each decode stage's output ids. Append-only by contract:
|
||||
# _refresh_fill_ids infers how many output tokens are already in
|
||||
# refresh_fill_ids infers how many output tokens are already in
|
||||
# full_untruncated_fill_ids from lengths alone, so in-place rewrites
|
||||
# that preserve length would silently corrupt fill_ids.
|
||||
self.output_ids = array("q")
|
||||
# Full untruncated sequence: origin + output (+ DLLM mask block).
|
||||
# Kept in sync by _refresh_fill_ids; admission only updates
|
||||
# Kept in sync by refresh_fill_ids; admission only updates
|
||||
# extend_range, never mutates this array's length.
|
||||
self.full_untruncated_fill_ids = array("q")
|
||||
self.extend_range: Optional[Range] = None
|
||||
@@ -1571,7 +1571,7 @@ class Req(ReqDllmMixin):
|
||||
def get_fill_ids(self) -> array:
|
||||
return self.full_untruncated_fill_ids[: self.extend_range.end]
|
||||
|
||||
def _refresh_fill_ids(self) -> None:
|
||||
def refresh_fill_ids(self) -> None:
|
||||
"""Keep full_untruncated_fill_ids == origin_input_ids + output_ids by
|
||||
appending only the new output tokens.
|
||||
|
||||
@@ -1600,7 +1600,7 @@ class Req(ReqDllmMixin):
|
||||
self._init_fill_ids_for_dllm()
|
||||
self.determine_dllm_phase()
|
||||
else:
|
||||
self._refresh_fill_ids()
|
||||
self.refresh_fill_ids()
|
||||
|
||||
input_len = len(self.full_untruncated_fill_ids)
|
||||
|
||||
@@ -3181,7 +3181,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
for req, prefix_len in zip(
|
||||
running_batch.reqs, running_prefix_lens, strict=True
|
||||
):
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
req.set_extend_range(prefix_len, prefix_len + 1)
|
||||
|
||||
# Decode tokens of the running portion live in future_map.output_tokens_buf.
|
||||
@@ -3259,7 +3259,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# cannot stand in for it. Rows past bs are a beam tail, not requests.
|
||||
seq_lens = self.seq_lens_cpu[:bs].tolist()
|
||||
for req, seq_len in zip(self.reqs, seq_lens, strict=True):
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
# end runs one past full_untruncated_fill_ids while output_ids
|
||||
# trails; safe only while decoding_reqs suppresses the checkpoint insert.
|
||||
req.set_extend_range(seq_len - 1, seq_len)
|
||||
|
||||
@@ -324,7 +324,7 @@ def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = Tr
|
||||
f"releasing unfinished request {req.rid} into a mamba tree"
|
||||
)
|
||||
# The fill-id array lags output_ids until the next prepare_for_decode.
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
tree_cache.insert_req(req, up_to=owned_kv_len)
|
||||
# The protected prefix is not this req's to free.
|
||||
tree_cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, owned_kv_len)])
|
||||
|
||||
@@ -157,7 +157,7 @@ class Session:
|
||||
or carry_fill is input_ids_unpadded
|
||||
):
|
||||
# Unexpected type or aliased with an origin array (extending it
|
||||
# below would double-append): let _refresh_fill_ids rebuild.
|
||||
# below would double-append): let refresh_fill_ids rebuild.
|
||||
carry_fill = None
|
||||
else:
|
||||
del carry_fill[self.committed_fill_len :]
|
||||
|
||||
@@ -8,8 +8,7 @@ def finish_req(cache, req, up_to):
|
||||
finished, insert_req, free the rest, drop the lock."""
|
||||
if getattr(req, "finished_reason", None) is None:
|
||||
req.finished_reason = FINISH_LENGTH(length=0)
|
||||
if hasattr(req, "_refresh_fill_ids"):
|
||||
req._refresh_fill_ids()
|
||||
req.refresh_fill_ids()
|
||||
cache.insert_req(req, up_to=up_to)
|
||||
cache.free_kv_row(req.kv, [(req.kv.cache_protected_len, up_to)])
|
||||
cache.unpin(req)
|
||||
|
||||
Reference in New Issue
Block a user