Fix the misnamed request finish-check method to reflect its mutating semantics (#25725)
This commit is contained in:
@@ -124,7 +124,7 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
|||||||
# Grammar accept_token can raise ValueError if the token is not in the grammar.
|
# Grammar accept_token can raise ValueError if the token is not in the grammar.
|
||||||
# This can happen if the grammar is not set correctly or the token is invalid.
|
# This can happen if the grammar is not set correctly or the token is invalid.
|
||||||
# Use to_finish (not finished_reason) so that process_batch_result_prebuilt
|
# Use to_finish (not finished_reason) so that process_batch_result_prebuilt
|
||||||
# handles the release via check_finished -> release_kv_cache in one place.
|
# handles the release via update_finish_state -> release_kv_cache in one place.
|
||||||
error_message = f"Grammar accept_token failed for req {req.rid} with token {req.output_ids[-1]}: {e}"
|
error_message = f"Grammar accept_token failed for req {req.rid} with token {req.output_ids[-1]}: {e}"
|
||||||
req.to_finish = FINISH_ABORT(
|
req.to_finish = FINISH_ABORT(
|
||||||
error_message, HTTPStatus.INTERNAL_SERVER_ERROR
|
error_message, HTTPStatus.INTERNAL_SERVER_ERROR
|
||||||
|
|||||||
@@ -83,7 +83,7 @@ class SchedulerDllmMixin:
|
|||||||
self.metrics_reporter.num_generated_tokens += new_tokens
|
self.metrics_reporter.num_generated_tokens += new_tokens
|
||||||
|
|
||||||
req.output_ids.extend(next_token_ids)
|
req.output_ids.extend(next_token_ids)
|
||||||
req.check_finished(new_accepted_len=new_tokens)
|
req.update_finish_state(new_accepted_len=new_tokens)
|
||||||
|
|
||||||
if req.finished():
|
if req.finished():
|
||||||
release_kv_cache(req, self.tree_cache)
|
release_kv_cache(req, self.tree_cache)
|
||||||
|
|||||||
@@ -1222,7 +1222,7 @@ class Req(ReqDllmMixin):
|
|||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def check_finished(self, new_accepted_len: int = 1):
|
def update_finish_state(self, new_accepted_len: int = 1):
|
||||||
if self.finished():
|
if self.finished():
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|||||||
@@ -960,7 +960,7 @@ class PrefillAdder:
|
|||||||
priority_sign = 1 if server_args.schedule_low_priority_values_first else -1
|
priority_sign = 1 if server_args.schedule_low_priority_values_first else -1
|
||||||
|
|
||||||
# NOTE: A request finishes in two phases:
|
# NOTE: A request finishes in two phases:
|
||||||
# 1) check_finished + release_kv_cache (in process_batch_result)
|
# 1) update_finish_state + release_kv_cache (in process_batch_result)
|
||||||
# 2) filter out of batch (in get_next_batch_to_run / update_running_batch)
|
# 2) filter out of batch (in get_next_batch_to_run / update_running_batch)
|
||||||
# Preemption runs between these two phases (inside get_new_batch_prefill),
|
# Preemption runs between these two phases (inside get_new_batch_prefill),
|
||||||
# so running_batch may still contain requests whose KV cache is already freed.
|
# so running_batch may still contain requests whose KV cache is already freed.
|
||||||
|
|||||||
@@ -86,7 +86,7 @@ class SchedulerBatchResultProcessor:
|
|||||||
self.token_to_kv_pool_allocator.free_group_begin()
|
self.token_to_kv_pool_allocator.free_group_begin()
|
||||||
for req in batch.reqs:
|
for req in batch.reqs:
|
||||||
req.time_stats.set_decode_prebuilt_finish_time()
|
req.time_stats.set_decode_prebuilt_finish_time()
|
||||||
req.check_finished()
|
req.update_finish_state()
|
||||||
if req.finished():
|
if req.finished():
|
||||||
req.time_stats.set_quick_finish_time()
|
req.time_stats.set_quick_finish_time()
|
||||||
if self.server_args.enable_hisparse:
|
if self.server_args.enable_hisparse:
|
||||||
@@ -223,7 +223,7 @@ class SchedulerBatchResultProcessor:
|
|||||||
|
|
||||||
self._maybe_update_reasoning_tokens(req, next_token_id)
|
self._maybe_update_reasoning_tokens(req, next_token_id)
|
||||||
|
|
||||||
req.check_finished()
|
req.update_finish_state()
|
||||||
if req.finished():
|
if req.finished():
|
||||||
self._maybe_collect_routed_experts(req)
|
self._maybe_collect_routed_experts(req)
|
||||||
self._maybe_collect_indexer_topk(req)
|
self._maybe_collect_indexer_topk(req)
|
||||||
@@ -308,7 +308,7 @@ class SchedulerBatchResultProcessor:
|
|||||||
req.time_stats.set_prefill_finished_time()
|
req.time_stats.set_prefill_finished_time()
|
||||||
# Dummy output token for embedding models
|
# Dummy output token for embedding models
|
||||||
req.output_ids.append(0)
|
req.output_ids.append(0)
|
||||||
req.check_finished()
|
req.update_finish_state()
|
||||||
|
|
||||||
if req.finished():
|
if req.finished():
|
||||||
release_kv_cache(req, self.tree_cache)
|
release_kv_cache(req, self.tree_cache)
|
||||||
@@ -578,7 +578,7 @@ class SchedulerBatchResultProcessor:
|
|||||||
|
|
||||||
self.token_to_kv_pool_allocator.free_group_begin()
|
self.token_to_kv_pool_allocator.free_group_begin()
|
||||||
|
|
||||||
# Spec V1 handles output_ids, check_finished, grammar, and reasoning tokens
|
# Spec V1 handles output_ids, update_finish_state, grammar, and reasoning tokens
|
||||||
# in the verify phase. Non-spec and V2 handle them here in post-processing.
|
# in the verify phase. Non-spec and V2 handle them here in post-processing.
|
||||||
is_spec_v1 = not batch.spec_algorithm.is_none() and not batch.is_spec_v2
|
is_spec_v1 = not batch.spec_algorithm.is_none() and not batch.is_spec_v2
|
||||||
|
|
||||||
@@ -618,7 +618,7 @@ class SchedulerBatchResultProcessor:
|
|||||||
# Update Mamba last track seqlen
|
# Update Mamba last track seqlen
|
||||||
self._mamba_prefix_cache_update(req, batch, result, i)
|
self._mamba_prefix_cache_update(req, batch, result, i)
|
||||||
req.time_stats.set_last_decode_finish_time()
|
req.time_stats.set_last_decode_finish_time()
|
||||||
req.check_finished(new_accepted_len)
|
req.update_finish_state(new_accepted_len)
|
||||||
|
|
||||||
self._handle_finished_req(req, i, logits_output)
|
self._handle_finished_req(req, i, logits_output)
|
||||||
|
|
||||||
@@ -686,7 +686,7 @@ class SchedulerBatchResultProcessor:
|
|||||||
v.tolist()
|
v.tolist()
|
||||||
for v in logits_output.next_token_token_ids_logprobs_val
|
for v in logits_output.next_token_token_ids_logprobs_val
|
||||||
]
|
]
|
||||||
# else: Spec V1 — output_ids, check_finished, grammar, and reasoning tokens
|
# else: Spec V1 — output_ids, update_finish_state, grammar, and reasoning tokens
|
||||||
# are already handled in the verify phase (eagle_info.py / ngram_info.py).
|
# are already handled in the verify phase (eagle_info.py / ngram_info.py).
|
||||||
return next_token_ids, next_token_logprobs
|
return next_token_ids, next_token_logprobs
|
||||||
|
|
||||||
|
|||||||
@@ -403,7 +403,7 @@ class DFlashVerifyInput(SpecInput):
|
|||||||
token_id = int(token_id)
|
token_id = int(token_id)
|
||||||
req.output_ids.append(token_id)
|
req.output_ids.append(token_id)
|
||||||
appended += 1
|
appended += 1
|
||||||
req.check_finished()
|
req.update_finish_state()
|
||||||
if req.finished():
|
if req.finished():
|
||||||
break
|
break
|
||||||
if req.grammar is not None:
|
if req.grammar is not None:
|
||||||
|
|||||||
@@ -438,7 +438,7 @@ class EagleVerifyInput(SpecInput, EagleVerifyInputV2Mixin):
|
|||||||
req.output_ids.append(id)
|
req.output_ids.append(id)
|
||||||
if req.require_reasoning and think_end_id is not None:
|
if req.require_reasoning and think_end_id is not None:
|
||||||
req.update_reasoning_tokens(id, think_end_id)
|
req.update_reasoning_tokens(id, think_end_id)
|
||||||
req.check_finished()
|
req.update_finish_state()
|
||||||
if not req.finished() and req.grammar is not None:
|
if not req.finished() and req.grammar is not None:
|
||||||
try:
|
try:
|
||||||
req.grammar.accept_token(id)
|
req.grammar.accept_token(id)
|
||||||
@@ -447,7 +447,7 @@ class EagleVerifyInput(SpecInput, EagleVerifyInputV2Mixin):
|
|||||||
f"{i=}, {req=}\n" f"{accept_index=}\n" f"{predict=}\n"
|
f"{i=}, {req=}\n" f"{accept_index=}\n" f"{predict=}\n"
|
||||||
)
|
)
|
||||||
raise e
|
raise e
|
||||||
req.check_finished()
|
req.update_finish_state()
|
||||||
if req.finished():
|
if req.finished():
|
||||||
has_finished = True
|
has_finished = True
|
||||||
# set all tokens after finished token to -1 and break
|
# set all tokens after finished token to -1 and break
|
||||||
|
|||||||
@@ -172,7 +172,7 @@ class NgramVerifyInput(SpecInput):
|
|||||||
req.output_ids.append(id)
|
req.output_ids.append(id)
|
||||||
if req.require_reasoning and think_end_id is not None:
|
if req.require_reasoning and think_end_id is not None:
|
||||||
req.update_reasoning_tokens(id, think_end_id)
|
req.update_reasoning_tokens(id, think_end_id)
|
||||||
req.check_finished()
|
req.update_finish_state()
|
||||||
if req.finished():
|
if req.finished():
|
||||||
has_finished = True
|
has_finished = True
|
||||||
# set all tokens after finished token to -1 and break
|
# set all tokens after finished token to -1 and break
|
||||||
|
|||||||
Reference in New Issue
Block a user