Fix AssertionError crash in disagg prefill inflight queue with PP (#20686)
This commit is contained in:
@@ -569,7 +569,20 @@ class SchedulerDisaggregationPrefillMixin:
|
|||||||
undone_reqs.append(req)
|
undone_reqs.append(req)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
assert poll == KVPoll.Success or poll == KVPoll.Failed
|
# In PP mode, the previous rank may have reached a terminal
|
||||||
|
# state (Success/Failed) while this rank's local poll is still
|
||||||
|
# in a transient state due to clock skew or propagation delay.
|
||||||
|
# Treat non-terminal states as undone instead of crashing.
|
||||||
|
if poll not in (
|
||||||
|
KVPoll.Success,
|
||||||
|
KVPoll.Failed,
|
||||||
|
):
|
||||||
|
logger.warning(
|
||||||
|
f"PP rank {self.pp_rank}: unexpected poll state {poll} for rid {req.rid} "
|
||||||
|
f"from consensus; treating as undone"
|
||||||
|
)
|
||||||
|
undone_reqs.append(req)
|
||||||
|
continue
|
||||||
|
|
||||||
if poll in [KVPoll.WaitingForInput, KVPoll.Transferring]:
|
if poll in [KVPoll.WaitingForInput, KVPoll.Transferring]:
|
||||||
undone_reqs.append(req)
|
undone_reqs.append(req)
|
||||||
@@ -597,7 +610,11 @@ class SchedulerDisaggregationPrefillMixin:
|
|||||||
if self.enable_metrics:
|
if self.enable_metrics:
|
||||||
self.metrics_collector.increment_transfer_failed_reqs()
|
self.metrics_collector.increment_transfer_failed_reqs()
|
||||||
else:
|
else:
|
||||||
assert False, f"Unexpected polling state {poll=}"
|
logger.warning(
|
||||||
|
f"Unexpected polling state {poll} for rid {req.rid} in inflight queue; "
|
||||||
|
f"treating as undone"
|
||||||
|
)
|
||||||
|
undone_reqs.append(req)
|
||||||
|
|
||||||
for req in done_reqs:
|
for req in done_reqs:
|
||||||
req.time_stats.set_completion_time()
|
req.time_stats.set_completion_time()
|
||||||
|
|||||||
Reference in New Issue
Block a user