[PD] Recompute retracted DeepSeek-V4 requests on prefill instead of crashing decode (#43936)
This commit is contained in:
@@ -90,7 +90,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
from sglang.srt.mem_cache.common import (
|
||||
RetractionBackup,
|
||||
discard_kv_cache_backup,
|
||||
dsv41_dspark_needs_rebootstrap,
|
||||
dsv4_needs_rebootstrap,
|
||||
kv_to_page_indices,
|
||||
match_kv_cache,
|
||||
page_align_floor,
|
||||
@@ -800,9 +800,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
if not is_retracted and not is_rebootstrap and is_unadmitted_reject(req):
|
||||
self.scheduler.retire_unadmitted_request(req)
|
||||
return
|
||||
if is_retracted and dsv41_dspark_needs_rebootstrap(
|
||||
self.token_to_kv_pool_allocator
|
||||
):
|
||||
if is_retracted and dsv4_needs_rebootstrap(self.token_to_kv_pool_allocator):
|
||||
if req.output_ids:
|
||||
req.pd_rebootstrap_forced_output_id = req.output_ids.pop()
|
||||
req.pd_rebootstrap_in_progress = True
|
||||
@@ -1083,9 +1081,15 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
# Deferred allocation cannot precede the recompute that makes its
|
||||
# source ready. Keep the existing leader election and error path.
|
||||
if decode_req.is_rebootstrap:
|
||||
self.kv_manager.submit_prefill_recompute(
|
||||
decode_req.kv_receiver, decode_req.req.build_rebootstrap_payload()
|
||||
)
|
||||
self._submit_prefill_recompute(decode_req)
|
||||
|
||||
def _submit_prefill_recompute(self, decode_req: DecodeRequest) -> None:
|
||||
# A fake transfer has no prefill; its receiver completes once metadata is sent.
|
||||
if _is_fake_transfer(decode_req.req):
|
||||
return
|
||||
self.kv_manager.submit_prefill_recompute(
|
||||
decode_req.kv_receiver, decode_req.req.build_rebootstrap_payload()
|
||||
)
|
||||
|
||||
def _update_handshake_waiters(
|
||||
self,
|
||||
@@ -1801,10 +1805,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
decode_req.is_rebootstrap
|
||||
and get_disagg().disaggregation_decode_allocation_policy == "early"
|
||||
):
|
||||
self.kv_manager.submit_prefill_recompute(
|
||||
decode_req.kv_receiver,
|
||||
decode_req.req.build_rebootstrap_payload(),
|
||||
)
|
||||
self._submit_prefill_recompute(decode_req)
|
||||
self._num_published_destinations += 1
|
||||
decode_req.req.time_stats.set_decode_transfer_queue_entry_time()
|
||||
|
||||
|
||||
@@ -3333,15 +3333,15 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
else:
|
||||
# No backup exists and the device KV is already freed, so the
|
||||
# request cannot resume.
|
||||
if get_disagg().disaggregation_decode_retraction_backup == "none":
|
||||
if get_disagg().disaggregation_decode_retraction_backup == "host_pool":
|
||||
message = "Retraction host KV pool exhausted. Aborting the request."
|
||||
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
else:
|
||||
message = (
|
||||
"Retracted under decode memory pressure without a KV "
|
||||
"backup. Retry later."
|
||||
)
|
||||
status_code = HTTPStatus.SERVICE_UNAVAILABLE
|
||||
else:
|
||||
message = "Retraction host KV pool exhausted. Aborting the request."
|
||||
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
req.to_finish = FINISH_ABORT(message, status_code=status_code)
|
||||
reqs_to_abort.append(req)
|
||||
logger.warning(
|
||||
|
||||
@@ -296,17 +296,14 @@ def _evict_until_allocatable(
|
||||
return
|
||||
|
||||
|
||||
def dsv41_dspark_needs_rebootstrap(
|
||||
def dsv4_needs_rebootstrap(
|
||||
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
|
||||
) -> bool:
|
||||
"""V4.1's request-scoped pair ring and draft KV cannot use CPU tensor backup."""
|
||||
if str(get_spec().speculative_algorithm).upper() != "DSPARK":
|
||||
return False
|
||||
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
|
||||
pool = token_to_kv_pool_allocator.get_kvcache()
|
||||
return isinstance(pool, DeepSeekV4TokenToKVPool) and 2 in pool.compression_ratios
|
||||
# No DeepSeek-V4 pool keeps a host copy of a request's KV; the prefill recomputes it.
|
||||
return isinstance(pool, DeepSeekV4TokenToKVPool)
|
||||
|
||||
|
||||
def backup_kv_cache(
|
||||
@@ -316,15 +313,18 @@ def backup_kv_cache(
|
||||
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
|
||||
backend: str,
|
||||
) -> bool:
|
||||
"""Returns False when no backup can be taken ('none' backend, or the host
|
||||
pool cannot hold it); the caller aborts the request."""
|
||||
if dsv41_dspark_needs_rebootstrap(token_to_kv_pool_allocator):
|
||||
# Drain the in-flight verify before its slots can receive recomputed KV.
|
||||
"""Returns False when the request can be neither backed up nor recomputed;
|
||||
the caller aborts it."""
|
||||
if backend == "none":
|
||||
return False
|
||||
if dsv4_needs_rebootstrap(token_to_kv_pool_allocator):
|
||||
if req.multimodal_inputs is not None:
|
||||
# The recompute payload carries token ids only; images would be lost.
|
||||
return False
|
||||
# Drain the in-flight forward before its slots can receive recomputed KV.
|
||||
device = token_to_kv_pool_allocator.get_kvcache().device
|
||||
torch.get_device_module(device).synchronize(device)
|
||||
return True
|
||||
if backend == "none":
|
||||
return False
|
||||
if backend == "cpu_tensor":
|
||||
req.offload_kv_cache(req_to_token_pool, token_to_kv_pool_allocator)
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user