[PD] Recompute retracted DeepSeek-V4 requests on prefill instead of crashing decode (#43936)

This commit is contained in:
Liangsheng Yin
2026-10-12 19:32:03 -07:00
committed by GitHub
parent 07566cbf0d
commit 43e4e84731
6 changed files with 119 additions and 29 deletions
+12 -11
View File
@@ -90,7 +90,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
from sglang.srt.mem_cache.common import (
RetractionBackup,
discard_kv_cache_backup,
dsv41_dspark_needs_rebootstrap,
dsv4_needs_rebootstrap,
kv_to_page_indices,
match_kv_cache,
page_align_floor,
@@ -800,9 +800,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
if not is_retracted and not is_rebootstrap and is_unadmitted_reject(req):
self.scheduler.retire_unadmitted_request(req)
return
if is_retracted and dsv41_dspark_needs_rebootstrap(
self.token_to_kv_pool_allocator
):
if is_retracted and dsv4_needs_rebootstrap(self.token_to_kv_pool_allocator):
if req.output_ids:
req.pd_rebootstrap_forced_output_id = req.output_ids.pop()
req.pd_rebootstrap_in_progress = True
@@ -1083,9 +1081,15 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
# Deferred allocation cannot precede the recompute that makes its
# source ready. Keep the existing leader election and error path.
if decode_req.is_rebootstrap:
self.kv_manager.submit_prefill_recompute(
decode_req.kv_receiver, decode_req.req.build_rebootstrap_payload()
)
self._submit_prefill_recompute(decode_req)
def _submit_prefill_recompute(self, decode_req: DecodeRequest) -> None:
# A fake transfer has no prefill; its receiver completes once metadata is sent.
if _is_fake_transfer(decode_req.req):
return
self.kv_manager.submit_prefill_recompute(
decode_req.kv_receiver, decode_req.req.build_rebootstrap_payload()
)
def _update_handshake_waiters(
self,
@@ -1801,10 +1805,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
decode_req.is_rebootstrap
and get_disagg().disaggregation_decode_allocation_policy == "early"
):
self.kv_manager.submit_prefill_recompute(
decode_req.kv_receiver,
decode_req.req.build_rebootstrap_payload(),
)
self._submit_prefill_recompute(decode_req)
self._num_published_destinations += 1
decode_req.req.time_stats.set_decode_transfer_queue_entry_time()
+4 -4
View File
@@ -3333,15 +3333,15 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
else:
# No backup exists and the device KV is already freed, so the
# request cannot resume.
if get_disagg().disaggregation_decode_retraction_backup == "none":
if get_disagg().disaggregation_decode_retraction_backup == "host_pool":
message = "Retraction host KV pool exhausted. Aborting the request."
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
else:
message = (
"Retracted under decode memory pressure without a KV "
"backup. Retry later."
)
status_code = HTTPStatus.SERVICE_UNAVAILABLE
else:
message = "Retraction host KV pool exhausted. Aborting the request."
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
req.to_finish = FINISH_ABORT(message, status_code=status_code)
reqs_to_abort.append(req)
logger.warning(
+12 -12
View File
@@ -296,17 +296,14 @@ def _evict_until_allocatable(
return
def dsv41_dspark_needs_rebootstrap(
def dsv4_needs_rebootstrap(
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
) -> bool:
"""V4.1's request-scoped pair ring and draft KV cannot use CPU tensor backup."""
if str(get_spec().speculative_algorithm).upper() != "DSPARK":
return False
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
pool = token_to_kv_pool_allocator.get_kvcache()
return isinstance(pool, DeepSeekV4TokenToKVPool) and 2 in pool.compression_ratios
# No DeepSeek-V4 pool keeps a host copy of a request's KV; the prefill recomputes it.
return isinstance(pool, DeepSeekV4TokenToKVPool)
def backup_kv_cache(
@@ -316,15 +313,18 @@ def backup_kv_cache(
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
backend: str,
) -> bool:
"""Returns False when no backup can be taken ('none' backend, or the host
pool cannot hold it); the caller aborts the request."""
if dsv41_dspark_needs_rebootstrap(token_to_kv_pool_allocator):
# Drain the in-flight verify before its slots can receive recomputed KV.
"""Returns False when the request can be neither backed up nor recomputed;
the caller aborts it."""
if backend == "none":
return False
if dsv4_needs_rebootstrap(token_to_kv_pool_allocator):
if req.multimodal_inputs is not None:
# The recompute payload carries token ids only; images would be lost.
return False
# Drain the in-flight forward before its slots can receive recomputed KV.
device = token_to_kv_pool_allocator.get_kvcache().device
torch.get_device_module(device).synchronize(device)
return True
if backend == "none":
return False
if backend == "cpu_tensor":
req.offload_kv_cache(req_to_token_pool, token_to_kv_pool_allocator)
return True