[PD] Add a none decode retraction backup and subclass seams in the PD queues (#41103)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: Zhiqiang Xie <zqx@meta.com>
This commit is contained in:
metamergebot
2026-09-24 16:13:54 -07:00
committed by GitHub
co-authored by metamergebot Zhiqiang Xie
parent 466e985f4b
commit 7c5bdb940c
5 changed files with 40 additions and 20 deletions
@@ -90,9 +90,11 @@ class Disagg(msgspec.Struct):
"Storage backend for KV preserved across PD decode retraction. "
"'cpu_tensor' uses per-request CPU tensors. 'host_pool' uses "
"a reserved HiCache pool and does not fall back on exhaustion. "
"'none' keeps no backup: a retracted request is aborted with "
"503 for the client to retry. "
"If omitted, the backend is inferred from the decode KV pool."
),
choices=["cpu_tensor", "host_pool"],
choices=["cpu_tensor", "host_pool", "none"],
),
] = None
num_reserved_decode_tokens: A[
+13 -2
View File
@@ -1714,7 +1714,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
self.req_to_metadata_buffer_idx_allocator.alloc()
)
assert decode_req.metadata_buffer_index is not None
page_indices = kv_to_page_indices(kv_indices, page_size).astype(np.int32)
page_indices = self._transfer_page_indices(decode_req, kv_indices, page_size)
if (
metadata_kwargs.get("destination") != KVTransferDestination.HOST
and self.transfer_queue.enable_staging
@@ -2041,6 +2041,17 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
)
return num_new_pages * page_size
def _alloc_for_decode_prealloc(
self, allocator: BaseTokenToKVPoolAllocator, **kwargs
) -> torch.Tensor:
return alloc_for_decode_prealloc(allocator, **kwargs)
def _transfer_page_indices(
self, decode_req: DecodeRequest, kv_indices: torch.Tensor, page_size: int
) -> np.ndarray:
# int32 for ZMQ serialization -- from_zmq reads np.int32.
return kv_to_page_indices(kv_indices, page_size).astype(np.int32)
def _pre_alloc(
self,
req: Req,
@@ -2149,7 +2160,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
coordinator.host_token_len(fill_len),
)
else:
kv_loc = alloc_for_decode_prealloc(
kv_loc = self._alloc_for_decode_prealloc(
allocator,
req=req,
fill_len=fill_len,
+7 -6
View File
@@ -400,19 +400,20 @@ class PrefillBootstrapQueue:
req.time_stats.set_bootstrap_done_time()
decode_prefix_len = req.disagg_kv_sender.pop_decode_prefix_len()
num_kv_indices = len(req.origin_input_ids)
req.start_send_idx = decode_prefix_len
# Base of the staging chunk grid (suffix-relative send coordinates).
req.disagg_decode_prefix_len = decode_prefix_len
num_kv_indices_to_send = num_kv_indices - decode_prefix_len
num_pages = kv_to_page_num(
num_kv_indices_to_send,
self.scheduler.token_to_kv_pool_allocator.page_size,
)
num_pages = self._num_pages_to_send(req, decode_prefix_len)
req.disagg_kv_sender.init(num_pages, req.metadata_buffer_index)
req.pending_bootstrap = False
return True
def _num_pages_to_send(self, req: Req, decode_prefix_len: int) -> int:
return kv_to_page_num(
len(req.origin_input_ids) - decode_prefix_len,
self.scheduler.token_to_kv_pool_allocator.page_size,
)
def add(self, req: Req, num_kv_heads: int) -> None:
# Rejected at intake: `set_finish_with_abort` left the verdict in
# `to_finish`, which `finished()` does not read, and swapped the prompt
+13 -9
View File
@@ -3301,17 +3301,21 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
if self.release_req(idx, len(sorted_indices)):
retracted_reqs.append(req)
else:
# The retraction host pool could not hold the backup and the
# device KV is already freed, so the request cannot resume.
req.to_finish = FINISH_ABORT(
"Retraction host KV pool exhausted. Aborting the request.",
status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
)
# No backup exists and the device KV is already freed, so the
# request cannot resume.
if get_disagg().disaggregation_decode_retraction_backup == "none":
message = (
"Retracted under decode memory pressure without a KV "
"backup. Retry later."
)
status_code = HTTPStatus.SERVICE_UNAVAILABLE
else:
message = "Retraction host KV pool exhausted. Aborting the request."
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
req.to_finish = FINISH_ABORT(message, status_code=status_code)
reqs_to_abort.append(req)
logger.warning(
"retract_decode: aborted request %s, retraction host pool "
"exhausted",
req.rid,
"retract_decode: aborted request %s: %s", req.rid, message
)
if len(sorted_indices) <= 1 and not self.check_decode_mem(
+4 -2
View File
@@ -220,13 +220,15 @@ def backup_kv_cache(
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
backend: str,
) -> bool:
"""Returns False when the host pool cannot hold the backup; the caller
aborts the request since its KV cannot be preserved."""
"""Returns False when no backup can be taken ('none' backend, or the host
pool cannot hold it); the caller aborts the request."""
if dsv41_dspark_needs_rebootstrap(token_to_kv_pool_allocator):
# Drain the in-flight verify before its slots can receive recomputed KV.
device = token_to_kv_pool_allocator.get_kvcache().device
torch.get_device_module(device).synchronize(device)
return True
if backend == "none":
return False
if backend == "cpu_tensor":
req.offload_kv_cache(req_to_token_pool, token_to_kv_pool_allocator)
return True