[PD] Add a none decode retraction backup and subclass seams in the PD queues (#41103)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: Zhiqiang Xie <zqx@meta.com>
This commit is contained in:
co-authored by
metamergebot
Zhiqiang Xie
parent
466e985f4b
commit
7c5bdb940c
@@ -90,9 +90,11 @@ class Disagg(msgspec.Struct):
|
||||
"Storage backend for KV preserved across PD decode retraction. "
|
||||
"'cpu_tensor' uses per-request CPU tensors. 'host_pool' uses "
|
||||
"a reserved HiCache pool and does not fall back on exhaustion. "
|
||||
"'none' keeps no backup: a retracted request is aborted with "
|
||||
"503 for the client to retry. "
|
||||
"If omitted, the backend is inferred from the decode KV pool."
|
||||
),
|
||||
choices=["cpu_tensor", "host_pool"],
|
||||
choices=["cpu_tensor", "host_pool", "none"],
|
||||
),
|
||||
] = None
|
||||
num_reserved_decode_tokens: A[
|
||||
|
||||
@@ -1714,7 +1714,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
self.req_to_metadata_buffer_idx_allocator.alloc()
|
||||
)
|
||||
assert decode_req.metadata_buffer_index is not None
|
||||
page_indices = kv_to_page_indices(kv_indices, page_size).astype(np.int32)
|
||||
page_indices = self._transfer_page_indices(decode_req, kv_indices, page_size)
|
||||
if (
|
||||
metadata_kwargs.get("destination") != KVTransferDestination.HOST
|
||||
and self.transfer_queue.enable_staging
|
||||
@@ -2041,6 +2041,17 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
)
|
||||
return num_new_pages * page_size
|
||||
|
||||
def _alloc_for_decode_prealloc(
|
||||
self, allocator: BaseTokenToKVPoolAllocator, **kwargs
|
||||
) -> torch.Tensor:
|
||||
return alloc_for_decode_prealloc(allocator, **kwargs)
|
||||
|
||||
def _transfer_page_indices(
|
||||
self, decode_req: DecodeRequest, kv_indices: torch.Tensor, page_size: int
|
||||
) -> np.ndarray:
|
||||
# int32 for ZMQ serialization -- from_zmq reads np.int32.
|
||||
return kv_to_page_indices(kv_indices, page_size).astype(np.int32)
|
||||
|
||||
def _pre_alloc(
|
||||
self,
|
||||
req: Req,
|
||||
@@ -2149,7 +2160,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
coordinator.host_token_len(fill_len),
|
||||
)
|
||||
else:
|
||||
kv_loc = alloc_for_decode_prealloc(
|
||||
kv_loc = self._alloc_for_decode_prealloc(
|
||||
allocator,
|
||||
req=req,
|
||||
fill_len=fill_len,
|
||||
|
||||
@@ -400,19 +400,20 @@ class PrefillBootstrapQueue:
|
||||
|
||||
req.time_stats.set_bootstrap_done_time()
|
||||
decode_prefix_len = req.disagg_kv_sender.pop_decode_prefix_len()
|
||||
num_kv_indices = len(req.origin_input_ids)
|
||||
req.start_send_idx = decode_prefix_len
|
||||
# Base of the staging chunk grid (suffix-relative send coordinates).
|
||||
req.disagg_decode_prefix_len = decode_prefix_len
|
||||
num_kv_indices_to_send = num_kv_indices - decode_prefix_len
|
||||
num_pages = kv_to_page_num(
|
||||
num_kv_indices_to_send,
|
||||
self.scheduler.token_to_kv_pool_allocator.page_size,
|
||||
)
|
||||
num_pages = self._num_pages_to_send(req, decode_prefix_len)
|
||||
req.disagg_kv_sender.init(num_pages, req.metadata_buffer_index)
|
||||
req.pending_bootstrap = False
|
||||
return True
|
||||
|
||||
def _num_pages_to_send(self, req: Req, decode_prefix_len: int) -> int:
|
||||
return kv_to_page_num(
|
||||
len(req.origin_input_ids) - decode_prefix_len,
|
||||
self.scheduler.token_to_kv_pool_allocator.page_size,
|
||||
)
|
||||
|
||||
def add(self, req: Req, num_kv_heads: int) -> None:
|
||||
# Rejected at intake: `set_finish_with_abort` left the verdict in
|
||||
# `to_finish`, which `finished()` does not read, and swapped the prompt
|
||||
|
||||
@@ -3301,17 +3301,21 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
if self.release_req(idx, len(sorted_indices)):
|
||||
retracted_reqs.append(req)
|
||||
else:
|
||||
# The retraction host pool could not hold the backup and the
|
||||
# device KV is already freed, so the request cannot resume.
|
||||
req.to_finish = FINISH_ABORT(
|
||||
"Retraction host KV pool exhausted. Aborting the request.",
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
# No backup exists and the device KV is already freed, so the
|
||||
# request cannot resume.
|
||||
if get_disagg().disaggregation_decode_retraction_backup == "none":
|
||||
message = (
|
||||
"Retracted under decode memory pressure without a KV "
|
||||
"backup. Retry later."
|
||||
)
|
||||
status_code = HTTPStatus.SERVICE_UNAVAILABLE
|
||||
else:
|
||||
message = "Retraction host KV pool exhausted. Aborting the request."
|
||||
status_code = HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
req.to_finish = FINISH_ABORT(message, status_code=status_code)
|
||||
reqs_to_abort.append(req)
|
||||
logger.warning(
|
||||
"retract_decode: aborted request %s, retraction host pool "
|
||||
"exhausted",
|
||||
req.rid,
|
||||
"retract_decode: aborted request %s: %s", req.rid, message
|
||||
)
|
||||
|
||||
if len(sorted_indices) <= 1 and not self.check_decode_mem(
|
||||
|
||||
@@ -220,13 +220,15 @@ def backup_kv_cache(
|
||||
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
|
||||
backend: str,
|
||||
) -> bool:
|
||||
"""Returns False when the host pool cannot hold the backup; the caller
|
||||
aborts the request since its KV cannot be preserved."""
|
||||
"""Returns False when no backup can be taken ('none' backend, or the host
|
||||
pool cannot hold it); the caller aborts the request."""
|
||||
if dsv41_dspark_needs_rebootstrap(token_to_kv_pool_allocator):
|
||||
# Drain the in-flight verify before its slots can receive recomputed KV.
|
||||
device = token_to_kv_pool_allocator.get_kvcache().device
|
||||
torch.get_device_module(device).synchronize(device)
|
||||
return True
|
||||
if backend == "none":
|
||||
return False
|
||||
if backend == "cpu_tensor":
|
||||
req.offload_kv_cache(req_to_token_pool, token_to_kv_pool_allocator)
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user