From 7c5bdb940cc24015aea6647c68e37e322e1d81d8 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Thu, 24 Sep 2026 16:13:54 -0700 Subject: [PATCH] [PD] Add a `none` decode retraction backup and subclass seams in the PD queues (#41103) Co-authored-by: metamergebot Co-authored-by: Zhiqiang Xie --- python/sglang/srt/arg_groups/fields/disagg.py | 4 +++- python/sglang/srt/disaggregation/decode.py | 15 +++++++++++-- python/sglang/srt/disaggregation/prefill.py | 13 ++++++----- python/sglang/srt/managers/schedule_batch.py | 22 +++++++++++-------- python/sglang/srt/mem_cache/common.py | 6 +++-- 5 files changed, 40 insertions(+), 20 deletions(-) diff --git a/python/sglang/srt/arg_groups/fields/disagg.py b/python/sglang/srt/arg_groups/fields/disagg.py index d16f10ceee..89985577dd 100644 --- a/python/sglang/srt/arg_groups/fields/disagg.py +++ b/python/sglang/srt/arg_groups/fields/disagg.py @@ -90,9 +90,11 @@ class Disagg(msgspec.Struct): "Storage backend for KV preserved across PD decode retraction. " "'cpu_tensor' uses per-request CPU tensors. 'host_pool' uses " "a reserved HiCache pool and does not fall back on exhaustion. " + "'none' keeps no backup: a retracted request is aborted with " + "503 for the client to retry. " "If omitted, the backend is inferred from the decode KV pool." ), - choices=["cpu_tensor", "host_pool"], + choices=["cpu_tensor", "host_pool", "none"], ), ] = None num_reserved_decode_tokens: A[ diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 62c584e2f6..4da8bc29b2 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -1714,7 +1714,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): self.req_to_metadata_buffer_idx_allocator.alloc() ) assert decode_req.metadata_buffer_index is not None - page_indices = kv_to_page_indices(kv_indices, page_size).astype(np.int32) + page_indices = self._transfer_page_indices(decode_req, kv_indices, page_size) if ( metadata_kwargs.get("destination") != KVTransferDestination.HOST and self.transfer_queue.enable_staging @@ -2041,6 +2041,17 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): ) return num_new_pages * page_size + def _alloc_for_decode_prealloc( + self, allocator: BaseTokenToKVPoolAllocator, **kwargs + ) -> torch.Tensor: + return alloc_for_decode_prealloc(allocator, **kwargs) + + def _transfer_page_indices( + self, decode_req: DecodeRequest, kv_indices: torch.Tensor, page_size: int + ) -> np.ndarray: + # int32 for ZMQ serialization -- from_zmq reads np.int32. + return kv_to_page_indices(kv_indices, page_size).astype(np.int32) + def _pre_alloc( self, req: Req, @@ -2149,7 +2160,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): coordinator.host_token_len(fill_len), ) else: - kv_loc = alloc_for_decode_prealloc( + kv_loc = self._alloc_for_decode_prealloc( allocator, req=req, fill_len=fill_len, diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 4019bda079..88cc5c05b9 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -400,19 +400,20 @@ class PrefillBootstrapQueue: req.time_stats.set_bootstrap_done_time() decode_prefix_len = req.disagg_kv_sender.pop_decode_prefix_len() - num_kv_indices = len(req.origin_input_ids) req.start_send_idx = decode_prefix_len # Base of the staging chunk grid (suffix-relative send coordinates). req.disagg_decode_prefix_len = decode_prefix_len - num_kv_indices_to_send = num_kv_indices - decode_prefix_len - num_pages = kv_to_page_num( - num_kv_indices_to_send, - self.scheduler.token_to_kv_pool_allocator.page_size, - ) + num_pages = self._num_pages_to_send(req, decode_prefix_len) req.disagg_kv_sender.init(num_pages, req.metadata_buffer_index) req.pending_bootstrap = False return True + def _num_pages_to_send(self, req: Req, decode_prefix_len: int) -> int: + return kv_to_page_num( + len(req.origin_input_ids) - decode_prefix_len, + self.scheduler.token_to_kv_pool_allocator.page_size, + ) + def add(self, req: Req, num_kv_heads: int) -> None: # Rejected at intake: `set_finish_with_abort` left the verdict in # `to_finish`, which `finished()` does not read, and swapped the prompt diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 105a44f970..1c6b2575c4 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -3301,17 +3301,21 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): if self.release_req(idx, len(sorted_indices)): retracted_reqs.append(req) else: - # The retraction host pool could not hold the backup and the - # device KV is already freed, so the request cannot resume. - req.to_finish = FINISH_ABORT( - "Retraction host KV pool exhausted. Aborting the request.", - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - ) + # No backup exists and the device KV is already freed, so the + # request cannot resume. + if get_disagg().disaggregation_decode_retraction_backup == "none": + message = ( + "Retracted under decode memory pressure without a KV " + "backup. Retry later." + ) + status_code = HTTPStatus.SERVICE_UNAVAILABLE + else: + message = "Retraction host KV pool exhausted. Aborting the request." + status_code = HTTPStatus.INTERNAL_SERVER_ERROR + req.to_finish = FINISH_ABORT(message, status_code=status_code) reqs_to_abort.append(req) logger.warning( - "retract_decode: aborted request %s, retraction host pool " - "exhausted", - req.rid, + "retract_decode: aborted request %s: %s", req.rid, message ) if len(sorted_indices) <= 1 and not self.check_decode_mem( diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 2d4d2c2b19..0c628fec9d 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -220,13 +220,15 @@ def backup_kv_cache( token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator, backend: str, ) -> bool: - """Returns False when the host pool cannot hold the backup; the caller - aborts the request since its KV cannot be preserved.""" + """Returns False when no backup can be taken ('none' backend, or the host + pool cannot hold it); the caller aborts the request.""" if dsv41_dspark_needs_rebootstrap(token_to_kv_pool_allocator): # Drain the in-flight verify before its slots can receive recomputed KV. device = token_to_kv_pool_allocator.get_kvcache().device torch.get_device_module(device).synchronize(device) return True + if backend == "none": + return False if backend == "cpu_tensor": req.offload_kv_cache(req_to_token_pool, token_to_kv_pool_allocator) return True