[PD] Receive decode KV into separate paged SWA host pools; let host-capable receivers own abort drain tracking (#43410)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: cctry <cctry@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-09 19:33:06 -07:00
committed by GitHub
co-authored by Hanming Lu cctry
parent 2ff3e6dce1
commit a338b9858a
6 changed files with 313 additions and 42 deletions
@@ -178,5 +178,5 @@ class Disagg(msgspec.Struct):
sm_group_num: A[int, "Number of sm partition groups."] = 8
disaggregation_decode_host_receive_threshold: A[
float,
"Device token usage fraction at which incoming KV is received in the decode retraction host pool, excluding evictable cache pages. Range [0, 1]; 1 disables host receive (default), 0 always uses host receive. Size with --hicache-size or --hicache-ratio; requires dense MHA and a transfer backend that supports host destinations. No built-in backend currently supports this.",
"Device token usage fraction at which incoming KV is received in the decode retraction host pool, excluding evictable cache pages. Range [0, 1]; 1 disables host receive (default), 0 always uses host receive. Size with --hicache-size or --hicache-ratio; requires dense NHD MHA buffers (including separate paged SWA pools) and a transfer backend that supports host destinations. No built-in backend currently supports this.",
] = 1.0
@@ -61,6 +61,9 @@ class KVArgs:
host_kv_data_ptrs: Optional[List[int]] = None
host_kv_data_lens: Optional[List[int]] = None
host_kv_item_lens: Optional[List[int]] = None
host_state_data_ptrs: Optional[List[List[int]]] = None
host_state_data_lens: Optional[List[List[int]]] = None
host_state_item_lens: Optional[List[List[int]]] = None
aux_data_ptrs: List[int]
aux_data_lens: List[int]
aux_item_lens: List[int]
@@ -232,6 +235,14 @@ class BaseKVReceiver(ABC):
"""Whether this receiver's peer and layout support host KV destinations."""
return False
def is_abort_release_safe(self) -> bool:
"""Whether all writes to an aborted host destination have drained.
Host-capable backends must retain this state until ``clear()``. A failed
poll or elapsed timeout alone does not prove the buffers can be reused.
"""
raise NotImplementedError
@abstractmethod
def __init__(
self,
+138 -29
View File
@@ -659,12 +659,12 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
if get_disagg().disaggregation_decode_host_receive_threshold < 1:
pool = self.token_to_kv_pool
group = self.tree_cache.host_pool_group
if kv_args.state_types or any(
spec.indices_from_pool != PoolName.KV
if (kv_args.state_types and kv_args.state_types != [StateType.SWA]) or any(
spec.indices_from_pool not in (PoolName.KV, PoolName.SWA)
for spec in self.tree_cache.sidecar_pool_specs
):
raise ValueError(
"Host receive requires KV pools sharing the primary indices"
"Host receive supports full KV and separate paged SWA pools"
)
self.host_pool = group.get_pool(PoolName.KV)
self.host_reserved_tokens = decode_retraction_max_tokens(
@@ -675,12 +675,66 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
"Host pool must hold a retraction and a receive page; "
"increase --hicache-size or --hicache-ratio"
)
_, host_buffers = group.get_contiguous_buf_infos()
def pool_names(source):
return {source} | {
spec.pool_name
for spec in self.tree_cache.sidecar_pool_specs
if spec.indices_from_pool == source
}
device_buffers, host_buffers = group.get_contiguous_buf_infos(
pool_names(PoolName.KV)
)
if device_buffers != (
kv_args.kv_data_ptrs,
kv_args.kv_data_lens,
kv_args.kv_item_lens,
):
raise ValueError(
"Host receive buffers must match the KV transfer layout"
)
(
kv_args.host_kv_data_ptrs,
kv_args.host_kv_data_lens,
kv_args.host_kv_item_lens,
) = host_buffers
self.host_swa_pool = None
self.host_swa_reserved_tokens = 0
if kv_args.state_types:
self.host_swa_pool = group.get_pool(PoolName.SWA)
self.host_swa_reserved_tokens = min(
decode_retraction_max_tokens(
self.req_to_token_pool, pool.swa_kv_pool
),
ceil_align(
self.scheduler.sliding_window_size + pool.page_size - 1,
pool.page_size,
),
)
if self.host_swa_pool.logical_size < (
self.host_swa_reserved_tokens + pool.page_size
):
raise ValueError(
"SWA host pool must hold a retraction and a receive page; "
"increase --hicache-size or --hicache-ratio"
)
device_state, host_state = group.get_contiguous_buf_infos(
pool_names(PoolName.SWA)
)
if device_state != (
kv_args.state_data_ptrs[0],
kv_args.state_data_lens[0],
kv_args.state_item_lens[0],
):
raise ValueError(
"Host receive buffers must match the SWA transfer layout"
)
(
kv_args.host_state_data_ptrs,
kv_args.host_state_data_lens,
kv_args.host_state_item_lens,
) = ([values] for values in host_state)
kv_args.ib_device = get_disagg().disaggregation_ib_device
kv_args.gpu_id = get_device().gpu_id
@@ -1755,9 +1809,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
decode_req.req.time_stats.set_decode_transfer_queue_entry_time()
def _pre_alloc_host(self, decode_req: DecodeRequest) -> bool:
num_tokens = ceil_align(
len(decode_req.req.origin_input_ids), self.host_pool.page_size
)
seq_len = len(decode_req.req.origin_input_ids)
page_size = self.host_pool.page_size
num_tokens = ceil_align(seq_len, page_size)
required_tokens = num_tokens + self.host_reserved_tokens
if (
self.host_pool.available_size() < required_tokens
@@ -1766,17 +1820,43 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
self.tree_cache.evict_host(required_tokens)
if self.host_pool.available_size() < required_tokens:
return False
swa_num_tokens = 0
if self.host_swa_pool is not None:
window_start = page_align_floor(
max(0, seq_len - self.scheduler.sliding_window_size), page_size
)
swa_num_tokens = num_tokens - window_start
required_swa_tokens = swa_num_tokens + self.host_swa_reserved_tokens
if self.host_swa_pool.available_size() < required_swa_tokens:
entry = self.tree_cache.host_pool_group.get_entry(PoolName.SWA)
if entry.host_evict_fn is not None:
entry.host_evict_fn(required_swa_tokens)
if self.host_swa_pool.available_size() < required_swa_tokens:
return False
host_indices = self.host_pool.alloc(num_tokens)
if host_indices is None:
return False
indices_by_pool = {PoolName.KV: host_indices}
transfers = []
state_indices = None
if self.host_swa_pool is not None:
swa_host_indices = self.host_swa_pool.alloc(swa_num_tokens)
if swa_host_indices is None:
self.host_pool.free(host_indices)
return False
indices_by_pool[PoolName.SWA] = swa_host_indices
transfers.append(
PoolTransfer(name=PoolName.SWA, host_indices=swa_host_indices)
)
state_indices = [kv_to_page_indices(swa_host_indices, page_size)]
assert decode_req.req.kv.retraction_backup is None
# All sidecars share the primary KV indices.
decode_req.req.kv.retraction_backup = RetractionBackup(
host_indices=host_indices,
pool_transfers=[
pool_transfers=transfers
+ [
PoolTransfer(
name=spec.pool_name,
host_indices=host_indices,
host_indices=indices_by_pool[spec.indices_from_pool],
indices_from_pool=spec.indices_from_pool,
hit_policy=spec.hit_policy,
)
@@ -1791,6 +1871,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
decode_req,
host_indices,
self.token_to_kv_pool_allocator.page_size,
state_indices,
decode_prefix_len=0,
destination=KVTransferDestination.HOST,
)
@@ -1815,13 +1896,29 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
0,
retractable_tokens,
)
swa_required_tokens = 0
swa_allocatable_tokens = None
if self._uses_swa_reservation():
_, swa_required_tokens = self._prealloc_required_tokens(req)
full_allocatable_tokens, swa_allocatable_tokens = (
self._swa_aware_allocatable_token_budgets(
retractable_tokens=retractable_tokens,
retractable_swa_tokens=sum(
self._swa_retractable_len(r)
for r in self.scheduler.running_batch.reqs
),
count_retracted=True,
)
)
else:
full_allocatable_tokens = self._allocatable_token_budgets(
retractable_tokens=retractable_tokens, count_retracted=True
)
if not self._prealloc_reservation_fits(
required_tokens,
0,
full_allocatable_tokens=self._allocatable_token_budgets(
retractable_tokens=retractable_tokens, count_retracted=True
),
swa_allocatable_tokens=None,
swa_required_tokens,
full_allocatable_tokens=full_allocatable_tokens,
swa_allocatable_tokens=swa_allocatable_tokens,
):
return False
self._pre_alloc(req)
@@ -2611,6 +2708,14 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
return
release_kv_cache(decode_req.req, self.tree_cache, checkpoint=False)
def _record_transfer_failure(self, failure_reason: str) -> None:
if self.scheduler.metrics_reporter.enable_metrics:
self.scheduler.metrics_collector.increment_transfer_failed_reqs()
def _abort_failed_transfer(self, decode_req: DecodeRequest) -> None:
if self.enable_host_receive and decode_req.host_staged:
decode_req.kv_receiver.abort()
def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]:
if not self.queue:
return []
@@ -2643,17 +2748,20 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
poll == KVPoll.Failed
or hicache_restore_status == HiCacheRestoreResult.FAILED
):
self._abort_failed_transfer(decode_req)
error_message = (
f"Decode transfer failed for request rank={self.tp_rank} "
f"{decode_req.req.rid=} {decode_req.req.bootstrap_room=}"
)
is_propagated = False
failure_reason = str(hicache_restore_status)
if poll == KVPoll.Failed:
try:
decode_req.kv_receiver.failure_exception()
except Exception as e:
error_message += f" with exception {e}"
is_propagated = getattr(e, "is_from_another_rank", False)
failure_reason = getattr(e, "failure_reason", str(e))
self._clean_hicache_prefetch_resources(decode_req)
# Mute error message for propagated exceptions to avoid duplicate logging
if is_propagated:
@@ -2674,8 +2782,6 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
requires_host_drain = (
self.enable_host_receive and decode_req.host_staged
)
if requires_host_drain:
decode_req.kv_receiver.abort()
deferrable = (
self.enable_deferred_kv_release
and decode_req.kv_receiver.kv_mgr.enable_deferred_decode_kv_release
@@ -2704,8 +2810,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
decode_req.kv_receiver.clear()
decode_req.kv_receiver = None
indices_to_remove.add(i)
if self.scheduler.metrics_reporter.enable_metrics:
self.scheduler.metrics_collector.increment_transfer_failed_reqs()
self._record_transfer_failure(failure_reason)
continue
elif poll == KVPoll.Success:
if self.enable_host_receive and decode_req.host_staged:
@@ -2741,8 +2846,9 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
)
self._clean_hicache_prefetch_resources(decode_req)
self._release_request(decode_req)
if self.scheduler.metrics_reporter.enable_metrics:
self.scheduler.metrics_collector.increment_transfer_failed_reqs()
self._record_transfer_failure(
decode_req.req.finished_reason.message
)
else:
transferred_reqs.append(decode_req.req)
elif poll in [
@@ -2780,9 +2886,13 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
def _defer_release(self, decode_req: DecodeRequest) -> None:
start_time = time.monotonic()
deadline = start_time + self.deferred_kv_release_timeout
# Require an ack from every notified prefill rank (dummy-proof). Snapshot
# now -- the receiver may be cleared by resolve time.
required_acks = len(decode_req.kv_receiver.bootstrap_infos)
# Host-capable receivers own their drain tracking. Device destinations
# snapshot the number of prefill acknowledgements required by the manager.
required_acks = (
0
if self.enable_host_receive and decode_req.host_staged
else len(decode_req.kv_receiver.bootstrap_infos)
)
self._deferred_releases.append(
(
decode_req,
@@ -2801,7 +2911,8 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
self._release_request(decode_req)
self.metadata_buffers.bootstrap_room[idx] = 0
self.req_to_metadata_buffer_idx_allocator.free(idx)
decode_req.kv_receiver.kv_mgr.clear_deferred_abort_state(room)
if not (self.enable_host_receive and decode_req.host_staged):
decode_req.kv_receiver.kv_mgr.clear_deferred_abort_state(room)
decode_req.kv_receiver.clear()
decode_req.kv_receiver = None
@@ -2826,10 +2937,8 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
return
host_ready = torch.tensor(
[
req.kv_receiver.kv_mgr.is_abort_release_safe(
req.req.bootstrap_room, acks
)
for req, _, _, _, acks in self._deferred_releases
req.kv_receiver.is_abort_release_safe()
for req, _, _, _, _ in self._deferred_releases
if self.enable_host_receive and req.host_staged
],
dtype=torch.int32,
+18 -4
View File
@@ -105,16 +105,18 @@ class HostPoolGroup:
element_sizes.add(element_size)
return element_sizes.pop() if len(element_sizes) == 1 else None
def get_contiguous_buf_infos(self):
def get_contiguous_buf_infos(self, pool_names: set[PoolName] | None = None):
"""Return (device_buffers, host_buffers), each (ptrs, sizes, item_sizes)."""
host_by_device_ptr = {}
device_infos = ([], [], [])
for entry in self.entries:
if pool_names is not None and entry.name not in pool_names:
continue
host = entry.host_pool
pools = (entry.device_pool, *entry.packed_draft_device_pools)
for pool in pools:
dense_mha = (
type(pool) is MHATokenToKVPool
isinstance(pool, MHATokenToKVPool)
and pool.kv_cache_layout == "nhd"
and pool.v_head_dim == pool.head_dim
)
@@ -127,9 +129,21 @@ class HostPoolGroup:
"Host receive requires dense NHD MHA target and "
"draft KV with matching page sizes"
)
for combined, values in zip(
device_infos, pool.get_contiguous_buf_infos(), strict=True
buffers = pool.k_buffer + pool.v_buffer
infos = pool.get_contiguous_buf_infos()
expected = (
[buffer.data_ptr() for buffer in buffers],
[buffer.nbytes for buffer in buffers],
[buffer[0].nbytes * self.page_size for buffer in buffers],
)
if infos != expected or any(
not buffer.is_contiguous() for buffer in buffers
):
raise ValueError(
"Host receive requires contiguous per-layer KV buffers "
"without embedded state"
)
for combined, values in zip(device_infos, infos, strict=True):
combined.extend(values)
# Packed MHA stores target/draft K followed by target/draft V,