[PD] Receive decode KV into separate paged SWA host pools; let host-capable receivers own abort drain tracking (#43410)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: cctry <cctry@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
cctry
parent
2ff3e6dce1
commit
a338b9858a
@@ -178,5 +178,5 @@ class Disagg(msgspec.Struct):
|
||||
sm_group_num: A[int, "Number of sm partition groups."] = 8
|
||||
disaggregation_decode_host_receive_threshold: A[
|
||||
float,
|
||||
"Device token usage fraction at which incoming KV is received in the decode retraction host pool, excluding evictable cache pages. Range [0, 1]; 1 disables host receive (default), 0 always uses host receive. Size with --hicache-size or --hicache-ratio; requires dense MHA and a transfer backend that supports host destinations. No built-in backend currently supports this.",
|
||||
"Device token usage fraction at which incoming KV is received in the decode retraction host pool, excluding evictable cache pages. Range [0, 1]; 1 disables host receive (default), 0 always uses host receive. Size with --hicache-size or --hicache-ratio; requires dense NHD MHA buffers (including separate paged SWA pools) and a transfer backend that supports host destinations. No built-in backend currently supports this.",
|
||||
] = 1.0
|
||||
|
||||
@@ -61,6 +61,9 @@ class KVArgs:
|
||||
host_kv_data_ptrs: Optional[List[int]] = None
|
||||
host_kv_data_lens: Optional[List[int]] = None
|
||||
host_kv_item_lens: Optional[List[int]] = None
|
||||
host_state_data_ptrs: Optional[List[List[int]]] = None
|
||||
host_state_data_lens: Optional[List[List[int]]] = None
|
||||
host_state_item_lens: Optional[List[List[int]]] = None
|
||||
aux_data_ptrs: List[int]
|
||||
aux_data_lens: List[int]
|
||||
aux_item_lens: List[int]
|
||||
@@ -232,6 +235,14 @@ class BaseKVReceiver(ABC):
|
||||
"""Whether this receiver's peer and layout support host KV destinations."""
|
||||
return False
|
||||
|
||||
def is_abort_release_safe(self) -> bool:
|
||||
"""Whether all writes to an aborted host destination have drained.
|
||||
|
||||
Host-capable backends must retain this state until ``clear()``. A failed
|
||||
poll or elapsed timeout alone does not prove the buffers can be reused.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -659,12 +659,12 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
if get_disagg().disaggregation_decode_host_receive_threshold < 1:
|
||||
pool = self.token_to_kv_pool
|
||||
group = self.tree_cache.host_pool_group
|
||||
if kv_args.state_types or any(
|
||||
spec.indices_from_pool != PoolName.KV
|
||||
if (kv_args.state_types and kv_args.state_types != [StateType.SWA]) or any(
|
||||
spec.indices_from_pool not in (PoolName.KV, PoolName.SWA)
|
||||
for spec in self.tree_cache.sidecar_pool_specs
|
||||
):
|
||||
raise ValueError(
|
||||
"Host receive requires KV pools sharing the primary indices"
|
||||
"Host receive supports full KV and separate paged SWA pools"
|
||||
)
|
||||
self.host_pool = group.get_pool(PoolName.KV)
|
||||
self.host_reserved_tokens = decode_retraction_max_tokens(
|
||||
@@ -675,12 +675,66 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
"Host pool must hold a retraction and a receive page; "
|
||||
"increase --hicache-size or --hicache-ratio"
|
||||
)
|
||||
_, host_buffers = group.get_contiguous_buf_infos()
|
||||
|
||||
def pool_names(source):
|
||||
return {source} | {
|
||||
spec.pool_name
|
||||
for spec in self.tree_cache.sidecar_pool_specs
|
||||
if spec.indices_from_pool == source
|
||||
}
|
||||
|
||||
device_buffers, host_buffers = group.get_contiguous_buf_infos(
|
||||
pool_names(PoolName.KV)
|
||||
)
|
||||
if device_buffers != (
|
||||
kv_args.kv_data_ptrs,
|
||||
kv_args.kv_data_lens,
|
||||
kv_args.kv_item_lens,
|
||||
):
|
||||
raise ValueError(
|
||||
"Host receive buffers must match the KV transfer layout"
|
||||
)
|
||||
(
|
||||
kv_args.host_kv_data_ptrs,
|
||||
kv_args.host_kv_data_lens,
|
||||
kv_args.host_kv_item_lens,
|
||||
) = host_buffers
|
||||
self.host_swa_pool = None
|
||||
self.host_swa_reserved_tokens = 0
|
||||
if kv_args.state_types:
|
||||
self.host_swa_pool = group.get_pool(PoolName.SWA)
|
||||
self.host_swa_reserved_tokens = min(
|
||||
decode_retraction_max_tokens(
|
||||
self.req_to_token_pool, pool.swa_kv_pool
|
||||
),
|
||||
ceil_align(
|
||||
self.scheduler.sliding_window_size + pool.page_size - 1,
|
||||
pool.page_size,
|
||||
),
|
||||
)
|
||||
if self.host_swa_pool.logical_size < (
|
||||
self.host_swa_reserved_tokens + pool.page_size
|
||||
):
|
||||
raise ValueError(
|
||||
"SWA host pool must hold a retraction and a receive page; "
|
||||
"increase --hicache-size or --hicache-ratio"
|
||||
)
|
||||
device_state, host_state = group.get_contiguous_buf_infos(
|
||||
pool_names(PoolName.SWA)
|
||||
)
|
||||
if device_state != (
|
||||
kv_args.state_data_ptrs[0],
|
||||
kv_args.state_data_lens[0],
|
||||
kv_args.state_item_lens[0],
|
||||
):
|
||||
raise ValueError(
|
||||
"Host receive buffers must match the SWA transfer layout"
|
||||
)
|
||||
(
|
||||
kv_args.host_state_data_ptrs,
|
||||
kv_args.host_state_data_lens,
|
||||
kv_args.host_state_item_lens,
|
||||
) = ([values] for values in host_state)
|
||||
|
||||
kv_args.ib_device = get_disagg().disaggregation_ib_device
|
||||
kv_args.gpu_id = get_device().gpu_id
|
||||
@@ -1755,9 +1809,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
decode_req.req.time_stats.set_decode_transfer_queue_entry_time()
|
||||
|
||||
def _pre_alloc_host(self, decode_req: DecodeRequest) -> bool:
|
||||
num_tokens = ceil_align(
|
||||
len(decode_req.req.origin_input_ids), self.host_pool.page_size
|
||||
)
|
||||
seq_len = len(decode_req.req.origin_input_ids)
|
||||
page_size = self.host_pool.page_size
|
||||
num_tokens = ceil_align(seq_len, page_size)
|
||||
required_tokens = num_tokens + self.host_reserved_tokens
|
||||
if (
|
||||
self.host_pool.available_size() < required_tokens
|
||||
@@ -1766,17 +1820,43 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
self.tree_cache.evict_host(required_tokens)
|
||||
if self.host_pool.available_size() < required_tokens:
|
||||
return False
|
||||
swa_num_tokens = 0
|
||||
if self.host_swa_pool is not None:
|
||||
window_start = page_align_floor(
|
||||
max(0, seq_len - self.scheduler.sliding_window_size), page_size
|
||||
)
|
||||
swa_num_tokens = num_tokens - window_start
|
||||
required_swa_tokens = swa_num_tokens + self.host_swa_reserved_tokens
|
||||
if self.host_swa_pool.available_size() < required_swa_tokens:
|
||||
entry = self.tree_cache.host_pool_group.get_entry(PoolName.SWA)
|
||||
if entry.host_evict_fn is not None:
|
||||
entry.host_evict_fn(required_swa_tokens)
|
||||
if self.host_swa_pool.available_size() < required_swa_tokens:
|
||||
return False
|
||||
host_indices = self.host_pool.alloc(num_tokens)
|
||||
if host_indices is None:
|
||||
return False
|
||||
indices_by_pool = {PoolName.KV: host_indices}
|
||||
transfers = []
|
||||
state_indices = None
|
||||
if self.host_swa_pool is not None:
|
||||
swa_host_indices = self.host_swa_pool.alloc(swa_num_tokens)
|
||||
if swa_host_indices is None:
|
||||
self.host_pool.free(host_indices)
|
||||
return False
|
||||
indices_by_pool[PoolName.SWA] = swa_host_indices
|
||||
transfers.append(
|
||||
PoolTransfer(name=PoolName.SWA, host_indices=swa_host_indices)
|
||||
)
|
||||
state_indices = [kv_to_page_indices(swa_host_indices, page_size)]
|
||||
assert decode_req.req.kv.retraction_backup is None
|
||||
# All sidecars share the primary KV indices.
|
||||
decode_req.req.kv.retraction_backup = RetractionBackup(
|
||||
host_indices=host_indices,
|
||||
pool_transfers=[
|
||||
pool_transfers=transfers
|
||||
+ [
|
||||
PoolTransfer(
|
||||
name=spec.pool_name,
|
||||
host_indices=host_indices,
|
||||
host_indices=indices_by_pool[spec.indices_from_pool],
|
||||
indices_from_pool=spec.indices_from_pool,
|
||||
hit_policy=spec.hit_policy,
|
||||
)
|
||||
@@ -1791,6 +1871,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
decode_req,
|
||||
host_indices,
|
||||
self.token_to_kv_pool_allocator.page_size,
|
||||
state_indices,
|
||||
decode_prefix_len=0,
|
||||
destination=KVTransferDestination.HOST,
|
||||
)
|
||||
@@ -1815,13 +1896,29 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
0,
|
||||
retractable_tokens,
|
||||
)
|
||||
swa_required_tokens = 0
|
||||
swa_allocatable_tokens = None
|
||||
if self._uses_swa_reservation():
|
||||
_, swa_required_tokens = self._prealloc_required_tokens(req)
|
||||
full_allocatable_tokens, swa_allocatable_tokens = (
|
||||
self._swa_aware_allocatable_token_budgets(
|
||||
retractable_tokens=retractable_tokens,
|
||||
retractable_swa_tokens=sum(
|
||||
self._swa_retractable_len(r)
|
||||
for r in self.scheduler.running_batch.reqs
|
||||
),
|
||||
count_retracted=True,
|
||||
)
|
||||
)
|
||||
else:
|
||||
full_allocatable_tokens = self._allocatable_token_budgets(
|
||||
retractable_tokens=retractable_tokens, count_retracted=True
|
||||
)
|
||||
if not self._prealloc_reservation_fits(
|
||||
required_tokens,
|
||||
0,
|
||||
full_allocatable_tokens=self._allocatable_token_budgets(
|
||||
retractable_tokens=retractable_tokens, count_retracted=True
|
||||
),
|
||||
swa_allocatable_tokens=None,
|
||||
swa_required_tokens,
|
||||
full_allocatable_tokens=full_allocatable_tokens,
|
||||
swa_allocatable_tokens=swa_allocatable_tokens,
|
||||
):
|
||||
return False
|
||||
self._pre_alloc(req)
|
||||
@@ -2611,6 +2708,14 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
return
|
||||
release_kv_cache(decode_req.req, self.tree_cache, checkpoint=False)
|
||||
|
||||
def _record_transfer_failure(self, failure_reason: str) -> None:
|
||||
if self.scheduler.metrics_reporter.enable_metrics:
|
||||
self.scheduler.metrics_collector.increment_transfer_failed_reqs()
|
||||
|
||||
def _abort_failed_transfer(self, decode_req: DecodeRequest) -> None:
|
||||
if self.enable_host_receive and decode_req.host_staged:
|
||||
decode_req.kv_receiver.abort()
|
||||
|
||||
def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]:
|
||||
if not self.queue:
|
||||
return []
|
||||
@@ -2643,17 +2748,20 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
poll == KVPoll.Failed
|
||||
or hicache_restore_status == HiCacheRestoreResult.FAILED
|
||||
):
|
||||
self._abort_failed_transfer(decode_req)
|
||||
error_message = (
|
||||
f"Decode transfer failed for request rank={self.tp_rank} "
|
||||
f"{decode_req.req.rid=} {decode_req.req.bootstrap_room=}"
|
||||
)
|
||||
is_propagated = False
|
||||
failure_reason = str(hicache_restore_status)
|
||||
if poll == KVPoll.Failed:
|
||||
try:
|
||||
decode_req.kv_receiver.failure_exception()
|
||||
except Exception as e:
|
||||
error_message += f" with exception {e}"
|
||||
is_propagated = getattr(e, "is_from_another_rank", False)
|
||||
failure_reason = getattr(e, "failure_reason", str(e))
|
||||
self._clean_hicache_prefetch_resources(decode_req)
|
||||
# Mute error message for propagated exceptions to avoid duplicate logging
|
||||
if is_propagated:
|
||||
@@ -2674,8 +2782,6 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
requires_host_drain = (
|
||||
self.enable_host_receive and decode_req.host_staged
|
||||
)
|
||||
if requires_host_drain:
|
||||
decode_req.kv_receiver.abort()
|
||||
deferrable = (
|
||||
self.enable_deferred_kv_release
|
||||
and decode_req.kv_receiver.kv_mgr.enable_deferred_decode_kv_release
|
||||
@@ -2704,8 +2810,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
decode_req.kv_receiver.clear()
|
||||
decode_req.kv_receiver = None
|
||||
indices_to_remove.add(i)
|
||||
if self.scheduler.metrics_reporter.enable_metrics:
|
||||
self.scheduler.metrics_collector.increment_transfer_failed_reqs()
|
||||
self._record_transfer_failure(failure_reason)
|
||||
continue
|
||||
elif poll == KVPoll.Success:
|
||||
if self.enable_host_receive and decode_req.host_staged:
|
||||
@@ -2741,8 +2846,9 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
)
|
||||
self._clean_hicache_prefetch_resources(decode_req)
|
||||
self._release_request(decode_req)
|
||||
if self.scheduler.metrics_reporter.enable_metrics:
|
||||
self.scheduler.metrics_collector.increment_transfer_failed_reqs()
|
||||
self._record_transfer_failure(
|
||||
decode_req.req.finished_reason.message
|
||||
)
|
||||
else:
|
||||
transferred_reqs.append(decode_req.req)
|
||||
elif poll in [
|
||||
@@ -2780,9 +2886,13 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
def _defer_release(self, decode_req: DecodeRequest) -> None:
|
||||
start_time = time.monotonic()
|
||||
deadline = start_time + self.deferred_kv_release_timeout
|
||||
# Require an ack from every notified prefill rank (dummy-proof). Snapshot
|
||||
# now -- the receiver may be cleared by resolve time.
|
||||
required_acks = len(decode_req.kv_receiver.bootstrap_infos)
|
||||
# Host-capable receivers own their drain tracking. Device destinations
|
||||
# snapshot the number of prefill acknowledgements required by the manager.
|
||||
required_acks = (
|
||||
0
|
||||
if self.enable_host_receive and decode_req.host_staged
|
||||
else len(decode_req.kv_receiver.bootstrap_infos)
|
||||
)
|
||||
self._deferred_releases.append(
|
||||
(
|
||||
decode_req,
|
||||
@@ -2801,7 +2911,8 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
self._release_request(decode_req)
|
||||
self.metadata_buffers.bootstrap_room[idx] = 0
|
||||
self.req_to_metadata_buffer_idx_allocator.free(idx)
|
||||
decode_req.kv_receiver.kv_mgr.clear_deferred_abort_state(room)
|
||||
if not (self.enable_host_receive and decode_req.host_staged):
|
||||
decode_req.kv_receiver.kv_mgr.clear_deferred_abort_state(room)
|
||||
decode_req.kv_receiver.clear()
|
||||
decode_req.kv_receiver = None
|
||||
|
||||
@@ -2826,10 +2937,8 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
return
|
||||
host_ready = torch.tensor(
|
||||
[
|
||||
req.kv_receiver.kv_mgr.is_abort_release_safe(
|
||||
req.req.bootstrap_room, acks
|
||||
)
|
||||
for req, _, _, _, acks in self._deferred_releases
|
||||
req.kv_receiver.is_abort_release_safe()
|
||||
for req, _, _, _, _ in self._deferred_releases
|
||||
if self.enable_host_receive and req.host_staged
|
||||
],
|
||||
dtype=torch.int32,
|
||||
|
||||
@@ -105,16 +105,18 @@ class HostPoolGroup:
|
||||
element_sizes.add(element_size)
|
||||
return element_sizes.pop() if len(element_sizes) == 1 else None
|
||||
|
||||
def get_contiguous_buf_infos(self):
|
||||
def get_contiguous_buf_infos(self, pool_names: set[PoolName] | None = None):
|
||||
"""Return (device_buffers, host_buffers), each (ptrs, sizes, item_sizes)."""
|
||||
host_by_device_ptr = {}
|
||||
device_infos = ([], [], [])
|
||||
for entry in self.entries:
|
||||
if pool_names is not None and entry.name not in pool_names:
|
||||
continue
|
||||
host = entry.host_pool
|
||||
pools = (entry.device_pool, *entry.packed_draft_device_pools)
|
||||
for pool in pools:
|
||||
dense_mha = (
|
||||
type(pool) is MHATokenToKVPool
|
||||
isinstance(pool, MHATokenToKVPool)
|
||||
and pool.kv_cache_layout == "nhd"
|
||||
and pool.v_head_dim == pool.head_dim
|
||||
)
|
||||
@@ -127,9 +129,21 @@ class HostPoolGroup:
|
||||
"Host receive requires dense NHD MHA target and "
|
||||
"draft KV with matching page sizes"
|
||||
)
|
||||
for combined, values in zip(
|
||||
device_infos, pool.get_contiguous_buf_infos(), strict=True
|
||||
buffers = pool.k_buffer + pool.v_buffer
|
||||
infos = pool.get_contiguous_buf_infos()
|
||||
expected = (
|
||||
[buffer.data_ptr() for buffer in buffers],
|
||||
[buffer.nbytes for buffer in buffers],
|
||||
[buffer[0].nbytes * self.page_size for buffer in buffers],
|
||||
)
|
||||
if infos != expected or any(
|
||||
not buffer.is_contiguous() for buffer in buffers
|
||||
):
|
||||
raise ValueError(
|
||||
"Host receive requires contiguous per-layer KV buffers "
|
||||
"without embedded state"
|
||||
)
|
||||
for combined, values in zip(device_infos, infos, strict=True):
|
||||
combined.extend(values)
|
||||
|
||||
# Packed MHA stores target/draft K followed by target/draft V,
|
||||
|
||||
Reference in New Issue
Block a user