[PD] Add decode host receive for custom transfer backends (#40238)

Co-authored-by: cctry <cctry@fb.com>
This commit is contained in:
cctry
2026-09-23 21:57:33 -07:00
committed by GitHub
co-authored by cctry
parent 7a9feacf91
commit efff836911
25 changed files with 1144 additions and 177 deletions
@@ -497,5 +497,25 @@ POSITIONAL_FIELD_ORDER = (
"weight_cache_timeout",
"forward_hooks",
"msprobe_dump_config",
"ple_offload_embedding",
"ple_offload_backend",
"ple_offload_dir",
"disaggregation_enable_kv_checksum",
"enable_pd_role_switch",
"speculative_domino_candidate_pool_size",
"flashinfer_a2a_dispatch_type",
"cuda_graph_max_seq_len_prefill",
"prefill_kv_cache_dequant_dtype",
"dsv4_attn_backend",
"otlp_service_name",
"_radix_eviction_policy_explicitly_set",
"hicache_host_memory_fraction",
"enable_linker_mla_dedup",
"_swa_full_tokens_ratio_explicitly_set",
"swa_prefix_tails",
"enable_encoder_swa_bounded_replay",
"enable_decoder_swa_bounded_replay",
"enable_response_store",
"disaggregation_decode_host_receive_threshold",
)
# fmt: on
@@ -168,3 +168,7 @@ class Disagg(msgspec.Struct):
"The path of the PD-Multiplexing config file.",
] = None
sm_group_num: A[int, "Number of sm partition groups."] = 8
disaggregation_decode_host_receive_threshold: A[
float,
"Device token usage fraction at which incoming KV is received in the decode retraction host pool, excluding evictable cache pages. Range [0, 1]; 0 disables host receive. Size with --hicache-size or --hicache-ratio; requires dense MHA and a transfer backend that supports host destinations. No built-in backend currently supports this.",
] = 0.0
@@ -54,11 +54,25 @@ def handle_hicache(server_args: Any):
validate_hicache_host_memory_mode(server_args)
if cfg.disaggregation_decode_host_receive_threshold > 0:
# The shared host pool must match KV transfer's per-layer buffers.
declare_resolution(
server_args, "handle_hicache", hicache_mem_layout="layer_first"
)
# Step 1: Initial layout-io compatibility normalization.
resolve_layout_io_compatibility(server_args)
# Step 2: Storage-layout normalization without changing io backend.
resolve_storage_layout_compatibility(server_args)
if (
cfg.disaggregation_decode_host_receive_threshold > 0
and cfg.hicache_mem_layout != "layer_first"
):
raise ValueError(
f"The resolved HiCache storage layout {cfg.hicache_mem_layout!r} "
"cannot share the layer_first decode host pool used by KV transfer"
)
# Step 3: DCP compatibility for the L2 (device<->host) path.
resolve_hicache_dcp_compatibility(server_args)
@@ -50,6 +50,24 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
"overhead without improving prefill performance."
)
if not 0 <= cfg.disaggregation_decode_host_receive_threshold <= 1:
raise ValueError(
"--disaggregation-decode-host-receive-threshold must be between 0 and 1"
)
if cfg.disaggregation_decode_host_receive_threshold > 0:
if cfg.enable_hisparse or cfg.enable_pd_role_switch:
raise ValueError(
"Decode host receive does not yet support HiSparse or role switching"
)
if cfg.disaggregation_decode_retraction_backup == "cpu_tensor":
raise ValueError("Decode host KV buffering requires host_pool retraction")
declare_resolution(
server_args,
"handle_pd_disaggregation",
disaggregation_decode_retraction_backup="host_pool",
)
if cfg.disaggregation_mode == "decode" and cfg.dcp_size > 1:
# Fake transfer moves no KV and is only used for synthetic decode
# benchmarks, so it does not need the DCP relayout from Mooncake/NIXL.
@@ -46,6 +46,11 @@ class KVTransferMetric:
transfer_total_bytes: Optional[int] = None
class KVTransferDestination(str, enum.Enum):
DEVICE = "device"
HOST = "host"
class KVArgs:
engine_rank: int
kv_data_ptrs: List[int]
@@ -53,6 +58,9 @@ class KVArgs:
kv_item_lens: List[int]
kv_layer_ids: List[int]
kv_cache_dtype_str: str
host_kv_data_ptrs: Optional[List[int]] = None
host_kv_data_lens: Optional[List[int]] = None
host_kv_item_lens: Optional[List[int]] = None
aux_data_ptrs: List[int]
aux_data_lens: List[int]
aux_item_lens: List[int]
@@ -113,6 +121,7 @@ class BaseKVManager(ABC):
"""Base class for managing transfer states"""
enable_deferred_decode_kv_release: bool = False
supports_host_destination: bool = False
@abstractmethod
def __init__(
@@ -212,6 +221,11 @@ class BaseKVSender(ABC):
class BaseKVReceiver(ABC):
@property
def supports_host_destination(self) -> bool:
"""Whether this receiver's peer and layout support host KV destinations."""
return False
@abstractmethod
def __init__(
self,
@@ -237,6 +251,7 @@ class BaseKVReceiver(ABC):
aux_index: Optional[int] = None,
state_indices: Optional[List] = None,
decode_prefix_len: Optional[int] = None,
destination: KVTransferDestination = KVTransferDestination.DEVICE,
):
"""
Notify the prefill server about the kv indices, aux index, and state_indices.
@@ -23,6 +23,7 @@ from sglang.srt.disaggregation.base.conn import (
BaseKVSender,
KVArgs,
KVPoll,
KVTransferDestination,
KVTransferMetric,
StateType,
)
@@ -1931,6 +1932,7 @@ class CommonKVReceiver(BaseKVReceiver):
aux_index: Optional[int] = None,
state_indices: Optional[List[int]] = None,
decode_prefix_len: Optional[int] = None,
destination: KVTransferDestination = KVTransferDestination.DEVICE,
):
raise NotImplementedError
+309 -87
View File
@@ -36,7 +36,7 @@ from torch.distributed import ProcessGroup
from sglang.srt.configs.mamba_utils import Mamba2CacheParams
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
from sglang.srt.disaggregation.base import KVPoll
from sglang.srt.disaggregation.base.conn import StateType
from sglang.srt.disaggregation.base.conn import KVTransferDestination, StateType
from sglang.srt.disaggregation.checksum import (
KvChecksumComputer,
is_health_check_req,
@@ -88,14 +88,17 @@ from sglang.srt.mem_cache.base_prefix_cache import (
EvictParams,
)
from sglang.srt.mem_cache.common import (
RetractionBackup,
discard_kv_cache_backup,
dsv41_dspark_needs_rebootstrap,
kv_to_page_indices,
page_align_floor,
release_kv_cache,
retraction_discard,
retraction_restore,
restore_kv_cache,
)
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
from sglang.srt.mem_cache.kv_cache_builder import decode_retraction_max_tokens
from sglang.srt.mem_cache.memory_pool import (
HybridReqToTokenPool,
KVCache,
@@ -330,6 +333,7 @@ class DecodeRequest:
waiting_for_input: bool = False
metadata_buffer_index: int = -1
is_rebootstrap: bool = False
host_staged: bool = False
# HiCache Status
prefix_match: Optional[DecodePrefixMatch] = None
@@ -382,6 +386,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
self.scheduler = scheduler
self.transfer_queue = transfer_queue
self.tree_cache = tree_cache
self.host_pool = None
self.host_reserved_tokens = 0
self.gloo_group = gloo_group
# Destinations visible to prefill but not yet on the transfer queue.
self._num_published_destinations = 0
@@ -460,6 +466,17 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
or self.token_to_kv_pool_allocator.prealloc_fits_assumes_reclaim()
)
def _required_admission_tokens(
self, req: Req, allocated_tokens: int, prefix_len: int, retractable_tokens: int
) -> int:
return max(
allocated_tokens + self.num_reserved_decode_tokens,
self._rebootstrap_prefill_len(req)
- prefix_len
+ min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKEN)
- retractable_tokens,
)
def _prealloc_reservation_fits(
self,
full_tokens: int,
@@ -630,6 +647,31 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
total_kv_layers=self.scheduler.model_config.num_hidden_layers,
req_to_token_pool=getattr(self, "req_to_token_pool", None),
)
if get_disagg().disaggregation_decode_host_receive_threshold > 0:
pool = self.token_to_kv_pool
group = self.tree_cache.host_pool_group
if kv_args.state_types or any(
spec.indices_from_pool != PoolName.KV
for spec in self.tree_cache.sidecar_pool_specs
):
raise ValueError(
"Host receive requires KV pools sharing the primary indices"
)
self.host_pool = group.get_pool(PoolName.KV)
self.host_reserved_tokens = decode_retraction_max_tokens(
self.req_to_token_pool, pool
)
if self.host_pool.logical_size < self.host_reserved_tokens + pool.page_size:
raise ValueError(
"Host pool must hold a retraction and a receive page; "
"increase --hicache-size or --hicache-ratio"
)
_, host_buffers = group.get_contiguous_buf_infos()
(
kv_args.host_kv_data_ptrs,
kv_args.host_kv_data_lens,
kv_args.host_kv_item_lens,
) = host_buffers
kv_args.ib_device = get_disagg().disaggregation_ib_device
kv_args.gpu_id = get_device().gpu_id
@@ -640,6 +682,13 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
self.scheduler.server_args,
self.is_mla_backend,
)
if (
get_disagg().disaggregation_decode_host_receive_threshold > 0
and not kv_manager.supports_host_destination
):
raise ValueError(
"Transfer backend does not support decode host KV destinations"
)
# Staging buffer setup (only when heterogeneous TP staging is enabled)
if self.enable_staging and not self.is_mla_backend:
kv_pool_for_heads = self.token_to_kv_pool
@@ -872,7 +921,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
self._cancel_prefill_dp_rank_queries()
self.queue.clear()
for req in self.retracted_queue:
retraction_discard(
discard_kv_cache_backup(
req,
self.tree_cache,
get_disagg().disaggregation_decode_retraction_backup,
@@ -941,7 +990,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
extra_reserved_reqs=len(resumed_reqs),
)
retraction_restore(
restore_kv_cache(
req,
self.tree_cache,
self.req_to_token_pool,
@@ -1186,6 +1235,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
failed_reqs = []
preallocated_reqs = []
num_device_preallocated = 0
indices_to_remove = set()
# We need to make sure that the sum of inflight tokens and allocatable tokens is greater than maximum input+output length of each inflight request
@@ -1285,6 +1335,30 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
if not decode_req.waiting_for_input:
continue
if self.req_to_metadata_buffer_idx_allocator.available_size() <= 0:
break
if self.scheduler.enable_lora and not self.scheduler.can_schedule_lora_req(
decode_req.req, running_loras
):
continue
if (
get_disagg().disaggregation_decode_host_receive_threshold > 0
and not decode_req.is_rebootstrap
and not _is_fake_transfer(decode_req.req)
and decode_req.kv_receiver.supports_host_destination
and self.scheduler.pool_stats_observer.get_pool_stats().full_token_usage
>= get_disagg().disaggregation_decode_host_receive_threshold
):
if not self._pre_alloc_host(decode_req):
break
preallocated_reqs.append(decode_req)
indices_to_remove.add(i)
if self.scheduler.enable_lora:
running_loras.add(decode_req.req.lora_id)
continue
if self.req_to_token_pool.available_size() <= 0:
break
@@ -1301,17 +1375,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
if mamba_allocator.available_size() <= 0:
break
if self.req_to_metadata_buffer_idx_allocator.available_size() <= 0:
break
if hisparse_req_budget <= 0:
break
if self.scheduler.enable_lora and not self.scheduler.can_schedule_lora_req(
decode_req.req, running_loras
):
continue
# Memory estimation: don't add if the projected memory cannot be met
# TODO: add new_token ratio
origin_input_len = self._rebootstrap_prefill_len(decode_req.req)
@@ -1369,7 +1435,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
full_allocatable_tokens = self._allocatable_token_budgets(
retractable_tokens=retractable_tokens,
count_retracted=True,
extra_reserved_reqs=len(preallocated_reqs),
extra_reserved_reqs=num_device_preallocated,
hicache_reserved_tokens=reserved_restore_tokens,
)
if uses_swa_tail_prealloc:
@@ -1377,7 +1443,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
retractable_tokens=retractable_tokens,
retractable_swa_tokens=retractable_swa_tokens,
count_retracted=True,
extra_reserved_reqs=len(preallocated_reqs),
extra_reserved_reqs=num_device_preallocated,
)
else:
prefix_indices = None
@@ -1385,19 +1451,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
total_prefix_len = 0
required_alloc_tokens = self._pre_alloc_fill_len(decode_req.req)
required_tokens_for_request = (
required_alloc_tokens + self.num_reserved_decode_tokens
)
full_required_for_admission = max(
required_tokens_for_request,
origin_input_len
- prefix_len
+ min(
decode_req.req.sampling_params.max_new_tokens,
CLIP_MAX_NEW_TOKEN,
)
- retractable_tokens,
full_required_for_admission = self._required_admission_tokens(
decode_req.req, required_alloc_tokens, prefix_len, retractable_tokens
)
swa_required_for_admission = 0
swa_len = required_alloc_tokens
@@ -1465,7 +1520,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
full_allocatable_tokens = self._allocatable_token_budgets(
retractable_tokens=retractable_tokens,
count_retracted=True,
extra_reserved_reqs=len(preallocated_reqs) + 1,
extra_reserved_reqs=num_device_preallocated + 1,
hicache_reserved_tokens=reserved_restore_tokens,
)
if swa_allocatable_tokens is not None:
@@ -1473,7 +1528,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
retractable_tokens=retractable_tokens,
retractable_swa_tokens=retractable_swa_tokens,
count_retracted=True,
extra_reserved_reqs=len(preallocated_reqs) + 1,
extra_reserved_reqs=num_device_preallocated + 1,
)
decode_req.req.kv.cache_protected_len = total_prefix_len
@@ -1601,14 +1656,6 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
payloads[st]() if st in payloads else None for st in state_types
]
decode_req.metadata_buffer_index = (
self.req_to_metadata_buffer_idx_allocator.alloc()
)
assert decode_req.metadata_buffer_index is not None
# int32 for ZMQ serialization -- from_zmq reads np.int32.
page_indices = kv_to_page_indices(kv_indices, kv_transfer_page_size).astype(
np.int32
)
device_page_indices = None
if (
self.scheduler.enable_hisparse
@@ -1635,33 +1682,18 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
metadata_kwargs = {"decode_prefix_len": total_prefix_len}
if device_page_indices is not None:
metadata_kwargs["device_kv_indices"] = device_page_indices
if (
self.transfer_queue.enable_staging
and hasattr(decode_req.kv_receiver, "require_staging")
and decode_req.kv_receiver.require_staging
):
# Register before send_metadata, which triggers the STAGING_REQ
# prefetch (dropped for an unregistered room); tiny race, correct order.
self.transfer_queue.staging_handler.register_decode_req(
decode_req.req.bootstrap_room, decode_req
)
decode_req.kv_receiver.send_metadata(
page_indices,
decode_req.metadata_buffer_index,
self._send_kv_metadata(
decode_req,
kv_indices,
kv_transfer_page_size,
state_indices,
**metadata_kwargs,
)
if decode_req.is_rebootstrap:
self.kv_manager.submit_prefill_recompute(
decode_req.kv_receiver,
decode_req.req.build_rebootstrap_payload(),
)
self._num_published_destinations += 1
num_device_preallocated += 1
preallocated_reqs.append(decode_req)
indices_to_remove.add(i)
if self.scheduler.enable_lora:
running_loras.add(decode_req.req.lora_id)
decode_req.req.time_stats.set_decode_transfer_queue_entry_time()
if failed_reqs:
failed_ids = {id(r) for r in failed_reqs}
@@ -1675,6 +1707,121 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
return preallocated_reqs, failed_reqs
def _send_kv_metadata(
self, decode_req, kv_indices, page_size, state_indices=None, **metadata_kwargs
) -> None:
decode_req.metadata_buffer_index = (
self.req_to_metadata_buffer_idx_allocator.alloc()
)
assert decode_req.metadata_buffer_index is not None
page_indices = kv_to_page_indices(kv_indices, page_size).astype(np.int32)
if (
metadata_kwargs.get("destination") != KVTransferDestination.HOST
and self.transfer_queue.enable_staging
and hasattr(decode_req.kv_receiver, "require_staging")
and decode_req.kv_receiver.require_staging
):
# Register before send_metadata, which triggers the STAGING_REQ
# prefetch (dropped for an unregistered room); tiny race, correct order.
self.transfer_queue.staging_handler.register_decode_req(
decode_req.req.bootstrap_room, decode_req
)
decode_req.kv_receiver.send_metadata(
page_indices,
decode_req.metadata_buffer_index,
state_indices,
**metadata_kwargs,
)
if decode_req.is_rebootstrap:
self.kv_manager.submit_prefill_recompute(
decode_req.kv_receiver,
decode_req.req.build_rebootstrap_payload(),
)
self._num_published_destinations += 1
decode_req.req.time_stats.set_decode_transfer_queue_entry_time()
def _pre_alloc_host(self, decode_req: DecodeRequest) -> bool:
num_tokens = ceil_align(
len(decode_req.req.origin_input_ids), self.host_pool.page_size
)
required_tokens = num_tokens + self.host_reserved_tokens
if (
self.host_pool.available_size() < required_tokens
and not self.tree_cache.disable
):
self.tree_cache.evict_host(required_tokens)
if self.host_pool.available_size() < required_tokens:
return False
host_indices = self.host_pool.alloc(num_tokens)
if host_indices is None:
return False
assert decode_req.req.kv.retraction_backup is None
# All sidecars share the primary KV indices.
decode_req.req.kv.retraction_backup = RetractionBackup(
host_indices=host_indices,
pool_transfers=[
PoolTransfer(
name=spec.pool_name,
host_indices=host_indices,
indices_from_pool=spec.indices_from_pool,
hit_policy=spec.hit_policy,
)
for spec in self.tree_cache.sidecar_pool_specs
if num_tokens
]
or None,
)
if get_disagg().disaggregation_decode_enable_radix_cache:
req = decode_req.req
req.prefix_indices = torch.empty((0,), dtype=torch.int64)
req.last_node = self.tree_cache.root_node_handle(req.extra_key)
req.last_host_node = req.last_node
req.best_match_node = req.last_node
req.lock_receipt = DecLockRefParams()
req.kv.cache_protected_len = 0
req.num_matched_prefix_tokens = 0
req.host_hit_length = 0
self._send_kv_metadata(
decode_req,
host_indices,
self.token_to_kv_pool_allocator.page_size,
decode_prefix_len=0,
destination=KVTransferDestination.HOST,
)
decode_req.host_staged = True
if self.scheduler.metrics_reporter.current_scheduler_metrics_enabled:
self.scheduler.metrics_collector.increment_decode_host_receive_reqs()
return True
def allocate_host_staged(self, decode_req: DecodeRequest) -> bool:
if self.req_to_token_pool.available_size() <= 0:
return False
req = decode_req.req
retractable_tokens = sum(
len(r.origin_input_ids) + len(r.output_ids)
for r in self.scheduler.running_batch.reqs
)
required_tokens = self._required_admission_tokens(
req,
self._required_alloc_tokens(
fill_len=len(req.origin_input_ids), prefix_len=0
),
0,
retractable_tokens,
)
if not self._prealloc_reservation_fits(
required_tokens,
0,
full_allocatable_tokens=self._allocatable_token_budgets(
retractable_tokens=retractable_tokens, count_retracted=True
),
swa_allocatable_tokens=None,
):
return False
self._pre_alloc(req)
decode_req.host_staged = False
return True
@property
def has_published_destinations(self) -> bool:
"""Whether any destination address is visible to prefill but not yet
@@ -1690,7 +1837,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
@property
def num_tokens_pre_allocated(self):
return sum(
decode_req.req.extend_range.end for decode_req in self.transfer_queue.queue
decode_req.req.extend_range.end
for decode_req in self.transfer_queue.queue
if not decode_req.host_staged
)
def _need_space_for_single_req(
@@ -1714,7 +1863,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
def _active_req_count(self, extra_reserved_reqs: int = 0) -> int:
return (
len(self.scheduler.running_batch.reqs)
+ len(self.transfer_queue.queue)
+ sum(not r.host_staged for r in self.transfer_queue.queue)
+ len(self.scheduler.waiting_queue)
+ extra_reserved_reqs
)
@@ -2193,6 +2342,8 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
Store the requests that is polling kv
"""
enable_host_receive: bool = False
def __init__(
self,
gloo_group: ProcessGroup,
@@ -2207,6 +2358,9 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
self.tp_rank = get_parallel().tp_rank
self.metadata_buffers = metadata_buffers
self.scheduler = scheduler
self.enable_host_receive = (
get_disagg().disaggregation_decode_host_receive_threshold > 0
)
self.tree_cache = tree_cache
self.spec_algorithm = scheduler.spec_algorithm
self.enable_staging = envs.SGLANG_DISAGG_STAGING_BUFFER.get()
@@ -2444,6 +2598,13 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
)
kv_manager._staging_handler = self.staging_handler
def _release_request(self, decode_req: DecodeRequest) -> None:
if self.enable_host_receive:
discard_kv_cache_backup(decode_req.req, self.tree_cache, "host_pool")
if decode_req.host_staged:
return
release_kv_cache(decode_req.req, self.tree_cache, is_insert=False)
def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]:
if not self.queue:
return []
@@ -2504,21 +2665,24 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
)
if self.scheduler.enable_hisparse:
self.scheduler.hisparse_coordinator.request_finished(decode_req.req)
if (
requires_host_drain = (
self.enable_host_receive and decode_req.host_staged
)
if requires_host_drain:
decode_req.kv_receiver.abort()
if requires_host_drain or (
self.enable_deferred_kv_release
and decode_req.kv_receiver.kv_mgr.enable_deferred_decode_kv_release
and decode_req.kv_receiver.abort_notified
):
# Decode-initiated abort: a prefill write may still target
# these pages, so hold them until the drain ack or timeout.
# (A prefill-initiated failure has already stopped writing ->
# immediate release below.)
# Host pages always await a drain ack. Device pages retain
# the existing opt-in deferred-release behavior.
self._defer_release(decode_req)
deferred_indices.add(i)
indices_to_remove.add(i)
else:
# release pre-allocated kv cache, but don't insert into the tree since it's failed
release_kv_cache(decode_req.req, self.tree_cache, is_insert=False)
self._release_request(decode_req)
decode_req.kv_receiver.clear()
decode_req.kv_receiver = None
indices_to_remove.add(i)
@@ -2526,6 +2690,20 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
self.scheduler.metrics_collector.increment_transfer_failed_reqs()
continue
elif poll == KVPoll.Success:
if self.enable_host_receive and decode_req.host_staged:
if isinstance(decode_req.req.finished_reason, FINISH_ABORT):
self.scheduler.output_streamer.stream_output(
[decode_req.req], decode_req.req.return_logprob
)
self._release_request(decode_req)
decode_req.kv_receiver.clear()
decode_req.kv_receiver = None
indices_to_remove.add(i)
continue
if not self.scheduler.disagg_decode_prealloc_queue.allocate_host_staged(
decode_req
):
continue
if (
self.scheduler.enable_decode_hicache
and hicache_restore_status == HiCacheRestoreResult.PENDING
@@ -2544,7 +2722,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
decode_req.req
)
self._clean_hicache_prefetch_resources(decode_req)
release_kv_cache(decode_req.req, self.tree_cache, is_insert=False)
self._release_request(decode_req)
if self.scheduler.metrics_reporter.enable_metrics:
self.scheduler.metrics_collector.increment_transfer_failed_reqs()
else:
@@ -2595,7 +2773,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
if self.enable_staging and self.staging_handler.is_staging_room(room):
self.staging_handler.unregister_decode_req(room)
# release pre-allocated kv cache, but don't insert into the tree since it's failed
release_kv_cache(decode_req.req, self.tree_cache, is_insert=False)
self._release_request(decode_req)
self.metadata_buffers.bootstrap_room[idx] = 0
self.req_to_metadata_buffer_idx_allocator.free(idx)
decode_req.kv_receiver.kv_mgr.clear_deferred_abort_state(room)
@@ -2606,15 +2784,37 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
return bool(self._deferred_releases)
def resolve_deferred_releases(self) -> None:
"""Release held requests once every prefill rank acks the drain, or the
hold times out."""
"""Release drained transfers; device destinations may also time out."""
if not self._deferred_releases:
return
host_ready = torch.tensor(
[
req.kv_receiver.kv_mgr.is_abort_release_safe(
req.req.bootstrap_room, acks
)
for req, _, _, acks in self._deferred_releases
if self.enable_host_receive and req.host_staged
],
dtype=torch.int32,
)
if host_ready.numel() and torch.distributed.get_world_size(self.gloo_group) > 1:
# Admission must see identical host capacity and page order on every rank.
torch.distributed.all_reduce(
host_ready, op=torch.distributed.ReduceOp.MIN, group=self.gloo_group
)
host_ready = iter(host_ready.tolist())
now = time.monotonic()
still_held = []
to_release = []
for decode_req, deadline, idx, required_acks in self._deferred_releases:
room = decode_req.req.bootstrap_room
if self.enable_host_receive and decode_req.host_staged:
if next(host_ready):
to_release.append((decode_req, idx, room, True))
else:
# A timeout cannot prove that a remote write has stopped.
still_held.append((decode_req, deadline, idx, required_acks))
continue
kv_mgr = decode_req.kv_receiver.kv_mgr
drained = kv_mgr.is_abort_release_safe(room, required_acks)
if not drained and now < deadline:
@@ -2797,25 +2997,39 @@ class SchedulerDisaggregationDecodeMixin:
def get_new_prebuilt_batch(
self, running_batch: ScheduleBatch
) -> Optional[ScheduleBatch]:
batch = self._get_new_prebuilt_batch(running_batch)
if batch is None:
return None
# Verify after host restore and before process_prebuilt caches any KV.
self._verify_kv_checksums(batch)
if batch.is_empty():
return None
self.ngram_embedding_manager.prepare_for_forward(
batch, chunked_req=self.chunked_req
)
batch.process_prebuilt(self.future_map)
return batch
def _verify_kv_checksums(self, batch: ScheduleBatch) -> None:
computer: Optional[KvChecksumComputer] = self.kv_checksum_computer
if computer is None:
return self._get_new_prebuilt_batch(running_batch)
return
verified: List[Req] = []
for req in self.waiting_queue:
verified: List[int] = []
for i, req in enumerate(batch.reqs):
if is_health_check_req(req):
verified.append(req)
verified.append(i)
continue
expected = req.expected_kv_checksum
if expected == 0:
verified.append(req)
verified.append(i)
continue
seq_len = len(req.origin_input_ids)
page_indices_gpu = page_indices_for_request(self, req, seq_len)
state_indices = state_indices_for_request(self, req, seq_len)
actual = computer.compute(page_indices_gpu, state_indices)
if actual == expected:
verified.append(req)
verified.append(i)
continue
msg = (
f"KV checksum mismatch req={req.rid} "
@@ -2824,9 +3038,8 @@ class SchedulerDisaggregationDecodeMixin:
)
logger.error(msg)
self._handle_kv_checksum_mismatch(req, msg)
self.waiting_queue = verified
return self._get_new_prebuilt_batch(running_batch)
if len(verified) != len(batch.reqs):
batch.filter_batch(keep_indices=verified)
def _handle_kv_checksum_mismatch(self, req: Req, msg: str) -> None:
# A mismatch means the KV this worker received is not what prefill sent,
@@ -2916,12 +3129,16 @@ class SchedulerDisaggregationDecodeMixin:
# A finished request can still have one redundant forward in flight.
# Drain it before a prebuilt request seeds a potentially reused row.
self.schedule_stream.wait_stream(self.forward_stream)
# The prebuilt batch never reaches the forward loop's prepare call.
self.ngram_embedding_manager.prepare_for_forward(
new_batch, chunked_req=self.chunked_req
)
new_batch.process_prebuilt(self.future_map)
if get_disagg().disaggregation_decode_host_receive_threshold > 0:
for req in new_batch.reqs:
if req.kv.retraction_backup is not None:
restore_kv_cache(
req,
self.tree_cache,
self.req_to_token_pool,
self.token_to_kv_pool_allocator,
"host_pool",
)
return new_batch
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE)
@@ -2950,8 +3167,9 @@ class SchedulerDisaggregationDecodeMixin:
self.polling_count = (self.polling_count + 1) % self.polling_interval
if self.polling_count % self.polling_interval == 0:
req_conns, _ = self.disagg_decode_prealloc_queue.pop_preallocated()
self.disagg_decode_transfer_queue.extend(req_conns)
if get_disagg().disaggregation_decode_host_receive_threshold == 0:
req_conns, _ = self.disagg_decode_prealloc_queue.pop_preallocated()
self.disagg_decode_transfer_queue.extend(req_conns)
transferred_reqs = (
self.disagg_decode_transfer_queue.pop_transferred()
) # the requests which kv has arrived
@@ -2960,3 +3178,7 @@ class SchedulerDisaggregationDecodeMixin:
# Direct-to-host: KV data already in host pool, skip staging
self.hisparse_coordinator.admit_request_direct(req)
self.waiting_queue.extend(transferred_reqs)
if get_disagg().disaggregation_decode_host_receive_threshold > 0:
# Give completed host transfers device space before new arrivals.
req_conns, _ = self.disagg_decode_prealloc_queue.pop_preallocated()
self.disagg_decode_transfer_queue.extend(req_conns)
+2 -2
View File
@@ -112,10 +112,10 @@ from sglang.srt.mem_cache.base_prefix_cache import (
)
from sglang.srt.mem_cache.common import (
RetractionBackup,
backup_kv_cache,
evict_from_tree_cache,
free_swa_out_of_window_slots,
release_kv_cache,
retraction_backup,
)
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool
from sglang.srt.mem_cache.radix_cache import RadixKey
@@ -2239,7 +2239,7 @@ def release_req(
backup_saved = True
# The config bag reflects role flips; server_args keeps the launch role.
if get_disagg().disaggregation_mode == "decode" and offload_kv:
backup_saved = retraction_backup(
backup_saved = backup_kv_cache(
req,
tree_cache,
req_to_token_pool,
+8 -2
View File
@@ -290,9 +290,9 @@ from sglang.srt.managers.utils import (
from sglang.srt.mem_cache import kv_cache_builder
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome
from sglang.srt.mem_cache.common import (
discard_kv_cache_backup,
maybe_cache_unfinished_req,
release_kv_cache,
retraction_discard,
)
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
from sglang.srt.model_executor.runner_utils.pool import prewarm_graph_pool_borrow
@@ -5434,6 +5434,8 @@ class Scheduler(
)
# For disaggregation decode mode, the request in the waiting queue has KV cache allocated.
if self.disaggregation_mode == DisaggregationMode.DECODE:
if get_disagg().disaggregation_decode_host_receive_threshold > 0:
discard_kv_cache_backup(req, self.tree_cache, "host_pool")
release_kv_cache(req, self.tree_cache)
if self.disaggregation_mode == DisaggregationMode.PREFILL:
self.release_aborted_prefill_waiting_req(req)
@@ -5497,6 +5499,10 @@ class Scheduler(
for decode_req in self.disagg_decode_transfer_queue.queue:
if recv_req.abort_all or decode_req.req.rid.startswith(recv_req.rid):
logger.debug(f"Abort transfer queue request. {decode_req.req.rid=}")
if decode_req.host_staged:
# Keep the host destination alive until prefill stops writing.
prepare_abort(decode_req.req, "Aborted by AbortReq.")
continue
receiver = decode_req.kv_receiver
receiver.abort()
# Arm drain-ack accounting once the ABORT is sent, so acks
@@ -5518,7 +5524,7 @@ class Scheduler(
remaining_retracted = []
for decode_req in self.disagg_decode_prealloc_queue.retracted_queue:
if recv_req.abort_all or decode_req.rid.startswith(recv_req.rid):
retraction_discard(
discard_kv_cache_backup(
decode_req,
self.tree_cache,
get_disagg().disaggregation_decode_retraction_backup,
@@ -843,12 +843,7 @@ class SchedulerMetricsReporter:
self.stats.kv_transfer_speed_gb_s = self.kv_transfer_speed_gb_s
self.stats.kv_transfer_latency_ms = self.kv_transfer_latency_ms
elif self.scheduler.disaggregation_mode == DisaggregationMode.DECODE:
self.stats.num_decode_prealloc_queue_reqs = QueueCount.from_reqs(
self.scheduler.disagg_decode_prealloc_queue.queue, priority_enabled
)
self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs(
self.scheduler.disagg_decode_transfer_queue.queue, priority_enabled
)
self._update_decode_queue_stats(priority_enabled)
# Utilization / LoRA / HiCache
self._calculate_utilization()
@@ -981,9 +976,12 @@ class SchedulerMetricsReporter:
spec_num_draft_tokens = spec_snapshot["num_draft_tokens"]
if self.scheduler.disaggregation_mode == DisaggregationMode.DECODE:
self._update_decode_queue_stats(self.scheduler.enable_priority_scheduling)
msg += f"pre-allocated usage: {self.scheduler.disagg_decode_prealloc_queue.num_tokens_pre_allocated / self.scheduler.max_total_num_tokens:.2f}, "
msg += f"#prealloc-req: {len(self.scheduler.disagg_decode_prealloc_queue.queue)}, "
msg += f"#transfer-req: {len(self.scheduler.disagg_decode_transfer_queue.queue)}, "
if get_disagg().disaggregation_decode_host_receive_threshold > 0:
msg += f"#host-receive-req: {self.stats.num_decode_host_receive_queue_reqs.total}, "
msg += f"#retracted-req: {len(self.scheduler.disagg_decode_prealloc_queue.retracted_queue)}, "
if (
@@ -1069,13 +1067,6 @@ class SchedulerMetricsReporter:
self.stats.num_prefill_inflight_queue_reqs = QueueCount.from_reqs(
self.scheduler.disagg_prefill_inflight_queue, priority_enabled
)
elif self.scheduler.disaggregation_mode == DisaggregationMode.DECODE:
self.stats.num_decode_prealloc_queue_reqs = QueueCount.from_reqs(
self.scheduler.disagg_decode_prealloc_queue.queue, priority_enabled
)
self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs(
self.scheduler.disagg_decode_transfer_queue.queue, priority_enabled
)
# Streaming session metrics
self.stats.num_streaming_sessions = (
@@ -1377,6 +1368,23 @@ class SchedulerMetricsReporter:
self.fwd_occupancy = float("nan")
self.stats.fwd_occupancy = float("nan")
def _update_decode_queue_stats(self, priority_enabled: bool) -> None:
transfer_queue = self.scheduler.disagg_decode_transfer_queue.queue
self.stats.num_decode_prealloc_queue_reqs = QueueCount.from_reqs(
self.scheduler.disagg_decode_prealloc_queue.queue, priority_enabled
)
self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs(
transfer_queue, priority_enabled
)
host_reqs = (
[req for req in transfer_queue if req.host_staged]
if get_disagg().disaggregation_decode_host_receive_threshold > 0
else []
)
self.stats.num_decode_host_receive_queue_reqs = QueueCount.from_reqs(
host_reqs, priority_enabled
)
def _maybe_log_idle_metrics(self):
"""Reset forward timing and publish idle metrics when needed."""
# Preserve the transition so the rate limit cannot leave a finite idle gauge.
@@ -1427,10 +1435,5 @@ class SchedulerMetricsReporter:
self.scheduler.disagg_prefill_inflight_queue, priority_enabled
)
if self.scheduler.disaggregation_mode == DisaggregationMode.DECODE:
self.stats.num_decode_prealloc_queue_reqs = QueueCount.from_reqs(
self.scheduler.disagg_decode_prealloc_queue.queue, priority_enabled
)
self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs(
self.scheduler.disagg_decode_transfer_queue.queue, priority_enabled
)
self._update_decode_queue_stats(priority_enabled)
self.metrics_collector.log_stats(self.stats)
+8 -6
View File
@@ -213,7 +213,7 @@ def dsv41_dspark_needs_rebootstrap(
return isinstance(pool, DeepSeekV4TokenToKVPool) and 2 in pool.compression_ratios
def retraction_backup(
def backup_kv_cache(
req: Req,
tree_cache: BasePrefixCache,
req_to_token_pool: ReqToTokenPool,
@@ -236,11 +236,11 @@ def retraction_backup(
return True
unified_cache = cast("UnifiedRadixCache", tree_cache)
req.kv.retraction_backup = unified_cache.retraction_backup(req)
req.kv.retraction_backup = unified_cache.backup_kv_cache(req)
return req.kv.retraction_backup is not None
def retraction_restore(
def restore_kv_cache(
req: Req,
tree_cache: BasePrefixCache,
req_to_token_pool: ReqToTokenPool,
@@ -257,11 +257,13 @@ def retraction_restore(
unified_cache = cast("UnifiedRadixCache", tree_cache)
assert req.kv.retraction_backup is not None
unified_cache.retraction_restore(req, req.kv.retraction_backup)
unified_cache.restore_kv_cache(req, req.kv.retraction_backup)
req.kv.retraction_backup = None
def retraction_discard(req: Req, tree_cache: BasePrefixCache, backend: str) -> None:
def discard_kv_cache_backup(
req: Req, tree_cache: BasePrefixCache, backend: str
) -> None:
if backend == "cpu_tensor":
req.kv.retraction_backup = None
return
@@ -271,7 +273,7 @@ def retraction_discard(req: Req, tree_cache: BasePrefixCache, backend: str) -> N
return
unified_cache = cast("UnifiedRadixCache", tree_cache)
unified_cache.retraction_discard(req.kv.retraction_backup)
unified_cache.discard_kv_cache_backup(req.kv.retraction_backup)
req.kv.retraction_backup = None
@@ -55,7 +55,7 @@ from sglang.srt.runtime_context import (
get_schedule,
)
from sglang.srt.speculative.base_spec_worker import HiCacheDraftMode
from sglang.srt.utils import is_hip
from sglang.srt.utils import ceil_align, is_hip
if TYPE_CHECKING:
from torch.distributed import ProcessGroup
@@ -142,11 +142,16 @@ def maybe_register_hicache_draft(
# Host slots a backup-only retraction pool gets, as a fraction of the device
# pool. Sized well under 1.0 because a retraction burst touches a fraction of
# the device tokens; overflow aborts the request rather than pre-reserving.
# pool, with a floor large enough for one maximum-length request.
BACKUP_ONLY_HICACHE_RATIO = 0.2
def decode_retraction_max_tokens(req_to_token_pool, kv_cache) -> int:
return ceil_align(
min(req_to_token_pool.max_context_len - 1, kv_cache.size), kv_cache.page_size
)
def uses_ssm_state(model_config) -> bool:
"""Whether the model keeps recurrent/conv state alongside its attention KV."""
spec = linear_attn_model_spec(model_config)
@@ -212,11 +217,19 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str:
if memory.hicache_ratio is None:
# Only a decode server reaches resolution with the ratio unset. A
# backup-only pool can be small: retractions that overflow it abort their
# request instead of crashing the scheduler. Sharing the pool with
# HiCache keeps the standard default.
# backup-only pool fits at least one runnable request. Sharing the pool
# with prefix caching keeps the standard HiCache default.
if backend == "host_pool" and not memory.enable_hierarchical_cache:
fields["hicache_ratio"] = BACKUP_ONLY_HICACHE_RATIO
ratio = BACKUP_ONLY_HICACHE_RATIO
if memory.hicache_size == 0:
req_pool, allocator = tp_worker.get_memory_pool()
kv_pool = allocator.get_kvcache()
min_tokens = decode_retraction_max_tokens(req_pool, kv_pool)
if disagg.disaggregation_decode_host_receive_threshold > 0:
# One request can receive while another is retracted.
min_tokens *= 2
ratio = max(ratio, min_tokens / kv_pool.size)
fields["hicache_ratio"] = ratio
else:
fields["hicache_ratio"] = 2.0
@@ -7,6 +7,7 @@ from typing import Any
import torch
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool
@dataclass
@@ -75,6 +76,63 @@ class HostPoolGroup:
def get_pool(self, name: PoolName):
return self.get_entry(name).host_pool
def get_contiguous_buf_infos(self):
"""Return (device_buffers, host_buffers), each (ptrs, sizes, item_sizes)."""
host_by_device_ptr = {}
device_infos = ([], [], [])
for entry in self.entries:
host = entry.host_pool
pools = (entry.device_pool, *entry.packed_draft_device_pools)
for pool in pools:
dense_mha = (
type(pool) is MHATokenToKVPool
and pool.kv_cache_layout == "nhd"
and pool.v_head_dim == pool.head_dim
)
if (
not dense_mha
or pool.layer_shard_enabled
or pool.page_size != self.page_size
):
raise ValueError(
"Host receive requires dense NHD MHA target and "
"draft KV with matching page sizes"
)
for combined, values in zip(
device_infos, pool.get_contiguous_buf_infos(), strict=True
):
combined.extend(values)
# Packed MHA stores target/draft K followed by target/draft V,
# while the wire lists target K/V followed by draft K/V. Associate
# each host view with its device buffer before applying wire order.
device_buffers = [b for p in pools for b in p.k_buffer] + [
b for p in pools for b in p.v_buffer
]
for device_buffer, host_buffer in zip(
device_buffers, host.host_kv_data_refs, strict=True
):
if (
not host_buffer.is_contiguous()
or host_buffer.shape[1:] != device_buffer.shape[1:]
):
raise ValueError(
"KV transfer requires matching contiguous per-layer "
"device and host buffers"
)
host_by_device_ptr[device_buffer.data_ptr()] = (
host_buffer.data_ptr(),
host_buffer.nbytes,
host.token_stride_size * self.page_size,
)
infos = [host_by_device_ptr[ptr] for ptr in device_infos[0]]
host_infos = (
[info[0] for info in infos],
[info[1] for info in infos],
[info[2] for info in infos],
)
return device_infos, host_infos
def alloc(
self,
need_size: int,
@@ -1469,7 +1469,7 @@ class UnifiedRadixCache(BasePrefixCache):
return 0
return self.evict_host(num_tokens)
def retraction_backup(self, req: Req) -> Optional[RetractionBackup]:
def backup_kv_cache(self, req: Req) -> Optional[RetractionBackup]:
"""Back up device KV to the host pool; None when it cannot fit after reclaim."""
assert req.seqlen > 1
@@ -1512,11 +1512,11 @@ class UnifiedRadixCache(BasePrefixCache):
)
completion.finish_event.synchronize()
except Exception:
self.retraction_discard(backup)
self.discard_kv_cache_backup(backup)
raise
return backup
def retraction_restore(self, req: Req, backup: RetractionBackup) -> None:
def restore_kv_cache(self, req: Req, backup: RetractionBackup) -> None:
device_indices, current_transfers = self._retraction_device_transfers(req)
assert len(backup.host_indices) == len(device_indices), (
f"Host backup has {len(backup.host_indices)} slots, but restore has "
@@ -1561,9 +1561,9 @@ class UnifiedRadixCache(BasePrefixCache):
transfer_layer_id_max=self.cache_controller.transfer_layer_id_max,
)
completion.finish_event.synchronize()
self.retraction_discard(backup)
self.discard_kv_cache_backup(backup)
def retraction_discard(self, backup: RetractionBackup) -> None:
def discard_kv_cache_backup(self, backup: RetractionBackup) -> None:
self.host_pool_group.free(backup.host_indices)
self.host_pool_group.release_transfers(backup.pool_transfers)
@@ -135,6 +135,7 @@ class SchedulerStats:
num_prefill_inflight_queue_reqs: QueueCount = field(default_factory=QueueCount)
num_decode_prealloc_queue_reqs: QueueCount = field(default_factory=QueueCount)
num_decode_transfer_queue_reqs: QueueCount = field(default_factory=QueueCount)
num_decode_host_receive_queue_reqs: QueueCount = field(default_factory=QueueCount)
kv_transfer_speed_gb_s: float = 0.0
kv_transfer_latency_ms: float = 0.0
pending_prealloc_token_usage: float = 0.0
@@ -519,6 +520,17 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
self.num_decode_host_receive_queue_reqs = Gauge(
name="sglang:num_decode_host_receive_queue_reqs",
documentation="Requests in the decode transfer queue receiving into host memory or waiting for device admission.",
labelnames=labels.keys(),
multiprocess_mode="mostrecent",
)
self.num_decode_host_receive_reqs = Counter(
name="sglang:num_decode_host_receive_reqs_total",
documentation="Total requests admitted to receive prefill KV in host memory.",
labelnames=labels.keys(),
)
self.kv_transfer_speed_gb_s = Histogram(
name="sglang:kv_transfer_speed_gb_s",
documentation="Histogram of KV cache transfer speed in GB/s.",
@@ -1186,6 +1198,9 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
def increment_transfer_failed_reqs(self) -> None:
self.num_transfer_failed_reqs.labels(**self.labels).inc(1)
def increment_decode_host_receive_reqs(self) -> None:
self.num_decode_host_receive_reqs.labels(**self.labels).inc(1)
def increment_prefill_retries(self, count: int) -> None:
if count > 0:
self.num_prefill_retries_total.labels(**self.labels).inc(count)
@@ -1414,6 +1429,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
self._log_gauge_queue_count(
self.num_decode_transfer_queue_reqs, stats.num_decode_transfer_queue_reqs
)
self._log_gauge_queue_count(
self.num_decode_host_receive_queue_reqs,
stats.num_decode_host_receive_queue_reqs,
)
self._log_gauge(
self.pending_prealloc_token_usage, stats.pending_prealloc_token_usage
)