From efff836911526784ce765780a6d326626b30ed8e Mon Sep 17 00:00:00 2001 From: cctry Date: Wed, 23 Sep 2026 21:57:33 -0700 Subject: [PATCH] [PD] Add decode host receive for custom transfer backends (#40238) Co-authored-by: cctry --- python/sglang/srt/arg_groups/field_order.py | 20 + python/sglang/srt/arg_groups/fields/disagg.py | 4 + python/sglang/srt/arg_groups/hicache_hook.py | 14 + .../srt/arg_groups/pd_disaggregation_hook.py | 18 + python/sglang/srt/disaggregation/base/conn.py | 15 + .../sglang/srt/disaggregation/common/conn.py | 2 + python/sglang/srt/disaggregation/decode.py | 396 +++++++++++---- python/sglang/srt/managers/schedule_batch.py | 4 +- python/sglang/srt/managers/scheduler.py | 10 +- .../scheduler_components/metrics_reporter.py | 41 +- python/sglang/srt/mem_cache/common.py | 14 +- .../sglang/srt/mem_cache/kv_cache_builder.py | 27 +- .../sglang/srt/mem_cache/pool_host/group.py | 58 +++ .../srt/mem_cache/unified_radix_cache.py | 10 +- .../srt/observability/metrics_collector.py | 19 + .../test_disaggregation_basic.py | 4 + .../ops/disaggregation/test_kv_checksum.py | 41 +- .../test_admission_abort_not_enqueued.py | 6 +- .../test_decode_queue_cleanup.py | 32 ++ .../test_deferred_decode_kv_release.py | 36 +- .../disaggregation/test_pd_role_switch.py | 1 + ...test_priority_scheduling_disaggregation.py | 3 + .../test_decode_retraction_backup.py | 456 ++++++++++++++++-- .../test_forward_pass_metrics.py | 61 ++- .../unit/server_args/test_server_args.py | 29 +- 25 files changed, 1144 insertions(+), 177 deletions(-) diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py index e20da4374b..7890b2bdbd 100644 --- a/python/sglang/srt/arg_groups/field_order.py +++ b/python/sglang/srt/arg_groups/field_order.py @@ -497,5 +497,25 @@ POSITIONAL_FIELD_ORDER = ( "weight_cache_timeout", "forward_hooks", "msprobe_dump_config", + "ple_offload_embedding", + "ple_offload_backend", + "ple_offload_dir", + "disaggregation_enable_kv_checksum", + "enable_pd_role_switch", + "speculative_domino_candidate_pool_size", + "flashinfer_a2a_dispatch_type", + "cuda_graph_max_seq_len_prefill", + "prefill_kv_cache_dequant_dtype", + "dsv4_attn_backend", + "otlp_service_name", + "_radix_eviction_policy_explicitly_set", + "hicache_host_memory_fraction", + "enable_linker_mla_dedup", + "_swa_full_tokens_ratio_explicitly_set", + "swa_prefix_tails", + "enable_encoder_swa_bounded_replay", + "enable_decoder_swa_bounded_replay", + "enable_response_store", + "disaggregation_decode_host_receive_threshold", ) # fmt: on diff --git a/python/sglang/srt/arg_groups/fields/disagg.py b/python/sglang/srt/arg_groups/fields/disagg.py index 2451ce6329..d16f10ceee 100644 --- a/python/sglang/srt/arg_groups/fields/disagg.py +++ b/python/sglang/srt/arg_groups/fields/disagg.py @@ -168,3 +168,7 @@ class Disagg(msgspec.Struct): "The path of the PD-Multiplexing config file.", ] = None sm_group_num: A[int, "Number of sm partition groups."] = 8 + disaggregation_decode_host_receive_threshold: A[ + float, + "Device token usage fraction at which incoming KV is received in the decode retraction host pool, excluding evictable cache pages. Range [0, 1]; 0 disables host receive. Size with --hicache-size or --hicache-ratio; requires dense MHA and a transfer backend that supports host destinations. No built-in backend currently supports this.", + ] = 0.0 diff --git a/python/sglang/srt/arg_groups/hicache_hook.py b/python/sglang/srt/arg_groups/hicache_hook.py index 757eaa92e5..457f6a0428 100644 --- a/python/sglang/srt/arg_groups/hicache_hook.py +++ b/python/sglang/srt/arg_groups/hicache_hook.py @@ -54,11 +54,25 @@ def handle_hicache(server_args: Any): validate_hicache_host_memory_mode(server_args) + if cfg.disaggregation_decode_host_receive_threshold > 0: + # The shared host pool must match KV transfer's per-layer buffers. + declare_resolution( + server_args, "handle_hicache", hicache_mem_layout="layer_first" + ) + # Step 1: Initial layout-io compatibility normalization. resolve_layout_io_compatibility(server_args) # Step 2: Storage-layout normalization without changing io backend. resolve_storage_layout_compatibility(server_args) + if ( + cfg.disaggregation_decode_host_receive_threshold > 0 + and cfg.hicache_mem_layout != "layer_first" + ): + raise ValueError( + f"The resolved HiCache storage layout {cfg.hicache_mem_layout!r} " + "cannot share the layer_first decode host pool used by KV transfer" + ) # Step 3: DCP compatibility for the L2 (device<->host) path. resolve_hicache_dcp_compatibility(server_args) diff --git a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py index e8c50177aa..4124b23c5d 100644 --- a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py +++ b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py @@ -50,6 +50,24 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: "overhead without improving prefill performance." ) + if not 0 <= cfg.disaggregation_decode_host_receive_threshold <= 1: + raise ValueError( + "--disaggregation-decode-host-receive-threshold must be between 0 and 1" + ) + if cfg.disaggregation_decode_host_receive_threshold > 0: + if cfg.enable_hisparse or cfg.enable_pd_role_switch: + raise ValueError( + "Decode host receive does not yet support HiSparse or role switching" + ) + + if cfg.disaggregation_decode_retraction_backup == "cpu_tensor": + raise ValueError("Decode host KV buffering requires host_pool retraction") + declare_resolution( + server_args, + "handle_pd_disaggregation", + disaggregation_decode_retraction_backup="host_pool", + ) + if cfg.disaggregation_mode == "decode" and cfg.dcp_size > 1: # Fake transfer moves no KV and is only used for synthetic decode # benchmarks, so it does not need the DCP relayout from Mooncake/NIXL. diff --git a/python/sglang/srt/disaggregation/base/conn.py b/python/sglang/srt/disaggregation/base/conn.py index 0cc63380b9..bcc8847895 100644 --- a/python/sglang/srt/disaggregation/base/conn.py +++ b/python/sglang/srt/disaggregation/base/conn.py @@ -46,6 +46,11 @@ class KVTransferMetric: transfer_total_bytes: Optional[int] = None +class KVTransferDestination(str, enum.Enum): + DEVICE = "device" + HOST = "host" + + class KVArgs: engine_rank: int kv_data_ptrs: List[int] @@ -53,6 +58,9 @@ class KVArgs: kv_item_lens: List[int] kv_layer_ids: List[int] kv_cache_dtype_str: str + host_kv_data_ptrs: Optional[List[int]] = None + host_kv_data_lens: Optional[List[int]] = None + host_kv_item_lens: Optional[List[int]] = None aux_data_ptrs: List[int] aux_data_lens: List[int] aux_item_lens: List[int] @@ -113,6 +121,7 @@ class BaseKVManager(ABC): """Base class for managing transfer states""" enable_deferred_decode_kv_release: bool = False + supports_host_destination: bool = False @abstractmethod def __init__( @@ -212,6 +221,11 @@ class BaseKVSender(ABC): class BaseKVReceiver(ABC): + @property + def supports_host_destination(self) -> bool: + """Whether this receiver's peer and layout support host KV destinations.""" + return False + @abstractmethod def __init__( self, @@ -237,6 +251,7 @@ class BaseKVReceiver(ABC): aux_index: Optional[int] = None, state_indices: Optional[List] = None, decode_prefix_len: Optional[int] = None, + destination: KVTransferDestination = KVTransferDestination.DEVICE, ): """ Notify the prefill server about the kv indices, aux index, and state_indices. diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index 5cf9d06089..85353f8755 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -23,6 +23,7 @@ from sglang.srt.disaggregation.base.conn import ( BaseKVSender, KVArgs, KVPoll, + KVTransferDestination, KVTransferMetric, StateType, ) @@ -1931,6 +1932,7 @@ class CommonKVReceiver(BaseKVReceiver): aux_index: Optional[int] = None, state_indices: Optional[List[int]] = None, decode_prefix_len: Optional[int] = None, + destination: KVTransferDestination = KVTransferDestination.DEVICE, ): raise NotImplementedError diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 3a2a8eacfe..62c584e2f6 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -36,7 +36,7 @@ from torch.distributed import ProcessGroup from sglang.srt.configs.mamba_utils import Mamba2CacheParams from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE from sglang.srt.disaggregation.base import KVPoll -from sglang.srt.disaggregation.base.conn import StateType +from sglang.srt.disaggregation.base.conn import KVTransferDestination, StateType from sglang.srt.disaggregation.checksum import ( KvChecksumComputer, is_health_check_req, @@ -88,14 +88,17 @@ from sglang.srt.mem_cache.base_prefix_cache import ( EvictParams, ) from sglang.srt.mem_cache.common import ( + RetractionBackup, + discard_kv_cache_backup, dsv41_dspark_needs_rebootstrap, kv_to_page_indices, page_align_floor, release_kv_cache, - retraction_discard, - retraction_restore, + restore_kv_cache, ) from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool +from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer +from sglang.srt.mem_cache.kv_cache_builder import decode_retraction_max_tokens from sglang.srt.mem_cache.memory_pool import ( HybridReqToTokenPool, KVCache, @@ -330,6 +333,7 @@ class DecodeRequest: waiting_for_input: bool = False metadata_buffer_index: int = -1 is_rebootstrap: bool = False + host_staged: bool = False # HiCache Status prefix_match: Optional[DecodePrefixMatch] = None @@ -382,6 +386,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): self.scheduler = scheduler self.transfer_queue = transfer_queue self.tree_cache = tree_cache + self.host_pool = None + self.host_reserved_tokens = 0 self.gloo_group = gloo_group # Destinations visible to prefill but not yet on the transfer queue. self._num_published_destinations = 0 @@ -460,6 +466,17 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): or self.token_to_kv_pool_allocator.prealloc_fits_assumes_reclaim() ) + def _required_admission_tokens( + self, req: Req, allocated_tokens: int, prefix_len: int, retractable_tokens: int + ) -> int: + return max( + allocated_tokens + self.num_reserved_decode_tokens, + self._rebootstrap_prefill_len(req) + - prefix_len + + min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKEN) + - retractable_tokens, + ) + def _prealloc_reservation_fits( self, full_tokens: int, @@ -630,6 +647,31 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): total_kv_layers=self.scheduler.model_config.num_hidden_layers, req_to_token_pool=getattr(self, "req_to_token_pool", None), ) + if get_disagg().disaggregation_decode_host_receive_threshold > 0: + pool = self.token_to_kv_pool + group = self.tree_cache.host_pool_group + if kv_args.state_types or any( + spec.indices_from_pool != PoolName.KV + for spec in self.tree_cache.sidecar_pool_specs + ): + raise ValueError( + "Host receive requires KV pools sharing the primary indices" + ) + self.host_pool = group.get_pool(PoolName.KV) + self.host_reserved_tokens = decode_retraction_max_tokens( + self.req_to_token_pool, pool + ) + if self.host_pool.logical_size < self.host_reserved_tokens + pool.page_size: + raise ValueError( + "Host pool must hold a retraction and a receive page; " + "increase --hicache-size or --hicache-ratio" + ) + _, host_buffers = group.get_contiguous_buf_infos() + ( + kv_args.host_kv_data_ptrs, + kv_args.host_kv_data_lens, + kv_args.host_kv_item_lens, + ) = host_buffers kv_args.ib_device = get_disagg().disaggregation_ib_device kv_args.gpu_id = get_device().gpu_id @@ -640,6 +682,13 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): self.scheduler.server_args, self.is_mla_backend, ) + if ( + get_disagg().disaggregation_decode_host_receive_threshold > 0 + and not kv_manager.supports_host_destination + ): + raise ValueError( + "Transfer backend does not support decode host KV destinations" + ) # Staging buffer setup (only when heterogeneous TP staging is enabled) if self.enable_staging and not self.is_mla_backend: kv_pool_for_heads = self.token_to_kv_pool @@ -872,7 +921,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): self._cancel_prefill_dp_rank_queries() self.queue.clear() for req in self.retracted_queue: - retraction_discard( + discard_kv_cache_backup( req, self.tree_cache, get_disagg().disaggregation_decode_retraction_backup, @@ -941,7 +990,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): extra_reserved_reqs=len(resumed_reqs), ) - retraction_restore( + restore_kv_cache( req, self.tree_cache, self.req_to_token_pool, @@ -1186,6 +1235,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): failed_reqs = [] preallocated_reqs = [] + num_device_preallocated = 0 indices_to_remove = set() # We need to make sure that the sum of inflight tokens and allocatable tokens is greater than maximum input+output length of each inflight request @@ -1285,6 +1335,30 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): if not decode_req.waiting_for_input: continue + if self.req_to_metadata_buffer_idx_allocator.available_size() <= 0: + break + + if self.scheduler.enable_lora and not self.scheduler.can_schedule_lora_req( + decode_req.req, running_loras + ): + continue + + if ( + get_disagg().disaggregation_decode_host_receive_threshold > 0 + and not decode_req.is_rebootstrap + and not _is_fake_transfer(decode_req.req) + and decode_req.kv_receiver.supports_host_destination + and self.scheduler.pool_stats_observer.get_pool_stats().full_token_usage + >= get_disagg().disaggregation_decode_host_receive_threshold + ): + if not self._pre_alloc_host(decode_req): + break + preallocated_reqs.append(decode_req) + indices_to_remove.add(i) + if self.scheduler.enable_lora: + running_loras.add(decode_req.req.lora_id) + continue + if self.req_to_token_pool.available_size() <= 0: break @@ -1301,17 +1375,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): if mamba_allocator.available_size() <= 0: break - if self.req_to_metadata_buffer_idx_allocator.available_size() <= 0: - break - if hisparse_req_budget <= 0: break - if self.scheduler.enable_lora and not self.scheduler.can_schedule_lora_req( - decode_req.req, running_loras - ): - continue - # Memory estimation: don't add if the projected memory cannot be met # TODO: add new_token ratio origin_input_len = self._rebootstrap_prefill_len(decode_req.req) @@ -1369,7 +1435,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): full_allocatable_tokens = self._allocatable_token_budgets( retractable_tokens=retractable_tokens, count_retracted=True, - extra_reserved_reqs=len(preallocated_reqs), + extra_reserved_reqs=num_device_preallocated, hicache_reserved_tokens=reserved_restore_tokens, ) if uses_swa_tail_prealloc: @@ -1377,7 +1443,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): retractable_tokens=retractable_tokens, retractable_swa_tokens=retractable_swa_tokens, count_retracted=True, - extra_reserved_reqs=len(preallocated_reqs), + extra_reserved_reqs=num_device_preallocated, ) else: prefix_indices = None @@ -1385,19 +1451,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): total_prefix_len = 0 required_alloc_tokens = self._pre_alloc_fill_len(decode_req.req) - required_tokens_for_request = ( - required_alloc_tokens + self.num_reserved_decode_tokens - ) - - full_required_for_admission = max( - required_tokens_for_request, - origin_input_len - - prefix_len - + min( - decode_req.req.sampling_params.max_new_tokens, - CLIP_MAX_NEW_TOKEN, - ) - - retractable_tokens, + full_required_for_admission = self._required_admission_tokens( + decode_req.req, required_alloc_tokens, prefix_len, retractable_tokens ) swa_required_for_admission = 0 swa_len = required_alloc_tokens @@ -1465,7 +1520,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): full_allocatable_tokens = self._allocatable_token_budgets( retractable_tokens=retractable_tokens, count_retracted=True, - extra_reserved_reqs=len(preallocated_reqs) + 1, + extra_reserved_reqs=num_device_preallocated + 1, hicache_reserved_tokens=reserved_restore_tokens, ) if swa_allocatable_tokens is not None: @@ -1473,7 +1528,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): retractable_tokens=retractable_tokens, retractable_swa_tokens=retractable_swa_tokens, count_retracted=True, - extra_reserved_reqs=len(preallocated_reqs) + 1, + extra_reserved_reqs=num_device_preallocated + 1, ) decode_req.req.kv.cache_protected_len = total_prefix_len @@ -1601,14 +1656,6 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): payloads[st]() if st in payloads else None for st in state_types ] - decode_req.metadata_buffer_index = ( - self.req_to_metadata_buffer_idx_allocator.alloc() - ) - assert decode_req.metadata_buffer_index is not None - # int32 for ZMQ serialization -- from_zmq reads np.int32. - page_indices = kv_to_page_indices(kv_indices, kv_transfer_page_size).astype( - np.int32 - ) device_page_indices = None if ( self.scheduler.enable_hisparse @@ -1635,33 +1682,18 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): metadata_kwargs = {"decode_prefix_len": total_prefix_len} if device_page_indices is not None: metadata_kwargs["device_kv_indices"] = device_page_indices - if ( - self.transfer_queue.enable_staging - and hasattr(decode_req.kv_receiver, "require_staging") - and decode_req.kv_receiver.require_staging - ): - # Register before send_metadata, which triggers the STAGING_REQ - # prefetch (dropped for an unregistered room); tiny race, correct order. - self.transfer_queue.staging_handler.register_decode_req( - decode_req.req.bootstrap_room, decode_req - ) - decode_req.kv_receiver.send_metadata( - page_indices, - decode_req.metadata_buffer_index, + self._send_kv_metadata( + decode_req, + kv_indices, + kv_transfer_page_size, state_indices, **metadata_kwargs, ) - if decode_req.is_rebootstrap: - self.kv_manager.submit_prefill_recompute( - decode_req.kv_receiver, - decode_req.req.build_rebootstrap_payload(), - ) - self._num_published_destinations += 1 + num_device_preallocated += 1 preallocated_reqs.append(decode_req) indices_to_remove.add(i) if self.scheduler.enable_lora: running_loras.add(decode_req.req.lora_id) - decode_req.req.time_stats.set_decode_transfer_queue_entry_time() if failed_reqs: failed_ids = {id(r) for r in failed_reqs} @@ -1675,6 +1707,121 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): return preallocated_reqs, failed_reqs + def _send_kv_metadata( + self, decode_req, kv_indices, page_size, state_indices=None, **metadata_kwargs + ) -> None: + decode_req.metadata_buffer_index = ( + self.req_to_metadata_buffer_idx_allocator.alloc() + ) + assert decode_req.metadata_buffer_index is not None + page_indices = kv_to_page_indices(kv_indices, page_size).astype(np.int32) + if ( + metadata_kwargs.get("destination") != KVTransferDestination.HOST + and self.transfer_queue.enable_staging + and hasattr(decode_req.kv_receiver, "require_staging") + and decode_req.kv_receiver.require_staging + ): + # Register before send_metadata, which triggers the STAGING_REQ + # prefetch (dropped for an unregistered room); tiny race, correct order. + self.transfer_queue.staging_handler.register_decode_req( + decode_req.req.bootstrap_room, decode_req + ) + decode_req.kv_receiver.send_metadata( + page_indices, + decode_req.metadata_buffer_index, + state_indices, + **metadata_kwargs, + ) + if decode_req.is_rebootstrap: + self.kv_manager.submit_prefill_recompute( + decode_req.kv_receiver, + decode_req.req.build_rebootstrap_payload(), + ) + self._num_published_destinations += 1 + decode_req.req.time_stats.set_decode_transfer_queue_entry_time() + + def _pre_alloc_host(self, decode_req: DecodeRequest) -> bool: + num_tokens = ceil_align( + len(decode_req.req.origin_input_ids), self.host_pool.page_size + ) + required_tokens = num_tokens + self.host_reserved_tokens + if ( + self.host_pool.available_size() < required_tokens + and not self.tree_cache.disable + ): + self.tree_cache.evict_host(required_tokens) + if self.host_pool.available_size() < required_tokens: + return False + host_indices = self.host_pool.alloc(num_tokens) + if host_indices is None: + return False + assert decode_req.req.kv.retraction_backup is None + # All sidecars share the primary KV indices. + decode_req.req.kv.retraction_backup = RetractionBackup( + host_indices=host_indices, + pool_transfers=[ + PoolTransfer( + name=spec.pool_name, + host_indices=host_indices, + indices_from_pool=spec.indices_from_pool, + hit_policy=spec.hit_policy, + ) + for spec in self.tree_cache.sidecar_pool_specs + if num_tokens + ] + or None, + ) + if get_disagg().disaggregation_decode_enable_radix_cache: + req = decode_req.req + req.prefix_indices = torch.empty((0,), dtype=torch.int64) + req.last_node = self.tree_cache.root_node_handle(req.extra_key) + req.last_host_node = req.last_node + req.best_match_node = req.last_node + req.lock_receipt = DecLockRefParams() + req.kv.cache_protected_len = 0 + req.num_matched_prefix_tokens = 0 + req.host_hit_length = 0 + self._send_kv_metadata( + decode_req, + host_indices, + self.token_to_kv_pool_allocator.page_size, + decode_prefix_len=0, + destination=KVTransferDestination.HOST, + ) + decode_req.host_staged = True + if self.scheduler.metrics_reporter.current_scheduler_metrics_enabled: + self.scheduler.metrics_collector.increment_decode_host_receive_reqs() + return True + + def allocate_host_staged(self, decode_req: DecodeRequest) -> bool: + if self.req_to_token_pool.available_size() <= 0: + return False + req = decode_req.req + retractable_tokens = sum( + len(r.origin_input_ids) + len(r.output_ids) + for r in self.scheduler.running_batch.reqs + ) + required_tokens = self._required_admission_tokens( + req, + self._required_alloc_tokens( + fill_len=len(req.origin_input_ids), prefix_len=0 + ), + 0, + retractable_tokens, + ) + if not self._prealloc_reservation_fits( + required_tokens, + 0, + full_allocatable_tokens=self._allocatable_token_budgets( + retractable_tokens=retractable_tokens, count_retracted=True + ), + swa_allocatable_tokens=None, + ): + return False + self._pre_alloc(req) + decode_req.host_staged = False + return True + @property def has_published_destinations(self) -> bool: """Whether any destination address is visible to prefill but not yet @@ -1690,7 +1837,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): @property def num_tokens_pre_allocated(self): return sum( - decode_req.req.extend_range.end for decode_req in self.transfer_queue.queue + decode_req.req.extend_range.end + for decode_req in self.transfer_queue.queue + if not decode_req.host_staged ) def _need_space_for_single_req( @@ -1714,7 +1863,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): def _active_req_count(self, extra_reserved_reqs: int = 0) -> int: return ( len(self.scheduler.running_batch.reqs) - + len(self.transfer_queue.queue) + + sum(not r.host_staged for r in self.transfer_queue.queue) + len(self.scheduler.waiting_queue) + extra_reserved_reqs ) @@ -2193,6 +2342,8 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): Store the requests that is polling kv """ + enable_host_receive: bool = False + def __init__( self, gloo_group: ProcessGroup, @@ -2207,6 +2358,9 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): self.tp_rank = get_parallel().tp_rank self.metadata_buffers = metadata_buffers self.scheduler = scheduler + self.enable_host_receive = ( + get_disagg().disaggregation_decode_host_receive_threshold > 0 + ) self.tree_cache = tree_cache self.spec_algorithm = scheduler.spec_algorithm self.enable_staging = envs.SGLANG_DISAGG_STAGING_BUFFER.get() @@ -2444,6 +2598,13 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): ) kv_manager._staging_handler = self.staging_handler + def _release_request(self, decode_req: DecodeRequest) -> None: + if self.enable_host_receive: + discard_kv_cache_backup(decode_req.req, self.tree_cache, "host_pool") + if decode_req.host_staged: + return + release_kv_cache(decode_req.req, self.tree_cache, is_insert=False) + def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]: if not self.queue: return [] @@ -2504,21 +2665,24 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): ) if self.scheduler.enable_hisparse: self.scheduler.hisparse_coordinator.request_finished(decode_req.req) - if ( + requires_host_drain = ( + self.enable_host_receive and decode_req.host_staged + ) + if requires_host_drain: + decode_req.kv_receiver.abort() + if requires_host_drain or ( self.enable_deferred_kv_release and decode_req.kv_receiver.kv_mgr.enable_deferred_decode_kv_release and decode_req.kv_receiver.abort_notified ): - # Decode-initiated abort: a prefill write may still target - # these pages, so hold them until the drain ack or timeout. - # (A prefill-initiated failure has already stopped writing -> - # immediate release below.) + # Host pages always await a drain ack. Device pages retain + # the existing opt-in deferred-release behavior. self._defer_release(decode_req) deferred_indices.add(i) indices_to_remove.add(i) else: # release pre-allocated kv cache, but don't insert into the tree since it's failed - release_kv_cache(decode_req.req, self.tree_cache, is_insert=False) + self._release_request(decode_req) decode_req.kv_receiver.clear() decode_req.kv_receiver = None indices_to_remove.add(i) @@ -2526,6 +2690,20 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): self.scheduler.metrics_collector.increment_transfer_failed_reqs() continue elif poll == KVPoll.Success: + if self.enable_host_receive and decode_req.host_staged: + if isinstance(decode_req.req.finished_reason, FINISH_ABORT): + self.scheduler.output_streamer.stream_output( + [decode_req.req], decode_req.req.return_logprob + ) + self._release_request(decode_req) + decode_req.kv_receiver.clear() + decode_req.kv_receiver = None + indices_to_remove.add(i) + continue + if not self.scheduler.disagg_decode_prealloc_queue.allocate_host_staged( + decode_req + ): + continue if ( self.scheduler.enable_decode_hicache and hicache_restore_status == HiCacheRestoreResult.PENDING @@ -2544,7 +2722,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): decode_req.req ) self._clean_hicache_prefetch_resources(decode_req) - release_kv_cache(decode_req.req, self.tree_cache, is_insert=False) + self._release_request(decode_req) if self.scheduler.metrics_reporter.enable_metrics: self.scheduler.metrics_collector.increment_transfer_failed_reqs() else: @@ -2595,7 +2773,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): if self.enable_staging and self.staging_handler.is_staging_room(room): self.staging_handler.unregister_decode_req(room) # release pre-allocated kv cache, but don't insert into the tree since it's failed - release_kv_cache(decode_req.req, self.tree_cache, is_insert=False) + self._release_request(decode_req) self.metadata_buffers.bootstrap_room[idx] = 0 self.req_to_metadata_buffer_idx_allocator.free(idx) decode_req.kv_receiver.kv_mgr.clear_deferred_abort_state(room) @@ -2606,15 +2784,37 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): return bool(self._deferred_releases) def resolve_deferred_releases(self) -> None: - """Release held requests once every prefill rank acks the drain, or the - hold times out.""" + """Release drained transfers; device destinations may also time out.""" if not self._deferred_releases: return + host_ready = torch.tensor( + [ + req.kv_receiver.kv_mgr.is_abort_release_safe( + req.req.bootstrap_room, acks + ) + for req, _, _, acks in self._deferred_releases + if self.enable_host_receive and req.host_staged + ], + dtype=torch.int32, + ) + if host_ready.numel() and torch.distributed.get_world_size(self.gloo_group) > 1: + # Admission must see identical host capacity and page order on every rank. + torch.distributed.all_reduce( + host_ready, op=torch.distributed.ReduceOp.MIN, group=self.gloo_group + ) + host_ready = iter(host_ready.tolist()) now = time.monotonic() still_held = [] to_release = [] for decode_req, deadline, idx, required_acks in self._deferred_releases: room = decode_req.req.bootstrap_room + if self.enable_host_receive and decode_req.host_staged: + if next(host_ready): + to_release.append((decode_req, idx, room, True)) + else: + # A timeout cannot prove that a remote write has stopped. + still_held.append((decode_req, deadline, idx, required_acks)) + continue kv_mgr = decode_req.kv_receiver.kv_mgr drained = kv_mgr.is_abort_release_safe(room, required_acks) if not drained and now < deadline: @@ -2797,25 +2997,39 @@ class SchedulerDisaggregationDecodeMixin: def get_new_prebuilt_batch( self, running_batch: ScheduleBatch ) -> Optional[ScheduleBatch]: + batch = self._get_new_prebuilt_batch(running_batch) + if batch is None: + return None + # Verify after host restore and before process_prebuilt caches any KV. + self._verify_kv_checksums(batch) + if batch.is_empty(): + return None + self.ngram_embedding_manager.prepare_for_forward( + batch, chunked_req=self.chunked_req + ) + batch.process_prebuilt(self.future_map) + return batch + + def _verify_kv_checksums(self, batch: ScheduleBatch) -> None: computer: Optional[KvChecksumComputer] = self.kv_checksum_computer if computer is None: - return self._get_new_prebuilt_batch(running_batch) + return - verified: List[Req] = [] - for req in self.waiting_queue: + verified: List[int] = [] + for i, req in enumerate(batch.reqs): if is_health_check_req(req): - verified.append(req) + verified.append(i) continue expected = req.expected_kv_checksum if expected == 0: - verified.append(req) + verified.append(i) continue seq_len = len(req.origin_input_ids) page_indices_gpu = page_indices_for_request(self, req, seq_len) state_indices = state_indices_for_request(self, req, seq_len) actual = computer.compute(page_indices_gpu, state_indices) if actual == expected: - verified.append(req) + verified.append(i) continue msg = ( f"KV checksum mismatch req={req.rid} " @@ -2824,9 +3038,8 @@ class SchedulerDisaggregationDecodeMixin: ) logger.error(msg) self._handle_kv_checksum_mismatch(req, msg) - self.waiting_queue = verified - - return self._get_new_prebuilt_batch(running_batch) + if len(verified) != len(batch.reqs): + batch.filter_batch(keep_indices=verified) def _handle_kv_checksum_mismatch(self, req: Req, msg: str) -> None: # A mismatch means the KV this worker received is not what prefill sent, @@ -2916,12 +3129,16 @@ class SchedulerDisaggregationDecodeMixin: # A finished request can still have one redundant forward in flight. # Drain it before a prebuilt request seeds a potentially reused row. self.schedule_stream.wait_stream(self.forward_stream) - # The prebuilt batch never reaches the forward loop's prepare call. - self.ngram_embedding_manager.prepare_for_forward( - new_batch, chunked_req=self.chunked_req - ) - new_batch.process_prebuilt(self.future_map) - + if get_disagg().disaggregation_decode_host_receive_threshold > 0: + for req in new_batch.reqs: + if req.kv.retraction_backup is not None: + restore_kv_cache( + req, + self.tree_cache, + self.req_to_token_pool, + self.token_to_kv_pool_allocator, + "host_pool", + ) return new_batch @scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE) @@ -2950,8 +3167,9 @@ class SchedulerDisaggregationDecodeMixin: self.polling_count = (self.polling_count + 1) % self.polling_interval if self.polling_count % self.polling_interval == 0: - req_conns, _ = self.disagg_decode_prealloc_queue.pop_preallocated() - self.disagg_decode_transfer_queue.extend(req_conns) + if get_disagg().disaggregation_decode_host_receive_threshold == 0: + req_conns, _ = self.disagg_decode_prealloc_queue.pop_preallocated() + self.disagg_decode_transfer_queue.extend(req_conns) transferred_reqs = ( self.disagg_decode_transfer_queue.pop_transferred() ) # the requests which kv has arrived @@ -2960,3 +3178,7 @@ class SchedulerDisaggregationDecodeMixin: # Direct-to-host: KV data already in host pool, skip staging self.hisparse_coordinator.admit_request_direct(req) self.waiting_queue.extend(transferred_reqs) + if get_disagg().disaggregation_decode_host_receive_threshold > 0: + # Give completed host transfers device space before new arrivals. + req_conns, _ = self.disagg_decode_prealloc_queue.pop_preallocated() + self.disagg_decode_transfer_queue.extend(req_conns) diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 840080d36c..816ca3041e 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -112,10 +112,10 @@ from sglang.srt.mem_cache.base_prefix_cache import ( ) from sglang.srt.mem_cache.common import ( RetractionBackup, + backup_kv_cache, evict_from_tree_cache, free_swa_out_of_window_slots, release_kv_cache, - retraction_backup, ) from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool from sglang.srt.mem_cache.radix_cache import RadixKey @@ -2239,7 +2239,7 @@ def release_req( backup_saved = True # The config bag reflects role flips; server_args keeps the launch role. if get_disagg().disaggregation_mode == "decode" and offload_kv: - backup_saved = retraction_backup( + backup_saved = backup_kv_cache( req, tree_cache, req_to_token_pool, diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 09f8d79bfd..9adf9d63d5 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -290,9 +290,9 @@ from sglang.srt.managers.utils import ( from sglang.srt.mem_cache import kv_cache_builder from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome from sglang.srt.mem_cache.common import ( + discard_kv_cache_backup, maybe_cache_unfinished_req, release_kv_cache, - retraction_discard, ) from sglang.srt.model_executor.forward_batch_info import PPProxyTensors from sglang.srt.model_executor.runner_utils.pool import prewarm_graph_pool_borrow @@ -5434,6 +5434,8 @@ class Scheduler( ) # For disaggregation decode mode, the request in the waiting queue has KV cache allocated. if self.disaggregation_mode == DisaggregationMode.DECODE: + if get_disagg().disaggregation_decode_host_receive_threshold > 0: + discard_kv_cache_backup(req, self.tree_cache, "host_pool") release_kv_cache(req, self.tree_cache) if self.disaggregation_mode == DisaggregationMode.PREFILL: self.release_aborted_prefill_waiting_req(req) @@ -5497,6 +5499,10 @@ class Scheduler( for decode_req in self.disagg_decode_transfer_queue.queue: if recv_req.abort_all or decode_req.req.rid.startswith(recv_req.rid): logger.debug(f"Abort transfer queue request. {decode_req.req.rid=}") + if decode_req.host_staged: + # Keep the host destination alive until prefill stops writing. + prepare_abort(decode_req.req, "Aborted by AbortReq.") + continue receiver = decode_req.kv_receiver receiver.abort() # Arm drain-ack accounting once the ABORT is sent, so acks @@ -5518,7 +5524,7 @@ class Scheduler( remaining_retracted = [] for decode_req in self.disagg_decode_prealloc_queue.retracted_queue: if recv_req.abort_all or decode_req.rid.startswith(recv_req.rid): - retraction_discard( + discard_kv_cache_backup( decode_req, self.tree_cache, get_disagg().disaggregation_decode_retraction_backup, diff --git a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py index e371915adb..8203e705cd 100644 --- a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py +++ b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py @@ -843,12 +843,7 @@ class SchedulerMetricsReporter: self.stats.kv_transfer_speed_gb_s = self.kv_transfer_speed_gb_s self.stats.kv_transfer_latency_ms = self.kv_transfer_latency_ms elif self.scheduler.disaggregation_mode == DisaggregationMode.DECODE: - self.stats.num_decode_prealloc_queue_reqs = QueueCount.from_reqs( - self.scheduler.disagg_decode_prealloc_queue.queue, priority_enabled - ) - self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs( - self.scheduler.disagg_decode_transfer_queue.queue, priority_enabled - ) + self._update_decode_queue_stats(priority_enabled) # Utilization / LoRA / HiCache self._calculate_utilization() @@ -981,9 +976,12 @@ class SchedulerMetricsReporter: spec_num_draft_tokens = spec_snapshot["num_draft_tokens"] if self.scheduler.disaggregation_mode == DisaggregationMode.DECODE: + self._update_decode_queue_stats(self.scheduler.enable_priority_scheduling) msg += f"pre-allocated usage: {self.scheduler.disagg_decode_prealloc_queue.num_tokens_pre_allocated / self.scheduler.max_total_num_tokens:.2f}, " msg += f"#prealloc-req: {len(self.scheduler.disagg_decode_prealloc_queue.queue)}, " msg += f"#transfer-req: {len(self.scheduler.disagg_decode_transfer_queue.queue)}, " + if get_disagg().disaggregation_decode_host_receive_threshold > 0: + msg += f"#host-receive-req: {self.stats.num_decode_host_receive_queue_reqs.total}, " msg += f"#retracted-req: {len(self.scheduler.disagg_decode_prealloc_queue.retracted_queue)}, " if ( @@ -1069,13 +1067,6 @@ class SchedulerMetricsReporter: self.stats.num_prefill_inflight_queue_reqs = QueueCount.from_reqs( self.scheduler.disagg_prefill_inflight_queue, priority_enabled ) - elif self.scheduler.disaggregation_mode == DisaggregationMode.DECODE: - self.stats.num_decode_prealloc_queue_reqs = QueueCount.from_reqs( - self.scheduler.disagg_decode_prealloc_queue.queue, priority_enabled - ) - self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs( - self.scheduler.disagg_decode_transfer_queue.queue, priority_enabled - ) # Streaming session metrics self.stats.num_streaming_sessions = ( @@ -1377,6 +1368,23 @@ class SchedulerMetricsReporter: self.fwd_occupancy = float("nan") self.stats.fwd_occupancy = float("nan") + def _update_decode_queue_stats(self, priority_enabled: bool) -> None: + transfer_queue = self.scheduler.disagg_decode_transfer_queue.queue + self.stats.num_decode_prealloc_queue_reqs = QueueCount.from_reqs( + self.scheduler.disagg_decode_prealloc_queue.queue, priority_enabled + ) + self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs( + transfer_queue, priority_enabled + ) + host_reqs = ( + [req for req in transfer_queue if req.host_staged] + if get_disagg().disaggregation_decode_host_receive_threshold > 0 + else [] + ) + self.stats.num_decode_host_receive_queue_reqs = QueueCount.from_reqs( + host_reqs, priority_enabled + ) + def _maybe_log_idle_metrics(self): """Reset forward timing and publish idle metrics when needed.""" # Preserve the transition so the rate limit cannot leave a finite idle gauge. @@ -1427,10 +1435,5 @@ class SchedulerMetricsReporter: self.scheduler.disagg_prefill_inflight_queue, priority_enabled ) if self.scheduler.disaggregation_mode == DisaggregationMode.DECODE: - self.stats.num_decode_prealloc_queue_reqs = QueueCount.from_reqs( - self.scheduler.disagg_decode_prealloc_queue.queue, priority_enabled - ) - self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs( - self.scheduler.disagg_decode_transfer_queue.queue, priority_enabled - ) + self._update_decode_queue_stats(priority_enabled) self.metrics_collector.log_stats(self.stats) diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 863111c408..2d4d2c2b19 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -213,7 +213,7 @@ def dsv41_dspark_needs_rebootstrap( return isinstance(pool, DeepSeekV4TokenToKVPool) and 2 in pool.compression_ratios -def retraction_backup( +def backup_kv_cache( req: Req, tree_cache: BasePrefixCache, req_to_token_pool: ReqToTokenPool, @@ -236,11 +236,11 @@ def retraction_backup( return True unified_cache = cast("UnifiedRadixCache", tree_cache) - req.kv.retraction_backup = unified_cache.retraction_backup(req) + req.kv.retraction_backup = unified_cache.backup_kv_cache(req) return req.kv.retraction_backup is not None -def retraction_restore( +def restore_kv_cache( req: Req, tree_cache: BasePrefixCache, req_to_token_pool: ReqToTokenPool, @@ -257,11 +257,13 @@ def retraction_restore( unified_cache = cast("UnifiedRadixCache", tree_cache) assert req.kv.retraction_backup is not None - unified_cache.retraction_restore(req, req.kv.retraction_backup) + unified_cache.restore_kv_cache(req, req.kv.retraction_backup) req.kv.retraction_backup = None -def retraction_discard(req: Req, tree_cache: BasePrefixCache, backend: str) -> None: +def discard_kv_cache_backup( + req: Req, tree_cache: BasePrefixCache, backend: str +) -> None: if backend == "cpu_tensor": req.kv.retraction_backup = None return @@ -271,7 +273,7 @@ def retraction_discard(req: Req, tree_cache: BasePrefixCache, backend: str) -> N return unified_cache = cast("UnifiedRadixCache", tree_cache) - unified_cache.retraction_discard(req.kv.retraction_backup) + unified_cache.discard_kv_cache_backup(req.kv.retraction_backup) req.kv.retraction_backup = None diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index 70c0df78d2..788f6bff15 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -55,7 +55,7 @@ from sglang.srt.runtime_context import ( get_schedule, ) from sglang.srt.speculative.base_spec_worker import HiCacheDraftMode -from sglang.srt.utils import is_hip +from sglang.srt.utils import ceil_align, is_hip if TYPE_CHECKING: from torch.distributed import ProcessGroup @@ -142,11 +142,16 @@ def maybe_register_hicache_draft( # Host slots a backup-only retraction pool gets, as a fraction of the device -# pool. Sized well under 1.0 because a retraction burst touches a fraction of -# the device tokens; overflow aborts the request rather than pre-reserving. +# pool, with a floor large enough for one maximum-length request. BACKUP_ONLY_HICACHE_RATIO = 0.2 +def decode_retraction_max_tokens(req_to_token_pool, kv_cache) -> int: + return ceil_align( + min(req_to_token_pool.max_context_len - 1, kv_cache.size), kv_cache.page_size + ) + + def uses_ssm_state(model_config) -> bool: """Whether the model keeps recurrent/conv state alongside its attention KV.""" spec = linear_attn_model_spec(model_config) @@ -212,11 +217,19 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str: if memory.hicache_ratio is None: # Only a decode server reaches resolution with the ratio unset. A - # backup-only pool can be small: retractions that overflow it abort their - # request instead of crashing the scheduler. Sharing the pool with - # HiCache keeps the standard default. + # backup-only pool fits at least one runnable request. Sharing the pool + # with prefix caching keeps the standard HiCache default. if backend == "host_pool" and not memory.enable_hierarchical_cache: - fields["hicache_ratio"] = BACKUP_ONLY_HICACHE_RATIO + ratio = BACKUP_ONLY_HICACHE_RATIO + if memory.hicache_size == 0: + req_pool, allocator = tp_worker.get_memory_pool() + kv_pool = allocator.get_kvcache() + min_tokens = decode_retraction_max_tokens(req_pool, kv_pool) + if disagg.disaggregation_decode_host_receive_threshold > 0: + # One request can receive while another is retracted. + min_tokens *= 2 + ratio = max(ratio, min_tokens / kv_pool.size) + fields["hicache_ratio"] = ratio else: fields["hicache_ratio"] = 2.0 diff --git a/python/sglang/srt/mem_cache/pool_host/group.py b/python/sglang/srt/mem_cache/pool_host/group.py index 48fa88775d..ca3fb53559 100644 --- a/python/sglang/srt/mem_cache/pool_host/group.py +++ b/python/sglang/srt/mem_cache/pool_host/group.py @@ -7,6 +7,7 @@ from typing import Any import torch from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer +from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool @dataclass @@ -75,6 +76,63 @@ class HostPoolGroup: def get_pool(self, name: PoolName): return self.get_entry(name).host_pool + def get_contiguous_buf_infos(self): + """Return (device_buffers, host_buffers), each (ptrs, sizes, item_sizes).""" + host_by_device_ptr = {} + device_infos = ([], [], []) + for entry in self.entries: + host = entry.host_pool + pools = (entry.device_pool, *entry.packed_draft_device_pools) + for pool in pools: + dense_mha = ( + type(pool) is MHATokenToKVPool + and pool.kv_cache_layout == "nhd" + and pool.v_head_dim == pool.head_dim + ) + if ( + not dense_mha + or pool.layer_shard_enabled + or pool.page_size != self.page_size + ): + raise ValueError( + "Host receive requires dense NHD MHA target and " + "draft KV with matching page sizes" + ) + for combined, values in zip( + device_infos, pool.get_contiguous_buf_infos(), strict=True + ): + combined.extend(values) + + # Packed MHA stores target/draft K followed by target/draft V, + # while the wire lists target K/V followed by draft K/V. Associate + # each host view with its device buffer before applying wire order. + device_buffers = [b for p in pools for b in p.k_buffer] + [ + b for p in pools for b in p.v_buffer + ] + for device_buffer, host_buffer in zip( + device_buffers, host.host_kv_data_refs, strict=True + ): + if ( + not host_buffer.is_contiguous() + or host_buffer.shape[1:] != device_buffer.shape[1:] + ): + raise ValueError( + "KV transfer requires matching contiguous per-layer " + "device and host buffers" + ) + host_by_device_ptr[device_buffer.data_ptr()] = ( + host_buffer.data_ptr(), + host_buffer.nbytes, + host.token_stride_size * self.page_size, + ) + infos = [host_by_device_ptr[ptr] for ptr in device_infos[0]] + host_infos = ( + [info[0] for info in infos], + [info[1] for info in infos], + [info[2] for info in infos], + ) + return device_infos, host_infos + def alloc( self, need_size: int, diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 3a69aff3c2..54c300544a 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -1469,7 +1469,7 @@ class UnifiedRadixCache(BasePrefixCache): return 0 return self.evict_host(num_tokens) - def retraction_backup(self, req: Req) -> Optional[RetractionBackup]: + def backup_kv_cache(self, req: Req) -> Optional[RetractionBackup]: """Back up device KV to the host pool; None when it cannot fit after reclaim.""" assert req.seqlen > 1 @@ -1512,11 +1512,11 @@ class UnifiedRadixCache(BasePrefixCache): ) completion.finish_event.synchronize() except Exception: - self.retraction_discard(backup) + self.discard_kv_cache_backup(backup) raise return backup - def retraction_restore(self, req: Req, backup: RetractionBackup) -> None: + def restore_kv_cache(self, req: Req, backup: RetractionBackup) -> None: device_indices, current_transfers = self._retraction_device_transfers(req) assert len(backup.host_indices) == len(device_indices), ( f"Host backup has {len(backup.host_indices)} slots, but restore has " @@ -1561,9 +1561,9 @@ class UnifiedRadixCache(BasePrefixCache): transfer_layer_id_max=self.cache_controller.transfer_layer_id_max, ) completion.finish_event.synchronize() - self.retraction_discard(backup) + self.discard_kv_cache_backup(backup) - def retraction_discard(self, backup: RetractionBackup) -> None: + def discard_kv_cache_backup(self, backup: RetractionBackup) -> None: self.host_pool_group.free(backup.host_indices) self.host_pool_group.release_transfers(backup.pool_transfers) diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 4307f511ea..bab9530251 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -135,6 +135,7 @@ class SchedulerStats: num_prefill_inflight_queue_reqs: QueueCount = field(default_factory=QueueCount) num_decode_prealloc_queue_reqs: QueueCount = field(default_factory=QueueCount) num_decode_transfer_queue_reqs: QueueCount = field(default_factory=QueueCount) + num_decode_host_receive_queue_reqs: QueueCount = field(default_factory=QueueCount) kv_transfer_speed_gb_s: float = 0.0 kv_transfer_latency_ms: float = 0.0 pending_prealloc_token_usage: float = 0.0 @@ -519,6 +520,17 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): labelnames=labels.keys(), multiprocess_mode="mostrecent", ) + self.num_decode_host_receive_queue_reqs = Gauge( + name="sglang:num_decode_host_receive_queue_reqs", + documentation="Requests in the decode transfer queue receiving into host memory or waiting for device admission.", + labelnames=labels.keys(), + multiprocess_mode="mostrecent", + ) + self.num_decode_host_receive_reqs = Counter( + name="sglang:num_decode_host_receive_reqs_total", + documentation="Total requests admitted to receive prefill KV in host memory.", + labelnames=labels.keys(), + ) self.kv_transfer_speed_gb_s = Histogram( name="sglang:kv_transfer_speed_gb_s", documentation="Histogram of KV cache transfer speed in GB/s.", @@ -1186,6 +1198,9 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): def increment_transfer_failed_reqs(self) -> None: self.num_transfer_failed_reqs.labels(**self.labels).inc(1) + def increment_decode_host_receive_reqs(self) -> None: + self.num_decode_host_receive_reqs.labels(**self.labels).inc(1) + def increment_prefill_retries(self, count: int) -> None: if count > 0: self.num_prefill_retries_total.labels(**self.labels).inc(count) @@ -1414,6 +1429,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): self._log_gauge_queue_count( self.num_decode_transfer_queue_reqs, stats.num_decode_transfer_queue_reqs ) + self._log_gauge_queue_count( + self.num_decode_host_receive_queue_reqs, + stats.num_decode_host_receive_queue_reqs, + ) self._log_gauge( self.pending_prealloc_token_usage, stats.pending_prealloc_token_usage ) diff --git a/test/registered/disaggregation/test_disaggregation_basic.py b/test/registered/disaggregation/test_disaggregation_basic.py index 1764b7f037..8fdec84038 100644 --- a/test/registered/disaggregation/test_disaggregation_basic.py +++ b/test/registered/disaggregation/test_disaggregation_basic.py @@ -252,6 +252,10 @@ class TestDisaggregationMooncakeSpec( *spec_args, "--disaggregation-decode-retraction-backup", "host_pool", + # Pin the ratio: auto-sizing floors the pool at one max-length request, + # which no single request can overflow. + "--hicache-ratio", + str(BACKUP_ONLY_HICACHE_RATIO), ] cls.extra_decode_env = {"SGLANG_TEST_RETRACT": "true"} cls.launch_all() diff --git a/test/registered/kernels/ops/disaggregation/test_kv_checksum.py b/test/registered/kernels/ops/disaggregation/test_kv_checksum.py index d7809c7787..70321df975 100644 --- a/test/registered/kernels/ops/disaggregation/test_kv_checksum.py +++ b/test/registered/kernels/ops/disaggregation/test_kv_checksum.py @@ -198,6 +198,9 @@ class _FakeScheduler(SchedulerDisaggregationDecodeMixin): self.metrics_reporter = SimpleNamespace(enable_metrics=True) self.metrics_collector = Mock() self.streamed_aborts = [] + self.ngram_embedding_manager = Mock() + self.chunked_req = None + self.future_map = None def stream_output(self, reqs, return_logprob): self.streamed_aborts.extend(reqs) @@ -258,32 +261,43 @@ class TestGetNewPrebuiltBatchChecksum(unittest.TestCase): return _FakeScheduler(computer, self.req_to_token) def _run_once(self, sched, batch_ret=None): + batch = Mock(reqs=sched.waiting_queue) if batch_ret is None else batch_ret + batch.is_empty.side_effect = lambda: not batch.reqs + + def filter_batch(*, keep_indices): + batch.reqs = [batch.reqs[i] for i in keep_indices] + + batch.filter_batch.side_effect = filter_batch + sched.waiting_queue = [] running_batch = SimpleNamespace() with patch.object( SchedulerDisaggregationDecodeMixin, "_get_new_prebuilt_batch", - lambda s, rb: batch_ret, + lambda s, rb: batch, ): return sched.get_new_prebuilt_batch(running_batch) def test_match_keeps_req(self): sched = self._make_sched(_SENTINEL) sched.waiting_queue = [_make_req(self.true_chksum, self.num_pages)] - self._run_once(sched) - self.assertEqual(len(sched.waiting_queue), 1) + batch = self._run_once(sched) + self.assertEqual(len(batch.reqs), 1) self.assertEqual(sched.streamed_aborts, []) def test_mismatch_aborts(self): sched = self._make_sched(_SENTINEL) req = _make_req(0xDEADBEEF, self.num_pages) - sched.waiting_queue = [req] + valid = _make_req(self.true_chksum, self.num_pages, rid="valid") + sched.waiting_queue = [req, valid] with ( envs.SGLANG_IS_IN_CI.override(False), patch("sglang.srt.disaggregation.decode.prepare_abort") as mock_abort, patch("sglang.srt.disaggregation.decode.release_kv_cache") as mock_release, ): - self._run_once(sched) - self._run_once(sched) + batch = self._run_once(sched) + self.assertIsNone(self._run_once(sched)) + self.assertEqual(batch.reqs, [valid]) + batch.process_prebuilt.assert_called_once_with(sched.future_map) self.assertEqual(sched.waiting_queue, []) self.assertEqual(sched.streamed_aborts, [req]) mock_abort.assert_called_once() @@ -310,8 +324,8 @@ class TestGetNewPrebuiltBatchChecksum(unittest.TestCase): patch("sglang.srt.disaggregation.decode.prepare_abort") as mock_abort, patch("sglang.srt.disaggregation.decode.release_kv_cache") as mock_release, ): - self._run_once(sched) - self.assertEqual(sched.waiting_queue, [req]) + batch = self._run_once(sched) + self.assertEqual(batch.reqs, [req]) self.assertEqual(sched.streamed_aborts, []) mock_abort.assert_not_called() mock_release.assert_not_called() @@ -320,22 +334,21 @@ class TestGetNewPrebuiltBatchChecksum(unittest.TestCase): sched = self._make_sched(_SENTINEL) req = _make_req(self.true_chksum, self.num_pages) sched.waiting_queue = [req] - self._run_once(sched) - self._run_once(sched) - self.assertEqual(sched.waiting_queue, [req]) + batch = self._run_once(sched) + sched.waiting_queue = batch.reqs + self.assertEqual(self._run_once(sched).reqs, [req]) self.assertEqual(sched.streamed_aborts, []) def test_disabled_delegates_to_batch_builder(self): sched = self._make_sched(computer=None) sched.waiting_queue = [_make_req(0xABCD, 4)] - sentinel = object() + sentinel = Mock(reqs=sched.waiting_queue) self.assertIs(self._run_once(sched, batch_ret=sentinel), sentinel) def test_zero_expected_skips_checksum(self): sched = self._make_sched(_SENTINEL) sched.waiting_queue = [_make_req(0, self.num_pages)] - self._run_once(sched) - self.assertEqual(len(sched.waiting_queue), 1) + self.assertEqual(len(self._run_once(sched).reqs), 1) _SENTINEL = object() diff --git a/test/registered/unit/disaggregation/test_admission_abort_not_enqueued.py b/test/registered/unit/disaggregation/test_admission_abort_not_enqueued.py index e4ef48bf9c..b8b7bdd981 100644 --- a/test/registered/unit/disaggregation/test_admission_abort_not_enqueued.py +++ b/test/registered/unit/disaggregation/test_admission_abort_not_enqueued.py @@ -183,10 +183,10 @@ class TestAdmissionAbortNotEnqueued(CustomTestCase): `kv.req_pool_idx`, `reset_for_retract` nulled `kv.mamba_pool_idx`, the decode metadata buffer lives on `DecodeRequest` rather than `Req`, and `add()` itself clears `retraction_mb_id`. Retiring it here would strand - the host pages `release_req` allocated, which only `retraction_restore` - or `retraction_discard` free. + the host pages `release_req` allocated, which only `restore_kv_cache` + or `discard_kv_cache_backup` free. - `backup=None` is the short-sequence case: `retraction_backup()` returns + `backup=None` is the short-sequence case: `backup_kv_cache()` returns early for `seqlen <= 1` without setting it, so the resource predicate alone says "unadmitted" and only the flag keeps the request safe. """ diff --git a/test/registered/unit/disaggregation/test_decode_queue_cleanup.py b/test/registered/unit/disaggregation/test_decode_queue_cleanup.py index ff0c001fd1..b7e6ecf160 100644 --- a/test/registered/unit/disaggregation/test_decode_queue_cleanup.py +++ b/test/registered/unit/disaggregation/test_decode_queue_cleanup.py @@ -422,6 +422,38 @@ class TestDecodeQueueCleanup(CustomTestCase): req, queue.tree_cache, is_insert=False ) + receiver = MagicMock() + decode_req.kv_receiver = receiver + decode_req.host_staged = True + queue.enable_host_receive = True + queue.enable_deferred_kv_release = False + queue._defer_release = MagicMock() + queue.queue = [decode_req] + queue.req_to_metadata_buffer_idx_allocator.reset_mock() + mock_release_kv_cache.reset_mock() + self.assertEqual(queue.pop_transferred(), []) + receiver.abort.assert_called_once_with() + queue._defer_release.assert_called_once_with(decode_req) + receiver.clear.assert_not_called() + queue.req_to_metadata_buffer_idx_allocator.free.assert_not_called() + mock_release_kv_cache.assert_not_called() + + queue.queue = [decode_req] + decode_req.req.finished_reason = FINISH_ABORT("cancelled") + with ( + patch.object( + queue, "_poll_with_metadata_gate", return_value=[KVPoll.Success] + ), + patch( + "sglang.srt.disaggregation.decode.discard_kv_cache_backup" + ) as discard, + ): + self.assertEqual(queue.pop_transferred(), []) + discard.assert_called_once_with(req, queue.tree_cache, "host_pool") + receiver.clear.assert_called_once_with() + queue.req_to_metadata_buffer_idx_allocator.free.assert_called_once_with(3) + mock_release_kv_cache.assert_not_called() + def test_fake_receiver_initializes_deferred_release_state(self): manager = MagicMock() receiver = FakeKVReceiver(manager, "") diff --git a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py index 0a8936f219..8527d73a01 100644 --- a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py +++ b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py @@ -4,8 +4,8 @@ When a decode request is aborted while its prefill->decode KV transfer may still be in flight, the decode side holds its KV pages / req-slot instead of freeing them immediately (which could let the still-in-flight write land on pages already reused by another request). The pages are released once every prefill rank acks -that its transfer drained (CommonKVManager.is_abort_release_safe), or a timeout -fires. See DecodeTransferQueue.resolve_deferred_releases. +that its transfer drained (CommonKVManager.is_abort_release_safe). Device +destinations may also release on timeout; host destinations require the ack. """ import unittest @@ -149,6 +149,38 @@ def _make_decode_req(room, idx, mgr, n_prefill_ranks=1): class TestResolveDeferredReleases(CustomTestCase): + def test_host_release_requires_drain_on_every_rank_even_after_timeout(self): + mgr = _make_manager() + q = _make_queue(timeout=-1) + q.enable_host_receive = True + q.gloo_group = object() + entries = [_make_decode_req(room, room, mgr) for room in (1, 2)] + for entry in entries: + entry.host_staged = True + mgr.register_deferred_abort_room(entry.req.bootstrap_room) + q._defer_release(entry) + with ( + patch.object(decode_mod, "discard_kv_cache_backup") as discard, + patch.object(decode_mod, "release_kv_cache") as device_release, + patch("torch.distributed.get_world_size", return_value=2), + patch("torch.distributed.all_reduce") as reduce, + ): + q.resolve_deferred_releases() + discard.assert_not_called() + mgr.note_abort_ack(2, 0) + reduce.side_effect = lambda ready, **_: ready.zero_() + q.resolve_deferred_releases() + discard.assert_not_called() + reduce.side_effect = None + q.resolve_deferred_releases() + discard.assert_called_once_with(entries[1].req, q.tree_cache, "host_pool") + self.assertEqual(q.req_to_metadata_buffer_idx_allocator.freed, [2]) + mgr.note_abort_ack(1, 0) + q.resolve_deferred_releases() + self.assertEqual(discard.call_count, 2) + self.assertEqual(q._deferred_releases, []) + device_release.assert_not_called() + def test_noop_when_nothing_deferred(self): q = _make_queue() with patch.object(decode_mod, "release_kv_cache") as rel: diff --git a/test/registered/unit/disaggregation/test_pd_role_switch.py b/test/registered/unit/disaggregation/test_pd_role_switch.py index f3fd17ea09..43498dca97 100644 --- a/test/registered/unit/disaggregation/test_pd_role_switch.py +++ b/test/registered/unit/disaggregation/test_pd_role_switch.py @@ -318,6 +318,7 @@ class TestPdRoleSwitchStartupValidation(unittest.TestCase): base = dict( disaggregation_transfer_backend="mori", disaggregation_mode="prefill", + disaggregation_decode_host_receive_threshold=0.0, enable_pd_role_switch=True, enable_dp_attention=False, ep_size=1, diff --git a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py index fdb90883dc..c772003c35 100644 --- a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py +++ b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py @@ -435,6 +435,7 @@ class TestDecodePreallocQueuePriority(unittest.TestCase): kv_receiver=MagicMock(), metadata_buffer_index=-1, is_rebootstrap=False, + host_staged=False, ) def _new_queue(self, decode_reqs, *, low_priority_values_first: bool = False): @@ -860,6 +861,7 @@ class TestDecodePrebuilt(unittest.TestCase): ) new_batch = MagicMock() + new_batch.is_empty.return_value = False # get_new_prebuilt_batch reads the published disagg config # (disaggregation_decode_enable_radix_cache). with ( @@ -889,6 +891,7 @@ class TestDecodePrebuilt(unittest.TestCase): call_order = [] new_batch = MagicMock() + new_batch.is_empty.return_value = False new_batch.prepare_for_prebuilt.side_effect = lambda: call_order.append( "prepare" ) diff --git a/test/registered/unit/mem_cache/test_decode_retraction_backup.py b/test/registered/unit/mem_cache/test_decode_retraction_backup.py index 2d05894381..911cf56dd5 100644 --- a/test/registered/unit/mem_cache/test_decode_retraction_backup.py +++ b/test/registered/unit/mem_cache/test_decode_retraction_backup.py @@ -1,12 +1,32 @@ +import ctypes import unittest +from array import array +from itertools import product from types import SimpleNamespace +from unittest.mock import Mock, patch import torch -from sglang.srt.managers.schedule_batch import ReqKvInfo -from sglang.srt.mem_cache.allocator import TokenToKVPoolAllocator +from sglang.srt.arg_groups.hicache_hook import handle_hicache +from sglang.srt.arg_groups.pd_disaggregation_hook import handle_pd_disaggregation +from sglang.srt.disaggregation.base.conn import KVPoll, KVTransferDestination +from sglang.srt.disaggregation.decode import DecodePreallocQueue, DecodeRequest +from sglang.srt.disaggregation.fake.conn import FakeKVManager +from sglang.srt.disaggregation.utils import ReqToMetadataIdxAllocator, TransferBackend +from sglang.srt.managers.schedule_batch import Req, ReqKvInfo +from sglang.srt.managers.scheduler_components.pool_stats_observer import ( + SchedulerPoolStatsObserver, +) +from sglang.srt.mem_cache.allocator import ( + PagedTokenToKVPoolAllocator, + TokenToKVPoolAllocator, +) from sglang.srt.mem_cache.cache_init_params import CacheInitParams -from sglang.srt.mem_cache.common import retraction_backup +from sglang.srt.mem_cache.common import ( + backup_kv_cache, + release_kv_cache, + restore_kv_cache, +) from sglang.srt.mem_cache.hicache_storage import PoolName from sglang.srt.mem_cache.kv_cache_builder import maybe_register_hicache_draft from sglang.srt.mem_cache.memory_pool import ( @@ -17,26 +37,28 @@ from sglang.srt.mem_cache.memory_pool import ( from sglang.srt.mem_cache.unified_cache.components import ComponentType from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache from sglang.srt.runtime_context import get_parallel +from sglang.srt.sampling.sampling_params import SamplingParams from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler from sglang.srt.speculative.base_spec_worker import ( HiCacheDraftMode, HiCacheDraftPlan, ) from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small") -class TestDecodeRetractionBackup(unittest.TestCase): +class TestDecodeRetractionBackup(CustomTestCase): pool_size = 32 num_tokens = 8 dtype = torch.bfloat16 device = "cuda" - def _make_pool(self, layer_num: int) -> MHATokenToKVPool: + def _make_pool(self, layer_num: int, *, page_size=1) -> MHATokenToKVPool: return MHATokenToKVPool( size=self.pool_size, - page_size=1, + page_size=page_size, head_num=2, head_dim=64, dtype=self.dtype, @@ -78,15 +100,36 @@ class TestDecodeRetractionBackup(unittest.TestCase): self.assertTrue(torch.equal(key[indices], expected_key)) self.assertTrue(torch.equal(value[indices], expected_value)) - def _build_cache(self, hicache_ratio: float): - """Bring up a UnifiedRadixCache with a draft sidecar over fresh pools.""" + def _build_cache( + self, + hicache_ratio: float, + *, + shared_receive=False, + page_size=1, + io_backend="kernel", + draft_mode=None, + radix_cache=False, + host_receive_threshold=0.5, + ): + """Bring up a UnifiedRadixCache over fresh pools, optionally with draft KV.""" server_args = ServerArgs( model_path="dummy", - page_size=1, + page_size=page_size, hicache_ratio=hicache_ratio, - hicache_io_backend="kernel", + hicache_io_backend=io_backend, hicache_mem_layout="page_first", + disable_radix_cache=not radix_cache, + hicache_write_policy="write_back", + disaggregation_mode="decode" if shared_receive else "null", + disaggregation_decode_host_receive_threshold=( + host_receive_threshold if shared_receive else 0.0 + ), + disaggregation_decode_enable_radix_cache=radix_cache, ) + if shared_receive: + # Exercise pool layout selection from the normal HiCache default. + handle_pd_disaggregation(server_args) + handle_hicache(server_args) set_global_server_args_for_scheduler(server_args) req_to_token_pool = ReqToTokenPool( @@ -95,35 +138,51 @@ class TestDecodeRetractionBackup(unittest.TestCase): device=self.device, enable_memory_saver=False, ) - target_pool = self._make_pool(layer_num=2) - allocator = TokenToKVPoolAllocator( + target_pool = self._make_pool(layer_num=2, page_size=page_size) + if draft_mode is None: + draft_mode = ( + HiCacheDraftMode.NONE if shared_receive else HiCacheDraftMode.SIDECAR + ) + draft_pool = ( + self._make_pool(layer_num=1, page_size=page_size) + if draft_mode != HiCacheDraftMode.NONE + else None + ) + allocator_cls = ( + PagedTokenToKVPoolAllocator if page_size > 1 else TokenToKVPoolAllocator + ) + allocator = allocator_cls( size=self.pool_size, dtype=self.dtype, device=self.device, kvcache=target_pool, need_sort=False, + **({"page_size": page_size} if page_size > 1 else {}), ) params = CacheInitParams( - disable=True, + disable=not radix_cache, req_to_token_pool=req_to_token_pool, token_to_kv_pool_allocator=allocator, - page_size=1, - is_eagle=True, + page_size=page_size, + is_eagle=draft_pool is not None, tree_components=(ComponentType.FULL,), + mtp_draft_device_pools=(draft_pool,) + if draft_mode == HiCacheDraftMode.PACKED + else (), ) cache = UnifiedRadixCache(params) cache.init_hicache(server_args, params) self.addCleanup(cache.release_host_resources) - draft_pool = self._make_pool(layer_num=1) - maybe_register_hicache_draft( - tree_cache=cache, - draft_plan=HiCacheDraftPlan( - mode=HiCacheDraftMode.SIDECAR, - device_pools=(draft_pool,), - ), - ) - self.assertIn(PoolName.DRAFT, cache.host_pool_group.entry_map) + if draft_mode == HiCacheDraftMode.SIDECAR: + maybe_register_hicache_draft( + tree_cache=cache, + draft_plan=HiCacheDraftPlan( + mode=HiCacheDraftMode.SIDECAR, + device_pools=(draft_pool,), + ), + ) + self.assertIn(PoolName.DRAFT, cache.host_pool_group.entry_map) cache.validate_retraction_host_capacity() return SimpleNamespace( server_args=server_args, @@ -153,13 +212,13 @@ class TestDecodeRetractionBackup(unittest.TestCase): req, source_indices = self._admit_req(env, self.num_tokens) host_free_before = env.cache.host_pool_group.available_size() - self.assertIsNone(env.cache.retraction_backup(req)) + self.assertIsNone(env.cache.backup_kv_cache(req)) # The declined backup must not leak host slots. self.assertEqual(env.cache.host_pool_group.available_size(), host_free_before) # This is the signal release_req propagates so retract_decode aborts. self.assertFalse( - retraction_backup( + backup_kv_cache( req, env.cache, env.req_to_token_pool, @@ -187,7 +246,7 @@ class TestDecodeRetractionBackup(unittest.TestCase): draft_expected = self._snapshot_pool(draft_pool, source_indices) host_free_before = cache.host_pool_group.available_size() - backup = cache.retraction_backup(req) + backup = cache.backup_kv_cache(req) self.assertEqual( {transfer.name for transfer in backup.pool_transfers or []}, {PoolName.DRAFT}, @@ -209,7 +268,7 @@ class TestDecodeRetractionBackup(unittest.TestCase): (req.kv.req_pool_idx, slice(0, self.num_tokens)), destination_indices ) - cache.retraction_restore(req, backup) + cache.restore_kv_cache(req, backup) self._assert_pool_equal(target_pool, destination_indices, target_expected) self._assert_pool_equal(draft_pool, destination_indices, draft_expected) @@ -219,6 +278,347 @@ class TestDecodeRetractionBackup(unittest.TestCase): allocator.free(destination_indices) req_to_token_pool.free(req) + def _receive_queue(self, env): + queue = object.__new__(DecodePreallocQueue) + queue.__dict__.update( + tree_cache=env.cache, + token_to_kv_pool=env.target_pool, + draft_token_to_kv_pool=env.draft_pool, + token_to_kv_pool_allocator=env.allocator, + req_to_token_pool=env.req_to_token_pool, + req_to_metadata_buffer_idx_allocator=ReqToMetadataIdxAllocator(2), + metadata_buffers=SimpleNamespace(get_buf_infos=lambda: ([], [], [])), + tp_rank=0, + pp_rank=0, + pp_size=1, + gloo_group=None, + transfer_backend=TransferBackend.FAKE, + is_mla_backend=False, + enable_staging=False, + transfer_queue=SimpleNamespace(queue=[], enable_staging=False), + queue=[], + pending_reqs=[], + retracted_queue=[], + _prefill_dp_rank_queries={}, + _num_published_destinations=0, + num_reserved_decode_tokens=0, + scheduler=SimpleNamespace( + server_args=env.server_args, + ps=SimpleNamespace(dp_rank=0, gpu_id=0), + model_config=SimpleNamespace( + num_hidden_layers=env.target_pool.layer_num + ), + running_batch=SimpleNamespace(reqs=[]), + waiting_queue=[], + last_batch=None, + enable_hisparse=False, + enable_decode_hicache=False, + enable_lora=False, + enable_priority_scheduling=False, + metrics_reporter=SimpleNamespace( + current_scheduler_metrics_enabled=False + ), + tp_worker=SimpleNamespace( + is_hybrid_swa=False, + model_runner=SimpleNamespace(kv_cache_dtype_str="bfloat16"), + ), + ), + ) + with ( + patch.object(FakeKVManager, "supports_host_destination", True), + get_parallel().override(dp_rank=0), + ): + queue.kv_manager = queue._init_kv_manager() + queue.scheduler.pool_stats_observer = SchedulerPoolStatsObserver( + tree_cache=env.cache, + token_to_kv_pool_allocator=env.allocator, + req_to_token_pool=env.req_to_token_pool, + session_controller=None, + hisparse_coordinator=None, + is_hybrid_swa=False, + is_hybrid_ssm=False, + enable_hisparse=False, + full_tokens_per_layer=self.pool_size, + swa_tokens_per_layer=None, + max_total_num_tokens=self.pool_size, + get_last_batch=lambda: queue.scheduler.last_batch, + get_running_batch=lambda: queue.scheduler.running_batch, + ) + return queue, queue.kv_manager.kv_args + + @patch("torch.distributed.get_world_size", return_value=1) + def test_host_receive_threshold_controls_device_allocation(self, _world_size): + """Transfers at the threshold leave device KV free; zero disables staging.""" + for threshold, used_tokens, host_staged in ( + (0.0, 16, False), + (0.5, 15, False), + (0.5, 16, True), + ): + with self.subTest(threshold=threshold, used_tokens=used_tokens): + env = self._build_cache( + hicache_ratio=2.0, + shared_receive=True, + host_receive_threshold=threshold, + ) + queue, _ = self._receive_queue(env) + host_admissions = [] + queue.scheduler.metrics_reporter.current_scheduler_metrics_enabled = ( + True + ) + queue.scheduler.metrics_collector = SimpleNamespace( + increment_decode_host_receive_reqs=lambda: host_admissions.append(1) + ) + pressure = env.allocator.alloc(used_tokens) + req = Req( + rid="threshold", + origin_input_text="", + bootstrap_host="localhost", + origin_input_ids=array("q", [1]), + sampling_params=SamplingParams(max_new_tokens=1), + ) + receiver = Mock(supports_host_destination=True) + receiver.poll.return_value = KVPoll.WaitingForInput + decode_req = DecodeRequest(req=req, kv_receiver=receiver) + queue.queue = [decode_req] + self.assertEqual(queue.pop_preallocated(), ([decode_req], [])) + self.assertEqual(decode_req.host_staged, host_staged) + self.assertEqual(len(host_admissions), int(host_staged)) + self.assertEqual(req.kv.req_pool_idx is None, host_staged) + self.assertEqual( + env.allocator.available_size(), + self.pool_size - used_tokens - (0 if host_staged else 1), + ) + if host_staged: + env.cache.discard_kv_cache_backup(req.kv.retraction_backup) + else: + release_kv_cache(req, env.cache, is_insert=False) + env.allocator.free(pressure) + self.assertEqual(env.allocator.available_size(), self.pool_size) + + @patch("torch.distributed.get_world_size", return_value=1) + def test_host_receive_restores_target_and_draft_kv(self, _world_size): + """Wire-order writes must restore both pools, including packed MHA K/V. + + Sidecar KV must follow the primary host indices through restore and + release without allocating or freeing those indices a second time. + """ + for draft_mode, io_backend in product( + (HiCacheDraftMode.SIDECAR, HiCacheDraftMode.PACKED), + ("kernel", "direct"), + ): + with self.subTest(draft_mode=draft_mode, io_backend=io_backend): + page_size = num_slots = 16 + num_tokens = num_slots - 1 + env = self._build_cache( + hicache_ratio=2.0, + shared_receive=True, + page_size=page_size, + io_backend=io_backend, + draft_mode=draft_mode, + ) + cache = env.cache + queue, kv_args = self._receive_queue(env) + + req = Req( + rid="host-receive", + origin_input_text="", + bootstrap_host="localhost", + origin_input_ids=array("q", [1] * num_tokens), + sampling_params=SamplingParams(max_new_tokens=1), + ) + receiver = Mock(supports_host_destination=False) + receiver.poll.return_value = KVPoll.WaitingForInput + decode_req = DecodeRequest(req=req, kv_receiver=receiver) + queue.queue = [decode_req] + host_free_before = cache.host_pool_group.available_size() + blocker = env.allocator.alloc(self.pool_size) + self.assertEqual(queue.pop_preallocated(), ([], [])) + receiver.send_metadata.assert_not_called() + self.assertEqual( + cache.host_pool_group.available_size(), host_free_before + ) + env.allocator.free(blocker) + pressure = env.allocator.alloc(num_slots) + receiver.supports_host_destination = True + self.assertEqual(queue.pop_preallocated(), ([decode_req], [])) + self.assertIsNone(req.kv.req_pool_idx) + self.assertEqual( + env.allocator.available_size(), self.pool_size - num_slots + ) + self.assertTrue(decode_req.host_staged) + backup = req.kv.retraction_backup + self.assertEqual( + cache.host_pool_group.available_size(), host_free_before - num_slots + ) + page_indices = receiver.send_metadata.call_args.args[0] + self.assertEqual( + receiver.send_metadata.call_args.kwargs["destination"], + KVTransferDestination.HOST, + ) + + # Emulate the transport's page writes using the advertised + # addresses and strides, independently of host-pool ordering. + device_buffers = [ + buffer + for pool in (env.target_pool, env.draft_pool) + for buffer in pool.k_buffer + pool.v_buffer + ] + expected = [] + for index, (buffer, host_ptr, host_len, item_len) in enumerate( + zip( + device_buffers, + kv_args.host_kv_data_ptrs, + kv_args.host_kv_data_lens, + kv_args.host_kv_item_lens, + strict=True, + ) + ): + values = torch.arange( + num_slots * buffer[0].numel(), dtype=torch.float32 + ).reshape(num_slots, *buffer.shape[1:]) + values = ((values + 37 * index) % 251).to(self.dtype) + for page, host_page in enumerate(page_indices): + page_values = values[page * page_size : (page + 1) * page_size] + offset = int(host_page) * item_len + self.assertEqual(page_values.nbytes, item_len) + self.assertLessEqual(offset + item_len, host_len) + ctypes.memmove( + host_ptr + offset, page_values.data_ptr(), item_len + ) + expected.append(values[:num_tokens]) + buffer.fill_(-1) + + # Incoming KV and retraction must coexist even with the host pool full. + group = cache.host_pool_group + blockers = group.alloc(group.available_size() - num_slots) + pending_req = Req( + rid="pending", + origin_input_text="", + bootstrap_host="localhost", + origin_input_ids=array("q", [1] * num_tokens), + sampling_params=SamplingParams(max_new_tokens=1), + ) + pending = DecodeRequest(req=pending_req, kv_receiver=receiver) + queue.queue = [pending] + self.assertEqual(queue.pop_preallocated(), ([], [])) + self.assertEqual(queue.queue, [pending]) + self.assertIsNone(pending_req.kv.req_pool_idx) + self.assertIsNone(pending_req.kv.retraction_backup) + self.assertEqual( + env.allocator.available_size(), self.pool_size - num_slots + ) + self.assertEqual(group.available_size(), num_slots) + retracted, source_indices = self._admit_req(env, num_slots) + for index, buffer in enumerate(device_buffers): + buffer[source_indices] = index + 100 + retraction = cache.backup_kv_cache(retracted) + self.assertIsNotNone(retraction) + self.assertEqual(group.available_size(), 0) + for buffer in device_buffers: + buffer[source_indices] = -1 + cache.restore_kv_cache(retracted, retraction) + for index, buffer in enumerate(device_buffers): + self.assertTrue(torch.all(buffer[source_indices] == index + 100)) + group.free(blockers) + env.allocator.free(source_indices) + env.req_to_token_pool.free(retracted) + + # Device slots are assigned only after the host transfer. + blocker = env.allocator.alloc(num_slots) + self.assertFalse(queue.allocate_host_staged(decode_req)) + self.assertIsNone(req.kv.req_pool_idx) + self.assertIs(req.kv.retraction_backup, backup) + env.allocator.free(blocker) + self.assertTrue(queue.allocate_host_staged(decode_req)) + self.assertFalse(decode_req.host_staged) + received_indices = env.req_to_token_pool.req_to_token[ + req.kv.req_pool_idx, :num_tokens + ].to(torch.int64) + req.output_ids.append(99) + cache.restore_kv_cache(req, backup) + for buffer, values in zip(device_buffers, expected, strict=True): + self.assertTrue( + torch.equal(buffer[received_indices[:num_tokens]].cpu(), values) + ) + self.assertEqual( + cache.host_pool_group.available_size(), host_free_before + ) + # Abort cleanup uses the same descriptor, including sidecars. + self.assertEqual(queue.pop_preallocated(), ([pending], [])) + cache.discard_kv_cache_backup(pending_req.kv.retraction_backup) + self.assertEqual( + cache.host_pool_group.available_size(), host_free_before + ) + env.allocator.free(received_indices) + env.req_to_token_pool.free(req) + env.allocator.free(pressure) + + @patch("torch.distributed.get_world_size", return_value=1) + def test_host_receive_merges_existing_radix_prefix_after_restore(self, _world_size): + """Receiving a cached prefix must not lock it or leak duplicate KV.""" + env = self._build_cache( + hicache_ratio=2.0, shared_receive=True, radix_cache=True + ) + cache = env.cache + queue, _ = self._receive_queue(env) + + def make_req(rid, num_tokens): + return Req( + rid=rid, + origin_input_text="", + origin_input_ids=array("q", range(num_tokens)), + sampling_params=SamplingParams(max_new_tokens=1), + bootstrap_host="localhost", + ) + + cached = make_req("cached", 4) + cached.output_ids.append(99) + cached.last_node = cache.root_node_handle() + cached_indices = queue._pre_alloc(cached) + self._seed_pool(env.target_pool, cached_indices, base=500) + cached_values = self._snapshot_pool(env.target_pool, cached_indices) + cache.cache_unfinished_req(cached) + pressure = env.allocator.alloc(self.pool_size // 2 - 4) + + req = make_req("receiving", 8) + receiver = Mock(supports_host_destination=True) + receiver.poll.return_value = KVPoll.WaitingForInput + decode_req = DecodeRequest(req=req, kv_receiver=receiver) + queue.queue = [decode_req] + host_free_before = cache.host_pool_group.available_size() + self.assertEqual(queue.pop_preallocated(), ([decode_req], [])) + self.assertIsNone(req.kv.req_pool_idx) + self.assertEqual(cache.protected_size(), 4) + env.allocator.free(pressure) + backup = req.kv.retraction_backup + for buffer in queue.host_pool.host_kv_data_refs: + buffer[backup.host_indices] = 7 + + self.assertTrue(queue.allocate_host_staged(decode_req)) + received_indices = env.req_to_token_pool.req_to_token[ + req.kv.req_pool_idx, :8 + ].clone() + req.output_ids.append(99) + # Prebuilt preparation retains the full-transfer/root state, then + # restores before normal cache insertion deduplicates the prefix. + req.init_next_round_input(None) + req.set_extend_range(0, 8) + restore_kv_cache(req, cache, env.req_to_token_pool, env.allocator, "host_pool") + self.assertEqual(cache.host_pool_group.available_size(), host_free_before) + free_before_insert = env.allocator.available_size() + cache.cache_unfinished_req(req) + + row = env.req_to_token_pool.req_to_token[req.kv.req_pool_idx, :8] + self.assertTrue(torch.equal(row[:4], cached_indices)) + self.assertTrue(torch.equal(row[4:], received_indices[4:])) + self._assert_pool_equal(env.target_pool, cached_indices, cached_values) + self.assertEqual(env.allocator.available_size(), free_before_insert + 4) + self.assertEqual(cache.protected_size(), 8) + release_kv_cache(req, cache, is_insert=False) + self.assertEqual(cache.protected_size(), 4) + release_kv_cache(cached, cache, is_insert=False) + self.assertEqual(cache.protected_size(), 0) + DCP_SIZE = 4 DCP_RANK = 1 diff --git a/test/registered/unit/observability/test_forward_pass_metrics.py b/test/registered/unit/observability/test_forward_pass_metrics.py index 6c27c4cd46..2a6e20f33c 100644 --- a/test/registered/unit/observability/test_forward_pass_metrics.py +++ b/test/registered/unit/observability/test_forward_pass_metrics.py @@ -6,14 +6,18 @@ register_cpu_ci(est_time=11, suite="base-a-test-cpu") import math import types import unittest +from functools import partial from unittest.mock import patch +import prometheus_client + from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.srt.managers.scheduler_components.metrics_reporter import ( PrefillStats, SchedulerMetricsReporter, _CacheHitRateWindow, ) +from sglang.srt.observability.metrics_collector import SchedulerMetricsCollector from sglang.test.test_utils import CustomTestCase, enter_scope @@ -338,7 +342,7 @@ class TestForwardPassMetrics(unittest.TestCase): self.assertFalse(scheduler.enable_fpm) -class TestIdleMetrics(unittest.TestCase): +class TestIdleMetrics(CustomTestCase): def setUp(self): self.scheduler = types.SimpleNamespace( running_batch=types.SimpleNamespace(reqs=[]), @@ -363,6 +367,61 @@ class TestIdleMetrics(unittest.TestCase): ), ) + def test_host_receive_metrics_survive_queue_drain(self): + registry = prometheus_client.CollectorRegistry() + labels = {"model_name": "test", "priority": "", "moe_ep_rank": 0} + sample_labels = {key: str(value) for key, value in labels.items()} + with patch.multiple( + prometheus_client, + **{ + kind: partial(getattr(prometheus_client, kind), registry=registry) + for kind in ("Counter", "Gauge", "Histogram", "Summary") + }, + ): + collector = SchedulerMetricsCollector( + labels=labels, server_args=self.scheduler.server_args + ) + self.reporter.metrics_collector = collector + self.reporter.current_scheduler_metrics_enabled = True + self.scheduler.disaggregation_mode = DisaggregationMode.DECODE + self.scheduler.enable_priority_scheduling = True + self.scheduler.disagg_decode_prealloc_queue = types.SimpleNamespace(queue=[]) + host_reqs = [ + types.SimpleNamespace(host_staged=True, priority=priority) + for priority in (1, 2) + ] + self.scheduler.disagg_decode_transfer_queue = types.SimpleNamespace( + queue=[*host_reqs, types.SimpleNamespace(host_staged=False, priority=1)] + ) + for _ in host_reqs: + collector.increment_decode_host_receive_reqs() + + with get_context().override_server_args( + disaggregation_decode_host_receive_threshold=0.8 + ): + for waiting in (True, False): + for req in host_reqs: + req.host_staged = waiting + with patch( + "sglang.srt.managers.scheduler_components.metrics_reporter.time.perf_counter", + return_value=collector.last_log_time + 31, + ): + self.reporter._maybe_log_idle_metrics() + for priority, expected in (("", 2), ("1", 1), ("2", 1)): + self.assertEqual( + registry.get_sample_value( + "sglang:num_decode_host_receive_queue_reqs", + {**sample_labels, "priority": priority}, + ), + expected if waiting else 0, + ) + self.assertEqual( + registry.get_sample_value( + "sglang:num_decode_host_receive_reqs_total", sample_labels + ), + 2, + ) + def test_idle_clears_cached_forward_occupancy_immediately(self): self.reporter.current_scheduler_metrics_enabled = True self.reporter.fwd_occupancy = 72.0 diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 6cf4026a24..64942ca3ff 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -1986,7 +1986,34 @@ class TestSSLArgs(unittest.TestCase): self.assertTrue(resolution_result(server_args, "enable_ssl_refresh")) -class TestHiCacheArgs(unittest.TestCase): +class TestHiCacheArgs(CustomTestCase): + def test_host_receive_speculative_uses_shared_retraction_pool(self): + """Speculation must still resolve host receive to the shared host pool.""" + for algorithm in ("EAGLE", "EAGLE3", "NGRAM"): + with self.subTest(algorithm=algorithm): + args = self._make_args( + disaggregation_mode="decode", + disaggregation_decode_host_receive_threshold=0.8, + speculative_algorithm=algorithm, + ) + handle_pd_disaggregation(args) + self.assertEqual( + resolution_result(args, "disaggregation_decode_retraction_backup"), + "host_pool", + ) + handle_hicache(args) + self.assertEqual( + resolution_result(args, "hicache_mem_layout"), "layer_first" + ) + + for threshold in (-0.1, 1.1, float("nan")): + with self.subTest(threshold=threshold): + args = self._make_args( + disaggregation_decode_host_receive_threshold=threshold + ) + with self.assertRaisesRegex(ValueError, "must be between 0 and 1"): + handle_pd_disaggregation(args) + def test_linker_mla_dedup_requires_mooncake_linker(self): for enabled, linker, backend in ( (False, False, "mooncake"),