[PD] Use 1 to disable decode host receive and 0 for always-on (#43407)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: cctry <cctry@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-09 15:50:52 -07:00
committed by GitHub
co-authored by Hanming Lu cctry
parent f6e3444b2e
commit f6c848c882
12 changed files with 42 additions and 27 deletions
@@ -178,5 +178,5 @@ class Disagg(msgspec.Struct):
sm_group_num: A[int, "Number of sm partition groups."] = 8
disaggregation_decode_host_receive_threshold: A[
float,
"Device token usage fraction at which incoming KV is received in the decode retraction host pool, excluding evictable cache pages. Range [0, 1]; 0 disables host receive. Size with --hicache-size or --hicache-ratio; requires dense MHA and a transfer backend that supports host destinations. No built-in backend currently supports this.",
] = 0.0
"Device token usage fraction at which incoming KV is received in the decode retraction host pool, excluding evictable cache pages. Range [0, 1]; 1 disables host receive (default), 0 always uses host receive. Size with --hicache-size or --hicache-ratio; requires dense MHA and a transfer backend that supports host destinations. No built-in backend currently supports this.",
] = 1.0
+2 -2
View File
@@ -54,7 +54,7 @@ def handle_hicache(server_args: Any):
validate_hicache_host_memory_mode(server_args)
if cfg.disaggregation_decode_host_receive_threshold > 0:
if cfg.disaggregation_decode_host_receive_threshold < 1:
# The shared host pool must match KV transfer's per-layer buffers.
declare_resolution(
server_args, "handle_hicache", hicache_mem_layout="layer_first"
@@ -66,7 +66,7 @@ def handle_hicache(server_args: Any):
# Step 2: Storage-layout normalization without changing io backend.
resolve_storage_layout_compatibility(server_args)
if (
cfg.disaggregation_decode_host_receive_threshold > 0
cfg.disaggregation_decode_host_receive_threshold < 1
and cfg.hicache_mem_layout != "layer_first"
):
raise ValueError(
@@ -65,7 +65,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
raise ValueError(
"--disaggregation-decode-host-receive-threshold must be between 0 and 1"
)
if cfg.disaggregation_decode_host_receive_threshold > 0:
if cfg.disaggregation_decode_host_receive_threshold < 1:
if cfg.enable_hisparse or cfg.enable_pd_role_switch:
raise ValueError(
"Decode host receive does not yet support HiSparse or role switching"
@@ -221,7 +221,7 @@ def _validate_prefill_complete(server_args: ServerArgs) -> None:
unsupported.append("encoder disaggregation")
if cfg.enable_hisparse:
unsupported.append("HiSparse")
if cfg.disaggregation_decode_host_receive_threshold > 0:
if cfg.disaggregation_decode_host_receive_threshold < 1:
unsupported.append("decode host KV buffering")
if cfg.disaggregation_mode == "decode" and (
cfg.disaggregation_decode_enable_radix_cache or cfg.enable_hierarchical_cache
+7 -7
View File
@@ -656,7 +656,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
total_kv_layers=self.scheduler.model_config.num_hidden_layers,
req_to_token_pool=getattr(self, "req_to_token_pool", None),
)
if get_disagg().disaggregation_decode_host_receive_threshold > 0:
if get_disagg().disaggregation_decode_host_receive_threshold < 1:
pool = self.token_to_kv_pool
group = self.tree_cache.host_pool_group
if kv_args.state_types or any(
@@ -692,7 +692,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
self.is_mla_backend,
)
if (
get_disagg().disaggregation_decode_host_receive_threshold > 0
get_disagg().disaggregation_decode_host_receive_threshold < 1
and not kv_manager.supports_host_destination
):
raise ValueError(
@@ -1371,7 +1371,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
continue
if (
get_disagg().disaggregation_decode_host_receive_threshold > 0
get_disagg().disaggregation_decode_host_receive_threshold < 1
and not decode_req.is_rebootstrap
and not _is_fake_transfer(decode_req.req)
and decode_req.kv_receiver.supports_host_destination
@@ -2372,7 +2372,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
self.metadata_buffers = metadata_buffers
self.scheduler = scheduler
self.enable_host_receive = (
get_disagg().disaggregation_decode_host_receive_threshold > 0
get_disagg().disaggregation_decode_host_receive_threshold < 1
)
self.tree_cache = tree_cache
self.spec_algorithm = scheduler.spec_algorithm
@@ -3187,7 +3187,7 @@ class SchedulerDisaggregationDecodeMixin:
# A finished request can still have one redundant forward in flight.
# Drain it before a prebuilt request seeds a potentially reused row.
self.schedule_stream.wait_stream(self.forward_stream)
if get_disagg().disaggregation_decode_host_receive_threshold > 0:
if get_disagg().disaggregation_decode_host_receive_threshold < 1:
for req in new_batch.reqs:
if req.kv.retraction_backup is not None:
restore_kv_cache(
@@ -3225,7 +3225,7 @@ class SchedulerDisaggregationDecodeMixin:
self.polling_count = (self.polling_count + 1) % self.polling_interval
if self.polling_count % self.polling_interval == 0:
if get_disagg().disaggregation_decode_host_receive_threshold == 0:
if get_disagg().disaggregation_decode_host_receive_threshold == 1:
req_conns, _ = self.disagg_decode_prealloc_queue.pop_preallocated()
self.disagg_decode_transfer_queue.extend(req_conns)
transferred_reqs = (
@@ -3236,7 +3236,7 @@ class SchedulerDisaggregationDecodeMixin:
# Direct-to-host: KV data already in host pool, skip staging
self.hisparse_coordinator.admit_request_direct(req)
self.waiting_queue.extend(transferred_reqs)
if get_disagg().disaggregation_decode_host_receive_threshold > 0:
if get_disagg().disaggregation_decode_host_receive_threshold < 1:
# Give completed host transfers device space before new arrivals.
req_conns, _ = self.disagg_decode_prealloc_queue.pop_preallocated()
self.disagg_decode_transfer_queue.extend(req_conns)
+1 -1
View File
@@ -5365,7 +5365,7 @@ class Scheduler(
)
# For disaggregation decode mode, the request in the waiting queue has KV cache allocated.
if self.disaggregation_mode == DisaggregationMode.DECODE:
if get_disagg().disaggregation_decode_host_receive_threshold > 0:
if get_disagg().disaggregation_decode_host_receive_threshold < 1:
discard_kv_cache_backup(req, self.tree_cache, "host_pool")
if self.enable_hisparse:
self.hisparse_coordinator.request_finished(req)
@@ -1001,7 +1001,7 @@ class SchedulerMetricsReporter:
msg += f"pre-allocated usage: {self.scheduler.disagg_decode_prealloc_queue.num_tokens_pre_allocated / self.scheduler.max_total_num_tokens:.2f}, "
msg += f"#prealloc-req: {len(self.scheduler.disagg_decode_prealloc_queue.queue)}, "
msg += f"#transfer-req: {len(self.scheduler.disagg_decode_transfer_queue.queue)}, "
if get_disagg().disaggregation_decode_host_receive_threshold > 0:
if get_disagg().disaggregation_decode_host_receive_threshold < 1:
msg += f"#host-receive-req: {self.stats.num_decode_host_receive_queue_reqs.total}, "
msg += f"#retracted-req: {len(self.scheduler.disagg_decode_prealloc_queue.retracted_queue)}, "
@@ -1419,7 +1419,7 @@ class SchedulerMetricsReporter:
)
host_reqs = (
[req for req in transfer_queue if req.host_staged]
if get_disagg().disaggregation_decode_host_receive_threshold > 0
if get_disagg().disaggregation_decode_host_receive_threshold < 1
else []
)
self.stats.num_decode_host_receive_queue_reqs = QueueCount.from_reqs(
@@ -231,7 +231,7 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str:
req_pool, allocator = tp_worker.get_memory_pool()
kv_pool = allocator.get_kvcache()
min_tokens = decode_retraction_max_tokens(req_pool, kv_pool)
if disagg.disaggregation_decode_host_receive_threshold > 0:
if disagg.disaggregation_decode_host_receive_threshold < 1:
# One request can receive while another is retracted.
min_tokens *= 2
ratio = max(ratio, min_tokens / kv_pool.size)