From f6c848c882ba8b60a80518b38088046e0bc5ce03 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Fri, 9 Oct 2026 15:50:52 -0700 Subject: [PATCH] [PD] Use 1 to disable decode host receive and 0 for always-on (#43407) Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: cctry --- python/sglang/srt/arg_groups/fields/disagg.py | 4 ++-- python/sglang/srt/arg_groups/hicache_hook.py | 4 ++-- .../srt/arg_groups/pd_disaggregation_hook.py | 4 ++-- python/sglang/srt/disaggregation/decode.py | 14 ++++++------- python/sglang/srt/managers/scheduler.py | 2 +- .../scheduler_components/metrics_reporter.py | 4 ++-- .../sglang/srt/mem_cache/kv_cache_builder.py | 2 +- .../disaggregation/test_pd_role_switch.py | 2 +- .../test_decode_retraction_backup.py | 7 ++++--- .../unit/mem_cache/test_mamba_capacity.py | 2 +- .../observability/test_stat_loggers_di.py | 4 ++-- .../unit/server_args/test_server_args.py | 20 ++++++++++++++++--- 12 files changed, 42 insertions(+), 27 deletions(-) diff --git a/python/sglang/srt/arg_groups/fields/disagg.py b/python/sglang/srt/arg_groups/fields/disagg.py index aab97a017a..da764f851c 100644 --- a/python/sglang/srt/arg_groups/fields/disagg.py +++ b/python/sglang/srt/arg_groups/fields/disagg.py @@ -178,5 +178,5 @@ class Disagg(msgspec.Struct): sm_group_num: A[int, "Number of sm partition groups."] = 8 disaggregation_decode_host_receive_threshold: A[ float, - "Device token usage fraction at which incoming KV is received in the decode retraction host pool, excluding evictable cache pages. Range [0, 1]; 0 disables host receive. Size with --hicache-size or --hicache-ratio; requires dense MHA and a transfer backend that supports host destinations. No built-in backend currently supports this.", - ] = 0.0 + "Device token usage fraction at which incoming KV is received in the decode retraction host pool, excluding evictable cache pages. Range [0, 1]; 1 disables host receive (default), 0 always uses host receive. Size with --hicache-size or --hicache-ratio; requires dense MHA and a transfer backend that supports host destinations. No built-in backend currently supports this.", + ] = 1.0 diff --git a/python/sglang/srt/arg_groups/hicache_hook.py b/python/sglang/srt/arg_groups/hicache_hook.py index 901fa2b14d..8ebb7bdd91 100644 --- a/python/sglang/srt/arg_groups/hicache_hook.py +++ b/python/sglang/srt/arg_groups/hicache_hook.py @@ -54,7 +54,7 @@ def handle_hicache(server_args: Any): validate_hicache_host_memory_mode(server_args) - if cfg.disaggregation_decode_host_receive_threshold > 0: + if cfg.disaggregation_decode_host_receive_threshold < 1: # The shared host pool must match KV transfer's per-layer buffers. declare_resolution( server_args, "handle_hicache", hicache_mem_layout="layer_first" @@ -66,7 +66,7 @@ def handle_hicache(server_args: Any): # Step 2: Storage-layout normalization without changing io backend. resolve_storage_layout_compatibility(server_args) if ( - cfg.disaggregation_decode_host_receive_threshold > 0 + cfg.disaggregation_decode_host_receive_threshold < 1 and cfg.hicache_mem_layout != "layer_first" ): raise ValueError( diff --git a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py index c0ef52335f..24fbbc4a87 100644 --- a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py +++ b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py @@ -65,7 +65,7 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: raise ValueError( "--disaggregation-decode-host-receive-threshold must be between 0 and 1" ) - if cfg.disaggregation_decode_host_receive_threshold > 0: + if cfg.disaggregation_decode_host_receive_threshold < 1: if cfg.enable_hisparse or cfg.enable_pd_role_switch: raise ValueError( "Decode host receive does not yet support HiSparse or role switching" @@ -221,7 +221,7 @@ def _validate_prefill_complete(server_args: ServerArgs) -> None: unsupported.append("encoder disaggregation") if cfg.enable_hisparse: unsupported.append("HiSparse") - if cfg.disaggregation_decode_host_receive_threshold > 0: + if cfg.disaggregation_decode_host_receive_threshold < 1: unsupported.append("decode host KV buffering") if cfg.disaggregation_mode == "decode" and ( cfg.disaggregation_decode_enable_radix_cache or cfg.enable_hierarchical_cache diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index bd6419285a..7911aad37b 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -656,7 +656,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): total_kv_layers=self.scheduler.model_config.num_hidden_layers, req_to_token_pool=getattr(self, "req_to_token_pool", None), ) - if get_disagg().disaggregation_decode_host_receive_threshold > 0: + if get_disagg().disaggregation_decode_host_receive_threshold < 1: pool = self.token_to_kv_pool group = self.tree_cache.host_pool_group if kv_args.state_types or any( @@ -692,7 +692,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): self.is_mla_backend, ) if ( - get_disagg().disaggregation_decode_host_receive_threshold > 0 + get_disagg().disaggregation_decode_host_receive_threshold < 1 and not kv_manager.supports_host_destination ): raise ValueError( @@ -1371,7 +1371,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): continue if ( - get_disagg().disaggregation_decode_host_receive_threshold > 0 + get_disagg().disaggregation_decode_host_receive_threshold < 1 and not decode_req.is_rebootstrap and not _is_fake_transfer(decode_req.req) and decode_req.kv_receiver.supports_host_destination @@ -2372,7 +2372,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): self.metadata_buffers = metadata_buffers self.scheduler = scheduler self.enable_host_receive = ( - get_disagg().disaggregation_decode_host_receive_threshold > 0 + get_disagg().disaggregation_decode_host_receive_threshold < 1 ) self.tree_cache = tree_cache self.spec_algorithm = scheduler.spec_algorithm @@ -3187,7 +3187,7 @@ class SchedulerDisaggregationDecodeMixin: # A finished request can still have one redundant forward in flight. # Drain it before a prebuilt request seeds a potentially reused row. self.schedule_stream.wait_stream(self.forward_stream) - if get_disagg().disaggregation_decode_host_receive_threshold > 0: + if get_disagg().disaggregation_decode_host_receive_threshold < 1: for req in new_batch.reqs: if req.kv.retraction_backup is not None: restore_kv_cache( @@ -3225,7 +3225,7 @@ class SchedulerDisaggregationDecodeMixin: self.polling_count = (self.polling_count + 1) % self.polling_interval if self.polling_count % self.polling_interval == 0: - if get_disagg().disaggregation_decode_host_receive_threshold == 0: + if get_disagg().disaggregation_decode_host_receive_threshold == 1: req_conns, _ = self.disagg_decode_prealloc_queue.pop_preallocated() self.disagg_decode_transfer_queue.extend(req_conns) transferred_reqs = ( @@ -3236,7 +3236,7 @@ class SchedulerDisaggregationDecodeMixin: # Direct-to-host: KV data already in host pool, skip staging self.hisparse_coordinator.admit_request_direct(req) self.waiting_queue.extend(transferred_reqs) - if get_disagg().disaggregation_decode_host_receive_threshold > 0: + if get_disagg().disaggregation_decode_host_receive_threshold < 1: # Give completed host transfers device space before new arrivals. req_conns, _ = self.disagg_decode_prealloc_queue.pop_preallocated() self.disagg_decode_transfer_queue.extend(req_conns) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 1a3f60b857..5875904fa0 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -5365,7 +5365,7 @@ class Scheduler( ) # For disaggregation decode mode, the request in the waiting queue has KV cache allocated. if self.disaggregation_mode == DisaggregationMode.DECODE: - if get_disagg().disaggregation_decode_host_receive_threshold > 0: + if get_disagg().disaggregation_decode_host_receive_threshold < 1: discard_kv_cache_backup(req, self.tree_cache, "host_pool") if self.enable_hisparse: self.hisparse_coordinator.request_finished(req) diff --git a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py index f66f53949a..3b4e0de877 100644 --- a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py +++ b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py @@ -1001,7 +1001,7 @@ class SchedulerMetricsReporter: msg += f"pre-allocated usage: {self.scheduler.disagg_decode_prealloc_queue.num_tokens_pre_allocated / self.scheduler.max_total_num_tokens:.2f}, " msg += f"#prealloc-req: {len(self.scheduler.disagg_decode_prealloc_queue.queue)}, " msg += f"#transfer-req: {len(self.scheduler.disagg_decode_transfer_queue.queue)}, " - if get_disagg().disaggregation_decode_host_receive_threshold > 0: + if get_disagg().disaggregation_decode_host_receive_threshold < 1: msg += f"#host-receive-req: {self.stats.num_decode_host_receive_queue_reqs.total}, " msg += f"#retracted-req: {len(self.scheduler.disagg_decode_prealloc_queue.retracted_queue)}, " @@ -1419,7 +1419,7 @@ class SchedulerMetricsReporter: ) host_reqs = ( [req for req in transfer_queue if req.host_staged] - if get_disagg().disaggregation_decode_host_receive_threshold > 0 + if get_disagg().disaggregation_decode_host_receive_threshold < 1 else [] ) self.stats.num_decode_host_receive_queue_reqs = QueueCount.from_reqs( diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index 2bb3ad2037..43830ce3b3 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -231,7 +231,7 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str: req_pool, allocator = tp_worker.get_memory_pool() kv_pool = allocator.get_kvcache() min_tokens = decode_retraction_max_tokens(req_pool, kv_pool) - if disagg.disaggregation_decode_host_receive_threshold > 0: + if disagg.disaggregation_decode_host_receive_threshold < 1: # One request can receive while another is retracted. min_tokens *= 2 ratio = max(ratio, min_tokens / kv_pool.size) diff --git a/test/registered/unit/disaggregation/test_pd_role_switch.py b/test/registered/unit/disaggregation/test_pd_role_switch.py index 473475fc91..493c2e5bd8 100644 --- a/test/registered/unit/disaggregation/test_pd_role_switch.py +++ b/test/registered/unit/disaggregation/test_pd_role_switch.py @@ -305,7 +305,7 @@ class TestPdRoleSwitchStartupValidation(unittest.TestCase): base = dict( disaggregation_transfer_backend="mori", disaggregation_mode="prefill", - disaggregation_decode_host_receive_threshold=0.0, + disaggregation_decode_host_receive_threshold=1.0, enable_pd_role_switch=True, attn_dp_size=1, ep_join_mode=None, diff --git a/test/registered/unit/mem_cache/test_decode_retraction_backup.py b/test/registered/unit/mem_cache/test_decode_retraction_backup.py index 0377c85bbc..8290850d50 100644 --- a/test/registered/unit/mem_cache/test_decode_retraction_backup.py +++ b/test/registered/unit/mem_cache/test_decode_retraction_backup.py @@ -122,7 +122,7 @@ class TestDecodeRetractionBackup(CustomTestCase): hicache_write_policy="write_back", disaggregation_mode="decode" if shared_receive else "null", disaggregation_decode_host_receive_threshold=( - host_receive_threshold if shared_receive else 0.0 + host_receive_threshold if shared_receive else 1.0 ), disaggregation_decode_enable_radix_cache=radix_cache, ) @@ -360,9 +360,10 @@ class TestDecodeRetractionBackup(CustomTestCase): @patch("torch.distributed.get_world_size", return_value=1) def test_host_receive_threshold_controls_device_allocation(self, _world_size): - """Transfers at the threshold leave device KV free; zero disables staging.""" + """Zero always stages, one disables, and intermediate values gate on usage.""" for threshold, used_tokens, host_staged in ( - (0.0, 16, False), + (0.0, 0, True), + (1.0, 16, False), (0.5, 15, False), (0.5, 16, True), ): diff --git a/test/registered/unit/mem_cache/test_mamba_capacity.py b/test/registered/unit/mem_cache/test_mamba_capacity.py index a2383b196e..549d6279bb 100644 --- a/test/registered/unit/mem_cache/test_mamba_capacity.py +++ b/test/registered/unit/mem_cache/test_mamba_capacity.py @@ -252,7 +252,7 @@ class TestMambaCapacity(unittest.TestCase): with patch( "sglang.srt.disaggregation.decode.get_disagg", return_value=SimpleNamespace( - disaggregation_decode_host_receive_threshold=0 + disaggregation_decode_host_receive_threshold=1 ), ): self.assertEqual(queue.pop_preallocated(), ([], [])) diff --git a/test/registered/unit/observability/test_stat_loggers_di.py b/test/registered/unit/observability/test_stat_loggers_di.py index bf658711ed..16250cfc4b 100644 --- a/test/registered/unit/observability/test_stat_loggers_di.py +++ b/test/registered/unit/observability/test_stat_loggers_di.py @@ -321,7 +321,7 @@ class TestDeferredKVReleaseMetrics(CustomTestCase): patch( "sglang.srt.managers.scheduler_components.metrics_reporter.get_disagg", return_value=SimpleNamespace( - disaggregation_decode_host_receive_threshold=0 + disaggregation_decode_host_receive_threshold=1 ), ), patch( @@ -419,7 +419,7 @@ class TestDeferredKVReleaseMetrics(CustomTestCase): patch( "sglang.srt.managers.scheduler_components.metrics_reporter.get_disagg", return_value=SimpleNamespace( - disaggregation_decode_host_receive_threshold=0, + disaggregation_decode_host_receive_threshold=1, language_only=False, ), ), diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index b962394f4c..542a6f43c2 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -2024,11 +2024,16 @@ class TestSSLArgs(unittest.TestCase): class TestHiCacheArgs(CustomTestCase): def test_host_receive_speculative_uses_shared_retraction_pool(self): """Speculation must still resolve host receive to the shared host pool.""" - for algorithm in ("EAGLE", "EAGLE3", "NGRAM"): - with self.subTest(algorithm=algorithm): + for algorithm, threshold in ( + ("EAGLE", 0.0), + ("EAGLE3", 0.0), + ("NGRAM", 0.0), + ("EAGLE", 0.8), + ): + with self.subTest(algorithm=algorithm, threshold=threshold): args = self._make_args( disaggregation_mode="decode", - disaggregation_decode_host_receive_threshold=0.8, + disaggregation_decode_host_receive_threshold=threshold, speculative_algorithm=algorithm, ) handle_pd_disaggregation(args) @@ -2041,6 +2046,15 @@ class TestHiCacheArgs(CustomTestCase): resolution_result(args, "hicache_mem_layout"), "layer_first" ) + for overrides in ({}, {"disaggregation_decode_host_receive_threshold": 1.0}): + with self.subTest(overrides=overrides): + args = self._make_args(disaggregation_mode="decode", **overrides) + self.assertEqual(args.disaggregation_decode_host_receive_threshold, 1.0) + handle_pd_disaggregation(args) + self.assertIsNone( + resolution_result(args, "disaggregation_decode_retraction_backup") + ) + for threshold in (-0.1, 1.1, float("nan")): with self.subTest(threshold=threshold): args = self._make_args(