diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 916f357d94..2430d9f3a7 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -90,7 +90,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( from sglang.srt.mem_cache.common import ( RetractionBackup, discard_kv_cache_backup, - dsv41_dspark_needs_rebootstrap, + dsv4_needs_rebootstrap, kv_to_page_indices, match_kv_cache, page_align_floor, @@ -800,9 +800,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): if not is_retracted and not is_rebootstrap and is_unadmitted_reject(req): self.scheduler.retire_unadmitted_request(req) return - if is_retracted and dsv41_dspark_needs_rebootstrap( - self.token_to_kv_pool_allocator - ): + if is_retracted and dsv4_needs_rebootstrap(self.token_to_kv_pool_allocator): if req.output_ids: req.pd_rebootstrap_forced_output_id = req.output_ids.pop() req.pd_rebootstrap_in_progress = True @@ -1083,9 +1081,15 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): # Deferred allocation cannot precede the recompute that makes its # source ready. Keep the existing leader election and error path. if decode_req.is_rebootstrap: - self.kv_manager.submit_prefill_recompute( - decode_req.kv_receiver, decode_req.req.build_rebootstrap_payload() - ) + self._submit_prefill_recompute(decode_req) + + def _submit_prefill_recompute(self, decode_req: DecodeRequest) -> None: + # A fake transfer has no prefill; its receiver completes once metadata is sent. + if _is_fake_transfer(decode_req.req): + return + self.kv_manager.submit_prefill_recompute( + decode_req.kv_receiver, decode_req.req.build_rebootstrap_payload() + ) def _update_handshake_waiters( self, @@ -1801,10 +1805,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): decode_req.is_rebootstrap and get_disagg().disaggregation_decode_allocation_policy == "early" ): - self.kv_manager.submit_prefill_recompute( - decode_req.kv_receiver, - decode_req.req.build_rebootstrap_payload(), - ) + self._submit_prefill_recompute(decode_req) self._num_published_destinations += 1 decode_req.req.time_stats.set_decode_transfer_queue_entry_time() diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 399b0c6a91..009a97c39b 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -3333,15 +3333,15 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): else: # No backup exists and the device KV is already freed, so the # request cannot resume. - if get_disagg().disaggregation_decode_retraction_backup == "none": + if get_disagg().disaggregation_decode_retraction_backup == "host_pool": + message = "Retraction host KV pool exhausted. Aborting the request." + status_code = HTTPStatus.INTERNAL_SERVER_ERROR + else: message = ( "Retracted under decode memory pressure without a KV " "backup. Retry later." ) status_code = HTTPStatus.SERVICE_UNAVAILABLE - else: - message = "Retraction host KV pool exhausted. Aborting the request." - status_code = HTTPStatus.INTERNAL_SERVER_ERROR req.to_finish = FINISH_ABORT(message, status_code=status_code) reqs_to_abort.append(req) logger.warning( diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 5d3c20978a..a1d714b15d 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -296,17 +296,14 @@ def _evict_until_allocatable( return -def dsv41_dspark_needs_rebootstrap( +def dsv4_needs_rebootstrap( token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator, ) -> bool: - """V4.1's request-scoped pair ring and draft KV cannot use CPU tensor backup.""" - if str(get_spec().speculative_algorithm).upper() != "DSPARK": - return False - from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool pool = token_to_kv_pool_allocator.get_kvcache() - return isinstance(pool, DeepSeekV4TokenToKVPool) and 2 in pool.compression_ratios + # No DeepSeek-V4 pool keeps a host copy of a request's KV; the prefill recomputes it. + return isinstance(pool, DeepSeekV4TokenToKVPool) def backup_kv_cache( @@ -316,15 +313,18 @@ def backup_kv_cache( token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator, backend: str, ) -> bool: - """Returns False when no backup can be taken ('none' backend, or the host - pool cannot hold it); the caller aborts the request.""" - if dsv41_dspark_needs_rebootstrap(token_to_kv_pool_allocator): - # Drain the in-flight verify before its slots can receive recomputed KV. + """Returns False when the request can be neither backed up nor recomputed; + the caller aborts it.""" + if backend == "none": + return False + if dsv4_needs_rebootstrap(token_to_kv_pool_allocator): + if req.multimodal_inputs is not None: + # The recompute payload carries token ids only; images would be lost. + return False + # Drain the in-flight forward before its slots can receive recomputed KV. device = token_to_kv_pool_allocator.get_kvcache().device torch.get_device_module(device).synchronize(device) return True - if backend == "none": - return False if backend == "cpu_tensor": req.offload_kv_cache(req_to_token_pool, token_to_kv_pool_allocator) return True diff --git a/test/registered/disaggregation/test_disaggregation_dsv4.py b/test/registered/disaggregation/test_disaggregation_dsv4.py index aa5caa7db3..390db2e506 100644 --- a/test/registered/disaggregation/test_disaggregation_dsv4.py +++ b/test/registered/disaggregation/test_disaggregation_dsv4.py @@ -1,10 +1,13 @@ import unittest +import requests + from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin from sglang.test.server_fixtures.disaggregation_fixture import ( PDDisaggregationServerBase, + assert_process_healthy, ) from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -23,6 +26,14 @@ DSV4_FLASH_ENV = { "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "256", } +# Force retraction on decode; the interval of 20 passes is arbitrary, sparse +# enough to keep the run short. +DSV4_FLASH_DECODE_ENV = { + **DSV4_FLASH_ENV, + "SGLANG_TEST_RETRACT": "1", + "SGLANG_TEST_RETRACT_INTERVAL": "20", +} + _EAGLE_SPEC_ARGS = [ "--speculative-algorithm", "EAGLE", @@ -116,6 +127,7 @@ class TestDisaggregationDSV4(SpecDecodingMixin, PDDisaggregationServerBase, GSM8 *_EAGLE_SPEC_ARGS, "--watchdog-timeout", "900", + "--enable-metrics", ] decode_args += cls.transfer_backend + cls.rdma_devices_for(range(4, 8)) cls.process_decode = popen_launch_pd_server( @@ -123,9 +135,20 @@ class TestDisaggregationDSV4(SpecDecodingMixin, PDDisaggregationServerBase, GSM8 cls.decode_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=decode_args, - env=DSV4_FLASH_ENV, + env=DSV4_FLASH_DECODE_ENV, ) + def test_gsm8k(self): + super().test_gsm8k() + assert_process_healthy(self, "decode", self.process_decode, self.decode_url) + metrics = requests.get(self.decode_url + "/metrics", timeout=30).text + retracted = sum( + float(line.rsplit(" ", 1)[1]) + for line in metrics.splitlines() + if line.startswith("sglang:num_retracted_requests_total") + ) + self.assertGreater(retracted, 0, "decode never retracted a request") + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/disaggregation/test_dsv4_retract_rebootstrap.py b/test/registered/unit/disaggregation/test_dsv4_retract_rebootstrap.py new file mode 100644 index 0000000000..99ecf34359 --- /dev/null +++ b/test/registered/unit/disaggregation/test_dsv4_retract_rebootstrap.py @@ -0,0 +1,62 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock + +from sglang.srt.disaggregation.decode import DecodePreallocQueue, DecodeRequest +from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST +from sglang.srt.managers.schedule_batch import ReqKvInfo +from sglang.srt.mem_cache.common import backup_kv_cache +from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +def _allocator(pool_cls): + # Buffers are never touched: the pool class alone decides the recovery path. + pool = pool_cls.__new__(pool_cls) + pool.device = "cpu" + return SimpleNamespace(get_kvcache=lambda: pool) + + +def _req(multimodal_inputs=None): + return SimpleNamespace( + kv=ReqKvInfo(), + seqlen=8, + multimodal_inputs=multimodal_inputs, + offload_kv_cache=MagicMock(), + ) + + +class TestDSV4RetractRebootstrap(CustomTestCase): + def test_dsv4_backup_decision(self): + cases = [ + # The default backend; the pool has no host copy, so the prefill recomputes. + ("default backend", _req(), "cpu_tensor", True), + # An explicit none still means abort, not recompute. + ("none backend", _req(), "none", False), + # The recompute payload carries token ids only. + ("image request", _req(multimodal_inputs=object()), "cpu_tensor", False), + ] + for name, req, backend, saved in cases: + with self.subTest(case=name): + self.assertEqual( + backup_kv_cache( + req, None, None, _allocator(DeepSeekV4TokenToKVPool), backend + ), + saved, + ) + req.offload_kv_cache.assert_not_called() + + def test_fake_transfer_skips_prefill_recompute(self): + # Neither the fake manager nor a fake receiver can dispatch a recompute. + queue = DecodePreallocQueue.__new__(DecodePreallocQueue) + queue.kv_manager = MagicMock() + req = MagicMock(bootstrap_host=FAKE_BOOTSTRAP_HOST) + queue._submit_prefill_recompute(DecodeRequest(req=req, kv_receiver=MagicMock())) + queue.kv_manager.submit_prefill_recompute.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/disaggregation/test_prefill_complete_integration.py b/test/registered/unit/disaggregation/test_prefill_complete_integration.py index e768655f8b..39a46c7b91 100644 --- a/test/registered/unit/disaggregation/test_prefill_complete_integration.py +++ b/test/registered/unit/disaggregation/test_prefill_complete_integration.py @@ -855,7 +855,10 @@ class TestAllocationIntegration(CustomTestCase): def test_rebootstrap_dispatch_precedes_ready_without_allocating(self): receiver = self.receiver() receiver.init = Mock() - req = SimpleNamespace(build_rebootstrap_payload=lambda: {"rid": "resume"}) + req = SimpleNamespace( + bootstrap_host="prefill", + build_rebootstrap_payload=lambda: {"rid": "resume"}, + ) entry = DecodeRequest(req=req, kv_receiver=receiver, is_rebootstrap=True) queue = DecodePreallocQueue.__new__(DecodePreallocQueue) dispatched = [] @@ -877,6 +880,7 @@ class TestAllocationIntegration(CustomTestCase): try: receiver = Mock(conclude_state=None) req = SimpleNamespace( + bootstrap_host="prefill", build_rebootstrap_payload=lambda: {"rid": "resume"}, time_stats=Mock(), )