[PD]: Support incremental transfer for mooncake transfer engine (#24257)

Co-authored-by: Shangming Cai <csmthu@gmail.com>

test/registered/distributed/test_disaggregation_decode_radix_cache.py

@@ -16,7 +16,7 @@ from sglang.test.test_utils import (
     try_cached_model,
 )
 
-register_cuda_ci(est_time=120, suite="stage-c-test-8-gpu-h20")
+register_cuda_ci(est_time=300, suite="stage-c-test-8-gpu-h20")
 
 
 def _has_nixl():
@@ -27,18 +27,26 @@ def _has_nixl():
     return True
 
 
-@unittest.skipUnless(
-    is_in_ci() or _has_nixl(),
-    "NIXL is required for decode radix cache disaggregation coverage.",
-)
-class TestDisaggregationDecodeRadixCache(PDDisaggregationServerBase):
+def _has_mooncake():
+    try:
+        import mooncake.engine  # noqa: F401
+    except ImportError:
+        return False
+    return True
+
+
+class DisaggregationDecodeRadixCacheTestMixin:
     extra_decode_args = ["--disaggregation-decode-enable-radix-cache"]
+    transfer_backend_name = None
 
     @classmethod
     def setUpClass(cls):
         super().setUpClass()
         cls.model = try_cached_model(DEFAULT_MODEL_NAME_FOR_TEST)
-        cls.transfer_backend = ["--disaggregation-transfer-backend", "nixl"]
+        cls.transfer_backend = [
+            "--disaggregation-transfer-backend",
+            cls.transfer_backend_name,
+        ]
         cls.launch_all()
 
     def _assert_process_healthy(self, name, process, url):
@@ -99,11 +107,9 @@ class TestDisaggregationDecodeRadixCache(PDDisaggregationServerBase):
         metrics_second = run_eval(args)
         print(f"Second run metrics: {metrics_second}")
 
-        # Both runs should have reasonable accuracy
         self.assertGreater(metrics_first["score"], 0.80)
         self.assertGreater(metrics_second["score"], 0.80)
 
-        # Second run accuracy should not drop more than 3% compared to first run
         accuracy_drop = metrics_first["score"] - metrics_second["score"]
         self.assertLessEqual(
             accuracy_drop,
@@ -114,5 +120,25 @@ class TestDisaggregationDecodeRadixCache(PDDisaggregationServerBase):
         )
 
 
+@unittest.skipUnless(
+    is_in_ci() or _has_nixl(),
+    "NIXL is required for decode radix cache disaggregation coverage.",
+)
+class TestDisaggregationDecodeRadixCacheNixl(
+    DisaggregationDecodeRadixCacheTestMixin, PDDisaggregationServerBase
+):
+    transfer_backend_name = "nixl"
+
+
+@unittest.skipUnless(
+    is_in_ci() or _has_mooncake(),
+    "Mooncake is required for decode radix cache disaggregation coverage.",
+)
+class TestDisaggregationDecodeRadixCacheMooncake(
+    DisaggregationDecodeRadixCacheTestMixin, PDDisaggregationServerBase
+):
+    transfer_backend_name = "mooncake"
+
+
 if __name__ == "__main__":
     unittest.main()

test/registered/unit/server_args/test_server_args.py

@@ -63,6 +63,28 @@ class TestLoadBalanceMethod(unittest.TestCase):
             str(context.exception),
         )
 
+    def test_pd_decode_radix_cache_allows_mooncake(self):
+        server_args = ServerArgs(
+            model_path="dummy",
+            disaggregation_mode="decode",
+            disaggregation_decode_enable_radix_cache=True,
+            disaggregation_transfer_backend="mooncake",
+        )
+
+        self.assertFalse(server_args.disable_radix_cache)
+
+    def test_pd_decode_radix_cache_rejects_unknown_backend(self):
+        with self.assertRaises(ValueError) as context:
+            ServerArgs(
+                model_path="dummy",
+                disaggregation_mode="decode",
+                disaggregation_decode_enable_radix_cache=True,
+                disaggregation_transfer_backend="fake",
+            )
+
+        self.assertIn("('nixl', 'mooncake')", str(context.exception))
+        self.assertIn("'fake'", str(context.exception))
+
 
 class TestPortArgs(unittest.TestCase):
     @patch("sglang.srt.server_args.get_free_port")