[HiCache] Allow a retraction host pool smaller than the device pool (#35543)

Co-authored-by: cctry <cctry@fb.com>
2026-08-19 22:59:57 -07:00
parent 50dae2d99d
commit 32d98aad13
7 changed files with 203 additions and 52 deletions
@@ -2701,7 +2701,7 @@ class ServerArgs:
    ] = "cache"
    hicache_ratio: A[
        Optional[float],
-        "The ratio of the size of host KV cache memory pool to the size of device pool. Defaults to 2.0 in cache mode, 1.2 in buffer_only mode, or 1.0 for host-pool decode retraction.",
+        "The ratio of the size of host KV cache memory pool to the size of device pool. Defaults to 2.0 in cache mode, 1.2 in buffer_only mode, or 0.2 for backup-only host-pool decode retraction.",
        NS("memory"),
    ] = None
    hicache_size: A[