[Fix] hicache: Keep unified memory off the direct page_first_direct copies (#43333)

This commit is contained in:
Shuwen Wang
2026-10-18 15:09:26 +08:00
committed by GitHub
parent df966f2345
commit e6c2cd47a9
6 changed files with 76 additions and 0 deletions
@@ -65,6 +65,7 @@ def handle_hicache(server_args: Any):
# Step 2: Storage-layout normalization without changing io backend.
resolve_storage_layout_compatibility(server_args)
resolve_unified_memory_layout_io(server_args)
if (
cfg.disaggregation_decode_host_receive_threshold < 1
and cfg.hicache_mem_layout != "layer_first"
@@ -203,6 +204,25 @@ def resolve_storage_layout_compatibility(server_args: Any):
)
def resolve_unified_memory_layout_io(server_args: Any):
# The unified pool's token-major entries make each layer's device rows
# strided; the direct page_first_direct copies assume packed rows.
cfg = resolving_view(server_args)
if not cfg.enable_unified_memory or cfg.hicache_mem_layout != "page_first_direct":
return
declare_resolution(
server_args,
"_resolve_unified_memory_layout_io",
hicache_mem_layout="page_first",
hicache_io_backend="kernel",
)
logger.warning(
"--enable-unified-memory: the direct io backend cannot copy the unified "
"pool's strided KV rows, switching to the page_first layout with the "
"kernel io backend"
)
def validate_hicache_host_memory_mode(server_args: Any):
cfg = resolving_view(server_args)
if cfg.hicache_host_memory_mode not in ("cache", "buffer_only"):
@@ -464,6 +464,7 @@ class MHATokenToKVPoolHost(HostKVCache):
page_size=self.page_size,
)
elif self.layout == "page_first_direct":
self._require_packed_device_rows("transfer_kv_per_layer_direct_pf_lf")
transfer_kv_per_layer_direct_pf_lf(
src_ptrs=[self.k_buffer, self.v_buffer],
dst_ptrs=[
@@ -621,6 +622,7 @@ class MHATokenToKVPoolHost(HostKVCache):
page_size=self.page_size,
)
elif self.layout == "page_first_direct":
self._require_packed_device_rows("transfer_kv_all_layer_direct_lf_pf")
transfer_kv_all_layer_direct_lf_pf(
src_ptrs=device_kv_buffers,
dst_ptrs=[self.k_buffer, self.v_buffer],
@@ -708,6 +708,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
page_size=self.page_size,
)
elif self.layout == "page_first_direct":
self._require_packed_device_rows("transfer_kv_per_layer_direct_pf_lf")
transfer_kv_per_layer_direct_pf_lf(
src_ptrs=[self.kv_buffer],
dst_ptrs=[device_buffer],
@@ -947,6 +948,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
page_size=self.page_size,
)
elif self.layout == "page_first_direct":
self._require_packed_device_rows("transfer_kv_all_layer_direct_lf_pf")
transfer_kv_all_layer_direct_lf_pf(
src_ptrs=device_kv_buffers,
dst_ptrs=[self.kv_buffer],
@@ -65,6 +65,7 @@ class TestHiCacheStridedDeviceRows(unittest.TestCase):
device="cpu",
allocator_type="default",
)
self.addCleanup(host.destroy)
if not host.can_use_jit:
self.skipTest("strided device rows need the JIT HiCache kernels")
@@ -63,6 +63,35 @@ class TestHostKVCache(CustomTestCase):
allocator_type="default",
)
def test_direct_page_first_direct_rejects_strided_rows(self):
device = self.device_pool
device.k_buffer = [
torch.empty((buf.shape[0] * 2, *buf.shape[1:]), dtype=buf.dtype)[::2]
for buf in device.k_buffer
]
device.v_buffer = [
torch.empty((buf.shape[0] * 2, *buf.shape[1:]), dtype=buf.dtype)[::2]
for buf in device.v_buffer
]
device._init_data_ptrs_and_strides()
host = MHATokenToKVPoolHost(
device_pool=device,
host_to_device_ratio=2.0,
host_size=0,
page_size=self.page_size,
layout="page_first_direct",
pin_memory=False,
device="cpu",
allocator_type="default",
)
self.addCleanup(host.destroy)
src = torch.tensor([0, 1])
host_slots = host.alloc(len(src))
with self.assertRaisesRegex(NotImplementedError, "direct_lf_pf"):
host.backup_from_device_all_layer(device, host_slots, src, "direct")
with self.assertRaisesRegex(NotImplementedError, "direct_pf_lf"):
host.load_to_device_per_layer(device, host_slots, src, 0, "direct")
def test_packed_transfers_use_captured_target_and_draft_buffers(self):
"""Target and draft rows retain their bound destinations during transfer."""
@@ -2325,6 +2325,28 @@ class TestHiCacheArgs(CustomTestCase):
"expected_io_backend": "direct",
"expected_mem_layout": "page_first_direct",
},
{
"name": "unified_memory_with_page_first_direct",
"overrides": {
"enable_hierarchical_cache": True,
"enable_unified_memory": True,
"hicache_mem_layout": "page_first_direct",
},
"expected_io_backend": "kernel",
"expected_mem_layout": "page_first",
},
{
"name": "unified_memory_mooncake_with_direct",
"overrides": {
"enable_hierarchical_cache": True,
"enable_unified_memory": True,
"hicache_storage_backend": "mooncake",
"hicache_io_backend": "direct",
"hicache_mem_layout": "layer_first",
},
"expected_io_backend": "kernel",
"expected_mem_layout": "page_first",
},
{
"name": "fa3_kernel_with_explicit_decode_backend",
"overrides": {