[Fix] hicache: Keep unified memory off the direct page_first_direct copies (#43333)
This commit is contained in:
@@ -65,6 +65,7 @@ def handle_hicache(server_args: Any):
|
||||
|
||||
# Step 2: Storage-layout normalization without changing io backend.
|
||||
resolve_storage_layout_compatibility(server_args)
|
||||
resolve_unified_memory_layout_io(server_args)
|
||||
if (
|
||||
cfg.disaggregation_decode_host_receive_threshold < 1
|
||||
and cfg.hicache_mem_layout != "layer_first"
|
||||
@@ -203,6 +204,25 @@ def resolve_storage_layout_compatibility(server_args: Any):
|
||||
)
|
||||
|
||||
|
||||
def resolve_unified_memory_layout_io(server_args: Any):
|
||||
# The unified pool's token-major entries make each layer's device rows
|
||||
# strided; the direct page_first_direct copies assume packed rows.
|
||||
cfg = resolving_view(server_args)
|
||||
if not cfg.enable_unified_memory or cfg.hicache_mem_layout != "page_first_direct":
|
||||
return
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_resolve_unified_memory_layout_io",
|
||||
hicache_mem_layout="page_first",
|
||||
hicache_io_backend="kernel",
|
||||
)
|
||||
logger.warning(
|
||||
"--enable-unified-memory: the direct io backend cannot copy the unified "
|
||||
"pool's strided KV rows, switching to the page_first layout with the "
|
||||
"kernel io backend"
|
||||
)
|
||||
|
||||
|
||||
def validate_hicache_host_memory_mode(server_args: Any):
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.hicache_host_memory_mode not in ("cache", "buffer_only"):
|
||||
|
||||
@@ -464,6 +464,7 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
page_size=self.page_size,
|
||||
)
|
||||
elif self.layout == "page_first_direct":
|
||||
self._require_packed_device_rows("transfer_kv_per_layer_direct_pf_lf")
|
||||
transfer_kv_per_layer_direct_pf_lf(
|
||||
src_ptrs=[self.k_buffer, self.v_buffer],
|
||||
dst_ptrs=[
|
||||
@@ -621,6 +622,7 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
page_size=self.page_size,
|
||||
)
|
||||
elif self.layout == "page_first_direct":
|
||||
self._require_packed_device_rows("transfer_kv_all_layer_direct_lf_pf")
|
||||
transfer_kv_all_layer_direct_lf_pf(
|
||||
src_ptrs=device_kv_buffers,
|
||||
dst_ptrs=[self.k_buffer, self.v_buffer],
|
||||
|
||||
@@ -708,6 +708,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
page_size=self.page_size,
|
||||
)
|
||||
elif self.layout == "page_first_direct":
|
||||
self._require_packed_device_rows("transfer_kv_per_layer_direct_pf_lf")
|
||||
transfer_kv_per_layer_direct_pf_lf(
|
||||
src_ptrs=[self.kv_buffer],
|
||||
dst_ptrs=[device_buffer],
|
||||
@@ -947,6 +948,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
page_size=self.page_size,
|
||||
)
|
||||
elif self.layout == "page_first_direct":
|
||||
self._require_packed_device_rows("transfer_kv_all_layer_direct_lf_pf")
|
||||
transfer_kv_all_layer_direct_lf_pf(
|
||||
src_ptrs=device_kv_buffers,
|
||||
dst_ptrs=[self.kv_buffer],
|
||||
|
||||
@@ -65,6 +65,7 @@ class TestHiCacheStridedDeviceRows(unittest.TestCase):
|
||||
device="cpu",
|
||||
allocator_type="default",
|
||||
)
|
||||
self.addCleanup(host.destroy)
|
||||
if not host.can_use_jit:
|
||||
self.skipTest("strided device rows need the JIT HiCache kernels")
|
||||
|
||||
|
||||
@@ -63,6 +63,35 @@ class TestHostKVCache(CustomTestCase):
|
||||
allocator_type="default",
|
||||
)
|
||||
|
||||
def test_direct_page_first_direct_rejects_strided_rows(self):
|
||||
device = self.device_pool
|
||||
device.k_buffer = [
|
||||
torch.empty((buf.shape[0] * 2, *buf.shape[1:]), dtype=buf.dtype)[::2]
|
||||
for buf in device.k_buffer
|
||||
]
|
||||
device.v_buffer = [
|
||||
torch.empty((buf.shape[0] * 2, *buf.shape[1:]), dtype=buf.dtype)[::2]
|
||||
for buf in device.v_buffer
|
||||
]
|
||||
device._init_data_ptrs_and_strides()
|
||||
host = MHATokenToKVPoolHost(
|
||||
device_pool=device,
|
||||
host_to_device_ratio=2.0,
|
||||
host_size=0,
|
||||
page_size=self.page_size,
|
||||
layout="page_first_direct",
|
||||
pin_memory=False,
|
||||
device="cpu",
|
||||
allocator_type="default",
|
||||
)
|
||||
self.addCleanup(host.destroy)
|
||||
src = torch.tensor([0, 1])
|
||||
host_slots = host.alloc(len(src))
|
||||
with self.assertRaisesRegex(NotImplementedError, "direct_lf_pf"):
|
||||
host.backup_from_device_all_layer(device, host_slots, src, "direct")
|
||||
with self.assertRaisesRegex(NotImplementedError, "direct_pf_lf"):
|
||||
host.load_to_device_per_layer(device, host_slots, src, 0, "direct")
|
||||
|
||||
def test_packed_transfers_use_captured_target_and_draft_buffers(self):
|
||||
"""Target and draft rows retain their bound destinations during transfer."""
|
||||
|
||||
|
||||
@@ -2325,6 +2325,28 @@ class TestHiCacheArgs(CustomTestCase):
|
||||
"expected_io_backend": "direct",
|
||||
"expected_mem_layout": "page_first_direct",
|
||||
},
|
||||
{
|
||||
"name": "unified_memory_with_page_first_direct",
|
||||
"overrides": {
|
||||
"enable_hierarchical_cache": True,
|
||||
"enable_unified_memory": True,
|
||||
"hicache_mem_layout": "page_first_direct",
|
||||
},
|
||||
"expected_io_backend": "kernel",
|
||||
"expected_mem_layout": "page_first",
|
||||
},
|
||||
{
|
||||
"name": "unified_memory_mooncake_with_direct",
|
||||
"overrides": {
|
||||
"enable_hierarchical_cache": True,
|
||||
"enable_unified_memory": True,
|
||||
"hicache_storage_backend": "mooncake",
|
||||
"hicache_io_backend": "direct",
|
||||
"hicache_mem_layout": "layer_first",
|
||||
},
|
||||
"expected_io_backend": "kernel",
|
||||
"expected_mem_layout": "page_first",
|
||||
},
|
||||
{
|
||||
"name": "fa3_kernel_with_explicit_decode_backend",
|
||||
"overrides": {
|
||||
|
||||
Reference in New Issue
Block a user