[Fix] hicache: Keep unified memory off the direct page_first_direct copies (#43333)

This commit is contained in:
Shuwen Wang
2026-10-18 15:09:26 +08:00
committed by GitHub
parent df966f2345
commit e6c2cd47a9
6 changed files with 76 additions and 0 deletions
@@ -65,6 +65,7 @@ def handle_hicache(server_args: Any):
# Step 2: Storage-layout normalization without changing io backend.
resolve_storage_layout_compatibility(server_args)
resolve_unified_memory_layout_io(server_args)
if (
cfg.disaggregation_decode_host_receive_threshold < 1
and cfg.hicache_mem_layout != "layer_first"
@@ -203,6 +204,25 @@ def resolve_storage_layout_compatibility(server_args: Any):
)
def resolve_unified_memory_layout_io(server_args: Any):
# The unified pool's token-major entries make each layer's device rows
# strided; the direct page_first_direct copies assume packed rows.
cfg = resolving_view(server_args)
if not cfg.enable_unified_memory or cfg.hicache_mem_layout != "page_first_direct":
return
declare_resolution(
server_args,
"_resolve_unified_memory_layout_io",
hicache_mem_layout="page_first",
hicache_io_backend="kernel",
)
logger.warning(
"--enable-unified-memory: the direct io backend cannot copy the unified "
"pool's strided KV rows, switching to the page_first layout with the "
"kernel io backend"
)
def validate_hicache_host_memory_mode(server_args: Any):
cfg = resolving_view(server_args)
if cfg.hicache_host_memory_mode not in ("cache", "buffer_only"):
@@ -464,6 +464,7 @@ class MHATokenToKVPoolHost(HostKVCache):
page_size=self.page_size,
)
elif self.layout == "page_first_direct":
self._require_packed_device_rows("transfer_kv_per_layer_direct_pf_lf")
transfer_kv_per_layer_direct_pf_lf(
src_ptrs=[self.k_buffer, self.v_buffer],
dst_ptrs=[
@@ -621,6 +622,7 @@ class MHATokenToKVPoolHost(HostKVCache):
page_size=self.page_size,
)
elif self.layout == "page_first_direct":
self._require_packed_device_rows("transfer_kv_all_layer_direct_lf_pf")
transfer_kv_all_layer_direct_lf_pf(
src_ptrs=device_kv_buffers,
dst_ptrs=[self.k_buffer, self.v_buffer],
@@ -708,6 +708,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
page_size=self.page_size,
)
elif self.layout == "page_first_direct":
self._require_packed_device_rows("transfer_kv_per_layer_direct_pf_lf")
transfer_kv_per_layer_direct_pf_lf(
src_ptrs=[self.kv_buffer],
dst_ptrs=[device_buffer],
@@ -947,6 +948,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
page_size=self.page_size,
)
elif self.layout == "page_first_direct":
self._require_packed_device_rows("transfer_kv_all_layer_direct_lf_pf")
transfer_kv_all_layer_direct_lf_pf(
src_ptrs=device_kv_buffers,
dst_ptrs=[self.kv_buffer],