[Fix] hicache: Keep unified memory off the direct page_first_direct copies (#43333)
This commit is contained in:
@@ -65,6 +65,7 @@ def handle_hicache(server_args: Any):
|
||||
|
||||
# Step 2: Storage-layout normalization without changing io backend.
|
||||
resolve_storage_layout_compatibility(server_args)
|
||||
resolve_unified_memory_layout_io(server_args)
|
||||
if (
|
||||
cfg.disaggregation_decode_host_receive_threshold < 1
|
||||
and cfg.hicache_mem_layout != "layer_first"
|
||||
@@ -203,6 +204,25 @@ def resolve_storage_layout_compatibility(server_args: Any):
|
||||
)
|
||||
|
||||
|
||||
def resolve_unified_memory_layout_io(server_args: Any):
|
||||
# The unified pool's token-major entries make each layer's device rows
|
||||
# strided; the direct page_first_direct copies assume packed rows.
|
||||
cfg = resolving_view(server_args)
|
||||
if not cfg.enable_unified_memory or cfg.hicache_mem_layout != "page_first_direct":
|
||||
return
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"_resolve_unified_memory_layout_io",
|
||||
hicache_mem_layout="page_first",
|
||||
hicache_io_backend="kernel",
|
||||
)
|
||||
logger.warning(
|
||||
"--enable-unified-memory: the direct io backend cannot copy the unified "
|
||||
"pool's strided KV rows, switching to the page_first layout with the "
|
||||
"kernel io backend"
|
||||
)
|
||||
|
||||
|
||||
def validate_hicache_host_memory_mode(server_args: Any):
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.hicache_host_memory_mode not in ("cache", "buffer_only"):
|
||||
|
||||
@@ -464,6 +464,7 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
page_size=self.page_size,
|
||||
)
|
||||
elif self.layout == "page_first_direct":
|
||||
self._require_packed_device_rows("transfer_kv_per_layer_direct_pf_lf")
|
||||
transfer_kv_per_layer_direct_pf_lf(
|
||||
src_ptrs=[self.k_buffer, self.v_buffer],
|
||||
dst_ptrs=[
|
||||
@@ -621,6 +622,7 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
page_size=self.page_size,
|
||||
)
|
||||
elif self.layout == "page_first_direct":
|
||||
self._require_packed_device_rows("transfer_kv_all_layer_direct_lf_pf")
|
||||
transfer_kv_all_layer_direct_lf_pf(
|
||||
src_ptrs=device_kv_buffers,
|
||||
dst_ptrs=[self.k_buffer, self.v_buffer],
|
||||
|
||||
@@ -708,6 +708,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
page_size=self.page_size,
|
||||
)
|
||||
elif self.layout == "page_first_direct":
|
||||
self._require_packed_device_rows("transfer_kv_per_layer_direct_pf_lf")
|
||||
transfer_kv_per_layer_direct_pf_lf(
|
||||
src_ptrs=[self.kv_buffer],
|
||||
dst_ptrs=[device_buffer],
|
||||
@@ -947,6 +948,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
page_size=self.page_size,
|
||||
)
|
||||
elif self.layout == "page_first_direct":
|
||||
self._require_packed_device_rows("transfer_kv_all_layer_direct_lf_pf")
|
||||
transfer_kv_all_layer_direct_lf_pf(
|
||||
src_ptrs=device_kv_buffers,
|
||||
dst_ptrs=[self.kv_buffer],
|
||||
|
||||
Reference in New Issue
Block a user