[Fix] [NPU] disaggregation: Fix host-pool backup for decode retraction (#43814)
This commit is contained in:
@@ -170,10 +170,15 @@ def set_default_server_args(args: "ServerArgs"):
|
||||
disable_custom_all_reduce=True,
|
||||
)
|
||||
|
||||
# handles hierarchical cache / decode-offload configs
|
||||
# Backup-only decode retraction also builds a HiCache host pool. The
|
||||
# inferred backend is resolved later, once the device KV pool exists.
|
||||
if (
|
||||
cfg.enable_hierarchical_cache
|
||||
or cfg.disaggregation_decode_enable_offload_kvcache
|
||||
or (
|
||||
cfg.disaggregation_mode == "decode"
|
||||
and cfg.disaggregation_decode_retraction_backup in (None, "host_pool")
|
||||
)
|
||||
):
|
||||
declare_resolution(
|
||||
args,
|
||||
|
||||
@@ -254,7 +254,7 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
# transfer kernels address the fused buffer directly instead.
|
||||
self.k_data_refs = []
|
||||
self.v_data_refs = []
|
||||
elif self.layout == "page_first":
|
||||
elif self.layout in ("page_first", "page_first_direct"):
|
||||
# Transpose [page, layer, ...] -> [layer, page, ...] to get per-layer views
|
||||
# This swaps strides without copying data
|
||||
k_transposed = self.k_buffer.transpose(0, 1)
|
||||
|
||||
Reference in New Issue
Block a user