[Disagg] Validate state strides before Mooncake transfers (#41607)

Co-authored-by: Aurick Qiao <6137920+aurickq@users.noreply.github.com>
(cherry picked from commit 1a0250c5b6)
This commit is contained in:
Aurick Qiao
2026-10-08 15:35:05 +08:00
committed by minke.yu
parent f577b61b52
commit 1be762368b
2 changed files with 318 additions and 0 deletions
@@ -960,6 +960,8 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
dst_layer_ids: Optional[List[int]] = None,
dst_device_data_indices: Optional[npt.NDArray[np.int32]] = None,
dst_device_data_ptrs: Optional[set[int]] = None,
dst_item_lens: list[int] | None = None,
bootstrap_room: Optional[int] = None,
) -> int:
"""
Generic KV cache transfer supporting both MHA and MLA architectures.
@@ -1011,6 +1013,21 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
len(dst_data_ptrs),
allow_positional_fallback=self.pp_size == 1,
)
if dst_item_lens is not None:
for i, j in pairs:
if item_lens[i] != dst_item_lens[j]:
assert bootstrap_room is not None
failure_reason = (
f"{state_type} item length mismatch for paired "
f"entries src[{i}]={item_lens[i]} "
f"dst[{j}]={dst_item_lens[j]}"
)
logger.error(failure_reason)
self.conclude_failure(
bootstrap_room=bootstrap_room,
failure_reason=failure_reason,
)
return -1
layers_params = [
(src_data_ptrs[i], dst_data_ptrs[j], item_lens[i]) for i, j in pairs
]
@@ -1020,6 +1037,24 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
src_data_ptrs, dst_data_ptrs, state_type
)
)
if dst_item_lens is not None:
_, mapped_dst_lens, _ = self.get_mla_kv_ptrs_with_pp(
item_lens, dst_item_lens, state_type
)
for layer_id in range(layers_current_pp_stage):
if item_lens[layer_id] != mapped_dst_lens[layer_id]:
assert bootstrap_room is not None
failure_reason = (
f"{state_type} item length mismatch for positional "
f"entry {layer_id}: prefill={item_lens[layer_id]} "
f"decode={mapped_dst_lens[layer_id]}"
)
logger.error(failure_reason)
self.conclude_failure(
bootstrap_room=bootstrap_room,
failure_reason=failure_reason,
)
return -1
layers_params = [
(
src_kv_ptrs[layer_id],
@@ -2110,9 +2145,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
),
src_layer_ids=src_state_layer_ids,
dst_layer_ids=dst_state_layer_ids,
dst_item_lens=dst_item_lens,
bootstrap_room=req.room,
)
or rc
)
if self.request_status.get(req.room, KVPoll.Failed) == KVPoll.Failed:
return rc
elif st in (StateType.MINIMAX_INDEX_K, StateType.MINIMAX_DENSE_KV):
# Compacted layer lists require equal TP and PP=1 on both peers.
if self.pp_size is not None and self.pp_size > 1:
@@ -2588,6 +2627,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
executor,
target_rank_registration_info,
)
if (
self.request_status.get(
kv_chunk.room, KVPoll.Failed
)
== KVPoll.Failed
):
break
if state_rc != 0:
with self.session_lock:
self.session_failures[