[Perf] NIXL: prepared dlists for hybrid-model state transfer (#44740)
Co-authored-by: ishandhanani <82981111+ishandhanani@users.noreply.github.com>
This commit is contained in:
co-authored by
ishandhanani
parent
80fef63786
commit
6c3db0f520
@@ -10,6 +10,7 @@ import uuid
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
import torch
|
||||
@@ -78,6 +79,98 @@ _PEER_RELOAD_MIN_INTERVAL_S = 1.0
|
||||
|
||||
GUARD = "NixlMsgGuard".encode("ascii")
|
||||
KV_MEM_KINDS = {"VRAM", "DRAM"}
|
||||
# Whole, fixed-stride slots on the generic path; DSA_TAIL encodes ring segments.
|
||||
_PREPARED_STATE_TYPES = frozenset(
|
||||
(
|
||||
StateType.MAMBA,
|
||||
StateType.SWA,
|
||||
StateType.DSA,
|
||||
StateType.QSA_PENDING,
|
||||
StateType.QSA_COMPRESSED,
|
||||
StateType.SWA_RING,
|
||||
StateType.DSV4_REQUEST_STATE,
|
||||
StateType.BLOCK_SCALE,
|
||||
StateType.BLOCK_SCALE_SWA,
|
||||
StateType.MINIMAX_INDEX_K,
|
||||
StateType.MINIMAX_DENSE_KV,
|
||||
)
|
||||
)
|
||||
# Flat per-entry layout on the generic path even on an MHA backend.
|
||||
_FLAT_STATE_TYPES = frozenset(
|
||||
(
|
||||
StateType.MAMBA,
|
||||
StateType.DSA,
|
||||
StateType.QSA_PENDING,
|
||||
StateType.QSA_COMPRESSED,
|
||||
StateType.MINIMAX_INDEX_K,
|
||||
StateType.MINIMAX_DENSE_KV,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _nth(lists, index: int, default=()):
|
||||
return lists[index] if index < len(lists) else default
|
||||
|
||||
|
||||
def _state_slot_capacities(
|
||||
*, ptrs: List[int], data_lens: List[int], item_lens: List[int]
|
||||
) -> Optional[List[int]]:
|
||||
# 0 for an elided or unregistered entry; None keeps the component generic.
|
||||
if not (len(ptrs) == len(data_lens) == len(item_lens)):
|
||||
return None
|
||||
capacities = []
|
||||
for ptr, data_len, item_len in zip(ptrs, data_lens, item_lens):
|
||||
if ptr == 0 or item_len == 0:
|
||||
capacities.append(0)
|
||||
elif data_len < item_len:
|
||||
return None
|
||||
else:
|
||||
capacities.append(data_len // item_len)
|
||||
return capacities
|
||||
|
||||
|
||||
def _prepared_state_runs(*, entries, base: int, gpu_id: int):
|
||||
runs, offsets = [], []
|
||||
for ptr, stride, capacity in entries:
|
||||
offsets.append(base)
|
||||
runs.append((ptr, stride, gpu_id, stride, capacity))
|
||||
base += capacity
|
||||
column = np.asarray(offsets, dtype=np.int32).reshape(-1, 1)
|
||||
column.flags.writeable = False
|
||||
return runs, column, base
|
||||
|
||||
|
||||
def _prepared_state_selectors(*, indices, offsets, capacity: int):
|
||||
values = np.asarray(indices)
|
||||
if values.ndim != 1 or values.dtype.kind not in "iu":
|
||||
raise ValueError("NIXL prepared state indices must be a 1-D integer sequence")
|
||||
if values.min() < 0 or values.max() >= capacity:
|
||||
raise ValueError(
|
||||
f"NIXL prepared state index outside the registered capacity {capacity}"
|
||||
)
|
||||
if values.dtype.kind == "u":
|
||||
values = values.astype(np.int64)
|
||||
# int32 in one pass: NIXL borrows such selectors without copying.
|
||||
return np.add(offsets, values, dtype=np.int32).ravel()
|
||||
|
||||
|
||||
class _PreparedStateComponent(msgspec.Struct, frozen=True):
|
||||
src_offsets: npt.NDArray[np.int32]
|
||||
dst_offsets: npt.NDArray[np.int32]
|
||||
src_capacity: int
|
||||
dst_capacity: int
|
||||
|
||||
|
||||
class _PreparedStatePlan(msgspec.Struct, frozen=True):
|
||||
src_handle: Any
|
||||
dst_handle: Any
|
||||
components: Dict[int, _PreparedStateComponent]
|
||||
|
||||
|
||||
# A decode stride that differs from the prefill's has no correct transfer path.
|
||||
class _StateLayoutMismatch(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
# Once one handle of a batch reports ERR, its siblings settle only when NIXL
|
||||
# notices their peer is gone, which for UCX means waiting out peer keepalive.
|
||||
@@ -255,6 +348,7 @@ class KVArgsRegisterInfo:
|
||||
kv_xfer_segments: Optional[List[_KVXferPreparedSegment]] = None
|
||||
staging_base_ptr: int = 0
|
||||
staging_total_size: int = 0
|
||||
dst_state_data_lens: List[List[int]] = dataclasses.field(default_factory=list)
|
||||
|
||||
@classmethod
|
||||
def from_zmq(cls, msg: List[bytes]):
|
||||
@@ -331,6 +425,12 @@ class KVArgsRegisterInfo:
|
||||
staging_total_size=(
|
||||
int(msg[15].decode("ascii")) if len(msg) > 15 and msg[15] != b"" else 0
|
||||
),
|
||||
# Frame 23 is absent on decodes that predate prepared state transfers.
|
||||
dst_state_data_lens=(
|
||||
unpack_int_lists(msg[23], "Q")
|
||||
if len(msg) > 23 and len(msg[23]) > 0
|
||||
else []
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -522,6 +622,8 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self.register_buffer_to_engine()
|
||||
|
||||
self.enable_staging = envs.SGLANG_DISAGG_STAGING_BUFFER.get()
|
||||
self._prepared_state_plans: Dict[str, _PreparedStatePlan] = {}
|
||||
self._state_layout_rejections: Dict[str, str] = {}
|
||||
self.kv_buffer_tensors = None
|
||||
self.prep_handles: Dict[str, Any] = {}
|
||||
self.prep_handle_slice_src: Optional[Tuple[Any, int, int, int]] = (
|
||||
@@ -539,6 +641,13 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self._num_slots_src = num_kv_slots(
|
||||
self.kv_args.kv_data_lens, self.kv_args.kv_item_lens
|
||||
)
|
||||
self._prepared_state_components = self._prepared_state_component_indices(
|
||||
backend
|
||||
)
|
||||
logger.info(
|
||||
"NIXL prepared state transfer: components=%s",
|
||||
sorted(self._prepared_state_components),
|
||||
)
|
||||
transfer_queue_size = envs.SGLANG_DISAGGREGATION_QUEUE_SIZE.get()
|
||||
self.transfer_queues: List[FastQueue] = [
|
||||
FastQueue() for _ in range(transfer_queue_size)
|
||||
@@ -575,6 +684,252 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
f"Unsupported DisaggregationMode: {self.disaggregation_mode}"
|
||||
)
|
||||
|
||||
def _prepared_state_component_indices(self, backend: str) -> frozenset[int]:
|
||||
if get_device().device != "cuda" or backend != "UCX":
|
||||
return frozenset()
|
||||
if self.pp_size != 1 or self.attn_cp_size != 1 or self.dcp_size != 1:
|
||||
return frozenset()
|
||||
return frozenset(
|
||||
index
|
||||
for index, state_type in enumerate(self.kv_args.state_types or [])
|
||||
if state_type in _PREPARED_STATE_TYPES
|
||||
)
|
||||
|
||||
def _prepare_state_xfer(self, peer_info: KVArgsRegisterInfo) -> None:
|
||||
peer_name = peer_info.agent_name
|
||||
self._prepared_state_plans.pop(peer_name, None)
|
||||
self._state_layout_rejections.pop(peer_name, None)
|
||||
if not self._prepared_state_components:
|
||||
return
|
||||
if peer_info.dst_dcp_size != 1 or peer_info.requires_dcp_relayout:
|
||||
reason = "dcp-peer"
|
||||
elif not peer_info.dst_state_data_lens:
|
||||
reason = "no-decode-pool-lengths"
|
||||
else:
|
||||
reason = None
|
||||
if reason is not None:
|
||||
logger.info(
|
||||
"NIXL prepared state: peer=%s all generic reason=%s", peer_name, reason
|
||||
)
|
||||
return
|
||||
planned, skipped, rejected = {}, [], []
|
||||
for index in sorted(self._prepared_state_components):
|
||||
try:
|
||||
entries = self._plan_prepared_state_component(
|
||||
index=index, peer_info=peer_info
|
||||
)
|
||||
except _StateLayoutMismatch as e:
|
||||
rejected.append(str(e))
|
||||
continue
|
||||
if isinstance(entries, str):
|
||||
skipped.append(f"{index}:{entries}")
|
||||
else:
|
||||
planned[index] = entries
|
||||
if rejected:
|
||||
# Every transfer to this peer fails instead of the bootstrap thread.
|
||||
self._state_layout_rejections[peer_name] = "; ".join(rejected)
|
||||
logger.error("NIXL state layout rejected: peer=%s %s", peer_name, rejected)
|
||||
return
|
||||
plan = self._build_prepared_state_plan(peer_info=peer_info, planned=planned)
|
||||
if plan is not None:
|
||||
self._prepared_state_plans[peer_name] = plan
|
||||
logger.info(
|
||||
"NIXL prepared state: peer=%s prepared=%s generic=%s",
|
||||
peer_name,
|
||||
sorted(plan.components) if plan is not None else [],
|
||||
skipped,
|
||||
)
|
||||
|
||||
def _plan_prepared_state_component(
|
||||
self, *, index: int, peer_info: KVArgsRegisterInfo
|
||||
):
|
||||
state_type = self.kv_args.state_types[index]
|
||||
if not self._state_component_is_whole_slot(
|
||||
state_type=state_type, decode_tp_size=peer_info.decode_tp_size
|
||||
):
|
||||
return "tp-layout"
|
||||
src_ptrs = _nth(self.kv_args.state_data_ptrs, index)
|
||||
src_strides = _nth(self.kv_args.state_item_lens, index)
|
||||
src_caps = _state_slot_capacities(
|
||||
ptrs=src_ptrs,
|
||||
data_lens=_nth(self.kv_args.state_data_lens, index),
|
||||
item_lens=src_strides,
|
||||
)
|
||||
dst_ptrs = _nth(peer_info.dst_state_data_ptrs, index)
|
||||
dst_strides = _nth(peer_info.dst_state_item_lens, index)
|
||||
dst_caps = _state_slot_capacities(
|
||||
ptrs=dst_ptrs,
|
||||
data_lens=_nth(peer_info.dst_state_data_lens, index),
|
||||
item_lens=dst_strides,
|
||||
)
|
||||
if src_caps is None or dst_caps is None:
|
||||
return "pool-geometry"
|
||||
pairs = self._prepared_state_pairs(
|
||||
state_type=state_type,
|
||||
src_layer_ids=_nth(self.kv_args.state_layer_ids, index),
|
||||
dst_layer_ids=_nth(peer_info.dst_state_layer_ids, index),
|
||||
n_src=len(src_ptrs),
|
||||
n_dst=len(dst_ptrs),
|
||||
)
|
||||
if pairs is None:
|
||||
return "entry-pairing"
|
||||
pairs = [(s, d) for s, d in pairs if src_caps[s] and dst_caps[d]]
|
||||
if not pairs:
|
||||
return "no-entries"
|
||||
for s, d in pairs:
|
||||
if src_strides[s] != dst_strides[d]:
|
||||
raise _StateLayoutMismatch(
|
||||
f"NIXL prepared state component {index} ({state_type}) "
|
||||
f"stride mismatch for peer {peer_info.agent_name}: "
|
||||
f"src[{s}]={src_strides[s]}, dst[{d}]={dst_strides[d]}"
|
||||
)
|
||||
return (
|
||||
[(src_ptrs[s], src_strides[s], src_caps[s]) for s, _ in pairs],
|
||||
[(dst_ptrs[d], dst_strides[d], dst_caps[d]) for _, d in pairs],
|
||||
)
|
||||
|
||||
def _prepared_state_pairs(
|
||||
self, *, state_type, src_layer_ids, dst_layer_ids, n_src: int, n_dst: int
|
||||
) -> Optional[List[Tuple[int, int]]]:
|
||||
if not (self.is_mla_backend or state_type in _FLAT_STATE_TYPES):
|
||||
# The generic MHA K/V split pairs positionally; unequal counts do not line up.
|
||||
return [(i, i) for i in range(n_src)] if n_src == n_dst else None
|
||||
if n_src != n_dst and self.kv_args.mla_compression_ratios:
|
||||
return None
|
||||
try:
|
||||
return build_transfer_entry_pairs(
|
||||
src_layer_ids,
|
||||
dst_layer_ids,
|
||||
n_src,
|
||||
n_dst,
|
||||
allow_positional_fallback=self.pp_size == 1,
|
||||
)
|
||||
except RuntimeError:
|
||||
return None
|
||||
|
||||
def _state_component_is_whole_slot(self, *, state_type, decode_tp_size) -> bool:
|
||||
# Mirrors the dispatch in maybe_send_extra; the other layouts slice per rank.
|
||||
if state_type == StateType.MAMBA:
|
||||
return self.state_tp_size == decode_tp_size
|
||||
if state_type in (StateType.QSA_PENDING, StateType.QSA_COMPRESSED):
|
||||
return self.is_mla_backend or self.state_tp_size == decode_tp_size
|
||||
if state_type in (StateType.MINIMAX_INDEX_K, StateType.MINIMAX_DENSE_KV):
|
||||
return self.attn_tp_size == decode_tp_size
|
||||
if state_type == StateType.DSA:
|
||||
return True
|
||||
return self.is_mla_backend or self.attn_tp_size == decode_tp_size
|
||||
|
||||
def _build_prepared_state_plan(
|
||||
self, *, peer_info: KVArgsRegisterInfo, planned
|
||||
) -> Optional[_PreparedStatePlan]:
|
||||
if not planned or not self._use_strided_descs:
|
||||
return None
|
||||
peer_name = peer_info.agent_name
|
||||
src_total = sum(c for src, _ in planned.values() for _, _, c in src)
|
||||
dst_total = sum(c for _, dst in planned.values() for _, _, c in dst)
|
||||
if max(src_total, dst_total) > 0x7FFFFFFF:
|
||||
# NIXL selects prepared descriptors with int32 indices.
|
||||
logger.warning(
|
||||
"NIXL prepared state: peer=%s all generic reason=index-space",
|
||||
peer_name,
|
||||
)
|
||||
return None
|
||||
src_runs, dst_runs, components = [], [], {}
|
||||
src_base = dst_base = 0
|
||||
for index, (src_entries, dst_entries) in planned.items():
|
||||
runs, src_offsets, src_base = _prepared_state_runs(
|
||||
entries=src_entries, base=src_base, gpu_id=self.kv_args.gpu_id
|
||||
)
|
||||
src_runs.extend(runs)
|
||||
runs, dst_offsets, dst_base = _prepared_state_runs(
|
||||
entries=dst_entries, base=dst_base, gpu_id=peer_info.gpu_id
|
||||
)
|
||||
dst_runs.extend(runs)
|
||||
components[index] = _PreparedStateComponent(
|
||||
src_offsets=src_offsets,
|
||||
dst_offsets=dst_offsets,
|
||||
src_capacity=min(c for _, _, c in src_entries),
|
||||
dst_capacity=min(c for _, _, c in dst_entries),
|
||||
)
|
||||
try:
|
||||
# Never expand to one descriptor per slot; a state pool holds millions.
|
||||
src_handle = self.agent.prep_xfer_dlist(
|
||||
"", self._pack_stride_descs(src_runs), "VRAM"
|
||||
)
|
||||
dst_handle = self.agent.prep_xfer_dlist(
|
||||
peer_name, self._pack_stride_descs(dst_runs), "VRAM"
|
||||
)
|
||||
except TypeError:
|
||||
logger.warning_once(
|
||||
"Installed NIXL does not support strided descriptors; "
|
||||
"state transfers stay generic."
|
||||
)
|
||||
self._use_strided_descs = False
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"NIXL prepared state: peer=%s all generic, dlist creation failed: %s",
|
||||
peer_name,
|
||||
e,
|
||||
)
|
||||
return None
|
||||
if src_handle is None or dst_handle is None:
|
||||
return None
|
||||
return _PreparedStatePlan(
|
||||
src_handle=src_handle, dst_handle=dst_handle, components=components
|
||||
)
|
||||
|
||||
def _send_prepared_states(
|
||||
self, *, plan: _PreparedStatePlan, prefill_state_indices, dst_state_indices
|
||||
):
|
||||
src_parts, dst_parts = [], []
|
||||
for index, component in plan.components.items():
|
||||
src_indices = _nth(prefill_state_indices, index, default=None)
|
||||
dst_indices = _nth(dst_state_indices, index)
|
||||
if src_indices is None or len(src_indices) == 0:
|
||||
continue
|
||||
if len(src_indices) != len(dst_indices):
|
||||
raise RuntimeError(
|
||||
f"State index length mismatch at component {index}: "
|
||||
f"prefill={len(src_indices)}, dst={len(dst_indices)}"
|
||||
)
|
||||
if (
|
||||
self.kv_args.state_types[index] == StateType.MAMBA
|
||||
and len(src_indices) != 1
|
||||
):
|
||||
raise RuntimeError("Mamba should have single state index")
|
||||
src_parts.append(
|
||||
_prepared_state_selectors(
|
||||
indices=src_indices,
|
||||
offsets=component.src_offsets,
|
||||
capacity=component.src_capacity,
|
||||
)
|
||||
)
|
||||
dst_parts.append(
|
||||
_prepared_state_selectors(
|
||||
indices=dst_indices,
|
||||
offsets=component.dst_offsets,
|
||||
capacity=component.dst_capacity,
|
||||
)
|
||||
)
|
||||
if not src_parts:
|
||||
return
|
||||
src_selectors = (
|
||||
src_parts[0] if len(src_parts) == 1 else np.concatenate(src_parts)
|
||||
)
|
||||
dst_selectors = (
|
||||
dst_parts[0] if len(dst_parts) == 1 else np.concatenate(dst_parts)
|
||||
)
|
||||
xfer_handle = self.agent.make_prepped_xfer(
|
||||
"WRITE", plan.src_handle, src_selectors, plan.dst_handle, dst_selectors, b""
|
||||
)
|
||||
if not xfer_handle:
|
||||
raise RuntimeError("Failed to create prepared state transfer")
|
||||
# The chunk owns the handle before posting, so a failed post settles too.
|
||||
yield xfer_handle
|
||||
if self.agent.transfer(xfer_handle) == "ERR":
|
||||
raise RuntimeError("Failed to post prepared state transfer")
|
||||
|
||||
def _init_staging_prefill_ctx(self):
|
||||
from sglang.srt.disaggregation.common.staging_handler import (
|
||||
PrefillStagingContext,
|
||||
@@ -1239,6 +1594,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
kv_chunk: TransferKVChunk = queue.get()
|
||||
room = kv_chunk.room
|
||||
handles: List[Any] = []
|
||||
state_notifs: List[Tuple[str, str]] = []
|
||||
settle_timed_out = False
|
||||
room_transfer_infos = None
|
||||
try:
|
||||
@@ -1476,8 +1832,20 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
dst_state_dim_per_tensor=dst_info.dst_state_dim_per_tensor,
|
||||
dst_state_layer_ids=dst_info.dst_state_layer_ids,
|
||||
)
|
||||
handles.extend(
|
||||
h for h in state_xfer_handles if h is not None
|
||||
for state_xfer_handle in state_xfer_handles:
|
||||
handles.append(state_xfer_handle)
|
||||
|
||||
if req.dst_state_indices:
|
||||
if not kv_chunk.state_indices:
|
||||
logger.warning_once(
|
||||
"Decode expects state but prefill has none to "
|
||||
"send; announcing state completion anyway"
|
||||
)
|
||||
state_notifs.append(
|
||||
(
|
||||
req.agent_name,
|
||||
f"{req.room}_state_{self.transfer_source_rank}",
|
||||
)
|
||||
)
|
||||
|
||||
if kv_chunk.prefill_aux_index is None:
|
||||
@@ -1520,6 +1888,10 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
if any_failed:
|
||||
raise RuntimeError(f"NIXL transfer encountered ERR room={room}")
|
||||
|
||||
if self.request_status.get(room) not in (None, KVPoll.Failed):
|
||||
for peer_name, notif in state_notifs:
|
||||
self.agent.send_notif(peer_name, notif.encode("ascii"))
|
||||
|
||||
# Clear with the decrement: the failure path below reads this
|
||||
# flag, so a raise after this point must not uncount twice.
|
||||
kv_chunk.staging_counted = False
|
||||
@@ -1667,6 +2039,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
self.agent.add_remote_agent(decode_kv_args.agent_metadata)
|
||||
if self.disaggregation_mode == DisaggregationMode.PREFILL:
|
||||
self._prepare_payload_xfer(decode_kv_args)
|
||||
self._prepare_state_xfer(decode_kv_args)
|
||||
|
||||
def _reload_invalidated_peers(self, room_transfer_infos: Dict[str, Any]) -> None:
|
||||
# NIXL drops a peer's metadata after a remote-disconnect error and never
|
||||
@@ -1694,10 +2067,12 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
# peer has the same peer set, so all shard to this worker.
|
||||
self.prep_handles.pop(agent_name, None)
|
||||
self.prep_handles_slice_dst.pop(agent_name, None)
|
||||
self._prepared_state_plans.pop(agent_name, None)
|
||||
peer_info.kv_xfer_segments = None
|
||||
self.agent.add_remote_agent(peer_info.agent_metadata)
|
||||
try:
|
||||
self._prepare_payload_xfer(peer_info)
|
||||
self._prepare_state_xfer(peer_info)
|
||||
except Exception:
|
||||
# Leave the peer invalid so a later failure retries.
|
||||
self.agent.remove_remote_agent(agent_name)
|
||||
@@ -2708,7 +3083,12 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
dst_state_dim_per_tensor: List[List[int]] | None = None,
|
||||
dst_state_layer_ids: List[List[int]] | None = None,
|
||||
):
|
||||
"""Send state per hybrid component, dispatching by state_type[i]."""
|
||||
"""Yield each state handle to the chunk owner; completion is announced there."""
|
||||
rejection = self._state_layout_rejections.get(peer_name)
|
||||
if rejection is not None:
|
||||
raise RuntimeError(
|
||||
f"NIXL state layout rejected for {peer_name}: {rejection}"
|
||||
)
|
||||
state_types = getattr(self.kv_args, "state_types", []) or []
|
||||
src_state_data_ptrs = self.kv_args.state_data_ptrs or []
|
||||
src_state_item_lens = self.kv_args.state_item_lens or []
|
||||
@@ -2726,8 +3106,18 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
dst_state_dim_per_tensor = dst_state_dim_per_tensor or []
|
||||
dst_state_layer_ids = dst_state_layer_ids or []
|
||||
|
||||
handles = []
|
||||
plan = self._prepared_state_plans.get(peer_name)
|
||||
if plan is not None:
|
||||
yield from self._send_prepared_states(
|
||||
plan=plan,
|
||||
prefill_state_indices=prefill_state_indices,
|
||||
dst_state_indices=dst_state_indices,
|
||||
)
|
||||
prepared_components = plan.components if plan is not None else {}
|
||||
|
||||
for i, st in enumerate(state_types):
|
||||
if i in prepared_components:
|
||||
continue
|
||||
src_indices = (
|
||||
prefill_state_indices[i] if i < len(prefill_state_indices) else None
|
||||
)
|
||||
@@ -2758,7 +3148,6 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
dst_state_dim_per_tensor[i] if i < len(dst_state_dim_per_tensor) else []
|
||||
)
|
||||
dst_lids = dst_state_layer_ids[i] if i < len(dst_state_layer_ids) else []
|
||||
comp_notif = f"{notif}_{i}"
|
||||
|
||||
if st == StateType.MAMBA:
|
||||
if self.state_tp_size != decode_tp_size:
|
||||
@@ -2778,7 +3167,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
dst_lens,
|
||||
dst_dims,
|
||||
dst_gpu_id,
|
||||
comp_notif,
|
||||
"",
|
||||
decode_tp_size,
|
||||
decode_tp_rank,
|
||||
src_conv,
|
||||
@@ -2795,7 +3184,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
dst_ptrs,
|
||||
dst_indices,
|
||||
dst_gpu_id,
|
||||
comp_notif,
|
||||
"",
|
||||
src_layer_ids=src_lids,
|
||||
dst_layer_ids=dst_lids,
|
||||
)
|
||||
@@ -2809,7 +3198,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
list(src_indices),
|
||||
list(dst_indices),
|
||||
dst_gpu_id,
|
||||
comp_notif,
|
||||
"",
|
||||
)
|
||||
elif st == StateType.DSA:
|
||||
if len(src_indices) != len(dst_indices):
|
||||
@@ -2825,7 +3214,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
prefill_data_indices=np.array(src_indices, dtype=np.int32),
|
||||
dst_data_indices=np.array(dst_indices, dtype=np.int32),
|
||||
dst_gpu_id=dst_gpu_id,
|
||||
notif=comp_notif,
|
||||
notif="",
|
||||
state_type=st,
|
||||
force_flat=True,
|
||||
)
|
||||
@@ -2843,7 +3232,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
dst_swa_page_indices=dst_indices,
|
||||
dst_state_item_lens=dst_lens,
|
||||
dst_gpu_id=dst_gpu_id,
|
||||
notif=comp_notif,
|
||||
notif="",
|
||||
decode_tp_size=decode_tp_size,
|
||||
decode_tp_rank=decode_tp_rank,
|
||||
)
|
||||
@@ -2885,7 +3274,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
prefill_data_indices=np.array(src_indices, dtype=np.int32),
|
||||
dst_data_indices=np.array(dst_indices, dtype=np.int32),
|
||||
dst_gpu_id=dst_gpu_id,
|
||||
notif=comp_notif,
|
||||
notif="",
|
||||
state_type=st,
|
||||
force_flat=st in (StateType.QSA_PENDING, StateType.QSA_COMPRESSED),
|
||||
src_layer_ids=src_lids,
|
||||
@@ -2916,7 +3305,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
prefill_data_indices=np.array(src_indices, dtype=np.int32),
|
||||
dst_data_indices=np.array(dst_indices, dtype=np.int32),
|
||||
dst_gpu_id=dst_gpu_id,
|
||||
notif=comp_notif,
|
||||
notif="",
|
||||
force_flat=True,
|
||||
)
|
||||
else:
|
||||
@@ -2924,8 +3313,7 @@ class NixlKVManager(StagingManagerMixin, CommonKVManager):
|
||||
f"PD Disaggregation via NIXL does NOT support {st} hybrid models yet."
|
||||
)
|
||||
if h is not None:
|
||||
handles.append(h)
|
||||
return handles
|
||||
yield h
|
||||
|
||||
def add_transfer_request(
|
||||
self,
|
||||
@@ -3550,6 +3938,9 @@ class NixlKVReceiver(CommonKVReceiver):
|
||||
packed_state_item_lens = pack_int_lists(
|
||||
self.kv_mgr.kv_args.state_item_lens or [], "I"
|
||||
)
|
||||
packed_state_data_lens = pack_int_lists(
|
||||
self.kv_mgr.kv_args.state_data_lens or [], "Q"
|
||||
)
|
||||
packed_state_dim_per_tensor = pack_int_lists(
|
||||
getattr(self.kv_mgr.kv_args, "state_dim_per_tensor", []) or [], "I"
|
||||
)
|
||||
@@ -3606,6 +3997,7 @@ class NixlKVReceiver(CommonKVReceiver):
|
||||
packed_kv_layer_ids,
|
||||
str(self.kv_mgr.dcp_size).encode("ascii"),
|
||||
str(self.kv_mgr.dcp_rank).encode("ascii"),
|
||||
packed_state_data_lens,
|
||||
]
|
||||
)
|
||||
except zmq.ZMQError:
|
||||
|
||||
Reference in New Issue
Block a user