refactor(hicache): flatten L2 transfer execution (#34793)
GB300 test fails unrelated
This commit is contained in:
@@ -16,7 +16,6 @@ limitations under the License.
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from functools import cache
|
||||
from queue import Empty, Queue
|
||||
from typing import TYPE_CHECKING, List, NamedTuple, Optional
|
||||
|
||||
@@ -38,6 +37,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
get_attention_dp_rank,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
|
||||
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_device_module
|
||||
@@ -47,26 +47,6 @@ logger = logging.getLogger(__name__)
|
||||
device_module = get_device_module()
|
||||
|
||||
|
||||
@cache
|
||||
def _timing_events_supported() -> bool:
|
||||
try:
|
||||
device_module.Event(enable_timing=True)
|
||||
return True
|
||||
except (TypeError, NotImplementedError):
|
||||
logger.warning(
|
||||
"%s.Event does not support enable_timing=True; load-back "
|
||||
"duration metric will be skipped on this backend.",
|
||||
device_module.__name__,
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def make_timing_event_pair():
|
||||
timing_enabled = _timing_events_supported()
|
||||
kwargs = {"enable_timing": True} if timing_enabled else {}
|
||||
return device_module.Event(**kwargs), device_module.Event(**kwargs), timing_enabled
|
||||
|
||||
|
||||
class LayerLoadingEvent:
|
||||
def __init__(self, num_layers: int):
|
||||
self._num_layers = num_layers
|
||||
@@ -126,30 +106,66 @@ class CacheOperation:
|
||||
device_indices: torch.Tensor,
|
||||
node_id: int,
|
||||
priority: Optional[int] = None,
|
||||
pool_transfers: Optional[List[PoolTransfer]] = None,
|
||||
):
|
||||
self.host_indices = host_indices
|
||||
self.device_indices = device_indices
|
||||
self.node_ids = [node_id]
|
||||
self.data = None
|
||||
self.pool_transfers = pool_transfers
|
||||
|
||||
self.id = CacheOperation.counter
|
||||
CacheOperation.counter += 1
|
||||
# default priority is the order of creation
|
||||
self.priority = priority if priority is not None else self.id
|
||||
|
||||
@staticmethod
|
||||
def _merge_pool_transfers(
|
||||
ops: List[CacheOperation],
|
||||
) -> Optional[List[PoolTransfer]]:
|
||||
grouped: dict[tuple[PoolName, Optional[PoolName]], List[PoolTransfer]] = {}
|
||||
for op in ops:
|
||||
for transfer in op.pool_transfers or []:
|
||||
grouped.setdefault(
|
||||
(transfer.name, transfer.indices_from_pool), []
|
||||
).append(transfer)
|
||||
if not grouped:
|
||||
return None
|
||||
|
||||
def cat_or_none(tensors):
|
||||
parts = [tensor for tensor in tensors if tensor is not None]
|
||||
return torch.cat(parts) if parts else None
|
||||
|
||||
return [
|
||||
PoolTransfer(
|
||||
name=transfers[0].name,
|
||||
host_indices=cat_or_none(t.host_indices for t in transfers),
|
||||
device_indices=cat_or_none(t.device_indices for t in transfers),
|
||||
keys=[key for t in transfers if t.keys for key in t.keys] or None,
|
||||
hit_policy=transfers[0].hit_policy,
|
||||
indices_from_pool=transfers[0].indices_from_pool,
|
||||
)
|
||||
for transfers in grouped.values()
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def merge_ops(ops: List[CacheOperation]) -> CacheOperation:
|
||||
assert len(ops) > 0
|
||||
assert ops
|
||||
if len(ops) == 1:
|
||||
return ops[0]
|
||||
|
||||
host_indices = torch.cat([op.host_indices for op in ops])
|
||||
device_indices = torch.cat([op.device_indices for op in ops])
|
||||
node_ids = []
|
||||
priority = min(op.priority for op in ops)
|
||||
for op in ops:
|
||||
node_ids.extend(op.node_ids)
|
||||
merged_op = CacheOperation(host_indices, device_indices, -1, priority)
|
||||
merged_op = CacheOperation(
|
||||
host_indices,
|
||||
device_indices,
|
||||
-1,
|
||||
priority,
|
||||
pool_transfers=CacheOperation._merge_pool_transfers(ops),
|
||||
)
|
||||
merged_op.node_ids = node_ids
|
||||
return merged_op
|
||||
|
||||
@@ -302,8 +318,7 @@ class HiCacheController:
|
||||
self.ack_load_queue: List[HiCacheAck] = []
|
||||
self.ack_write_queue: List[HiCacheAck] = []
|
||||
|
||||
self.write_stream = device_module.Stream()
|
||||
self.load_stream = device_module.Stream()
|
||||
self.l2_transfer_engine = L2TransferEngine(io_backend)
|
||||
|
||||
# If a storage backend is provided at startup, treat it as an implicit attach,
|
||||
# so init/runtime share the same lifecycle semantics and code paths.
|
||||
@@ -698,60 +713,21 @@ class HiCacheController:
|
||||
return
|
||||
|
||||
op = CacheOperation.merge_ops(self.write_queue)
|
||||
# Kernel write-back keeps host indices on CPU only for page_first AND only
|
||||
# when the staged JIT write-back kernel is available (it stages through
|
||||
# device memory and accepts CPU destination indices). Otherwise we fall back
|
||||
# to the plain transfer kernel, whose CUDA/HIP implementation requires
|
||||
# device-resident destination indices -- so the indices must be moved to the
|
||||
# device first. Without the can_use_write_back_jit check this crashes on
|
||||
# backends where the JIT kernel is unavailable, with
|
||||
# "Destination indices must be a CUDA tensor".
|
||||
if (
|
||||
self.io_backend == "kernel"
|
||||
and self.mem_pool_host.layout == "page_first"
|
||||
and getattr(self.mem_pool_host, "can_use_write_back_jit", False)
|
||||
):
|
||||
host_indices, device_indices = op.host_indices, op.device_indices
|
||||
else:
|
||||
host_indices, device_indices = self.move_indices(
|
||||
op.host_indices, op.device_indices
|
||||
)
|
||||
host_indices, device_indices, pool_transfers = self._move_write_operation(op)
|
||||
self.write_queue.clear()
|
||||
|
||||
start_event = device_module.Event()
|
||||
ack_start_event, ack_finish_event, timing_enabled = make_timing_event_pair()
|
||||
|
||||
start_event.record()
|
||||
with device_module.stream(self.write_stream):
|
||||
start_event.wait(self.write_stream)
|
||||
ack_start_event.record()
|
||||
self.mem_pool_host.backup_from_device_all_layer(
|
||||
self.mem_pool_device, host_indices, device_indices, self.io_backend
|
||||
)
|
||||
if self.has_draft:
|
||||
self.mem_pool_host_draft.backup_from_device_all_layer(
|
||||
self.mem_pool_device_draft,
|
||||
host_indices,
|
||||
device_indices,
|
||||
self.io_backend,
|
||||
)
|
||||
ack_finish_event.record()
|
||||
# NOTE: We must save the host indices and device indices here,
|
||||
# this is because we need to guarantee that these tensors are
|
||||
# still alive when the write stream is executing.
|
||||
if host_indices.is_cuda:
|
||||
host_indices.record_stream(self.write_stream)
|
||||
if device_indices.is_cuda:
|
||||
device_indices.record_stream(self.write_stream)
|
||||
completion = self.l2_transfer_engine.submit_device_to_host(
|
||||
self._l2_transfers(host_indices, device_indices, pool_transfers)
|
||||
)
|
||||
|
||||
self.ack_write_queue.append(
|
||||
HiCacheAck(
|
||||
start_event=ack_start_event,
|
||||
finish_event=ack_finish_event,
|
||||
start_event=completion.start_event,
|
||||
finish_event=completion.finish_event,
|
||||
node_ids=op.node_ids,
|
||||
num_tokens=len(op.device_indices),
|
||||
timing_enabled=timing_enabled,
|
||||
num_tokens_by_pool={PoolName.KV.value: len(op.device_indices)},
|
||||
timing_enabled=completion.timing_enabled,
|
||||
num_tokens_by_pool=self._num_tokens_by_pool(op),
|
||||
num_bytes=self._transfer_num_bytes(op),
|
||||
)
|
||||
)
|
||||
@@ -764,6 +740,9 @@ class HiCacheController:
|
||||
num_bytes += num_tokens * self.mem_pool_host_draft.size_per_token
|
||||
return num_bytes
|
||||
|
||||
def _num_tokens_by_pool(self, op: CacheOperation) -> dict[str, int]:
|
||||
return {PoolName.KV.value: len(op.device_indices)}
|
||||
|
||||
def load(
|
||||
self,
|
||||
host_indices: torch.Tensor,
|
||||
@@ -781,7 +760,9 @@ class HiCacheController:
|
||||
)
|
||||
return device_indices
|
||||
|
||||
def move_indices(self, host_indices: torch.Tensor, device_indices: torch.Tensor):
|
||||
def move_indices(
|
||||
self, host_indices: torch.Tensor, device_indices: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# move indices to GPU if using kernels, to host if using direct indexing
|
||||
if self.io_backend == "kernel":
|
||||
if not host_indices.is_cuda:
|
||||
@@ -803,58 +784,82 @@ class HiCacheController:
|
||||
else:
|
||||
raise ValueError(f"Unsupported io backend")
|
||||
|
||||
def _move_write_operation(
|
||||
self, op: CacheOperation
|
||||
) -> tuple[torch.Tensor, torch.Tensor, Optional[List[PoolTransfer]]]:
|
||||
"""Keep CPU host indices only for page-first staged write-back."""
|
||||
if (
|
||||
self.io_backend == "kernel"
|
||||
and self.mem_pool_host.layout == "page_first"
|
||||
and getattr(self.mem_pool_host, "can_use_write_back_jit", False)
|
||||
):
|
||||
return op.host_indices, op.device_indices, op.pool_transfers
|
||||
return self._move_op_indices(op)
|
||||
|
||||
def _move_op_indices(
|
||||
self, op: CacheOperation
|
||||
) -> tuple[torch.Tensor, torch.Tensor, Optional[List[PoolTransfer]]]:
|
||||
return (*self.move_indices(op.host_indices, op.device_indices), None)
|
||||
|
||||
def _l2_transfers(
|
||||
self,
|
||||
host_indices: torch.Tensor,
|
||||
device_indices: torch.Tensor,
|
||||
pool_transfers: Optional[List[PoolTransfer]] = None,
|
||||
) -> list[L2Transfer]:
|
||||
transfers = [
|
||||
L2Transfer(
|
||||
host_pool=self.mem_pool_host,
|
||||
device_pool=self.mem_pool_device,
|
||||
host_indices=host_indices,
|
||||
device_indices=device_indices,
|
||||
)
|
||||
]
|
||||
if self.has_draft and host_indices.numel() > 0:
|
||||
transfers.append(
|
||||
L2Transfer(
|
||||
host_pool=self.mem_pool_host_draft,
|
||||
device_pool=self.mem_pool_device_draft,
|
||||
host_indices=host_indices,
|
||||
device_indices=device_indices,
|
||||
)
|
||||
)
|
||||
return transfers
|
||||
|
||||
def _l2_load_transfers(
|
||||
self,
|
||||
host_indices: torch.Tensor,
|
||||
device_indices: torch.Tensor,
|
||||
pool_transfers: Optional[List[PoolTransfer]] = None,
|
||||
) -> list[L2Transfer]:
|
||||
return self._l2_transfers(host_indices, device_indices, pool_transfers)
|
||||
|
||||
def start_loading(self) -> int:
|
||||
if len(self.load_queue) == 0:
|
||||
return -1
|
||||
|
||||
producer_id = self.layer_done_counter.update_producer()
|
||||
op = CacheOperation.merge_ops(self.load_queue)
|
||||
host_indices, device_indices = self.move_indices(
|
||||
op.host_indices, op.device_indices
|
||||
)
|
||||
host_indices, device_indices, pool_transfers = self._move_op_indices(op)
|
||||
self.load_queue.clear()
|
||||
producer_event = self.layer_done_counter.events[producer_id]
|
||||
producer_event.start_event.record()
|
||||
|
||||
ack_start_event, ack_finish_event, timing_enabled = make_timing_event_pair()
|
||||
|
||||
with device_module.stream(self.load_stream):
|
||||
producer_event.start_event.wait(self.load_stream)
|
||||
ack_start_event.record()
|
||||
for i in range(self.layer_num):
|
||||
self.mem_pool_host.load_to_device_per_layer(
|
||||
self.mem_pool_device,
|
||||
host_indices,
|
||||
device_indices,
|
||||
i,
|
||||
self.io_backend,
|
||||
)
|
||||
if self.has_draft and i < self.mem_pool_host_draft.layer_num:
|
||||
self.mem_pool_host_draft.load_to_device_per_layer(
|
||||
self.mem_pool_device_draft,
|
||||
host_indices,
|
||||
device_indices,
|
||||
i,
|
||||
self.io_backend,
|
||||
)
|
||||
producer_event.complete(i)
|
||||
ack_finish_event.record()
|
||||
# NOTE: We must save the host indices and device indices here,
|
||||
# this is because we need to guarantee that these tensors are
|
||||
# still alive when the load stream is executing.
|
||||
if host_indices.is_cuda:
|
||||
host_indices.record_stream(self.load_stream)
|
||||
if device_indices.is_cuda:
|
||||
device_indices.record_stream(self.load_stream)
|
||||
completion = self.l2_transfer_engine.submit_host_to_device(
|
||||
self._l2_load_transfers(host_indices, device_indices, pool_transfers),
|
||||
start_event=producer_event.start_event,
|
||||
on_layer_done=producer_event.complete,
|
||||
layer_num=self.layer_num,
|
||||
)
|
||||
|
||||
self.ack_load_queue.append(
|
||||
HiCacheAck(
|
||||
start_event=ack_start_event,
|
||||
finish_event=ack_finish_event,
|
||||
start_event=completion.start_event,
|
||||
finish_event=completion.finish_event,
|
||||
node_ids=op.node_ids,
|
||||
num_tokens=len(op.device_indices),
|
||||
timing_enabled=timing_enabled,
|
||||
num_tokens_by_pool={PoolName.KV.value: len(op.device_indices)},
|
||||
timing_enabled=completion.timing_enabled,
|
||||
num_tokens_by_pool=self._num_tokens_by_pool(op),
|
||||
num_bytes=self._transfer_num_bytes(op),
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user