[PD] Add decode host receive for custom transfer backends (#40238)
Co-authored-by: cctry <cctry@fb.com>
This commit is contained in:
@@ -497,5 +497,25 @@ POSITIONAL_FIELD_ORDER = (
|
||||
"weight_cache_timeout",
|
||||
"forward_hooks",
|
||||
"msprobe_dump_config",
|
||||
"ple_offload_embedding",
|
||||
"ple_offload_backend",
|
||||
"ple_offload_dir",
|
||||
"disaggregation_enable_kv_checksum",
|
||||
"enable_pd_role_switch",
|
||||
"speculative_domino_candidate_pool_size",
|
||||
"flashinfer_a2a_dispatch_type",
|
||||
"cuda_graph_max_seq_len_prefill",
|
||||
"prefill_kv_cache_dequant_dtype",
|
||||
"dsv4_attn_backend",
|
||||
"otlp_service_name",
|
||||
"_radix_eviction_policy_explicitly_set",
|
||||
"hicache_host_memory_fraction",
|
||||
"enable_linker_mla_dedup",
|
||||
"_swa_full_tokens_ratio_explicitly_set",
|
||||
"swa_prefix_tails",
|
||||
"enable_encoder_swa_bounded_replay",
|
||||
"enable_decoder_swa_bounded_replay",
|
||||
"enable_response_store",
|
||||
"disaggregation_decode_host_receive_threshold",
|
||||
)
|
||||
# fmt: on
|
||||
|
||||
@@ -168,3 +168,7 @@ class Disagg(msgspec.Struct):
|
||||
"The path of the PD-Multiplexing config file.",
|
||||
] = None
|
||||
sm_group_num: A[int, "Number of sm partition groups."] = 8
|
||||
disaggregation_decode_host_receive_threshold: A[
|
||||
float,
|
||||
"Device token usage fraction at which incoming KV is received in the decode retraction host pool, excluding evictable cache pages. Range [0, 1]; 0 disables host receive. Size with --hicache-size or --hicache-ratio; requires dense MHA and a transfer backend that supports host destinations. No built-in backend currently supports this.",
|
||||
] = 0.0
|
||||
|
||||
@@ -54,11 +54,25 @@ def handle_hicache(server_args: Any):
|
||||
|
||||
validate_hicache_host_memory_mode(server_args)
|
||||
|
||||
if cfg.disaggregation_decode_host_receive_threshold > 0:
|
||||
# The shared host pool must match KV transfer's per-layer buffers.
|
||||
declare_resolution(
|
||||
server_args, "handle_hicache", hicache_mem_layout="layer_first"
|
||||
)
|
||||
|
||||
# Step 1: Initial layout-io compatibility normalization.
|
||||
resolve_layout_io_compatibility(server_args)
|
||||
|
||||
# Step 2: Storage-layout normalization without changing io backend.
|
||||
resolve_storage_layout_compatibility(server_args)
|
||||
if (
|
||||
cfg.disaggregation_decode_host_receive_threshold > 0
|
||||
and cfg.hicache_mem_layout != "layer_first"
|
||||
):
|
||||
raise ValueError(
|
||||
f"The resolved HiCache storage layout {cfg.hicache_mem_layout!r} "
|
||||
"cannot share the layer_first decode host pool used by KV transfer"
|
||||
)
|
||||
|
||||
# Step 3: DCP compatibility for the L2 (device<->host) path.
|
||||
resolve_hicache_dcp_compatibility(server_args)
|
||||
|
||||
@@ -50,6 +50,24 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None:
|
||||
"overhead without improving prefill performance."
|
||||
)
|
||||
|
||||
if not 0 <= cfg.disaggregation_decode_host_receive_threshold <= 1:
|
||||
raise ValueError(
|
||||
"--disaggregation-decode-host-receive-threshold must be between 0 and 1"
|
||||
)
|
||||
if cfg.disaggregation_decode_host_receive_threshold > 0:
|
||||
if cfg.enable_hisparse or cfg.enable_pd_role_switch:
|
||||
raise ValueError(
|
||||
"Decode host receive does not yet support HiSparse or role switching"
|
||||
)
|
||||
|
||||
if cfg.disaggregation_decode_retraction_backup == "cpu_tensor":
|
||||
raise ValueError("Decode host KV buffering requires host_pool retraction")
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"handle_pd_disaggregation",
|
||||
disaggregation_decode_retraction_backup="host_pool",
|
||||
)
|
||||
|
||||
if cfg.disaggregation_mode == "decode" and cfg.dcp_size > 1:
|
||||
# Fake transfer moves no KV and is only used for synthetic decode
|
||||
# benchmarks, so it does not need the DCP relayout from Mooncake/NIXL.
|
||||
|
||||
@@ -46,6 +46,11 @@ class KVTransferMetric:
|
||||
transfer_total_bytes: Optional[int] = None
|
||||
|
||||
|
||||
class KVTransferDestination(str, enum.Enum):
|
||||
DEVICE = "device"
|
||||
HOST = "host"
|
||||
|
||||
|
||||
class KVArgs:
|
||||
engine_rank: int
|
||||
kv_data_ptrs: List[int]
|
||||
@@ -53,6 +58,9 @@ class KVArgs:
|
||||
kv_item_lens: List[int]
|
||||
kv_layer_ids: List[int]
|
||||
kv_cache_dtype_str: str
|
||||
host_kv_data_ptrs: Optional[List[int]] = None
|
||||
host_kv_data_lens: Optional[List[int]] = None
|
||||
host_kv_item_lens: Optional[List[int]] = None
|
||||
aux_data_ptrs: List[int]
|
||||
aux_data_lens: List[int]
|
||||
aux_item_lens: List[int]
|
||||
@@ -113,6 +121,7 @@ class BaseKVManager(ABC):
|
||||
"""Base class for managing transfer states"""
|
||||
|
||||
enable_deferred_decode_kv_release: bool = False
|
||||
supports_host_destination: bool = False
|
||||
|
||||
@abstractmethod
|
||||
def __init__(
|
||||
@@ -212,6 +221,11 @@ class BaseKVSender(ABC):
|
||||
|
||||
|
||||
class BaseKVReceiver(ABC):
|
||||
@property
|
||||
def supports_host_destination(self) -> bool:
|
||||
"""Whether this receiver's peer and layout support host KV destinations."""
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def __init__(
|
||||
self,
|
||||
@@ -237,6 +251,7 @@ class BaseKVReceiver(ABC):
|
||||
aux_index: Optional[int] = None,
|
||||
state_indices: Optional[List] = None,
|
||||
decode_prefix_len: Optional[int] = None,
|
||||
destination: KVTransferDestination = KVTransferDestination.DEVICE,
|
||||
):
|
||||
"""
|
||||
Notify the prefill server about the kv indices, aux index, and state_indices.
|
||||
|
||||
@@ -23,6 +23,7 @@ from sglang.srt.disaggregation.base.conn import (
|
||||
BaseKVSender,
|
||||
KVArgs,
|
||||
KVPoll,
|
||||
KVTransferDestination,
|
||||
KVTransferMetric,
|
||||
StateType,
|
||||
)
|
||||
@@ -1931,6 +1932,7 @@ class CommonKVReceiver(BaseKVReceiver):
|
||||
aux_index: Optional[int] = None,
|
||||
state_indices: Optional[List[int]] = None,
|
||||
decode_prefix_len: Optional[int] = None,
|
||||
destination: KVTransferDestination = KVTransferDestination.DEVICE,
|
||||
):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ from torch.distributed import ProcessGroup
|
||||
from sglang.srt.configs.mamba_utils import Mamba2CacheParams
|
||||
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
|
||||
from sglang.srt.disaggregation.base import KVPoll
|
||||
from sglang.srt.disaggregation.base.conn import StateType
|
||||
from sglang.srt.disaggregation.base.conn import KVTransferDestination, StateType
|
||||
from sglang.srt.disaggregation.checksum import (
|
||||
KvChecksumComputer,
|
||||
is_health_check_req,
|
||||
@@ -88,14 +88,17 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
EvictParams,
|
||||
)
|
||||
from sglang.srt.mem_cache.common import (
|
||||
RetractionBackup,
|
||||
discard_kv_cache_backup,
|
||||
dsv41_dspark_needs_rebootstrap,
|
||||
kv_to_page_indices,
|
||||
page_align_floor,
|
||||
release_kv_cache,
|
||||
retraction_discard,
|
||||
retraction_restore,
|
||||
restore_kv_cache,
|
||||
)
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
|
||||
from sglang.srt.mem_cache.kv_cache_builder import decode_retraction_max_tokens
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
HybridReqToTokenPool,
|
||||
KVCache,
|
||||
@@ -330,6 +333,7 @@ class DecodeRequest:
|
||||
waiting_for_input: bool = False
|
||||
metadata_buffer_index: int = -1
|
||||
is_rebootstrap: bool = False
|
||||
host_staged: bool = False
|
||||
|
||||
# HiCache Status
|
||||
prefix_match: Optional[DecodePrefixMatch] = None
|
||||
@@ -382,6 +386,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
self.scheduler = scheduler
|
||||
self.transfer_queue = transfer_queue
|
||||
self.tree_cache = tree_cache
|
||||
self.host_pool = None
|
||||
self.host_reserved_tokens = 0
|
||||
self.gloo_group = gloo_group
|
||||
# Destinations visible to prefill but not yet on the transfer queue.
|
||||
self._num_published_destinations = 0
|
||||
@@ -460,6 +466,17 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
or self.token_to_kv_pool_allocator.prealloc_fits_assumes_reclaim()
|
||||
)
|
||||
|
||||
def _required_admission_tokens(
|
||||
self, req: Req, allocated_tokens: int, prefix_len: int, retractable_tokens: int
|
||||
) -> int:
|
||||
return max(
|
||||
allocated_tokens + self.num_reserved_decode_tokens,
|
||||
self._rebootstrap_prefill_len(req)
|
||||
- prefix_len
|
||||
+ min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKEN)
|
||||
- retractable_tokens,
|
||||
)
|
||||
|
||||
def _prealloc_reservation_fits(
|
||||
self,
|
||||
full_tokens: int,
|
||||
@@ -630,6 +647,31 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
total_kv_layers=self.scheduler.model_config.num_hidden_layers,
|
||||
req_to_token_pool=getattr(self, "req_to_token_pool", None),
|
||||
)
|
||||
if get_disagg().disaggregation_decode_host_receive_threshold > 0:
|
||||
pool = self.token_to_kv_pool
|
||||
group = self.tree_cache.host_pool_group
|
||||
if kv_args.state_types or any(
|
||||
spec.indices_from_pool != PoolName.KV
|
||||
for spec in self.tree_cache.sidecar_pool_specs
|
||||
):
|
||||
raise ValueError(
|
||||
"Host receive requires KV pools sharing the primary indices"
|
||||
)
|
||||
self.host_pool = group.get_pool(PoolName.KV)
|
||||
self.host_reserved_tokens = decode_retraction_max_tokens(
|
||||
self.req_to_token_pool, pool
|
||||
)
|
||||
if self.host_pool.logical_size < self.host_reserved_tokens + pool.page_size:
|
||||
raise ValueError(
|
||||
"Host pool must hold a retraction and a receive page; "
|
||||
"increase --hicache-size or --hicache-ratio"
|
||||
)
|
||||
_, host_buffers = group.get_contiguous_buf_infos()
|
||||
(
|
||||
kv_args.host_kv_data_ptrs,
|
||||
kv_args.host_kv_data_lens,
|
||||
kv_args.host_kv_item_lens,
|
||||
) = host_buffers
|
||||
|
||||
kv_args.ib_device = get_disagg().disaggregation_ib_device
|
||||
kv_args.gpu_id = get_device().gpu_id
|
||||
@@ -640,6 +682,13 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
self.scheduler.server_args,
|
||||
self.is_mla_backend,
|
||||
)
|
||||
if (
|
||||
get_disagg().disaggregation_decode_host_receive_threshold > 0
|
||||
and not kv_manager.supports_host_destination
|
||||
):
|
||||
raise ValueError(
|
||||
"Transfer backend does not support decode host KV destinations"
|
||||
)
|
||||
# Staging buffer setup (only when heterogeneous TP staging is enabled)
|
||||
if self.enable_staging and not self.is_mla_backend:
|
||||
kv_pool_for_heads = self.token_to_kv_pool
|
||||
@@ -872,7 +921,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
self._cancel_prefill_dp_rank_queries()
|
||||
self.queue.clear()
|
||||
for req in self.retracted_queue:
|
||||
retraction_discard(
|
||||
discard_kv_cache_backup(
|
||||
req,
|
||||
self.tree_cache,
|
||||
get_disagg().disaggregation_decode_retraction_backup,
|
||||
@@ -941,7 +990,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
extra_reserved_reqs=len(resumed_reqs),
|
||||
)
|
||||
|
||||
retraction_restore(
|
||||
restore_kv_cache(
|
||||
req,
|
||||
self.tree_cache,
|
||||
self.req_to_token_pool,
|
||||
@@ -1186,6 +1235,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
|
||||
failed_reqs = []
|
||||
preallocated_reqs = []
|
||||
num_device_preallocated = 0
|
||||
indices_to_remove = set()
|
||||
|
||||
# We need to make sure that the sum of inflight tokens and allocatable tokens is greater than maximum input+output length of each inflight request
|
||||
@@ -1285,6 +1335,30 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
if not decode_req.waiting_for_input:
|
||||
continue
|
||||
|
||||
if self.req_to_metadata_buffer_idx_allocator.available_size() <= 0:
|
||||
break
|
||||
|
||||
if self.scheduler.enable_lora and not self.scheduler.can_schedule_lora_req(
|
||||
decode_req.req, running_loras
|
||||
):
|
||||
continue
|
||||
|
||||
if (
|
||||
get_disagg().disaggregation_decode_host_receive_threshold > 0
|
||||
and not decode_req.is_rebootstrap
|
||||
and not _is_fake_transfer(decode_req.req)
|
||||
and decode_req.kv_receiver.supports_host_destination
|
||||
and self.scheduler.pool_stats_observer.get_pool_stats().full_token_usage
|
||||
>= get_disagg().disaggregation_decode_host_receive_threshold
|
||||
):
|
||||
if not self._pre_alloc_host(decode_req):
|
||||
break
|
||||
preallocated_reqs.append(decode_req)
|
||||
indices_to_remove.add(i)
|
||||
if self.scheduler.enable_lora:
|
||||
running_loras.add(decode_req.req.lora_id)
|
||||
continue
|
||||
|
||||
if self.req_to_token_pool.available_size() <= 0:
|
||||
break
|
||||
|
||||
@@ -1301,17 +1375,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
if mamba_allocator.available_size() <= 0:
|
||||
break
|
||||
|
||||
if self.req_to_metadata_buffer_idx_allocator.available_size() <= 0:
|
||||
break
|
||||
|
||||
if hisparse_req_budget <= 0:
|
||||
break
|
||||
|
||||
if self.scheduler.enable_lora and not self.scheduler.can_schedule_lora_req(
|
||||
decode_req.req, running_loras
|
||||
):
|
||||
continue
|
||||
|
||||
# Memory estimation: don't add if the projected memory cannot be met
|
||||
# TODO: add new_token ratio
|
||||
origin_input_len = self._rebootstrap_prefill_len(decode_req.req)
|
||||
@@ -1369,7 +1435,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
full_allocatable_tokens = self._allocatable_token_budgets(
|
||||
retractable_tokens=retractable_tokens,
|
||||
count_retracted=True,
|
||||
extra_reserved_reqs=len(preallocated_reqs),
|
||||
extra_reserved_reqs=num_device_preallocated,
|
||||
hicache_reserved_tokens=reserved_restore_tokens,
|
||||
)
|
||||
if uses_swa_tail_prealloc:
|
||||
@@ -1377,7 +1443,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
retractable_tokens=retractable_tokens,
|
||||
retractable_swa_tokens=retractable_swa_tokens,
|
||||
count_retracted=True,
|
||||
extra_reserved_reqs=len(preallocated_reqs),
|
||||
extra_reserved_reqs=num_device_preallocated,
|
||||
)
|
||||
else:
|
||||
prefix_indices = None
|
||||
@@ -1385,19 +1451,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
total_prefix_len = 0
|
||||
required_alloc_tokens = self._pre_alloc_fill_len(decode_req.req)
|
||||
|
||||
required_tokens_for_request = (
|
||||
required_alloc_tokens + self.num_reserved_decode_tokens
|
||||
)
|
||||
|
||||
full_required_for_admission = max(
|
||||
required_tokens_for_request,
|
||||
origin_input_len
|
||||
- prefix_len
|
||||
+ min(
|
||||
decode_req.req.sampling_params.max_new_tokens,
|
||||
CLIP_MAX_NEW_TOKEN,
|
||||
)
|
||||
- retractable_tokens,
|
||||
full_required_for_admission = self._required_admission_tokens(
|
||||
decode_req.req, required_alloc_tokens, prefix_len, retractable_tokens
|
||||
)
|
||||
swa_required_for_admission = 0
|
||||
swa_len = required_alloc_tokens
|
||||
@@ -1465,7 +1520,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
full_allocatable_tokens = self._allocatable_token_budgets(
|
||||
retractable_tokens=retractable_tokens,
|
||||
count_retracted=True,
|
||||
extra_reserved_reqs=len(preallocated_reqs) + 1,
|
||||
extra_reserved_reqs=num_device_preallocated + 1,
|
||||
hicache_reserved_tokens=reserved_restore_tokens,
|
||||
)
|
||||
if swa_allocatable_tokens is not None:
|
||||
@@ -1473,7 +1528,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
retractable_tokens=retractable_tokens,
|
||||
retractable_swa_tokens=retractable_swa_tokens,
|
||||
count_retracted=True,
|
||||
extra_reserved_reqs=len(preallocated_reqs) + 1,
|
||||
extra_reserved_reqs=num_device_preallocated + 1,
|
||||
)
|
||||
decode_req.req.kv.cache_protected_len = total_prefix_len
|
||||
|
||||
@@ -1601,14 +1656,6 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
payloads[st]() if st in payloads else None for st in state_types
|
||||
]
|
||||
|
||||
decode_req.metadata_buffer_index = (
|
||||
self.req_to_metadata_buffer_idx_allocator.alloc()
|
||||
)
|
||||
assert decode_req.metadata_buffer_index is not None
|
||||
# int32 for ZMQ serialization -- from_zmq reads np.int32.
|
||||
page_indices = kv_to_page_indices(kv_indices, kv_transfer_page_size).astype(
|
||||
np.int32
|
||||
)
|
||||
device_page_indices = None
|
||||
if (
|
||||
self.scheduler.enable_hisparse
|
||||
@@ -1635,33 +1682,18 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
metadata_kwargs = {"decode_prefix_len": total_prefix_len}
|
||||
if device_page_indices is not None:
|
||||
metadata_kwargs["device_kv_indices"] = device_page_indices
|
||||
if (
|
||||
self.transfer_queue.enable_staging
|
||||
and hasattr(decode_req.kv_receiver, "require_staging")
|
||||
and decode_req.kv_receiver.require_staging
|
||||
):
|
||||
# Register before send_metadata, which triggers the STAGING_REQ
|
||||
# prefetch (dropped for an unregistered room); tiny race, correct order.
|
||||
self.transfer_queue.staging_handler.register_decode_req(
|
||||
decode_req.req.bootstrap_room, decode_req
|
||||
)
|
||||
decode_req.kv_receiver.send_metadata(
|
||||
page_indices,
|
||||
decode_req.metadata_buffer_index,
|
||||
self._send_kv_metadata(
|
||||
decode_req,
|
||||
kv_indices,
|
||||
kv_transfer_page_size,
|
||||
state_indices,
|
||||
**metadata_kwargs,
|
||||
)
|
||||
if decode_req.is_rebootstrap:
|
||||
self.kv_manager.submit_prefill_recompute(
|
||||
decode_req.kv_receiver,
|
||||
decode_req.req.build_rebootstrap_payload(),
|
||||
)
|
||||
self._num_published_destinations += 1
|
||||
num_device_preallocated += 1
|
||||
preallocated_reqs.append(decode_req)
|
||||
indices_to_remove.add(i)
|
||||
if self.scheduler.enable_lora:
|
||||
running_loras.add(decode_req.req.lora_id)
|
||||
decode_req.req.time_stats.set_decode_transfer_queue_entry_time()
|
||||
|
||||
if failed_reqs:
|
||||
failed_ids = {id(r) for r in failed_reqs}
|
||||
@@ -1675,6 +1707,121 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
|
||||
return preallocated_reqs, failed_reqs
|
||||
|
||||
def _send_kv_metadata(
|
||||
self, decode_req, kv_indices, page_size, state_indices=None, **metadata_kwargs
|
||||
) -> None:
|
||||
decode_req.metadata_buffer_index = (
|
||||
self.req_to_metadata_buffer_idx_allocator.alloc()
|
||||
)
|
||||
assert decode_req.metadata_buffer_index is not None
|
||||
page_indices = kv_to_page_indices(kv_indices, page_size).astype(np.int32)
|
||||
if (
|
||||
metadata_kwargs.get("destination") != KVTransferDestination.HOST
|
||||
and self.transfer_queue.enable_staging
|
||||
and hasattr(decode_req.kv_receiver, "require_staging")
|
||||
and decode_req.kv_receiver.require_staging
|
||||
):
|
||||
# Register before send_metadata, which triggers the STAGING_REQ
|
||||
# prefetch (dropped for an unregistered room); tiny race, correct order.
|
||||
self.transfer_queue.staging_handler.register_decode_req(
|
||||
decode_req.req.bootstrap_room, decode_req
|
||||
)
|
||||
decode_req.kv_receiver.send_metadata(
|
||||
page_indices,
|
||||
decode_req.metadata_buffer_index,
|
||||
state_indices,
|
||||
**metadata_kwargs,
|
||||
)
|
||||
if decode_req.is_rebootstrap:
|
||||
self.kv_manager.submit_prefill_recompute(
|
||||
decode_req.kv_receiver,
|
||||
decode_req.req.build_rebootstrap_payload(),
|
||||
)
|
||||
self._num_published_destinations += 1
|
||||
decode_req.req.time_stats.set_decode_transfer_queue_entry_time()
|
||||
|
||||
def _pre_alloc_host(self, decode_req: DecodeRequest) -> bool:
|
||||
num_tokens = ceil_align(
|
||||
len(decode_req.req.origin_input_ids), self.host_pool.page_size
|
||||
)
|
||||
required_tokens = num_tokens + self.host_reserved_tokens
|
||||
if (
|
||||
self.host_pool.available_size() < required_tokens
|
||||
and not self.tree_cache.disable
|
||||
):
|
||||
self.tree_cache.evict_host(required_tokens)
|
||||
if self.host_pool.available_size() < required_tokens:
|
||||
return False
|
||||
host_indices = self.host_pool.alloc(num_tokens)
|
||||
if host_indices is None:
|
||||
return False
|
||||
assert decode_req.req.kv.retraction_backup is None
|
||||
# All sidecars share the primary KV indices.
|
||||
decode_req.req.kv.retraction_backup = RetractionBackup(
|
||||
host_indices=host_indices,
|
||||
pool_transfers=[
|
||||
PoolTransfer(
|
||||
name=spec.pool_name,
|
||||
host_indices=host_indices,
|
||||
indices_from_pool=spec.indices_from_pool,
|
||||
hit_policy=spec.hit_policy,
|
||||
)
|
||||
for spec in self.tree_cache.sidecar_pool_specs
|
||||
if num_tokens
|
||||
]
|
||||
or None,
|
||||
)
|
||||
if get_disagg().disaggregation_decode_enable_radix_cache:
|
||||
req = decode_req.req
|
||||
req.prefix_indices = torch.empty((0,), dtype=torch.int64)
|
||||
req.last_node = self.tree_cache.root_node_handle(req.extra_key)
|
||||
req.last_host_node = req.last_node
|
||||
req.best_match_node = req.last_node
|
||||
req.lock_receipt = DecLockRefParams()
|
||||
req.kv.cache_protected_len = 0
|
||||
req.num_matched_prefix_tokens = 0
|
||||
req.host_hit_length = 0
|
||||
self._send_kv_metadata(
|
||||
decode_req,
|
||||
host_indices,
|
||||
self.token_to_kv_pool_allocator.page_size,
|
||||
decode_prefix_len=0,
|
||||
destination=KVTransferDestination.HOST,
|
||||
)
|
||||
decode_req.host_staged = True
|
||||
if self.scheduler.metrics_reporter.current_scheduler_metrics_enabled:
|
||||
self.scheduler.metrics_collector.increment_decode_host_receive_reqs()
|
||||
return True
|
||||
|
||||
def allocate_host_staged(self, decode_req: DecodeRequest) -> bool:
|
||||
if self.req_to_token_pool.available_size() <= 0:
|
||||
return False
|
||||
req = decode_req.req
|
||||
retractable_tokens = sum(
|
||||
len(r.origin_input_ids) + len(r.output_ids)
|
||||
for r in self.scheduler.running_batch.reqs
|
||||
)
|
||||
required_tokens = self._required_admission_tokens(
|
||||
req,
|
||||
self._required_alloc_tokens(
|
||||
fill_len=len(req.origin_input_ids), prefix_len=0
|
||||
),
|
||||
0,
|
||||
retractable_tokens,
|
||||
)
|
||||
if not self._prealloc_reservation_fits(
|
||||
required_tokens,
|
||||
0,
|
||||
full_allocatable_tokens=self._allocatable_token_budgets(
|
||||
retractable_tokens=retractable_tokens, count_retracted=True
|
||||
),
|
||||
swa_allocatable_tokens=None,
|
||||
):
|
||||
return False
|
||||
self._pre_alloc(req)
|
||||
decode_req.host_staged = False
|
||||
return True
|
||||
|
||||
@property
|
||||
def has_published_destinations(self) -> bool:
|
||||
"""Whether any destination address is visible to prefill but not yet
|
||||
@@ -1690,7 +1837,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
@property
|
||||
def num_tokens_pre_allocated(self):
|
||||
return sum(
|
||||
decode_req.req.extend_range.end for decode_req in self.transfer_queue.queue
|
||||
decode_req.req.extend_range.end
|
||||
for decode_req in self.transfer_queue.queue
|
||||
if not decode_req.host_staged
|
||||
)
|
||||
|
||||
def _need_space_for_single_req(
|
||||
@@ -1714,7 +1863,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
def _active_req_count(self, extra_reserved_reqs: int = 0) -> int:
|
||||
return (
|
||||
len(self.scheduler.running_batch.reqs)
|
||||
+ len(self.transfer_queue.queue)
|
||||
+ sum(not r.host_staged for r in self.transfer_queue.queue)
|
||||
+ len(self.scheduler.waiting_queue)
|
||||
+ extra_reserved_reqs
|
||||
)
|
||||
@@ -2193,6 +2342,8 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
Store the requests that is polling kv
|
||||
"""
|
||||
|
||||
enable_host_receive: bool = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
gloo_group: ProcessGroup,
|
||||
@@ -2207,6 +2358,9 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
self.metadata_buffers = metadata_buffers
|
||||
self.scheduler = scheduler
|
||||
self.enable_host_receive = (
|
||||
get_disagg().disaggregation_decode_host_receive_threshold > 0
|
||||
)
|
||||
self.tree_cache = tree_cache
|
||||
self.spec_algorithm = scheduler.spec_algorithm
|
||||
self.enable_staging = envs.SGLANG_DISAGG_STAGING_BUFFER.get()
|
||||
@@ -2444,6 +2598,13 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
)
|
||||
kv_manager._staging_handler = self.staging_handler
|
||||
|
||||
def _release_request(self, decode_req: DecodeRequest) -> None:
|
||||
if self.enable_host_receive:
|
||||
discard_kv_cache_backup(decode_req.req, self.tree_cache, "host_pool")
|
||||
if decode_req.host_staged:
|
||||
return
|
||||
release_kv_cache(decode_req.req, self.tree_cache, is_insert=False)
|
||||
|
||||
def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]:
|
||||
if not self.queue:
|
||||
return []
|
||||
@@ -2504,21 +2665,24 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
)
|
||||
if self.scheduler.enable_hisparse:
|
||||
self.scheduler.hisparse_coordinator.request_finished(decode_req.req)
|
||||
if (
|
||||
requires_host_drain = (
|
||||
self.enable_host_receive and decode_req.host_staged
|
||||
)
|
||||
if requires_host_drain:
|
||||
decode_req.kv_receiver.abort()
|
||||
if requires_host_drain or (
|
||||
self.enable_deferred_kv_release
|
||||
and decode_req.kv_receiver.kv_mgr.enable_deferred_decode_kv_release
|
||||
and decode_req.kv_receiver.abort_notified
|
||||
):
|
||||
# Decode-initiated abort: a prefill write may still target
|
||||
# these pages, so hold them until the drain ack or timeout.
|
||||
# (A prefill-initiated failure has already stopped writing ->
|
||||
# immediate release below.)
|
||||
# Host pages always await a drain ack. Device pages retain
|
||||
# the existing opt-in deferred-release behavior.
|
||||
self._defer_release(decode_req)
|
||||
deferred_indices.add(i)
|
||||
indices_to_remove.add(i)
|
||||
else:
|
||||
# release pre-allocated kv cache, but don't insert into the tree since it's failed
|
||||
release_kv_cache(decode_req.req, self.tree_cache, is_insert=False)
|
||||
self._release_request(decode_req)
|
||||
decode_req.kv_receiver.clear()
|
||||
decode_req.kv_receiver = None
|
||||
indices_to_remove.add(i)
|
||||
@@ -2526,6 +2690,20 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
self.scheduler.metrics_collector.increment_transfer_failed_reqs()
|
||||
continue
|
||||
elif poll == KVPoll.Success:
|
||||
if self.enable_host_receive and decode_req.host_staged:
|
||||
if isinstance(decode_req.req.finished_reason, FINISH_ABORT):
|
||||
self.scheduler.output_streamer.stream_output(
|
||||
[decode_req.req], decode_req.req.return_logprob
|
||||
)
|
||||
self._release_request(decode_req)
|
||||
decode_req.kv_receiver.clear()
|
||||
decode_req.kv_receiver = None
|
||||
indices_to_remove.add(i)
|
||||
continue
|
||||
if not self.scheduler.disagg_decode_prealloc_queue.allocate_host_staged(
|
||||
decode_req
|
||||
):
|
||||
continue
|
||||
if (
|
||||
self.scheduler.enable_decode_hicache
|
||||
and hicache_restore_status == HiCacheRestoreResult.PENDING
|
||||
@@ -2544,7 +2722,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
decode_req.req
|
||||
)
|
||||
self._clean_hicache_prefetch_resources(decode_req)
|
||||
release_kv_cache(decode_req.req, self.tree_cache, is_insert=False)
|
||||
self._release_request(decode_req)
|
||||
if self.scheduler.metrics_reporter.enable_metrics:
|
||||
self.scheduler.metrics_collector.increment_transfer_failed_reqs()
|
||||
else:
|
||||
@@ -2595,7 +2773,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
if self.enable_staging and self.staging_handler.is_staging_room(room):
|
||||
self.staging_handler.unregister_decode_req(room)
|
||||
# release pre-allocated kv cache, but don't insert into the tree since it's failed
|
||||
release_kv_cache(decode_req.req, self.tree_cache, is_insert=False)
|
||||
self._release_request(decode_req)
|
||||
self.metadata_buffers.bootstrap_room[idx] = 0
|
||||
self.req_to_metadata_buffer_idx_allocator.free(idx)
|
||||
decode_req.kv_receiver.kv_mgr.clear_deferred_abort_state(room)
|
||||
@@ -2606,15 +2784,37 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin):
|
||||
return bool(self._deferred_releases)
|
||||
|
||||
def resolve_deferred_releases(self) -> None:
|
||||
"""Release held requests once every prefill rank acks the drain, or the
|
||||
hold times out."""
|
||||
"""Release drained transfers; device destinations may also time out."""
|
||||
if not self._deferred_releases:
|
||||
return
|
||||
host_ready = torch.tensor(
|
||||
[
|
||||
req.kv_receiver.kv_mgr.is_abort_release_safe(
|
||||
req.req.bootstrap_room, acks
|
||||
)
|
||||
for req, _, _, acks in self._deferred_releases
|
||||
if self.enable_host_receive and req.host_staged
|
||||
],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
if host_ready.numel() and torch.distributed.get_world_size(self.gloo_group) > 1:
|
||||
# Admission must see identical host capacity and page order on every rank.
|
||||
torch.distributed.all_reduce(
|
||||
host_ready, op=torch.distributed.ReduceOp.MIN, group=self.gloo_group
|
||||
)
|
||||
host_ready = iter(host_ready.tolist())
|
||||
now = time.monotonic()
|
||||
still_held = []
|
||||
to_release = []
|
||||
for decode_req, deadline, idx, required_acks in self._deferred_releases:
|
||||
room = decode_req.req.bootstrap_room
|
||||
if self.enable_host_receive and decode_req.host_staged:
|
||||
if next(host_ready):
|
||||
to_release.append((decode_req, idx, room, True))
|
||||
else:
|
||||
# A timeout cannot prove that a remote write has stopped.
|
||||
still_held.append((decode_req, deadline, idx, required_acks))
|
||||
continue
|
||||
kv_mgr = decode_req.kv_receiver.kv_mgr
|
||||
drained = kv_mgr.is_abort_release_safe(room, required_acks)
|
||||
if not drained and now < deadline:
|
||||
@@ -2797,25 +2997,39 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
def get_new_prebuilt_batch(
|
||||
self, running_batch: ScheduleBatch
|
||||
) -> Optional[ScheduleBatch]:
|
||||
batch = self._get_new_prebuilt_batch(running_batch)
|
||||
if batch is None:
|
||||
return None
|
||||
# Verify after host restore and before process_prebuilt caches any KV.
|
||||
self._verify_kv_checksums(batch)
|
||||
if batch.is_empty():
|
||||
return None
|
||||
self.ngram_embedding_manager.prepare_for_forward(
|
||||
batch, chunked_req=self.chunked_req
|
||||
)
|
||||
batch.process_prebuilt(self.future_map)
|
||||
return batch
|
||||
|
||||
def _verify_kv_checksums(self, batch: ScheduleBatch) -> None:
|
||||
computer: Optional[KvChecksumComputer] = self.kv_checksum_computer
|
||||
if computer is None:
|
||||
return self._get_new_prebuilt_batch(running_batch)
|
||||
return
|
||||
|
||||
verified: List[Req] = []
|
||||
for req in self.waiting_queue:
|
||||
verified: List[int] = []
|
||||
for i, req in enumerate(batch.reqs):
|
||||
if is_health_check_req(req):
|
||||
verified.append(req)
|
||||
verified.append(i)
|
||||
continue
|
||||
expected = req.expected_kv_checksum
|
||||
if expected == 0:
|
||||
verified.append(req)
|
||||
verified.append(i)
|
||||
continue
|
||||
seq_len = len(req.origin_input_ids)
|
||||
page_indices_gpu = page_indices_for_request(self, req, seq_len)
|
||||
state_indices = state_indices_for_request(self, req, seq_len)
|
||||
actual = computer.compute(page_indices_gpu, state_indices)
|
||||
if actual == expected:
|
||||
verified.append(req)
|
||||
verified.append(i)
|
||||
continue
|
||||
msg = (
|
||||
f"KV checksum mismatch req={req.rid} "
|
||||
@@ -2824,9 +3038,8 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
)
|
||||
logger.error(msg)
|
||||
self._handle_kv_checksum_mismatch(req, msg)
|
||||
self.waiting_queue = verified
|
||||
|
||||
return self._get_new_prebuilt_batch(running_batch)
|
||||
if len(verified) != len(batch.reqs):
|
||||
batch.filter_batch(keep_indices=verified)
|
||||
|
||||
def _handle_kv_checksum_mismatch(self, req: Req, msg: str) -> None:
|
||||
# A mismatch means the KV this worker received is not what prefill sent,
|
||||
@@ -2916,12 +3129,16 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
# A finished request can still have one redundant forward in flight.
|
||||
# Drain it before a prebuilt request seeds a potentially reused row.
|
||||
self.schedule_stream.wait_stream(self.forward_stream)
|
||||
# The prebuilt batch never reaches the forward loop's prepare call.
|
||||
self.ngram_embedding_manager.prepare_for_forward(
|
||||
new_batch, chunked_req=self.chunked_req
|
||||
)
|
||||
new_batch.process_prebuilt(self.future_map)
|
||||
|
||||
if get_disagg().disaggregation_decode_host_receive_threshold > 0:
|
||||
for req in new_batch.reqs:
|
||||
if req.kv.retraction_backup is not None:
|
||||
restore_kv_cache(
|
||||
req,
|
||||
self.tree_cache,
|
||||
self.req_to_token_pool,
|
||||
self.token_to_kv_pool_allocator,
|
||||
"host_pool",
|
||||
)
|
||||
return new_batch
|
||||
|
||||
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE)
|
||||
@@ -2950,8 +3167,9 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
self.polling_count = (self.polling_count + 1) % self.polling_interval
|
||||
|
||||
if self.polling_count % self.polling_interval == 0:
|
||||
req_conns, _ = self.disagg_decode_prealloc_queue.pop_preallocated()
|
||||
self.disagg_decode_transfer_queue.extend(req_conns)
|
||||
if get_disagg().disaggregation_decode_host_receive_threshold == 0:
|
||||
req_conns, _ = self.disagg_decode_prealloc_queue.pop_preallocated()
|
||||
self.disagg_decode_transfer_queue.extend(req_conns)
|
||||
transferred_reqs = (
|
||||
self.disagg_decode_transfer_queue.pop_transferred()
|
||||
) # the requests which kv has arrived
|
||||
@@ -2960,3 +3178,7 @@ class SchedulerDisaggregationDecodeMixin:
|
||||
# Direct-to-host: KV data already in host pool, skip staging
|
||||
self.hisparse_coordinator.admit_request_direct(req)
|
||||
self.waiting_queue.extend(transferred_reqs)
|
||||
if get_disagg().disaggregation_decode_host_receive_threshold > 0:
|
||||
# Give completed host transfers device space before new arrivals.
|
||||
req_conns, _ = self.disagg_decode_prealloc_queue.pop_preallocated()
|
||||
self.disagg_decode_transfer_queue.extend(req_conns)
|
||||
|
||||
@@ -112,10 +112,10 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
)
|
||||
from sglang.srt.mem_cache.common import (
|
||||
RetractionBackup,
|
||||
backup_kv_cache,
|
||||
evict_from_tree_cache,
|
||||
free_swa_out_of_window_slots,
|
||||
release_kv_cache,
|
||||
retraction_backup,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
@@ -2239,7 +2239,7 @@ def release_req(
|
||||
backup_saved = True
|
||||
# The config bag reflects role flips; server_args keeps the launch role.
|
||||
if get_disagg().disaggregation_mode == "decode" and offload_kv:
|
||||
backup_saved = retraction_backup(
|
||||
backup_saved = backup_kv_cache(
|
||||
req,
|
||||
tree_cache,
|
||||
req_to_token_pool,
|
||||
|
||||
@@ -290,9 +290,9 @@ from sglang.srt.managers.utils import (
|
||||
from sglang.srt.mem_cache import kv_cache_builder
|
||||
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome
|
||||
from sglang.srt.mem_cache.common import (
|
||||
discard_kv_cache_backup,
|
||||
maybe_cache_unfinished_req,
|
||||
release_kv_cache,
|
||||
retraction_discard,
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
|
||||
from sglang.srt.model_executor.runner_utils.pool import prewarm_graph_pool_borrow
|
||||
@@ -5434,6 +5434,8 @@ class Scheduler(
|
||||
)
|
||||
# For disaggregation decode mode, the request in the waiting queue has KV cache allocated.
|
||||
if self.disaggregation_mode == DisaggregationMode.DECODE:
|
||||
if get_disagg().disaggregation_decode_host_receive_threshold > 0:
|
||||
discard_kv_cache_backup(req, self.tree_cache, "host_pool")
|
||||
release_kv_cache(req, self.tree_cache)
|
||||
if self.disaggregation_mode == DisaggregationMode.PREFILL:
|
||||
self.release_aborted_prefill_waiting_req(req)
|
||||
@@ -5497,6 +5499,10 @@ class Scheduler(
|
||||
for decode_req in self.disagg_decode_transfer_queue.queue:
|
||||
if recv_req.abort_all or decode_req.req.rid.startswith(recv_req.rid):
|
||||
logger.debug(f"Abort transfer queue request. {decode_req.req.rid=}")
|
||||
if decode_req.host_staged:
|
||||
# Keep the host destination alive until prefill stops writing.
|
||||
prepare_abort(decode_req.req, "Aborted by AbortReq.")
|
||||
continue
|
||||
receiver = decode_req.kv_receiver
|
||||
receiver.abort()
|
||||
# Arm drain-ack accounting once the ABORT is sent, so acks
|
||||
@@ -5518,7 +5524,7 @@ class Scheduler(
|
||||
remaining_retracted = []
|
||||
for decode_req in self.disagg_decode_prealloc_queue.retracted_queue:
|
||||
if recv_req.abort_all or decode_req.rid.startswith(recv_req.rid):
|
||||
retraction_discard(
|
||||
discard_kv_cache_backup(
|
||||
decode_req,
|
||||
self.tree_cache,
|
||||
get_disagg().disaggregation_decode_retraction_backup,
|
||||
|
||||
@@ -843,12 +843,7 @@ class SchedulerMetricsReporter:
|
||||
self.stats.kv_transfer_speed_gb_s = self.kv_transfer_speed_gb_s
|
||||
self.stats.kv_transfer_latency_ms = self.kv_transfer_latency_ms
|
||||
elif self.scheduler.disaggregation_mode == DisaggregationMode.DECODE:
|
||||
self.stats.num_decode_prealloc_queue_reqs = QueueCount.from_reqs(
|
||||
self.scheduler.disagg_decode_prealloc_queue.queue, priority_enabled
|
||||
)
|
||||
self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs(
|
||||
self.scheduler.disagg_decode_transfer_queue.queue, priority_enabled
|
||||
)
|
||||
self._update_decode_queue_stats(priority_enabled)
|
||||
|
||||
# Utilization / LoRA / HiCache
|
||||
self._calculate_utilization()
|
||||
@@ -981,9 +976,12 @@ class SchedulerMetricsReporter:
|
||||
spec_num_draft_tokens = spec_snapshot["num_draft_tokens"]
|
||||
|
||||
if self.scheduler.disaggregation_mode == DisaggregationMode.DECODE:
|
||||
self._update_decode_queue_stats(self.scheduler.enable_priority_scheduling)
|
||||
msg += f"pre-allocated usage: {self.scheduler.disagg_decode_prealloc_queue.num_tokens_pre_allocated / self.scheduler.max_total_num_tokens:.2f}, "
|
||||
msg += f"#prealloc-req: {len(self.scheduler.disagg_decode_prealloc_queue.queue)}, "
|
||||
msg += f"#transfer-req: {len(self.scheduler.disagg_decode_transfer_queue.queue)}, "
|
||||
if get_disagg().disaggregation_decode_host_receive_threshold > 0:
|
||||
msg += f"#host-receive-req: {self.stats.num_decode_host_receive_queue_reqs.total}, "
|
||||
msg += f"#retracted-req: {len(self.scheduler.disagg_decode_prealloc_queue.retracted_queue)}, "
|
||||
|
||||
if (
|
||||
@@ -1069,13 +1067,6 @@ class SchedulerMetricsReporter:
|
||||
self.stats.num_prefill_inflight_queue_reqs = QueueCount.from_reqs(
|
||||
self.scheduler.disagg_prefill_inflight_queue, priority_enabled
|
||||
)
|
||||
elif self.scheduler.disaggregation_mode == DisaggregationMode.DECODE:
|
||||
self.stats.num_decode_prealloc_queue_reqs = QueueCount.from_reqs(
|
||||
self.scheduler.disagg_decode_prealloc_queue.queue, priority_enabled
|
||||
)
|
||||
self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs(
|
||||
self.scheduler.disagg_decode_transfer_queue.queue, priority_enabled
|
||||
)
|
||||
|
||||
# Streaming session metrics
|
||||
self.stats.num_streaming_sessions = (
|
||||
@@ -1377,6 +1368,23 @@ class SchedulerMetricsReporter:
|
||||
self.fwd_occupancy = float("nan")
|
||||
self.stats.fwd_occupancy = float("nan")
|
||||
|
||||
def _update_decode_queue_stats(self, priority_enabled: bool) -> None:
|
||||
transfer_queue = self.scheduler.disagg_decode_transfer_queue.queue
|
||||
self.stats.num_decode_prealloc_queue_reqs = QueueCount.from_reqs(
|
||||
self.scheduler.disagg_decode_prealloc_queue.queue, priority_enabled
|
||||
)
|
||||
self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs(
|
||||
transfer_queue, priority_enabled
|
||||
)
|
||||
host_reqs = (
|
||||
[req for req in transfer_queue if req.host_staged]
|
||||
if get_disagg().disaggregation_decode_host_receive_threshold > 0
|
||||
else []
|
||||
)
|
||||
self.stats.num_decode_host_receive_queue_reqs = QueueCount.from_reqs(
|
||||
host_reqs, priority_enabled
|
||||
)
|
||||
|
||||
def _maybe_log_idle_metrics(self):
|
||||
"""Reset forward timing and publish idle metrics when needed."""
|
||||
# Preserve the transition so the rate limit cannot leave a finite idle gauge.
|
||||
@@ -1427,10 +1435,5 @@ class SchedulerMetricsReporter:
|
||||
self.scheduler.disagg_prefill_inflight_queue, priority_enabled
|
||||
)
|
||||
if self.scheduler.disaggregation_mode == DisaggregationMode.DECODE:
|
||||
self.stats.num_decode_prealloc_queue_reqs = QueueCount.from_reqs(
|
||||
self.scheduler.disagg_decode_prealloc_queue.queue, priority_enabled
|
||||
)
|
||||
self.stats.num_decode_transfer_queue_reqs = QueueCount.from_reqs(
|
||||
self.scheduler.disagg_decode_transfer_queue.queue, priority_enabled
|
||||
)
|
||||
self._update_decode_queue_stats(priority_enabled)
|
||||
self.metrics_collector.log_stats(self.stats)
|
||||
|
||||
@@ -213,7 +213,7 @@ def dsv41_dspark_needs_rebootstrap(
|
||||
return isinstance(pool, DeepSeekV4TokenToKVPool) and 2 in pool.compression_ratios
|
||||
|
||||
|
||||
def retraction_backup(
|
||||
def backup_kv_cache(
|
||||
req: Req,
|
||||
tree_cache: BasePrefixCache,
|
||||
req_to_token_pool: ReqToTokenPool,
|
||||
@@ -236,11 +236,11 @@ def retraction_backup(
|
||||
return True
|
||||
|
||||
unified_cache = cast("UnifiedRadixCache", tree_cache)
|
||||
req.kv.retraction_backup = unified_cache.retraction_backup(req)
|
||||
req.kv.retraction_backup = unified_cache.backup_kv_cache(req)
|
||||
return req.kv.retraction_backup is not None
|
||||
|
||||
|
||||
def retraction_restore(
|
||||
def restore_kv_cache(
|
||||
req: Req,
|
||||
tree_cache: BasePrefixCache,
|
||||
req_to_token_pool: ReqToTokenPool,
|
||||
@@ -257,11 +257,13 @@ def retraction_restore(
|
||||
|
||||
unified_cache = cast("UnifiedRadixCache", tree_cache)
|
||||
assert req.kv.retraction_backup is not None
|
||||
unified_cache.retraction_restore(req, req.kv.retraction_backup)
|
||||
unified_cache.restore_kv_cache(req, req.kv.retraction_backup)
|
||||
req.kv.retraction_backup = None
|
||||
|
||||
|
||||
def retraction_discard(req: Req, tree_cache: BasePrefixCache, backend: str) -> None:
|
||||
def discard_kv_cache_backup(
|
||||
req: Req, tree_cache: BasePrefixCache, backend: str
|
||||
) -> None:
|
||||
if backend == "cpu_tensor":
|
||||
req.kv.retraction_backup = None
|
||||
return
|
||||
@@ -271,7 +273,7 @@ def retraction_discard(req: Req, tree_cache: BasePrefixCache, backend: str) -> N
|
||||
return
|
||||
|
||||
unified_cache = cast("UnifiedRadixCache", tree_cache)
|
||||
unified_cache.retraction_discard(req.kv.retraction_backup)
|
||||
unified_cache.discard_kv_cache_backup(req.kv.retraction_backup)
|
||||
req.kv.retraction_backup = None
|
||||
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ from sglang.srt.runtime_context import (
|
||||
get_schedule,
|
||||
)
|
||||
from sglang.srt.speculative.base_spec_worker import HiCacheDraftMode
|
||||
from sglang.srt.utils import is_hip
|
||||
from sglang.srt.utils import ceil_align, is_hip
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch.distributed import ProcessGroup
|
||||
@@ -142,11 +142,16 @@ def maybe_register_hicache_draft(
|
||||
|
||||
|
||||
# Host slots a backup-only retraction pool gets, as a fraction of the device
|
||||
# pool. Sized well under 1.0 because a retraction burst touches a fraction of
|
||||
# the device tokens; overflow aborts the request rather than pre-reserving.
|
||||
# pool, with a floor large enough for one maximum-length request.
|
||||
BACKUP_ONLY_HICACHE_RATIO = 0.2
|
||||
|
||||
|
||||
def decode_retraction_max_tokens(req_to_token_pool, kv_cache) -> int:
|
||||
return ceil_align(
|
||||
min(req_to_token_pool.max_context_len - 1, kv_cache.size), kv_cache.page_size
|
||||
)
|
||||
|
||||
|
||||
def uses_ssm_state(model_config) -> bool:
|
||||
"""Whether the model keeps recurrent/conv state alongside its attention KV."""
|
||||
spec = linear_attn_model_spec(model_config)
|
||||
@@ -212,11 +217,19 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str:
|
||||
|
||||
if memory.hicache_ratio is None:
|
||||
# Only a decode server reaches resolution with the ratio unset. A
|
||||
# backup-only pool can be small: retractions that overflow it abort their
|
||||
# request instead of crashing the scheduler. Sharing the pool with
|
||||
# HiCache keeps the standard default.
|
||||
# backup-only pool fits at least one runnable request. Sharing the pool
|
||||
# with prefix caching keeps the standard HiCache default.
|
||||
if backend == "host_pool" and not memory.enable_hierarchical_cache:
|
||||
fields["hicache_ratio"] = BACKUP_ONLY_HICACHE_RATIO
|
||||
ratio = BACKUP_ONLY_HICACHE_RATIO
|
||||
if memory.hicache_size == 0:
|
||||
req_pool, allocator = tp_worker.get_memory_pool()
|
||||
kv_pool = allocator.get_kvcache()
|
||||
min_tokens = decode_retraction_max_tokens(req_pool, kv_pool)
|
||||
if disagg.disaggregation_decode_host_receive_threshold > 0:
|
||||
# One request can receive while another is retracted.
|
||||
min_tokens *= 2
|
||||
ratio = max(ratio, min_tokens / kv_pool.size)
|
||||
fields["hicache_ratio"] = ratio
|
||||
else:
|
||||
fields["hicache_ratio"] = 2.0
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from typing import Any
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
|
||||
from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -75,6 +76,63 @@ class HostPoolGroup:
|
||||
def get_pool(self, name: PoolName):
|
||||
return self.get_entry(name).host_pool
|
||||
|
||||
def get_contiguous_buf_infos(self):
|
||||
"""Return (device_buffers, host_buffers), each (ptrs, sizes, item_sizes)."""
|
||||
host_by_device_ptr = {}
|
||||
device_infos = ([], [], [])
|
||||
for entry in self.entries:
|
||||
host = entry.host_pool
|
||||
pools = (entry.device_pool, *entry.packed_draft_device_pools)
|
||||
for pool in pools:
|
||||
dense_mha = (
|
||||
type(pool) is MHATokenToKVPool
|
||||
and pool.kv_cache_layout == "nhd"
|
||||
and pool.v_head_dim == pool.head_dim
|
||||
)
|
||||
if (
|
||||
not dense_mha
|
||||
or pool.layer_shard_enabled
|
||||
or pool.page_size != self.page_size
|
||||
):
|
||||
raise ValueError(
|
||||
"Host receive requires dense NHD MHA target and "
|
||||
"draft KV with matching page sizes"
|
||||
)
|
||||
for combined, values in zip(
|
||||
device_infos, pool.get_contiguous_buf_infos(), strict=True
|
||||
):
|
||||
combined.extend(values)
|
||||
|
||||
# Packed MHA stores target/draft K followed by target/draft V,
|
||||
# while the wire lists target K/V followed by draft K/V. Associate
|
||||
# each host view with its device buffer before applying wire order.
|
||||
device_buffers = [b for p in pools for b in p.k_buffer] + [
|
||||
b for p in pools for b in p.v_buffer
|
||||
]
|
||||
for device_buffer, host_buffer in zip(
|
||||
device_buffers, host.host_kv_data_refs, strict=True
|
||||
):
|
||||
if (
|
||||
not host_buffer.is_contiguous()
|
||||
or host_buffer.shape[1:] != device_buffer.shape[1:]
|
||||
):
|
||||
raise ValueError(
|
||||
"KV transfer requires matching contiguous per-layer "
|
||||
"device and host buffers"
|
||||
)
|
||||
host_by_device_ptr[device_buffer.data_ptr()] = (
|
||||
host_buffer.data_ptr(),
|
||||
host_buffer.nbytes,
|
||||
host.token_stride_size * self.page_size,
|
||||
)
|
||||
infos = [host_by_device_ptr[ptr] for ptr in device_infos[0]]
|
||||
host_infos = (
|
||||
[info[0] for info in infos],
|
||||
[info[1] for info in infos],
|
||||
[info[2] for info in infos],
|
||||
)
|
||||
return device_infos, host_infos
|
||||
|
||||
def alloc(
|
||||
self,
|
||||
need_size: int,
|
||||
|
||||
@@ -1469,7 +1469,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
return 0
|
||||
return self.evict_host(num_tokens)
|
||||
|
||||
def retraction_backup(self, req: Req) -> Optional[RetractionBackup]:
|
||||
def backup_kv_cache(self, req: Req) -> Optional[RetractionBackup]:
|
||||
"""Back up device KV to the host pool; None when it cannot fit after reclaim."""
|
||||
assert req.seqlen > 1
|
||||
|
||||
@@ -1512,11 +1512,11 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
)
|
||||
completion.finish_event.synchronize()
|
||||
except Exception:
|
||||
self.retraction_discard(backup)
|
||||
self.discard_kv_cache_backup(backup)
|
||||
raise
|
||||
return backup
|
||||
|
||||
def retraction_restore(self, req: Req, backup: RetractionBackup) -> None:
|
||||
def restore_kv_cache(self, req: Req, backup: RetractionBackup) -> None:
|
||||
device_indices, current_transfers = self._retraction_device_transfers(req)
|
||||
assert len(backup.host_indices) == len(device_indices), (
|
||||
f"Host backup has {len(backup.host_indices)} slots, but restore has "
|
||||
@@ -1561,9 +1561,9 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
transfer_layer_id_max=self.cache_controller.transfer_layer_id_max,
|
||||
)
|
||||
completion.finish_event.synchronize()
|
||||
self.retraction_discard(backup)
|
||||
self.discard_kv_cache_backup(backup)
|
||||
|
||||
def retraction_discard(self, backup: RetractionBackup) -> None:
|
||||
def discard_kv_cache_backup(self, backup: RetractionBackup) -> None:
|
||||
self.host_pool_group.free(backup.host_indices)
|
||||
self.host_pool_group.release_transfers(backup.pool_transfers)
|
||||
|
||||
|
||||
@@ -135,6 +135,7 @@ class SchedulerStats:
|
||||
num_prefill_inflight_queue_reqs: QueueCount = field(default_factory=QueueCount)
|
||||
num_decode_prealloc_queue_reqs: QueueCount = field(default_factory=QueueCount)
|
||||
num_decode_transfer_queue_reqs: QueueCount = field(default_factory=QueueCount)
|
||||
num_decode_host_receive_queue_reqs: QueueCount = field(default_factory=QueueCount)
|
||||
kv_transfer_speed_gb_s: float = 0.0
|
||||
kv_transfer_latency_ms: float = 0.0
|
||||
pending_prealloc_token_usage: float = 0.0
|
||||
@@ -519,6 +520,17 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
labelnames=labels.keys(),
|
||||
multiprocess_mode="mostrecent",
|
||||
)
|
||||
self.num_decode_host_receive_queue_reqs = Gauge(
|
||||
name="sglang:num_decode_host_receive_queue_reqs",
|
||||
documentation="Requests in the decode transfer queue receiving into host memory or waiting for device admission.",
|
||||
labelnames=labels.keys(),
|
||||
multiprocess_mode="mostrecent",
|
||||
)
|
||||
self.num_decode_host_receive_reqs = Counter(
|
||||
name="sglang:num_decode_host_receive_reqs_total",
|
||||
documentation="Total requests admitted to receive prefill KV in host memory.",
|
||||
labelnames=labels.keys(),
|
||||
)
|
||||
self.kv_transfer_speed_gb_s = Histogram(
|
||||
name="sglang:kv_transfer_speed_gb_s",
|
||||
documentation="Histogram of KV cache transfer speed in GB/s.",
|
||||
@@ -1186,6 +1198,9 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
def increment_transfer_failed_reqs(self) -> None:
|
||||
self.num_transfer_failed_reqs.labels(**self.labels).inc(1)
|
||||
|
||||
def increment_decode_host_receive_reqs(self) -> None:
|
||||
self.num_decode_host_receive_reqs.labels(**self.labels).inc(1)
|
||||
|
||||
def increment_prefill_retries(self, count: int) -> None:
|
||||
if count > 0:
|
||||
self.num_prefill_retries_total.labels(**self.labels).inc(count)
|
||||
@@ -1414,6 +1429,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
self._log_gauge_queue_count(
|
||||
self.num_decode_transfer_queue_reqs, stats.num_decode_transfer_queue_reqs
|
||||
)
|
||||
self._log_gauge_queue_count(
|
||||
self.num_decode_host_receive_queue_reqs,
|
||||
stats.num_decode_host_receive_queue_reqs,
|
||||
)
|
||||
self._log_gauge(
|
||||
self.pending_prealloc_token_usage, stats.pending_prealloc_token_usage
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user