[HiSparse] Add speculative decoding support with multi-step KV cache swapping (#37771)
This commit is contained in:
@@ -79,10 +79,7 @@ def validate_hisparse_kv_cache_dtype(server_args: ServerArgs) -> None:
|
||||
def validate_hisparse_feature_combos(server_args: ServerArgs) -> None:
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
# Both crash on the first request: the draft pool has no HiSparse mapping,
|
||||
# and a mixed batch cannot merge the HiSparse decode batch.
|
||||
if cfg.speculative_algorithm is not None:
|
||||
raise ValueError("--enable-hisparse does not support speculative decoding yet.")
|
||||
# A mixed batch cannot merge the HiSparse decode batch.
|
||||
if cfg.enable_mixed_chunk:
|
||||
raise ValueError("--enable-hisparse does not support --enable-mixed-chunk yet.")
|
||||
|
||||
@@ -114,6 +111,21 @@ def validate_hisparse(server_args: ServerArgs) -> None:
|
||||
)
|
||||
validate_hisparse_feature_combos(server_args)
|
||||
|
||||
enable_spec = (
|
||||
cfg.speculative_algorithm is not None
|
||||
and cfg.speculative_algorithm.upper() != "NONE"
|
||||
)
|
||||
if enable_spec and (cfg.speculative_eagle_topk or 1) != 1:
|
||||
raise ValueError(
|
||||
"The minimal HiSparse spec integration supports linear speculation "
|
||||
"only; use --speculative-eagle-topk 1."
|
||||
)
|
||||
if enable_spec and cfg.speculative_adaptive:
|
||||
raise ValueError(
|
||||
"HiSparse speculative decoding uses statically sized swap state; "
|
||||
"--speculative-adaptive is not supported."
|
||||
)
|
||||
|
||||
# DSv4 hisparse handles its own dtype/backend pairing elsewhere; the dtype-
|
||||
# aware checks below only apply to the DSA hisparse path.
|
||||
if is_hip and is_v4_hisparse:
|
||||
|
||||
@@ -1688,8 +1688,16 @@ class CommonKVManager(BaseKVManager):
|
||||
|
||||
# Regular MLA PP slicing
|
||||
# Decode pp size should be equal to prefill pp size or 1
|
||||
start_layer, end_layer = self._mla_kv_entry_span_with_pp(len(src_kv_ptrs))
|
||||
num_draft = (
|
||||
getattr(self.kv_args, "num_draft_entries", 0) if state_type is None else 0
|
||||
)
|
||||
start_layer, end_layer = self._mla_kv_entry_span_with_pp(
|
||||
len(src_kv_ptrs) - num_draft
|
||||
)
|
||||
sliced_dst_kv_ptrs = dst_kv_ptrs[start_layer:end_layer]
|
||||
if num_draft:
|
||||
# Draft entries follow the target's global layer range on both peers.
|
||||
sliced_dst_kv_ptrs += dst_kv_ptrs[-num_draft:]
|
||||
return src_kv_ptrs, sliced_dst_kv_ptrs, len(src_kv_ptrs)
|
||||
|
||||
def _mla_kv_entry_span_with_pp(self, n_src: int) -> Tuple[int, int]:
|
||||
|
||||
@@ -1382,17 +1382,21 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
]
|
||||
|
||||
# HiSparse physical constraint: max requests by device buffer capacity.
|
||||
# Each admitted req needs padded_buffer_size from hisparse device pool.
|
||||
# Each admitted req needs a padded buffer plus its speculative scratch.
|
||||
# waiting_queue reqs already have device buffers (allocated in admit_request_direct),
|
||||
# only transfer_queue reqs are pending device buffer allocation.
|
||||
hisparse_req_budget = float("inf")
|
||||
if self.scheduler.enable_hisparse:
|
||||
coordinator = self.scheduler.hisparse_coordinator
|
||||
per_req_device_tokens = (
|
||||
coordinator.padded_buffer_size + coordinator.spec_swap.scratch_capacity
|
||||
)
|
||||
hisparse_avail = (
|
||||
self.token_to_kv_pool_allocator.hisparse_attn_allocator.available_size()
|
||||
)
|
||||
hisparse_req_budget = max(
|
||||
0,
|
||||
hisparse_avail // self.scheduler.hisparse_coordinator.padded_buffer_size
|
||||
hisparse_avail // per_req_device_tokens
|
||||
- len(self.transfer_queue.queue),
|
||||
)
|
||||
|
||||
@@ -1725,13 +1729,16 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
device_page_indices = None
|
||||
if (
|
||||
self.scheduler.enable_hisparse
|
||||
and isinstance(self.token_to_kv_pool, DeepSeekV4TokenToKVPool)
|
||||
and (
|
||||
isinstance(self.token_to_kv_pool, DeepSeekV4TokenToKVPool)
|
||||
or self.draft_token_to_kv_pool is not None
|
||||
)
|
||||
and not _is_fake_transfer(decode_req.req)
|
||||
):
|
||||
# alloc_logical_only() already allocated the shared logical pages
|
||||
# used by C4 indexer and C128 KV. These device buffers do not use
|
||||
# the C4 sparse physical-slot mapping; carry their logical page IDs
|
||||
# alongside the independently allocated C4 host page IDs.
|
||||
# used by the indexer and draft KV. These device buffers do not
|
||||
# use target sparse physical slots; carry logical page IDs beside
|
||||
# the independently allocated target host page IDs.
|
||||
full_kv_indices = self.req_to_token_pool.req_to_token[
|
||||
decode_req.req.kv.req_pool_idx,
|
||||
prefix_len:origin_input_len,
|
||||
@@ -1742,8 +1749,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
).astype(np.int32)
|
||||
if self.transfer_backend != TransferBackend.MOONCAKE:
|
||||
raise NotImplementedError(
|
||||
"DSV4 HiSparse direct PD transfer currently requires "
|
||||
"the Mooncake backend"
|
||||
"HiSparse device-only PD transfer currently requires the "
|
||||
"Mooncake backend"
|
||||
)
|
||||
metadata_kwargs = {"decode_prefix_len": total_prefix_len}
|
||||
if device_page_indices is not None:
|
||||
|
||||
@@ -1320,14 +1320,24 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
dst_device_kv_ptrs = None
|
||||
if dst_device_kv_indices is not None:
|
||||
compression_ratios = self.kv_args.mla_compression_ratios
|
||||
assert compression_ratios is not None
|
||||
if len(dst_kv_ptrs) == len(self.kv_args.kv_data_ptrs):
|
||||
start = self.kv_args.prefill_start_layer
|
||||
end = self.kv_args.prefill_end_layer
|
||||
assert end is not None
|
||||
compression_ratios = compression_ratios[start:end]
|
||||
c4_layer_num = sum(ratio == 4 for ratio in compression_ratios)
|
||||
dst_device_kv_ptrs = set(dst_kv_ptrs[c4_layer_num:])
|
||||
if compression_ratios is not None:
|
||||
if len(dst_kv_ptrs) == len(self.kv_args.kv_data_ptrs):
|
||||
start = self.kv_args.prefill_start_layer
|
||||
end = self.kv_args.prefill_end_layer
|
||||
assert end is not None
|
||||
compression_ratios = compression_ratios[start:end]
|
||||
host_backed_kv_count = sum(ratio == 4 for ratio in compression_ratios)
|
||||
else:
|
||||
# The remote target prefix is host-backed, even across PP stages;
|
||||
# only the draft suffix uses logical device page IDs.
|
||||
num_draft_entries = self.kv_args.num_draft_entries
|
||||
if not 0 < num_draft_entries < len(dst_kv_ptrs):
|
||||
raise ValueError(
|
||||
"HiSparse spec PD transfer requires an appended device-only "
|
||||
"draft KV buffer"
|
||||
)
|
||||
host_backed_kv_count = len(dst_kv_ptrs) - num_draft_entries
|
||||
dst_device_kv_ptrs = set(dst_kv_ptrs[host_backed_kv_count:])
|
||||
return self._send_kvcache_generic(
|
||||
mooncake_session_id=mooncake_session_id,
|
||||
src_data_ptrs=self.kv_args.kv_data_ptrs,
|
||||
|
||||
@@ -2262,10 +2262,23 @@ class DeepseekSparseAttnBackend(
|
||||
|
||||
# todo hisparse: to cover more backends
|
||||
if self.hisparse_coordinator is not None:
|
||||
# flash_mla_sparse_fwd / tilelang require int32 page indices.
|
||||
page_table_1 = self.token_to_kv_pool.translate_loc_to_hisparse_device(
|
||||
page_table_1
|
||||
).to(torch.int32)
|
||||
if forward_batch.forward_mode.is_target_verify():
|
||||
num_reqs = forward_batch.req_pool_indices.shape[0]
|
||||
num_steps = self.speculative_num_draft_tokens
|
||||
assert topk_indices is not None
|
||||
grouped_topk_indices = topk_indices.view(num_reqs, num_steps, -1)
|
||||
assert metadata.dsa_seqlens_expanded is not None
|
||||
page_table_1 = self.hisparse_coordinator.swap_in_selected_pages(
|
||||
forward_batch.req_pool_indices,
|
||||
metadata.dsa_seqlens_expanded,
|
||||
grouped_topk_indices,
|
||||
layer.layer_id,
|
||||
).view(num_reqs * num_steps, -1)
|
||||
else:
|
||||
# flash_mla_sparse_fwd / tilelang require int32 page indices.
|
||||
page_table_1 = self.token_to_kv_pool.translate_loc_to_hisparse_device(
|
||||
page_table_1
|
||||
).to(torch.int32)
|
||||
|
||||
if dsa_impl == "tilelang":
|
||||
if q_rope is not None:
|
||||
@@ -4019,7 +4032,10 @@ class DeepseekSparseAttnBackend(
|
||||
) -> DSAIndexerMetadata:
|
||||
force_unfused = not self.use_fused_topk or (
|
||||
self.hisparse_coordinator is not None
|
||||
and forward_batch.forward_mode.is_decode_or_idle()
|
||||
and (
|
||||
forward_batch.forward_mode.is_decode_or_idle()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
)
|
||||
)
|
||||
return DSAIndexerMetadata(
|
||||
attn_metadata=self.forward_metadata,
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
# to be combined with the sparse coordinator class and sparse algorithm family
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Dict, List, NamedTuple, Optional, Tuple, Union
|
||||
import weakref
|
||||
from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
|
||||
@@ -27,10 +28,12 @@ if is_xpu():
|
||||
|
||||
else:
|
||||
from sglang.kernels.ops.kvcache.hisparse import (
|
||||
HiSparseSpecState,
|
||||
copy_cache_planned_mla,
|
||||
load_blocks_to_device_buffer_mha,
|
||||
load_cache_to_device_buffer_dsv4_mla,
|
||||
load_cache_to_device_buffer_mla,
|
||||
load_cache_to_device_buffer_spec_mla,
|
||||
)
|
||||
|
||||
from sglang.srt.configs.model_config import dsa_layer_skips_topk, is_deepseek_dsa
|
||||
@@ -49,6 +52,10 @@ from sglang.srt.mem_cache.pool_host.mha import HiSparseMHATokenToKVPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.kernels.ops.kvcache.hisparse import HiSparseSpecState
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
|
||||
device_module = get_device_module()
|
||||
|
||||
_is_hip = is_hip()
|
||||
@@ -73,13 +80,12 @@ class HiSparseTokenStats(NamedTuple):
|
||||
def resolve_shared_index_layers(
|
||||
*,
|
||||
hf_text_config,
|
||||
is_speculative: bool,
|
||||
) -> Optional[List[bool]]:
|
||||
"""Per-layer "reuses the previous layer's top-k index" pattern, or None.
|
||||
|
||||
Mirrors DeepseekV2AttentionMLA's skip_topk derivation (index_topk_pattern /
|
||||
index_topk_freq / cli_factor); None when the model has no sharing or the
|
||||
prefetch cannot run (PP, speculative decoding, kill-switch).
|
||||
prefetch cannot run (pipeline parallelism or the kill-switch).
|
||||
"""
|
||||
if not is_deepseek_dsa(hf_text_config):
|
||||
return None
|
||||
@@ -91,11 +97,12 @@ def resolve_shared_index_layers(
|
||||
pattern = [dsa_layer_skips_topk(hf_text_config, i) for i in range(num_layers)]
|
||||
if not any(pattern):
|
||||
return None
|
||||
if get_parallel().pp_size != 1 or is_speculative:
|
||||
# Speculative decoding is supported by the HiSparse multi-step swap path;
|
||||
# only pipeline parallelism still prevents the shared-index prefetch plan.
|
||||
if get_parallel().pp_size != 1:
|
||||
logger.warning(
|
||||
"HiSparse shared-index prefetch is unsupported under pipeline "
|
||||
"parallelism / speculative decoding; falling back to synchronous "
|
||||
"swap-in."
|
||||
"parallelism; falling back to synchronous swap-in."
|
||||
)
|
||||
return None
|
||||
if envs.SGLANG_DISABLE_HISPARSE_PREFETCH.get():
|
||||
@@ -132,6 +139,397 @@ def _build_prefetch_groups(
|
||||
return groups, slot
|
||||
|
||||
|
||||
class HiSparseSpecSwapManager:
|
||||
"""Own HiSparse speculative-swap state and request lifecycle."""
|
||||
|
||||
HASH_MULTIPLIER = 2654435761
|
||||
MIN_SCRATCH_CAPACITY = 1024
|
||||
NUM_METADATA_VALUES_PER_OCCURRENCE = 5
|
||||
NUM_COUNTERS_PER_REQUEST = 4
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
coordinator: HiSparseCoordinator,
|
||||
*,
|
||||
num_draft_tokens: int,
|
||||
) -> None:
|
||||
self._coordinator = weakref.proxy(coordinator)
|
||||
self.enabled = num_draft_tokens > 0
|
||||
self.num_draft_tokens = num_draft_tokens
|
||||
self.scratch_capacity = 0
|
||||
self._scratch_reqs: set[int] = set()
|
||||
|
||||
layer_num = coordinator.mem_pool_device.layer_num
|
||||
max_num_req_slots = coordinator.req_to_token_pool.req_to_token.shape[0]
|
||||
self.req_to_scratch = torch.empty(
|
||||
(layer_num, max_num_req_slots, 0),
|
||||
dtype=torch.int32,
|
||||
device=coordinator.device,
|
||||
)
|
||||
self.states: tuple[HiSparseSpecState, ...] = ()
|
||||
self.top_k_device_locs: Optional[torch.Tensor] = None
|
||||
if not self.enabled:
|
||||
return
|
||||
|
||||
if _is_xpu:
|
||||
raise ValueError("HiSparse speculative swap is unsupported on XPU.")
|
||||
if coordinator.is_dsv4_hisparse:
|
||||
raise ValueError("HiSparse spec swap currently supports DSA caches only.")
|
||||
if not 2 <= num_draft_tokens <= 4 or coordinator.top_k < 1024:
|
||||
raise ValueError(
|
||||
"HiSparse spec swap requires 2-4 draft tokens and top_k >= 1024."
|
||||
)
|
||||
workspace_capacity = num_draft_tokens * coordinator.top_k
|
||||
if workspace_capacity > 8192:
|
||||
raise ValueError("HiSparse spec swap supports at most 8192 occurrences.")
|
||||
if max_num_req_slots > coordinator.device_buffer_size:
|
||||
raise ValueError(
|
||||
"HiSparse spec request capacity must not exceed device_buffer_size."
|
||||
)
|
||||
|
||||
self.scratch_capacity = min(
|
||||
workspace_capacity,
|
||||
max(
|
||||
self.MIN_SCRATCH_CAPACITY,
|
||||
workspace_capacity - coordinator.device_buffer_size,
|
||||
),
|
||||
)
|
||||
hash_size = 1 << max(1, (2 * coordinator.device_buffer_size - 1).bit_length())
|
||||
self._cache_index = torch.full(
|
||||
(layer_num, max_num_req_slots, 2, hash_size),
|
||||
-1,
|
||||
dtype=torch.int64,
|
||||
device=coordinator.device,
|
||||
)
|
||||
self._cache_policy = torch.zeros(
|
||||
(layer_num, max_num_req_slots + 1, coordinator.device_buffer_size),
|
||||
dtype=torch.int32,
|
||||
device=coordinator.device,
|
||||
)
|
||||
metadata_width = max(
|
||||
self.NUM_COUNTERS_PER_REQUEST * max_num_req_slots,
|
||||
self.NUM_METADATA_VALUES_PER_OCCURRENCE * workspace_capacity,
|
||||
)
|
||||
self._scratch_state = torch.full(
|
||||
(max_num_req_slots + 1, metadata_width),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=coordinator.device,
|
||||
)
|
||||
self._scratch_state[0].zero_()
|
||||
self.req_to_scratch = torch.zeros(
|
||||
(layer_num, max_num_req_slots, self.scratch_capacity),
|
||||
dtype=torch.int32,
|
||||
device=coordinator.device,
|
||||
)
|
||||
self.states = tuple(
|
||||
HiSparseSpecState(
|
||||
cache_index=self._cache_index[layer_id],
|
||||
cache_policy=self._cache_policy[layer_id],
|
||||
scratch_locs=self.req_to_scratch[layer_id],
|
||||
scratch_state=self._scratch_state,
|
||||
)
|
||||
for layer_id in range(layer_num)
|
||||
)
|
||||
|
||||
tokens = torch.arange(
|
||||
coordinator.device_buffer_size,
|
||||
dtype=torch.int64,
|
||||
device=coordinator.device,
|
||||
)
|
||||
self._hash_slots = ((tokens * self.HASH_MULTIPLIER) & (hash_size - 1)).to(
|
||||
torch.long
|
||||
)
|
||||
self._hash_entries = (tokens << 32) | tokens
|
||||
self.top_k_device_locs = torch.full(
|
||||
(max_num_req_slots, num_draft_tokens, coordinator.top_k),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=coordinator.device,
|
||||
)
|
||||
|
||||
@property
|
||||
def miss_plan_capacity(self) -> int:
|
||||
return self._coordinator.top_k * max(1, self.num_draft_tokens)
|
||||
|
||||
def invalidate_cache(self, req_pool_idx: int) -> None:
|
||||
if self.enabled:
|
||||
self._cache_index[:, req_pool_idx].fill_(-1)
|
||||
|
||||
def reset(self, req_pool_idx: int) -> None:
|
||||
if not self.enabled:
|
||||
return
|
||||
self._cache_index[:, req_pool_idx].fill_(-1)
|
||||
self._cache_index[:, req_pool_idx, 0, self._hash_slots] = (
|
||||
self._hash_entries.view(1, -1)
|
||||
)
|
||||
self._cache_policy[:, 0, req_pool_idx].zero_()
|
||||
self._cache_policy[:, req_pool_idx + 1].zero_()
|
||||
max_num_req_slots = self.req_to_scratch.shape[1]
|
||||
self._scratch_state[
|
||||
0,
|
||||
req_pool_idx : self.NUM_COUNTERS_PER_REQUEST
|
||||
* max_num_req_slots : max_num_req_slots,
|
||||
] = 0
|
||||
self._scratch_state[req_pool_idx + 1].fill_(-1)
|
||||
|
||||
def allocate_scratch(self, req_pool_idx: int) -> None:
|
||||
"""Reserve one request's graph-stable swap workspace at admission."""
|
||||
if not self.enabled or req_pool_idx in self._scratch_reqs:
|
||||
return
|
||||
allocator = self._coordinator.token_to_kv_pool_allocator
|
||||
scratch_locs = allocator.hisparse_attn_allocator.alloc(self.scratch_capacity)
|
||||
if scratch_locs is None:
|
||||
raise RuntimeError(
|
||||
"HiSparse spec failed to allocate "
|
||||
f"{self.scratch_capacity} scratch slots."
|
||||
)
|
||||
# Physical slots are shared by all KV layers, but every full-index
|
||||
# layer rotates those IDs between persistent cache and scratch
|
||||
# independently. Keep per-layer location views so one layer cannot
|
||||
# overwrite the next layer's ownership state.
|
||||
self.req_to_scratch[:, req_pool_idx].copy_(scratch_locs.to(torch.int32))
|
||||
self._scratch_reqs.add(req_pool_idx)
|
||||
|
||||
def free_unrotated_scratch(self, req_pool_idx: int) -> None:
|
||||
"""Free scratch before any swap kernel has changed its ownership."""
|
||||
if not self.enabled or req_pool_idx not in self._scratch_reqs:
|
||||
return
|
||||
self._coordinator.token_to_kv_pool_allocator.free_hisparse_indices(
|
||||
self.req_to_scratch[0, req_pool_idx].clone()
|
||||
)
|
||||
self.clear_scratch(req_pool_idx)
|
||||
|
||||
def clear_scratch(self, req_pool_idx: int) -> None:
|
||||
if not self.enabled or req_pool_idx not in self._scratch_reqs:
|
||||
return
|
||||
self.req_to_scratch[:, req_pool_idx].zero_()
|
||||
self._scratch_reqs.remove(req_pool_idx)
|
||||
|
||||
def extend_owned_locs(
|
||||
self, req_pool_idx: int, owned_locs: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
if not self.enabled or req_pool_idx not in self._scratch_reqs:
|
||||
return owned_locs
|
||||
return torch.cat([owned_locs, self.req_to_scratch[:, req_pool_idx].reshape(-1)])
|
||||
|
||||
def prepare_verify(self, batch: ScheduleBatch) -> None:
|
||||
"""Bind target-verify KV writes to the side buffer's extra page."""
|
||||
coordinator = self._coordinator
|
||||
req_pool_indices = batch.req_pool_indices
|
||||
verify_cache_locs = batch.out_cache_loc
|
||||
start_positions = batch.seq_lens
|
||||
|
||||
if not self.enabled:
|
||||
raise RuntimeError("HiSparse spec is not initialized.")
|
||||
if self.num_draft_tokens > coordinator.page_size - 1:
|
||||
raise ValueError(
|
||||
f"HiSparse spec needs {self.num_draft_tokens} verify slots, but the "
|
||||
f"extra page has only {coordinator.page_size - 1} usable slots."
|
||||
)
|
||||
expected_slots = req_pool_indices.numel() * self.num_draft_tokens
|
||||
if verify_cache_locs.numel() != expected_slots:
|
||||
raise ValueError(
|
||||
f"HiSparse verify slot mismatch: expected {expected_slots}, "
|
||||
f"got {verify_cache_locs.numel()}."
|
||||
)
|
||||
|
||||
# CPU-side logical reservations cover verify even when overlap lags seq_lens.
|
||||
for req in batch.reqs:
|
||||
req_pool_idx = req.kv.req_pool_idx
|
||||
allocated_len = int(
|
||||
coordinator.req_to_host_pool_allocated_len[req_pool_idx]
|
||||
)
|
||||
if req.kv.kv_allocated_len > allocated_len:
|
||||
coordinator.mem_pool_host.alloc_paged_token_slots(
|
||||
coordinator.req_to_host_pool,
|
||||
coordinator.req_to_host_pool_allocated_len,
|
||||
req_pool_idx,
|
||||
allocated_len,
|
||||
req.kv.kv_allocated_len - allocated_len,
|
||||
)
|
||||
|
||||
extra_start = coordinator.device_buffer_size + 1
|
||||
row_indices = torch.repeat_interleave(req_pool_indices, self.num_draft_tokens)
|
||||
offsets = (
|
||||
torch.arange(
|
||||
expected_slots, dtype=torch.int64, device=req_pool_indices.device
|
||||
)
|
||||
% self.num_draft_tokens
|
||||
)
|
||||
token_positions = (
|
||||
torch.repeat_interleave(
|
||||
start_positions.to(torch.int64), self.num_draft_tokens
|
||||
)
|
||||
+ offsets
|
||||
)
|
||||
columns = extra_start + offsets
|
||||
device_locs = coordinator.req_to_device_buffer[row_indices, columns]
|
||||
coordinator.req_device_buffer_tokens[:, row_indices, columns] = (
|
||||
token_positions.to(torch.int32).unsqueeze(0)
|
||||
)
|
||||
coordinator.req_device_buffer_token_locs[:, row_indices, columns] = (
|
||||
device_locs.to(torch.int32).unsqueeze(0)
|
||||
)
|
||||
coordinator.token_to_kv_pool_allocator.full_to_hisparse_device_index_mapping[
|
||||
verify_cache_locs
|
||||
] = device_locs
|
||||
|
||||
def _backup_device_locs_to_host(
|
||||
self, host_locs: torch.Tensor, device_locs: torch.Tensor
|
||||
) -> None:
|
||||
if host_locs.numel() == 0:
|
||||
return
|
||||
coordinator = self._coordinator
|
||||
coordinator.wait_for_pending_backup()
|
||||
schedule_stream = device_module.current_stream()
|
||||
device_locs = device_locs.contiguous()
|
||||
with device_module.stream(coordinator.decode_backup_stream):
|
||||
coordinator.decode_backup_stream.wait_stream(schedule_stream)
|
||||
if coordinator.decode_producer_stream is not None:
|
||||
coordinator.decode_backup_stream.wait_stream(
|
||||
coordinator.decode_producer_stream
|
||||
)
|
||||
coordinator.mem_pool_host.backup_from_device_all_layer(
|
||||
coordinator.mem_pool_device,
|
||||
host_locs,
|
||||
device_locs,
|
||||
io_backend="kernel",
|
||||
)
|
||||
if host_locs.is_cuda:
|
||||
host_locs.record_stream(coordinator.decode_backup_stream)
|
||||
if device_locs.is_cuda:
|
||||
device_locs.record_stream(coordinator.decode_backup_stream)
|
||||
coordinator._backup_done_event.record()
|
||||
coordinator._has_pending_backup = True
|
||||
coordinator.wait_for_pending_backup()
|
||||
|
||||
def commit_accept_tokens(
|
||||
self, batch: ScheduleBatch, accept_indices: torch.Tensor
|
||||
) -> None:
|
||||
"""Persist accepted target-verify KV in host and the persistent buffer."""
|
||||
if batch.forward_mode.is_idle():
|
||||
return
|
||||
coordinator = self._coordinator
|
||||
req_pool_indices = batch.req_pool_indices[:, None]
|
||||
verify_cache_locs = batch.out_cache_loc
|
||||
|
||||
if verify_cache_locs.numel() == 0:
|
||||
return
|
||||
batch_size = req_pool_indices.numel()
|
||||
if batch_size == 0 or verify_cache_locs.numel() % batch_size != 0:
|
||||
raise ValueError("HiSparse verify cache locations are not request-aligned.")
|
||||
|
||||
valid = accept_indices >= 0
|
||||
counts = valid.sum(dim=1)
|
||||
steps = torch.arange(accept_indices.shape[1], device=accept_indices.device)
|
||||
positions = batch.seq_lens[:, None].to(torch.int64) + steps
|
||||
mapping = (
|
||||
coordinator.token_to_kv_pool_allocator.full_to_hisparse_device_index_mapping
|
||||
)
|
||||
# Keep a fixed width: boolean indexing and GPU-sized repeat_interleave sync.
|
||||
verify_width = verify_cache_locs.numel() // batch_size
|
||||
row_offsets = (
|
||||
torch.arange(batch_size, device=accept_indices.device)[:, None]
|
||||
* verify_width
|
||||
)
|
||||
source_offsets = torch.where(valid, accept_indices, row_offsets + steps)
|
||||
source_device_locs = mapping[verify_cache_locs[source_offsets]].clone()
|
||||
canonical_locs = coordinator.req_to_token_pool.req_to_token[
|
||||
req_pool_indices, positions
|
||||
]
|
||||
|
||||
# Keep hot tokens and each request's newest token in persistent slots.
|
||||
cache_mask = valid & (
|
||||
(positions < coordinator.device_buffer_size)
|
||||
| (steps == counts[:, None] - 1)
|
||||
)
|
||||
cache_positions = torch.where(
|
||||
cache_mask,
|
||||
positions.clamp(max=coordinator.device_buffer_size),
|
||||
coordinator.device_buffer_size + 1 + steps,
|
||||
)
|
||||
cache_locs = coordinator.req_to_device_buffer[req_pool_indices, cache_positions]
|
||||
destination_locs = torch.where(cache_mask, cache_locs, source_device_locs)
|
||||
coordinator.mem_pool_device.transfer_values_on_device(
|
||||
dst_indices=destination_locs.flatten(),
|
||||
src_indices=source_device_locs.flatten(),
|
||||
)
|
||||
|
||||
# Rejected padding goes to this request's reserved, not-yet-visible host slots.
|
||||
host_locs = coordinator.req_to_host_pool[req_pool_indices, positions]
|
||||
self._backup_device_locs_to_host(
|
||||
host_locs.flatten(), destination_locs.flatten()
|
||||
)
|
||||
|
||||
# Scalar indexed assignment performs a blocking CPU-to-CUDA copy.
|
||||
mapping.index_fill_(0, verify_cache_locs, 0)
|
||||
mapping[canonical_locs] = torch.where(cache_mask, cache_locs, 0)
|
||||
coordinator.req_device_buffer_tokens[:, req_pool_indices, cache_positions] = (
|
||||
torch.where(
|
||||
cache_mask,
|
||||
positions.to(torch.int32),
|
||||
coordinator.req_device_buffer_tokens[
|
||||
:, req_pool_indices, cache_positions
|
||||
],
|
||||
)
|
||||
)
|
||||
coordinator.req_device_buffer_token_locs[
|
||||
:, req_pool_indices, cache_positions
|
||||
] = torch.where(
|
||||
cache_mask,
|
||||
cache_locs.to(torch.int32),
|
||||
coordinator.req_device_buffer_token_locs[
|
||||
:, req_pool_indices, cache_positions
|
||||
],
|
||||
)
|
||||
for req in batch.reqs:
|
||||
coordinator._skip_first_backup[req.kv.req_pool_idx] = True
|
||||
|
||||
def swap_in(
|
||||
self,
|
||||
req_pool_indices: torch.Tensor,
|
||||
compressed_seq_lens: torch.Tensor,
|
||||
top_k_result: torch.Tensor,
|
||||
layer_id: int,
|
||||
**plan,
|
||||
) -> torch.Tensor:
|
||||
if not self.enabled:
|
||||
raise RuntimeError("HiSparse spec swap is not initialized.")
|
||||
num_reqs = req_pool_indices.size(0)
|
||||
num_steps, num_top_k = top_k_result.shape[1:]
|
||||
if num_steps != self.num_draft_tokens:
|
||||
raise ValueError(
|
||||
f"HiSparse spec step mismatch: expected {self.num_draft_tokens}, "
|
||||
f"got {num_steps}."
|
||||
)
|
||||
assert self.top_k_device_locs is not None
|
||||
top_k_indices = self.top_k_device_locs[:num_reqs, :num_steps, :num_top_k]
|
||||
coordinator = self._coordinator
|
||||
load_cache_to_device_buffer_spec_mla(
|
||||
top_k_tokens=top_k_result,
|
||||
device_buffer_tokens=coordinator.req_device_buffer_tokens[layer_id],
|
||||
host_cache_locs=coordinator.req_to_host_pool,
|
||||
device_buffer_locs=coordinator.req_device_buffer_token_locs[layer_id],
|
||||
host_cache=coordinator.mem_pool_host.kv_buffer[layer_id],
|
||||
device_buffer=coordinator.mem_pool_device.kv_buffer[layer_id],
|
||||
top_k_device_locs=top_k_indices,
|
||||
req_pool_indices=req_pool_indices,
|
||||
seq_lens=compressed_seq_lens,
|
||||
state=self.states[layer_id],
|
||||
num_real_reqs=coordinator.num_real_reqs,
|
||||
**plan,
|
||||
)
|
||||
return top_k_indices
|
||||
|
||||
def output_locs(self, top_k_result: torch.Tensor, num_reqs: int) -> torch.Tensor:
|
||||
if not self.enabled or self.top_k_device_locs is None:
|
||||
raise RuntimeError("HiSparse spec swap is not initialized.")
|
||||
return self.top_k_device_locs[
|
||||
:num_reqs, : top_k_result.shape[1], : top_k_result.shape[2]
|
||||
]
|
||||
|
||||
|
||||
class HiSparseCoordinator:
|
||||
def __init__(
|
||||
self,
|
||||
@@ -147,6 +545,7 @@ class HiSparseCoordinator:
|
||||
host_to_device_ratio: int = 2,
|
||||
swap_in_block_size: int = 960,
|
||||
shared_index_layers: Optional[List[bool]] = None,
|
||||
num_draft_tokens: int = 0,
|
||||
):
|
||||
self.req_to_token_pool = req_to_token_pool
|
||||
self.token_to_kv_pool_allocator = token_to_kv_pool_allocator
|
||||
@@ -281,6 +680,15 @@ class HiSparseCoordinator:
|
||||
.repeat(layer_num, max_num_req_slots, 1)
|
||||
.contiguous()
|
||||
)
|
||||
self.spec_swap = HiSparseSpecSwapManager(
|
||||
self,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
)
|
||||
if isinstance(self.token_to_kv_pool_allocator, HiSparseTokenToKVPoolAllocator):
|
||||
self.token_to_kv_pool_allocator.configure_spec_scratch(
|
||||
self.spec_swap.scratch_capacity,
|
||||
device_buffer_size=self.padded_buffer_size,
|
||||
)
|
||||
self._device_buffer_arange_i32 = torch.arange(
|
||||
self.device_buffer_size, dtype=torch.int32, device=device
|
||||
)
|
||||
@@ -356,11 +764,16 @@ class HiSparseCoordinator:
|
||||
# Plan recorded by the current anchor, replayed by its skip layers. One
|
||||
# buffer set suffices: the last skip layer's event wait orders the next
|
||||
# anchor's writes after this group's copies.
|
||||
miss_plan_capacity = self.spec_swap.miss_plan_capacity
|
||||
self._miss_src = torch.zeros(
|
||||
(max_num_req_slots, self.top_k), dtype=torch.int64, device=self.device
|
||||
(max_num_req_slots, miss_plan_capacity),
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
self._miss_dst = torch.zeros(
|
||||
(max_num_req_slots, self.top_k), dtype=torch.int32, device=self.device
|
||||
(max_num_req_slots, miss_plan_capacity),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
self._miss_count = torch.zeros(
|
||||
(max_num_req_slots,), dtype=torch.int32, device=self.device
|
||||
@@ -376,6 +789,10 @@ class HiSparseCoordinator:
|
||||
def set_decode_producer_stream(self, stream) -> None:
|
||||
self.decode_producer_stream = stream
|
||||
|
||||
def allocate_spec_scratch(self, req_pool_indices: List[int]) -> None:
|
||||
for req_pool_idx in req_pool_indices:
|
||||
self.spec_swap.allocate_scratch(req_pool_idx)
|
||||
|
||||
def destroy(self) -> None:
|
||||
# Drain in-flight transfers so the buffer is idle, then unregister it.
|
||||
# See HostKVCache.destroy for why the explicit unregister matters.
|
||||
@@ -471,6 +888,7 @@ class HiSparseCoordinator:
|
||||
self.req_device_buffer_tokens[
|
||||
:, req.kv.req_pool_idx, : self.device_buffer_size
|
||||
] = -1
|
||||
self.spec_swap.invalidate_cache(req.kv.req_pool_idx)
|
||||
|
||||
req.hisparse_staging = False
|
||||
self._skip_first_backup[req.kv.req_pool_idx] = True
|
||||
@@ -514,7 +932,8 @@ class HiSparseCoordinator:
|
||||
((allocated_len + page_size - 1) // page_size) * page_size,
|
||||
self.device_buffer_size,
|
||||
)
|
||||
if alloc_size == self.device_buffer_size:
|
||||
# Spec verify needs the complete, graph-stable buffer even for short prompts.
|
||||
if self.spec_swap.enabled or alloc_size == self.device_buffer_size:
|
||||
alloc_size = self.padded_buffer_size
|
||||
|
||||
compressed_logical_indices = (
|
||||
@@ -547,6 +966,8 @@ class HiSparseCoordinator:
|
||||
self.req_device_buffer_token_locs[:, req.kv.req_pool_idx, :alloc_size] = (
|
||||
buffer_indices[:alloc_size]
|
||||
)
|
||||
self.spec_swap.allocate_scratch(req.kv.req_pool_idx)
|
||||
self.spec_swap.reset(req.kv.req_pool_idx)
|
||||
|
||||
def _grow_device_buffers(
|
||||
self,
|
||||
@@ -653,6 +1074,14 @@ class HiSparseCoordinator:
|
||||
ready_reqs.append(req)
|
||||
return ready_reqs
|
||||
|
||||
def prepare_spec_verify(self, batch: ScheduleBatch) -> None:
|
||||
self.spec_swap.prepare_verify(batch)
|
||||
|
||||
def commit_spec_accept_tokens(
|
||||
self, batch: ScheduleBatch, accept_indices: torch.Tensor
|
||||
) -> None:
|
||||
self.spec_swap.commit_accept_tokens(batch, accept_indices)
|
||||
|
||||
def map_last_loc_to_buffer(
|
||||
self,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -939,6 +1368,7 @@ class HiSparseCoordinator:
|
||||
self.req_to_host_pool[req.kv.req_pool_idx, :] = -1
|
||||
self.req_to_host_pool_allocated_len[req.kv.req_pool_idx] = 0
|
||||
self._skip_first_backup[req.kv.req_pool_idx] = False
|
||||
self.spec_swap.free_unrotated_scratch(req.kv.req_pool_idx)
|
||||
req.hisparse_staging = False
|
||||
|
||||
def retract_req(self, req: Req) -> None:
|
||||
@@ -960,22 +1390,38 @@ class HiSparseCoordinator:
|
||||
# subsequent release_kv_cache -> allocator.free -> free_hisparse path
|
||||
# re-frees them (double-free into the page allocator's free list).
|
||||
allocated_len = req.kv.kv_allocated_len
|
||||
req_pool_idx = req.kv.req_pool_idx
|
||||
allocated_locs = self.req_to_token_pool.req_to_token[
|
||||
req_pool_idx, :allocated_len
|
||||
]
|
||||
|
||||
# release memory -- only free actually-allocated buffer indices
|
||||
current_cap = int(self.req_device_buffer_size[req.kv.req_pool_idx])
|
||||
# The spec commit kernel rotates physical locations between each
|
||||
# layer's persistent cache and scratch. Their current union is the
|
||||
# request's allocation ownership; the original side-buffer row is no
|
||||
# longer authoritative after the first rotation.
|
||||
current_cap = int(self.req_device_buffer_size[req_pool_idx])
|
||||
if current_cap > 0:
|
||||
side_buf_hi = self.req_to_device_buffer[req.kv.req_pool_idx, :current_cap]
|
||||
all_hi = torch.unique(side_buf_hi[side_buf_hi > 0])
|
||||
owned_locs = self.req_device_buffer_token_locs[
|
||||
:, req_pool_idx, :current_cap
|
||||
].reshape(-1)
|
||||
owned_locs = self.spec_swap.extend_owned_locs(req_pool_idx, owned_locs)
|
||||
all_hi = torch.unique(owned_locs[owned_locs > 0])
|
||||
if all_hi.numel() > 0:
|
||||
self.token_to_kv_pool_allocator.free_hisparse_indices(all_hi)
|
||||
|
||||
allocated_locs = self.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, :allocated_len
|
||||
]
|
||||
compressed_locs = self.mem_pool_device.translate_loc_from_full_to_compressed(
|
||||
allocated_locs
|
||||
)
|
||||
self.mem_pool_device.full_to_hisparse_device_index_mapping[compressed_locs] = 0
|
||||
compressed_locs = (
|
||||
self.mem_pool_device.translate_loc_from_full_to_compressed(
|
||||
allocated_locs
|
||||
)
|
||||
)
|
||||
self.mem_pool_device.full_to_hisparse_device_index_mapping[
|
||||
compressed_locs
|
||||
] = 0
|
||||
else:
|
||||
# The request can finish or be aborted during prefill, before a
|
||||
# device side buffer exists. Its ordinary prefill mapping is still
|
||||
# owned by release_kv_cache; only release our separate workspace.
|
||||
self.spec_swap.free_unrotated_scratch(req_pool_idx)
|
||||
|
||||
host_indices = self.mem_pool_host.allocated_host_indices(
|
||||
self.req_to_host_pool,
|
||||
@@ -993,6 +1439,7 @@ class HiSparseCoordinator:
|
||||
self.req_to_host_pool[req.kv.req_pool_idx, :] = -1
|
||||
self.req_to_host_pool_allocated_len[req.kv.req_pool_idx] = 0
|
||||
self.lru_slots[:, req.kv.req_pool_idx, :].copy_(self._lru_init)
|
||||
self.spec_swap.clear_scratch(req_pool_idx)
|
||||
self._skip_first_backup[req.kv.req_pool_idx] = False
|
||||
|
||||
def _run_swap_in_kernel(
|
||||
@@ -1009,12 +1456,6 @@ class HiSparseCoordinator:
|
||||
miss plan into self._miss_{src,dst,count} for the skip layers to replay.
|
||||
"""
|
||||
num_reqs = req_pool_indices.size(0)
|
||||
top_k_indices = self.top_k_device_locs_buffer[:num_reqs, : self.top_k]
|
||||
swap_in_fn = (
|
||||
load_cache_to_device_buffer_dsv4_mla
|
||||
if self.is_dsv4_hisparse
|
||||
else load_cache_to_device_buffer_mla
|
||||
)
|
||||
plan = (
|
||||
dict(
|
||||
miss_src=self._miss_src[:num_reqs],
|
||||
@@ -1024,6 +1465,24 @@ class HiSparseCoordinator:
|
||||
if record_plan
|
||||
else {}
|
||||
)
|
||||
|
||||
if top_k_result.ndim == 3:
|
||||
return self.spec_swap.swap_in(
|
||||
req_pool_indices,
|
||||
compressed_seq_lens,
|
||||
top_k_result,
|
||||
layer_id,
|
||||
**plan,
|
||||
)
|
||||
if top_k_result.ndim != 2:
|
||||
raise ValueError("HiSparse top-k must be two- or three-dimensional.")
|
||||
|
||||
top_k_indices = self.top_k_device_locs_buffer[:num_reqs, : self.top_k]
|
||||
swap_in_fn = (
|
||||
load_cache_to_device_buffer_dsv4_mla
|
||||
if self.is_dsv4_hisparse
|
||||
else load_cache_to_device_buffer_mla
|
||||
)
|
||||
skip_io_kwargs = {} if _is_xpu else dict(skip_io=self.skip_io)
|
||||
swap_in_fn(
|
||||
top_k_tokens=top_k_result,
|
||||
@@ -1129,6 +1588,8 @@ class HiSparseCoordinator:
|
||||
# applies (shared index + lockstep buffers).
|
||||
slot = self._prefetch_slot[layer_id]
|
||||
self._prefetch_events[slot].wait(device_module.current_stream())
|
||||
if top_k_result.ndim == 3:
|
||||
return self.spec_swap.output_locs(top_k_result, num_reqs)
|
||||
return self.top_k_device_locs_buffer[:num_reqs, : self.top_k]
|
||||
|
||||
# Anchor: swap in synchronously (recording the plan), then prefetch the
|
||||
|
||||
@@ -399,6 +399,26 @@ class FutureMap:
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
def make_staged_spec_input(self, future_indices: torch.Tensor):
|
||||
"""Build the relay handle for a request leaving HiSparse staging."""
|
||||
if not (self.spec_algo.is_eagle() or self.spec_algo.is_standalone()):
|
||||
raise RuntimeError(
|
||||
"HiSparse staging only supports EAGLE-family speculative inputs."
|
||||
)
|
||||
|
||||
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode
|
||||
from sglang.srt.speculative.eagle_info import EagleDraftInput
|
||||
|
||||
return EagleDraftInput(
|
||||
capture_hidden_mode=(
|
||||
CaptureHiddenMode.NULL
|
||||
if self.spec_algo.is_standalone()
|
||||
else CaptureHiddenMode.LAST
|
||||
),
|
||||
future_indices=future_indices,
|
||||
future_dsa_topk_indices_available=(self.dsa_topk_indices_buf is not None),
|
||||
)
|
||||
|
||||
def resolve_confidence_cpu(
|
||||
self, batch: ScheduleBatch
|
||||
) -> Optional[ResolvedConfidence]:
|
||||
@@ -430,7 +450,8 @@ class FutureMap:
|
||||
return
|
||||
# FIXME: indices = batch.req_pool_indices, pinned 2 iters via
|
||||
# record_batch_in_overlap; record_stream here is redundant.
|
||||
indices.record_stream(torch.get_device_module(self.device).current_stream())
|
||||
if indices.device.type != "cpu":
|
||||
indices.record_stream(torch.get_device_module(self.device).current_stream())
|
||||
if self.need_topk:
|
||||
hidden_states_buf = (
|
||||
self.hidden_states_buf if self.need_hidden_states else None
|
||||
@@ -450,7 +471,10 @@ class FutureMap:
|
||||
draft_input.bonus_tokens = bonus_tokens
|
||||
if hidden_states is not None:
|
||||
draft_input.hidden_states = hidden_states
|
||||
if self.draft_probs_buf is not None and draft_input.draft_probs is not None:
|
||||
# A staged HiSparse request carries only future_indices. The
|
||||
# rejection-sampling probabilities live in this relay buffer, so
|
||||
# checking the empty draft_input would silently drop them.
|
||||
if self.draft_probs_buf is not None:
|
||||
draft_input.draft_probs = self.draft_probs_buf[indices]
|
||||
else:
|
||||
draft_input.bonus_tokens = self.output_tokens_buf[indices]
|
||||
@@ -615,12 +639,16 @@ class FutureMap:
|
||||
self.output_tokens_buf[indices] = payload.bonus_tokens.to(
|
||||
self.output_tokens_buf.dtype
|
||||
)
|
||||
if self.need_topk:
|
||||
|
||||
# Staging transitions refresh only the bonus token. Preserve top-k and
|
||||
# hidden state already published by the preceding target forward.
|
||||
if self.need_topk and payload.topk_p is not None:
|
||||
assert payload.topk_index is not None
|
||||
self.topk_p_buf[indices] = payload.topk_p.to(self.topk_p_buf.dtype)
|
||||
self.topk_index_buf[indices] = payload.topk_index.to(
|
||||
self.topk_index_buf.dtype
|
||||
)
|
||||
if self.need_hidden_states:
|
||||
if self.need_hidden_states and payload.hidden_states is not None:
|
||||
self.hidden_states_buf[indices] = payload.hidden_states.to(
|
||||
self.hidden_states_buf.dtype
|
||||
)
|
||||
|
||||
@@ -814,6 +814,16 @@ class PrefillAdder:
|
||||
slots += req.mamba_host_hit_length
|
||||
return slots * self._mamba_slot_cost
|
||||
|
||||
def _request_slot_reserve_for_req(
|
||||
self, req: Req, *, extend_input_len: Optional[int] = None
|
||||
) -> int:
|
||||
if extend_input_len is None:
|
||||
extend_input_len = len(req.full_untruncated_fill_ids) - req.prefix_len
|
||||
return self.token_to_kv_pool_allocator.request_slot_reserve(
|
||||
has_req_pool_slot=req.kv.req_pool_idx is not None,
|
||||
prefill_tokens=req.prefix_len + extend_input_len,
|
||||
)
|
||||
|
||||
def ceil_paged_tokens(self, tokens: int) -> int:
|
||||
return -(-tokens // self.page_size) * self.page_size
|
||||
|
||||
@@ -845,6 +855,7 @@ class PrefillAdder:
|
||||
max_new_tokens: int,
|
||||
retracted_stain: bool,
|
||||
mamba_gap_reserve: int = 0,
|
||||
request_slot_reserve: int = 0,
|
||||
is_chunked_continuation: bool = False,
|
||||
compute_charge: Optional[int] = None,
|
||||
):
|
||||
@@ -870,7 +881,10 @@ class PrefillAdder:
|
||||
max_new_tokens,
|
||||
# reserve() already charges one page; add the remaining shard pages.
|
||||
extra_tokens=(
|
||||
mamba_gap_reserve + self.per_req_token_overhead - self.page_size
|
||||
mamba_gap_reserve
|
||||
+ request_slot_reserve
|
||||
+ self.per_req_token_overhead
|
||||
- self.page_size
|
||||
),
|
||||
chunk_limit=self.rem_chunk_tokens,
|
||||
is_chunked_continuation=is_chunked_continuation,
|
||||
@@ -1118,6 +1132,20 @@ class PrefillAdder:
|
||||
return req
|
||||
truncated = cand_extend_input_len > _rem_tokens
|
||||
new_len = min(cand_extend_input_len, _rem_tokens)
|
||||
request_slot_reserve = self._request_slot_reserve_for_req(
|
||||
req, extend_input_len=new_len
|
||||
)
|
||||
if request_slot_reserve and not self.memory_budget.can_allocate_prefill(
|
||||
paged_input=(
|
||||
self.ceil_paged_tokens(new_len)
|
||||
+ self.per_req_token_overhead
|
||||
+ request_slot_reserve
|
||||
),
|
||||
extend_input_len=new_len,
|
||||
max_new_tokens=self._swa_new_tokens(req),
|
||||
chunk_limit=self.rem_chunk_tokens,
|
||||
):
|
||||
return req
|
||||
# The continuing chunk must fit. Keep reservation outside assert for -O.
|
||||
reserved = self._kv_shard_reserve_scratch(
|
||||
prefix_len=req.prefix_len, extend_len=new_len
|
||||
@@ -1136,6 +1164,7 @@ class PrefillAdder:
|
||||
),
|
||||
req.retracted_stain,
|
||||
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
|
||||
request_slot_reserve=request_slot_reserve,
|
||||
is_chunked_continuation=True,
|
||||
compute_charge=req.extend_len if self.exact_chunk_fill else None,
|
||||
)
|
||||
@@ -1168,6 +1197,7 @@ class PrefillAdder:
|
||||
# Shared Mamba pool: fold the new mamba state's shared-gap cost into the
|
||||
# budget gate so admission can't over-commit (0 for baseline / non-Mamba).
|
||||
paged_input += self._mamba_gap_budget_for_req(req)
|
||||
paged_input += self._request_slot_reserve_for_req(req)
|
||||
fits = self.memory_budget.can_allocate_prefill(
|
||||
paged_input=paged_input,
|
||||
extend_input_len=cand_extend_input_len,
|
||||
@@ -1268,6 +1298,7 @@ class PrefillAdder:
|
||||
min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS),
|
||||
req.retracted_stain,
|
||||
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
|
||||
request_slot_reserve=self._request_slot_reserve_for_req(req),
|
||||
compute_charge=(req.extend_len if self.exact_chunk_fill else None),
|
||||
)
|
||||
else:
|
||||
@@ -1294,6 +1325,9 @@ class PrefillAdder:
|
||||
0,
|
||||
req.retracted_stain,
|
||||
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
|
||||
request_slot_reserve=self._request_slot_reserve_for_req(
|
||||
req, extend_input_len=trunc_len
|
||||
),
|
||||
compute_charge=trunc_len if self.exact_chunk_fill else None,
|
||||
)
|
||||
|
||||
@@ -1330,6 +1364,8 @@ class PrefillAdder:
|
||||
):
|
||||
self.tree_cache.buffer_pipeline.defer_staged_admission(req, "mamba")
|
||||
return AddReqResult.NO_TOKEN
|
||||
request_slot_reserve = self._request_slot_reserve_for_req(req)
|
||||
total_tokens += request_slot_reserve
|
||||
|
||||
# The temporary pin excludes this prefix from the evictable budget.
|
||||
# Selection itself neither allocates slots nor materializes host hits.
|
||||
@@ -1562,6 +1598,9 @@ class PrefillAdder:
|
||||
admission.max_new_tokens,
|
||||
req.retracted_stain,
|
||||
mamba_gap_reserve=mamba_gap_reserve,
|
||||
request_slot_reserve=self._request_slot_reserve_for_req(
|
||||
req, extend_input_len=admission.extend_len
|
||||
),
|
||||
# Compute budgets are billed forward-pass tokens under exact-chunk-fill.
|
||||
compute_charge=admission.extend_len if self.exact_chunk_fill else None,
|
||||
)
|
||||
@@ -1600,6 +1639,7 @@ class PrefillAdder:
|
||||
len(req.full_untruncated_fill_ids)
|
||||
- req.prefix_len
|
||||
+ min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS)
|
||||
+ self._request_slot_reserve_for_req(req)
|
||||
- self.rem_total_tokens
|
||||
)
|
||||
for running_req in sorted_valid_running_reqs:
|
||||
|
||||
@@ -3725,6 +3725,11 @@ class Scheduler(
|
||||
)
|
||||
batch.input_ids = None
|
||||
|
||||
if not batch.spec_algorithm.is_none():
|
||||
batch.spec_info = self.future_map.make_staged_spec_input(
|
||||
batch.req_pool_indices
|
||||
)
|
||||
|
||||
if batch.return_logprob:
|
||||
batch.top_logprobs_nums = [r.logprob.top_logprobs_num for r in reqs]
|
||||
batch.token_ids_logprobs = [list(r.origin_input_ids) for r in reqs]
|
||||
@@ -4210,6 +4215,7 @@ class Scheduler(
|
||||
new_batch.contains_last_prefill_chunk = (
|
||||
self.chunked_req is None or len(can_run_list) != 1
|
||||
)
|
||||
new_batch.hisparse_coordinator = self.hisparse_coordinator
|
||||
|
||||
if self.enable_hierarchical_cache or self.enable_unified_cache_external_linker:
|
||||
# todo (zhiqiang): disable cuda graph execution if hicache loading triggered
|
||||
|
||||
@@ -380,6 +380,8 @@ class SchedulerBatchResultProcessor:
|
||||
if discard_finish_reason is None:
|
||||
self._maybe_collect_routed_experts(req)
|
||||
self._maybe_collect_indexer_topk(req)
|
||||
if get_memory().enable_hisparse:
|
||||
self.hisparse_coordinator.request_finished(req)
|
||||
if batch.decoding_reqs and req in batch.decoding_reqs:
|
||||
self._prepare_kv_cache_release(req)
|
||||
release_kv_cache(
|
||||
|
||||
@@ -403,6 +403,11 @@ def alloc_for_extend(
|
||||
)
|
||||
req_pool_indices_device = req_pool_indices_cpu.to(batch.device, non_blocking=True)
|
||||
|
||||
# Admission already charged HiSparse scratch; allocate it with the request
|
||||
# slot so subsequent scheduling sees the remaining physical capacity.
|
||||
if batch.hisparse_coordinator is not None:
|
||||
batch.hisparse_coordinator.allocate_spec_scratch(req_pool_indices)
|
||||
|
||||
# Allocate KV cache (throws exception on failure)
|
||||
alloc_page_size = _alloc_page_size(batch)
|
||||
if reuse_kv is not None and any(reuse_kv):
|
||||
@@ -785,17 +790,41 @@ def alloc_for_spec_decode(
|
||||
last_loc = get_last_loc(
|
||||
req_to_token_pool.req_to_token, req_pool_indices, cur_kv_lens
|
||||
)
|
||||
out_cache_loc = ALLOC_EXTEND_FUNCS[_device_type(batch)](
|
||||
tree_cache,
|
||||
cur_kv_lens,
|
||||
cur_kv_lens_cpu,
|
||||
nxt_kv_lens,
|
||||
nxt_kv_lens_cpu,
|
||||
last_loc,
|
||||
num_needed_tokens,
|
||||
req_pool_indices=req_pool_indices,
|
||||
batch=batch,
|
||||
)
|
||||
allocator = tree_cache.token_to_kv_pool_allocator
|
||||
if batch is not None and batch.hisparse_coordinator is not None:
|
||||
# Spec decode reserves logical target slots here. Target verify
|
||||
# later binds those slots to HiSparse's extra page; allocating
|
||||
# target physical pages first both wastes memory and breaks the
|
||||
# physical paged allocator's last_loc chain after the reserve is
|
||||
# released. Draft KV has its own logical GPU-page mapping.
|
||||
evict_from_tree_cache(
|
||||
tree_cache,
|
||||
num_needed_tokens + len(nxt_kv_lens_cpu) * allocator.page_size,
|
||||
)
|
||||
out_cache_loc = allocator.alloc_logical_only(
|
||||
cur_kv_lens,
|
||||
cur_kv_lens_cpu,
|
||||
nxt_kv_lens,
|
||||
nxt_kv_lens_cpu,
|
||||
last_loc,
|
||||
num_needed_tokens,
|
||||
)
|
||||
if out_cache_loc is None:
|
||||
raise RuntimeError(
|
||||
"HiSparse spec decode failed to allocate logical KV slots."
|
||||
)
|
||||
else:
|
||||
out_cache_loc = ALLOC_EXTEND_FUNCS[_device_type(batch)](
|
||||
tree_cache,
|
||||
cur_kv_lens,
|
||||
cur_kv_lens_cpu,
|
||||
nxt_kv_lens,
|
||||
nxt_kv_lens_cpu,
|
||||
last_loc,
|
||||
num_needed_tokens,
|
||||
req_pool_indices=req_pool_indices,
|
||||
batch=batch,
|
||||
)
|
||||
# Updating req_to_token is a write to a shared tensor: it must not overlap
|
||||
# with the previous batch's forward, which also reads req_to_token.
|
||||
assign_req_to_token_pool_func(
|
||||
|
||||
@@ -76,6 +76,15 @@ class BaseTokenToKVPoolAllocator(abc.ABC):
|
||||
self, tree_cache, num_mixed_decode_tokens=num_mixed_decode_tokens
|
||||
)
|
||||
|
||||
def request_slot_reserve(
|
||||
self, *, has_req_pool_slot: bool, prefill_tokens: int = 0
|
||||
) -> int:
|
||||
"""Extra KV slots beyond prefill, including request-owned workspace.
|
||||
|
||||
``prefill_tokens`` includes the cached prefix and selected extension.
|
||||
"""
|
||||
return 0
|
||||
|
||||
def max_new_tokens_for_memory(
|
||||
self,
|
||||
input_tokens: int,
|
||||
|
||||
@@ -38,6 +38,8 @@ class HiSparseTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
self.device = device
|
||||
self.page_size = page_size
|
||||
self.need_sort = need_sort
|
||||
self._spec_scratch_capacity = 0
|
||||
self._spec_device_buffer_size = 0
|
||||
|
||||
paged_allocator_cls = (
|
||||
current_platform.get_paged_allocator_cls() or PagedTokenToKVPoolAllocator
|
||||
@@ -77,6 +79,29 @@ class HiSparseTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
weakref.proxy(self.full_to_hisparse_device_index_mapping)
|
||||
)
|
||||
|
||||
def configure_spec_scratch(
|
||||
self, capacity: int, *, device_buffer_size: int = 0
|
||||
) -> None:
|
||||
if any(
|
||||
size < 0 or size % self.page_size for size in (capacity, device_buffer_size)
|
||||
):
|
||||
raise ValueError(
|
||||
"HiSparse spec scratch and device buffer sizes must be "
|
||||
f"non-negative multiples of page_size={self.page_size}."
|
||||
)
|
||||
self._spec_scratch_capacity = capacity
|
||||
self._spec_device_buffer_size = device_buffer_size if capacity else 0
|
||||
|
||||
def request_slot_reserve(
|
||||
self, *, has_req_pool_slot: bool, prefill_tokens: int = 0
|
||||
) -> int:
|
||||
paged_prefill = -(-prefill_tokens // self.page_size) * self.page_size
|
||||
# Prefill KV is reused by the persistent buffer. Charge only its
|
||||
# missing slots, including on chunks whose scratch was already allocated.
|
||||
buffer_reserve = max(self._spec_device_buffer_size - paged_prefill, 0)
|
||||
scratch_reserve = 0 if has_req_pool_slot else self._spec_scratch_capacity
|
||||
return buffer_reserve + scratch_reserve
|
||||
|
||||
@property
|
||||
def size_full(self) -> int:
|
||||
return self._size_full
|
||||
@@ -169,7 +194,10 @@ class HiSparseTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
return buffer_indices
|
||||
|
||||
def free_hisparse_indices(self, buffer_indices: torch.Tensor):
|
||||
self.hisparse_attn_allocator.free(buffer_indices[buffer_indices > 0])
|
||||
# Page zero is the padding sink and is never owned by a request.
|
||||
self.hisparse_attn_allocator.free(
|
||||
buffer_indices[buffer_indices >= self.page_size]
|
||||
)
|
||||
|
||||
def get_last_loc_compressed(self, last_locs: torch.Tensor):
|
||||
return last_locs
|
||||
@@ -514,7 +542,9 @@ class DeepSeekV4HiSparseTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
return buffer_indices
|
||||
|
||||
def free_hisparse_indices(self, buffer_indices: torch.Tensor):
|
||||
self.hisparse_attn_allocator.free(buffer_indices[buffer_indices > 0])
|
||||
self.hisparse_attn_allocator.free(
|
||||
buffer_indices[buffer_indices >= self.hisparse_page_size]
|
||||
)
|
||||
|
||||
def get_last_loc_compressed(self, last_locs: torch.Tensor):
|
||||
# Last complete C4 block of a prefix of last_loc + 1 tokens; -1 stays -1.
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.kvcache.cache_move import copy_all_layer_kv_cache_func
|
||||
from sglang.kernels.ops.kvcache.hisparse_slot_mapping import (
|
||||
translate_padded_hisparse_locations,
|
||||
)
|
||||
@@ -76,6 +77,7 @@ class HiSparseDSATokenToKVPool(DSATokenToKVPool):
|
||||
skip_topk_layers=skip_topk_layers,
|
||||
)
|
||||
self.bytes_per_token = self.kv_cache_dim * self.dtype.itemsize
|
||||
self.data_strides = torch.full_like(self.data_ptrs, self.bytes_per_token)
|
||||
|
||||
def register_mapping(self, full_to_hisparse_device_index_mapping: torch.Tensor):
|
||||
self.full_to_hisparse_device_index_mapping = (
|
||||
@@ -136,6 +138,30 @@ class HiSparseDSATokenToKVPool(DSATokenToKVPool):
|
||||
loc = self.translate_loc_to_hisparse_device(loc)
|
||||
return super().get_mla_kv_buffer(layer, loc, dst_dtype)
|
||||
|
||||
def transfer_values_on_device(
|
||||
self, dst_indices: torch.Tensor, src_indices: torch.Tensor
|
||||
) -> None:
|
||||
"""Copy target MLA KV between physical slots, without remapping IDs."""
|
||||
if src_indices.numel() == 0:
|
||||
return
|
||||
config = {
|
||||
"bytes_per_tile": 128,
|
||||
"byte_tiles": (self.bytes_per_token + 127) // 128,
|
||||
"num_warps": 4,
|
||||
}
|
||||
for start in range(0, src_indices.numel(), 256):
|
||||
src = src_indices[start : start + 256].contiguous()
|
||||
dst = dst_indices[start : start + 256].contiguous()
|
||||
copy_all_layer_kv_cache_func(
|
||||
self.data_ptrs,
|
||||
self.data_strides,
|
||||
dst,
|
||||
src,
|
||||
src.numel(),
|
||||
1 << (src.numel() - 1).bit_length(),
|
||||
config,
|
||||
)
|
||||
|
||||
def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None):
|
||||
raise NotImplementedError("HiSparseDevicePool does not support get_cpu_copy")
|
||||
|
||||
|
||||
@@ -688,6 +688,18 @@ class KVCacheConfigurator:
|
||||
draft_virtual_id_space
|
||||
)
|
||||
sizes = msgspec.structs.replace(sizes, **size_overrides)
|
||||
elif isinstance(token_to_kv_pool_allocator, HiSparseTokenToKVPoolAllocator):
|
||||
# HiSparse's target allocator hands out logical IDs from its
|
||||
# host-expanded address space. Draft KV is direct-indexed by
|
||||
# those shared IDs, so its GPU pool must span the same space.
|
||||
draft_virtual_id_space = token_to_kv_pool_allocator.size_full
|
||||
page = max(int(self.pool_page_size or 1), 1)
|
||||
draft_virtual_id_space = (
|
||||
(draft_virtual_id_space + page - 1) // page * page
|
||||
)
|
||||
sizes = msgspec.structs.replace(
|
||||
sizes, max_total_num_tokens=draft_virtual_id_space
|
||||
)
|
||||
|
||||
# Initialize req_to_token_pool
|
||||
if req_to_token_pool is None:
|
||||
@@ -2018,7 +2030,7 @@ class KVCacheConfigurator:
|
||||
dsa_cp_layer_shard_size,
|
||||
) = get_glm_dsa_cp_layer_shard_info(self)
|
||||
pool_kwargs = {}
|
||||
if get_memory().enable_hisparse:
|
||||
if get_memory().enable_hisparse and not self.is_draft_worker:
|
||||
reject_out_of_tree_path(
|
||||
current_platform, subsystem="the HiSparse DSA KV pool"
|
||||
)
|
||||
|
||||
@@ -396,7 +396,9 @@ class ModelRunner:
|
||||
self._pending_elastic_scale_update = None
|
||||
self.init_new_workspace = False
|
||||
self.draft_model_idx = draft_model_idx
|
||||
self.enable_hisparse = get_memory().enable_hisparse
|
||||
# HiSparse owns target KV only. Draft KV keeps its logical GPU page
|
||||
# space and must not construct a second sparse coordinator.
|
||||
self.enable_hisparse = get_memory().enable_hisparse and not is_draft_worker
|
||||
self._sampling_observer: Optional[SamplingObserver] = None
|
||||
self.sampling_prewarm_result = SamplingPrewarmResult()
|
||||
|
||||
@@ -1026,7 +1028,11 @@ class ModelRunner:
|
||||
swap_in_block_size=hisparse_cfg.swap_in_block_size,
|
||||
shared_index_layers=resolve_shared_index_layers(
|
||||
hf_text_config=self.model_config.hf_text_config,
|
||||
is_speculative=self.spec_algorithm.is_speculative(),
|
||||
),
|
||||
num_draft_tokens=(
|
||||
0
|
||||
if self.spec_algorithm.is_none()
|
||||
else get_spec().speculative_num_draft_tokens or 0
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -265,6 +265,14 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
and int(num_layers) > 0
|
||||
):
|
||||
draft_num_layers = int(eagle_draft_num_layers)
|
||||
draft_loc_space_scale = 1
|
||||
if get_memory().enable_hisparse:
|
||||
from sglang.srt.mem_cache.sparsity import parse_hisparse_config
|
||||
|
||||
# The draft pool is direct-indexed by the target allocator's
|
||||
# host-expanded logical IDs. Account for the entire dense
|
||||
# address space that KVCacheConfigurator allocates below.
|
||||
draft_loc_space_scale = parse_hisparse_config().host_to_device_ratio
|
||||
if is_deepseek_dsa(kvc.model_config.hf_config):
|
||||
target_indexer_size = self._compute_dsa_indexer_cell_size(
|
||||
kvc=kvc,
|
||||
@@ -279,9 +287,12 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
target_kv_num_layers = get_glm_dsa_layer_split_effective_num_layers(
|
||||
kvc, num_layers
|
||||
)
|
||||
draft_kv_size = int(
|
||||
target_kv_size * draft_num_layers / target_kv_num_layers
|
||||
draft_kv_size = (
|
||||
int(target_kv_size * draft_num_layers / target_kv_num_layers)
|
||||
* draft_loc_space_scale
|
||||
)
|
||||
# _compute_dsa_indexer_cell_size already applies the
|
||||
# HiSparse logical-space ratio to the draft indexer.
|
||||
draft_indexer_size = self._compute_dsa_indexer_cell_size(
|
||||
kvc=kvc,
|
||||
num_layers=draft_num_layers,
|
||||
@@ -291,9 +302,10 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
|
||||
draft_indexer_size *= dcp_size
|
||||
self._cell_size += draft_kv_size + draft_indexer_size
|
||||
else:
|
||||
self._cell_size = int(
|
||||
self._cell_size * (1 + draft_num_layers / int(num_layers))
|
||||
draft_kv_size = int(
|
||||
self._cell_size * draft_num_layers / int(num_layers)
|
||||
)
|
||||
self._cell_size += draft_kv_size * draft_loc_space_scale
|
||||
|
||||
# DFLASH/DSPARK: the draft's per-token KV cost can differ from the target's
|
||||
# (e.g. MLA target, per-head K/V draft), so size from the draft config.
|
||||
|
||||
@@ -580,6 +580,8 @@ def eagle_prepare_for_verify(
|
||||
draft_token_num=verify_input.draft_token_num,
|
||||
device=device,
|
||||
)
|
||||
if batch.hisparse_coordinator is not None:
|
||||
batch.hisparse_coordinator.prepare_spec_verify(batch)
|
||||
|
||||
batch.out_cache_loc_dsv4 = maybe_build_dsv4_verify_bundle(
|
||||
batch, verify_input.draft_token_num
|
||||
|
||||
@@ -732,6 +732,11 @@ def run_eagle_verify(
|
||||
)
|
||||
else:
|
||||
new_seq_lens = batch.seq_lens + accept_lens
|
||||
if batch.hisparse_coordinator is not None:
|
||||
batch.hisparse_coordinator.commit_spec_accept_tokens(
|
||||
batch=batch,
|
||||
accept_indices=accept_index,
|
||||
)
|
||||
clear_unaccepted_c128 = getattr(
|
||||
token_to_kv_pool_allocator.get_kvcache(),
|
||||
"clear_unaccepted_c128_draft_states",
|
||||
|
||||
Reference in New Issue
Block a user