[HiSparse] Add speculative decoding support with multi-step KV cache swapping (#37771)

This commit is contained in:
huangtingwei
2026-10-13 13:38:31 -07:00
committed by GitHub
parent 7dc2930585
commit 18210ed849
31 changed files with 1580 additions and 117 deletions
+16 -4
View File
@@ -79,10 +79,7 @@ def validate_hisparse_kv_cache_dtype(server_args: ServerArgs) -> None:
def validate_hisparse_feature_combos(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
# Both crash on the first request: the draft pool has no HiSparse mapping,
# and a mixed batch cannot merge the HiSparse decode batch.
if cfg.speculative_algorithm is not None:
raise ValueError("--enable-hisparse does not support speculative decoding yet.")
# A mixed batch cannot merge the HiSparse decode batch.
if cfg.enable_mixed_chunk:
raise ValueError("--enable-hisparse does not support --enable-mixed-chunk yet.")
@@ -114,6 +111,21 @@ def validate_hisparse(server_args: ServerArgs) -> None:
)
validate_hisparse_feature_combos(server_args)
enable_spec = (
cfg.speculative_algorithm is not None
and cfg.speculative_algorithm.upper() != "NONE"
)
if enable_spec and (cfg.speculative_eagle_topk or 1) != 1:
raise ValueError(
"The minimal HiSparse spec integration supports linear speculation "
"only; use --speculative-eagle-topk 1."
)
if enable_spec and cfg.speculative_adaptive:
raise ValueError(
"HiSparse speculative decoding uses statically sized swap state; "
"--speculative-adaptive is not supported."
)
# DSv4 hisparse handles its own dtype/backend pairing elsewhere; the dtype-
# aware checks below only apply to the DSA hisparse path.
if is_hip and is_v4_hisparse:
@@ -1688,8 +1688,16 @@ class CommonKVManager(BaseKVManager):
# Regular MLA PP slicing
# Decode pp size should be equal to prefill pp size or 1
start_layer, end_layer = self._mla_kv_entry_span_with_pp(len(src_kv_ptrs))
num_draft = (
getattr(self.kv_args, "num_draft_entries", 0) if state_type is None else 0
)
start_layer, end_layer = self._mla_kv_entry_span_with_pp(
len(src_kv_ptrs) - num_draft
)
sliced_dst_kv_ptrs = dst_kv_ptrs[start_layer:end_layer]
if num_draft:
# Draft entries follow the target's global layer range on both peers.
sliced_dst_kv_ptrs += dst_kv_ptrs[-num_draft:]
return src_kv_ptrs, sliced_dst_kv_ptrs, len(src_kv_ptrs)
def _mla_kv_entry_span_with_pp(self, n_src: int) -> Tuple[int, int]:
+15 -8
View File
@@ -1382,17 +1382,21 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
]
# HiSparse physical constraint: max requests by device buffer capacity.
# Each admitted req needs padded_buffer_size from hisparse device pool.
# Each admitted req needs a padded buffer plus its speculative scratch.
# waiting_queue reqs already have device buffers (allocated in admit_request_direct),
# only transfer_queue reqs are pending device buffer allocation.
hisparse_req_budget = float("inf")
if self.scheduler.enable_hisparse:
coordinator = self.scheduler.hisparse_coordinator
per_req_device_tokens = (
coordinator.padded_buffer_size + coordinator.spec_swap.scratch_capacity
)
hisparse_avail = (
self.token_to_kv_pool_allocator.hisparse_attn_allocator.available_size()
)
hisparse_req_budget = max(
0,
hisparse_avail // self.scheduler.hisparse_coordinator.padded_buffer_size
hisparse_avail // per_req_device_tokens
- len(self.transfer_queue.queue),
)
@@ -1725,13 +1729,16 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
device_page_indices = None
if (
self.scheduler.enable_hisparse
and isinstance(self.token_to_kv_pool, DeepSeekV4TokenToKVPool)
and (
isinstance(self.token_to_kv_pool, DeepSeekV4TokenToKVPool)
or self.draft_token_to_kv_pool is not None
)
and not _is_fake_transfer(decode_req.req)
):
# alloc_logical_only() already allocated the shared logical pages
# used by C4 indexer and C128 KV. These device buffers do not use
# the C4 sparse physical-slot mapping; carry their logical page IDs
# alongside the independently allocated C4 host page IDs.
# used by the indexer and draft KV. These device buffers do not
# use target sparse physical slots; carry logical page IDs beside
# the independently allocated target host page IDs.
full_kv_indices = self.req_to_token_pool.req_to_token[
decode_req.req.kv.req_pool_idx,
prefix_len:origin_input_len,
@@ -1742,8 +1749,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
).astype(np.int32)
if self.transfer_backend != TransferBackend.MOONCAKE:
raise NotImplementedError(
"DSV4 HiSparse direct PD transfer currently requires "
"the Mooncake backend"
"HiSparse device-only PD transfer currently requires the "
"Mooncake backend"
)
metadata_kwargs = {"decode_prefix_len": total_prefix_len}
if device_page_indices is not None:
@@ -1320,14 +1320,24 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
dst_device_kv_ptrs = None
if dst_device_kv_indices is not None:
compression_ratios = self.kv_args.mla_compression_ratios
assert compression_ratios is not None
if len(dst_kv_ptrs) == len(self.kv_args.kv_data_ptrs):
start = self.kv_args.prefill_start_layer
end = self.kv_args.prefill_end_layer
assert end is not None
compression_ratios = compression_ratios[start:end]
c4_layer_num = sum(ratio == 4 for ratio in compression_ratios)
dst_device_kv_ptrs = set(dst_kv_ptrs[c4_layer_num:])
if compression_ratios is not None:
if len(dst_kv_ptrs) == len(self.kv_args.kv_data_ptrs):
start = self.kv_args.prefill_start_layer
end = self.kv_args.prefill_end_layer
assert end is not None
compression_ratios = compression_ratios[start:end]
host_backed_kv_count = sum(ratio == 4 for ratio in compression_ratios)
else:
# The remote target prefix is host-backed, even across PP stages;
# only the draft suffix uses logical device page IDs.
num_draft_entries = self.kv_args.num_draft_entries
if not 0 < num_draft_entries < len(dst_kv_ptrs):
raise ValueError(
"HiSparse spec PD transfer requires an appended device-only "
"draft KV buffer"
)
host_backed_kv_count = len(dst_kv_ptrs) - num_draft_entries
dst_device_kv_ptrs = set(dst_kv_ptrs[host_backed_kv_count:])
return self._send_kvcache_generic(
mooncake_session_id=mooncake_session_id,
src_data_ptrs=self.kv_args.kv_data_ptrs,
@@ -2262,10 +2262,23 @@ class DeepseekSparseAttnBackend(
# todo hisparse: to cover more backends
if self.hisparse_coordinator is not None:
# flash_mla_sparse_fwd / tilelang require int32 page indices.
page_table_1 = self.token_to_kv_pool.translate_loc_to_hisparse_device(
page_table_1
).to(torch.int32)
if forward_batch.forward_mode.is_target_verify():
num_reqs = forward_batch.req_pool_indices.shape[0]
num_steps = self.speculative_num_draft_tokens
assert topk_indices is not None
grouped_topk_indices = topk_indices.view(num_reqs, num_steps, -1)
assert metadata.dsa_seqlens_expanded is not None
page_table_1 = self.hisparse_coordinator.swap_in_selected_pages(
forward_batch.req_pool_indices,
metadata.dsa_seqlens_expanded,
grouped_topk_indices,
layer.layer_id,
).view(num_reqs * num_steps, -1)
else:
# flash_mla_sparse_fwd / tilelang require int32 page indices.
page_table_1 = self.token_to_kv_pool.translate_loc_to_hisparse_device(
page_table_1
).to(torch.int32)
if dsa_impl == "tilelang":
if q_rope is not None:
@@ -4019,7 +4032,10 @@ class DeepseekSparseAttnBackend(
) -> DSAIndexerMetadata:
force_unfused = not self.use_fused_topk or (
self.hisparse_coordinator is not None
and forward_batch.forward_mode.is_decode_or_idle()
and (
forward_batch.forward_mode.is_decode_or_idle()
or forward_batch.forward_mode.is_target_verify()
)
)
return DSAIndexerMetadata(
attn_metadata=self.forward_metadata,
@@ -1,7 +1,8 @@
# to be combined with the sparse coordinator class and sparse algorithm family
from __future__ import annotations
import logging
from typing import Dict, List, NamedTuple, Optional, Tuple, Union
import weakref
from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional, Tuple, Union
import torch
@@ -27,10 +28,12 @@ if is_xpu():
else:
from sglang.kernels.ops.kvcache.hisparse import (
HiSparseSpecState,
copy_cache_planned_mla,
load_blocks_to_device_buffer_mha,
load_cache_to_device_buffer_dsv4_mla,
load_cache_to_device_buffer_mla,
load_cache_to_device_buffer_spec_mla,
)
from sglang.srt.configs.model_config import dsa_layer_skips_topk, is_deepseek_dsa
@@ -49,6 +52,10 @@ from sglang.srt.mem_cache.pool_host.mha import HiSparseMHATokenToKVPoolHost
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
from sglang.srt.runtime_context import get_parallel
if TYPE_CHECKING:
from sglang.kernels.ops.kvcache.hisparse import HiSparseSpecState
from sglang.srt.managers.schedule_batch import ScheduleBatch
device_module = get_device_module()
_is_hip = is_hip()
@@ -73,13 +80,12 @@ class HiSparseTokenStats(NamedTuple):
def resolve_shared_index_layers(
*,
hf_text_config,
is_speculative: bool,
) -> Optional[List[bool]]:
"""Per-layer "reuses the previous layer's top-k index" pattern, or None.
Mirrors DeepseekV2AttentionMLA's skip_topk derivation (index_topk_pattern /
index_topk_freq / cli_factor); None when the model has no sharing or the
prefetch cannot run (PP, speculative decoding, kill-switch).
prefetch cannot run (pipeline parallelism or the kill-switch).
"""
if not is_deepseek_dsa(hf_text_config):
return None
@@ -91,11 +97,12 @@ def resolve_shared_index_layers(
pattern = [dsa_layer_skips_topk(hf_text_config, i) for i in range(num_layers)]
if not any(pattern):
return None
if get_parallel().pp_size != 1 or is_speculative:
# Speculative decoding is supported by the HiSparse multi-step swap path;
# only pipeline parallelism still prevents the shared-index prefetch plan.
if get_parallel().pp_size != 1:
logger.warning(
"HiSparse shared-index prefetch is unsupported under pipeline "
"parallelism / speculative decoding; falling back to synchronous "
"swap-in."
"parallelism; falling back to synchronous swap-in."
)
return None
if envs.SGLANG_DISABLE_HISPARSE_PREFETCH.get():
@@ -132,6 +139,397 @@ def _build_prefetch_groups(
return groups, slot
class HiSparseSpecSwapManager:
"""Own HiSparse speculative-swap state and request lifecycle."""
HASH_MULTIPLIER = 2654435761
MIN_SCRATCH_CAPACITY = 1024
NUM_METADATA_VALUES_PER_OCCURRENCE = 5
NUM_COUNTERS_PER_REQUEST = 4
def __init__(
self,
coordinator: HiSparseCoordinator,
*,
num_draft_tokens: int,
) -> None:
self._coordinator = weakref.proxy(coordinator)
self.enabled = num_draft_tokens > 0
self.num_draft_tokens = num_draft_tokens
self.scratch_capacity = 0
self._scratch_reqs: set[int] = set()
layer_num = coordinator.mem_pool_device.layer_num
max_num_req_slots = coordinator.req_to_token_pool.req_to_token.shape[0]
self.req_to_scratch = torch.empty(
(layer_num, max_num_req_slots, 0),
dtype=torch.int32,
device=coordinator.device,
)
self.states: tuple[HiSparseSpecState, ...] = ()
self.top_k_device_locs: Optional[torch.Tensor] = None
if not self.enabled:
return
if _is_xpu:
raise ValueError("HiSparse speculative swap is unsupported on XPU.")
if coordinator.is_dsv4_hisparse:
raise ValueError("HiSparse spec swap currently supports DSA caches only.")
if not 2 <= num_draft_tokens <= 4 or coordinator.top_k < 1024:
raise ValueError(
"HiSparse spec swap requires 2-4 draft tokens and top_k >= 1024."
)
workspace_capacity = num_draft_tokens * coordinator.top_k
if workspace_capacity > 8192:
raise ValueError("HiSparse spec swap supports at most 8192 occurrences.")
if max_num_req_slots > coordinator.device_buffer_size:
raise ValueError(
"HiSparse spec request capacity must not exceed device_buffer_size."
)
self.scratch_capacity = min(
workspace_capacity,
max(
self.MIN_SCRATCH_CAPACITY,
workspace_capacity - coordinator.device_buffer_size,
),
)
hash_size = 1 << max(1, (2 * coordinator.device_buffer_size - 1).bit_length())
self._cache_index = torch.full(
(layer_num, max_num_req_slots, 2, hash_size),
-1,
dtype=torch.int64,
device=coordinator.device,
)
self._cache_policy = torch.zeros(
(layer_num, max_num_req_slots + 1, coordinator.device_buffer_size),
dtype=torch.int32,
device=coordinator.device,
)
metadata_width = max(
self.NUM_COUNTERS_PER_REQUEST * max_num_req_slots,
self.NUM_METADATA_VALUES_PER_OCCURRENCE * workspace_capacity,
)
self._scratch_state = torch.full(
(max_num_req_slots + 1, metadata_width),
-1,
dtype=torch.int32,
device=coordinator.device,
)
self._scratch_state[0].zero_()
self.req_to_scratch = torch.zeros(
(layer_num, max_num_req_slots, self.scratch_capacity),
dtype=torch.int32,
device=coordinator.device,
)
self.states = tuple(
HiSparseSpecState(
cache_index=self._cache_index[layer_id],
cache_policy=self._cache_policy[layer_id],
scratch_locs=self.req_to_scratch[layer_id],
scratch_state=self._scratch_state,
)
for layer_id in range(layer_num)
)
tokens = torch.arange(
coordinator.device_buffer_size,
dtype=torch.int64,
device=coordinator.device,
)
self._hash_slots = ((tokens * self.HASH_MULTIPLIER) & (hash_size - 1)).to(
torch.long
)
self._hash_entries = (tokens << 32) | tokens
self.top_k_device_locs = torch.full(
(max_num_req_slots, num_draft_tokens, coordinator.top_k),
-1,
dtype=torch.int32,
device=coordinator.device,
)
@property
def miss_plan_capacity(self) -> int:
return self._coordinator.top_k * max(1, self.num_draft_tokens)
def invalidate_cache(self, req_pool_idx: int) -> None:
if self.enabled:
self._cache_index[:, req_pool_idx].fill_(-1)
def reset(self, req_pool_idx: int) -> None:
if not self.enabled:
return
self._cache_index[:, req_pool_idx].fill_(-1)
self._cache_index[:, req_pool_idx, 0, self._hash_slots] = (
self._hash_entries.view(1, -1)
)
self._cache_policy[:, 0, req_pool_idx].zero_()
self._cache_policy[:, req_pool_idx + 1].zero_()
max_num_req_slots = self.req_to_scratch.shape[1]
self._scratch_state[
0,
req_pool_idx : self.NUM_COUNTERS_PER_REQUEST
* max_num_req_slots : max_num_req_slots,
] = 0
self._scratch_state[req_pool_idx + 1].fill_(-1)
def allocate_scratch(self, req_pool_idx: int) -> None:
"""Reserve one request's graph-stable swap workspace at admission."""
if not self.enabled or req_pool_idx in self._scratch_reqs:
return
allocator = self._coordinator.token_to_kv_pool_allocator
scratch_locs = allocator.hisparse_attn_allocator.alloc(self.scratch_capacity)
if scratch_locs is None:
raise RuntimeError(
"HiSparse spec failed to allocate "
f"{self.scratch_capacity} scratch slots."
)
# Physical slots are shared by all KV layers, but every full-index
# layer rotates those IDs between persistent cache and scratch
# independently. Keep per-layer location views so one layer cannot
# overwrite the next layer's ownership state.
self.req_to_scratch[:, req_pool_idx].copy_(scratch_locs.to(torch.int32))
self._scratch_reqs.add(req_pool_idx)
def free_unrotated_scratch(self, req_pool_idx: int) -> None:
"""Free scratch before any swap kernel has changed its ownership."""
if not self.enabled or req_pool_idx not in self._scratch_reqs:
return
self._coordinator.token_to_kv_pool_allocator.free_hisparse_indices(
self.req_to_scratch[0, req_pool_idx].clone()
)
self.clear_scratch(req_pool_idx)
def clear_scratch(self, req_pool_idx: int) -> None:
if not self.enabled or req_pool_idx not in self._scratch_reqs:
return
self.req_to_scratch[:, req_pool_idx].zero_()
self._scratch_reqs.remove(req_pool_idx)
def extend_owned_locs(
self, req_pool_idx: int, owned_locs: torch.Tensor
) -> torch.Tensor:
if not self.enabled or req_pool_idx not in self._scratch_reqs:
return owned_locs
return torch.cat([owned_locs, self.req_to_scratch[:, req_pool_idx].reshape(-1)])
def prepare_verify(self, batch: ScheduleBatch) -> None:
"""Bind target-verify KV writes to the side buffer's extra page."""
coordinator = self._coordinator
req_pool_indices = batch.req_pool_indices
verify_cache_locs = batch.out_cache_loc
start_positions = batch.seq_lens
if not self.enabled:
raise RuntimeError("HiSparse spec is not initialized.")
if self.num_draft_tokens > coordinator.page_size - 1:
raise ValueError(
f"HiSparse spec needs {self.num_draft_tokens} verify slots, but the "
f"extra page has only {coordinator.page_size - 1} usable slots."
)
expected_slots = req_pool_indices.numel() * self.num_draft_tokens
if verify_cache_locs.numel() != expected_slots:
raise ValueError(
f"HiSparse verify slot mismatch: expected {expected_slots}, "
f"got {verify_cache_locs.numel()}."
)
# CPU-side logical reservations cover verify even when overlap lags seq_lens.
for req in batch.reqs:
req_pool_idx = req.kv.req_pool_idx
allocated_len = int(
coordinator.req_to_host_pool_allocated_len[req_pool_idx]
)
if req.kv.kv_allocated_len > allocated_len:
coordinator.mem_pool_host.alloc_paged_token_slots(
coordinator.req_to_host_pool,
coordinator.req_to_host_pool_allocated_len,
req_pool_idx,
allocated_len,
req.kv.kv_allocated_len - allocated_len,
)
extra_start = coordinator.device_buffer_size + 1
row_indices = torch.repeat_interleave(req_pool_indices, self.num_draft_tokens)
offsets = (
torch.arange(
expected_slots, dtype=torch.int64, device=req_pool_indices.device
)
% self.num_draft_tokens
)
token_positions = (
torch.repeat_interleave(
start_positions.to(torch.int64), self.num_draft_tokens
)
+ offsets
)
columns = extra_start + offsets
device_locs = coordinator.req_to_device_buffer[row_indices, columns]
coordinator.req_device_buffer_tokens[:, row_indices, columns] = (
token_positions.to(torch.int32).unsqueeze(0)
)
coordinator.req_device_buffer_token_locs[:, row_indices, columns] = (
device_locs.to(torch.int32).unsqueeze(0)
)
coordinator.token_to_kv_pool_allocator.full_to_hisparse_device_index_mapping[
verify_cache_locs
] = device_locs
def _backup_device_locs_to_host(
self, host_locs: torch.Tensor, device_locs: torch.Tensor
) -> None:
if host_locs.numel() == 0:
return
coordinator = self._coordinator
coordinator.wait_for_pending_backup()
schedule_stream = device_module.current_stream()
device_locs = device_locs.contiguous()
with device_module.stream(coordinator.decode_backup_stream):
coordinator.decode_backup_stream.wait_stream(schedule_stream)
if coordinator.decode_producer_stream is not None:
coordinator.decode_backup_stream.wait_stream(
coordinator.decode_producer_stream
)
coordinator.mem_pool_host.backup_from_device_all_layer(
coordinator.mem_pool_device,
host_locs,
device_locs,
io_backend="kernel",
)
if host_locs.is_cuda:
host_locs.record_stream(coordinator.decode_backup_stream)
if device_locs.is_cuda:
device_locs.record_stream(coordinator.decode_backup_stream)
coordinator._backup_done_event.record()
coordinator._has_pending_backup = True
coordinator.wait_for_pending_backup()
def commit_accept_tokens(
self, batch: ScheduleBatch, accept_indices: torch.Tensor
) -> None:
"""Persist accepted target-verify KV in host and the persistent buffer."""
if batch.forward_mode.is_idle():
return
coordinator = self._coordinator
req_pool_indices = batch.req_pool_indices[:, None]
verify_cache_locs = batch.out_cache_loc
if verify_cache_locs.numel() == 0:
return
batch_size = req_pool_indices.numel()
if batch_size == 0 or verify_cache_locs.numel() % batch_size != 0:
raise ValueError("HiSparse verify cache locations are not request-aligned.")
valid = accept_indices >= 0
counts = valid.sum(dim=1)
steps = torch.arange(accept_indices.shape[1], device=accept_indices.device)
positions = batch.seq_lens[:, None].to(torch.int64) + steps
mapping = (
coordinator.token_to_kv_pool_allocator.full_to_hisparse_device_index_mapping
)
# Keep a fixed width: boolean indexing and GPU-sized repeat_interleave sync.
verify_width = verify_cache_locs.numel() // batch_size
row_offsets = (
torch.arange(batch_size, device=accept_indices.device)[:, None]
* verify_width
)
source_offsets = torch.where(valid, accept_indices, row_offsets + steps)
source_device_locs = mapping[verify_cache_locs[source_offsets]].clone()
canonical_locs = coordinator.req_to_token_pool.req_to_token[
req_pool_indices, positions
]
# Keep hot tokens and each request's newest token in persistent slots.
cache_mask = valid & (
(positions < coordinator.device_buffer_size)
| (steps == counts[:, None] - 1)
)
cache_positions = torch.where(
cache_mask,
positions.clamp(max=coordinator.device_buffer_size),
coordinator.device_buffer_size + 1 + steps,
)
cache_locs = coordinator.req_to_device_buffer[req_pool_indices, cache_positions]
destination_locs = torch.where(cache_mask, cache_locs, source_device_locs)
coordinator.mem_pool_device.transfer_values_on_device(
dst_indices=destination_locs.flatten(),
src_indices=source_device_locs.flatten(),
)
# Rejected padding goes to this request's reserved, not-yet-visible host slots.
host_locs = coordinator.req_to_host_pool[req_pool_indices, positions]
self._backup_device_locs_to_host(
host_locs.flatten(), destination_locs.flatten()
)
# Scalar indexed assignment performs a blocking CPU-to-CUDA copy.
mapping.index_fill_(0, verify_cache_locs, 0)
mapping[canonical_locs] = torch.where(cache_mask, cache_locs, 0)
coordinator.req_device_buffer_tokens[:, req_pool_indices, cache_positions] = (
torch.where(
cache_mask,
positions.to(torch.int32),
coordinator.req_device_buffer_tokens[
:, req_pool_indices, cache_positions
],
)
)
coordinator.req_device_buffer_token_locs[
:, req_pool_indices, cache_positions
] = torch.where(
cache_mask,
cache_locs.to(torch.int32),
coordinator.req_device_buffer_token_locs[
:, req_pool_indices, cache_positions
],
)
for req in batch.reqs:
coordinator._skip_first_backup[req.kv.req_pool_idx] = True
def swap_in(
self,
req_pool_indices: torch.Tensor,
compressed_seq_lens: torch.Tensor,
top_k_result: torch.Tensor,
layer_id: int,
**plan,
) -> torch.Tensor:
if not self.enabled:
raise RuntimeError("HiSparse spec swap is not initialized.")
num_reqs = req_pool_indices.size(0)
num_steps, num_top_k = top_k_result.shape[1:]
if num_steps != self.num_draft_tokens:
raise ValueError(
f"HiSparse spec step mismatch: expected {self.num_draft_tokens}, "
f"got {num_steps}."
)
assert self.top_k_device_locs is not None
top_k_indices = self.top_k_device_locs[:num_reqs, :num_steps, :num_top_k]
coordinator = self._coordinator
load_cache_to_device_buffer_spec_mla(
top_k_tokens=top_k_result,
device_buffer_tokens=coordinator.req_device_buffer_tokens[layer_id],
host_cache_locs=coordinator.req_to_host_pool,
device_buffer_locs=coordinator.req_device_buffer_token_locs[layer_id],
host_cache=coordinator.mem_pool_host.kv_buffer[layer_id],
device_buffer=coordinator.mem_pool_device.kv_buffer[layer_id],
top_k_device_locs=top_k_indices,
req_pool_indices=req_pool_indices,
seq_lens=compressed_seq_lens,
state=self.states[layer_id],
num_real_reqs=coordinator.num_real_reqs,
**plan,
)
return top_k_indices
def output_locs(self, top_k_result: torch.Tensor, num_reqs: int) -> torch.Tensor:
if not self.enabled or self.top_k_device_locs is None:
raise RuntimeError("HiSparse spec swap is not initialized.")
return self.top_k_device_locs[
:num_reqs, : top_k_result.shape[1], : top_k_result.shape[2]
]
class HiSparseCoordinator:
def __init__(
self,
@@ -147,6 +545,7 @@ class HiSparseCoordinator:
host_to_device_ratio: int = 2,
swap_in_block_size: int = 960,
shared_index_layers: Optional[List[bool]] = None,
num_draft_tokens: int = 0,
):
self.req_to_token_pool = req_to_token_pool
self.token_to_kv_pool_allocator = token_to_kv_pool_allocator
@@ -281,6 +680,15 @@ class HiSparseCoordinator:
.repeat(layer_num, max_num_req_slots, 1)
.contiguous()
)
self.spec_swap = HiSparseSpecSwapManager(
self,
num_draft_tokens=num_draft_tokens,
)
if isinstance(self.token_to_kv_pool_allocator, HiSparseTokenToKVPoolAllocator):
self.token_to_kv_pool_allocator.configure_spec_scratch(
self.spec_swap.scratch_capacity,
device_buffer_size=self.padded_buffer_size,
)
self._device_buffer_arange_i32 = torch.arange(
self.device_buffer_size, dtype=torch.int32, device=device
)
@@ -356,11 +764,16 @@ class HiSparseCoordinator:
# Plan recorded by the current anchor, replayed by its skip layers. One
# buffer set suffices: the last skip layer's event wait orders the next
# anchor's writes after this group's copies.
miss_plan_capacity = self.spec_swap.miss_plan_capacity
self._miss_src = torch.zeros(
(max_num_req_slots, self.top_k), dtype=torch.int64, device=self.device
(max_num_req_slots, miss_plan_capacity),
dtype=torch.int64,
device=self.device,
)
self._miss_dst = torch.zeros(
(max_num_req_slots, self.top_k), dtype=torch.int32, device=self.device
(max_num_req_slots, miss_plan_capacity),
dtype=torch.int32,
device=self.device,
)
self._miss_count = torch.zeros(
(max_num_req_slots,), dtype=torch.int32, device=self.device
@@ -376,6 +789,10 @@ class HiSparseCoordinator:
def set_decode_producer_stream(self, stream) -> None:
self.decode_producer_stream = stream
def allocate_spec_scratch(self, req_pool_indices: List[int]) -> None:
for req_pool_idx in req_pool_indices:
self.spec_swap.allocate_scratch(req_pool_idx)
def destroy(self) -> None:
# Drain in-flight transfers so the buffer is idle, then unregister it.
# See HostKVCache.destroy for why the explicit unregister matters.
@@ -471,6 +888,7 @@ class HiSparseCoordinator:
self.req_device_buffer_tokens[
:, req.kv.req_pool_idx, : self.device_buffer_size
] = -1
self.spec_swap.invalidate_cache(req.kv.req_pool_idx)
req.hisparse_staging = False
self._skip_first_backup[req.kv.req_pool_idx] = True
@@ -514,7 +932,8 @@ class HiSparseCoordinator:
((allocated_len + page_size - 1) // page_size) * page_size,
self.device_buffer_size,
)
if alloc_size == self.device_buffer_size:
# Spec verify needs the complete, graph-stable buffer even for short prompts.
if self.spec_swap.enabled or alloc_size == self.device_buffer_size:
alloc_size = self.padded_buffer_size
compressed_logical_indices = (
@@ -547,6 +966,8 @@ class HiSparseCoordinator:
self.req_device_buffer_token_locs[:, req.kv.req_pool_idx, :alloc_size] = (
buffer_indices[:alloc_size]
)
self.spec_swap.allocate_scratch(req.kv.req_pool_idx)
self.spec_swap.reset(req.kv.req_pool_idx)
def _grow_device_buffers(
self,
@@ -653,6 +1074,14 @@ class HiSparseCoordinator:
ready_reqs.append(req)
return ready_reqs
def prepare_spec_verify(self, batch: ScheduleBatch) -> None:
self.spec_swap.prepare_verify(batch)
def commit_spec_accept_tokens(
self, batch: ScheduleBatch, accept_indices: torch.Tensor
) -> None:
self.spec_swap.commit_accept_tokens(batch, accept_indices)
def map_last_loc_to_buffer(
self,
seq_lens: torch.Tensor,
@@ -939,6 +1368,7 @@ class HiSparseCoordinator:
self.req_to_host_pool[req.kv.req_pool_idx, :] = -1
self.req_to_host_pool_allocated_len[req.kv.req_pool_idx] = 0
self._skip_first_backup[req.kv.req_pool_idx] = False
self.spec_swap.free_unrotated_scratch(req.kv.req_pool_idx)
req.hisparse_staging = False
def retract_req(self, req: Req) -> None:
@@ -960,22 +1390,38 @@ class HiSparseCoordinator:
# subsequent release_kv_cache -> allocator.free -> free_hisparse path
# re-frees them (double-free into the page allocator's free list).
allocated_len = req.kv.kv_allocated_len
req_pool_idx = req.kv.req_pool_idx
allocated_locs = self.req_to_token_pool.req_to_token[
req_pool_idx, :allocated_len
]
# release memory -- only free actually-allocated buffer indices
current_cap = int(self.req_device_buffer_size[req.kv.req_pool_idx])
# The spec commit kernel rotates physical locations between each
# layer's persistent cache and scratch. Their current union is the
# request's allocation ownership; the original side-buffer row is no
# longer authoritative after the first rotation.
current_cap = int(self.req_device_buffer_size[req_pool_idx])
if current_cap > 0:
side_buf_hi = self.req_to_device_buffer[req.kv.req_pool_idx, :current_cap]
all_hi = torch.unique(side_buf_hi[side_buf_hi > 0])
owned_locs = self.req_device_buffer_token_locs[
:, req_pool_idx, :current_cap
].reshape(-1)
owned_locs = self.spec_swap.extend_owned_locs(req_pool_idx, owned_locs)
all_hi = torch.unique(owned_locs[owned_locs > 0])
if all_hi.numel() > 0:
self.token_to_kv_pool_allocator.free_hisparse_indices(all_hi)
allocated_locs = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, :allocated_len
]
compressed_locs = self.mem_pool_device.translate_loc_from_full_to_compressed(
allocated_locs
)
self.mem_pool_device.full_to_hisparse_device_index_mapping[compressed_locs] = 0
compressed_locs = (
self.mem_pool_device.translate_loc_from_full_to_compressed(
allocated_locs
)
)
self.mem_pool_device.full_to_hisparse_device_index_mapping[
compressed_locs
] = 0
else:
# The request can finish or be aborted during prefill, before a
# device side buffer exists. Its ordinary prefill mapping is still
# owned by release_kv_cache; only release our separate workspace.
self.spec_swap.free_unrotated_scratch(req_pool_idx)
host_indices = self.mem_pool_host.allocated_host_indices(
self.req_to_host_pool,
@@ -993,6 +1439,7 @@ class HiSparseCoordinator:
self.req_to_host_pool[req.kv.req_pool_idx, :] = -1
self.req_to_host_pool_allocated_len[req.kv.req_pool_idx] = 0
self.lru_slots[:, req.kv.req_pool_idx, :].copy_(self._lru_init)
self.spec_swap.clear_scratch(req_pool_idx)
self._skip_first_backup[req.kv.req_pool_idx] = False
def _run_swap_in_kernel(
@@ -1009,12 +1456,6 @@ class HiSparseCoordinator:
miss plan into self._miss_{src,dst,count} for the skip layers to replay.
"""
num_reqs = req_pool_indices.size(0)
top_k_indices = self.top_k_device_locs_buffer[:num_reqs, : self.top_k]
swap_in_fn = (
load_cache_to_device_buffer_dsv4_mla
if self.is_dsv4_hisparse
else load_cache_to_device_buffer_mla
)
plan = (
dict(
miss_src=self._miss_src[:num_reqs],
@@ -1024,6 +1465,24 @@ class HiSparseCoordinator:
if record_plan
else {}
)
if top_k_result.ndim == 3:
return self.spec_swap.swap_in(
req_pool_indices,
compressed_seq_lens,
top_k_result,
layer_id,
**plan,
)
if top_k_result.ndim != 2:
raise ValueError("HiSparse top-k must be two- or three-dimensional.")
top_k_indices = self.top_k_device_locs_buffer[:num_reqs, : self.top_k]
swap_in_fn = (
load_cache_to_device_buffer_dsv4_mla
if self.is_dsv4_hisparse
else load_cache_to_device_buffer_mla
)
skip_io_kwargs = {} if _is_xpu else dict(skip_io=self.skip_io)
swap_in_fn(
top_k_tokens=top_k_result,
@@ -1129,6 +1588,8 @@ class HiSparseCoordinator:
# applies (shared index + lockstep buffers).
slot = self._prefetch_slot[layer_id]
self._prefetch_events[slot].wait(device_module.current_stream())
if top_k_result.ndim == 3:
return self.spec_swap.output_locs(top_k_result, num_reqs)
return self.top_k_device_locs_buffer[:num_reqs, : self.top_k]
# Anchor: swap in synchronously (recording the plan), then prefetch the
+32 -4
View File
@@ -399,6 +399,26 @@ class FutureMap:
device=self.device,
)
def make_staged_spec_input(self, future_indices: torch.Tensor):
"""Build the relay handle for a request leaving HiSparse staging."""
if not (self.spec_algo.is_eagle() or self.spec_algo.is_standalone()):
raise RuntimeError(
"HiSparse staging only supports EAGLE-family speculative inputs."
)
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode
from sglang.srt.speculative.eagle_info import EagleDraftInput
return EagleDraftInput(
capture_hidden_mode=(
CaptureHiddenMode.NULL
if self.spec_algo.is_standalone()
else CaptureHiddenMode.LAST
),
future_indices=future_indices,
future_dsa_topk_indices_available=(self.dsa_topk_indices_buf is not None),
)
def resolve_confidence_cpu(
self, batch: ScheduleBatch
) -> Optional[ResolvedConfidence]:
@@ -430,7 +450,8 @@ class FutureMap:
return
# FIXME: indices = batch.req_pool_indices, pinned 2 iters via
# record_batch_in_overlap; record_stream here is redundant.
indices.record_stream(torch.get_device_module(self.device).current_stream())
if indices.device.type != "cpu":
indices.record_stream(torch.get_device_module(self.device).current_stream())
if self.need_topk:
hidden_states_buf = (
self.hidden_states_buf if self.need_hidden_states else None
@@ -450,7 +471,10 @@ class FutureMap:
draft_input.bonus_tokens = bonus_tokens
if hidden_states is not None:
draft_input.hidden_states = hidden_states
if self.draft_probs_buf is not None and draft_input.draft_probs is not None:
# A staged HiSparse request carries only future_indices. The
# rejection-sampling probabilities live in this relay buffer, so
# checking the empty draft_input would silently drop them.
if self.draft_probs_buf is not None:
draft_input.draft_probs = self.draft_probs_buf[indices]
else:
draft_input.bonus_tokens = self.output_tokens_buf[indices]
@@ -615,12 +639,16 @@ class FutureMap:
self.output_tokens_buf[indices] = payload.bonus_tokens.to(
self.output_tokens_buf.dtype
)
if self.need_topk:
# Staging transitions refresh only the bonus token. Preserve top-k and
# hidden state already published by the preceding target forward.
if self.need_topk and payload.topk_p is not None:
assert payload.topk_index is not None
self.topk_p_buf[indices] = payload.topk_p.to(self.topk_p_buf.dtype)
self.topk_index_buf[indices] = payload.topk_index.to(
self.topk_index_buf.dtype
)
if self.need_hidden_states:
if self.need_hidden_states and payload.hidden_states is not None:
self.hidden_states_buf[indices] = payload.hidden_states.to(
self.hidden_states_buf.dtype
)
+41 -1
View File
@@ -814,6 +814,16 @@ class PrefillAdder:
slots += req.mamba_host_hit_length
return slots * self._mamba_slot_cost
def _request_slot_reserve_for_req(
self, req: Req, *, extend_input_len: Optional[int] = None
) -> int:
if extend_input_len is None:
extend_input_len = len(req.full_untruncated_fill_ids) - req.prefix_len
return self.token_to_kv_pool_allocator.request_slot_reserve(
has_req_pool_slot=req.kv.req_pool_idx is not None,
prefill_tokens=req.prefix_len + extend_input_len,
)
def ceil_paged_tokens(self, tokens: int) -> int:
return -(-tokens // self.page_size) * self.page_size
@@ -845,6 +855,7 @@ class PrefillAdder:
max_new_tokens: int,
retracted_stain: bool,
mamba_gap_reserve: int = 0,
request_slot_reserve: int = 0,
is_chunked_continuation: bool = False,
compute_charge: Optional[int] = None,
):
@@ -870,7 +881,10 @@ class PrefillAdder:
max_new_tokens,
# reserve() already charges one page; add the remaining shard pages.
extra_tokens=(
mamba_gap_reserve + self.per_req_token_overhead - self.page_size
mamba_gap_reserve
+ request_slot_reserve
+ self.per_req_token_overhead
- self.page_size
),
chunk_limit=self.rem_chunk_tokens,
is_chunked_continuation=is_chunked_continuation,
@@ -1118,6 +1132,20 @@ class PrefillAdder:
return req
truncated = cand_extend_input_len > _rem_tokens
new_len = min(cand_extend_input_len, _rem_tokens)
request_slot_reserve = self._request_slot_reserve_for_req(
req, extend_input_len=new_len
)
if request_slot_reserve and not self.memory_budget.can_allocate_prefill(
paged_input=(
self.ceil_paged_tokens(new_len)
+ self.per_req_token_overhead
+ request_slot_reserve
),
extend_input_len=new_len,
max_new_tokens=self._swa_new_tokens(req),
chunk_limit=self.rem_chunk_tokens,
):
return req
# The continuing chunk must fit. Keep reservation outside assert for -O.
reserved = self._kv_shard_reserve_scratch(
prefix_len=req.prefix_len, extend_len=new_len
@@ -1136,6 +1164,7 @@ class PrefillAdder:
),
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
request_slot_reserve=request_slot_reserve,
is_chunked_continuation=True,
compute_charge=req.extend_len if self.exact_chunk_fill else None,
)
@@ -1168,6 +1197,7 @@ class PrefillAdder:
# Shared Mamba pool: fold the new mamba state's shared-gap cost into the
# budget gate so admission can't over-commit (0 for baseline / non-Mamba).
paged_input += self._mamba_gap_budget_for_req(req)
paged_input += self._request_slot_reserve_for_req(req)
fits = self.memory_budget.can_allocate_prefill(
paged_input=paged_input,
extend_input_len=cand_extend_input_len,
@@ -1268,6 +1298,7 @@ class PrefillAdder:
min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS),
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
request_slot_reserve=self._request_slot_reserve_for_req(req),
compute_charge=(req.extend_len if self.exact_chunk_fill else None),
)
else:
@@ -1294,6 +1325,9 @@ class PrefillAdder:
0,
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
request_slot_reserve=self._request_slot_reserve_for_req(
req, extend_input_len=trunc_len
),
compute_charge=trunc_len if self.exact_chunk_fill else None,
)
@@ -1330,6 +1364,8 @@ class PrefillAdder:
):
self.tree_cache.buffer_pipeline.defer_staged_admission(req, "mamba")
return AddReqResult.NO_TOKEN
request_slot_reserve = self._request_slot_reserve_for_req(req)
total_tokens += request_slot_reserve
# The temporary pin excludes this prefix from the evictable budget.
# Selection itself neither allocates slots nor materializes host hits.
@@ -1562,6 +1598,9 @@ class PrefillAdder:
admission.max_new_tokens,
req.retracted_stain,
mamba_gap_reserve=mamba_gap_reserve,
request_slot_reserve=self._request_slot_reserve_for_req(
req, extend_input_len=admission.extend_len
),
# Compute budgets are billed forward-pass tokens under exact-chunk-fill.
compute_charge=admission.extend_len if self.exact_chunk_fill else None,
)
@@ -1600,6 +1639,7 @@ class PrefillAdder:
len(req.full_untruncated_fill_ids)
- req.prefix_len
+ min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS)
+ self._request_slot_reserve_for_req(req)
- self.rem_total_tokens
)
for running_req in sorted_valid_running_reqs:
+6
View File
@@ -3725,6 +3725,11 @@ class Scheduler(
)
batch.input_ids = None
if not batch.spec_algorithm.is_none():
batch.spec_info = self.future_map.make_staged_spec_input(
batch.req_pool_indices
)
if batch.return_logprob:
batch.top_logprobs_nums = [r.logprob.top_logprobs_num for r in reqs]
batch.token_ids_logprobs = [list(r.origin_input_ids) for r in reqs]
@@ -4210,6 +4215,7 @@ class Scheduler(
new_batch.contains_last_prefill_chunk = (
self.chunked_req is None or len(can_run_list) != 1
)
new_batch.hisparse_coordinator = self.hisparse_coordinator
if self.enable_hierarchical_cache or self.enable_unified_cache_external_linker:
# todo (zhiqiang): disable cuda graph execution if hicache loading triggered
@@ -380,6 +380,8 @@ class SchedulerBatchResultProcessor:
if discard_finish_reason is None:
self._maybe_collect_routed_experts(req)
self._maybe_collect_indexer_topk(req)
if get_memory().enable_hisparse:
self.hisparse_coordinator.request_finished(req)
if batch.decoding_reqs and req in batch.decoding_reqs:
self._prepare_kv_cache_release(req)
release_kv_cache(
+40 -11
View File
@@ -403,6 +403,11 @@ def alloc_for_extend(
)
req_pool_indices_device = req_pool_indices_cpu.to(batch.device, non_blocking=True)
# Admission already charged HiSparse scratch; allocate it with the request
# slot so subsequent scheduling sees the remaining physical capacity.
if batch.hisparse_coordinator is not None:
batch.hisparse_coordinator.allocate_spec_scratch(req_pool_indices)
# Allocate KV cache (throws exception on failure)
alloc_page_size = _alloc_page_size(batch)
if reuse_kv is not None and any(reuse_kv):
@@ -785,17 +790,41 @@ def alloc_for_spec_decode(
last_loc = get_last_loc(
req_to_token_pool.req_to_token, req_pool_indices, cur_kv_lens
)
out_cache_loc = ALLOC_EXTEND_FUNCS[_device_type(batch)](
tree_cache,
cur_kv_lens,
cur_kv_lens_cpu,
nxt_kv_lens,
nxt_kv_lens_cpu,
last_loc,
num_needed_tokens,
req_pool_indices=req_pool_indices,
batch=batch,
)
allocator = tree_cache.token_to_kv_pool_allocator
if batch is not None and batch.hisparse_coordinator is not None:
# Spec decode reserves logical target slots here. Target verify
# later binds those slots to HiSparse's extra page; allocating
# target physical pages first both wastes memory and breaks the
# physical paged allocator's last_loc chain after the reserve is
# released. Draft KV has its own logical GPU-page mapping.
evict_from_tree_cache(
tree_cache,
num_needed_tokens + len(nxt_kv_lens_cpu) * allocator.page_size,
)
out_cache_loc = allocator.alloc_logical_only(
cur_kv_lens,
cur_kv_lens_cpu,
nxt_kv_lens,
nxt_kv_lens_cpu,
last_loc,
num_needed_tokens,
)
if out_cache_loc is None:
raise RuntimeError(
"HiSparse spec decode failed to allocate logical KV slots."
)
else:
out_cache_loc = ALLOC_EXTEND_FUNCS[_device_type(batch)](
tree_cache,
cur_kv_lens,
cur_kv_lens_cpu,
nxt_kv_lens,
nxt_kv_lens_cpu,
last_loc,
num_needed_tokens,
req_pool_indices=req_pool_indices,
batch=batch,
)
# Updating req_to_token is a write to a shared tensor: it must not overlap
# with the previous batch's forward, which also reads req_to_token.
assign_req_to_token_pool_func(
@@ -76,6 +76,15 @@ class BaseTokenToKVPoolAllocator(abc.ABC):
self, tree_cache, num_mixed_decode_tokens=num_mixed_decode_tokens
)
def request_slot_reserve(
self, *, has_req_pool_slot: bool, prefill_tokens: int = 0
) -> int:
"""Extra KV slots beyond prefill, including request-owned workspace.
``prefill_tokens`` includes the cached prefix and selected extension.
"""
return 0
def max_new_tokens_for_memory(
self,
input_tokens: int,
@@ -38,6 +38,8 @@ class HiSparseTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
self.device = device
self.page_size = page_size
self.need_sort = need_sort
self._spec_scratch_capacity = 0
self._spec_device_buffer_size = 0
paged_allocator_cls = (
current_platform.get_paged_allocator_cls() or PagedTokenToKVPoolAllocator
@@ -77,6 +79,29 @@ class HiSparseTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
weakref.proxy(self.full_to_hisparse_device_index_mapping)
)
def configure_spec_scratch(
self, capacity: int, *, device_buffer_size: int = 0
) -> None:
if any(
size < 0 or size % self.page_size for size in (capacity, device_buffer_size)
):
raise ValueError(
"HiSparse spec scratch and device buffer sizes must be "
f"non-negative multiples of page_size={self.page_size}."
)
self._spec_scratch_capacity = capacity
self._spec_device_buffer_size = device_buffer_size if capacity else 0
def request_slot_reserve(
self, *, has_req_pool_slot: bool, prefill_tokens: int = 0
) -> int:
paged_prefill = -(-prefill_tokens // self.page_size) * self.page_size
# Prefill KV is reused by the persistent buffer. Charge only its
# missing slots, including on chunks whose scratch was already allocated.
buffer_reserve = max(self._spec_device_buffer_size - paged_prefill, 0)
scratch_reserve = 0 if has_req_pool_slot else self._spec_scratch_capacity
return buffer_reserve + scratch_reserve
@property
def size_full(self) -> int:
return self._size_full
@@ -169,7 +194,10 @@ class HiSparseTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
return buffer_indices
def free_hisparse_indices(self, buffer_indices: torch.Tensor):
self.hisparse_attn_allocator.free(buffer_indices[buffer_indices > 0])
# Page zero is the padding sink and is never owned by a request.
self.hisparse_attn_allocator.free(
buffer_indices[buffer_indices >= self.page_size]
)
def get_last_loc_compressed(self, last_locs: torch.Tensor):
return last_locs
@@ -514,7 +542,9 @@ class DeepSeekV4HiSparseTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
return buffer_indices
def free_hisparse_indices(self, buffer_indices: torch.Tensor):
self.hisparse_attn_allocator.free(buffer_indices[buffer_indices > 0])
self.hisparse_attn_allocator.free(
buffer_indices[buffer_indices >= self.hisparse_page_size]
)
def get_last_loc_compressed(self, last_locs: torch.Tensor):
# Last complete C4 block of a prefix of last_loc + 1 tokens; -1 stays -1.
@@ -5,6 +5,7 @@ from typing import Optional
import torch
from sglang.kernels.ops.kvcache.cache_move import copy_all_layer_kv_cache_func
from sglang.kernels.ops.kvcache.hisparse_slot_mapping import (
translate_padded_hisparse_locations,
)
@@ -76,6 +77,7 @@ class HiSparseDSATokenToKVPool(DSATokenToKVPool):
skip_topk_layers=skip_topk_layers,
)
self.bytes_per_token = self.kv_cache_dim * self.dtype.itemsize
self.data_strides = torch.full_like(self.data_ptrs, self.bytes_per_token)
def register_mapping(self, full_to_hisparse_device_index_mapping: torch.Tensor):
self.full_to_hisparse_device_index_mapping = (
@@ -136,6 +138,30 @@ class HiSparseDSATokenToKVPool(DSATokenToKVPool):
loc = self.translate_loc_to_hisparse_device(loc)
return super().get_mla_kv_buffer(layer, loc, dst_dtype)
def transfer_values_on_device(
self, dst_indices: torch.Tensor, src_indices: torch.Tensor
) -> None:
"""Copy target MLA KV between physical slots, without remapping IDs."""
if src_indices.numel() == 0:
return
config = {
"bytes_per_tile": 128,
"byte_tiles": (self.bytes_per_token + 127) // 128,
"num_warps": 4,
}
for start in range(0, src_indices.numel(), 256):
src = src_indices[start : start + 256].contiguous()
dst = dst_indices[start : start + 256].contiguous()
copy_all_layer_kv_cache_func(
self.data_ptrs,
self.data_strides,
dst,
src,
src.numel(),
1 << (src.numel() - 1).bit_length(),
config,
)
def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None):
raise NotImplementedError("HiSparseDevicePool does not support get_cpu_copy")
@@ -688,6 +688,18 @@ class KVCacheConfigurator:
draft_virtual_id_space
)
sizes = msgspec.structs.replace(sizes, **size_overrides)
elif isinstance(token_to_kv_pool_allocator, HiSparseTokenToKVPoolAllocator):
# HiSparse's target allocator hands out logical IDs from its
# host-expanded address space. Draft KV is direct-indexed by
# those shared IDs, so its GPU pool must span the same space.
draft_virtual_id_space = token_to_kv_pool_allocator.size_full
page = max(int(self.pool_page_size or 1), 1)
draft_virtual_id_space = (
(draft_virtual_id_space + page - 1) // page * page
)
sizes = msgspec.structs.replace(
sizes, max_total_num_tokens=draft_virtual_id_space
)
# Initialize req_to_token_pool
if req_to_token_pool is None:
@@ -2018,7 +2030,7 @@ class KVCacheConfigurator:
dsa_cp_layer_shard_size,
) = get_glm_dsa_cp_layer_shard_info(self)
pool_kwargs = {}
if get_memory().enable_hisparse:
if get_memory().enable_hisparse and not self.is_draft_worker:
reject_out_of_tree_path(
current_platform, subsystem="the HiSparse DSA KV pool"
)
@@ -396,7 +396,9 @@ class ModelRunner:
self._pending_elastic_scale_update = None
self.init_new_workspace = False
self.draft_model_idx = draft_model_idx
self.enable_hisparse = get_memory().enable_hisparse
# HiSparse owns target KV only. Draft KV keeps its logical GPU page
# space and must not construct a second sparse coordinator.
self.enable_hisparse = get_memory().enable_hisparse and not is_draft_worker
self._sampling_observer: Optional[SamplingObserver] = None
self.sampling_prewarm_result = SamplingPrewarmResult()
@@ -1026,7 +1028,11 @@ class ModelRunner:
swap_in_block_size=hisparse_cfg.swap_in_block_size,
shared_index_layers=resolve_shared_index_layers(
hf_text_config=self.model_config.hf_text_config,
is_speculative=self.spec_algorithm.is_speculative(),
),
num_draft_tokens=(
0
if self.spec_algorithm.is_none()
else get_spec().speculative_num_draft_tokens or 0
),
)
@@ -265,6 +265,14 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
and int(num_layers) > 0
):
draft_num_layers = int(eagle_draft_num_layers)
draft_loc_space_scale = 1
if get_memory().enable_hisparse:
from sglang.srt.mem_cache.sparsity import parse_hisparse_config
# The draft pool is direct-indexed by the target allocator's
# host-expanded logical IDs. Account for the entire dense
# address space that KVCacheConfigurator allocates below.
draft_loc_space_scale = parse_hisparse_config().host_to_device_ratio
if is_deepseek_dsa(kvc.model_config.hf_config):
target_indexer_size = self._compute_dsa_indexer_cell_size(
kvc=kvc,
@@ -279,9 +287,12 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
target_kv_num_layers = get_glm_dsa_layer_split_effective_num_layers(
kvc, num_layers
)
draft_kv_size = int(
target_kv_size * draft_num_layers / target_kv_num_layers
draft_kv_size = (
int(target_kv_size * draft_num_layers / target_kv_num_layers)
* draft_loc_space_scale
)
# _compute_dsa_indexer_cell_size already applies the
# HiSparse logical-space ratio to the draft indexer.
draft_indexer_size = self._compute_dsa_indexer_cell_size(
kvc=kvc,
num_layers=draft_num_layers,
@@ -291,9 +302,10 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
draft_indexer_size *= dcp_size
self._cell_size += draft_kv_size + draft_indexer_size
else:
self._cell_size = int(
self._cell_size * (1 + draft_num_layers / int(num_layers))
draft_kv_size = int(
self._cell_size * draft_num_layers / int(num_layers)
)
self._cell_size += draft_kv_size * draft_loc_space_scale
# DFLASH/DSPARK: the draft's per-token KV cost can differ from the target's
# (e.g. MLA target, per-head K/V draft), so size from the draft config.
@@ -580,6 +580,8 @@ def eagle_prepare_for_verify(
draft_token_num=verify_input.draft_token_num,
device=device,
)
if batch.hisparse_coordinator is not None:
batch.hisparse_coordinator.prepare_spec_verify(batch)
batch.out_cache_loc_dsv4 = maybe_build_dsv4_verify_bundle(
batch, verify_input.draft_token_num
@@ -732,6 +732,11 @@ def run_eagle_verify(
)
else:
new_seq_lens = batch.seq_lens + accept_lens
if batch.hisparse_coordinator is not None:
batch.hisparse_coordinator.commit_spec_accept_tokens(
batch=batch,
accept_indices=accept_index,
)
clear_unaccepted_c128 = getattr(
token_to_kv_pool_allocator.get_kvcache(),
"clear_unaccepted_c128_draft_states",