diff --git a/python/sglang/srt/arg_groups/hisparse_hook.py b/python/sglang/srt/arg_groups/hisparse_hook.py index 34ad4e51c5..38ddd24ca1 100644 --- a/python/sglang/srt/arg_groups/hisparse_hook.py +++ b/python/sglang/srt/arg_groups/hisparse_hook.py @@ -79,10 +79,7 @@ def validate_hisparse_kv_cache_dtype(server_args: ServerArgs) -> None: def validate_hisparse_feature_combos(server_args: ServerArgs) -> None: cfg = resolving_view(server_args) - # Both crash on the first request: the draft pool has no HiSparse mapping, - # and a mixed batch cannot merge the HiSparse decode batch. - if cfg.speculative_algorithm is not None: - raise ValueError("--enable-hisparse does not support speculative decoding yet.") + # A mixed batch cannot merge the HiSparse decode batch. if cfg.enable_mixed_chunk: raise ValueError("--enable-hisparse does not support --enable-mixed-chunk yet.") @@ -114,6 +111,21 @@ def validate_hisparse(server_args: ServerArgs) -> None: ) validate_hisparse_feature_combos(server_args) + enable_spec = ( + cfg.speculative_algorithm is not None + and cfg.speculative_algorithm.upper() != "NONE" + ) + if enable_spec and (cfg.speculative_eagle_topk or 1) != 1: + raise ValueError( + "The minimal HiSparse spec integration supports linear speculation " + "only; use --speculative-eagle-topk 1." + ) + if enable_spec and cfg.speculative_adaptive: + raise ValueError( + "HiSparse speculative decoding uses statically sized swap state; " + "--speculative-adaptive is not supported." + ) + # DSv4 hisparse handles its own dtype/backend pairing elsewhere; the dtype- # aware checks below only apply to the DSA hisparse path. if is_hip and is_v4_hisparse: diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index ee137d1e3b..a6834df2c5 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -1688,8 +1688,16 @@ class CommonKVManager(BaseKVManager): # Regular MLA PP slicing # Decode pp size should be equal to prefill pp size or 1 - start_layer, end_layer = self._mla_kv_entry_span_with_pp(len(src_kv_ptrs)) + num_draft = ( + getattr(self.kv_args, "num_draft_entries", 0) if state_type is None else 0 + ) + start_layer, end_layer = self._mla_kv_entry_span_with_pp( + len(src_kv_ptrs) - num_draft + ) sliced_dst_kv_ptrs = dst_kv_ptrs[start_layer:end_layer] + if num_draft: + # Draft entries follow the target's global layer range on both peers. + sliced_dst_kv_ptrs += dst_kv_ptrs[-num_draft:] return src_kv_ptrs, sliced_dst_kv_ptrs, len(src_kv_ptrs) def _mla_kv_entry_span_with_pp(self, n_src: int) -> Tuple[int, int]: diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 2430d9f3a7..d6621f7995 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -1382,17 +1382,21 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): ] # HiSparse physical constraint: max requests by device buffer capacity. - # Each admitted req needs padded_buffer_size from hisparse device pool. + # Each admitted req needs a padded buffer plus its speculative scratch. # waiting_queue reqs already have device buffers (allocated in admit_request_direct), # only transfer_queue reqs are pending device buffer allocation. hisparse_req_budget = float("inf") if self.scheduler.enable_hisparse: + coordinator = self.scheduler.hisparse_coordinator + per_req_device_tokens = ( + coordinator.padded_buffer_size + coordinator.spec_swap.scratch_capacity + ) hisparse_avail = ( self.token_to_kv_pool_allocator.hisparse_attn_allocator.available_size() ) hisparse_req_budget = max( 0, - hisparse_avail // self.scheduler.hisparse_coordinator.padded_buffer_size + hisparse_avail // per_req_device_tokens - len(self.transfer_queue.queue), ) @@ -1725,13 +1729,16 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): device_page_indices = None if ( self.scheduler.enable_hisparse - and isinstance(self.token_to_kv_pool, DeepSeekV4TokenToKVPool) + and ( + isinstance(self.token_to_kv_pool, DeepSeekV4TokenToKVPool) + or self.draft_token_to_kv_pool is not None + ) and not _is_fake_transfer(decode_req.req) ): # alloc_logical_only() already allocated the shared logical pages - # used by C4 indexer and C128 KV. These device buffers do not use - # the C4 sparse physical-slot mapping; carry their logical page IDs - # alongside the independently allocated C4 host page IDs. + # used by the indexer and draft KV. These device buffers do not + # use target sparse physical slots; carry logical page IDs beside + # the independently allocated target host page IDs. full_kv_indices = self.req_to_token_pool.req_to_token[ decode_req.req.kv.req_pool_idx, prefix_len:origin_input_len, @@ -1742,8 +1749,8 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): ).astype(np.int32) if self.transfer_backend != TransferBackend.MOONCAKE: raise NotImplementedError( - "DSV4 HiSparse direct PD transfer currently requires " - "the Mooncake backend" + "HiSparse device-only PD transfer currently requires the " + "Mooncake backend" ) metadata_kwargs = {"decode_prefix_len": total_prefix_len} if device_page_indices is not None: diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index ab7bf2aefc..04714abf24 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -1320,14 +1320,24 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager): dst_device_kv_ptrs = None if dst_device_kv_indices is not None: compression_ratios = self.kv_args.mla_compression_ratios - assert compression_ratios is not None - if len(dst_kv_ptrs) == len(self.kv_args.kv_data_ptrs): - start = self.kv_args.prefill_start_layer - end = self.kv_args.prefill_end_layer - assert end is not None - compression_ratios = compression_ratios[start:end] - c4_layer_num = sum(ratio == 4 for ratio in compression_ratios) - dst_device_kv_ptrs = set(dst_kv_ptrs[c4_layer_num:]) + if compression_ratios is not None: + if len(dst_kv_ptrs) == len(self.kv_args.kv_data_ptrs): + start = self.kv_args.prefill_start_layer + end = self.kv_args.prefill_end_layer + assert end is not None + compression_ratios = compression_ratios[start:end] + host_backed_kv_count = sum(ratio == 4 for ratio in compression_ratios) + else: + # The remote target prefix is host-backed, even across PP stages; + # only the draft suffix uses logical device page IDs. + num_draft_entries = self.kv_args.num_draft_entries + if not 0 < num_draft_entries < len(dst_kv_ptrs): + raise ValueError( + "HiSparse spec PD transfer requires an appended device-only " + "draft KV buffer" + ) + host_backed_kv_count = len(dst_kv_ptrs) - num_draft_entries + dst_device_kv_ptrs = set(dst_kv_ptrs[host_backed_kv_count:]) return self._send_kvcache_generic( mooncake_session_id=mooncake_session_id, src_data_ptrs=self.kv_args.kv_data_ptrs, diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py index d81753d910..b8392091b6 100644 --- a/python/sglang/srt/layers/attention/dsa_backend.py +++ b/python/sglang/srt/layers/attention/dsa_backend.py @@ -2262,10 +2262,23 @@ class DeepseekSparseAttnBackend( # todo hisparse: to cover more backends if self.hisparse_coordinator is not None: - # flash_mla_sparse_fwd / tilelang require int32 page indices. - page_table_1 = self.token_to_kv_pool.translate_loc_to_hisparse_device( - page_table_1 - ).to(torch.int32) + if forward_batch.forward_mode.is_target_verify(): + num_reqs = forward_batch.req_pool_indices.shape[0] + num_steps = self.speculative_num_draft_tokens + assert topk_indices is not None + grouped_topk_indices = topk_indices.view(num_reqs, num_steps, -1) + assert metadata.dsa_seqlens_expanded is not None + page_table_1 = self.hisparse_coordinator.swap_in_selected_pages( + forward_batch.req_pool_indices, + metadata.dsa_seqlens_expanded, + grouped_topk_indices, + layer.layer_id, + ).view(num_reqs * num_steps, -1) + else: + # flash_mla_sparse_fwd / tilelang require int32 page indices. + page_table_1 = self.token_to_kv_pool.translate_loc_to_hisparse_device( + page_table_1 + ).to(torch.int32) if dsa_impl == "tilelang": if q_rope is not None: @@ -4019,7 +4032,10 @@ class DeepseekSparseAttnBackend( ) -> DSAIndexerMetadata: force_unfused = not self.use_fused_topk or ( self.hisparse_coordinator is not None - and forward_batch.forward_mode.is_decode_or_idle() + and ( + forward_batch.forward_mode.is_decode_or_idle() + or forward_batch.forward_mode.is_target_verify() + ) ) return DSAIndexerMetadata( attn_metadata=self.forward_metadata, diff --git a/python/sglang/srt/managers/hisparse_coordinator.py b/python/sglang/srt/managers/hisparse_coordinator.py index 4f65dc999a..ecf25fb72e 100644 --- a/python/sglang/srt/managers/hisparse_coordinator.py +++ b/python/sglang/srt/managers/hisparse_coordinator.py @@ -1,7 +1,8 @@ -# to be combined with the sparse coordinator class and sparse algorithm family +from __future__ import annotations import logging -from typing import Dict, List, NamedTuple, Optional, Tuple, Union +import weakref +from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional, Tuple, Union import torch @@ -27,10 +28,12 @@ if is_xpu(): else: from sglang.kernels.ops.kvcache.hisparse import ( + HiSparseSpecState, copy_cache_planned_mla, load_blocks_to_device_buffer_mha, load_cache_to_device_buffer_dsv4_mla, load_cache_to_device_buffer_mla, + load_cache_to_device_buffer_spec_mla, ) from sglang.srt.configs.model_config import dsa_layer_skips_topk, is_deepseek_dsa @@ -49,6 +52,10 @@ from sglang.srt.mem_cache.pool_host.mha import HiSparseMHATokenToKVPoolHost from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost from sglang.srt.runtime_context import get_parallel +if TYPE_CHECKING: + from sglang.kernels.ops.kvcache.hisparse import HiSparseSpecState + from sglang.srt.managers.schedule_batch import ScheduleBatch + device_module = get_device_module() _is_hip = is_hip() @@ -73,13 +80,12 @@ class HiSparseTokenStats(NamedTuple): def resolve_shared_index_layers( *, hf_text_config, - is_speculative: bool, ) -> Optional[List[bool]]: """Per-layer "reuses the previous layer's top-k index" pattern, or None. Mirrors DeepseekV2AttentionMLA's skip_topk derivation (index_topk_pattern / index_topk_freq / cli_factor); None when the model has no sharing or the - prefetch cannot run (PP, speculative decoding, kill-switch). + prefetch cannot run (pipeline parallelism or the kill-switch). """ if not is_deepseek_dsa(hf_text_config): return None @@ -91,11 +97,12 @@ def resolve_shared_index_layers( pattern = [dsa_layer_skips_topk(hf_text_config, i) for i in range(num_layers)] if not any(pattern): return None - if get_parallel().pp_size != 1 or is_speculative: + # Speculative decoding is supported by the HiSparse multi-step swap path; + # only pipeline parallelism still prevents the shared-index prefetch plan. + if get_parallel().pp_size != 1: logger.warning( "HiSparse shared-index prefetch is unsupported under pipeline " - "parallelism / speculative decoding; falling back to synchronous " - "swap-in." + "parallelism; falling back to synchronous swap-in." ) return None if envs.SGLANG_DISABLE_HISPARSE_PREFETCH.get(): @@ -132,6 +139,397 @@ def _build_prefetch_groups( return groups, slot +class HiSparseSpecSwapManager: + """Own HiSparse speculative-swap state and request lifecycle.""" + + HASH_MULTIPLIER = 2654435761 + MIN_SCRATCH_CAPACITY = 1024 + NUM_METADATA_VALUES_PER_OCCURRENCE = 5 + NUM_COUNTERS_PER_REQUEST = 4 + + def __init__( + self, + coordinator: HiSparseCoordinator, + *, + num_draft_tokens: int, + ) -> None: + self._coordinator = weakref.proxy(coordinator) + self.enabled = num_draft_tokens > 0 + self.num_draft_tokens = num_draft_tokens + self.scratch_capacity = 0 + self._scratch_reqs: set[int] = set() + + layer_num = coordinator.mem_pool_device.layer_num + max_num_req_slots = coordinator.req_to_token_pool.req_to_token.shape[0] + self.req_to_scratch = torch.empty( + (layer_num, max_num_req_slots, 0), + dtype=torch.int32, + device=coordinator.device, + ) + self.states: tuple[HiSparseSpecState, ...] = () + self.top_k_device_locs: Optional[torch.Tensor] = None + if not self.enabled: + return + + if _is_xpu: + raise ValueError("HiSparse speculative swap is unsupported on XPU.") + if coordinator.is_dsv4_hisparse: + raise ValueError("HiSparse spec swap currently supports DSA caches only.") + if not 2 <= num_draft_tokens <= 4 or coordinator.top_k < 1024: + raise ValueError( + "HiSparse spec swap requires 2-4 draft tokens and top_k >= 1024." + ) + workspace_capacity = num_draft_tokens * coordinator.top_k + if workspace_capacity > 8192: + raise ValueError("HiSparse spec swap supports at most 8192 occurrences.") + if max_num_req_slots > coordinator.device_buffer_size: + raise ValueError( + "HiSparse spec request capacity must not exceed device_buffer_size." + ) + + self.scratch_capacity = min( + workspace_capacity, + max( + self.MIN_SCRATCH_CAPACITY, + workspace_capacity - coordinator.device_buffer_size, + ), + ) + hash_size = 1 << max(1, (2 * coordinator.device_buffer_size - 1).bit_length()) + self._cache_index = torch.full( + (layer_num, max_num_req_slots, 2, hash_size), + -1, + dtype=torch.int64, + device=coordinator.device, + ) + self._cache_policy = torch.zeros( + (layer_num, max_num_req_slots + 1, coordinator.device_buffer_size), + dtype=torch.int32, + device=coordinator.device, + ) + metadata_width = max( + self.NUM_COUNTERS_PER_REQUEST * max_num_req_slots, + self.NUM_METADATA_VALUES_PER_OCCURRENCE * workspace_capacity, + ) + self._scratch_state = torch.full( + (max_num_req_slots + 1, metadata_width), + -1, + dtype=torch.int32, + device=coordinator.device, + ) + self._scratch_state[0].zero_() + self.req_to_scratch = torch.zeros( + (layer_num, max_num_req_slots, self.scratch_capacity), + dtype=torch.int32, + device=coordinator.device, + ) + self.states = tuple( + HiSparseSpecState( + cache_index=self._cache_index[layer_id], + cache_policy=self._cache_policy[layer_id], + scratch_locs=self.req_to_scratch[layer_id], + scratch_state=self._scratch_state, + ) + for layer_id in range(layer_num) + ) + + tokens = torch.arange( + coordinator.device_buffer_size, + dtype=torch.int64, + device=coordinator.device, + ) + self._hash_slots = ((tokens * self.HASH_MULTIPLIER) & (hash_size - 1)).to( + torch.long + ) + self._hash_entries = (tokens << 32) | tokens + self.top_k_device_locs = torch.full( + (max_num_req_slots, num_draft_tokens, coordinator.top_k), + -1, + dtype=torch.int32, + device=coordinator.device, + ) + + @property + def miss_plan_capacity(self) -> int: + return self._coordinator.top_k * max(1, self.num_draft_tokens) + + def invalidate_cache(self, req_pool_idx: int) -> None: + if self.enabled: + self._cache_index[:, req_pool_idx].fill_(-1) + + def reset(self, req_pool_idx: int) -> None: + if not self.enabled: + return + self._cache_index[:, req_pool_idx].fill_(-1) + self._cache_index[:, req_pool_idx, 0, self._hash_slots] = ( + self._hash_entries.view(1, -1) + ) + self._cache_policy[:, 0, req_pool_idx].zero_() + self._cache_policy[:, req_pool_idx + 1].zero_() + max_num_req_slots = self.req_to_scratch.shape[1] + self._scratch_state[ + 0, + req_pool_idx : self.NUM_COUNTERS_PER_REQUEST + * max_num_req_slots : max_num_req_slots, + ] = 0 + self._scratch_state[req_pool_idx + 1].fill_(-1) + + def allocate_scratch(self, req_pool_idx: int) -> None: + """Reserve one request's graph-stable swap workspace at admission.""" + if not self.enabled or req_pool_idx in self._scratch_reqs: + return + allocator = self._coordinator.token_to_kv_pool_allocator + scratch_locs = allocator.hisparse_attn_allocator.alloc(self.scratch_capacity) + if scratch_locs is None: + raise RuntimeError( + "HiSparse spec failed to allocate " + f"{self.scratch_capacity} scratch slots." + ) + # Physical slots are shared by all KV layers, but every full-index + # layer rotates those IDs between persistent cache and scratch + # independently. Keep per-layer location views so one layer cannot + # overwrite the next layer's ownership state. + self.req_to_scratch[:, req_pool_idx].copy_(scratch_locs.to(torch.int32)) + self._scratch_reqs.add(req_pool_idx) + + def free_unrotated_scratch(self, req_pool_idx: int) -> None: + """Free scratch before any swap kernel has changed its ownership.""" + if not self.enabled or req_pool_idx not in self._scratch_reqs: + return + self._coordinator.token_to_kv_pool_allocator.free_hisparse_indices( + self.req_to_scratch[0, req_pool_idx].clone() + ) + self.clear_scratch(req_pool_idx) + + def clear_scratch(self, req_pool_idx: int) -> None: + if not self.enabled or req_pool_idx not in self._scratch_reqs: + return + self.req_to_scratch[:, req_pool_idx].zero_() + self._scratch_reqs.remove(req_pool_idx) + + def extend_owned_locs( + self, req_pool_idx: int, owned_locs: torch.Tensor + ) -> torch.Tensor: + if not self.enabled or req_pool_idx not in self._scratch_reqs: + return owned_locs + return torch.cat([owned_locs, self.req_to_scratch[:, req_pool_idx].reshape(-1)]) + + def prepare_verify(self, batch: ScheduleBatch) -> None: + """Bind target-verify KV writes to the side buffer's extra page.""" + coordinator = self._coordinator + req_pool_indices = batch.req_pool_indices + verify_cache_locs = batch.out_cache_loc + start_positions = batch.seq_lens + + if not self.enabled: + raise RuntimeError("HiSparse spec is not initialized.") + if self.num_draft_tokens > coordinator.page_size - 1: + raise ValueError( + f"HiSparse spec needs {self.num_draft_tokens} verify slots, but the " + f"extra page has only {coordinator.page_size - 1} usable slots." + ) + expected_slots = req_pool_indices.numel() * self.num_draft_tokens + if verify_cache_locs.numel() != expected_slots: + raise ValueError( + f"HiSparse verify slot mismatch: expected {expected_slots}, " + f"got {verify_cache_locs.numel()}." + ) + + # CPU-side logical reservations cover verify even when overlap lags seq_lens. + for req in batch.reqs: + req_pool_idx = req.kv.req_pool_idx + allocated_len = int( + coordinator.req_to_host_pool_allocated_len[req_pool_idx] + ) + if req.kv.kv_allocated_len > allocated_len: + coordinator.mem_pool_host.alloc_paged_token_slots( + coordinator.req_to_host_pool, + coordinator.req_to_host_pool_allocated_len, + req_pool_idx, + allocated_len, + req.kv.kv_allocated_len - allocated_len, + ) + + extra_start = coordinator.device_buffer_size + 1 + row_indices = torch.repeat_interleave(req_pool_indices, self.num_draft_tokens) + offsets = ( + torch.arange( + expected_slots, dtype=torch.int64, device=req_pool_indices.device + ) + % self.num_draft_tokens + ) + token_positions = ( + torch.repeat_interleave( + start_positions.to(torch.int64), self.num_draft_tokens + ) + + offsets + ) + columns = extra_start + offsets + device_locs = coordinator.req_to_device_buffer[row_indices, columns] + coordinator.req_device_buffer_tokens[:, row_indices, columns] = ( + token_positions.to(torch.int32).unsqueeze(0) + ) + coordinator.req_device_buffer_token_locs[:, row_indices, columns] = ( + device_locs.to(torch.int32).unsqueeze(0) + ) + coordinator.token_to_kv_pool_allocator.full_to_hisparse_device_index_mapping[ + verify_cache_locs + ] = device_locs + + def _backup_device_locs_to_host( + self, host_locs: torch.Tensor, device_locs: torch.Tensor + ) -> None: + if host_locs.numel() == 0: + return + coordinator = self._coordinator + coordinator.wait_for_pending_backup() + schedule_stream = device_module.current_stream() + device_locs = device_locs.contiguous() + with device_module.stream(coordinator.decode_backup_stream): + coordinator.decode_backup_stream.wait_stream(schedule_stream) + if coordinator.decode_producer_stream is not None: + coordinator.decode_backup_stream.wait_stream( + coordinator.decode_producer_stream + ) + coordinator.mem_pool_host.backup_from_device_all_layer( + coordinator.mem_pool_device, + host_locs, + device_locs, + io_backend="kernel", + ) + if host_locs.is_cuda: + host_locs.record_stream(coordinator.decode_backup_stream) + if device_locs.is_cuda: + device_locs.record_stream(coordinator.decode_backup_stream) + coordinator._backup_done_event.record() + coordinator._has_pending_backup = True + coordinator.wait_for_pending_backup() + + def commit_accept_tokens( + self, batch: ScheduleBatch, accept_indices: torch.Tensor + ) -> None: + """Persist accepted target-verify KV in host and the persistent buffer.""" + if batch.forward_mode.is_idle(): + return + coordinator = self._coordinator + req_pool_indices = batch.req_pool_indices[:, None] + verify_cache_locs = batch.out_cache_loc + + if verify_cache_locs.numel() == 0: + return + batch_size = req_pool_indices.numel() + if batch_size == 0 or verify_cache_locs.numel() % batch_size != 0: + raise ValueError("HiSparse verify cache locations are not request-aligned.") + + valid = accept_indices >= 0 + counts = valid.sum(dim=1) + steps = torch.arange(accept_indices.shape[1], device=accept_indices.device) + positions = batch.seq_lens[:, None].to(torch.int64) + steps + mapping = ( + coordinator.token_to_kv_pool_allocator.full_to_hisparse_device_index_mapping + ) + # Keep a fixed width: boolean indexing and GPU-sized repeat_interleave sync. + verify_width = verify_cache_locs.numel() // batch_size + row_offsets = ( + torch.arange(batch_size, device=accept_indices.device)[:, None] + * verify_width + ) + source_offsets = torch.where(valid, accept_indices, row_offsets + steps) + source_device_locs = mapping[verify_cache_locs[source_offsets]].clone() + canonical_locs = coordinator.req_to_token_pool.req_to_token[ + req_pool_indices, positions + ] + + # Keep hot tokens and each request's newest token in persistent slots. + cache_mask = valid & ( + (positions < coordinator.device_buffer_size) + | (steps == counts[:, None] - 1) + ) + cache_positions = torch.where( + cache_mask, + positions.clamp(max=coordinator.device_buffer_size), + coordinator.device_buffer_size + 1 + steps, + ) + cache_locs = coordinator.req_to_device_buffer[req_pool_indices, cache_positions] + destination_locs = torch.where(cache_mask, cache_locs, source_device_locs) + coordinator.mem_pool_device.transfer_values_on_device( + dst_indices=destination_locs.flatten(), + src_indices=source_device_locs.flatten(), + ) + + # Rejected padding goes to this request's reserved, not-yet-visible host slots. + host_locs = coordinator.req_to_host_pool[req_pool_indices, positions] + self._backup_device_locs_to_host( + host_locs.flatten(), destination_locs.flatten() + ) + + # Scalar indexed assignment performs a blocking CPU-to-CUDA copy. + mapping.index_fill_(0, verify_cache_locs, 0) + mapping[canonical_locs] = torch.where(cache_mask, cache_locs, 0) + coordinator.req_device_buffer_tokens[:, req_pool_indices, cache_positions] = ( + torch.where( + cache_mask, + positions.to(torch.int32), + coordinator.req_device_buffer_tokens[ + :, req_pool_indices, cache_positions + ], + ) + ) + coordinator.req_device_buffer_token_locs[ + :, req_pool_indices, cache_positions + ] = torch.where( + cache_mask, + cache_locs.to(torch.int32), + coordinator.req_device_buffer_token_locs[ + :, req_pool_indices, cache_positions + ], + ) + for req in batch.reqs: + coordinator._skip_first_backup[req.kv.req_pool_idx] = True + + def swap_in( + self, + req_pool_indices: torch.Tensor, + compressed_seq_lens: torch.Tensor, + top_k_result: torch.Tensor, + layer_id: int, + **plan, + ) -> torch.Tensor: + if not self.enabled: + raise RuntimeError("HiSparse spec swap is not initialized.") + num_reqs = req_pool_indices.size(0) + num_steps, num_top_k = top_k_result.shape[1:] + if num_steps != self.num_draft_tokens: + raise ValueError( + f"HiSparse spec step mismatch: expected {self.num_draft_tokens}, " + f"got {num_steps}." + ) + assert self.top_k_device_locs is not None + top_k_indices = self.top_k_device_locs[:num_reqs, :num_steps, :num_top_k] + coordinator = self._coordinator + load_cache_to_device_buffer_spec_mla( + top_k_tokens=top_k_result, + device_buffer_tokens=coordinator.req_device_buffer_tokens[layer_id], + host_cache_locs=coordinator.req_to_host_pool, + device_buffer_locs=coordinator.req_device_buffer_token_locs[layer_id], + host_cache=coordinator.mem_pool_host.kv_buffer[layer_id], + device_buffer=coordinator.mem_pool_device.kv_buffer[layer_id], + top_k_device_locs=top_k_indices, + req_pool_indices=req_pool_indices, + seq_lens=compressed_seq_lens, + state=self.states[layer_id], + num_real_reqs=coordinator.num_real_reqs, + **plan, + ) + return top_k_indices + + def output_locs(self, top_k_result: torch.Tensor, num_reqs: int) -> torch.Tensor: + if not self.enabled or self.top_k_device_locs is None: + raise RuntimeError("HiSparse spec swap is not initialized.") + return self.top_k_device_locs[ + :num_reqs, : top_k_result.shape[1], : top_k_result.shape[2] + ] + + class HiSparseCoordinator: def __init__( self, @@ -147,6 +545,7 @@ class HiSparseCoordinator: host_to_device_ratio: int = 2, swap_in_block_size: int = 960, shared_index_layers: Optional[List[bool]] = None, + num_draft_tokens: int = 0, ): self.req_to_token_pool = req_to_token_pool self.token_to_kv_pool_allocator = token_to_kv_pool_allocator @@ -281,6 +680,15 @@ class HiSparseCoordinator: .repeat(layer_num, max_num_req_slots, 1) .contiguous() ) + self.spec_swap = HiSparseSpecSwapManager( + self, + num_draft_tokens=num_draft_tokens, + ) + if isinstance(self.token_to_kv_pool_allocator, HiSparseTokenToKVPoolAllocator): + self.token_to_kv_pool_allocator.configure_spec_scratch( + self.spec_swap.scratch_capacity, + device_buffer_size=self.padded_buffer_size, + ) self._device_buffer_arange_i32 = torch.arange( self.device_buffer_size, dtype=torch.int32, device=device ) @@ -356,11 +764,16 @@ class HiSparseCoordinator: # Plan recorded by the current anchor, replayed by its skip layers. One # buffer set suffices: the last skip layer's event wait orders the next # anchor's writes after this group's copies. + miss_plan_capacity = self.spec_swap.miss_plan_capacity self._miss_src = torch.zeros( - (max_num_req_slots, self.top_k), dtype=torch.int64, device=self.device + (max_num_req_slots, miss_plan_capacity), + dtype=torch.int64, + device=self.device, ) self._miss_dst = torch.zeros( - (max_num_req_slots, self.top_k), dtype=torch.int32, device=self.device + (max_num_req_slots, miss_plan_capacity), + dtype=torch.int32, + device=self.device, ) self._miss_count = torch.zeros( (max_num_req_slots,), dtype=torch.int32, device=self.device @@ -376,6 +789,10 @@ class HiSparseCoordinator: def set_decode_producer_stream(self, stream) -> None: self.decode_producer_stream = stream + def allocate_spec_scratch(self, req_pool_indices: List[int]) -> None: + for req_pool_idx in req_pool_indices: + self.spec_swap.allocate_scratch(req_pool_idx) + def destroy(self) -> None: # Drain in-flight transfers so the buffer is idle, then unregister it. # See HostKVCache.destroy for why the explicit unregister matters. @@ -471,6 +888,7 @@ class HiSparseCoordinator: self.req_device_buffer_tokens[ :, req.kv.req_pool_idx, : self.device_buffer_size ] = -1 + self.spec_swap.invalidate_cache(req.kv.req_pool_idx) req.hisparse_staging = False self._skip_first_backup[req.kv.req_pool_idx] = True @@ -514,7 +932,8 @@ class HiSparseCoordinator: ((allocated_len + page_size - 1) // page_size) * page_size, self.device_buffer_size, ) - if alloc_size == self.device_buffer_size: + # Spec verify needs the complete, graph-stable buffer even for short prompts. + if self.spec_swap.enabled or alloc_size == self.device_buffer_size: alloc_size = self.padded_buffer_size compressed_logical_indices = ( @@ -547,6 +966,8 @@ class HiSparseCoordinator: self.req_device_buffer_token_locs[:, req.kv.req_pool_idx, :alloc_size] = ( buffer_indices[:alloc_size] ) + self.spec_swap.allocate_scratch(req.kv.req_pool_idx) + self.spec_swap.reset(req.kv.req_pool_idx) def _grow_device_buffers( self, @@ -653,6 +1074,14 @@ class HiSparseCoordinator: ready_reqs.append(req) return ready_reqs + def prepare_spec_verify(self, batch: ScheduleBatch) -> None: + self.spec_swap.prepare_verify(batch) + + def commit_spec_accept_tokens( + self, batch: ScheduleBatch, accept_indices: torch.Tensor + ) -> None: + self.spec_swap.commit_accept_tokens(batch, accept_indices) + def map_last_loc_to_buffer( self, seq_lens: torch.Tensor, @@ -939,6 +1368,7 @@ class HiSparseCoordinator: self.req_to_host_pool[req.kv.req_pool_idx, :] = -1 self.req_to_host_pool_allocated_len[req.kv.req_pool_idx] = 0 self._skip_first_backup[req.kv.req_pool_idx] = False + self.spec_swap.free_unrotated_scratch(req.kv.req_pool_idx) req.hisparse_staging = False def retract_req(self, req: Req) -> None: @@ -960,22 +1390,38 @@ class HiSparseCoordinator: # subsequent release_kv_cache -> allocator.free -> free_hisparse path # re-frees them (double-free into the page allocator's free list). allocated_len = req.kv.kv_allocated_len + req_pool_idx = req.kv.req_pool_idx + allocated_locs = self.req_to_token_pool.req_to_token[ + req_pool_idx, :allocated_len + ] - # release memory -- only free actually-allocated buffer indices - current_cap = int(self.req_device_buffer_size[req.kv.req_pool_idx]) + # The spec commit kernel rotates physical locations between each + # layer's persistent cache and scratch. Their current union is the + # request's allocation ownership; the original side-buffer row is no + # longer authoritative after the first rotation. + current_cap = int(self.req_device_buffer_size[req_pool_idx]) if current_cap > 0: - side_buf_hi = self.req_to_device_buffer[req.kv.req_pool_idx, :current_cap] - all_hi = torch.unique(side_buf_hi[side_buf_hi > 0]) + owned_locs = self.req_device_buffer_token_locs[ + :, req_pool_idx, :current_cap + ].reshape(-1) + owned_locs = self.spec_swap.extend_owned_locs(req_pool_idx, owned_locs) + all_hi = torch.unique(owned_locs[owned_locs > 0]) if all_hi.numel() > 0: self.token_to_kv_pool_allocator.free_hisparse_indices(all_hi) - allocated_locs = self.req_to_token_pool.req_to_token[ - req.kv.req_pool_idx, :allocated_len - ] - compressed_locs = self.mem_pool_device.translate_loc_from_full_to_compressed( - allocated_locs - ) - self.mem_pool_device.full_to_hisparse_device_index_mapping[compressed_locs] = 0 + compressed_locs = ( + self.mem_pool_device.translate_loc_from_full_to_compressed( + allocated_locs + ) + ) + self.mem_pool_device.full_to_hisparse_device_index_mapping[ + compressed_locs + ] = 0 + else: + # The request can finish or be aborted during prefill, before a + # device side buffer exists. Its ordinary prefill mapping is still + # owned by release_kv_cache; only release our separate workspace. + self.spec_swap.free_unrotated_scratch(req_pool_idx) host_indices = self.mem_pool_host.allocated_host_indices( self.req_to_host_pool, @@ -993,6 +1439,7 @@ class HiSparseCoordinator: self.req_to_host_pool[req.kv.req_pool_idx, :] = -1 self.req_to_host_pool_allocated_len[req.kv.req_pool_idx] = 0 self.lru_slots[:, req.kv.req_pool_idx, :].copy_(self._lru_init) + self.spec_swap.clear_scratch(req_pool_idx) self._skip_first_backup[req.kv.req_pool_idx] = False def _run_swap_in_kernel( @@ -1009,12 +1456,6 @@ class HiSparseCoordinator: miss plan into self._miss_{src,dst,count} for the skip layers to replay. """ num_reqs = req_pool_indices.size(0) - top_k_indices = self.top_k_device_locs_buffer[:num_reqs, : self.top_k] - swap_in_fn = ( - load_cache_to_device_buffer_dsv4_mla - if self.is_dsv4_hisparse - else load_cache_to_device_buffer_mla - ) plan = ( dict( miss_src=self._miss_src[:num_reqs], @@ -1024,6 +1465,24 @@ class HiSparseCoordinator: if record_plan else {} ) + + if top_k_result.ndim == 3: + return self.spec_swap.swap_in( + req_pool_indices, + compressed_seq_lens, + top_k_result, + layer_id, + **plan, + ) + if top_k_result.ndim != 2: + raise ValueError("HiSparse top-k must be two- or three-dimensional.") + + top_k_indices = self.top_k_device_locs_buffer[:num_reqs, : self.top_k] + swap_in_fn = ( + load_cache_to_device_buffer_dsv4_mla + if self.is_dsv4_hisparse + else load_cache_to_device_buffer_mla + ) skip_io_kwargs = {} if _is_xpu else dict(skip_io=self.skip_io) swap_in_fn( top_k_tokens=top_k_result, @@ -1129,6 +1588,8 @@ class HiSparseCoordinator: # applies (shared index + lockstep buffers). slot = self._prefetch_slot[layer_id] self._prefetch_events[slot].wait(device_module.current_stream()) + if top_k_result.ndim == 3: + return self.spec_swap.output_locs(top_k_result, num_reqs) return self.top_k_device_locs_buffer[:num_reqs, : self.top_k] # Anchor: swap in synchronously (recording the plan), then prefetch the diff --git a/python/sglang/srt/managers/overlap_utils.py b/python/sglang/srt/managers/overlap_utils.py index 4fb1981633..faf3010aba 100644 --- a/python/sglang/srt/managers/overlap_utils.py +++ b/python/sglang/srt/managers/overlap_utils.py @@ -399,6 +399,26 @@ class FutureMap: device=self.device, ) + def make_staged_spec_input(self, future_indices: torch.Tensor): + """Build the relay handle for a request leaving HiSparse staging.""" + if not (self.spec_algo.is_eagle() or self.spec_algo.is_standalone()): + raise RuntimeError( + "HiSparse staging only supports EAGLE-family speculative inputs." + ) + + from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode + from sglang.srt.speculative.eagle_info import EagleDraftInput + + return EagleDraftInput( + capture_hidden_mode=( + CaptureHiddenMode.NULL + if self.spec_algo.is_standalone() + else CaptureHiddenMode.LAST + ), + future_indices=future_indices, + future_dsa_topk_indices_available=(self.dsa_topk_indices_buf is not None), + ) + def resolve_confidence_cpu( self, batch: ScheduleBatch ) -> Optional[ResolvedConfidence]: @@ -430,7 +450,8 @@ class FutureMap: return # FIXME: indices = batch.req_pool_indices, pinned 2 iters via # record_batch_in_overlap; record_stream here is redundant. - indices.record_stream(torch.get_device_module(self.device).current_stream()) + if indices.device.type != "cpu": + indices.record_stream(torch.get_device_module(self.device).current_stream()) if self.need_topk: hidden_states_buf = ( self.hidden_states_buf if self.need_hidden_states else None @@ -450,7 +471,10 @@ class FutureMap: draft_input.bonus_tokens = bonus_tokens if hidden_states is not None: draft_input.hidden_states = hidden_states - if self.draft_probs_buf is not None and draft_input.draft_probs is not None: + # A staged HiSparse request carries only future_indices. The + # rejection-sampling probabilities live in this relay buffer, so + # checking the empty draft_input would silently drop them. + if self.draft_probs_buf is not None: draft_input.draft_probs = self.draft_probs_buf[indices] else: draft_input.bonus_tokens = self.output_tokens_buf[indices] @@ -615,12 +639,16 @@ class FutureMap: self.output_tokens_buf[indices] = payload.bonus_tokens.to( self.output_tokens_buf.dtype ) - if self.need_topk: + + # Staging transitions refresh only the bonus token. Preserve top-k and + # hidden state already published by the preceding target forward. + if self.need_topk and payload.topk_p is not None: + assert payload.topk_index is not None self.topk_p_buf[indices] = payload.topk_p.to(self.topk_p_buf.dtype) self.topk_index_buf[indices] = payload.topk_index.to( self.topk_index_buf.dtype ) - if self.need_hidden_states: + if self.need_hidden_states and payload.hidden_states is not None: self.hidden_states_buf[indices] = payload.hidden_states.to( self.hidden_states_buf.dtype ) diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index d54c4c819b..9f6d152173 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -814,6 +814,16 @@ class PrefillAdder: slots += req.mamba_host_hit_length return slots * self._mamba_slot_cost + def _request_slot_reserve_for_req( + self, req: Req, *, extend_input_len: Optional[int] = None + ) -> int: + if extend_input_len is None: + extend_input_len = len(req.full_untruncated_fill_ids) - req.prefix_len + return self.token_to_kv_pool_allocator.request_slot_reserve( + has_req_pool_slot=req.kv.req_pool_idx is not None, + prefill_tokens=req.prefix_len + extend_input_len, + ) + def ceil_paged_tokens(self, tokens: int) -> int: return -(-tokens // self.page_size) * self.page_size @@ -845,6 +855,7 @@ class PrefillAdder: max_new_tokens: int, retracted_stain: bool, mamba_gap_reserve: int = 0, + request_slot_reserve: int = 0, is_chunked_continuation: bool = False, compute_charge: Optional[int] = None, ): @@ -870,7 +881,10 @@ class PrefillAdder: max_new_tokens, # reserve() already charges one page; add the remaining shard pages. extra_tokens=( - mamba_gap_reserve + self.per_req_token_overhead - self.page_size + mamba_gap_reserve + + request_slot_reserve + + self.per_req_token_overhead + - self.page_size ), chunk_limit=self.rem_chunk_tokens, is_chunked_continuation=is_chunked_continuation, @@ -1118,6 +1132,20 @@ class PrefillAdder: return req truncated = cand_extend_input_len > _rem_tokens new_len = min(cand_extend_input_len, _rem_tokens) + request_slot_reserve = self._request_slot_reserve_for_req( + req, extend_input_len=new_len + ) + if request_slot_reserve and not self.memory_budget.can_allocate_prefill( + paged_input=( + self.ceil_paged_tokens(new_len) + + self.per_req_token_overhead + + request_slot_reserve + ), + extend_input_len=new_len, + max_new_tokens=self._swa_new_tokens(req), + chunk_limit=self.rem_chunk_tokens, + ): + return req # The continuing chunk must fit. Keep reservation outside assert for -O. reserved = self._kv_shard_reserve_scratch( prefix_len=req.prefix_len, extend_len=new_len @@ -1136,6 +1164,7 @@ class PrefillAdder: ), req.retracted_stain, mamba_gap_reserve=self._mamba_gap_budget_for_req(req), + request_slot_reserve=request_slot_reserve, is_chunked_continuation=True, compute_charge=req.extend_len if self.exact_chunk_fill else None, ) @@ -1168,6 +1197,7 @@ class PrefillAdder: # Shared Mamba pool: fold the new mamba state's shared-gap cost into the # budget gate so admission can't over-commit (0 for baseline / non-Mamba). paged_input += self._mamba_gap_budget_for_req(req) + paged_input += self._request_slot_reserve_for_req(req) fits = self.memory_budget.can_allocate_prefill( paged_input=paged_input, extend_input_len=cand_extend_input_len, @@ -1268,6 +1298,7 @@ class PrefillAdder: min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS), req.retracted_stain, mamba_gap_reserve=self._mamba_gap_budget_for_req(req), + request_slot_reserve=self._request_slot_reserve_for_req(req), compute_charge=(req.extend_len if self.exact_chunk_fill else None), ) else: @@ -1294,6 +1325,9 @@ class PrefillAdder: 0, req.retracted_stain, mamba_gap_reserve=self._mamba_gap_budget_for_req(req), + request_slot_reserve=self._request_slot_reserve_for_req( + req, extend_input_len=trunc_len + ), compute_charge=trunc_len if self.exact_chunk_fill else None, ) @@ -1330,6 +1364,8 @@ class PrefillAdder: ): self.tree_cache.buffer_pipeline.defer_staged_admission(req, "mamba") return AddReqResult.NO_TOKEN + request_slot_reserve = self._request_slot_reserve_for_req(req) + total_tokens += request_slot_reserve # The temporary pin excludes this prefix from the evictable budget. # Selection itself neither allocates slots nor materializes host hits. @@ -1562,6 +1598,9 @@ class PrefillAdder: admission.max_new_tokens, req.retracted_stain, mamba_gap_reserve=mamba_gap_reserve, + request_slot_reserve=self._request_slot_reserve_for_req( + req, extend_input_len=admission.extend_len + ), # Compute budgets are billed forward-pass tokens under exact-chunk-fill. compute_charge=admission.extend_len if self.exact_chunk_fill else None, ) @@ -1600,6 +1639,7 @@ class PrefillAdder: len(req.full_untruncated_fill_ids) - req.prefix_len + min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS) + + self._request_slot_reserve_for_req(req) - self.rem_total_tokens ) for running_req in sorted_valid_running_reqs: diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 93c50767d7..eff66f6c33 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -3725,6 +3725,11 @@ class Scheduler( ) batch.input_ids = None + if not batch.spec_algorithm.is_none(): + batch.spec_info = self.future_map.make_staged_spec_input( + batch.req_pool_indices + ) + if batch.return_logprob: batch.top_logprobs_nums = [r.logprob.top_logprobs_num for r in reqs] batch.token_ids_logprobs = [list(r.origin_input_ids) for r in reqs] @@ -4210,6 +4215,7 @@ class Scheduler( new_batch.contains_last_prefill_chunk = ( self.chunked_req is None or len(can_run_list) != 1 ) + new_batch.hisparse_coordinator = self.hisparse_coordinator if self.enable_hierarchical_cache or self.enable_unified_cache_external_linker: # todo (zhiqiang): disable cuda graph execution if hicache loading triggered diff --git a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py index d7894327d5..928e3cb410 100644 --- a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py @@ -380,6 +380,8 @@ class SchedulerBatchResultProcessor: if discard_finish_reason is None: self._maybe_collect_routed_experts(req) self._maybe_collect_indexer_topk(req) + if get_memory().enable_hisparse: + self.hisparse_coordinator.request_finished(req) if batch.decoding_reqs and req in batch.decoding_reqs: self._prepare_kv_cache_release(req) release_kv_cache( diff --git a/python/sglang/srt/mem_cache/allocation.py b/python/sglang/srt/mem_cache/allocation.py index 287e6db4ca..4a5393779c 100644 --- a/python/sglang/srt/mem_cache/allocation.py +++ b/python/sglang/srt/mem_cache/allocation.py @@ -403,6 +403,11 @@ def alloc_for_extend( ) req_pool_indices_device = req_pool_indices_cpu.to(batch.device, non_blocking=True) + # Admission already charged HiSparse scratch; allocate it with the request + # slot so subsequent scheduling sees the remaining physical capacity. + if batch.hisparse_coordinator is not None: + batch.hisparse_coordinator.allocate_spec_scratch(req_pool_indices) + # Allocate KV cache (throws exception on failure) alloc_page_size = _alloc_page_size(batch) if reuse_kv is not None and any(reuse_kv): @@ -785,17 +790,41 @@ def alloc_for_spec_decode( last_loc = get_last_loc( req_to_token_pool.req_to_token, req_pool_indices, cur_kv_lens ) - out_cache_loc = ALLOC_EXTEND_FUNCS[_device_type(batch)]( - tree_cache, - cur_kv_lens, - cur_kv_lens_cpu, - nxt_kv_lens, - nxt_kv_lens_cpu, - last_loc, - num_needed_tokens, - req_pool_indices=req_pool_indices, - batch=batch, - ) + allocator = tree_cache.token_to_kv_pool_allocator + if batch is not None and batch.hisparse_coordinator is not None: + # Spec decode reserves logical target slots here. Target verify + # later binds those slots to HiSparse's extra page; allocating + # target physical pages first both wastes memory and breaks the + # physical paged allocator's last_loc chain after the reserve is + # released. Draft KV has its own logical GPU-page mapping. + evict_from_tree_cache( + tree_cache, + num_needed_tokens + len(nxt_kv_lens_cpu) * allocator.page_size, + ) + out_cache_loc = allocator.alloc_logical_only( + cur_kv_lens, + cur_kv_lens_cpu, + nxt_kv_lens, + nxt_kv_lens_cpu, + last_loc, + num_needed_tokens, + ) + if out_cache_loc is None: + raise RuntimeError( + "HiSparse spec decode failed to allocate logical KV slots." + ) + else: + out_cache_loc = ALLOC_EXTEND_FUNCS[_device_type(batch)]( + tree_cache, + cur_kv_lens, + cur_kv_lens_cpu, + nxt_kv_lens, + nxt_kv_lens_cpu, + last_loc, + num_needed_tokens, + req_pool_indices=req_pool_indices, + batch=batch, + ) # Updating req_to_token is a write to a shared tensor: it must not overlap # with the previous batch's forward, which also reads req_to_token. assign_req_to_token_pool_func( diff --git a/python/sglang/srt/mem_cache/allocator/base.py b/python/sglang/srt/mem_cache/allocator/base.py index ab3831bb43..a5d1aca7dc 100644 --- a/python/sglang/srt/mem_cache/allocator/base.py +++ b/python/sglang/srt/mem_cache/allocator/base.py @@ -76,6 +76,15 @@ class BaseTokenToKVPoolAllocator(abc.ABC): self, tree_cache, num_mixed_decode_tokens=num_mixed_decode_tokens ) + def request_slot_reserve( + self, *, has_req_pool_slot: bool, prefill_tokens: int = 0 + ) -> int: + """Extra KV slots beyond prefill, including request-owned workspace. + + ``prefill_tokens`` includes the cached prefix and selected extension. + """ + return 0 + def max_new_tokens_for_memory( self, input_tokens: int, diff --git a/python/sglang/srt/mem_cache/allocator/hisparse.py b/python/sglang/srt/mem_cache/allocator/hisparse.py index dd59c1416a..668373d66f 100644 --- a/python/sglang/srt/mem_cache/allocator/hisparse.py +++ b/python/sglang/srt/mem_cache/allocator/hisparse.py @@ -38,6 +38,8 @@ class HiSparseTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator): self.device = device self.page_size = page_size self.need_sort = need_sort + self._spec_scratch_capacity = 0 + self._spec_device_buffer_size = 0 paged_allocator_cls = ( current_platform.get_paged_allocator_cls() or PagedTokenToKVPoolAllocator @@ -77,6 +79,29 @@ class HiSparseTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator): weakref.proxy(self.full_to_hisparse_device_index_mapping) ) + def configure_spec_scratch( + self, capacity: int, *, device_buffer_size: int = 0 + ) -> None: + if any( + size < 0 or size % self.page_size for size in (capacity, device_buffer_size) + ): + raise ValueError( + "HiSparse spec scratch and device buffer sizes must be " + f"non-negative multiples of page_size={self.page_size}." + ) + self._spec_scratch_capacity = capacity + self._spec_device_buffer_size = device_buffer_size if capacity else 0 + + def request_slot_reserve( + self, *, has_req_pool_slot: bool, prefill_tokens: int = 0 + ) -> int: + paged_prefill = -(-prefill_tokens // self.page_size) * self.page_size + # Prefill KV is reused by the persistent buffer. Charge only its + # missing slots, including on chunks whose scratch was already allocated. + buffer_reserve = max(self._spec_device_buffer_size - paged_prefill, 0) + scratch_reserve = 0 if has_req_pool_slot else self._spec_scratch_capacity + return buffer_reserve + scratch_reserve + @property def size_full(self) -> int: return self._size_full @@ -169,7 +194,10 @@ class HiSparseTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator): return buffer_indices def free_hisparse_indices(self, buffer_indices: torch.Tensor): - self.hisparse_attn_allocator.free(buffer_indices[buffer_indices > 0]) + # Page zero is the padding sink and is never owned by a request. + self.hisparse_attn_allocator.free( + buffer_indices[buffer_indices >= self.page_size] + ) def get_last_loc_compressed(self, last_locs: torch.Tensor): return last_locs @@ -514,7 +542,9 @@ class DeepSeekV4HiSparseTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator): return buffer_indices def free_hisparse_indices(self, buffer_indices: torch.Tensor): - self.hisparse_attn_allocator.free(buffer_indices[buffer_indices > 0]) + self.hisparse_attn_allocator.free( + buffer_indices[buffer_indices >= self.hisparse_page_size] + ) def get_last_loc_compressed(self, last_locs: torch.Tensor): # Last complete C4 block of a prefix of last_loc + 1 tokens; -1 stays -1. diff --git a/python/sglang/srt/mem_cache/hisparse_memory_pool.py b/python/sglang/srt/mem_cache/hisparse_memory_pool.py index f3208a6392..76278abb53 100644 --- a/python/sglang/srt/mem_cache/hisparse_memory_pool.py +++ b/python/sglang/srt/mem_cache/hisparse_memory_pool.py @@ -5,6 +5,7 @@ from typing import Optional import torch +from sglang.kernels.ops.kvcache.cache_move import copy_all_layer_kv_cache_func from sglang.kernels.ops.kvcache.hisparse_slot_mapping import ( translate_padded_hisparse_locations, ) @@ -76,6 +77,7 @@ class HiSparseDSATokenToKVPool(DSATokenToKVPool): skip_topk_layers=skip_topk_layers, ) self.bytes_per_token = self.kv_cache_dim * self.dtype.itemsize + self.data_strides = torch.full_like(self.data_ptrs, self.bytes_per_token) def register_mapping(self, full_to_hisparse_device_index_mapping: torch.Tensor): self.full_to_hisparse_device_index_mapping = ( @@ -136,6 +138,30 @@ class HiSparseDSATokenToKVPool(DSATokenToKVPool): loc = self.translate_loc_to_hisparse_device(loc) return super().get_mla_kv_buffer(layer, loc, dst_dtype) + def transfer_values_on_device( + self, dst_indices: torch.Tensor, src_indices: torch.Tensor + ) -> None: + """Copy target MLA KV between physical slots, without remapping IDs.""" + if src_indices.numel() == 0: + return + config = { + "bytes_per_tile": 128, + "byte_tiles": (self.bytes_per_token + 127) // 128, + "num_warps": 4, + } + for start in range(0, src_indices.numel(), 256): + src = src_indices[start : start + 256].contiguous() + dst = dst_indices[start : start + 256].contiguous() + copy_all_layer_kv_cache_func( + self.data_ptrs, + self.data_strides, + dst, + src, + src.numel(), + 1 << (src.numel() - 1).bit_length(), + config, + ) + def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None): raise NotImplementedError("HiSparseDevicePool does not support get_cpu_copy") diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 6fe104b842..f49dd58f7e 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -688,6 +688,18 @@ class KVCacheConfigurator: draft_virtual_id_space ) sizes = msgspec.structs.replace(sizes, **size_overrides) + elif isinstance(token_to_kv_pool_allocator, HiSparseTokenToKVPoolAllocator): + # HiSparse's target allocator hands out logical IDs from its + # host-expanded address space. Draft KV is direct-indexed by + # those shared IDs, so its GPU pool must span the same space. + draft_virtual_id_space = token_to_kv_pool_allocator.size_full + page = max(int(self.pool_page_size or 1), 1) + draft_virtual_id_space = ( + (draft_virtual_id_space + page - 1) // page * page + ) + sizes = msgspec.structs.replace( + sizes, max_total_num_tokens=draft_virtual_id_space + ) # Initialize req_to_token_pool if req_to_token_pool is None: @@ -2018,7 +2030,7 @@ class KVCacheConfigurator: dsa_cp_layer_shard_size, ) = get_glm_dsa_cp_layer_shard_info(self) pool_kwargs = {} - if get_memory().enable_hisparse: + if get_memory().enable_hisparse and not self.is_draft_worker: reject_out_of_tree_path( current_platform, subsystem="the HiSparse DSA KV pool" ) diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 9d87400b4f..e3618f516f 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -396,7 +396,9 @@ class ModelRunner: self._pending_elastic_scale_update = None self.init_new_workspace = False self.draft_model_idx = draft_model_idx - self.enable_hisparse = get_memory().enable_hisparse + # HiSparse owns target KV only. Draft KV keeps its logical GPU page + # space and must not construct a second sparse coordinator. + self.enable_hisparse = get_memory().enable_hisparse and not is_draft_worker self._sampling_observer: Optional[SamplingObserver] = None self.sampling_prewarm_result = SamplingPrewarmResult() @@ -1026,7 +1028,11 @@ class ModelRunner: swap_in_block_size=hisparse_cfg.swap_in_block_size, shared_index_layers=resolve_shared_index_layers( hf_text_config=self.model_config.hf_text_config, - is_speculative=self.spec_algorithm.is_speculative(), + ), + num_draft_tokens=( + 0 + if self.spec_algorithm.is_none() + else get_spec().speculative_num_draft_tokens or 0 ), ) diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index b3dc523a52..4efd2fda8a 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -265,6 +265,14 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): and int(num_layers) > 0 ): draft_num_layers = int(eagle_draft_num_layers) + draft_loc_space_scale = 1 + if get_memory().enable_hisparse: + from sglang.srt.mem_cache.sparsity import parse_hisparse_config + + # The draft pool is direct-indexed by the target allocator's + # host-expanded logical IDs. Account for the entire dense + # address space that KVCacheConfigurator allocates below. + draft_loc_space_scale = parse_hisparse_config().host_to_device_ratio if is_deepseek_dsa(kvc.model_config.hf_config): target_indexer_size = self._compute_dsa_indexer_cell_size( kvc=kvc, @@ -279,9 +287,12 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): target_kv_num_layers = get_glm_dsa_layer_split_effective_num_layers( kvc, num_layers ) - draft_kv_size = int( - target_kv_size * draft_num_layers / target_kv_num_layers + draft_kv_size = ( + int(target_kv_size * draft_num_layers / target_kv_num_layers) + * draft_loc_space_scale ) + # _compute_dsa_indexer_cell_size already applies the + # HiSparse logical-space ratio to the draft indexer. draft_indexer_size = self._compute_dsa_indexer_cell_size( kvc=kvc, num_layers=draft_num_layers, @@ -291,9 +302,10 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator): draft_indexer_size *= dcp_size self._cell_size += draft_kv_size + draft_indexer_size else: - self._cell_size = int( - self._cell_size * (1 + draft_num_layers / int(num_layers)) + draft_kv_size = int( + self._cell_size * draft_num_layers / int(num_layers) ) + self._cell_size += draft_kv_size * draft_loc_space_scale # DFLASH/DSPARK: the draft's per-token KV cost can differ from the target's # (e.g. MLA target, per-head K/V draft), so size from the draft config. diff --git a/python/sglang/srt/speculative/eagle_utils.py b/python/sglang/srt/speculative/eagle_utils.py index 533aa2d79e..149b7e56c1 100644 --- a/python/sglang/srt/speculative/eagle_utils.py +++ b/python/sglang/srt/speculative/eagle_utils.py @@ -580,6 +580,8 @@ def eagle_prepare_for_verify( draft_token_num=verify_input.draft_token_num, device=device, ) + if batch.hisparse_coordinator is not None: + batch.hisparse_coordinator.prepare_spec_verify(batch) batch.out_cache_loc_dsv4 = maybe_build_dsv4_verify_bundle( batch, verify_input.draft_token_num diff --git a/python/sglang/srt/speculative/eagle_worker_common.py b/python/sglang/srt/speculative/eagle_worker_common.py index 1045075133..7507a695fb 100644 --- a/python/sglang/srt/speculative/eagle_worker_common.py +++ b/python/sglang/srt/speculative/eagle_worker_common.py @@ -732,6 +732,11 @@ def run_eagle_verify( ) else: new_seq_lens = batch.seq_lens + accept_lens + if batch.hisparse_coordinator is not None: + batch.hisparse_coordinator.commit_spec_accept_tokens( + batch=batch, + accept_indices=accept_index, + ) clear_unaccepted_c128 = getattr( token_to_kv_pool_allocator.get_kvcache(), "clear_unaccepted_c128_draft_states", diff --git a/test/registered/e2e/models/test_dsa_glm52_hisparse.py b/test/registered/e2e/models/test_dsa_glm52_hisparse.py index bd5add5e6b..9ce5444edb 100644 --- a/test/registered/e2e/models/test_dsa_glm52_hisparse.py +++ b/test/registered/e2e/models/test_dsa_glm52_hisparse.py @@ -1,62 +1,156 @@ -import time import unittest +import requests + +from sglang.srt.utils.hf_transformers_utils import get_tokenizer from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.eval_accuracy_kit import GSM8KMixin -from sglang.test.server_fixtures.default_fixture import DefaultServerBase -from sglang.test.test_utils import terminate_and_kill_process_tree +from sglang.test.kl_test_utils import get_input_ids +from sglang.test.server_fixtures.disaggregation_fixture import ( + PDDisaggregationServerBase, +) -register_cuda_ci(est_time=483, stage="extra-b", runner_config="8-gpu-h200") +register_cuda_ci(est_time=700, stage="extra-b", runner_config="8-gpu-b300") -GLM52_FP8_MODEL_PATH = "zai-org/GLM-5.2-FP8" +GLM52_NVFP4_MODEL_PATH = "nvidia/GLM-5.2-NVFP4" -class TestGLM52HiSparse(DefaultServerBase, GSM8KMixin): - """GLM-5.2 FP8 with HiSparse (host-to-device sparse KV offload) on DSA decode. +class TestGLM52PDHiSparseSpec(PDDisaggregationServerBase, GSM8KMixin): + """1P1D GLM-5.2 MTP with HiSparse target KV and dense draft KV transfer.""" - HiSparse targets the high-concurrency regime and is not used together with - EAGLE MTP, so this variant runs without speculative decoding (unlike the - DSA-MTP variants in test_dsa_glm52_{dp,tp}_mtp.py). - """ - - model = GLM52_FP8_MODEL_PATH - other_args = [ - "--trust-remote-code", - "--tp", - "8", - "--attn-dp-size", - "8", + # Match the existing GLM-5.2 PD test: NVFP4 fits TP4 + TP4 on eight B300s. + model = GLM52_NVFP4_MODEL_PATH + prefill_tp_size = decode_tp_size = 4 + decode_base_gpu_id = 4 + common_args = [ "--page-size", "64", "--max-running-requests", "200", + "--max-total-tokens", + "262144", "--mem-fraction-static", "0.85", "--disable-radix-cache", "--kv-cache-dtype", - "bfloat16", + "fp8_e4m3", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--speculative-use-rejection-sampling", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 6}', + ] + extra_prefill_args = common_args + extra_decode_args = common_args + [ + "--attn-dp-size", + "4", "--dsa-decode-backend", - "flashmla_sparse", + "flashmla_kv", "--enable-hisparse", "--hisparse-config", '{"top_k": 2048, "device_buffer_size": 4096, "host_to_device_ratio": 5}', - "--model-loader-extra-config", - '{"enable_multithread_load": true, "num_threads": 64}', ] - # Match the original standalone hisparse eval config. - gsm8k_accuracy_thres = 0.94 + # Reuse the NVFP4 accuracy floor from test_dsa_glm52_pd_mtp_cp_layersplit.py. + gsm8k_accuracy_thres = 0.935 gsm8k_num_questions = 500 gsm8k_num_threads = 100 gsm8k_num_shots = 24 + gsm8k_accept_length_thres = 2.5 @classmethod - def tearDownClass(cls): - # HiSparse's pinned host buffer needs longer than the base class's 60s. - terminate_and_kill_process_tree( - cls.process, terminate_timeout=90, wait_timeout=60 + def setUpClass(cls): + super().setUpClass() + cls.launch_all() + + def test_gsm8k(self): + GSM8KMixin.test_gsm8k(self) + # The PD router does not expose the decode server's speculative metrics. + response = requests.get(self.decode_url + "/server_info", timeout=30) + self.assertEqual(response.status_code, 200, response.text) + states = response.json()["internal_states"] + self.assertTrue(states) + for rank, state in enumerate(states): + with self.subTest(rank=rank): + self.assertGreater( + state["avg_spec_accept_length"], self.gsm8k_accept_length_thres + ) + + def test_long_context_spec_swap(self): + self._test_long_context_spec_swap(temperature=0) + + def test_non_greedy_spec_swap(self): + self._test_long_context_spec_swap(temperature=0.6) + + def _test_long_context_spec_swap(self, temperature): + """Retrieve LongBench needles beyond the GPU cache after PD+MTP transfer.""" + tokenizer = get_tokenizer(self.model, trust_remote_code=True) + contexts = get_input_ids( + self.model, + max_prompt_tokens=32768, + num_samples=3, + trust_remote_code=True, ) - time.sleep(2) + self.assertEqual(len(contexts), 3) + prompts, answers = [], [] + for context, length, depth, answer in zip( + contexts, + (8192, 16384, 16384), + (0.1, 0.5, 0.9), + ("739391", "482617", "956083"), + ): + self.assertGreaterEqual(len(context), length) + context = context[:length] + split = int(length * depth) + document = ( + tokenizer.decode(context[:split]) + + f"\nImportant record: the verification code is {answer}.\n" + + tokenizer.decode(context[split:]) + + "\nWhat is the verification code in the important record? " + "Reply with the digits only." + ) + prompts.append( + tokenizer.apply_chat_template( + [{"role": "user", "content": document}], + tokenize=True, + return_dict=False, + add_generation_prompt=True, + enable_thinking=False, + ) + ) + answers.append(answer) + + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": prompts, + "sampling_params": { + "temperature": temperature, + "top_p": 0.95, + "max_new_tokens": 128, + "ignore_eos": True, + }, + }, + timeout=600, + ) + self.assertEqual(response.status_code, 200, response.text) + results = response.json() + self.assertEqual(len(results), len(prompts)) + for result, prompt, answer in zip(results, prompts, answers): + with self.subTest(prompt_tokens=len(prompt), answer=answer): + meta = result["meta_info"] + self.assertEqual(meta["prompt_tokens"], len(prompt)) + self.assertGreater(meta["prompt_tokens"], 4096) + self.assertEqual(meta["completion_tokens"], 128) + self.assertGreater(meta["spec_verify_ct"], 0) + # ignore_eos can append more text immediately after the answer. + self.assertRegex(result["text"], rf"^\s*{answer}(?!\d)") if __name__ == "__main__": diff --git a/test/registered/kernels/ops/kvcache/test_hisparse_spec.py b/test/registered/kernels/ops/kvcache/test_hisparse_spec.py index 7f46073a2b..c821dac919 100644 --- a/test/registered/kernels/ops/kvcache/test_hisparse_spec.py +++ b/test/registered/kernels/ops/kvcache/test_hisparse_spec.py @@ -174,6 +174,40 @@ def _assert_output_matches_tokens( class TestHiSparseSpec(CustomTestCase): + def test_fixed_width_device_commit_copy_supports_cuda_graph(self) -> None: + from sglang.srt.mem_cache.hisparse_memory_pool import HiSparseDSATokenToKVPool + + pool = object.__new__(HiSparseDSATokenToKVPool) + pool.bytes_per_token = 576 + pool.kv_buffer = [ + torch.arange(32 * 144, dtype=torch.int32, device=DEVICE).reshape(32, 144) + + layer * 10000 + for layer in range(2) + ] + originals = [buf.clone() for buf in pool.kv_buffer] + pool.data_ptrs = torch.tensor( + [buf.data_ptr() for buf in pool.kv_buffer], + dtype=torch.uint64, + device=DEVICE, + ) + pool.data_strides = torch.full_like(pool.data_ptrs, pool.bytes_per_token) + # Verify slots and persistent destinations are disjoint; padding stays put. + src = torch.tensor([26, 27, 28, 29], device=DEVICE) + dst = torch.tensor([24, 27, 25, 29], device=DEVICE) + + pool.transfer_values_on_device(dst, src) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + pool.transfer_values_on_device(dst, src) + for buf, original in zip(pool.kv_buffer, originals): + buf.copy_(original) + graph.replay() + + for buf, original in zip(pool.kv_buffer, originals): + original[[24, 25]] = original[[26, 28]] + torch.testing.assert_close(buf, original) + def test_deduplicates_repeated_misses_and_copies_full_items(self) -> None: hot_size, page_size = 4096, 64 num_steps, top_k, item_words = 4, 2048, 72 diff --git a/test/registered/unit/disaggregation/test_disaggregation_wire.py b/test/registered/unit/disaggregation/test_disaggregation_wire.py index e8e7affc85..5443bea7ac 100644 --- a/test/registered/unit/disaggregation/test_disaggregation_wire.py +++ b/test/registered/unit/disaggregation/test_disaggregation_wire.py @@ -69,6 +69,7 @@ from sglang.srt.sampling.sampling_mask import SamplingMaskRows from sglang.srt.speculative.eagle_disaggregation import ( build_eagle_disagg_draft_input, ) +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -1549,6 +1550,98 @@ class TestStagingWatermark(unittest.TestCase): ) +class TestMooncakeHiSparseSpecIndices(CustomTestCase): + def test_draft_buffers_use_logical_device_indices(self): + """A local target layer count must not turn remote target KV into draft KV.""" + for pp_size, stage_start in ((1, 0), (2, 0), (2, 2)): + with self.subTest(pp_size=pp_size, stage_start=stage_start): + manager = object.__new__(MooncakeKVManager) + manager.kv_args = SimpleNamespace( + mla_compression_ratios=None, + prefill_start_layer=stage_start, + prefill_end_layer=stage_start + 2, + kv_data_ptrs=[1000, 2000, 3000], + kv_item_lens=[100, 100, 100], + kv_layer_ids=[], + num_draft_entries=1, + ) + manager.is_mla_backend = True + manager.is_hybrid_mla_backend = False + manager.enable_custom_mem_pool = False + manager.max_transfer_batch_indices = 0 + manager.pp_size = pp_size + manager._transfer_data = Mock(return_value=0) + dst_ptrs = ( + [10000, 20000, 50000] + if pp_size == 1 + else [10000, 20000, 30000, 40000, 50000] + ) + with patch( + "sglang.srt.disaggregation.mooncake.conn.get_memory", + return_value=SimpleNamespace(enable_unified_memory=False), + ): + status = manager.send_kvcache( + mooncake_session_id="session", + prefill_kv_indices=np.array([1, 2], dtype=np.int32), + dst_kv_ptrs=dst_ptrs, + dst_kv_indices=np.array([7, 8], dtype=np.int32), + dst_device_kv_indices=np.array([21, 22], dtype=np.int32), + executor=None, + ) + + self.assertEqual(status, 0) + blocks = manager._transfer_data.call_args.args[1] + self.assertEqual( + blocks, + [ + (1100, dst_ptrs[stage_start] + 700, 200), + (2100, dst_ptrs[stage_start + 1] + 700, 200), + (3100, dst_ptrs[-1] + 2100, 200), + ], + ) + + +class TestHiSparseStagedSpecRelay(CustomTestCase): + def test_builds_lightweight_eagle_relay_handle(self): + future_map = object.__new__(FutureMap) + future_map.spec_algo = SpeculativeAlgorithm.EAGLE + future_map.dsa_topk_indices_buf = None + future_indices = torch.tensor([2, 5], device="cpu") + + spec_input = future_map.make_staged_spec_input(future_indices) + + self.assertIs(spec_input.future_indices, future_indices) + self.assertFalse(spec_input.future_dsa_topk_indices_available) + + @patch("sglang.srt.managers.overlap_utils.gather_spec_extras") + def test_staged_relay_restores_rejection_sampling_probs(self, gather_spec_extras): + future_map = object.__new__(FutureMap) + future_map.spec_algo = SpeculativeAlgorithm.EAGLE + future_map.device = torch.device("cpu") + future_map.need_topk = True + future_map.need_hidden_states = False + future_map.topk_p_buf = torch.zeros((4, 1), dtype=torch.float32) + future_map.topk_index_buf = torch.zeros((4, 1), dtype=torch.int64) + future_map.output_tokens_buf = torch.arange(4, dtype=torch.int64) + future_map.hidden_states_buf = None + future_map.draft_probs_buf = torch.arange(12, dtype=torch.float32).view(4, 3) + future_map.dsa_topk_indices_buf = None + future_indices = torch.tensor([2, 3], dtype=torch.int64) + spec_input = future_map.make_staged_spec_input(future_indices) + gather_spec_extras.return_value = ( + torch.zeros((2, 1)), + torch.zeros((2, 1), dtype=torch.int64), + torch.tensor([2, 3]), + None, + ) + + future_map._resolve_spec_extras(SimpleNamespace(spec_info=spec_input)) + + torch.testing.assert_close( + spec_input.draft_probs, future_map.draft_probs_buf[future_indices] + ) + + class TestMooncakePPStaging(unittest.TestCase): def test_staging_response_targets_requesting_pp_rank(self): sock = Mock() diff --git a/test/registered/unit/managers/test_hisparse_spec_coordinator.py b/test/registered/unit/managers/test_hisparse_spec_coordinator.py new file mode 100644 index 0000000000..297220762e --- /dev/null +++ b/test/registered/unit/managers/test_hisparse_spec_coordinator.py @@ -0,0 +1,289 @@ +"""Unit tests for the minimal HiSparse spec coordinator lifecycle.""" + +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import torch +from torch.utils._python_dispatch import TorchDispatchMode + +from sglang.srt.managers.hisparse_coordinator import ( + HiSparseCoordinator, + HiSparseSpecSwapManager, +) +from sglang.srt.mem_cache import allocation +from sglang.srt.mem_cache.allocator.hisparse import HiSparseTokenToKVPoolAllocator +from sglang.srt.mem_cache.pool_host.hisparse import HiSparseHostPoolMixin +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + + +class TestHiSparseSpecCoordinator(CustomTestCase): + def test_spec_decode_reserve_allocates_logical_pages_only(self): + logical_locs = torch.tensor([64, 65, 66, 67], dtype=torch.int64) + allocator = SimpleNamespace( + page_size=64, + alloc_logical_only=MagicMock(return_value=logical_locs), + ) + tree_cache = SimpleNamespace(token_to_kv_pool_allocator=allocator) + req_to_token = torch.full((1, 16), -1, dtype=torch.int64) + req_to_token[0, 1] = 63 + req_to_token_pool = SimpleNamespace(req_to_token=req_to_token) + req = SimpleNamespace(kv=SimpleNamespace(kv_allocated_len=2)) + batch = SimpleNamespace( + device=torch.device("cpu"), + hisparse_coordinator=object(), + ) + + def assign_on_cpu(req_indices, table, start, end, locs, batch_size): + table[int(req_indices[0]), int(start[0]) : int(end[0])] = locs + + with ( + patch.object(allocation, "evict_from_tree_cache"), + patch.object(allocation, "get_last_loc", return_value=torch.tensor([63])), + patch.object( + allocation, + "assign_req_to_token_pool_func", + side_effect=assign_on_cpu, + ), + ): + allocation.alloc_for_spec_decode( + tree_cache, + req_to_token_pool, + reqs=[req], + req_pool_indices=torch.tensor([0]), + cur_kv_lens=torch.tensor([2]), + cur_kv_lens_cpu=torch.tensor([2]), + nxt_kv_lens=torch.tensor([6]), + nxt_kv_lens_cpu=torch.tensor([6]), + num_needed_tokens=4, + batch=batch, + ) + + allocator.alloc_logical_only.assert_called_once() + torch.testing.assert_close(req_to_token[0, 2:6], logical_locs) + + def test_physical_free_excludes_reserved_page_zero(self): + allocator = object.__new__(HiSparseTokenToKVPoolAllocator) + allocator.page_size = 64 + allocator.hisparse_attn_allocator = MagicMock() + + allocator.free_hisparse_indices(torch.tensor([0, 1, 63, 64, 65])) + + freed = allocator.hisparse_attn_allocator.free.call_args.args[0] + torch.testing.assert_close(freed, torch.tensor([64, 65])) + + def test_scratch_allocation_has_per_layer_location_views(self): + scratch_allocator = MagicMock() + scratch_allocator.alloc.return_value = torch.arange(64, 1088) + token_allocator = SimpleNamespace( + hisparse_attn_allocator=scratch_allocator, + free_hisparse_indices=MagicMock(), + ) + coordinator = MagicMock( + token_to_kv_pool_allocator=token_allocator, + mem_pool_device=SimpleNamespace(layer_num=3), + req_to_token_pool=SimpleNamespace(req_to_token=torch.empty((2, 16))), + device="cpu", + is_dsv4_hisparse=False, + top_k=1024, + device_buffer_size=4096, + ) + spec_swap = HiSparseSpecSwapManager(coordinator, num_draft_tokens=2) + + spec_swap.allocate_scratch(1) + + expected = torch.arange(64, 1088, dtype=torch.int32) + for layer_id in range(3): + torch.testing.assert_close(spec_swap.req_to_scratch[layer_id, 1], expected) + self.assertIn(1, spec_swap._scratch_reqs) + self.assertIs( + spec_swap.states[0].scratch_state, spec_swap.states[1].scratch_state + ) + + owned_locs = spec_swap.extend_owned_locs( + 1, torch.tensor([20], dtype=torch.int32) + ) + torch.testing.assert_close( + owned_locs, + torch.cat([torch.tensor([20], dtype=torch.int32), expected.repeat(3)]), + ) + + spec_swap.free_unrotated_scratch(1) + token_allocator.free_hisparse_indices.assert_called_once() + torch.testing.assert_close( + token_allocator.free_hisparse_indices.call_args.args[0], expected + ) + self.assertNotIn(1, spec_swap._scratch_reqs) + self.assertEqual(spec_swap.req_to_scratch[:, 1].count_nonzero(), 0) + + def test_spec_workspace_is_reserved_before_request_becomes_runnable(self): + coordinator = object.__new__(HiSparseCoordinator) + coordinator.is_dsv4_hisparse = False + coordinator.device_buffer_size = 4096 + coordinator.padded_buffer_size = 4160 + coordinator.mem_pool_device = SimpleNamespace( + page_size=64, + translate_loc_from_full_to_compressed=lambda locs: locs, + ) + coordinator.req_to_token_pool = SimpleNamespace( + req_to_token=torch.arange(16, dtype=torch.int64).view(1, -1) + ) + buffer_indices = torch.arange(64, 4224, dtype=torch.int64) + coordinator.token_to_kv_pool_allocator = SimpleNamespace( + alloc_device_buffer=MagicMock(return_value=buffer_indices) + ) + coordinator.req_to_device_buffer = torch.zeros((1, 4160), dtype=torch.int32) + coordinator.req_device_buffer_size = torch.zeros(1, dtype=torch.int32) + coordinator.req_device_buffer_tokens = torch.full( + (2, 1, 4160), -1, dtype=torch.int32 + ) + coordinator.req_device_buffer_token_locs = torch.full( + (2, 1, 4160), -1, dtype=torch.int32 + ) + coordinator._device_buffer_arange_i32 = torch.arange(4096, dtype=torch.int32) + coordinator.spec_swap = SimpleNamespace( + enabled=True, + allocate_scratch=MagicMock(), + reset=MagicMock(), + ) + req = SimpleNamespace( + rid="req-0", + kv=SimpleNamespace(req_pool_idx=0, kv_allocated_len=8), + ) + + coordinator.alloc_device_buffer(req) + + self.assertEqual(int(coordinator.req_device_buffer_size[0]), 4160) + torch.testing.assert_close( + coordinator.req_to_device_buffer[0], buffer_indices.to(torch.int32) + ) + coordinator.spec_swap.allocate_scratch.assert_called_once_with(0) + coordinator.spec_swap.reset.assert_called_once_with(0) + + def test_accept_commit_keeps_hot_and_newest_device_slots(self): + coordinator = object.__new__(HiSparseCoordinator) + coordinator.device_buffer_size = 5 + coordinator.page_size = 8 + coordinator.req_to_device_buffer = torch.tensor( + [list(range(20, 33))], dtype=torch.int64 + ) + coordinator.req_device_buffer_tokens = torch.full( + (2, 1, 13), -1, dtype=torch.int32 + ) + coordinator.req_device_buffer_token_locs = torch.full( + (2, 1, 13), -1, dtype=torch.int32 + ) + coordinator._skip_first_backup = [False] + + verify_cache_locs = torch.tensor([50, 51, 52, 53], dtype=torch.int64) + req_to_token = torch.zeros((1, 16), dtype=torch.int64) + req_to_token[0, 4:8] = verify_cache_locs + coordinator.req_to_token_pool = SimpleNamespace(req_to_token=req_to_token) + + full_to_device_mapping = torch.zeros(128, dtype=torch.int64) + transfer_values = MagicMock() + coordinator.mem_pool_device = SimpleNamespace( + transfer_values_on_device=transfer_values + ) + coordinator.token_to_kv_pool_allocator = SimpleNamespace( + full_to_hisparse_device_index_mapping=full_to_device_mapping + ) + + coordinator.mem_pool_host = HiSparseHostPoolMixin() + coordinator.mem_pool_host.page_size = 8 + coordinator.mem_pool_host.alloc_page = MagicMock( + return_value=torch.arange(1000, 1008) + ) + coordinator.req_to_host_pool = torch.full((1, 16), -1, dtype=torch.int64) + coordinator.req_to_host_pool_allocated_len = torch.zeros(1, dtype=torch.int64) + + spec_swap = object.__new__(HiSparseSpecSwapManager) + spec_swap._coordinator = coordinator + spec_swap.enabled = True + spec_swap.num_draft_tokens = 4 + spec_swap._backup_device_locs_to_host = MagicMock() + + batch = SimpleNamespace( + reqs=[ + SimpleNamespace(kv=SimpleNamespace(req_pool_idx=0, kv_allocated_len=8)) + ], + forward_mode=SimpleNamespace(is_idle=lambda: False), + req_pool_indices=torch.tensor([0]), + req_pool_indices_cpu=torch.tensor([0]), + seq_lens=torch.tensor([4]), + seq_lens_cpu=torch.tensor([2]), # CPU metadata can lag during overlap. + out_cache_loc=verify_cache_locs, + ) + spec_swap.prepare_verify(batch) + spec_swap.prepare_verify(batch) + coordinator.mem_pool_host.alloc_page.assert_called_once_with(1) + self.assertEqual(int(coordinator.req_to_host_pool_allocated_len[0]), 8) + accept_indices = torch.tensor([[0, 1, 2, -1]], dtype=torch.int32) + + class NoDataDependentOps(TorchDispatchMode): + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + if func in ( + torch.ops.aten._local_scalar_dense.default, + torch.ops.aten.nonzero.default, + torch.ops.aten.repeat_interleave.Tensor, + ): + raise AssertionError(f"Synchronizing operation: {func}") + if func in ( + torch.ops.aten.index.Tensor, + torch.ops.aten.index_put_.default, + ): + if any( + idx is not None and idx.dtype == torch.bool for idx in args[1] + ): + raise AssertionError("Dynamically sized boolean indexing") + if func == torch.ops.aten.index_put_.default: + value = args[2] + if value.device.type == "cpu" and value.ndim == 0: + raise AssertionError( + "Blocking CPU scalar copy in CUDA index_put" + ) + return func(*args, **(kwargs or {})) + + with ( + patch.object( + torch.Tensor, "cpu", side_effect=AssertionError("CPU readback") + ), + NoDataDependentOps(), + ): + spec_swap.commit_accept_tokens(batch=batch, accept_indices=accept_indices) + + transfer_call = transfer_values.call_args + torch.testing.assert_close( + transfer_call.kwargs["src_indices"], torch.tensor([26, 27, 28, 29]) + ) + torch.testing.assert_close( + transfer_call.kwargs["dst_indices"], torch.tensor([24, 27, 25, 29]) + ) + backup_call = spec_swap._backup_device_locs_to_host.call_args + torch.testing.assert_close( + backup_call.args[0], torch.tensor([1004, 1005, 1006, 1007]) + ) + torch.testing.assert_close(backup_call.args[1], torch.tensor([24, 27, 25, 29])) + torch.testing.assert_close( + full_to_device_mapping[verify_cache_locs], torch.tensor([24, 0, 25, 0]) + ) + torch.testing.assert_close( + coordinator.req_device_buffer_tokens[:, 0, 4], + torch.tensor([4, 4], dtype=torch.int32), + ) + torch.testing.assert_close( + coordinator.req_device_buffer_tokens[:, 0, 5], + torch.tensor([6, 6], dtype=torch.int32), + ) + torch.testing.assert_close( + coordinator.req_device_buffer_tokens[:, 0, 6:10], + torch.tensor([[4, 5, 6, 7], [4, 5, 6, 7]], dtype=torch.int32), + ) + self.assertTrue(coordinator._skip_first_backup[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/managers/test_prefill_adder.py b/test/registered/unit/managers/test_prefill_adder.py index 6bd25c2312..702140ff4b 100644 --- a/test/registered/unit/managers/test_prefill_adder.py +++ b/test/registered/unit/managers/test_prefill_adder.py @@ -12,6 +12,7 @@ from sglang.srt.managers.schedule_policy import ( SchedulePolicy, estimate_prefill_extend_tile_metrics, ) +from sglang.srt.mem_cache.allocator.hisparse import HiSparseTokenToKVPoolAllocator from sglang.srt.mem_cache.allocator.page_interleave import PageInterleavePoolAllocator from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( @@ -96,6 +97,7 @@ class TestPrefillAdder(CustomTestCase): allocator.available_size.return_value = available_size allocator.size_swa = size_swa allocator.swa_req_ring = False + allocator.request_slot_reserve.return_value = 0 allocator.create_prefill_budget.side_effect = lambda tree_cache, **kwargs: ( PrefillBudget(allocator, tree_cache, **kwargs) ) @@ -119,6 +121,7 @@ class TestPrefillAdder(CustomTestCase): def create_mock_req(self, rid, priority, max_new_tokens, output_len=0, wait_time=0): req = MagicMock(spec=Req) + req.kv = SimpleNamespace(req_pool_idx=None) req.rid = str(rid) req.cache_request_handle = CacheRequestHandle(req.rid, 0) req.priority = priority @@ -283,6 +286,56 @@ class TestPrefillAdder(CustomTestCase): ) self.assertEqual(adder.can_run_list, []) + def test_hisparse_short_prompts_reserve_full_buffers_and_scratch(self): + """Short prompts and chunks must fit persistent buffers plus one scratch.""" + for available, chunk, prefix, ignore_eos, admitted in ( + (10496, None, 0, False, 2), + (10496, None, 0, True, 2), + (5248, 64, 0, False, 1), + (4096, 64, 64, False, 0), + (4160, 64, 64, False, 1), + ): + with self.subTest(available=available, chunk=chunk, prefix=prefix): + self.mock_tree_cache.supports_mamba.return_value = False + self.mock_tree_cache.disable = ignore_eos + allocator = HiSparseTokenToKVPoolAllocator( + size=available, + page_size=64, + dtype=torch.float32, + device="cpu", + kvcache=SimpleNamespace(register_mapping=lambda mapping: None), + need_sort=False, + ) + allocator.configure_spec_scratch(1024, device_buffer_size=4160) + adder = self.create_adder( + self.create_running_batch(), + page_size=64, + rem_chunk_tokens=chunk, + token_to_kv_pool_allocator=allocator, + ) + reqs = [] + for i in range(3): + req = self.create_shared_req(f"hisparse-{i}", max_new_tokens=0) + req.sampling_params.ignore_eos = ignore_eos + req.full_untruncated_fill_ids = list(range(100)) + req.origin_input_ids = req.full_untruncated_fill_ids + req.prefix_len = prefix + req.kv = SimpleNamespace(req_pool_idx=i if prefix else None) + reqs.append(req) + if prefix: + adder.add_chunked_req(req) + else: + adder.add_one_req( + req, has_chunked_req=False, truncation_align_size=None + ) + self.assertEqual(adder.can_run_list, reqs[:admitted]) + if admitted: + self.assertEqual(adder.cur_rem_tokens, 0) + self.assertEqual( + reqs[0].extend_len, + 64 if chunk and not prefix else 100 - prefix, + ) + def test_continuation_without_limit_keeps_normal_chunk_size(self): adder = self.create_shortest_prefill_adder() req = self.create_shared_req("continuation") @@ -1185,7 +1238,7 @@ class TestPrefillAdder(CustomTestCase): req.needs_host_load_back.return_value = True req.last_node = MagicMock() req.best_match_node = MagicMock() - req.kv = SimpleNamespace(cache_protected_len=0) + req.kv.cache_protected_len = 0 req.sampling_params = SimpleNamespace(max_new_tokens=8, ignore_eos=False) @@ -1412,7 +1465,7 @@ class TestPrefillAdder(CustomTestCase): req.host_hit_length = host_hit req.needs_host_load_back.return_value = True req.best_match_node = req.last_node - req.kv = SimpleNamespace(cache_protected_len=prefix_len) + req.kv.cache_protected_len = prefix_len return req def _build_mamba_restore_admission(self, *, holds_state, available_slots): diff --git a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py index 4a6729ea35..89616c7505 100644 --- a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py +++ b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py @@ -37,6 +37,7 @@ from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.separate_buffer_allocator_double import ( bind_separate_buffer_capacity, ) +from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=12, suite="base-a-test-cpu") @@ -432,7 +433,7 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase): release.assert_not_called() -class TestDecodePreallocQueuePriority(unittest.TestCase): +class TestDecodePreallocQueuePriority(CustomTestCase): def setUp(self): # The code under test reads its config from the bags. reset_context() @@ -519,6 +520,46 @@ class TestDecodePreallocQueuePriority(unittest.TestCase): queue.scheduler = scheduler return queue + def test_hisparse_prealloc_reserves_scratch_for_pending_transfers(self): + """Pending PD requests consume a full buffer and scratch, not just a buffer.""" + for available, pending, admitted in ( + (4160, 0, 0), + (8320, 1, 0), + (10368, 1, 1), + (10368, 0, 2), + ): + with self.subTest(available=available, pending=pending): + reqs = [self._new_decode_req(f"hisparse-{i}", i) for i in range(3)] + queue = self._new_queue(reqs) + queue.scheduler.enable_priority_scheduling = False + queue.scheduler.enable_hisparse = True + queue.scheduler.enable_decode_hicache = False + queue.scheduler.hisparse_coordinator = SimpleNamespace( + padded_buffer_size=4160, + spec_swap=SimpleNamespace(scratch_capacity=1024), + ) + queue.token_to_kv_pool_allocator.logical_attn_allocator = ( + SimpleNamespace(available_size=lambda: 32768) + ) + queue.token_to_kv_pool_allocator.hisparse_attn_allocator = ( + SimpleNamespace(available_size=lambda: available) + ) + queue.token_to_kv_pool_allocator.page_size = 64 + queue.token_to_kv_pool.page_size = 64 + queue.draft_token_to_kv_pool = None + queue.transfer_queue.queue = [ + SimpleNamespace(host_staged=False) for _ in range(pending) + ] + queue._uses_swa_tail_prealloc = MagicMock(return_value=False) + queue._uses_swa_reservation = MagicMock(return_value=False) + queue._send_kv_metadata = MagicMock() + + preallocated, failed = queue.pop_preallocated() + + self.assertEqual(failed, []) + self.assertEqual(preallocated, reqs[:admitted]) + self.assertEqual(queue.queue, reqs[admitted:]) + def test_prealloc_lora_slots_cover_inflight_microbatches_and_queues(self): """In-flight requests retain their adapter slots.""" running, finished, transferring, waiting = [ diff --git a/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py b/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py index d600d6b396..cc92a6b4b1 100644 --- a/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py +++ b/test/registered/unit/mem_cache/test_dllm_fdfo_kv_reuse.py @@ -99,6 +99,7 @@ def _make_batch(pool, allocator, reqs, extend_lens): req_to_token_pool=pool, token_to_kv_pool_allocator=allocator, tree_cache=_FakeTreeCache(allocator), + hisparse_coordinator=None, prefix_lens=[req.prefix_len for req in reqs], extend_lens=extend_lens, seq_lens=seq_lens_cpu, diff --git a/test/registered/unit/mem_cache/test_hisparse_allocator.py b/test/registered/unit/mem_cache/test_hisparse_allocator.py index c2e4b58f09..dd39122f00 100644 --- a/test/registered/unit/mem_cache/test_hisparse_allocator.py +++ b/test/registered/unit/mem_cache/test_hisparse_allocator.py @@ -9,6 +9,7 @@ import torch from sglang.srt.managers.schedule_batch import ReqKvInfo from sglang.srt.mem_cache.allocator.hisparse import ( DeepSeekV4HiSparseTokenToKVPoolAllocator, + HiSparseTokenToKVPoolAllocator, ) from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.runtime_context import get_context, publish, reset_context @@ -20,12 +21,18 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu") class TestHiSparseDecodeRemap(CustomTestCase): + def test_spec_scratch_is_charged_only_before_request_slot_allocation(self): + allocator = object.__new__(HiSparseTokenToKVPoolAllocator) + allocator.page_size = 64 + + allocator.configure_spec_scratch(1024) + + self.assertEqual(allocator.request_slot_reserve(has_req_pool_slot=False), 1024) + self.assertEqual(allocator.request_slot_reserve(has_req_pool_slot=True), 0) + def test_page_size_one_reclaims_temporary_device_slot(self): """Decode remapping must reclaim its temporary slot without freeing the live slot.""" from sglang.srt.managers.hisparse_coordinator import HiSparseCoordinator - from sglang.srt.mem_cache.allocator.hisparse import ( - HiSparseTokenToKVPoolAllocator, - ) from sglang.srt.mem_cache.memory_pool import MiniMaxSparseKVPool pool = MiniMaxSparseKVPool( diff --git a/test/registered/unit/mem_cache/test_hisparse_slot_translation.py b/test/registered/unit/mem_cache/test_hisparse_slot_translation.py index b0833784d7..f58cf3a01f 100644 --- a/test/registered/unit/mem_cache/test_hisparse_slot_translation.py +++ b/test/registered/unit/mem_cache/test_hisparse_slot_translation.py @@ -17,6 +17,58 @@ class TestHiSparseSlotTranslation(CustomTestCase): def make_pool(self, pool_type): return pool_type.__new__(pool_type) + def test_device_transfer_copies_physical_slots_without_remapping(self): + """Accept commit moves target KV, not logical IDs or indexer KV.""" + pool = self.make_pool(HiSparseDSATokenToKVPool) + pool.layer_num = 2 + pool.bytes_per_token = 4 + pool.kv_buffer = [ + torch.arange(12, dtype=torch.float32) + 100 * layer + for layer in range(pool.layer_num) + ] + pool.data_ptrs = torch.tensor( + [buf.data_ptr() for buf in pool.kv_buffer], dtype=torch.uint64 + ) + pool.data_strides = torch.full_like(pool.data_ptrs, 4) + pool.register_mapping(torch.arange(12).flip(0)) + pool.index_key_cache = [torch.arange(12)] + expected = [buf.clone() for buf in pool.kv_buffer] + for buf in expected: + buf[[1, 3]] = buf[[6, 8]] + + def copy_physical_rows( + data_ptrs, + strides, + dst_indices, + src_indices, + num_locs, + num_locs_upper, + config, + ): + self.assertEqual(num_locs, 2) + self.assertEqual(num_locs_upper, 2) + torch.testing.assert_close(data_ptrs, pool.data_ptrs) + torch.testing.assert_close(strides, pool.data_strides) + for buf in pool.kv_buffer: + buf[dst_indices] = buf[src_indices] + + with patch.object( + hisparse_memory_pool, + "copy_all_layer_kv_cache_func", + side_effect=copy_physical_rows, + ): + pool.transfer_values_on_device( + dst_indices=torch.tensor([1, 0, 3, 0])[::2], + src_indices=torch.tensor([6, 0, 8, 0])[::2], + ) + + for actual, reference in zip(pool.kv_buffer, expected): + torch.testing.assert_close(actual, reference) + torch.testing.assert_close( + pool.full_to_hisparse_device_index_mapping, torch.arange(12).flip(0) + ) + torch.testing.assert_close(pool.index_key_cache[0], torch.arange(12)) + def test_pool_translation_selects_fused_kernel_for_gpu_slot_lists(self): pool = self.make_pool(HiSparseDSATokenToKVPool) mapping = torch.arange(32, dtype=torch.int64) + 100 diff --git a/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py b/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py index 974113068c..8a341b0f17 100644 --- a/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py +++ b/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py @@ -30,7 +30,7 @@ from unittest import mock import torch -from sglang.srt.managers.schedule_batch import Req +from sglang.srt.managers.schedule_batch import Req, ReqKvInfo from sglang.srt.managers.schedule_policy import AddReqResult, PrefillAdder from sglang.srt.mem_cache.base_prefix_cache import ( CacheRequestHandle, @@ -523,7 +523,7 @@ class TestLoadBackAdmission(_UnifiedHiCacheCase): req.finished.return_value = False req.materialized_host_hit_len.return_value = match.host_hit_length req.fulfilled_storage_hit_len.return_value = 0 - req.kv = SimpleNamespace( + req.kv = ReqKvInfo( cache_protected_len=match.device_prefix_len, mamba_pool_idx=None ) type(req).extend_len = mock.PropertyMock( diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index 397e241f53..11c125cd4b 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -85,6 +85,8 @@ def _make_model_runner( max_running_requests=None, disaggregation_decode_extra_slots=0, enable_unified_memory=False, + enable_hisparse=False, + hisparse_host_to_device_ratio=1, kv_lora_rank=512, qk_rope_head_dim=64, swa_kv_lora_rank=128, @@ -152,7 +154,10 @@ def _make_model_runner( max_running_requests=max_running_requests, disaggregation_decode_extra_slots=disaggregation_decode_extra_slots, enable_unified_memory=enable_unified_memory, - enable_hisparse=False, + enable_hisparse=enable_hisparse, + hisparse_config=( + f'{{"host_to_device_ratio": {hisparse_host_to_device_ratio}}}' + ), enable_hierarchical_cache=False, enable_dsa_cache_layer_split=False, kv_cache_dtype="auto", @@ -949,6 +954,52 @@ class TestEagleConfigurator(CustomTestCase): available, ) + @patch( + "sglang.srt.mem_cache.kv_cache_configurator.calculate_mla_kv_cache_dim", + return_value=576, + ) + def test_hisparse_draft_expanded_id_space_is_budgeted( + self, + _mock_calculate_mla_kv_cache_dim, + ): + """The dense draft pool spans HiSparse's host-expanded logical IDs.""" + available = 10_000_000 + num_layers = 78 + draft_num_layers = 1 + indexer_bytes_per_token = 132 + host_to_device_ratio = 4 + + mr = _make_model_runner( + self, + num_layers=num_layers, + use_mla_backend=True, + enable_hisparse=True, + hisparse_host_to_device_ratio=host_to_device_ratio, + ) + _configure_dsa_model(mr) + mr.spec_algorithm.is_eagle.return_value = True + mr.spec_algorithm.is_none.return_value = False + mr.spec_aux_config.eagle_draft_num_layers = draft_num_layers + + with mock_cpu_env(kv_size=1): + from sglang.srt.model_executor.pool_configurator import ( + create_memory_pool_configurator, + ) + + cfg = create_memory_pool_configurator(mr) + config = cfg.calculate_pool_sizes(available, page_size=1) + + actual_bytes_per_token = ( + 576 * num_layers + + indexer_bytes_per_token * num_layers * host_to_device_ratio + + (576 + indexer_bytes_per_token) * draft_num_layers * host_to_device_ratio + ) + self.assertEqual(cfg._cell_size, actual_bytes_per_token) + self.assertLessEqual( + config.max_total_num_tokens * actual_bytes_per_token, + available, + ) + def test_hybrid_swa_draft_uses_swa_geometry_and_capacity(self): """SWA draft layers use SWA KV geometry and capacity.""" available = 10_000_000 diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index dba4973d92..b9ec877344 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -33,6 +33,7 @@ from sglang.srt.arg_groups.hicache_hook import ( handle_hicache_ratio_default, ) from sglang.srt.arg_groups.hisparse_hook import ( + validate_hisparse, validate_hisparse_dsa_backend, validate_hisparse_kv_cache_dtype, ) @@ -1407,6 +1408,32 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase): with self.assertRaisesRegex(ValueError, r"fp8_e4m3"): validate_hisparse_kv_cache_dtype(server_args) + def test_hisparse_rejects_adaptive_speculation(self): + server_args = ServerArgs( + model_path="dummy", + enable_hisparse=True, + disable_radix_cache=True, + speculative_algorithm="EAGLE", + speculative_eagle_topk=1, + speculative_adaptive=True, + ) + model_config = SimpleNamespace( + hf_config=SimpleNamespace(architectures=["GlmMoeDsaForCausalLM"]) + ) + + with ( + patch( + "sglang.srt.arg_groups.hisparse_hook.model_config_of", + return_value=model_config, + ), + patch( + "sglang.srt.configs.model_config.is_deepseek_dsa", + return_value=True, + ), + ): + with self.assertRaisesRegex(ValueError, "speculative-adaptive"): + validate_hisparse(server_args) + class TestFa4PageSizeAutoForce(CustomTestCase): """FA4 requires page_size 128 for non-MLA models on SM100. The auto-force