Files
sglang/python/sglang/srt/managers/schedule_policy.py
T

1714 lines
70 KiB
Python

from __future__ import annotations
import logging
from sglang.srt.environ import envs
from sglang.srt.managers.prefill_delayer import PrefillDelayerSinglePassExecutor
from sglang.srt.runtime_context import (
get_disagg,
get_exec,
get_memory,
get_schedule,
)
from sglang.srt.utils import get_bool_env_var, is_gfx95_supported, is_hip
_ROUTING_KEY_POLICY_DEBUG_LOG = get_bool_env_var("SGLANG_ROUTING_KEY_POLICY_DEBUG_LOG")
logger = logging.getLogger(__name__)
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Request scheduler policy"""
import os
import random
from collections import Counter
from contextlib import contextmanager, nullcontext
from dataclasses import dataclass
from enum import Enum, auto
from functools import lru_cache
from typing import TYPE_CHECKING, Dict, List, Optional, Set, Union
import torch
from sglang.srt.dllm.config import DllmConfig
from sglang.srt.managers.schedule_batch import (
Req,
ScheduleBatch,
split_cached_prefix_by_tier,
)
from sglang.srt.mem_cache.allocator.hisparse import (
DeepSeekV4HiSparseTokenToKVPoolAllocator,
)
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
from sglang.srt.mem_cache.allocator.swa import (
SWATokenToKVPoolAllocator,
)
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedMambaSWATokenToKVPoolAllocator,
)
from sglang.srt.mem_cache.allocator.unified_mamba import (
UnifiedMambaTokenToKVPoolAllocator,
)
from sglang.srt.mem_cache.base_prefix_cache import (
BasePrefixCache,
InitLoadBackParams,
InsertParams,
MatchPrefixParams,
zero_match_result,
)
from sglang.srt.mem_cache.common import match_kv_cache, touch_waiting_prefix
from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode
from sglang.srt.mem_cache.unified_cache.components import (
CacheTransferPhase,
ComponentType,
)
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
from sglang.srt.model_executor import encoder_swa_replay
if TYPE_CHECKING:
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
# Clip the estimation of max_new_tokens for the request whose max_new_tokens is very large.
# This can prevent the server from being too conservative.
# Note that this only clips the estimation in the scheduler but does not change the stop
# condition. The request can still generate tokens until it hits the unclipped max_new_tokens.
CLIP_MAX_NEW_TOKENS = int(
os.environ.get("SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION", "4096")
)
@lru_cache(maxsize=1)
def _use_exact_chunk_fill() -> bool:
"""Whether to charge the chunked-prefill compute budget in tokens (gfx95 only).
Gated on gfx95 because that is where the win is: the aiter absorb bmm picks
its EVEN_MN specialization from M % BLOCK_SIZE_M, so a short batch costs 2x
on that kernel, against 1% for the hipBLASLt GEMMs that just run one extra
partial tile.
"""
return envs.SGLANG_EXACT_CHUNK_FILL.get() and is_gfx95_supported()
# Threshold for in-batch prefix cache.
# If a request has a matched prefix length (against existing cache) less than this value,
# the scheduler runs the in-batch prefix caching check for this request.
# If we set it to -1, it means we disable in-batch prefix caching.
IN_BATCH_PREFIX_CACHING_CHECK_THRESHOLD = int(
os.environ.get("IN_BATCH_PREFIX_CACHING_CHECK_THRESHOLD", "32")
)
# Threshold for in-batch prefix cache.
# If a request has a matched prefix length (within the waiting queue) larger than this value,
# the scheduler deprioritizes this request
IN_BATCH_PREFIX_CACHING_DEPRIORITIZE_THRESHOLD = int(
os.environ.get("IN_BATCH_PREFIX_CACHING_DEPRIORITIZE_THRESHOLD", "32")
)
IGNORE_EOS_RESERVE_TOKENS = 1
# AMD/HIP-only: the prefill tile-budget admission control is part of the AMD
# compact extend-attention work and is gated on HIP (see _check_prefill_tile_budget),
# so non-AMD vendors keep the exact legacy scheduler behavior.
_IS_HIP = is_hip()
PREFILL_TILE_BUDGET = envs.SGLANG_PREFILL_TILE_BUDGET.get()
PREFILL_TILE_BUDGET_MODE = envs.SGLANG_PREFILL_TILE_BUDGET_MODE.get().strip().lower()
if PREFILL_TILE_BUDGET_MODE not in {"legacy", "compact"}:
logger.warning(
"Unsupported SGLANG_PREFILL_TILE_BUDGET_MODE=%s. Falling back to compact.",
PREFILL_TILE_BUDGET_MODE,
)
PREFILL_TILE_BUDGET_MODE = "compact"
# Arbitrary; bounds per-round prefix work on the waiting queue.
WAITING_QUEUE_PREFIX_MATCH_MAX = 128
# Bound the walk cost using scheduling passes, which agree across TP ranks.
WAITING_PREFIX_REFRESH_INTERVAL = 32
def _ceil_div(value: int, divisor: int) -> int:
return -(-value // divisor)
def estimate_prefill_extend_tile_metrics(
extend_lens: List[int], block_m: int
) -> Dict[str, Union[int, float, List[int], None]]:
"""Estimate extend-attention query tiles per head for a prefill batch."""
normalized_lens = [max(0, int(length)) for length in extend_lens]
q_tiles = [
_ceil_div(length, block_m) if length > 0 else 0 for length in normalized_lens
]
legacy_tiles = len(q_tiles) * max(q_tiles) if q_tiles else 0
compact_tiles = sum(q_tiles)
saved_tiles = legacy_tiles - compact_tiles
saved_ratio = saved_tiles / legacy_tiles if legacy_tiles else None
return {
"block_m": int(block_m),
"request_count": len(normalized_lens),
"extend_lens": normalized_lens,
"q_tiles_per_request": q_tiles,
"max_extend_len": max(normalized_lens) if normalized_lens else 0,
"sum_extend_len": sum(normalized_lens),
"legacy_q_tiles_per_head": legacy_tiles,
"compact_q_tiles_per_head": compact_tiles,
"saved_q_tiles_per_head": saved_tiles,
"saved_q_tile_ratio": saved_ratio,
}
class CacheAwarePolicy(Enum):
"""Scheduling policies that are aware of the tree cache."""
LPM = "lpm" # longest prefix match
DFS_WEIGHT = "dfs-weight" # depth-first search weighting
HRRN = "hrrn" # highest response ratio next, token-based aging
SHORTEST_PREFILL_FIRST = "shortest-prefill-first"
class CacheAgnosticPolicy(Enum):
"""Scheduling policies that are not aware of the tree cache."""
FCFS = "fcfs" # first come first serve
LOF = "lof" # longest output first
RANDOM = "random"
ROUTING_KEY = "routing-key" # prioritize by routing key frequency in running batch
class SchedulePolicy:
Policy = Union[CacheAwarePolicy, CacheAgnosticPolicy]
def __init__(
self,
policy: str,
tree_cache: BasePrefixCache,
enable_hierarchical_cache: bool,
enable_priority_scheduling: bool,
schedule_low_priority_values_first: bool,
):
self.policy = self._validate_and_adjust_policy(policy, tree_cache)
self.tree_cache = tree_cache
self.enable_hierarchical_cache = enable_hierarchical_cache
self.enable_priority_scheduling = enable_priority_scheduling
self.schedule_low_priority_values_first = schedule_low_priority_values_first
self.priority_sign = 1 if schedule_low_priority_values_first else -1
self._shortest_prefill_calls = 0
self._waiting_prefix_refresh_calls = 0
# It is used to find the matching prefix for in-batch prefix caching.
self.waiting_queue_radix_tree = RadixCache.create_simulated()
def calc_priority(
self,
waiting_queue: List[Req],
running_batch: Optional[ScheduleBatch] = None,
processed_tokens: int = 0,
) -> None:
policy = self._determine_active_policy(waiting_queue)
# Cache-agnostic policies sort without matching; match anyway so the load
# snapshot sees num_matched_prefix_tokens. Decode never prefills.
if (
not isinstance(policy, CacheAwarePolicy)
and self.tree_cache.supports_fast_match_prefix()
and get_disagg().disaggregation_mode != "decode"
):
for r in waiting_queue:
match_kv_cache(r, self.tree_cache)
if self.policy == CacheAgnosticPolicy.FCFS:
if self.enable_priority_scheduling:
SchedulePolicy._sort_by_priority_and_fcfs(
waiting_queue, self.priority_sign
)
elif isinstance(policy, CacheAwarePolicy):
temporary_deprioritized = self._compute_prefix_matches(
waiting_queue, policy
)
if policy == CacheAwarePolicy.LPM:
SchedulePolicy._sort_by_longest_prefix(
waiting_queue, temporary_deprioritized
)
elif policy == CacheAwarePolicy.DFS_WEIGHT:
SchedulePolicy._sort_by_dfs_weight(waiting_queue, self.tree_cache)
elif policy == CacheAwarePolicy.HRRN:
SchedulePolicy._sort_by_hrrn(
waiting_queue, temporary_deprioritized, processed_tokens
)
elif policy == CacheAwarePolicy.SHORTEST_PREFILL_FIRST:
SchedulePolicy._sort_by_shortest_prefill(
waiting_queue, temporary_deprioritized
)
self._shortest_prefill_calls += 1
if waiting_queue and self._shortest_prefill_calls % 128 == 1:
logger.info(
"Experimental shortest-prefill-first: queue=%d shortest_uncached=%d",
len(waiting_queue),
self._shortest_prefill_work(waiting_queue[0]),
)
else:
raise ValueError(f"Unknown CacheAware Policy: {policy=}")
else:
if policy == CacheAgnosticPolicy.FCFS:
pass
elif policy == CacheAgnosticPolicy.LOF:
SchedulePolicy._sort_by_longest_output(
waiting_queue,
self.enable_priority_scheduling,
self.priority_sign,
)
elif policy == CacheAgnosticPolicy.RANDOM:
SchedulePolicy._sort_randomly(waiting_queue)
elif policy == CacheAgnosticPolicy.ROUTING_KEY:
if running_batch is not None:
SchedulePolicy._sort_by_routing_key(waiting_queue, running_batch)
else:
raise ValueError(f"Unknown CacheAgnostic Policy: {policy=}")
self._touch_waiting_prefixes(policy, waiting_queue)
def _touch_waiting_prefixes(self, policy: Policy, waiting_queue: List[Req]) -> None:
# Head last, so LRU evicts in reverse admission order; cache-aware policies
# already touch while matching, and other eviction strategies ignore recency.
if (
isinstance(policy, CacheAwarePolicy)
or not waiting_queue
or not envs.SGLANG_ENABLE_WAITING_PREFIX_REFRESH.get()
or get_disagg().disaggregation_mode == "decode"
or get_memory().radix_eviction_policy.lower() != "lru"
or self.tree_cache.supports_fast_match_prefix()
or not self.tree_cache.supports_prefix_sharing()
):
return
refresh = (
self._waiting_prefix_refresh_calls % WAITING_PREFIX_REFRESH_INTERVAL == 0
)
self._waiting_prefix_refresh_calls += 1
if not refresh:
return
for r in reversed(waiting_queue[:WAITING_QUEUE_PREFIX_MATCH_MAX]):
touch_waiting_prefix(r, self.tree_cache)
def _determine_active_policy(self, waiting_queue: List[Req]) -> Policy:
if (
self.policy
in (
CacheAwarePolicy.LPM,
CacheAwarePolicy.HRRN,
)
and len(waiting_queue) > WAITING_QUEUE_PREFIX_MATCH_MAX
):
# Turn off the expensive prefix matching and sorting when the #queue is large.
return CacheAgnosticPolicy.FCFS
return self.policy
def waiting_queue_prefix_matched(self, waiting_queue: List[Req]) -> bool:
policy = self._determine_active_policy(waiting_queue)
return (
isinstance(policy, CacheAwarePolicy)
or self.tree_cache.supports_fast_match_prefix()
)
def _validate_and_adjust_policy(
self, policy: str, tree_cache: BasePrefixCache
) -> Policy:
"""
Validates the policy and adjusts it if necessary based on tree cache settings.
"""
try:
policy_enum = CacheAwarePolicy(policy)
if getattr(tree_cache, "disable", True):
if policy_enum == CacheAwarePolicy.SHORTEST_PREFILL_FIRST:
raise ValueError(
"Experimental shortest-prefill-first requires prefix caching"
)
# If tree_cache is disabled, using CacheAgnosticPolicy policy
return CacheAgnosticPolicy.FCFS
return policy_enum
except ValueError:
try:
return CacheAgnosticPolicy(policy)
except ValueError:
raise ValueError(f"Unknown schedule_policy: {policy=}")
def _compute_prefix_matches(
self, waiting_queue: List[Req], policy: CacheAwarePolicy
) -> Set[int]:
temporary_deprioritized: Set[int] = set()
self.waiting_queue_radix_tree.reset()
for r in waiting_queue:
prefix_ids = r.origin_input_ids + r.output_ids
extra_key = r.extra_key
cache_salt = r.cache_salt
match_result = match_kv_cache(r, self.tree_cache, prefix_ids)
# NOTE(sang): This logic is for in-batch prefix caching;
# If there are more than 1 request that have small matching prefix from
# existing cache, but all those requests share the same prefix, we prefer
# to schedule only one of them so that we can increase the cache hit rate.
# We prefer to set IN_BATCH_PREFIX_CACHING_CHECK_THRESHOLD > 0 because too small
# threshold means we cannot use in-batch prefix caching for short prefixes.
# It is kind of common when the engine is long running (e.g., imagine the prefix "the").
if r.prefix_len <= IN_BATCH_PREFIX_CACHING_CHECK_THRESHOLD:
match_result = self.waiting_queue_radix_tree.match_prefix(
MatchPrefixParams(
key=RadixKey(
token_ids=prefix_ids,
extra_key=extra_key,
cache_salt=cache_salt,
)
)
)
if envs.SGLANG_RADIX_FORCE_MISS.get():
match_result = zero_match_result(
self.waiting_queue_radix_tree, match_result, extra_key=extra_key
)
if (
match_result.device_prefix_len
>= IN_BATCH_PREFIX_CACHING_DEPRIORITIZE_THRESHOLD
):
temporary_deprioritized.add(r.rid)
else:
# Insert with a dummy key
self.waiting_queue_radix_tree.insert(
InsertParams(
key=RadixKey(
token_ids=prefix_ids,
extra_key=extra_key,
cache_salt=cache_salt,
),
value=torch.empty(len(prefix_ids), dtype=torch.bool),
)
)
return temporary_deprioritized
@staticmethod
def _shortest_prefill_work(r: Req) -> int:
return max(
1,
len(r.origin_input_ids) + len(r.output_ids) - r.num_matched_prefix_tokens,
)
@staticmethod
def _sort_by_shortest_prefill(
waiting_queue: List[Req], temporary_deprioritized: Set[int]
) -> None:
# Prioritize short uncached prefills while deferring duplicate prefixes.
waiting_queue.sort(
key=lambda r: (
r.rid in temporary_deprioritized,
SchedulePolicy._shortest_prefill_work(r),
r.time_stats.wait_queue_entry_time,
)
)
def shortest_prefill_chunk_limit(
self, chunked_req: Req, waiting_queue: List[Req], budget: int, page_size: int
) -> Optional[int]:
"""Cap the active prefill chunk to reserve tokens for shorter waiting requests."""
if (
self.policy != CacheAwarePolicy.SHORTEST_PREFILL_FIRST
or budget < 2 * page_size
):
return None
remaining = len(chunked_req.full_untruncated_fill_ids) - chunked_req.prefix_len
reserved = 0
for req in waiting_queue:
work = self._shortest_prefill_work(req)
charge = _ceil_div(work, page_size) * page_size
if work >= remaining or reserved + charge > budget - page_size:
break
reserved += charge
if not reserved:
return None
# Page alignment keeps continuation boundaries allocator-compatible.
return (budget - reserved) // page_size * page_size
@staticmethod
def _sort_by_longest_prefix(
waiting_queue: List[Req], temporary_deprioritized: Set[int]
) -> None:
"""Sorts the waiting queue based on the longest prefix match."""
waiting_queue.sort(
key=lambda r: (
-r.num_matched_prefix_tokens
if r.rid not in temporary_deprioritized
else float("inf")
)
)
@staticmethod
def _uncached_len(r: Req) -> int:
"""Number of tokens that must actually be prefilled for this req
(all cache levels — device + host via hicache — counted as cached)."""
return max(0, len(r.origin_input_ids) - r.num_matched_prefix_tokens)
@staticmethod
def _sort_by_hrrn(
waiting_queue: List[Req],
temporary_deprioritized: Set[int],
processed_tokens: int,
) -> None:
"""Highest Response Ratio Next, with token-based aging.
Equivalence with classic HRRN when throughput is constant:
ratio = 1 + wait_sec / est_prefill_time
= 1 + (processed_tokens - arrival_processed_tokens) / uncached
Caller (Scheduler) contract:
- Maintain a monotonically increasing counter of prefill tokens processed so far
(accumulate batch.extend_num_tokens per forward). Pass it in as `processed_tokens`.
- Snapshot `req.arrival_processed_tokens = counter` when the req enters waiting_queue
(pop_bootstrapped for disagg prefill, _add_request_to_queue for unified).
Call sites that omit `processed_tokens` (dllm, disagg decode)
degrade to rid-lexicographic order; those queues carry no prefill work.
"""
def _key(r: Req):
rid = r.rid
if rid in temporary_deprioritized:
return (float("inf"), rid)
uncached = SchedulePolicy._uncached_len(r)
if uncached <= 0:
# No prefill work; drain immediately.
return (-float("inf"), rid)
waited_tokens = max(0, processed_tokens - r.arrival_processed_tokens)
ratio_delta = waited_tokens / uncached
return (-ratio_delta, rid)
waiting_queue.sort(key=_key)
@staticmethod
def _sort_by_dfs_weight(
waiting_queue: List[Req], tree_cache: BasePrefixCache
) -> None:
"""Sorts the waiting queue based on a depth-first search weighting."""
order = tree_cache.dfs_weight_order([req.last_node for req in waiting_queue])
waiting_queue[:] = [waiting_queue[index] for index in order]
@staticmethod
def _sort_by_longest_output(
waiting_queue: List[Req],
enable_priority_scheduling: bool,
priority_sign: int,
) -> None:
"""Sorts the waiting queue based on the longest output (max_new_tokens). If using priority scheduling, sort by priority first."""
if enable_priority_scheduling:
waiting_queue.sort(
key=lambda x: (
x.priority * priority_sign,
-x.sampling_params.max_new_tokens,
)
)
else:
waiting_queue.sort(key=lambda x: -x.sampling_params.max_new_tokens)
@staticmethod
def _sort_randomly(waiting_queue: List[Req]) -> None:
"""Shuffles the waiting queue randomly."""
random.shuffle(waiting_queue)
@staticmethod
def _sort_by_priority_and_fcfs(
waiting_queue: List[Req], priority_sign: int
) -> None:
"""Sorts the waiting queue based on the request priority then received titmestamp."""
waiting_queue.sort(
key=lambda x: (
x.priority * priority_sign,
x.time_stats.wait_queue_entry_time,
)
)
@staticmethod
def _sort_by_routing_key(
waiting_queue: List[Req], running_batch: ScheduleBatch
) -> None:
"""Sorts waiting queue by routing key frequency in running batch."""
routing_key_counts = Counter(
r.routing_key for r in running_batch.reqs if r.routing_key
)
if _ROUTING_KEY_POLICY_DEBUG_LOG:
waiting_keys_before = [r.routing_key for r in waiting_queue]
logger.info(
f"routing_key_counts={dict(routing_key_counts)}, "
f"waiting_keys_before={waiting_keys_before}"
)
if not routing_key_counts:
return
def sort_key(req: Req):
key = req.routing_key
if key and key in routing_key_counts:
count = routing_key_counts[key]
return (0, -count, key)
else:
return (1, 0, key or "")
waiting_queue.sort(key=sort_key)
if _ROUTING_KEY_POLICY_DEBUG_LOG:
waiting_keys_after = [r.routing_key for r in waiting_queue]
logger.info(f"waiting_keys_after={waiting_keys_after}")
class AddReqResult(Enum):
CONTINUE = auto() # Continue to add requests
NO_TOKEN = auto() # No token left
OTHER = auto() # Other reasons to stop adding requests
@dataclass(frozen=True, slots=True)
class _PrefillAdmission:
prefix_len: int
extend_len: int
max_new_tokens: int
is_chunked: bool
replay_len: int
class PrefillAdder:
def __init__(
self,
page_size: int,
tree_cache: BasePrefixCache,
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
running_batch: ScheduleBatch,
new_token_ratio: float,
rem_input_tokens: int,
rem_chunk_tokens: Optional[int],
num_mixed_decode_tokens: int = 0,
priority_scheduling_preemption_threshold: int = 0,
max_prefill_bs: int = 0,
max_running_requests: Optional[int] = None,
prefill_max_requests: Optional[int] = None,
prefill_delayer_single_pass: Optional[PrefillDelayerSinglePassExecutor] = None,
dllm_config: Optional[DllmConfig] = None,
waiting_queue_len: int = 0,
prefill_tile_block_m: int = 64,
):
self.page_size = page_size
self.prefill_tile_block_m = prefill_tile_block_m
self.tree_cache = tree_cache
self.token_to_kv_pool_allocator = token_to_kv_pool_allocator
self.running_batch = running_batch
self.new_token_ratio = new_token_ratio
self.rem_input_tokens = rem_input_tokens - num_mixed_decode_tokens
self.rem_chunk_tokens = rem_chunk_tokens
self.chunked_req_limit: Optional[int] = None
self.dllm_config = dllm_config
# Mamba checkpoints only land on page-aligned chunk ends.
self.exact_chunk_fill = (
_use_exact_chunk_fill()
and dllm_config is None
and not (
tree_cache.supports_mamba() and tree_cache.supports_prefix_sharing()
)
)
if self.dllm_config is not None:
self._init_dllm_meta(dllm_config)
if self.rem_chunk_tokens is not None:
self.rem_chunk_tokens -= num_mixed_decode_tokens
self.memory_budget = token_to_kv_pool_allocator.create_prefill_budget(
tree_cache, num_mixed_decode_tokens=num_mixed_decode_tokens
)
self.req_states = None
self.can_run_list = []
self.preempt_list = []
self.new_chunked_req = None
self.log_hit_tokens = 0
self.reprocessed_log_hit_tokens = 0
self.log_device_hit_tokens = 0
self.log_host_hit_tokens = 0
self.log_storage_hit_tokens = 0
self.log_input_tokens = 0
self.log_replay_tokens = 0
self.reprocessed_log_input_tokens = 0
if running_batch is not None:
# Estimate the offset in the remaining token space
self.memory_budget.total_offset += sum(
[
self._get_running_request_total_token_offset(r)
for r in running_batch.reqs
]
)
# DeepSeek V4 HiSparse wraps an SWATokenToKVPoolAllocator internally and
# exposes the full SWA allocator interface.
self.is_hybrid_swa = isinstance(
self.token_to_kv_pool_allocator,
(SWATokenToKVPoolAllocator, DeepSeekV4HiSparseTokenToKVPoolAllocator),
)
self.is_hybrid_ssm_cache = self.tree_cache.supports_mamba()
# A new state slot eats shared-gap bytes that `rem_total_tokens` counts
# as free, so reserve per slot or admission over-commits. Gate on the
# ALLOCATOR: the reservation holds with the radix cache disabled too.
self._mamba_slot_cost = 0
if isinstance(
self.token_to_kv_pool_allocator,
(
UnifiedMambaTokenToKVPoolAllocator,
UnifiedMambaSWATokenToKVPoolAllocator,
),
):
self._mamba_slot_cost = (
self.token_to_kv_pool_allocator.mamba_slot_full_token_cost()
)
# `mamba_gap_reserve` is charged to `rem_total_tokens`, which INCLUDES
# `full_evictable_size()` — but `alloc_req_slots` can only recover
# MAMBA-recoverable bytes for a mamba slot (shared gap + peer holes +
# mamba-evictable radix), NOT full-evictable. Gate new mamba slots on
# that mamba-recoverable budget separately or an over-admit hits the
# fail-loud `RuntimeError`. `None` outside the unified Mamba pool.
self.rem_mamba_slots = None
if self._mamba_slot_cost:
self.rem_mamba_slots = self.token_to_kv_pool_allocator.mamba_allocator.schedulable_available_size()
if self.is_hybrid_ssm_cache:
self.rem_mamba_slots += self.tree_cache.mamba_evictable_size()
self.priority_scheduling_preemption_threshold = (
priority_scheduling_preemption_threshold
)
self.max_running_requests = max_running_requests
# Align sharded chunk boundaries to physical pages; 0 disables alignment.
kv_shard_size = page_interleave_shard_size(self.token_to_kv_pool_allocator)
self.kv_shard_granule = (
self.token_to_kv_pool_allocator.page_size if kv_shard_size > 1 else 0
)
self.kv_shard_size = kv_shard_size
# Each admission must fit the batch's prefix and chunk scratch regions.
self.kv_shard_scratch_spec = (
self.token_to_kv_pool_allocator.shard_spec if kv_shard_size > 1 else None
)
self.kv_shard_block_bound_pages = 0
self.kv_shard_chunk_pages = 0
# Reserve one extra page per shard class for allocation rounding.
self.per_req_token_overhead = (
kv_shard_size * self.kv_shard_granule
if self.kv_shard_granule
else self.page_size
)
self.prefill_max_requests = prefill_max_requests
self.prefill_delayer_single_pass = prefill_delayer_single_pass
self.max_prefill_bs = max_prefill_bs
# Snapshot of scheduler waiting_queue length at the start of this
# prefill pass. Used by PrefillDelayer's queue-based trigger.
self.waiting_queue_len = waiting_queue_len
def _admitted_extend_lens(self) -> List[int]:
return [int(getattr(req, "extend_input_len", 0)) for req in self.can_run_list]
def _tile_admission_metric_key(self) -> str:
return f"{PREFILL_TILE_BUDGET_MODE}_q_tiles_per_head"
def _candidate_tile_metrics(self, candidate_extend_len: int) -> Dict[str, object]:
return estimate_prefill_extend_tile_metrics(
[*self._admitted_extend_lens(), int(candidate_extend_len)],
block_m=self.prefill_tile_block_m,
)
def _check_prefill_tile_budget(
self, candidate_extend_len: int
) -> Optional[AddReqResult]:
# AMD-only: leave non-AMD scheduler admission unchanged even if the env
# budget is set.
if not _IS_HIP or PREFILL_TILE_BUDGET <= 0:
return None
if not self.can_run_list:
return None
metrics = self._candidate_tile_metrics(candidate_extend_len)
candidate_metric = int(metrics.get(self._tile_admission_metric_key()) or 0)
if candidate_metric <= PREFILL_TILE_BUDGET:
return None
return AddReqResult.OTHER
def _init_dllm_meta(self, dllm_config: DllmConfig):
self.dllm_block_size = dllm_config.block_size
max_running_reqs = dllm_config.max_running_requests
self.rem_dllm_tokens = (
self.rem_input_tokens
if dllm_config.requires_separate_context_encoding
else max_running_reqs * self.dllm_block_size
)
def _get_running_request_total_token_offset(self, req: Req) -> int:
# Fixed-length requests cannot benefit from the early-EOS estimate.
ratio = 1.0 if req.sampling_params.ignore_eos else self.new_token_ratio
return (
min(
(req.sampling_params.max_new_tokens - len(req.output_ids)),
CLIP_MAX_NEW_TOKENS,
)
* ratio
)
@property
def rem_total_tokens(self):
return self.memory_budget.remaining_total
@property
def cur_rem_tokens(self):
return self.memory_budget.remaining_current
def _swa_new_tokens(self, req: Req) -> int:
"""Tokens a request may still decode, for SWA headroom sizing. Mirrors
the remaining-then-clip order of add_one_req's max_new: clip-then-subtract
would zero out a request that has already generated >= CLIP tokens but
still has a long decode ahead, under-reserving its window."""
return min(
max(req.sampling_params.max_new_tokens - len(req.output_ids), 0),
CLIP_MAX_NEW_TOKENS,
)
def _check_prefill_budget(
self,
req: Req,
*,
extend_input_len: int,
total_tokens: int,
swa_host_hit_length: int,
chunk_limit: Optional[int],
) -> tuple[bool, Optional[int]]:
return self.memory_budget.check_prefill(
extend_input_len=extend_input_len,
total_tokens=total_tokens,
max_new_tokens=self._swa_new_tokens(req),
input_tokens=len(req.full_untruncated_fill_ids),
swa_host_hit_length=swa_host_hit_length,
chunk_limit=chunk_limit,
auxiliary_swa_host_hit_length=req.auxiliary_swa_host_hit_length,
)
def _mamba_gap_budget_for_req(
self, req: Req, *, include_host_hit: bool = False
) -> int:
"""Reserve full-token equivalents for shared-pool Mamba state slots.
Admission includes the host checkpoint before load-back binds the
request slot. Continuing chunks only need the request-state budget;
their host-hit metadata may still describe an already consumed hit.
Tracking buffers and radix CoW headroom are not included here.
"""
if not self._mamba_slot_cost:
return 0
slots = int(not req.kv.holds_mamba)
if include_host_hit:
slots += req.mamba_host_hit_length
return slots * self._mamba_slot_cost
def _request_slot_reserve_for_req(
self, req: Req, *, extend_input_len: Optional[int] = None
) -> int:
if extend_input_len is None:
extend_input_len = len(req.full_untruncated_fill_ids) - req.prefix_len
return self.token_to_kv_pool_allocator.request_slot_reserve(
has_req_pool_slot=req.kv.req_pool_idx is not None,
prefill_tokens=req.prefix_len + extend_input_len,
)
def ceil_paged_tokens(self, tokens: int) -> int:
return -(-tokens // self.page_size) * self.page_size
def budget_state(self):
no_token = not self.memory_budget.has_capacity()
# Gate new mamba slots separately: rem_total_tokens' full_evictable can't
# cover a mamba slot, which needs mamba-recoverable bytes (see __init__).
if not no_token and self.rem_mamba_slots is not None:
no_token = self.rem_mamba_slots <= 0
if no_token:
return AddReqResult.NO_TOKEN
if self.rem_input_tokens <= 0:
return AddReqResult.OTHER
if self.dllm_config is not None:
if self.rem_dllm_tokens <= 0:
return AddReqResult.OTHER
else:
if self.rem_chunk_tokens is not None and self.rem_chunk_tokens <= 0:
return AddReqResult.OTHER
return AddReqResult.CONTINUE
def _update_prefill_budget(
self,
prefix_len: int,
extend_input_len: int,
max_new_tokens: int,
retracted_stain: bool,
mamba_gap_reserve: int = 0,
request_slot_reserve: int = 0,
is_chunked_continuation: bool = False,
replay_len: int = 0,
):
"""Charge one admitted request against the prefill budgets.
The compute budgets (`rem_chunk_tokens`, `rem_input_tokens`,
`rem_dllm_tokens`) take the rows the step runs: the extend, plus
`replay_len` cached tokens that encoder replay recomputes. The KV
budgets take only the page-ceiled extend. Exact-chunk-fill bills the
compute budgets the raw extend instead, all alike, so none runs out
first with rounding slack unspent.
"""
# TODO(lsyin): check this workaround logic, which only ensures the prefill will not out of memory, and may be too conservative
raw_extend_input_len = extend_input_len
extend_input_len = self.ceil_paged_tokens(extend_input_len)
compute_charge = (
raw_extend_input_len if self.exact_chunk_fill else extend_input_len
) + replay_len
self.memory_budget.reserve(
extend_input_len,
max_new_tokens,
# reserve() already charges one page; add the remaining shard pages.
extra_tokens=(
mamba_gap_reserve
+ request_slot_reserve
+ self.per_req_token_overhead
- self.page_size
),
chunk_limit=self.rem_chunk_tokens,
is_chunked_continuation=is_chunked_continuation,
)
# Debit every reserved state slot, including the load-back checkpoint.
if mamba_gap_reserve and self.rem_mamba_slots is not None:
self.rem_mamba_slots -= mamba_gap_reserve // self._mamba_slot_cost
self.rem_input_tokens -= compute_charge
if self.dllm_config is not None:
self.rem_dllm_tokens -= compute_charge
elif self.rem_chunk_tokens is not None:
self.rem_chunk_tokens -= compute_charge
# reprocessed_log_* is a subset of log_*; metrics_reporter subtracts it
# when computing the first-attempt prefix cache hit rate.
self.log_hit_tokens += prefix_len
self.log_input_tokens += raw_extend_input_len
self.log_replay_tokens += replay_len
if retracted_stain:
self.reprocessed_log_hit_tokens += prefix_len
self.reprocessed_log_input_tokens += raw_extend_input_len
def _replay_len(self, req: Req, prefix_len: int) -> int:
if (
get_exec().features.enable_encoder_swa_bounded_replay
and encoder_swa_replay.resets_swa_window(req)
):
return encoder_swa_replay.replay_len(prefix_len)
return 0
def _account_prefill_cache_admission(self, req: Req, prefix_len: int) -> None:
if req.retracted_stain:
# Retraction attribution is intentionally omitted for now; discard
# its lifecycle state so a later abort cannot report it as a drop.
self.tree_cache.discard_storage_prefetch_accounting(
req.cache_request_handle
)
self.tree_cache.finish_storage_prefetch_admission(
req.cache_request_handle, fulfilled_tokens=0, reason=None
)
return
if prefix_len > 0:
device_hit, host_hit, storage_hit = split_cached_prefix_by_tier(
prefix_len=prefix_len,
host_hit_len=req.materialized_host_hit_len(),
storage_hit_len=req.storage_hit_length,
storage_hit_start=req.storage_hit_start,
host_hit_is_storage=req.host_hit_is_storage,
)
self.log_device_hit_tokens += device_hit
self.log_host_hit_tokens += host_hit
self.log_storage_hit_tokens += storage_hit
fulfilled_storage_hit = req.fulfilled_storage_hit_len(prefix_len)
reason = None
if fulfilled_storage_hit < req.storage_hit_length:
reason = (
"device_capacity"
if req.needs_host_load_back()
and req.host_loaded_length < req.host_hit_length
else "cache_admission_shortfall"
)
self.tree_cache.finish_storage_prefetch_admission(
req.cache_request_handle,
fulfilled_tokens=fulfilled_storage_hit,
reason=reason,
)
def _get_dllm_remain_tokens(self, req: Optional[Req] = None) -> int:
_rem_tokens = min(self.rem_dllm_tokens, int(self.rem_total_tokens))
if not (
self.dllm_config.requires_separate_context_encoding
and req is not None
and req.is_dllm_prefill()
):
_rem_tokens = min(_rem_tokens, self.dllm_block_size)
if _rem_tokens <= 0:
_rem_tokens = self.rem_dllm_tokens
if self.dllm_config.requires_separate_context_encoding:
_rem_tokens = min(_rem_tokens, int(self.cur_rem_tokens) - self.page_size)
if self.is_hybrid_swa:
_rem_tokens = min(
_rem_tokens, int(self.memory_budget.remaining_swa) - self.page_size
)
return _rem_tokens
def _get_dllm_extend_len(self, req: Req, prefix_len: int) -> int:
if self.dllm_config.requires_separate_context_encoding:
remaining = (
req.dllm_block_offset - prefix_len
if req.is_dllm_prefill()
else len(req.full_untruncated_fill_ids) - prefix_len
)
limit = self.rem_dllm_tokens
if not req.is_dllm_prefill():
limit = min(limit, self.dllm_block_size)
trunc_len = min(limit, remaining)
else:
trunc_len = (
min(self.rem_dllm_tokens, self.dllm_block_size)
// self.page_size
* self.page_size
)
return trunc_len
def _add_dllm_req(self, req: Req):
prefix_len = req.prefix_len
trunc_len = self._get_dllm_extend_len(req, prefix_len)
req.extend_end = prefix_len + trunc_len
self.can_run_list.append(req)
self._update_prefill_budget(
prefix_len,
trunc_len,
0,
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
replay_len=self._replay_len(req, prefix_len),
)
self._account_prefill_cache_admission(req, prefix_len)
def _kv_shard_reserve_scratch(self, prefix_len: int, extend_len: int) -> bool:
"""Reserve scratch or return False to defer; no-op when sharding is off.
Prefixes use N * sum_i ceil(prefix_pages_i / N) pages; extends use
sum_i ceil(extend_i / ps). A request must fit an empty batch or raise.
"""
if self.kv_shard_scratch_spec is None:
return True
ps = self.kv_shard_granule
shard_size = self.kv_shard_size
block = self.kv_shard_block_bound_pages + -(-(prefix_len // ps) // shard_size)
chunk = self.kv_shard_chunk_pages + -(-extend_len // ps)
if (
shard_size * block * ps > self.kv_shard_scratch_spec.max_prefix_tokens
or chunk * ps > self.kv_shard_scratch_spec.chunk_tokens
):
if self.kv_shard_block_bound_pages == 0 and self.kv_shard_chunk_pages == 0:
# Deferring a request that cannot fit an empty batch would livelock.
raise RuntimeError(
"request cannot fit the sharded assembly scratch even in "
f"an empty batch (prefix_len={prefix_len}, extend_len="
f"{extend_len}, max_prefix_tokens="
f"{self.kv_shard_scratch_spec.max_prefix_tokens}, "
f"chunk_tokens={self.kv_shard_scratch_spec.chunk_tokens}); "
"KV sharding requires chunked prefill sized within the "
"assembly scratch"
)
return False
self.kv_shard_block_bound_pages = block
self.kv_shard_chunk_pages = chunk
return True
def add_dllm_staging_req(self, req: Req):
assert self.dllm_config is not None
_rem_tokens = self._get_dllm_remain_tokens(req)
if _rem_tokens <= 0:
return AddReqResult.NO_TOKEN
# Truncate input length to available tokens and update request metadata
cand_extend_input_len = len(req.full_untruncated_fill_ids) - req.prefix_len
if (
self.dllm_config.requires_separate_context_encoding
and req.is_dllm_prefill()
):
cand_extend_input_len = min(
cand_extend_input_len,
req.dllm_block_offset - req.prefix_len,
)
if (
req.dllm_incomplete_ids
or (
self.dllm_config.requires_separate_context_encoding
and not req.is_dllm_prefill()
)
) and cand_extend_input_len > _rem_tokens:
return AddReqResult.NO_TOKEN
truncated = cand_extend_input_len > _rem_tokens
new_len = min(cand_extend_input_len, _rem_tokens)
req.extend_end = req.prefix_len + new_len
self.can_run_list.append(req)
# Update budget: reserve max_new_tokens only if not truncated
max_new_tokens = (
min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS)
if not truncated
else 0
)
self._update_prefill_budget(
0,
req.extend_len,
max_new_tokens,
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
)
# Return based on remaining token availability
return (
AddReqResult.NO_TOKEN
if self._get_dllm_remain_tokens(req) <= 0
else AddReqResult.CONTINUE
)
def add_chunked_req(self, req: Req):
if self.dllm_config is not None:
_rem_tokens = self._get_dllm_remain_tokens(req)
else:
_rem_tokens = self.memory_budget.available_chunk_tokens(
self.rem_chunk_tokens
)
if _rem_tokens is None:
return req
if self.kv_shard_granule:
# Align the absolute boundary to keep pages within one scratch region.
# If no budget remains, use a full chunk to make progress.
prefix_len = req.prefix_len
floored = (
prefix_len + _rem_tokens
) // self.kv_shard_granule * self.kv_shard_granule - prefix_len
_rem_tokens = floored if floored > 0 else self.rem_chunk_tokens
# A mid-chunk rank prefills this pass regardless of the delayer
# verdict, so report prefillable=True and ignore the result.
if self.prefill_delayer_single_pass is not None:
self.prefill_delayer_single_pass.negotiate_should_allow_prefill(
local_prefillable=True,
running_batch=self.running_batch.batch_size(),
max_prefill_bs=self.max_prefill_bs,
max_running_requests=self.max_running_requests,
waiting_queue_len=self.waiting_queue_len,
)
if self.chunked_req_limit is not None:
assert self.chunked_req_limit > 0
_rem_tokens = min(_rem_tokens, self.chunked_req_limit)
cand_extend_input_len = len(req.full_untruncated_fill_ids) - req.prefix_len
_rem_tokens = self.memory_budget.fit_chunk(
extend_input_len=cand_extend_input_len,
max_new_tokens=self._swa_new_tokens(req),
chunk_limit=_rem_tokens,
)
if _rem_tokens is None:
return req
truncated = cand_extend_input_len > _rem_tokens
new_len = min(cand_extend_input_len, _rem_tokens)
request_slot_reserve = self._request_slot_reserve_for_req(
req, extend_input_len=new_len
)
if request_slot_reserve and not self.memory_budget.can_allocate_prefill(
paged_input=(
self.ceil_paged_tokens(new_len)
+ self.per_req_token_overhead
+ request_slot_reserve
),
extend_input_len=new_len,
max_new_tokens=self._swa_new_tokens(req),
chunk_limit=self.rem_chunk_tokens,
):
return req
# The continuing chunk must fit. Keep reservation outside assert for -O.
reserved = self._kv_shard_reserve_scratch(
prefix_len=req.prefix_len, extend_len=new_len
)
if not reserved:
raise RuntimeError("chunked request exceeds the sharded assembly scratch")
req.extend_end = req.prefix_len + new_len
self.can_run_list.append(req)
self._update_prefill_budget(
0,
req.extend_len,
(
min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS)
if not truncated
else 0
),
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
request_slot_reserve=request_slot_reserve,
is_chunked_continuation=True,
)
# Return if chunked prefill not finished
return req if truncated else None
@contextmanager
def _lock_node(self, last_node: TreeNode, *, lock_host: bool = False):
host_lock_params = (
self.tree_cache.inc_host_lock_ref(last_node).to_dec_params()
if lock_host
else None
)
try:
lock = self.tree_cache.lock(last_node)
try:
yield None
finally:
self.tree_cache.unlock(lock)
finally:
if host_lock_params is not None:
self.tree_cache.dec_host_lock_ref(last_node, host_lock_params)
def add_one_req_ignore_eos(self, req: Req):
cand_extend_input_len = len(req.full_untruncated_fill_ids) - req.prefix_len
paged_input = (
self.ceil_paged_tokens(cand_extend_input_len) + self.per_req_token_overhead
)
# Shared Mamba pool: fold the new mamba state's shared-gap cost into the
# budget gate so admission can't over-commit (0 for baseline / non-Mamba).
paged_input += self._mamba_gap_budget_for_req(req)
paged_input += self._request_slot_reserve_for_req(req)
fits = self.memory_budget.can_allocate_prefill(
paged_input=paged_input,
extend_input_len=cand_extend_input_len,
max_new_tokens=self._swa_new_tokens(req),
chunk_limit=self.rem_chunk_tokens,
)
if not fits:
return AddReqResult.NO_TOKEN
def add_req_state(r, insert_sort=False):
new_token_ratio = (
1.0 if r.sampling_params.ignore_eos else self.new_token_ratio
)
tokens_left = r.sampling_params.max_new_tokens * new_token_ratio - len(
r.output_ids
)
tokens_occupied = len(r.origin_input_ids) + len(r.output_ids)
if tokens_left <= 0:
return
if not insert_sort:
self.req_states.append((tokens_left, tokens_occupied))
else:
i = 0
for i in range(len(self.req_states)):
if tokens_left <= self.req_states[i][0]:
break
self.req_states.insert(i, (tokens_left, tokens_occupied))
if self.req_states is None:
self.req_states = []
add_req_state(req)
if self.running_batch is not None:
for r in self.running_batch.reqs:
add_req_state(r)
for r in self.can_run_list:
add_req_state(r)
self.req_states.sort(key=lambda x: x[0])
else:
add_req_state(req, insert_sort=True)
if not self.is_hybrid_swa:
# Skip this logic for swa. The SWA has different memory management, and
# this mechanism is underestimating the memory usage.
cur_rem_tokens = self.cur_rem_tokens - paged_input
tokens_freed = 0
for i, (tokens_left, tokens_occupied) in enumerate(self.req_states):
# tokens_left gives a reservative calculation as the last token is not stored
bs = len(self.req_states) - i
min_free_tokens = cur_rem_tokens + tokens_freed - tokens_left * bs
# reserve tokens for corner cases
if min_free_tokens <= IGNORE_EOS_RESERVE_TOKENS * bs:
return AddReqResult.NO_TOKEN
tokens_freed += tokens_occupied
if (self.prefill_delayer_single_pass is not None) and (
not self.prefill_delayer_single_pass.negotiate_should_allow_prefill(
local_prefillable=True,
running_batch=self.running_batch.batch_size(),
max_prefill_bs=self.max_prefill_bs,
max_running_requests=self.max_running_requests,
waiting_queue_len=self.waiting_queue_len,
)
):
return AddReqResult.OTHER
if self.dllm_config is not None:
if self.rem_dllm_tokens <= 0:
return AddReqResult.OTHER
if (
tile_stop := self._check_prefill_tile_budget(cand_extend_input_len)
) is not None:
return tile_stop
self._add_dllm_req(req)
elif (
self.rem_chunk_tokens is None # chunked prefill is disabled
or cand_extend_input_len <= self.rem_chunk_tokens # it is the last chunk
):
if (
tile_stop := self._check_prefill_tile_budget(cand_extend_input_len)
) is not None:
return tile_stop
# Non-chunked prefill — the whole sequence is committed this iter.
if not self._kv_shard_reserve_scratch(
prefix_len=req.prefix_len,
extend_len=cand_extend_input_len,
):
return AddReqResult.OTHER
req.extend_end = len(req.full_untruncated_fill_ids)
self.can_run_list.append(req)
self._update_prefill_budget(
0,
req.extend_len,
min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS),
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
request_slot_reserve=self._request_slot_reserve_for_req(req),
)
else:
if self.rem_chunk_tokens <= 0:
return AddReqResult.OTHER
# Chunked prefill
trunc_len = self.rem_chunk_tokens
if (tile_stop := self._check_prefill_tile_budget(trunc_len)) is not None:
return tile_stop
# Reserve after the tile gate so rejected requests consume no scratch.
if not self._kv_shard_reserve_scratch(prefix_len=0, extend_len=trunc_len):
return AddReqResult.OTHER
assert req.prefix_len == 0
req.extend_end = req.prefix_len + trunc_len
self.can_run_list.append(req)
self.new_chunked_req = req
self._update_prefill_budget(
0,
trunc_len,
0,
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
request_slot_reserve=self._request_slot_reserve_for_req(
req, extend_input_len=trunc_len
),
)
return self.budget_state()
def _defer_staged_for_capacity_if_idle(self, req: Req, *, pool: str) -> None:
# Running work and admitted prefills (including continuing chunks) can
# release capacity. Only idle budget failures should force recomputation.
if self.can_run_list or (
self.running_batch is not None and self.running_batch.reqs
):
return
if self.tree_cache.buffer_pipeline is not None:
self.tree_cache.buffer_pipeline.defer_staged_admission(req, pool)
def add_one_req(
self, req: Req, has_chunked_req: bool, truncation_align_size: Optional[int]
):
if (x := self.prefill_max_requests) is not None and len(self.can_run_list) >= x:
return AddReqResult.OTHER
if req.sampling_params.ignore_eos and getattr(self.tree_cache, "disable", True):
return self.add_one_req_ignore_eos(req)
# Reserve page_size for page-alignment overhead: the paged allocator may
# consume one extra page per request (see alloc_extend), which
# _update_prefill_budget also deducts.
max_new = min(
max(req.sampling_params.max_new_tokens - len(req.output_ids), 0),
CLIP_MAX_NEW_TOKENS,
)
cand_extend_input_len = len(req.full_untruncated_fill_ids) - req.prefix_len
total_tokens = cand_extend_input_len + max_new + self.per_req_token_overhead
# Reserve request and load-back state slots before init_load_back binds
# the request slot. Reuse this reservation when committing admission.
mamba_gap_reserve = self._mamba_gap_budget_for_req(req, include_host_hit=True)
total_tokens += mamba_gap_reserve
if self.rem_mamba_slots is not None:
required_mamba_slots = mamba_gap_reserve // self._mamba_slot_cost
if required_mamba_slots > self.rem_mamba_slots:
if (
isinstance(self.tree_cache, UnifiedRadixCache)
and self.tree_cache.buffer_pipeline is not None
):
self.tree_cache.buffer_pipeline.defer_staged_admission(req, "mamba")
return AddReqResult.NO_TOKEN
request_slot_reserve = self._request_slot_reserve_for_req(req)
total_tokens += request_slot_reserve
# The temporary pin excludes this prefix from the evictable budget.
# Selection itself neither allocates slots nor materializes host hits.
with self._lock_node(req.last_node):
admission = self._select_prefill_admission(
req,
total_tokens=total_tokens,
host_hit_length=req.host_hit_length,
swa_host_hit_length=req.swa_host_hit_length,
truncation_align_size=truncation_align_size,
has_chunked_req=has_chunked_req,
)
if isinstance(admission, AddReqResult):
if admission is AddReqResult.NO_TOKEN:
self._defer_staged_for_capacity_if_idle(req, pool="prefill_budget")
return admission
# A rejected candidate must not report prefillable or queue H2D.
if (self.prefill_delayer_single_pass is not None) and (
not self.prefill_delayer_single_pass.negotiate_should_allow_prefill(
local_prefillable=True,
running_batch=self.running_batch.batch_size(),
max_prefill_bs=self.max_prefill_bs,
max_running_requests=self.max_running_requests,
waiting_queue_len=self.waiting_queue_len,
)
):
return AddReqResult.OTHER
if req.needs_host_load_back():
load_max_new = min(max_new, admission.max_new_tokens)
# Reclaim can write back device victims and evict host leaves.
# Pin the selected host/aux match until load-back owns its locks.
with (
self._lock_node(req.best_match_node, lock_host=True)
if isinstance(self.tree_cache, UnifiedRadixCache)
else nullcontext()
):
full_load_tokens = req.host_hit_length
if (
self.memory_budget.load_back_needs_full_tokens
and isinstance(self.tree_cache, UnifiedRadixCache)
and self.tree_cache.buffer_pipeline is None
and not (
self.tree_cache.linker is not None
and self.tree_cache.linker.has_hit(req.rid)
)
):
# Host hits can include resident FULL behind host-only aux.
# Reuse the FULL transfer spec to count only new slots.
full_transfer = (
self.tree_cache.tree_core.build_hicache_transfers(
ComponentType.FULL,
req.best_match_node,
CacheTransferPhase.LOAD_BACK,
)[0]
)
full_load_tokens = len(full_transfer.host_indices)
if not self.memory_budget.prepare_load_back(
full_tokens=(
full_load_tokens
+ admission.extend_len
+ load_max_new
+ self.page_size
+ mamba_gap_reserve
),
extend_input_len=admission.extend_len,
max_new_tokens=load_max_new,
swa_host_hit_length=req.swa_host_hit_length,
chunk_limit=self.rem_chunk_tokens,
):
self._defer_staged_for_capacity_if_idle(
req, pool="load_back_budget"
)
return AddReqResult.NO_TOKEN
promised_host_hit = req.host_hit_length
loaded = self.tree_cache.init_load_back(
InitLoadBackParams(
best_match_node=req.best_match_node,
host_hit_length=req.host_hit_length,
req=req,
)
)
if loaded is None:
return AddReqResult.OTHER
req.host_loaded_length, req.last_node = loaded
if 0 < req.host_loaded_length < promised_host_hit:
raise RuntimeError(
"HiCache load-back must commit all promised FULL tokens or none: "
f"req={req.rid} promised={promised_host_hit} "
f"loaded={req.host_loaded_length}"
)
if req.host_loaded_length > promised_host_hit:
# A load can expose resident FULL behind host-only aux state; its
# H2D is queued, so keep the approved budget and shrink the work.
prefix_len = req.prefix_len + req.host_loaded_length
extend_len = admission.extend_len
if self.dllm_config is None:
extend_len = min(
extend_len, len(req.full_untruncated_fill_ids) - prefix_len
)
is_chunked = admission.is_chunked and (
prefix_len + extend_len < len(req.full_untruncated_fill_ids)
)
max_new_tokens = admission.max_new_tokens
if admission.is_chunked and not is_chunked:
max_new_tokens = min(
req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS
)
admission = _PrefillAdmission(
prefix_len,
extend_len,
max_new_tokens,
is_chunked,
self._replay_len(req, prefix_len),
)
elif req.host_loaded_length < promised_host_hit:
# No FULL was loaded; recomputation may no longer fit.
admission = self._select_prefill_admission(
req,
total_tokens=total_tokens,
host_hit_length=0,
swa_host_hit_length=0,
truncation_align_size=truncation_align_size,
has_chunked_req=has_chunked_req,
)
if isinstance(admission, AddReqResult):
return admission
req.prefix_len += req.host_loaded_length
if self.tree_cache.load_back_is_cache_owned():
req.kv.cache_protected_len = req.prefix_len
# Sharded pools cannot load host KV; reserve scratch after all other gates.
if not self._kv_shard_reserve_scratch(
prefix_len=admission.prefix_len, extend_len=admission.extend_len
):
return AddReqResult.OTHER
self._commit_prefill_admission(req, admission, mamba_gap_reserve)
# This verdict controls the next candidate, not the committed request.
return self.budget_state()
def _select_prefill_admission(
self,
req: Req,
*,
total_tokens: int,
host_hit_length: int,
swa_host_hit_length: int,
truncation_align_size: Optional[int],
has_chunked_req: bool = False,
) -> _PrefillAdmission | AddReqResult:
"""Select a prefill shape without allocating or publishing cached KV."""
prefix_len = req.prefix_len + host_hit_length
extend_len = len(req.full_untruncated_fill_ids) - prefix_len
input_tokens = self.ceil_paged_tokens(extend_len)
# Whether the request fits whole. Against the raw length under
# exact-chunk-fill, so a request whose ceiled length would spill is
# not needlessly split into a second chunk.
chunk_fit_tokens = extend_len if self.exact_chunk_fill else input_tokens
replay_len = self._replay_len(req, prefix_len)
chunk_limit = self.rem_chunk_tokens
if chunk_limit is not None and replay_len:
# The replay rows share the chunk with the extend.
chunk_limit -= replay_len
if chunk_limit <= 0:
return AddReqResult.OTHER
can_admit, chunk_tokens_limit = self._check_prefill_budget(
req,
extend_input_len=extend_len,
total_tokens=total_tokens,
swa_host_hit_length=swa_host_hit_length,
chunk_limit=chunk_limit,
)
if not can_admit:
return AddReqResult.NO_TOKEN
# Without chunking, allow the first request even above the input cap.
if (
self.rem_chunk_tokens is None
and self.can_run_list
and input_tokens >= self.rem_input_tokens
):
return AddReqResult.OTHER
is_chunked = False
max_new_tokens = min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS)
tile_tokens = input_tokens
if self.dllm_config is not None:
assert truncation_align_size is None, (
"truncation_align_size is not supported for dllm prefill"
)
extend_len = self._get_dllm_extend_len(req, prefix_len)
if extend_len <= 0:
return AddReqResult.OTHER
max_new_tokens = 0
elif chunk_tokens_limit is not None and chunk_fit_tokens > chunk_tokens_limit:
if (
has_chunked_req
and get_schedule().schedule_policy == "shortest-prefill-first"
):
# Only one unfinished chunked request can be tracked.
return AddReqResult.OTHER
if self.exact_chunk_fill:
# Take the remainder verbatim so the batch hits exactly
# chunked_prefill_size. `chunk_fit_tokens > chunk_tokens_limit`
# here, so this never runs past the end of the prompt. Uses the
# limit rather than rem_chunk_tokens so an SWA-capped chunk stays
# capped.
extend_len = chunk_tokens_limit
if truncation_align_size is not None:
extend_len = (
extend_len // truncation_align_size * truncation_align_size
)
else:
extend_len = chunk_tokens_limit // self.page_size * self.page_size
if truncation_align_size is not None:
extend_len = (
extend_len // truncation_align_size * truncation_align_size
)
end = (prefix_len + extend_len) // self.page_size * self.page_size
extend_len = end - prefix_len
if extend_len <= 0:
return AddReqResult.OTHER
is_chunked = True
max_new_tokens = 0
tile_tokens = extend_len
if (verdict := self._check_prefill_tile_budget(tile_tokens)) is not None:
return verdict
return _PrefillAdmission(
prefix_len, extend_len, max_new_tokens, is_chunked, replay_len
)
def _commit_prefill_admission(
self, req: Req, admission: _PrefillAdmission, mamba_gap_reserve: int
) -> None:
assert req.prefix_len == admission.prefix_len
req.extend_end = admission.prefix_len + admission.extend_len
req.lock = self.tree_cache.lock(req.last_node)
self.can_run_list.append(req)
if admission.is_chunked:
self.new_chunked_req = req
self._update_prefill_budget(
admission.prefix_len,
admission.extend_len,
admission.max_new_tokens,
req.retracted_stain,
mamba_gap_reserve=mamba_gap_reserve,
request_slot_reserve=self._request_slot_reserve_for_req(
req, extend_input_len=admission.extend_len
),
replay_len=admission.replay_len,
)
self._account_prefill_cache_admission(req, admission.prefix_len)
def preempt_to_schedule(self, req: Req) -> bool:
"""
Preempt running requests to serve the new request if the priority threshold is met and token count sum is verified.
Returns True if preemption was committed, and the new request can be scheduled.
"""
# Iterate running requests to find preemptible requests
priority_sign = 1 if get_schedule().schedule_low_priority_values_first else -1
# NOTE: A request finishes in two phases:
# 1) update_finish_state + release_kv_cache (in process_batch_result)
# 2) filter out of batch (in get_next_batch_to_run / update_running_batch)
# Preemption runs between these two phases (inside get_new_batch_prefill),
# so running_batch may still contain requests whose KV cache is already freed.
# We must skip them here to avoid a double-free on release_req.
valid_running_reqs = (
r
for r in self.running_batch.reqs
if r not in self.preempt_list and not r.finished()
)
sorted_valid_running_reqs = sorted(
valid_running_reqs,
key=lambda x: (
x.priority * (-priority_sign),
-x.time_stats.wait_queue_entry_time,
),
)
preemptible_reqs = []
min_tokens_to_remove = (
len(req.full_untruncated_fill_ids)
- req.prefix_len
+ min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS)
+ self._request_slot_reserve_for_req(req)
- self.rem_total_tokens
)
for running_req in sorted_valid_running_reqs:
# Priority difference needs to meet the threshold to be preemptible.
priority_diff = (req.priority - running_req.priority) * (-priority_sign)
if priority_diff > self.priority_scheduling_preemption_threshold:
preemptible_reqs.append(running_req)
min_tokens_to_remove -= self._get_running_request_total_token_offset(
running_req
)
if min_tokens_to_remove <= 0:
break
else:
break
# Check max token count limit can be met
if len(preemptible_reqs) == 0 or min_tokens_to_remove > 0:
return False
# Preempt running requests. Release allocated resources for immediate usage.
preemptible_reqs = set(preemptible_reqs)
keep_indices = []
release_counter = 0
for i, running_req in enumerate(self.running_batch.reqs):
if running_req in preemptible_reqs:
self.memory_budget.total_offset -= (
self._get_running_request_total_token_offset(running_req)
)
release_counter += 1
self.running_batch.release_req(
i, len(self.running_batch.reqs) - release_counter
)
else:
keep_indices.append(i)
self.running_batch.filter_batch(keep_indices=keep_indices)
self.preempt_list.extend(preemptible_reqs)
return True