[kv-shard 3/4] Enable Control Plane B (#39964)
Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
@@ -50,6 +50,7 @@ from sglang.srt.managers.schedule_batch import (
|
||||
from sglang.srt.mem_cache.allocator.hisparse import (
|
||||
DeepSeekV4HiSparseTokenToKVPoolAllocator,
|
||||
)
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
|
||||
from sglang.srt.mem_cache.allocator.swa import (
|
||||
SWATokenToKVPoolAllocator,
|
||||
)
|
||||
@@ -719,6 +720,24 @@ class PrefillAdder:
|
||||
priority_scheduling_preemption_threshold
|
||||
)
|
||||
self.max_running_requests = max_running_requests
|
||||
# Align sharded chunk boundaries to physical pages; 0 disables alignment.
|
||||
kv_shard_size = page_interleave_shard_size(self.token_to_kv_pool_allocator)
|
||||
self.kv_shard_granule = (
|
||||
self.token_to_kv_pool_allocator.page_size if kv_shard_size > 1 else 0
|
||||
)
|
||||
self.kv_shard_size = kv_shard_size
|
||||
# Each admission must fit the batch's prefix and chunk scratch regions.
|
||||
self.kv_shard_scratch_spec = (
|
||||
self.token_to_kv_pool_allocator.shard_spec if kv_shard_size > 1 else None
|
||||
)
|
||||
self.kv_shard_block_bound_pages = 0
|
||||
self.kv_shard_chunk_pages = 0
|
||||
# Reserve one extra page per shard class for allocation rounding.
|
||||
self.per_req_token_overhead = (
|
||||
kv_shard_size * self.kv_shard_granule
|
||||
if self.kv_shard_granule
|
||||
else self.page_size
|
||||
)
|
||||
self.prefill_max_requests = prefill_max_requests
|
||||
self.prefill_delayer_single_pass = prefill_delayer_single_pass
|
||||
self.max_prefill_bs = max_prefill_bs
|
||||
@@ -879,7 +898,10 @@ class PrefillAdder:
|
||||
self.memory_budget.reserve(
|
||||
extend_input_len,
|
||||
max_new_tokens,
|
||||
extra_tokens=mamba_gap_reserve,
|
||||
# reserve() already charges one page; add the remaining shard pages.
|
||||
extra_tokens=(
|
||||
mamba_gap_reserve + self.per_req_token_overhead - self.page_size
|
||||
),
|
||||
chunk_limit=self.rem_chunk_tokens,
|
||||
is_chunked_continuation=is_chunked_continuation,
|
||||
)
|
||||
@@ -1004,6 +1026,38 @@ class PrefillAdder:
|
||||
# Persist the release receipt.
|
||||
req.lock_receipt = self.tree_cache.inc_lock_ref(req.last_node).to_dec_params()
|
||||
|
||||
def _kv_shard_reserve_scratch(self, prefix_len: int, extend_len: int) -> bool:
|
||||
"""Reserve scratch or return False to defer; no-op when sharding is off.
|
||||
|
||||
Prefixes use N * sum_i ceil(prefix_pages_i / N) pages; extends use
|
||||
sum_i ceil(extend_i / ps). A request must fit an empty batch or raise.
|
||||
"""
|
||||
if self.kv_shard_scratch_spec is None:
|
||||
return True
|
||||
ps = self.kv_shard_granule
|
||||
shard_size = self.kv_shard_size
|
||||
block = self.kv_shard_block_bound_pages + -(-(prefix_len // ps) // shard_size)
|
||||
chunk = self.kv_shard_chunk_pages + -(-extend_len // ps)
|
||||
if (
|
||||
shard_size * block * ps > self.kv_shard_scratch_spec.max_prefix_tokens
|
||||
or chunk * ps > self.kv_shard_scratch_spec.chunk_tokens
|
||||
):
|
||||
if self.kv_shard_block_bound_pages == 0 and self.kv_shard_chunk_pages == 0:
|
||||
# Deferring a request that cannot fit an empty batch would livelock.
|
||||
raise RuntimeError(
|
||||
"request cannot fit the sharded assembly scratch even in "
|
||||
f"an empty batch (prefix_len={prefix_len}, extend_len="
|
||||
f"{extend_len}, max_prefix_tokens="
|
||||
f"{self.kv_shard_scratch_spec.max_prefix_tokens}, "
|
||||
f"chunk_tokens={self.kv_shard_scratch_spec.chunk_tokens}); "
|
||||
"KV sharding requires chunked prefill sized within the "
|
||||
"assembly scratch"
|
||||
)
|
||||
return False
|
||||
self.kv_shard_block_bound_pages = block
|
||||
self.kv_shard_chunk_pages = chunk
|
||||
return True
|
||||
|
||||
def add_dllm_staging_req(self, req: Req):
|
||||
assert self.dllm_config is not None
|
||||
_rem_tokens = self._get_dllm_remain_tokens(req)
|
||||
@@ -1066,6 +1120,14 @@ class PrefillAdder:
|
||||
)
|
||||
if _rem_tokens is None:
|
||||
return req
|
||||
if self.kv_shard_granule:
|
||||
# Align the absolute boundary to keep pages within one scratch region.
|
||||
# If no budget remains, use a full chunk to make progress.
|
||||
prefix_len = len(req.prefix_indices)
|
||||
floored = (
|
||||
prefix_len + _rem_tokens
|
||||
) // self.kv_shard_granule * self.kv_shard_granule - prefix_len
|
||||
_rem_tokens = floored if floored > 0 else self.rem_chunk_tokens
|
||||
|
||||
# A mid-chunk rank prefills this pass regardless of the delayer
|
||||
# verdict, so report prefillable=True and ignore the result.
|
||||
@@ -1094,6 +1156,12 @@ class PrefillAdder:
|
||||
return req
|
||||
truncated = cand_extend_input_len > _rem_tokens
|
||||
new_len = min(cand_extend_input_len, _rem_tokens)
|
||||
# The continuing chunk must fit. Keep reservation outside assert for -O.
|
||||
reserved = self._kv_shard_reserve_scratch(
|
||||
prefix_len=len(req.prefix_indices), extend_len=new_len
|
||||
)
|
||||
if not reserved:
|
||||
raise RuntimeError("chunked request exceeds the sharded assembly scratch")
|
||||
req.set_extend_range(len(req.prefix_indices), len(req.prefix_indices) + new_len)
|
||||
self.can_run_list.append(req)
|
||||
self._update_prefill_budget(
|
||||
@@ -1133,7 +1201,9 @@ class PrefillAdder:
|
||||
cand_extend_input_len = len(req.full_untruncated_fill_ids) - len(
|
||||
req.prefix_indices
|
||||
)
|
||||
paged_input = self.ceil_paged_tokens(cand_extend_input_len)
|
||||
paged_input = (
|
||||
self.ceil_paged_tokens(cand_extend_input_len) + self.per_req_token_overhead
|
||||
)
|
||||
# Shared Mamba pool: fold the new mamba state's shared-gap cost into the
|
||||
# budget gate so admission can't over-commit (0 for baseline / non-Mamba).
|
||||
paged_input += self._mamba_gap_budget_for_req(req)
|
||||
@@ -1182,9 +1252,7 @@ class PrefillAdder:
|
||||
if not self.is_hybrid_swa:
|
||||
# Skip this logic for swa. The SWA has different memory management, and
|
||||
# this mechanism is underestimating the memory usage.
|
||||
cur_rem_tokens = self.cur_rem_tokens - self.ceil_paged_tokens(
|
||||
cand_extend_input_len
|
||||
)
|
||||
cur_rem_tokens = self.cur_rem_tokens - paged_input
|
||||
tokens_freed = 0
|
||||
for i, (tokens_left, tokens_occupied) in enumerate(self.req_states):
|
||||
# tokens_left gives a reservative calculation as the last token is not stored
|
||||
@@ -1226,6 +1294,11 @@ class PrefillAdder:
|
||||
return tile_stop
|
||||
|
||||
# Non-chunked prefill — the whole sequence is committed this iter.
|
||||
if not self._kv_shard_reserve_scratch(
|
||||
prefix_len=len(req.prefix_indices),
|
||||
extend_len=cand_extend_input_len,
|
||||
):
|
||||
return AddReqResult.OTHER
|
||||
req.set_extend_range(
|
||||
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
|
||||
)
|
||||
@@ -1250,6 +1323,10 @@ class PrefillAdder:
|
||||
if (tile_stop := self._check_prefill_tile_budget(trunc_len)) is not None:
|
||||
return tile_stop
|
||||
|
||||
# Reserve after the tile gate so rejected requests consume no scratch.
|
||||
if not self._kv_shard_reserve_scratch(prefix_len=0, extend_len=trunc_len):
|
||||
return AddReqResult.OTHER
|
||||
|
||||
assert len(req.prefix_indices) == 0
|
||||
req.set_extend_range(
|
||||
len(req.prefix_indices), len(req.prefix_indices) + trunc_len
|
||||
@@ -1286,7 +1363,7 @@ class PrefillAdder:
|
||||
cand_extend_input_len = len(req.full_untruncated_fill_ids) - len(
|
||||
req.prefix_indices
|
||||
)
|
||||
total_tokens = cand_extend_input_len + max_new + self.page_size
|
||||
total_tokens = cand_extend_input_len + max_new + self.per_req_token_overhead
|
||||
# Shared Mamba pool: fold the new mamba state's shared-gap cost into
|
||||
# `total_tokens` so both `rem_total_tokens` gates reflect the joint budget.
|
||||
# Read before `init_load_back` binds `req.mamba_pool_idx` — after that
|
||||
@@ -1374,7 +1451,12 @@ class PrefillAdder:
|
||||
req.prefix_indices = torch.cat([req.prefix_indices, new_indices])
|
||||
req.kv.cache_protected_len = len(req.prefix_indices)
|
||||
|
||||
# Successful materialization has no remaining admission gates.
|
||||
# Sharded pools cannot load host KV; reserve scratch after all other gates.
|
||||
if not self._kv_shard_reserve_scratch(
|
||||
prefix_len=admission.prefix_len, extend_len=admission.extend_len
|
||||
):
|
||||
return AddReqResult.OTHER
|
||||
|
||||
self._commit_prefill_admission(req, admission, mamba_gap_reserve)
|
||||
|
||||
# This verdict controls the next candidate, not the committed request.
|
||||
|
||||
@@ -288,6 +288,7 @@ from sglang.srt.managers.utils import (
|
||||
validate_input_length,
|
||||
)
|
||||
from sglang.srt.mem_cache import kv_cache_builder
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
|
||||
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome
|
||||
from sglang.srt.mem_cache.common import (
|
||||
discard_kv_cache_backup,
|
||||
@@ -442,6 +443,9 @@ class Scheduler(
|
||||
):
|
||||
"""A scheduler that manages a tensor parallel GPU worker."""
|
||||
|
||||
# Logical-to-physical KV capacity ratio; defaults to 1 before pool init.
|
||||
kv_shard_widening: int = 1
|
||||
|
||||
# Class-level default so on_idle's stall gate works even if a fork
|
||||
# overrides init_load_publisher (which would otherwise not set it).
|
||||
_last_stall_publish_ts: float = float("-inf")
|
||||
@@ -588,6 +592,9 @@ class Scheduler(
|
||||
self.swa_tokens_per_layer = result.swa_tokens_per_layer
|
||||
self.req_to_token_pool = result.req_to_token_pool
|
||||
self.token_to_kv_pool_allocator = result.token_to_kv_pool_allocator
|
||||
self.kv_shard_widening = page_interleave_shard_size(
|
||||
self.token_to_kv_pool_allocator
|
||||
)
|
||||
self.disable_radix_cache = result.disable_radix_cache
|
||||
self.tree_cache = result.tree_cache
|
||||
if self.enable_hierarchical_cache:
|
||||
@@ -2347,7 +2354,8 @@ class Scheduler(
|
||||
enable_hisparse=self.enable_hisparse,
|
||||
full_tokens_per_layer=self.full_tokens_per_layer,
|
||||
swa_tokens_per_layer=self.swa_tokens_per_layer,
|
||||
max_total_num_tokens=self.max_total_num_tokens,
|
||||
# Match the allocator and radix counters' logical units.
|
||||
max_total_num_tokens=self.max_total_num_tokens * self.kv_shard_widening,
|
||||
get_last_batch=lambda: self.last_batch,
|
||||
get_running_batch=lambda: self.running_batch,
|
||||
)
|
||||
@@ -2360,7 +2368,7 @@ class Scheduler(
|
||||
page_size=self.page_size,
|
||||
full_tokens_per_layer=self.full_tokens_per_layer,
|
||||
swa_tokens_per_layer=self.swa_tokens_per_layer,
|
||||
max_total_num_tokens=self.max_total_num_tokens,
|
||||
max_total_num_tokens=self.max_total_num_tokens * self.kv_shard_widening,
|
||||
tree_cache=self.tree_cache,
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
req_to_token_pool=self.req_to_token_pool,
|
||||
@@ -2412,7 +2420,8 @@ class Scheduler(
|
||||
self.load_inquirer = SchedulerLoadInquirer(
|
||||
disaggregation_mode=self.disaggregation_mode,
|
||||
server_args=self.server_args,
|
||||
max_total_num_tokens=self.max_total_num_tokens,
|
||||
# The worker reports logical DCP capacity; add KV shard widening.
|
||||
max_total_num_tokens=self.max_total_num_tokens * self.kv_shard_widening,
|
||||
max_running_requests=self.max_running_requests,
|
||||
pool_stats_observer=self.pool_stats_observer,
|
||||
tp_worker=self.tp_worker,
|
||||
@@ -2527,10 +2536,16 @@ class Scheduler(
|
||||
self.max_req_len - input_len - 1,
|
||||
),
|
||||
)
|
||||
# PrefillAdder reserves one page per shard; the allocator reserves one.
|
||||
# Subtract the other N - 1 pages to keep queue admission schedulable.
|
||||
token_capacity = (
|
||||
self.max_total_num_tokens * self.kv_shard_widening
|
||||
- self.page_size * (self.kv_shard_widening - 1)
|
||||
)
|
||||
max_new_tokens = self.token_to_kv_pool_allocator.max_new_tokens_for_memory(
|
||||
input_len,
|
||||
max_new_tokens,
|
||||
token_capacity=self.max_total_num_tokens,
|
||||
token_capacity=token_capacity,
|
||||
sliding_window_size=self.sliding_window_size,
|
||||
chunk_size=self.chunked_prefill_size,
|
||||
)
|
||||
|
||||
@@ -36,6 +36,7 @@ from sglang.srt.managers.io_struct import (
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.scheduler import GenerationBatchResult
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
@@ -535,9 +536,14 @@ class TpModelWorker(BaseTpWorker):
|
||||
self.model_runner.hisparse_coordinator = coordinator
|
||||
|
||||
def get_worker_info(self):
|
||||
# The runner already reports logical DCP capacity.
|
||||
kv_capacity = (
|
||||
self.model_runner.effective_logical_max_total_num_tokens
|
||||
* page_interleave_shard_size(self.model_runner.token_to_kv_pool_allocator)
|
||||
)
|
||||
max_req_len = min(
|
||||
self.model_config.context_len - 1,
|
||||
self.model_runner.effective_logical_max_total_num_tokens - 1,
|
||||
kv_capacity - 1,
|
||||
)
|
||||
max_req_input_len = max_req_len - 5
|
||||
if self.dllm_algorithm is not None:
|
||||
|
||||
Reference in New Issue
Block a user