[kv-shard 3/4] Enable Control Plane B (#39964)

Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
Shunkangz
2026-09-27 14:19:08 +08:00
committed by GitHub
co-authored by Zhangheng
parent acc15c193b
commit df1fc6ab64
6 changed files with 416 additions and 17 deletions
+89 -7
View File
@@ -50,6 +50,7 @@ from sglang.srt.managers.schedule_batch import (
from sglang.srt.mem_cache.allocator.hisparse import (
DeepSeekV4HiSparseTokenToKVPoolAllocator,
)
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
from sglang.srt.mem_cache.allocator.swa import (
SWATokenToKVPoolAllocator,
)
@@ -719,6 +720,24 @@ class PrefillAdder:
priority_scheduling_preemption_threshold
)
self.max_running_requests = max_running_requests
# Align sharded chunk boundaries to physical pages; 0 disables alignment.
kv_shard_size = page_interleave_shard_size(self.token_to_kv_pool_allocator)
self.kv_shard_granule = (
self.token_to_kv_pool_allocator.page_size if kv_shard_size > 1 else 0
)
self.kv_shard_size = kv_shard_size
# Each admission must fit the batch's prefix and chunk scratch regions.
self.kv_shard_scratch_spec = (
self.token_to_kv_pool_allocator.shard_spec if kv_shard_size > 1 else None
)
self.kv_shard_block_bound_pages = 0
self.kv_shard_chunk_pages = 0
# Reserve one extra page per shard class for allocation rounding.
self.per_req_token_overhead = (
kv_shard_size * self.kv_shard_granule
if self.kv_shard_granule
else self.page_size
)
self.prefill_max_requests = prefill_max_requests
self.prefill_delayer_single_pass = prefill_delayer_single_pass
self.max_prefill_bs = max_prefill_bs
@@ -879,7 +898,10 @@ class PrefillAdder:
self.memory_budget.reserve(
extend_input_len,
max_new_tokens,
extra_tokens=mamba_gap_reserve,
# reserve() already charges one page; add the remaining shard pages.
extra_tokens=(
mamba_gap_reserve + self.per_req_token_overhead - self.page_size
),
chunk_limit=self.rem_chunk_tokens,
is_chunked_continuation=is_chunked_continuation,
)
@@ -1004,6 +1026,38 @@ class PrefillAdder:
# Persist the release receipt.
req.lock_receipt = self.tree_cache.inc_lock_ref(req.last_node).to_dec_params()
def _kv_shard_reserve_scratch(self, prefix_len: int, extend_len: int) -> bool:
"""Reserve scratch or return False to defer; no-op when sharding is off.
Prefixes use N * sum_i ceil(prefix_pages_i / N) pages; extends use
sum_i ceil(extend_i / ps). A request must fit an empty batch or raise.
"""
if self.kv_shard_scratch_spec is None:
return True
ps = self.kv_shard_granule
shard_size = self.kv_shard_size
block = self.kv_shard_block_bound_pages + -(-(prefix_len // ps) // shard_size)
chunk = self.kv_shard_chunk_pages + -(-extend_len // ps)
if (
shard_size * block * ps > self.kv_shard_scratch_spec.max_prefix_tokens
or chunk * ps > self.kv_shard_scratch_spec.chunk_tokens
):
if self.kv_shard_block_bound_pages == 0 and self.kv_shard_chunk_pages == 0:
# Deferring a request that cannot fit an empty batch would livelock.
raise RuntimeError(
"request cannot fit the sharded assembly scratch even in "
f"an empty batch (prefix_len={prefix_len}, extend_len="
f"{extend_len}, max_prefix_tokens="
f"{self.kv_shard_scratch_spec.max_prefix_tokens}, "
f"chunk_tokens={self.kv_shard_scratch_spec.chunk_tokens}); "
"KV sharding requires chunked prefill sized within the "
"assembly scratch"
)
return False
self.kv_shard_block_bound_pages = block
self.kv_shard_chunk_pages = chunk
return True
def add_dllm_staging_req(self, req: Req):
assert self.dllm_config is not None
_rem_tokens = self._get_dllm_remain_tokens(req)
@@ -1066,6 +1120,14 @@ class PrefillAdder:
)
if _rem_tokens is None:
return req
if self.kv_shard_granule:
# Align the absolute boundary to keep pages within one scratch region.
# If no budget remains, use a full chunk to make progress.
prefix_len = len(req.prefix_indices)
floored = (
prefix_len + _rem_tokens
) // self.kv_shard_granule * self.kv_shard_granule - prefix_len
_rem_tokens = floored if floored > 0 else self.rem_chunk_tokens
# A mid-chunk rank prefills this pass regardless of the delayer
# verdict, so report prefillable=True and ignore the result.
@@ -1094,6 +1156,12 @@ class PrefillAdder:
return req
truncated = cand_extend_input_len > _rem_tokens
new_len = min(cand_extend_input_len, _rem_tokens)
# The continuing chunk must fit. Keep reservation outside assert for -O.
reserved = self._kv_shard_reserve_scratch(
prefix_len=len(req.prefix_indices), extend_len=new_len
)
if not reserved:
raise RuntimeError("chunked request exceeds the sharded assembly scratch")
req.set_extend_range(len(req.prefix_indices), len(req.prefix_indices) + new_len)
self.can_run_list.append(req)
self._update_prefill_budget(
@@ -1133,7 +1201,9 @@ class PrefillAdder:
cand_extend_input_len = len(req.full_untruncated_fill_ids) - len(
req.prefix_indices
)
paged_input = self.ceil_paged_tokens(cand_extend_input_len)
paged_input = (
self.ceil_paged_tokens(cand_extend_input_len) + self.per_req_token_overhead
)
# Shared Mamba pool: fold the new mamba state's shared-gap cost into the
# budget gate so admission can't over-commit (0 for baseline / non-Mamba).
paged_input += self._mamba_gap_budget_for_req(req)
@@ -1182,9 +1252,7 @@ class PrefillAdder:
if not self.is_hybrid_swa:
# Skip this logic for swa. The SWA has different memory management, and
# this mechanism is underestimating the memory usage.
cur_rem_tokens = self.cur_rem_tokens - self.ceil_paged_tokens(
cand_extend_input_len
)
cur_rem_tokens = self.cur_rem_tokens - paged_input
tokens_freed = 0
for i, (tokens_left, tokens_occupied) in enumerate(self.req_states):
# tokens_left gives a reservative calculation as the last token is not stored
@@ -1226,6 +1294,11 @@ class PrefillAdder:
return tile_stop
# Non-chunked prefill — the whole sequence is committed this iter.
if not self._kv_shard_reserve_scratch(
prefix_len=len(req.prefix_indices),
extend_len=cand_extend_input_len,
):
return AddReqResult.OTHER
req.set_extend_range(
len(req.prefix_indices), len(req.full_untruncated_fill_ids)
)
@@ -1250,6 +1323,10 @@ class PrefillAdder:
if (tile_stop := self._check_prefill_tile_budget(trunc_len)) is not None:
return tile_stop
# Reserve after the tile gate so rejected requests consume no scratch.
if not self._kv_shard_reserve_scratch(prefix_len=0, extend_len=trunc_len):
return AddReqResult.OTHER
assert len(req.prefix_indices) == 0
req.set_extend_range(
len(req.prefix_indices), len(req.prefix_indices) + trunc_len
@@ -1286,7 +1363,7 @@ class PrefillAdder:
cand_extend_input_len = len(req.full_untruncated_fill_ids) - len(
req.prefix_indices
)
total_tokens = cand_extend_input_len + max_new + self.page_size
total_tokens = cand_extend_input_len + max_new + self.per_req_token_overhead
# Shared Mamba pool: fold the new mamba state's shared-gap cost into
# `total_tokens` so both `rem_total_tokens` gates reflect the joint budget.
# Read before `init_load_back` binds `req.mamba_pool_idx` — after that
@@ -1374,7 +1451,12 @@ class PrefillAdder:
req.prefix_indices = torch.cat([req.prefix_indices, new_indices])
req.kv.cache_protected_len = len(req.prefix_indices)
# Successful materialization has no remaining admission gates.
# Sharded pools cannot load host KV; reserve scratch after all other gates.
if not self._kv_shard_reserve_scratch(
prefix_len=admission.prefix_len, extend_len=admission.extend_len
):
return AddReqResult.OTHER
self._commit_prefill_admission(req, admission, mamba_gap_reserve)
# This verdict controls the next candidate, not the committed request.
+19 -4
View File
@@ -288,6 +288,7 @@ from sglang.srt.managers.utils import (
validate_input_length,
)
from sglang.srt.mem_cache import kv_cache_builder
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome
from sglang.srt.mem_cache.common import (
discard_kv_cache_backup,
@@ -442,6 +443,9 @@ class Scheduler(
):
"""A scheduler that manages a tensor parallel GPU worker."""
# Logical-to-physical KV capacity ratio; defaults to 1 before pool init.
kv_shard_widening: int = 1
# Class-level default so on_idle's stall gate works even if a fork
# overrides init_load_publisher (which would otherwise not set it).
_last_stall_publish_ts: float = float("-inf")
@@ -588,6 +592,9 @@ class Scheduler(
self.swa_tokens_per_layer = result.swa_tokens_per_layer
self.req_to_token_pool = result.req_to_token_pool
self.token_to_kv_pool_allocator = result.token_to_kv_pool_allocator
self.kv_shard_widening = page_interleave_shard_size(
self.token_to_kv_pool_allocator
)
self.disable_radix_cache = result.disable_radix_cache
self.tree_cache = result.tree_cache
if self.enable_hierarchical_cache:
@@ -2347,7 +2354,8 @@ class Scheduler(
enable_hisparse=self.enable_hisparse,
full_tokens_per_layer=self.full_tokens_per_layer,
swa_tokens_per_layer=self.swa_tokens_per_layer,
max_total_num_tokens=self.max_total_num_tokens,
# Match the allocator and radix counters' logical units.
max_total_num_tokens=self.max_total_num_tokens * self.kv_shard_widening,
get_last_batch=lambda: self.last_batch,
get_running_batch=lambda: self.running_batch,
)
@@ -2360,7 +2368,7 @@ class Scheduler(
page_size=self.page_size,
full_tokens_per_layer=self.full_tokens_per_layer,
swa_tokens_per_layer=self.swa_tokens_per_layer,
max_total_num_tokens=self.max_total_num_tokens,
max_total_num_tokens=self.max_total_num_tokens * self.kv_shard_widening,
tree_cache=self.tree_cache,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
req_to_token_pool=self.req_to_token_pool,
@@ -2412,7 +2420,8 @@ class Scheduler(
self.load_inquirer = SchedulerLoadInquirer(
disaggregation_mode=self.disaggregation_mode,
server_args=self.server_args,
max_total_num_tokens=self.max_total_num_tokens,
# The worker reports logical DCP capacity; add KV shard widening.
max_total_num_tokens=self.max_total_num_tokens * self.kv_shard_widening,
max_running_requests=self.max_running_requests,
pool_stats_observer=self.pool_stats_observer,
tp_worker=self.tp_worker,
@@ -2527,10 +2536,16 @@ class Scheduler(
self.max_req_len - input_len - 1,
),
)
# PrefillAdder reserves one page per shard; the allocator reserves one.
# Subtract the other N - 1 pages to keep queue admission schedulable.
token_capacity = (
self.max_total_num_tokens * self.kv_shard_widening
- self.page_size * (self.kv_shard_widening - 1)
)
max_new_tokens = self.token_to_kv_pool_allocator.max_new_tokens_for_memory(
input_len,
max_new_tokens,
token_capacity=self.max_total_num_tokens,
token_capacity=token_capacity,
sliding_window_size=self.sliding_window_size,
chunk_size=self.chunked_prefill_size,
)
+7 -1
View File
@@ -36,6 +36,7 @@ from sglang.srt.managers.io_struct import (
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.scheduler import GenerationBatchResult
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
@@ -535,9 +536,14 @@ class TpModelWorker(BaseTpWorker):
self.model_runner.hisparse_coordinator = coordinator
def get_worker_info(self):
# The runner already reports logical DCP capacity.
kv_capacity = (
self.model_runner.effective_logical_max_total_num_tokens
* page_interleave_shard_size(self.model_runner.token_to_kv_pool_allocator)
)
max_req_len = min(
self.model_config.context_len - 1,
self.model_runner.effective_logical_max_total_num_tokens - 1,
kv_capacity - 1,
)
max_req_input_len = max_req_len - 5
if self.dllm_algorithm is not None: