diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index d84b136af0..d5f1b80ae7 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -50,6 +50,7 @@ from sglang.srt.managers.schedule_batch import ( from sglang.srt.mem_cache.allocator.hisparse import ( DeepSeekV4HiSparseTokenToKVPoolAllocator, ) +from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size from sglang.srt.mem_cache.allocator.swa import ( SWATokenToKVPoolAllocator, ) @@ -719,6 +720,24 @@ class PrefillAdder: priority_scheduling_preemption_threshold ) self.max_running_requests = max_running_requests + # Align sharded chunk boundaries to physical pages; 0 disables alignment. + kv_shard_size = page_interleave_shard_size(self.token_to_kv_pool_allocator) + self.kv_shard_granule = ( + self.token_to_kv_pool_allocator.page_size if kv_shard_size > 1 else 0 + ) + self.kv_shard_size = kv_shard_size + # Each admission must fit the batch's prefix and chunk scratch regions. + self.kv_shard_scratch_spec = ( + self.token_to_kv_pool_allocator.shard_spec if kv_shard_size > 1 else None + ) + self.kv_shard_block_bound_pages = 0 + self.kv_shard_chunk_pages = 0 + # Reserve one extra page per shard class for allocation rounding. + self.per_req_token_overhead = ( + kv_shard_size * self.kv_shard_granule + if self.kv_shard_granule + else self.page_size + ) self.prefill_max_requests = prefill_max_requests self.prefill_delayer_single_pass = prefill_delayer_single_pass self.max_prefill_bs = max_prefill_bs @@ -879,7 +898,10 @@ class PrefillAdder: self.memory_budget.reserve( extend_input_len, max_new_tokens, - extra_tokens=mamba_gap_reserve, + # reserve() already charges one page; add the remaining shard pages. + extra_tokens=( + mamba_gap_reserve + self.per_req_token_overhead - self.page_size + ), chunk_limit=self.rem_chunk_tokens, is_chunked_continuation=is_chunked_continuation, ) @@ -1004,6 +1026,38 @@ class PrefillAdder: # Persist the release receipt. req.lock_receipt = self.tree_cache.inc_lock_ref(req.last_node).to_dec_params() + def _kv_shard_reserve_scratch(self, prefix_len: int, extend_len: int) -> bool: + """Reserve scratch or return False to defer; no-op when sharding is off. + + Prefixes use N * sum_i ceil(prefix_pages_i / N) pages; extends use + sum_i ceil(extend_i / ps). A request must fit an empty batch or raise. + """ + if self.kv_shard_scratch_spec is None: + return True + ps = self.kv_shard_granule + shard_size = self.kv_shard_size + block = self.kv_shard_block_bound_pages + -(-(prefix_len // ps) // shard_size) + chunk = self.kv_shard_chunk_pages + -(-extend_len // ps) + if ( + shard_size * block * ps > self.kv_shard_scratch_spec.max_prefix_tokens + or chunk * ps > self.kv_shard_scratch_spec.chunk_tokens + ): + if self.kv_shard_block_bound_pages == 0 and self.kv_shard_chunk_pages == 0: + # Deferring a request that cannot fit an empty batch would livelock. + raise RuntimeError( + "request cannot fit the sharded assembly scratch even in " + f"an empty batch (prefix_len={prefix_len}, extend_len=" + f"{extend_len}, max_prefix_tokens=" + f"{self.kv_shard_scratch_spec.max_prefix_tokens}, " + f"chunk_tokens={self.kv_shard_scratch_spec.chunk_tokens}); " + "KV sharding requires chunked prefill sized within the " + "assembly scratch" + ) + return False + self.kv_shard_block_bound_pages = block + self.kv_shard_chunk_pages = chunk + return True + def add_dllm_staging_req(self, req: Req): assert self.dllm_config is not None _rem_tokens = self._get_dllm_remain_tokens(req) @@ -1066,6 +1120,14 @@ class PrefillAdder: ) if _rem_tokens is None: return req + if self.kv_shard_granule: + # Align the absolute boundary to keep pages within one scratch region. + # If no budget remains, use a full chunk to make progress. + prefix_len = len(req.prefix_indices) + floored = ( + prefix_len + _rem_tokens + ) // self.kv_shard_granule * self.kv_shard_granule - prefix_len + _rem_tokens = floored if floored > 0 else self.rem_chunk_tokens # A mid-chunk rank prefills this pass regardless of the delayer # verdict, so report prefillable=True and ignore the result. @@ -1094,6 +1156,12 @@ class PrefillAdder: return req truncated = cand_extend_input_len > _rem_tokens new_len = min(cand_extend_input_len, _rem_tokens) + # The continuing chunk must fit. Keep reservation outside assert for -O. + reserved = self._kv_shard_reserve_scratch( + prefix_len=len(req.prefix_indices), extend_len=new_len + ) + if not reserved: + raise RuntimeError("chunked request exceeds the sharded assembly scratch") req.set_extend_range(len(req.prefix_indices), len(req.prefix_indices) + new_len) self.can_run_list.append(req) self._update_prefill_budget( @@ -1133,7 +1201,9 @@ class PrefillAdder: cand_extend_input_len = len(req.full_untruncated_fill_ids) - len( req.prefix_indices ) - paged_input = self.ceil_paged_tokens(cand_extend_input_len) + paged_input = ( + self.ceil_paged_tokens(cand_extend_input_len) + self.per_req_token_overhead + ) # Shared Mamba pool: fold the new mamba state's shared-gap cost into the # budget gate so admission can't over-commit (0 for baseline / non-Mamba). paged_input += self._mamba_gap_budget_for_req(req) @@ -1182,9 +1252,7 @@ class PrefillAdder: if not self.is_hybrid_swa: # Skip this logic for swa. The SWA has different memory management, and # this mechanism is underestimating the memory usage. - cur_rem_tokens = self.cur_rem_tokens - self.ceil_paged_tokens( - cand_extend_input_len - ) + cur_rem_tokens = self.cur_rem_tokens - paged_input tokens_freed = 0 for i, (tokens_left, tokens_occupied) in enumerate(self.req_states): # tokens_left gives a reservative calculation as the last token is not stored @@ -1226,6 +1294,11 @@ class PrefillAdder: return tile_stop # Non-chunked prefill — the whole sequence is committed this iter. + if not self._kv_shard_reserve_scratch( + prefix_len=len(req.prefix_indices), + extend_len=cand_extend_input_len, + ): + return AddReqResult.OTHER req.set_extend_range( len(req.prefix_indices), len(req.full_untruncated_fill_ids) ) @@ -1250,6 +1323,10 @@ class PrefillAdder: if (tile_stop := self._check_prefill_tile_budget(trunc_len)) is not None: return tile_stop + # Reserve after the tile gate so rejected requests consume no scratch. + if not self._kv_shard_reserve_scratch(prefix_len=0, extend_len=trunc_len): + return AddReqResult.OTHER + assert len(req.prefix_indices) == 0 req.set_extend_range( len(req.prefix_indices), len(req.prefix_indices) + trunc_len @@ -1286,7 +1363,7 @@ class PrefillAdder: cand_extend_input_len = len(req.full_untruncated_fill_ids) - len( req.prefix_indices ) - total_tokens = cand_extend_input_len + max_new + self.page_size + total_tokens = cand_extend_input_len + max_new + self.per_req_token_overhead # Shared Mamba pool: fold the new mamba state's shared-gap cost into # `total_tokens` so both `rem_total_tokens` gates reflect the joint budget. # Read before `init_load_back` binds `req.mamba_pool_idx` — after that @@ -1374,7 +1451,12 @@ class PrefillAdder: req.prefix_indices = torch.cat([req.prefix_indices, new_indices]) req.kv.cache_protected_len = len(req.prefix_indices) - # Successful materialization has no remaining admission gates. + # Sharded pools cannot load host KV; reserve scratch after all other gates. + if not self._kv_shard_reserve_scratch( + prefix_len=admission.prefix_len, extend_len=admission.extend_len + ): + return AddReqResult.OTHER + self._commit_prefill_admission(req, admission, mamba_gap_reserve) # This verdict controls the next candidate, not the committed request. diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index dd05564d85..e1ce5e02d4 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -288,6 +288,7 @@ from sglang.srt.managers.utils import ( validate_input_length, ) from sglang.srt.mem_cache import kv_cache_builder +from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome from sglang.srt.mem_cache.common import ( discard_kv_cache_backup, @@ -442,6 +443,9 @@ class Scheduler( ): """A scheduler that manages a tensor parallel GPU worker.""" + # Logical-to-physical KV capacity ratio; defaults to 1 before pool init. + kv_shard_widening: int = 1 + # Class-level default so on_idle's stall gate works even if a fork # overrides init_load_publisher (which would otherwise not set it). _last_stall_publish_ts: float = float("-inf") @@ -588,6 +592,9 @@ class Scheduler( self.swa_tokens_per_layer = result.swa_tokens_per_layer self.req_to_token_pool = result.req_to_token_pool self.token_to_kv_pool_allocator = result.token_to_kv_pool_allocator + self.kv_shard_widening = page_interleave_shard_size( + self.token_to_kv_pool_allocator + ) self.disable_radix_cache = result.disable_radix_cache self.tree_cache = result.tree_cache if self.enable_hierarchical_cache: @@ -2347,7 +2354,8 @@ class Scheduler( enable_hisparse=self.enable_hisparse, full_tokens_per_layer=self.full_tokens_per_layer, swa_tokens_per_layer=self.swa_tokens_per_layer, - max_total_num_tokens=self.max_total_num_tokens, + # Match the allocator and radix counters' logical units. + max_total_num_tokens=self.max_total_num_tokens * self.kv_shard_widening, get_last_batch=lambda: self.last_batch, get_running_batch=lambda: self.running_batch, ) @@ -2360,7 +2368,7 @@ class Scheduler( page_size=self.page_size, full_tokens_per_layer=self.full_tokens_per_layer, swa_tokens_per_layer=self.swa_tokens_per_layer, - max_total_num_tokens=self.max_total_num_tokens, + max_total_num_tokens=self.max_total_num_tokens * self.kv_shard_widening, tree_cache=self.tree_cache, token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, req_to_token_pool=self.req_to_token_pool, @@ -2412,7 +2420,8 @@ class Scheduler( self.load_inquirer = SchedulerLoadInquirer( disaggregation_mode=self.disaggregation_mode, server_args=self.server_args, - max_total_num_tokens=self.max_total_num_tokens, + # The worker reports logical DCP capacity; add KV shard widening. + max_total_num_tokens=self.max_total_num_tokens * self.kv_shard_widening, max_running_requests=self.max_running_requests, pool_stats_observer=self.pool_stats_observer, tp_worker=self.tp_worker, @@ -2527,10 +2536,16 @@ class Scheduler( self.max_req_len - input_len - 1, ), ) + # PrefillAdder reserves one page per shard; the allocator reserves one. + # Subtract the other N - 1 pages to keep queue admission schedulable. + token_capacity = ( + self.max_total_num_tokens * self.kv_shard_widening + - self.page_size * (self.kv_shard_widening - 1) + ) max_new_tokens = self.token_to_kv_pool_allocator.max_new_tokens_for_memory( input_len, max_new_tokens, - token_capacity=self.max_total_num_tokens, + token_capacity=token_capacity, sliding_window_size=self.sliding_window_size, chunk_size=self.chunked_prefill_size, ) diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index c8a0426874..6adfbe0437 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -36,6 +36,7 @@ from sglang.srt.managers.io_struct import ( from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.managers.scheduler import GenerationBatchResult from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator +from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size from sglang.srt.mem_cache.memory_pool import ReqToTokenPool from sglang.srt.model_executor.forward_batch_info import ( CaptureHiddenMode, @@ -535,9 +536,14 @@ class TpModelWorker(BaseTpWorker): self.model_runner.hisparse_coordinator = coordinator def get_worker_info(self): + # The runner already reports logical DCP capacity. + kv_capacity = ( + self.model_runner.effective_logical_max_total_num_tokens + * page_interleave_shard_size(self.model_runner.token_to_kv_pool_allocator) + ) max_req_len = min( self.model_config.context_len - 1, - self.model_runner.effective_logical_max_total_num_tokens - 1, + kv_capacity - 1, ) max_req_input_len = max_req_len - 5 if self.dllm_algorithm is not None: diff --git a/test/registered/unit/managers/test_dcp_logical_capacity.py b/test/registered/unit/managers/test_dcp_logical_capacity.py index 3d7a92322d..ef20cf1344 100644 --- a/test/registered/unit/managers/test_dcp_logical_capacity.py +++ b/test/registered/unit/managers/test_dcp_logical_capacity.py @@ -7,6 +7,7 @@ from unittest.mock import patch import torch from sglang.srt.managers.tp_worker import TpModelWorker +from sglang.srt.mem_cache.allocator.page_interleave import PageInterleavePoolAllocator from sglang.srt.mem_cache.allocator.unified_mamba import ( UnifiedMambaTokenToKVPoolAllocator, ) @@ -78,6 +79,7 @@ def make_worker(dcp_size): runner.max_total_num_tokens = PHYSICAL runner.max_running_requests = 64 runner.token_to_kv_pool = kv + runner.token_to_kv_pool_allocator = None runner.req_to_token_pool = ReqToTokenPool.__new__(ReqToTokenPool) runner.req_to_token_pool.size = 96 runner.req_to_token_pool.max_context_len = CONTEXT @@ -107,6 +109,20 @@ class TestDcpLogicalCapacity(CustomTestCase): config.start() self.addCleanup(config.stop) + def test_sharded_worker_request_limits(self): + worker = self.make_worker(1) + runner = worker.model_runner + runner.max_total_num_tokens = 64 + allocator = PageInterleavePoolAllocator.__new__(PageInterleavePoolAllocator) + allocator.shard_size = 3 + runner.token_to_kv_pool_allocator = allocator + + info = TpModelWorker.get_worker_info(worker) + + self.assertEqual(info[0], 64) + self.assertEqual(info[4], 64 * 3 - 1) + self.assertEqual(info[5], 64 * 3 - 6) + def test_logical_capacity(self): for dcp_size in (1, 2, 8): with self.subTest(dcp_size=dcp_size): diff --git a/test/registered/unit/managers/test_prefill_adder.py b/test/registered/unit/managers/test_prefill_adder.py index 2486b2915f..cf5d25fba1 100644 --- a/test/registered/unit/managers/test_prefill_adder.py +++ b/test/registered/unit/managers/test_prefill_adder.py @@ -12,11 +12,13 @@ from sglang.srt.managers.schedule_policy import ( SchedulePolicy, estimate_prefill_extend_tile_metrics, ) +from sglang.srt.mem_cache.allocator.page_interleave import PageInterleavePoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( CacheRequestHandle, DecLockRefResult, IncLockRefResult, ) +from sglang.srt.mem_cache.page_interleave import PageShardSpec, make_page_shard_spec from sglang.srt.mem_cache.prefill_budget import ( PrefillBudget, SWAPrefillBudget, @@ -757,6 +759,251 @@ class TestPrefillAdder(CustomTestCase): self.assertEqual(adder2.rem_chunk_tokens, 0) # 3 - 3 = 0 self.assertEqual(result3, AddReqResult.OTHER) + @patch( + "sglang.srt.managers.schedule_policy.page_interleave_shard_size", + return_value=4, + ) + def test_ignore_eos_reserves_all_shard_class_pages(self, _shard_size): + """Disabled-radix admission must charge one page per shard class.""" + self.mock_tree_cache.disable = True + self.mock_token_allocator.page_size = 16 + self.mock_token_allocator.shard_spec = SimpleNamespace( + max_prefix_tokens=1024, chunk_tokens=1024 + ) + self.mock_token_allocator.available_size.return_value = 64 + adder = self.create_adder(self.create_running_batch(), page_size=16) + + req = self.create_mock_req("ignore_eos", priority=0, max_new_tokens=1) + req.sampling_params.ignore_eos = True + req.origin_input_ids = list(range(16)) + req.full_untruncated_fill_ids = list(range(16)) + req.last_node = MagicMock() + req.set_extend_range = MagicMock( + side_effect=lambda start, end: setattr( + req, "extend_range", Range(start, end) + ) + ) + + result = adder.add_one_req( + req, has_chunked_req=False, truncation_align_size=None + ) + + self.assertEqual(result, AddReqResult.NO_TOKEN) + self.assertEqual(adder.can_run_list, []) + req.set_extend_range.assert_not_called() + + def create_sharded_adder( + self, + *, + chunk_tokens=16, + max_prefix_tokens=16, + shard_spec=None, + allocator_size=1024, + **kwargs, + ): + self.mock_tree_cache.supports_mamba.return_value = False + self.mock_tree_cache.is_tree_cache.return_value = False + if shard_spec is None: + shard_spec = PageShardSpec( + shard_rank=0, + shard_size=2, + page_size=4, + max_prefix_tokens=max_prefix_tokens, + chunk_tokens=chunk_tokens, + ) + allocator = PageInterleavePoolAllocator( + size=allocator_size, + physical_page_size=shard_spec.page_size, + shard_size=shard_spec.shard_size, + dtype=torch.int64, + device="cpu", + kvcache=None, + need_sort=False, + shard_spec=shard_spec, + ) + return self.create_adder( + self.create_running_batch(), + page_size=shard_spec.page_size, + rem_chunk_tokens=shard_spec.chunk_tokens, + token_to_kv_pool_allocator=allocator, + **kwargs, + ) + + def create_sharded_req(self, rid, *, prefix_len=12, extend_len=4): + req = self.create_shared_req(rid, max_new_tokens=1) + req.prefix_indices = list(range(prefix_len)) + req.full_untruncated_fill_ids = list(range(prefix_len + extend_len)) + return req + + def test_sharded_admission_defers_prefix_scratch_overflow(self): + adder = self.create_sharded_adder() + first = self.create_sharded_req("first") + second = self.create_sharded_req("second") + self.assertEqual( + adder.add_one_req(first, has_chunked_req=False, truncation_align_size=None), + AddReqResult.CONTINUE, + ) + remaining = (adder.rem_total_tokens, adder.rem_chunk_tokens) + + # Both requests fit KV capacity, but prefix scratch fits only one. + self.assertEqual( + adder.add_one_req( + second, has_chunked_req=False, truncation_align_size=None + ), + AddReqResult.OTHER, + ) + self.assertEqual(adder.can_run_list, [first]) + second.set_extend_range.assert_not_called() + self.assertEqual((adder.rem_total_tokens, adder.rem_chunk_tokens), remaining) + self.assertEqual(adder.kv_shard_block_bound_pages, 2) + self.assertEqual(adder.kv_shard_chunk_pages, 1) + + # Rejection leaves the remaining chunk scratch usable. + third = self.create_sharded_req("no-prefix", prefix_len=0) + self.assertEqual( + adder.add_one_req(third, has_chunked_req=False, truncation_align_size=None), + AddReqResult.CONTINUE, + ) + self.assertEqual(adder.can_run_list, [first, third]) + self.assertEqual(adder.kv_shard_block_bound_pages, 2) + self.assertEqual(adder.kv_shard_chunk_pages, 2) + + def test_eight_long_prefixes_fit_an_eight_k_chunk(self): + context_len = 16384 + extend_len = 1024 + prefix_len = context_len - extend_len + kvc = SimpleNamespace( + page_size=4, + model_config=SimpleNamespace(context_len=context_len), + ) + for chunk_tokens in (8192, 16384): + with self.subTest(chunk_tokens=chunk_tokens): + with ( + patch( + "sglang.srt.mem_cache.page_interleave.get_kv_shard_group_info", + return_value=(0, 2), + ), + patch( + "sglang.srt.mem_cache.page_interleave.get_schedule", + return_value=SimpleNamespace(chunked_prefill_size=chunk_tokens), + ), + ): + spec = make_page_shard_spec(kvc) + self.assertEqual(spec.max_prefix_tokens, 8 * context_len) + adder = self.create_sharded_adder( + shard_spec=spec, + allocator_size=8 * context_len, + rem_input_tokens=32768, + ) + reqs = [ + self.create_sharded_req( + str(i), prefix_len=prefix_len, extend_len=extend_len + ) + for i in range(9) + ] + for i, req in enumerate(reqs[:8]): + result = adder.add_one_req( + req, has_chunked_req=False, truncation_align_size=None + ) + # The eighth request commits even if it exhausts the budget. + expected = ( + AddReqResult.OTHER + if i == 7 and chunk_tokens == 8192 + else AddReqResult.CONTINUE + ) + self.assertEqual(result, expected) + self.assertEqual(adder.can_run_list, reqs[:8]) + self.assertEqual(adder.rem_chunk_tokens, chunk_tokens - 8192) + self.assertEqual(adder.kv_shard_chunk_pages * spec.page_size, 8192) + self.assertEqual( + adder.kv_shard_block_bound_pages * spec.logical_page_size, + 8 * prefix_len, + ) + remaining = ( + adder.rem_total_tokens, + adder.rem_chunk_tokens, + adder.kv_shard_block_bound_pages, + adder.kv_shard_chunk_pages, + ) + self.assertEqual( + adder.add_one_req( + reqs[8], has_chunked_req=False, truncation_align_size=None + ), + AddReqResult.OTHER, + ) + self.assertEqual(adder.can_run_list, reqs[:8]) + reqs[8].set_extend_range.assert_not_called() + self.assertEqual( + ( + adder.rem_total_tokens, + adder.rem_chunk_tokens, + adder.kv_shard_block_bound_pages, + adder.kv_shard_chunk_pages, + ), + remaining, + ) + if chunk_tokens == 16384: + # Prefix scratch is full; a request without a prefix still fits. + no_prefix = self.create_sharded_req( + "no-prefix", prefix_len=0, extend_len=extend_len + ) + self.assertEqual( + adder.add_one_req( + no_prefix, + has_chunked_req=False, + truncation_align_size=None, + ), + AddReqResult.CONTINUE, + ) + self.assertEqual(adder.can_run_list, reqs[:8] + [no_prefix]) + + def test_sharded_admission_reserves_only_selected_chunk(self): + # Use normal page rounding without a sharding alignment override. + for remaining, selected, verdict in ( + (8, 8, AddReqResult.OTHER), + (7, 4, AddReqResult.CONTINUE), + ): + with self.subTest(remaining=remaining): + adder = self.create_sharded_adder(chunk_tokens=8) + adder.exact_chunk_fill = False + adder.rem_chunk_tokens = remaining + req = self.create_sharded_req("chunked", extend_len=28) + + self.assertEqual( + adder.add_one_req( + req, has_chunked_req=False, truncation_align_size=None + ), + verdict, + ) + self.assertEqual(adder.can_run_list, [req]) + self.assertIs(adder.new_chunked_req, req) + self.assertEqual(req.extend_range, Range(12, 12 + selected)) + self.assertEqual(adder.kv_shard_block_bound_pages, 2) + self.assertEqual(adder.kv_shard_chunk_pages, selected // 4) + + def test_sharded_admission_reserves_nothing_when_delayed(self): + delayer = _RecordingDelayer(allow=False) + adder = self.create_sharded_adder(prefill_delayer_single_pass=delayer) + req = self.create_sharded_req("delayed") + + self.assertEqual( + adder.add_one_req(req, has_chunked_req=False, truncation_align_size=None), + AddReqResult.OTHER, + ) + self.assertEqual(adder.can_run_list, []) + req.set_extend_range.assert_not_called() + self.assertEqual(adder.kv_shard_block_bound_pages, 0) + self.assertEqual(adder.kv_shard_chunk_pages, 0) + + delayer.allow = True + self.assertEqual( + adder.add_one_req(req, has_chunked_req=False, truncation_align_size=None), + AddReqResult.CONTINUE, + ) + self.assertEqual(adder.can_run_list, [req]) + self.assertEqual(adder.kv_shard_block_bound_pages, 2) + self.assertEqual(adder.kv_shard_chunk_pages, 1) + def _build_hybrid_swa_chunked_req( self, *, diff --git a/test/registered/unit/managers/test_scheduler_init_req_max_new_tokens.py b/test/registered/unit/managers/test_scheduler_init_req_max_new_tokens.py index d651071e5d..d2d034dba8 100644 --- a/test/registered/unit/managers/test_scheduler_init_req_max_new_tokens.py +++ b/test/registered/unit/managers/test_scheduler_init_req_max_new_tokens.py @@ -20,7 +20,8 @@ class TestSchedulerInitReqMaxNewTokens(unittest.TestCase): Rules enforced when clipping a request's max_new_tokens: 1. context: input_len + max_new_tokens < max_req_len 2. admission budget (PrefillAdder): - ceil_page(input_len) + max_new_tokens + page_size < max_total_num_tokens + ceil_page(input_len) + max_new_tokens + page_size * shard_widening + < max_total_num_tokens * shard_widening 3. env limit: <= SGLANG_MAX_NEW_TOKENS_LIMIT when set and positive 4. never above the requested value 5. min_new_tokens <= max_new_tokens afterwards @@ -42,9 +43,7 @@ class TestSchedulerInitReqMaxNewTokens(unittest.TestCase): cls._scheduler_logger.setLevel(cls._old_level) def setUp(self): - # The scheduler scales the budget by the live DCP size - # (`get_parallel().attn_dcp_size`), so the double states a topology - # rather than publishing a config it does not otherwise need. + # Tests use an unsharded topology unless they explicitly override it. cm = get_parallel().override(attn_dcp_size=1) cm.__enter__() self.addCleanup(cm.__exit__, None, None, None) @@ -54,11 +53,13 @@ class TestSchedulerInitReqMaxNewTokens(unittest.TestCase): max_req_len: int = 128, max_total_num_tokens: int = 1024, page_size: int = 1, + kv_shard_widening: int = 1, ) -> Scheduler: scheduler = Scheduler.__new__(Scheduler) scheduler.max_req_len = max_req_len scheduler.max_total_num_tokens = max_total_num_tokens scheduler.page_size = page_size + scheduler.kv_shard_widening = kv_shard_widening scheduler.max_new_tokens_limit = envs.SGLANG_MAX_NEW_TOKENS_LIMIT.get() scheduler.sliding_window_size = None scheduler.chunked_prefill_size = None @@ -90,6 +91,7 @@ class TestSchedulerInitReqMaxNewTokens(unittest.TestCase): input_len = len(req.origin_input_ids) page_size = scheduler.page_size + shard_widening = scheduler.kv_shard_widening paged_input_len = -(-input_len // page_size) * page_size limit = scheduler.max_new_tokens_limit limit_active = limit is not None and limit > 0 @@ -97,7 +99,8 @@ class TestSchedulerInitReqMaxNewTokens(unittest.TestCase): def satisfies_rules(candidate: int) -> bool: context_ok = input_len + candidate < scheduler.max_req_len budget_ok = ( - paged_input_len + candidate + page_size < scheduler.max_total_num_tokens + paged_input_len + candidate + page_size * shard_widening + < scheduler.max_total_num_tokens * shard_widening ) limit_ok = not limit_active or candidate <= limit requested_ok = requested is None or candidate <= requested @@ -158,6 +161,36 @@ class TestSchedulerInitReqMaxNewTokens(unittest.TestCase): max_total_num_tokens - paged_input_len - page_size - 1, ) + def test_dcp_budget_is_already_in_logical_tokens(self): + with get_parallel().override(attn_dcp_size=4): + scheduler = self._new_scheduler(max_total_num_tokens=24, page_size=4) + req = self._new_req(max_new_tokens=64, input_len=8) + + self.assertEqual(self._init_and_check(scheduler, req), 11) + + def test_sharded_budget_reserves_one_page_per_shard(self): + max_total_num_tokens, page_size, input_len, shard_widening = 8, 4, 8, 3 + scheduler = self._new_scheduler( + max_req_len=128, + max_total_num_tokens=max_total_num_tokens, + page_size=page_size, + kv_shard_widening=shard_widening, + ) + req = self._new_req(max_new_tokens=64, input_len=input_len) + + max_new_tokens = self._init_and_check(scheduler, req) + + widened_capacity = max_total_num_tokens * shard_widening + per_req_overhead = page_size * shard_widening + self.assertEqual( + max_new_tokens, + widened_capacity - input_len - per_req_overhead - 1, + ) + self.assertLess( + input_len + max_new_tokens + per_req_overhead, + widened_capacity, + ) + def test_min_new_tokens_clamped_to_limit(self): with envs.SGLANG_MAX_NEW_TOKENS_LIMIT.override(16): scheduler = self._new_scheduler()