Files
sglang/test/registered/unit/managers/test_prefill_adder.py
T
2026-09-27 14:19:08 +08:00

1633 lines
69 KiB
Python

import unittest
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import torch
import sglang.srt.managers.schedule_policy as schedule_policy
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.managers.schedule_policy import (
AddReqResult,
PrefillAdder,
SchedulePolicy,
estimate_prefill_extend_tile_metrics,
)
from sglang.srt.mem_cache.allocator.page_interleave import PageInterleavePoolAllocator
from sglang.srt.mem_cache.base_prefix_cache import (
CacheRequestHandle,
DecLockRefResult,
IncLockRefResult,
)
from sglang.srt.mem_cache.page_interleave import PageShardSpec, make_page_shard_spec
from sglang.srt.mem_cache.prefill_budget import (
PrefillBudget,
SWAPrefillBudget,
estimate_swa_kv_tokens,
)
from sglang.srt.mem_cache.radix_cache import RadixCache
from sglang.srt.mem_cache.unified_memory_pool import init_unified_swa_pools
from sglang.srt.runtime_context import get_context
from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler
from sglang.srt.utils.common import Range
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
class _RecordingDelayer:
"""Duck-typed stand-in for PrefillDelayerSinglePassExecutor that records
the local_prefillable value of every negotiate call and returns a fixed
verdict."""
def __init__(self, allow: bool):
self.allow = allow
self.calls = []
def negotiate_should_allow_prefill(self, local_prefillable, **kwargs):
self.calls.append(local_prefillable)
return self.allow
class TestPrefillAdder(CustomTestCase):
def setUp(self):
set_global_server_args_for_scheduler(ServerArgs(model_path="dummy"))
self.mock_tree_cache = self.create_tree_cache()
self.mock_token_allocator = self.create_token_allocator()
def create_tree_cache(
self,
*,
full_evictable_size: int = 0,
swa_evictable_size: int = 0,
evictable_size: int = 0,
) -> MagicMock:
tree_cache = MagicMock()
tree_cache.full_evictable_size.return_value = full_evictable_size
tree_cache.swa_evictable_size.return_value = swa_evictable_size
tree_cache.evictable_size.return_value = evictable_size
tree_cache.disable = False
tree_cache.inc_lock_ref.return_value = IncLockRefResult()
tree_cache.dec_lock_ref.return_value = DecLockRefResult()
tree_cache.buffer_pipeline = None
return tree_cache
def create_token_allocator(
self,
*,
full_available_size: int = 0,
swa_available_size: int = 0,
available_size: int = 0,
size_swa: int = 1_000_000,
) -> MagicMock:
allocator = MagicMock()
allocator.full_available_size.return_value = full_available_size
allocator.swa_available_size.return_value = swa_available_size
allocator.available_size.return_value = available_size
allocator.size_swa = size_swa
allocator.swa_req_ring = False
allocator.create_prefill_budget.side_effect = lambda tree_cache, **kwargs: (
PrefillBudget(allocator, tree_cache, **kwargs)
)
return allocator
def create_running_batch(self, reqs=None) -> MagicMock:
batch = MagicMock()
batch.reqs = list(reqs or [])
batch.release_req.return_value = None
batch.filter_batch.return_value = None
return batch
def scheduling_order(self, *, schedule_low_priority_values_first: bool):
"""State the policy on the context, which is where the scheduler reads
it: `preempt_to_schedule` takes no record to state it on."""
override = get_context().override_server_args(
schedule_low_priority_values_first=schedule_low_priority_values_first
)
override.install()
self.addCleanup(override.restore)
def create_mock_req(self, rid, priority, max_new_tokens, output_len=0, wait_time=0):
req = MagicMock(spec=Req)
req.rid = str(rid)
req.cache_request_handle = CacheRequestHandle(req.rid, 0)
req.priority = priority
req.prefix_indices = []
req.full_untruncated_fill_ids = []
req.output_ids = [0] * output_len
req.sampling_params = SimpleNamespace(max_new_tokens=max_new_tokens)
req.time_stats = SimpleNamespace(wait_queue_entry_time=wait_time)
req.retracted_stain = False
req.host_hit_length = 0
req.swa_host_hit_length = 0
req.storage_hit_length = 0
req.storage_hit_start = None
req.host_hit_is_storage = False
req.host_loaded_length = 0
req.materialized_host_hit_len.return_value = 0
req.fulfilled_storage_hit_len.return_value = 0
req.finished.return_value = False
req.needs_host_load_back.return_value = False
return req
def create_adder(self, running_batch, **kwargs):
defaults = dict(
page_size=1,
tree_cache=self.mock_tree_cache,
token_to_kv_pool_allocator=self.mock_token_allocator,
running_batch=running_batch,
new_token_ratio=1.0,
rem_input_tokens=10000,
rem_chunk_tokens=None,
num_mixed_decode_tokens=0,
priority_scheduling_preemption_threshold=0,
)
defaults.update(kwargs)
defaults["token_to_kv_pool_allocator"].page_size = defaults["page_size"]
return PrefillAdder(**defaults)
def create_shared_adder(self, *, num_mixed_decode_tokens=0):
self.mock_tree_cache.supports_mamba.return_value = False
self.mock_tree_cache.sliding_window_size = 8
self.mock_tree_cache.is_tree_cache.return_value = False
allocator = init_unified_swa_pools(
device="cpu",
kv_cache_dtype=torch.float16,
head_num=1,
head_dim=4,
v_head_dim=4,
swa_head_num=1,
swa_head_dim=4,
swa_v_head_dim=4,
page_size=4,
start_layer=0,
end_layer=2,
swa_attention_layer_ids=[1],
full_attention_layer_ids=[0],
total_bytes=1024,
enable_memory_saver=False,
need_sort=False,
lazy_compaction=True,
).token_to_kv_pool_allocator
return self.create_adder(
self.create_running_batch(),
page_size=4,
rem_chunk_tokens=16,
num_mixed_decode_tokens=num_mixed_decode_tokens,
token_to_kv_pool_allocator=allocator,
)
def create_shared_req(self, rid, max_new_tokens=4):
req = self.create_mock_req(rid, priority=0, max_new_tokens=max_new_tokens)
req.sampling_params.ignore_eos = False
req.swa_host_hit_length = 0
req.last_node = MagicMock()
req.full_untruncated_fill_ids = list(range(12))
req.set_extend_range = MagicMock(
side_effect=lambda start, end: setattr(
req, "extend_range", Range(start, end)
)
)
return req
def create_shortest_prefill_adder(self, *, chunk_tokens=4096):
override = get_context().override_server_args(
schedule_policy="shortest-prefill-first"
)
override.install()
self.addCleanup(override.restore)
self.mock_tree_cache.supports_mamba.return_value = False
self.mock_tree_cache.is_tree_cache.return_value = False
self.mock_token_allocator.available_size.return_value = 32768
return self.create_adder(
self.create_running_batch(), page_size=256, rem_chunk_tokens=chunk_tokens
)
def test_shortest_prefill_reserves_space_for_complete_waiting_requests(self):
adder = self.create_shortest_prefill_adder()
policy = SchedulePolicy(
policy="shortest-prefill-first",
tree_cache=RadixCache.create_simulated(),
enable_hierarchical_cache=True,
enable_priority_scheduling=False,
schedule_low_priority_values_first=False,
)
continuation = self.create_shared_req("continuation")
continuation.full_untruncated_fill_ids = list(range(16384))
waiting = [self.create_shared_req("a"), self.create_shared_req("b")]
for req, length in zip(waiting, [512, 1024]):
req.origin_input_ids = list(range(length))
req.full_untruncated_fill_ids = list(range(length))
req.num_matched_prefix_tokens = 0
adder.chunked_req_limit = policy.shortest_prefill_chunk_limit(
continuation, waiting, adder.rem_chunk_tokens, adder.page_size
)
self.assertIs(adder.add_chunked_req(continuation), continuation)
self.assertEqual(continuation.extend_range.length, 2560)
for req in waiting:
adder.add_one_req(req, has_chunked_req=True, truncation_align_size=None)
self.assertEqual(adder.can_run_list, [continuation, *waiting])
self.assertIsNone(adder.new_chunked_req)
self.assertEqual(adder.rem_chunk_tokens, 0)
self.assertGreaterEqual(adder.rem_total_tokens, 0)
def test_shortest_prefill_rejects_second_unfinished_chunk(self):
adder = self.create_shortest_prefill_adder(chunk_tokens=512)
req = self.create_shared_req("second-chunk")
req.full_untruncated_fill_ids = list(range(1024))
self.assertEqual(
adder.add_one_req(req, has_chunked_req=True, truncation_align_size=None),
AddReqResult.OTHER,
)
self.assertEqual(adder.can_run_list, [])
self.assertIsNone(adder.new_chunked_req)
req.set_extend_range.assert_not_called()
self.mock_tree_cache.init_load_back.assert_not_called()
def test_shortest_prefill_rechecks_chunk_limit_after_host_miss(self):
adder = self.create_shortest_prefill_adder(chunk_tokens=512)
req = self.create_shared_req("host-miss")
req.full_untruncated_fill_ids = list(range(1024))
req.prefix_indices = torch.empty(0, dtype=torch.int64)
req.host_hit_length = 768
req.best_match_node = req.last_node
req.needs_host_load_back.return_value = True
self.mock_tree_cache.init_load_back.return_value = (
torch.empty(0, dtype=torch.int64),
req.last_node,
)
self.assertEqual(
adder.add_one_req(req, has_chunked_req=True, truncation_align_size=None),
AddReqResult.OTHER,
)
self.mock_tree_cache.init_load_back.assert_called_once()
self.assertEqual(adder.can_run_list, [])
req.set_extend_range.assert_not_called()
def test_shortest_prefill_preserves_memory_admission(self):
adder = self.create_shortest_prefill_adder()
self.mock_token_allocator.available_size.return_value = 256
req = self.create_shared_req("no-memory")
req.full_untruncated_fill_ids = list(range(512))
self.assertEqual(
adder.add_one_req(req, has_chunked_req=True, truncation_align_size=None),
AddReqResult.NO_TOKEN,
)
self.assertEqual(adder.can_run_list, [])
def test_continuation_without_limit_keeps_normal_chunk_size(self):
adder = self.create_shortest_prefill_adder()
req = self.create_shared_req("continuation")
req.full_untruncated_fill_ids = list(range(8192))
self.assertIs(adder.add_chunked_req(req), req)
self.assertEqual(req.extend_range.length, 4096)
def test_shared_admission_reserves_all_pending_requests(self):
adder = self.create_shared_adder()
first, second = (
self.create_shared_req("first"),
self.create_shared_req("second"),
)
adder.add_one_req(first, has_chunked_req=False, truncation_align_size=None)
self.assertEqual(adder.can_run_list, [first])
self.assertEqual(
adder.add_one_req(
second, has_chunked_req=False, truncation_align_size=None
),
AddReqResult.NO_TOKEN,
)
self.assertEqual(adder.can_run_list, [first])
def test_shared_admission_rechecks_after_prefix_lock(self):
adder = self.create_shared_adder()
self.assertIsNotNone(adder.token_to_kv_pool_allocator.alloc(24))
self.mock_tree_cache.full_evictable_size.return_value = 24
self.mock_tree_cache.swa_evictable_size.return_value = 24
def lock_prefix(_):
self.mock_tree_cache.full_evictable_size.return_value = 0
self.mock_tree_cache.swa_evictable_size.return_value = 0
return IncLockRefResult()
self.mock_tree_cache.inc_lock_ref.side_effect = lock_prefix
req = self.create_shared_req("locked-prefix")
self.assertEqual(
adder.add_one_req(req, has_chunked_req=False, truncation_align_size=None),
AddReqResult.NO_TOKEN,
)
self.mock_tree_cache.inc_lock_ref.assert_called_once()
self.assertEqual(adder.can_run_list, [])
def test_shared_continuation_defers_when_decode_consumes_chunk_budget(self):
"""Mixed decode must not commit an empty or negatively sliced prompt."""
for decode_tokens in (16, 28):
with self.subTest(decode_tokens=decode_tokens):
adder = self.create_shared_adder(num_mixed_decode_tokens=decode_tokens)
req = self.create_shared_req("continuation")
before = (
adder.memory_budget.total_offset,
adder.memory_budget.swa_offset,
)
self.assertIs(adder.add_chunked_req(req), req)
self.assertEqual(adder.can_run_list, [])
req.set_extend_range.assert_not_called()
self.assertEqual(
(adder.memory_budget.total_offset, adder.memory_budget.swa_offset),
before,
)
def test_shared_continuation_uses_memory_chunk_limit(self):
adder = self.create_shared_adder()
req = self.create_shared_req("continuation", max_new_tokens=80)
self.assertIs(adder.add_chunked_req(req), req)
self.assertEqual(req.extend_range.length, 8)
self.assertEqual(adder.memory_budget.total_offset, 12)
self.assertEqual(adder.memory_budget.swa_offset, 12)
def test_storage_prefetch_fulfillment_resolves_at_admission(self):
adder = self.create_adder(self.create_running_batch())
req = self.create_mock_req("storage-hit", priority=0, max_new_tokens=1)
req.host_hit_length = 12
req.host_loaded_length = 4
req.storage_hit_length = 8
req.storage_hit_start = 4
req.materialized_host_hit_len.return_value = 4
req.fulfilled_storage_hit_len.return_value = 8
req.needs_host_load_back.return_value = True
adder._account_prefill_cache_admission(req, prefix_len=12)
self.mock_tree_cache.finish_storage_prefetch_admission.assert_called_once_with(
req.cache_request_handle,
fulfilled_tokens=8,
reason=None,
)
self.assertEqual(adder.log_device_hit_tokens, 4)
self.assertEqual(adder.log_host_hit_tokens, 0)
self.assertEqual(adder.log_storage_hit_tokens, 8)
self.mock_tree_cache.finish_storage_prefetch_admission.reset_mock()
req.host_loaded_length = 0
req.materialized_host_hit_len.return_value = 0
req.fulfilled_storage_hit_len.return_value = 0
adder._account_prefill_cache_admission(req, prefix_len=0)
self.mock_tree_cache.finish_storage_prefetch_admission.assert_called_once_with(
req.cache_request_handle, fulfilled_tokens=0, reason="device_capacity"
)
self.mock_tree_cache.finish_storage_prefetch_admission.reset_mock()
req.host_hit_length = 4
req.host_loaded_length = 4
req.storage_hit_length = 8
req.storage_hit_start = 4
req.materialized_host_hit_len.return_value = 4
req.fulfilled_storage_hit_len.return_value = 4
req.needs_host_load_back.return_value = True
adder._account_prefill_cache_admission(req, prefix_len=8)
self.mock_tree_cache.finish_storage_prefetch_admission.assert_called_once_with(
req.cache_request_handle,
fulfilled_tokens=4,
reason="cache_admission_shortfall",
)
self.assertEqual(adder.log_device_hit_tokens, 8)
self.assertEqual(adder.log_host_hit_tokens, 0)
self.assertEqual(adder.log_storage_hit_tokens, 12)
def test_retracted_storage_prefetch_accounting_is_omitted(self):
adder = self.create_adder(self.create_running_batch())
req = self.create_mock_req(
"retracted-storage-hit", priority=0, max_new_tokens=1
)
req.retracted_stain = True
adder._account_prefill_cache_admission(req, prefix_len=8)
self.mock_tree_cache.discard_storage_prefetch_accounting.assert_called_once_with(
req.cache_request_handle
)
self.mock_tree_cache.finish_storage_prefetch_admission.assert_not_called()
def test_preempt_success_high_priority_values_first(self):
params = [
("run1", 0, 50),
("run2", 1, 75),
("run3", 2, 100),
]
running_reqs = [
self.create_mock_req(rid, priority, max_new_tokens)
for rid, priority, max_new_tokens in params
]
self.scheduling_order(schedule_low_priority_values_first=False)
running_batch = self.create_running_batch(running_reqs)
adder = self.create_adder(running_batch)
self.assertEqual(adder.memory_budget.total_offset, 225)
self.mock_token_allocator.full_available_size.return_value = (
225 # full occupation of GRam
)
self.mock_token_allocator.available_size.return_value = 225
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=49)
success = adder.preempt_to_schedule(new_req)
self.assertTrue(success)
self.assertIn(running_reqs[0], adder.preempt_list)
self.assertEqual(
adder.memory_budget.total_offset, 175
) # 50 + 75 + 100 - 50 = 175
running_batch.release_req.assert_called_once()
def test_preempt_success_low_priority_values_first(self):
params = [
("run1", 0, 50),
("run2", 1, 75),
("run3", 2, 100),
]
running_reqs = [
self.create_mock_req(rid, priority, max_new_tokens)
for rid, priority, max_new_tokens in params
]
self.scheduling_order(schedule_low_priority_values_first=True)
running_batch = self.create_running_batch(running_reqs)
adder = self.create_adder(running_batch)
self.assertEqual(adder.memory_budget.total_offset, 225)
self.mock_token_allocator.full_available_size.return_value = (
225 # full occupation of GRam
)
self.mock_token_allocator.available_size.return_value = 225
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=49)
success = adder.preempt_to_schedule(new_req)
self.assertTrue(success)
self.assertIn(running_reqs[2], adder.preempt_list)
self.assertEqual(
adder.memory_budget.total_offset, 125
) # 50 + 75 + 100 - 100 = 125
running_batch.release_req.assert_called_once()
def test_preempt_fail_low_priority_values_first(self):
params = [
("run1", 0, 50),
("run2", 1, 75),
("run3", 2, 100),
]
running_reqs = [
self.create_mock_req(rid, priority, max_new_tokens)
for rid, priority, max_new_tokens in params
]
self.scheduling_order(schedule_low_priority_values_first=True)
running_batch = self.create_running_batch(running_reqs)
adder = self.create_adder(running_batch)
self.assertEqual(adder.memory_budget.total_offset, 225)
self.mock_token_allocator.full_available_size.return_value = (
225 # full occupation of GRam
)
self.mock_token_allocator.available_size.return_value = 225
new_req_fail_by_priority_check = self.create_mock_req(
"new1", priority=2, max_new_tokens=49
)
success_by_priority_check = adder.preempt_to_schedule(
new_req_fail_by_priority_check
)
self.assertFalse(success_by_priority_check)
new_req_fail_by_priority_check = self.create_mock_req(
"new2", priority=1, max_new_tokens=110
)
success_by_capacity_check = adder.preempt_to_schedule(
new_req_fail_by_priority_check
)
self.assertFalse(success_by_capacity_check)
def test_preempt_fail_high_priority_values_first(self):
params = [
("run1", 0, 50),
("run2", 1, 75),
("run3", 2, 100),
]
running_reqs = [
self.create_mock_req(rid, priority, max_new_tokens)
for rid, priority, max_new_tokens in params
]
self.scheduling_order(schedule_low_priority_values_first=False)
running_batch = self.create_running_batch(running_reqs)
adder = self.create_adder(running_batch)
self.assertEqual(adder.memory_budget.total_offset, 225)
self.mock_token_allocator.full_available_size.return_value = (
225 # full occupation of GRam
)
self.mock_token_allocator.available_size.return_value = 225
new_req_fail_by_priority_check = self.create_mock_req(
"new1", priority=0, max_new_tokens=49
)
success_by_priority_check = adder.preempt_to_schedule(
new_req_fail_by_priority_check
)
self.assertFalse(success_by_priority_check)
new_req_fail_by_priority_check = self.create_mock_req(
"new2", priority=-1, max_new_tokens=110
)
success_by_capacity_check = adder.preempt_to_schedule(
new_req_fail_by_priority_check
)
self.assertFalse(success_by_capacity_check)
def test_preempt_skip_already_preempted_request(self):
params = [
("req_prio_0", 0, 50),
("req_prio_1", 1, 75),
("req_prio_2", 2, 100),
]
running_reqs = [
self.create_mock_req(rid, priority, max_new_tokens)
for rid, priority, max_new_tokens in params
]
self.scheduling_order(schedule_low_priority_values_first=False)
running_batch = self.create_running_batch(running_reqs)
adder = self.create_adder(running_batch)
self.assertEqual(adder.memory_budget.total_offset, 225)
self.mock_token_allocator.full_available_size.return_value = 225
self.mock_token_allocator.available_size.return_value = 225
# New request preempts req_prio_0
first_req = self.create_mock_req(
"new_req_prio_1", priority=1, max_new_tokens=49
)
first_success = adder.preempt_to_schedule(first_req)
self.assertTrue(first_success)
self.assertIn(running_reqs[0], adder.preempt_list)
self.assertEqual(adder.memory_budget.total_offset, 175)
running_batch.release_req.assert_called_once()
# Second call needs more tokens than currently free, so it would need to
# preempt req_prio_0 again if already-preempted requests were not filtered out.
second_req = self.create_mock_req(
"second_new_req_prio_1", priority=1, max_new_tokens=76
)
second_success = adder.preempt_to_schedule(second_req)
self.assertFalse(second_success)
self.assertEqual(adder.memory_budget.total_offset, 175)
self.assertEqual(adder.preempt_list.count(running_reqs[0]), 1)
running_batch.release_req.assert_called_once()
def test_preempt_success_low_priority_values_first_exact_once(self):
params = [
("run1", 0, 50),
("run2", 1, 75),
("run3", 2, 100),
("run4", 2, 125),
("run4", 2, 125),
]
running_reqs = [
self.create_mock_req(rid, priority, max_new_tokens)
for rid, priority, max_new_tokens in params
]
self.scheduling_order(schedule_low_priority_values_first=True)
running_batch = self.create_running_batch(running_reqs)
adder = self.create_adder(running_batch)
self.assertEqual(adder.memory_budget.total_offset, 475)
self.mock_token_allocator.full_available_size.return_value = (
475 # full occupation of GRam
)
self.mock_token_allocator.available_size.return_value = 475
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=75)
success = adder.preempt_to_schedule(new_req)
self.assertTrue(success)
self.assertIn(running_reqs[2], adder.preempt_list)
self.assertEqual(
adder.memory_budget.total_offset, 375
) # 50 + 75 + 100 + 125 + 125 - 100 = 375
running_batch.release_req.assert_called_once()
def test_preempt_success_low_priority_values_first_exact_twice(self):
params = [
("run1", 0, 50),
("run2", 1, 75),
("run3", 2, 100),
("run4", 2, 125),
("run4", 2, 125),
]
running_reqs = [
self.create_mock_req(rid, priority, max_new_tokens)
for rid, priority, max_new_tokens in params
]
self.scheduling_order(schedule_low_priority_values_first=True)
running_batch = self.create_running_batch(running_reqs)
adder = self.create_adder(running_batch)
self.assertEqual(adder.memory_budget.total_offset, 475)
self.mock_token_allocator.full_available_size.return_value = (
475 # full occupation of GRam
)
self.mock_token_allocator.available_size.return_value = 475
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=200)
success = adder.preempt_to_schedule(new_req)
self.assertTrue(success)
self.assertIn(running_reqs[2], adder.preempt_list)
self.assertIn(running_reqs[3], adder.preempt_list)
self.assertEqual(
adder.memory_budget.total_offset, 250
) # 50 + 75 + 100 + 125 + 125 - 100 - 125 = 250
self.assertEqual(running_batch.release_req.call_count, 2)
def test_mixed_chunk_prefill_budgets(self):
self.mock_token_allocator.available_size.return_value = 1000
decode_reqs = [
self.create_mock_req(f"decode_{i}", priority=0, max_new_tokens=50)
for i in range(8)
]
running_batch = self.create_running_batch(decode_reqs)
adder = self.create_adder(
running_batch,
rem_input_tokens=200,
rem_chunk_tokens=64,
num_mixed_decode_tokens=len(decode_reqs),
)
self.assertEqual(adder.rem_input_tokens, 192) # 200 - 8
self.assertEqual(adder.rem_chunk_tokens, 56) # 64 - 8
self.assertEqual(adder.memory_budget.total_offset, 408) # 8 + 8 * 50
self.assertEqual(adder.memory_budget.current_offset, 8)
self.assertEqual(adder.budget_state(), AddReqResult.CONTINUE)
# Add a prefill that exactly consumes the chunk budget
req1 = self.create_mock_req("req1", priority=0, max_new_tokens=64)
req1.host_hit_length = 0
req1.prefix_indices = []
req1.full_untruncated_fill_ids = list(range(56))
req1.last_node = MagicMock()
req1.sampling_params.ignore_eos = False
# add_one_req reads req.extend_range.length after set_extend_range;
# emulate the real Req writer (a spec=Req mock lacks the attribute).
req1.set_extend_range = MagicMock(
side_effect=lambda start, end: setattr(
req1, "extend_range", Range(start, end)
)
)
result1 = adder.add_one_req(
req1, has_chunked_req=False, truncation_align_size=None
)
self.assertEqual(len(adder.can_run_list), 1)
self.assertEqual(adder.rem_chunk_tokens, 0) # 56 - 56
self.assertEqual(adder.rem_input_tokens, 136) # 192 - 56
self.assertEqual(result1, AddReqResult.OTHER)
# 3 decode requests finished
remaining_decode_reqs = decode_reqs[3:]
running_batch2 = self.create_running_batch(remaining_decode_reqs)
adder2 = self.create_adder(
running_batch2,
rem_input_tokens=200,
rem_chunk_tokens=64,
num_mixed_decode_tokens=len(remaining_decode_reqs),
)
self.assertEqual(adder2.rem_input_tokens, 195) # 200 - 5
self.assertEqual(adder2.rem_chunk_tokens, 59) # 64 - 5
self.assertEqual(adder2.memory_budget.total_offset, 255) # 5 + 5 * 50
self.assertEqual(adder2.budget_state(), AddReqResult.CONTINUE)
# Same prefill no longer exhausts the chunk budget
req2 = self.create_mock_req("req2", priority=0, max_new_tokens=64)
req2.host_hit_length = 0
req2.prefix_indices = []
req2.full_untruncated_fill_ids = list(range(56))
req2.last_node = MagicMock()
req2.sampling_params.ignore_eos = False
req2.set_extend_range = MagicMock(
side_effect=lambda start, end: setattr(
req2, "extend_range", Range(start, end)
)
)
result2 = adder2.add_one_req(
req2, has_chunked_req=False, truncation_align_size=None
)
self.assertEqual(len(adder2.can_run_list), 1)
self.assertEqual(adder2.rem_chunk_tokens, 3) # 59 - 56 = 3 remaining
self.assertEqual(result2, AddReqResult.CONTINUE)
# Fit last small prefill request
req3 = self.create_mock_req("req3", priority=0, max_new_tokens=16)
req3.host_hit_length = 0
req3.prefix_indices = []
req3.full_untruncated_fill_ids = list(range(3))
req3.last_node = MagicMock()
req3.sampling_params.ignore_eos = False
req3.set_extend_range = MagicMock(
side_effect=lambda start, end: setattr(
req3, "extend_range", Range(start, end)
)
)
result3 = adder2.add_one_req(
req3, has_chunked_req=False, truncation_align_size=None
)
self.assertEqual(len(adder2.can_run_list), 2)
self.assertEqual(adder2.rem_chunk_tokens, 0) # 3 - 3 = 0
self.assertEqual(result3, AddReqResult.OTHER)
@patch(
"sglang.srt.managers.schedule_policy.page_interleave_shard_size",
return_value=4,
)
def test_ignore_eos_reserves_all_shard_class_pages(self, _shard_size):
"""Disabled-radix admission must charge one page per shard class."""
self.mock_tree_cache.disable = True
self.mock_token_allocator.page_size = 16
self.mock_token_allocator.shard_spec = SimpleNamespace(
max_prefix_tokens=1024, chunk_tokens=1024
)
self.mock_token_allocator.available_size.return_value = 64
adder = self.create_adder(self.create_running_batch(), page_size=16)
req = self.create_mock_req("ignore_eos", priority=0, max_new_tokens=1)
req.sampling_params.ignore_eos = True
req.origin_input_ids = list(range(16))
req.full_untruncated_fill_ids = list(range(16))
req.last_node = MagicMock()
req.set_extend_range = MagicMock(
side_effect=lambda start, end: setattr(
req, "extend_range", Range(start, end)
)
)
result = adder.add_one_req(
req, has_chunked_req=False, truncation_align_size=None
)
self.assertEqual(result, AddReqResult.NO_TOKEN)
self.assertEqual(adder.can_run_list, [])
req.set_extend_range.assert_not_called()
def create_sharded_adder(
self,
*,
chunk_tokens=16,
max_prefix_tokens=16,
shard_spec=None,
allocator_size=1024,
**kwargs,
):
self.mock_tree_cache.supports_mamba.return_value = False
self.mock_tree_cache.is_tree_cache.return_value = False
if shard_spec is None:
shard_spec = PageShardSpec(
shard_rank=0,
shard_size=2,
page_size=4,
max_prefix_tokens=max_prefix_tokens,
chunk_tokens=chunk_tokens,
)
allocator = PageInterleavePoolAllocator(
size=allocator_size,
physical_page_size=shard_spec.page_size,
shard_size=shard_spec.shard_size,
dtype=torch.int64,
device="cpu",
kvcache=None,
need_sort=False,
shard_spec=shard_spec,
)
return self.create_adder(
self.create_running_batch(),
page_size=shard_spec.page_size,
rem_chunk_tokens=shard_spec.chunk_tokens,
token_to_kv_pool_allocator=allocator,
**kwargs,
)
def create_sharded_req(self, rid, *, prefix_len=12, extend_len=4):
req = self.create_shared_req(rid, max_new_tokens=1)
req.prefix_indices = list(range(prefix_len))
req.full_untruncated_fill_ids = list(range(prefix_len + extend_len))
return req
def test_sharded_admission_defers_prefix_scratch_overflow(self):
adder = self.create_sharded_adder()
first = self.create_sharded_req("first")
second = self.create_sharded_req("second")
self.assertEqual(
adder.add_one_req(first, has_chunked_req=False, truncation_align_size=None),
AddReqResult.CONTINUE,
)
remaining = (adder.rem_total_tokens, adder.rem_chunk_tokens)
# Both requests fit KV capacity, but prefix scratch fits only one.
self.assertEqual(
adder.add_one_req(
second, has_chunked_req=False, truncation_align_size=None
),
AddReqResult.OTHER,
)
self.assertEqual(adder.can_run_list, [first])
second.set_extend_range.assert_not_called()
self.assertEqual((adder.rem_total_tokens, adder.rem_chunk_tokens), remaining)
self.assertEqual(adder.kv_shard_block_bound_pages, 2)
self.assertEqual(adder.kv_shard_chunk_pages, 1)
# Rejection leaves the remaining chunk scratch usable.
third = self.create_sharded_req("no-prefix", prefix_len=0)
self.assertEqual(
adder.add_one_req(third, has_chunked_req=False, truncation_align_size=None),
AddReqResult.CONTINUE,
)
self.assertEqual(adder.can_run_list, [first, third])
self.assertEqual(adder.kv_shard_block_bound_pages, 2)
self.assertEqual(adder.kv_shard_chunk_pages, 2)
def test_eight_long_prefixes_fit_an_eight_k_chunk(self):
context_len = 16384
extend_len = 1024
prefix_len = context_len - extend_len
kvc = SimpleNamespace(
page_size=4,
model_config=SimpleNamespace(context_len=context_len),
)
for chunk_tokens in (8192, 16384):
with self.subTest(chunk_tokens=chunk_tokens):
with (
patch(
"sglang.srt.mem_cache.page_interleave.get_kv_shard_group_info",
return_value=(0, 2),
),
patch(
"sglang.srt.mem_cache.page_interleave.get_schedule",
return_value=SimpleNamespace(chunked_prefill_size=chunk_tokens),
),
):
spec = make_page_shard_spec(kvc)
self.assertEqual(spec.max_prefix_tokens, 8 * context_len)
adder = self.create_sharded_adder(
shard_spec=spec,
allocator_size=8 * context_len,
rem_input_tokens=32768,
)
reqs = [
self.create_sharded_req(
str(i), prefix_len=prefix_len, extend_len=extend_len
)
for i in range(9)
]
for i, req in enumerate(reqs[:8]):
result = adder.add_one_req(
req, has_chunked_req=False, truncation_align_size=None
)
# The eighth request commits even if it exhausts the budget.
expected = (
AddReqResult.OTHER
if i == 7 and chunk_tokens == 8192
else AddReqResult.CONTINUE
)
self.assertEqual(result, expected)
self.assertEqual(adder.can_run_list, reqs[:8])
self.assertEqual(adder.rem_chunk_tokens, chunk_tokens - 8192)
self.assertEqual(adder.kv_shard_chunk_pages * spec.page_size, 8192)
self.assertEqual(
adder.kv_shard_block_bound_pages * spec.logical_page_size,
8 * prefix_len,
)
remaining = (
adder.rem_total_tokens,
adder.rem_chunk_tokens,
adder.kv_shard_block_bound_pages,
adder.kv_shard_chunk_pages,
)
self.assertEqual(
adder.add_one_req(
reqs[8], has_chunked_req=False, truncation_align_size=None
),
AddReqResult.OTHER,
)
self.assertEqual(adder.can_run_list, reqs[:8])
reqs[8].set_extend_range.assert_not_called()
self.assertEqual(
(
adder.rem_total_tokens,
adder.rem_chunk_tokens,
adder.kv_shard_block_bound_pages,
adder.kv_shard_chunk_pages,
),
remaining,
)
if chunk_tokens == 16384:
# Prefix scratch is full; a request without a prefix still fits.
no_prefix = self.create_sharded_req(
"no-prefix", prefix_len=0, extend_len=extend_len
)
self.assertEqual(
adder.add_one_req(
no_prefix,
has_chunked_req=False,
truncation_align_size=None,
),
AddReqResult.CONTINUE,
)
self.assertEqual(adder.can_run_list, reqs[:8] + [no_prefix])
def test_sharded_admission_reserves_only_selected_chunk(self):
# Use normal page rounding without a sharding alignment override.
for remaining, selected, verdict in (
(8, 8, AddReqResult.OTHER),
(7, 4, AddReqResult.CONTINUE),
):
with self.subTest(remaining=remaining):
adder = self.create_sharded_adder(chunk_tokens=8)
adder.exact_chunk_fill = False
adder.rem_chunk_tokens = remaining
req = self.create_sharded_req("chunked", extend_len=28)
self.assertEqual(
adder.add_one_req(
req, has_chunked_req=False, truncation_align_size=None
),
verdict,
)
self.assertEqual(adder.can_run_list, [req])
self.assertIs(adder.new_chunked_req, req)
self.assertEqual(req.extend_range, Range(12, 12 + selected))
self.assertEqual(adder.kv_shard_block_bound_pages, 2)
self.assertEqual(adder.kv_shard_chunk_pages, selected // 4)
def test_sharded_admission_reserves_nothing_when_delayed(self):
delayer = _RecordingDelayer(allow=False)
adder = self.create_sharded_adder(prefill_delayer_single_pass=delayer)
req = self.create_sharded_req("delayed")
self.assertEqual(
adder.add_one_req(req, has_chunked_req=False, truncation_align_size=None),
AddReqResult.OTHER,
)
self.assertEqual(adder.can_run_list, [])
req.set_extend_range.assert_not_called()
self.assertEqual(adder.kv_shard_block_bound_pages, 0)
self.assertEqual(adder.kv_shard_chunk_pages, 0)
delayer.allow = True
self.assertEqual(
adder.add_one_req(req, has_chunked_req=False, truncation_align_size=None),
AddReqResult.CONTINUE,
)
self.assertEqual(adder.can_run_list, [req])
self.assertEqual(adder.kv_shard_block_bound_pages, 2)
self.assertEqual(adder.kv_shard_chunk_pages, 1)
def _build_hybrid_swa_chunked_req(
self,
*,
page_size,
rem_swa,
rem_chunk=2048,
extend_input_len=500,
is_hybrid_swa=True,
full_available=100_000,
):
self.mock_token_allocator.swa_available_size.return_value = rem_swa
self.mock_token_allocator.full_available_size.return_value = full_available
self.mock_token_allocator.available_size.return_value = full_available
self.mock_tree_cache.sliding_window_size = 128
adder = self.create_adder(
self.create_running_batch(),
page_size=page_size,
rem_chunk_tokens=rem_chunk,
)
adder.is_hybrid_swa = is_hybrid_swa
if is_hybrid_swa:
adder.memory_budget = SWAPrefillBudget(
self.mock_token_allocator, self.mock_tree_cache
)
req = self.create_mock_req("chunked", priority=0, max_new_tokens=128)
req.prefix_indices = []
req.full_untruncated_fill_ids = list(range(extend_input_len))
# set_extend_range is the only writer of extend_range; the production
# path reads req.extend_range.length right after calling it, so the mock
# must actually set the attribute (a spec=Req mock has the method but
# not the instance attribute).
req.set_extend_range = MagicMock(
side_effect=lambda start, end: setattr(
req, "extend_range", Range(start, end)
)
)
return adder, req
def test_add_chunked_req_hybrid_swa_reserves_page_for_alloc_extend(self):
# alloc_extend needs extend_num_tokens + page_size per request. If the
# scheduler hands out all of rem_swa_tokens, alloc_extend cannot get its
# extra page and OOMs. With the fix, extend_input_len must cap at
# rem_swa_tokens - page_size so the page is reserved.
PAGE_SIZE = 64
REM_SWA = 100
adder, req = self._build_hybrid_swa_chunked_req(
page_size=PAGE_SIZE, rem_swa=REM_SWA
)
result = adder.add_chunked_req(req)
self.assertIs(result, req) # truncated → chunked prefill continues
req.set_extend_range.assert_called_once()
start, end = req.set_extend_range.call_args.args
new_len = end - start
self.assertLessEqual(new_len + PAGE_SIZE, REM_SWA)
self.assertEqual(new_len, REM_SWA - PAGE_SIZE)
def test_add_chunked_req_hybrid_swa_defers_when_swa_below_page(self):
# When rem_swa_tokens <= page_size there is no room to serve even the
# reservation, so the chunked req must be deferred (returned unchanged)
# instead of falling back to rem_chunk_tokens and bypassing SWA budget.
PAGE_SIZE = 64
adder, req = self._build_hybrid_swa_chunked_req(
page_size=PAGE_SIZE, rem_swa=PAGE_SIZE
)
result = adder.add_chunked_req(req)
self.assertIs(result, req)
req.set_extend_range.assert_not_called()
self.assertEqual(len(adder.can_run_list), 0)
def test_swa_budget_for_req(self):
# budget = max(alloc - window, 0) + min(extend + max_new, window) + page,
# where alloc = min(extend, rem_chunk). The decode headroom is the SWA the
# request adds to its own sliding window (extend + decode), capped at the
# window -- a cached prefix funds the rest -- rather than a constant
# window, which over-charged short requests into an admission livelock.
cases = [
# (extend, max_new, rem_chunk, window, page, expected, label)
(64, 512, None, 128, 16, 128 + 16, "long_decode_hits_window_floor"),
(64, 32, None, 128, 16, 96 + 16, "short_req_reserves_below_window"),
(10, 20, None, 512, 8, 30 + 8, "short_resume_tiny_budget"),
(300, 512, None, 128, 16, 300 + 16, "big_extend_over_window"),
(200, 512, 50, 64, 8, 64 + 8, "chunk_capped_alloc_hits_floor"),
# Multi-chunk: alloc = rem_chunk (1024) caps a huge extend, and the
# chunk itself exceeds the window, so term1 = alloc - window is driven
# by the chunk, not the full extend -> budget = chunk + page.
(2000, 256, 1024, 512, 16, 1024 + 16, "multichunk_alloc_over_window"),
(0, 512, None, 128, 16, 128 + 16, "extend_zero_long_decode"),
(0, 40, None, 128, 16, 40 + 16, "extend_zero_short_decode"),
]
for extend, max_new, rem_chunk, window, page, expected, label in cases:
with self.subTest(label=label):
self.mock_tree_cache.sliding_window_size = window
adder = self.create_adder(
self.create_running_batch(),
page_size=page,
rem_chunk_tokens=rem_chunk,
)
self.assertEqual(
estimate_swa_kv_tokens(
extend,
max_new,
sliding_window_size=window,
page_size=page,
allocation_limit=rem_chunk,
),
expected,
)
def test_swa_admission_admits_short_cached_resume_at_two_window_pool(self):
# Livelock regression (real incident). At an SWA pool ~= 2 sliding
# windows, a cached-prefix resume matches >= 1 window (locked, excluded
# from rem_swa) and has only a short uncached tail + a little decode
# left. The pre-fix constant-window reservation charged a second full
# window, so the admission gate (swa_needed >= rem_swa_tokens) rejected
# it every scheduler iteration while LPM kept it at the queue head --
# 100% scheduler CPU, idle GPU. Capping the reservation at
# min(extend + decode, window) admits it.
WINDOW, PAGE, REM_SWA = 128, 8, 100
PREFIX, EXTEND = 200, 16 # cached prefix > window; short uncached tail
self.mock_token_allocator.swa_available_size.return_value = REM_SWA
self.mock_token_allocator.full_available_size.return_value = 100_000
self.mock_token_allocator.available_size.return_value = 100_000
self.mock_tree_cache.sliding_window_size = WINDOW
self.mock_tree_cache.is_tree_cache.return_value = False
adder = self.create_adder(self.create_running_batch(), page_size=PAGE)
adder.is_hybrid_swa = True
adder.memory_budget = SWAPrefillBudget(
self.mock_token_allocator, self.mock_tree_cache
)
req = self.create_mock_req(
"resume", priority=0, max_new_tokens=40, output_len=10
)
req.prefix_indices = list(range(PREFIX))
req.full_untruncated_fill_ids = list(range(PREFIX + EXTEND))
req.host_hit_length = 0
req.swa_host_hit_length = 0
req.last_node = MagicMock()
req.set_extend_range = MagicMock(
side_effect=lambda start, end: setattr(
req, "extend_range", Range(start, end)
)
)
req.sampling_params = SimpleNamespace(max_new_tokens=40, ignore_eos=False)
# Pre-fix: a constant sliding-window reservation rejects the resume.
with patch.object(
adder.memory_budget, "swa_tokens", return_value=WINDOW + PAGE
):
self.assertIs(
adder.add_one_req(
req, has_chunked_req=False, truncation_align_size=None
),
AddReqResult.NO_TOKEN,
)
self.assertEqual(len(adder.can_run_list), 0)
# Fix: min(extend + decode, window) reservation admits it.
adder.add_one_req(req, has_chunked_req=False, truncation_align_size=None)
self.assertIn(req, adder.can_run_list)
def test_load_back_delivery_mismatch_reselects_the_prefill_shape(self):
# Two incidents: a load that delivers nothing left the SWA gate sized
# for the tail and the allocator OOMed; a cache-mode load that also
# surfaces FULL device tokens behind a host-only SWA window tripped a
# strict promised==loaded check and crashed the scheduler.
WINDOW, PAGE = 128, 8
SPAN, HOST_HIT = 1024, 1016
self.mock_token_allocator.swa_available_size.return_value = 400
self.mock_token_allocator.full_available_size.return_value = 100_000
self.mock_token_allocator.available_size.return_value = 100_000
self.mock_tree_cache.sliding_window_size = WINDOW
self.mock_tree_cache.is_tree_cache.return_value = False
def run(delivered: int, remaining_after_load: int = 100_000):
self.mock_token_allocator.full_available_size.return_value = 100_000
self.mock_token_allocator.swa_available_size.return_value = 400
adder = self.create_adder(self.create_running_batch(), page_size=PAGE)
adder.is_hybrid_swa = True
adder.memory_budget = SWAPrefillBudget(
self.mock_token_allocator, self.mock_tree_cache
)
req = self.create_mock_req("dropped-fetch", priority=0, max_new_tokens=8)
req.prefix_indices = torch.empty(0, dtype=torch.int64)
req.full_untruncated_fill_ids = list(range(SPAN))
req.host_hit_length = HOST_HIT
req.swa_host_hit_length = WINDOW
req.needs_host_load_back.return_value = True
req.last_node = MagicMock()
req.best_match_node = MagicMock()
req.kv = SimpleNamespace(cache_protected_len=0)
def set_extend_range(start, end):
req.extend_range = Range(start, end)
req.set_extend_range = MagicMock(side_effect=set_extend_range)
req.sampling_params = SimpleNamespace(max_new_tokens=8, ignore_eos=False)
def load_back(params):
self.mock_token_allocator.full_available_size.return_value = (
remaining_after_load
)
if remaining_after_load == 0:
self.mock_token_allocator.swa_available_size.return_value = 0
return torch.arange(delivered, dtype=torch.int64), req.last_node
self.mock_tree_cache.init_load_back.side_effect = load_back
verdict = adder.add_one_req(
req, has_chunked_req=False, truncation_align_size=None
)
return verdict, list(adder.can_run_list), req
# Promise kept: only the 8-token tail is prefilled, which fits.
_, admitted, _ = run(HOST_HIT)
self.assertEqual(len(admitted), 1)
# Nothing delivered: the whole span is prefilled and no longer fits, so
# admission must decline rather than OOM the pool.
verdict, admitted, _ = run(0)
self.assertIs(verdict, AddReqResult.NO_TOKEN)
self.assertEqual(admitted, [])
# Over-delivery: admitted with the loaded prefix, not the promise.
# The loaded prefix is now pinned and no longer part of the evictable
# budget. A successful load must not run admission gates again.
_, admitted, req = run(HOST_HIT + 4, remaining_after_load=0)
self.assertEqual(len(admitted), 1)
self.assertEqual(len(req.prefix_indices), HOST_HIT + 4)
self.assertEqual(req.kv.cache_protected_len, HOST_HIT + 4)
req.set_extend_range.assert_called_once_with(HOST_HIT + 4, SPAN)
# A partial FULL load stays fatal.
with self.assertRaisesRegex(RuntimeError, "promised"):
run(HOST_HIT // 2)
def _create_host_hit_req(self, *, prefix_len=0, host_hit=8192, tail=1024):
req = self._create_delayer_req(prefix_len + host_hit + tail)
req.prefix_indices = torch.arange(prefix_len)
req.host_hit_length = host_hit
req.needs_host_load_back.return_value = True
req.best_match_node = req.last_node
req.kv = SimpleNamespace(cache_protected_len=prefix_len)
return req
def test_successful_load_back_commits_the_selected_shape_once(self):
cases = (
("full", 0, 24, None, None, 8, 8),
("full_unaligned", 0, 24, None, None, 7, 8),
("retracted_unaligned", 0, 24, None, None, 7, 8),
("chunk", 0, 24, 4, None, 4, 0),
("aux_only", 24, 0, None, None, 8, 8),
("overdelivery_full", 0, 24, None, None, 8, 8),
("overdelivery_chunk", 0, 24, 4, None, 4, 0),
("overdelivery_chunk_end", 0, 24, 8, None, 8, 8),
(
"dllm",
0,
24,
None,
SimpleNamespace(
block_size=4,
max_running_requests=2,
requires_separate_context_encoding=False,
),
4,
0,
),
(
"overdelivery_dllm",
0,
24,
None,
SimpleNamespace(
block_size=4,
max_running_requests=2,
requires_separate_context_encoding=False,
),
4,
0,
),
)
for name, prefix_len, host_hit, chunk, dllm, extend, decode in cases:
with self.subTest(mode=name):
self.mock_tree_cache.reset_mock()
adder = self._create_delayer_adder(
available_tokens=100_000,
delayer=None,
page_size=2,
rem_chunk_tokens=chunk,
dllm_config=dllm,
)
req = self._create_host_hit_req(
prefix_len=prefix_len,
host_hit=host_hit,
tail=extend if chunk is None and dllm is None else 8,
)
req.retracted_stain = name == "retracted_unaligned"
if name.startswith("overdelivery"):
req.host_hit_length -= 4
old_node, restored_node = req.last_node, object()
if name == "aux_only":
req.swa_host_hit_length = 8
def load_back(params):
self.assertIs(params.req, req)
tile_gate.assert_called_once()
tile_gate.return_value = AddReqResult.OTHER
return torch.arange(host_hit), restored_node
self.mock_tree_cache.init_load_back.side_effect = load_back
with patch.object(
adder, "_check_prefill_tile_budget", return_value=None
) as tile_gate:
adder.add_one_req(req, False, None)
tile_gate.assert_called_once()
self.mock_tree_cache.init_load_back.assert_called_once()
self.assertEqual(adder.can_run_list, [req])
req.set_extend_range.assert_called_once_with(24, 24 + extend)
self.mock_tree_cache.inc_lock_ref.assert_any_call(restored_node)
self.assertIs(
self.mock_tree_cache.dec_lock_ref.call_args.args[0], old_node
)
self.assertEqual(adder.log_hit_tokens, 24)
self.assertEqual(adder.log_input_tokens, extend)
self.assertEqual(
adder.reprocessed_log_input_tokens,
extend if req.retracted_stain else 0,
)
self.assertEqual(
adder.memory_budget.total_offset,
adder.ceil_paged_tokens(extend) + decode + 2,
)
self.assertEqual(
adder.new_chunked_req is req,
name in ("chunk", "overdelivery_chunk"),
)
self.mock_tree_cache.init_load_back.side_effect = None
def test_swa_new_tokens_clamps_remaining_not_total(self):
# Remaining decode headroom must be min(max_new - generated, CLIP)
# (subtract-then-clip). The reversed order (clip-then-subtract) zeroes
# out a request that has already generated >= CLIP tokens but still has a
# long decode ahead, under-reserving its SWA window -> OOM risk on resume
# of a long-generation request.
from sglang.srt.managers.schedule_policy import CLIP_MAX_NEW_TOKENS as CLIP
adder = self.create_adder(self.create_running_batch())
cases = [
# (max_new, generated, expected, label)
(100, 10, 90, "below_clip_normal"),
(40, 100, 0, "already_finished"),
# clip-then-subtract would give 3996; subtract-then-clip caps at CLIP.
(CLIP + 6000, 100, CLIP, "long_gen_small_output_caps_at_clip"),
# clip-then-subtract would give 0; the long decode still needs CLIP.
(CLIP + 6000, CLIP + 100, CLIP, "long_gen_output_over_clip"),
]
for max_new, generated, expected, label in cases:
with self.subTest(label=label):
req = self.create_mock_req(
label, priority=0, max_new_tokens=max_new, output_len=generated
)
self.assertEqual(adder._swa_new_tokens(req), expected)
def test_delayer_not_consulted_when_kv_budget_rejects(self):
"""A rank whose first candidate fails the KV-budget gate must NOT
report local_prefillable=True: add_one_req returns NO_TOKEN before
negotiating, and finalize() later reports the rank as not
prefillable. Regression guard: the negotiate used to run at the top
of add_one_req, so under KV pressure a full rank still claimed
prefillable=True, the delayer saw "all prefillable" and allowed, the
full ranks then NO_TOKEN'ed out, and the DP-synced forward mixed
prefill and decode — the exact pattern the delayer exists to
prevent."""
delayer = _RecordingDelayer(allow=True)
adder = self._create_delayer_adder(available_tokens=10, delayer=delayer)
result = adder.add_one_req(
self._create_delayer_req(50),
has_chunked_req=False,
truncation_align_size=None,
)
self.assertEqual(result, AddReqResult.NO_TOKEN)
self.assertEqual(delayer.calls, [])
self.assertEqual(adder.can_run_list, [])
def test_delayer_not_consulted_when_post_lock_recheck_rejects(self):
"""Locking the request's own prefix converts evictable tokens into
protected ones, so a request can pass the pre-lock KV gate yet fail
the post-lock recheck — precisely the high-utilization regime the
delayer targets. The negotiate must sit after that recheck too, or
the rank claims prefillable=True and then runs decode."""
delayer = _RecordingDelayer(allow=True)
adder = self._create_delayer_adder(available_tokens=10, delayer=delayer)
# Pre-lock budget is covered by evictable tokens; inc_lock_ref pins
# them (evictable -> protected), shrinking the budget below demand.
self.mock_tree_cache.evictable_size.return_value = 1000
self.mock_tree_cache.full_evictable_size.return_value = 1000
def _pin_prefix(node):
self.mock_tree_cache.evictable_size.return_value = 0
self.mock_tree_cache.full_evictable_size.return_value = 0
return IncLockRefResult()
self.mock_tree_cache.inc_lock_ref.side_effect = _pin_prefix
result = adder.add_one_req(
self._create_delayer_req(50),
has_chunked_req=False,
truncation_align_size=None,
)
self.assertEqual(result, AddReqResult.NO_TOKEN)
self.assertEqual(delayer.calls, [])
self.assertEqual(adder.can_run_list, [])
def test_delay_verdict_blocks_admissible_request(self):
"""An admissible request must still be gated by the (relocated)
negotiate: on a delay verdict it is not admitted, and the rank
reported prefillable=True exactly once."""
delayer = _RecordingDelayer(allow=False)
adder = self._create_delayer_adder(available_tokens=100_000, delayer=delayer)
result = adder.add_one_req(
self._create_delayer_req(50),
has_chunked_req=False,
truncation_align_size=None,
)
self.assertEqual(result, AddReqResult.OTHER)
self.assertEqual(delayer.calls, [True])
self.assertEqual(adder.can_run_list, [])
def test_allow_verdict_admits_request(self):
"""On an allow verdict the request proceeds through admission — the
relocated negotiate must not block the commit path."""
delayer = _RecordingDelayer(allow=True)
adder = self._create_delayer_adder(available_tokens=100_000, delayer=delayer)
req = self._create_delayer_req(50)
result = adder.add_one_req(
req, has_chunked_req=False, truncation_align_size=None
)
self.assertEqual(result, AddReqResult.CONTINUE)
self.assertEqual(delayer.calls, [True])
self.assertIn(req, adder.can_run_list)
def test_chunked_req_negotiates_prefillable_and_proceeds(self):
"""A rank resuming a chunked prefill runs it this pass regardless of
the verdict, so add_chunked_req must report prefillable=True (else a
rank with an empty waiting queue reports False via finalize() and
peers delay while it prefills alone) and must not drop the chunk on
a delay verdict (that would leak memory)."""
delayer = _RecordingDelayer(allow=False)
adder = self._create_delayer_adder(
available_tokens=100_000, delayer=delayer, rem_chunk_tokens=500
)
req = self._create_delayer_req(200)
result = adder.add_chunked_req(req)
self.assertIsNone(result) # chunk fully admitted, not truncated
self.assertEqual(delayer.calls, [True])
self.assertIn(req, adder.can_run_list)
def _create_delayer_adder(self, *, available_tokens, delayer, **kwargs):
self.mock_token_allocator.available_size.return_value = available_tokens
self.mock_token_allocator.full_available_size.return_value = available_tokens
return self.create_adder(
self.create_running_batch(),
prefill_delayer_single_pass=delayer,
**kwargs,
)
def _create_delayer_req(self, num_tokens: int):
req = self.create_mock_req("delayer_req", priority=0, max_new_tokens=8)
req.full_untruncated_fill_ids = list(range(num_tokens))
req.host_hit_length = 0
req.last_node = MagicMock()
req.sampling_params.ignore_eos = False
req.set_extend_range = MagicMock(
side_effect=lambda start, end: setattr(
req, "extend_range", Range(start, end)
)
)
return req
def test_add_chunked_req_non_hybrid_no_swa_reservation(self):
# Non-hybrid path: the SWA-pool reservation must NOT apply, otherwise
# the fix would regress non-SWA models.
PAGE_SIZE = 16
adder, req = self._build_hybrid_swa_chunked_req(
page_size=PAGE_SIZE,
rem_swa=10,
rem_chunk=500,
extend_input_len=200,
is_hybrid_swa=False,
full_available=300,
)
result = adder.add_chunked_req(req)
self.assertIsNone(result)
req.set_extend_range.assert_called_once_with(0, 200)
self.assertIn(req, adder.can_run_list)
def _adder_with_extend_lens(self, extend_lens):
adder = PrefillAdder.__new__(PrefillAdder)
adder.can_run_list = [
SimpleNamespace(extend_input_len=length) for length in extend_lens
]
# BLOCK_M is auto-detected from the attention backend in production; the
# __new__ helper bypasses __init__, so set it explicitly. 64 matches the
# block_m the tile-count assertions below are computed against.
adder.prefill_tile_block_m = 64
return adder
def test_estimate_prefill_extend_tile_metrics(self):
metrics = estimate_prefill_extend_tile_metrics([1, 7, 13, 129], block_m=64)
self.assertEqual(metrics["q_tiles_per_request"], [1, 1, 1, 3])
self.assertEqual(metrics["legacy_q_tiles_per_head"], 12)
self.assertEqual(metrics["compact_q_tiles_per_head"], 6)
self.assertEqual(metrics["saved_q_tiles_per_head"], 6)
self.assertEqual(metrics["saved_q_tile_ratio"], 0.5)
def test_compact_prefill_tile_budget_admits_more_than_legacy(self):
adder = self._adder_with_extend_lens([1, 7, 13])
# The tile-budget admission is gated on HIP in production; force the gate
# on so this vendor-neutral admission-math check runs on any CI runner.
with (
patch.object(schedule_policy, "_IS_HIP", True),
patch.object(schedule_policy, "PREFILL_TILE_BUDGET", 6),
patch.object(schedule_policy, "PREFILL_TILE_BUDGET_MODE", "compact"),
):
self.assertIsNone(adder._check_prefill_tile_budget(129))
with (
patch.object(schedule_policy, "_IS_HIP", True),
patch.object(schedule_policy, "PREFILL_TILE_BUDGET", 6),
patch.object(schedule_policy, "PREFILL_TILE_BUDGET_MODE", "legacy"),
):
self.assertEqual(adder._check_prefill_tile_budget(129), AddReqResult.OTHER)
def test_prefill_tile_budget_always_allows_first_request(self):
adder = self._adder_with_extend_lens([])
with (
patch.object(schedule_policy, "_IS_HIP", True),
patch.object(schedule_policy, "PREFILL_TILE_BUDGET", 1),
):
self.assertIsNone(adder._check_prefill_tile_budget(4096))
def test_prefill_tile_budget_disabled_on_non_hip(self):
# AMD-only: on non-HIP vendors the tile-budget admission must be a no-op
# even when the budget env is set, so scheduler behavior is unchanged.
adder = self._adder_with_extend_lens([1, 7, 13])
with (
patch.object(schedule_policy, "_IS_HIP", False),
patch.object(schedule_policy, "PREFILL_TILE_BUDGET", 6),
patch.object(schedule_policy, "PREFILL_TILE_BUDGET_MODE", "legacy"),
):
self.assertIsNone(adder._check_prefill_tile_budget(129))
# ---- _swa_req_never_fits: the gate on the _swa_chunk_cap escape hatch ----
# The hatch shrinks a chunk and admits it instead of waiting; it must fire
# ONLY for a request whose SWA budget can never fit the drained pool (true
# head-of-line livelock). Admitting transient-pressure requests instead
# collapses the SWA evictable cushion and causes a retraction storm.
def create_swa_adder(
self, *, size_swa: int, sliding_window: int, page_size: int = 16
) -> PrefillAdder:
self.mock_tree_cache.sliding_window_size = sliding_window
self.mock_token_allocator = self.create_token_allocator(size_swa=size_swa)
self.mock_token_allocator.create_prefill_budget.side_effect = (
lambda tree_cache, **kwargs: SWAPrefillBudget(
self.mock_token_allocator, tree_cache, **kwargs
)
)
return self.create_adder(
self.create_running_batch(),
page_size=page_size,
rem_chunk_tokens=512,
)
def test_swa_never_fits_false_under_transient_pressure(self):
# Small request against an ample pool: budget << pool, so it would fit
# once running decodes drain -> must wait, not take the hatch.
adder = self.create_swa_adder(size_swa=1024, sliding_window=128)
self.assertFalse(
adder.memory_budget.swa_never_fits(extend_input_len=256, max_new_tokens=64)
)
def test_swa_never_fits_true_when_budget_exceeds_whole_pool(self):
# A large host-hit load-back charge pushes the budget past the entire
# pool: it can never fit however far the pool drains -> hatch.
adder = self.create_swa_adder(size_swa=1024, sliding_window=128)
self.assertTrue(
adder.memory_budget.swa_never_fits(
extend_input_len=256, max_new_tokens=64, swa_host_hit_length=4096
)
)
def test_swa_never_fits_is_gated_by_pool_capacity(self):
# Same request; only the pool size changes. Proves the check compares
# the budget against size_swa (guards against a wrong-accessor bug).
req = dict(extend_input_len=256, max_new_tokens=64, swa_host_hit_length=600)
self.assertTrue(
self.create_swa_adder(
size_swa=512, sliding_window=128
).memory_budget.swa_never_fits(**req)
)
self.assertFalse(
self.create_swa_adder(
size_swa=4096, sliding_window=128
).memory_budget.swa_never_fits(**req)
)
if __name__ == "__main__":
unittest.main()