1633 lines
69 KiB
Python
1633 lines
69 KiB
Python
import unittest
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import torch
|
|
|
|
import sglang.srt.managers.schedule_policy as schedule_policy
|
|
from sglang.srt.managers.schedule_batch import Req
|
|
from sglang.srt.managers.schedule_policy import (
|
|
AddReqResult,
|
|
PrefillAdder,
|
|
SchedulePolicy,
|
|
estimate_prefill_extend_tile_metrics,
|
|
)
|
|
from sglang.srt.mem_cache.allocator.page_interleave import PageInterleavePoolAllocator
|
|
from sglang.srt.mem_cache.base_prefix_cache import (
|
|
CacheRequestHandle,
|
|
DecLockRefResult,
|
|
IncLockRefResult,
|
|
)
|
|
from sglang.srt.mem_cache.page_interleave import PageShardSpec, make_page_shard_spec
|
|
from sglang.srt.mem_cache.prefill_budget import (
|
|
PrefillBudget,
|
|
SWAPrefillBudget,
|
|
estimate_swa_kv_tokens,
|
|
)
|
|
from sglang.srt.mem_cache.radix_cache import RadixCache
|
|
from sglang.srt.mem_cache.unified_memory_pool import init_unified_swa_pools
|
|
from sglang.srt.runtime_context import get_context
|
|
from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler
|
|
from sglang.srt.utils.common import Range
|
|
from sglang.test.ci.ci_register import register_cpu_ci
|
|
from sglang.test.test_utils import CustomTestCase
|
|
|
|
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
|
|
|
|
|
|
class _RecordingDelayer:
|
|
"""Duck-typed stand-in for PrefillDelayerSinglePassExecutor that records
|
|
the local_prefillable value of every negotiate call and returns a fixed
|
|
verdict."""
|
|
|
|
def __init__(self, allow: bool):
|
|
self.allow = allow
|
|
self.calls = []
|
|
|
|
def negotiate_should_allow_prefill(self, local_prefillable, **kwargs):
|
|
self.calls.append(local_prefillable)
|
|
return self.allow
|
|
|
|
|
|
class TestPrefillAdder(CustomTestCase):
|
|
def setUp(self):
|
|
set_global_server_args_for_scheduler(ServerArgs(model_path="dummy"))
|
|
self.mock_tree_cache = self.create_tree_cache()
|
|
self.mock_token_allocator = self.create_token_allocator()
|
|
|
|
def create_tree_cache(
|
|
self,
|
|
*,
|
|
full_evictable_size: int = 0,
|
|
swa_evictable_size: int = 0,
|
|
evictable_size: int = 0,
|
|
) -> MagicMock:
|
|
tree_cache = MagicMock()
|
|
tree_cache.full_evictable_size.return_value = full_evictable_size
|
|
tree_cache.swa_evictable_size.return_value = swa_evictable_size
|
|
tree_cache.evictable_size.return_value = evictable_size
|
|
tree_cache.disable = False
|
|
tree_cache.inc_lock_ref.return_value = IncLockRefResult()
|
|
tree_cache.dec_lock_ref.return_value = DecLockRefResult()
|
|
tree_cache.buffer_pipeline = None
|
|
return tree_cache
|
|
|
|
def create_token_allocator(
|
|
self,
|
|
*,
|
|
full_available_size: int = 0,
|
|
swa_available_size: int = 0,
|
|
available_size: int = 0,
|
|
size_swa: int = 1_000_000,
|
|
) -> MagicMock:
|
|
allocator = MagicMock()
|
|
allocator.full_available_size.return_value = full_available_size
|
|
allocator.swa_available_size.return_value = swa_available_size
|
|
allocator.available_size.return_value = available_size
|
|
allocator.size_swa = size_swa
|
|
allocator.swa_req_ring = False
|
|
allocator.create_prefill_budget.side_effect = lambda tree_cache, **kwargs: (
|
|
PrefillBudget(allocator, tree_cache, **kwargs)
|
|
)
|
|
return allocator
|
|
|
|
def create_running_batch(self, reqs=None) -> MagicMock:
|
|
batch = MagicMock()
|
|
batch.reqs = list(reqs or [])
|
|
batch.release_req.return_value = None
|
|
batch.filter_batch.return_value = None
|
|
return batch
|
|
|
|
def scheduling_order(self, *, schedule_low_priority_values_first: bool):
|
|
"""State the policy on the context, which is where the scheduler reads
|
|
it: `preempt_to_schedule` takes no record to state it on."""
|
|
override = get_context().override_server_args(
|
|
schedule_low_priority_values_first=schedule_low_priority_values_first
|
|
)
|
|
override.install()
|
|
self.addCleanup(override.restore)
|
|
|
|
def create_mock_req(self, rid, priority, max_new_tokens, output_len=0, wait_time=0):
|
|
req = MagicMock(spec=Req)
|
|
req.rid = str(rid)
|
|
req.cache_request_handle = CacheRequestHandle(req.rid, 0)
|
|
req.priority = priority
|
|
req.prefix_indices = []
|
|
req.full_untruncated_fill_ids = []
|
|
req.output_ids = [0] * output_len
|
|
req.sampling_params = SimpleNamespace(max_new_tokens=max_new_tokens)
|
|
req.time_stats = SimpleNamespace(wait_queue_entry_time=wait_time)
|
|
req.retracted_stain = False
|
|
req.host_hit_length = 0
|
|
req.swa_host_hit_length = 0
|
|
req.storage_hit_length = 0
|
|
req.storage_hit_start = None
|
|
req.host_hit_is_storage = False
|
|
req.host_loaded_length = 0
|
|
req.materialized_host_hit_len.return_value = 0
|
|
req.fulfilled_storage_hit_len.return_value = 0
|
|
req.finished.return_value = False
|
|
req.needs_host_load_back.return_value = False
|
|
return req
|
|
|
|
def create_adder(self, running_batch, **kwargs):
|
|
defaults = dict(
|
|
page_size=1,
|
|
tree_cache=self.mock_tree_cache,
|
|
token_to_kv_pool_allocator=self.mock_token_allocator,
|
|
running_batch=running_batch,
|
|
new_token_ratio=1.0,
|
|
rem_input_tokens=10000,
|
|
rem_chunk_tokens=None,
|
|
num_mixed_decode_tokens=0,
|
|
priority_scheduling_preemption_threshold=0,
|
|
)
|
|
defaults.update(kwargs)
|
|
defaults["token_to_kv_pool_allocator"].page_size = defaults["page_size"]
|
|
return PrefillAdder(**defaults)
|
|
|
|
def create_shared_adder(self, *, num_mixed_decode_tokens=0):
|
|
self.mock_tree_cache.supports_mamba.return_value = False
|
|
self.mock_tree_cache.sliding_window_size = 8
|
|
self.mock_tree_cache.is_tree_cache.return_value = False
|
|
allocator = init_unified_swa_pools(
|
|
device="cpu",
|
|
kv_cache_dtype=torch.float16,
|
|
head_num=1,
|
|
head_dim=4,
|
|
v_head_dim=4,
|
|
swa_head_num=1,
|
|
swa_head_dim=4,
|
|
swa_v_head_dim=4,
|
|
page_size=4,
|
|
start_layer=0,
|
|
end_layer=2,
|
|
swa_attention_layer_ids=[1],
|
|
full_attention_layer_ids=[0],
|
|
total_bytes=1024,
|
|
enable_memory_saver=False,
|
|
need_sort=False,
|
|
lazy_compaction=True,
|
|
).token_to_kv_pool_allocator
|
|
return self.create_adder(
|
|
self.create_running_batch(),
|
|
page_size=4,
|
|
rem_chunk_tokens=16,
|
|
num_mixed_decode_tokens=num_mixed_decode_tokens,
|
|
token_to_kv_pool_allocator=allocator,
|
|
)
|
|
|
|
def create_shared_req(self, rid, max_new_tokens=4):
|
|
req = self.create_mock_req(rid, priority=0, max_new_tokens=max_new_tokens)
|
|
req.sampling_params.ignore_eos = False
|
|
req.swa_host_hit_length = 0
|
|
req.last_node = MagicMock()
|
|
req.full_untruncated_fill_ids = list(range(12))
|
|
req.set_extend_range = MagicMock(
|
|
side_effect=lambda start, end: setattr(
|
|
req, "extend_range", Range(start, end)
|
|
)
|
|
)
|
|
return req
|
|
|
|
def create_shortest_prefill_adder(self, *, chunk_tokens=4096):
|
|
override = get_context().override_server_args(
|
|
schedule_policy="shortest-prefill-first"
|
|
)
|
|
override.install()
|
|
self.addCleanup(override.restore)
|
|
self.mock_tree_cache.supports_mamba.return_value = False
|
|
self.mock_tree_cache.is_tree_cache.return_value = False
|
|
self.mock_token_allocator.available_size.return_value = 32768
|
|
return self.create_adder(
|
|
self.create_running_batch(), page_size=256, rem_chunk_tokens=chunk_tokens
|
|
)
|
|
|
|
def test_shortest_prefill_reserves_space_for_complete_waiting_requests(self):
|
|
adder = self.create_shortest_prefill_adder()
|
|
policy = SchedulePolicy(
|
|
policy="shortest-prefill-first",
|
|
tree_cache=RadixCache.create_simulated(),
|
|
enable_hierarchical_cache=True,
|
|
enable_priority_scheduling=False,
|
|
schedule_low_priority_values_first=False,
|
|
)
|
|
continuation = self.create_shared_req("continuation")
|
|
continuation.full_untruncated_fill_ids = list(range(16384))
|
|
waiting = [self.create_shared_req("a"), self.create_shared_req("b")]
|
|
for req, length in zip(waiting, [512, 1024]):
|
|
req.origin_input_ids = list(range(length))
|
|
req.full_untruncated_fill_ids = list(range(length))
|
|
req.num_matched_prefix_tokens = 0
|
|
adder.chunked_req_limit = policy.shortest_prefill_chunk_limit(
|
|
continuation, waiting, adder.rem_chunk_tokens, adder.page_size
|
|
)
|
|
self.assertIs(adder.add_chunked_req(continuation), continuation)
|
|
self.assertEqual(continuation.extend_range.length, 2560)
|
|
for req in waiting:
|
|
adder.add_one_req(req, has_chunked_req=True, truncation_align_size=None)
|
|
self.assertEqual(adder.can_run_list, [continuation, *waiting])
|
|
self.assertIsNone(adder.new_chunked_req)
|
|
self.assertEqual(adder.rem_chunk_tokens, 0)
|
|
self.assertGreaterEqual(adder.rem_total_tokens, 0)
|
|
|
|
def test_shortest_prefill_rejects_second_unfinished_chunk(self):
|
|
adder = self.create_shortest_prefill_adder(chunk_tokens=512)
|
|
req = self.create_shared_req("second-chunk")
|
|
req.full_untruncated_fill_ids = list(range(1024))
|
|
self.assertEqual(
|
|
adder.add_one_req(req, has_chunked_req=True, truncation_align_size=None),
|
|
AddReqResult.OTHER,
|
|
)
|
|
self.assertEqual(adder.can_run_list, [])
|
|
self.assertIsNone(adder.new_chunked_req)
|
|
req.set_extend_range.assert_not_called()
|
|
self.mock_tree_cache.init_load_back.assert_not_called()
|
|
|
|
def test_shortest_prefill_rechecks_chunk_limit_after_host_miss(self):
|
|
adder = self.create_shortest_prefill_adder(chunk_tokens=512)
|
|
req = self.create_shared_req("host-miss")
|
|
req.full_untruncated_fill_ids = list(range(1024))
|
|
req.prefix_indices = torch.empty(0, dtype=torch.int64)
|
|
req.host_hit_length = 768
|
|
req.best_match_node = req.last_node
|
|
req.needs_host_load_back.return_value = True
|
|
self.mock_tree_cache.init_load_back.return_value = (
|
|
torch.empty(0, dtype=torch.int64),
|
|
req.last_node,
|
|
)
|
|
self.assertEqual(
|
|
adder.add_one_req(req, has_chunked_req=True, truncation_align_size=None),
|
|
AddReqResult.OTHER,
|
|
)
|
|
self.mock_tree_cache.init_load_back.assert_called_once()
|
|
self.assertEqual(adder.can_run_list, [])
|
|
req.set_extend_range.assert_not_called()
|
|
|
|
def test_shortest_prefill_preserves_memory_admission(self):
|
|
adder = self.create_shortest_prefill_adder()
|
|
self.mock_token_allocator.available_size.return_value = 256
|
|
req = self.create_shared_req("no-memory")
|
|
req.full_untruncated_fill_ids = list(range(512))
|
|
self.assertEqual(
|
|
adder.add_one_req(req, has_chunked_req=True, truncation_align_size=None),
|
|
AddReqResult.NO_TOKEN,
|
|
)
|
|
self.assertEqual(adder.can_run_list, [])
|
|
|
|
def test_continuation_without_limit_keeps_normal_chunk_size(self):
|
|
adder = self.create_shortest_prefill_adder()
|
|
req = self.create_shared_req("continuation")
|
|
req.full_untruncated_fill_ids = list(range(8192))
|
|
self.assertIs(adder.add_chunked_req(req), req)
|
|
self.assertEqual(req.extend_range.length, 4096)
|
|
|
|
def test_shared_admission_reserves_all_pending_requests(self):
|
|
adder = self.create_shared_adder()
|
|
first, second = (
|
|
self.create_shared_req("first"),
|
|
self.create_shared_req("second"),
|
|
)
|
|
adder.add_one_req(first, has_chunked_req=False, truncation_align_size=None)
|
|
self.assertEqual(adder.can_run_list, [first])
|
|
self.assertEqual(
|
|
adder.add_one_req(
|
|
second, has_chunked_req=False, truncation_align_size=None
|
|
),
|
|
AddReqResult.NO_TOKEN,
|
|
)
|
|
self.assertEqual(adder.can_run_list, [first])
|
|
|
|
def test_shared_admission_rechecks_after_prefix_lock(self):
|
|
adder = self.create_shared_adder()
|
|
self.assertIsNotNone(adder.token_to_kv_pool_allocator.alloc(24))
|
|
self.mock_tree_cache.full_evictable_size.return_value = 24
|
|
self.mock_tree_cache.swa_evictable_size.return_value = 24
|
|
|
|
def lock_prefix(_):
|
|
self.mock_tree_cache.full_evictable_size.return_value = 0
|
|
self.mock_tree_cache.swa_evictable_size.return_value = 0
|
|
return IncLockRefResult()
|
|
|
|
self.mock_tree_cache.inc_lock_ref.side_effect = lock_prefix
|
|
req = self.create_shared_req("locked-prefix")
|
|
self.assertEqual(
|
|
adder.add_one_req(req, has_chunked_req=False, truncation_align_size=None),
|
|
AddReqResult.NO_TOKEN,
|
|
)
|
|
self.mock_tree_cache.inc_lock_ref.assert_called_once()
|
|
self.assertEqual(adder.can_run_list, [])
|
|
|
|
def test_shared_continuation_defers_when_decode_consumes_chunk_budget(self):
|
|
"""Mixed decode must not commit an empty or negatively sliced prompt."""
|
|
for decode_tokens in (16, 28):
|
|
with self.subTest(decode_tokens=decode_tokens):
|
|
adder = self.create_shared_adder(num_mixed_decode_tokens=decode_tokens)
|
|
req = self.create_shared_req("continuation")
|
|
before = (
|
|
adder.memory_budget.total_offset,
|
|
adder.memory_budget.swa_offset,
|
|
)
|
|
self.assertIs(adder.add_chunked_req(req), req)
|
|
self.assertEqual(adder.can_run_list, [])
|
|
req.set_extend_range.assert_not_called()
|
|
self.assertEqual(
|
|
(adder.memory_budget.total_offset, adder.memory_budget.swa_offset),
|
|
before,
|
|
)
|
|
|
|
def test_shared_continuation_uses_memory_chunk_limit(self):
|
|
adder = self.create_shared_adder()
|
|
req = self.create_shared_req("continuation", max_new_tokens=80)
|
|
self.assertIs(adder.add_chunked_req(req), req)
|
|
self.assertEqual(req.extend_range.length, 8)
|
|
self.assertEqual(adder.memory_budget.total_offset, 12)
|
|
self.assertEqual(adder.memory_budget.swa_offset, 12)
|
|
|
|
def test_storage_prefetch_fulfillment_resolves_at_admission(self):
|
|
adder = self.create_adder(self.create_running_batch())
|
|
req = self.create_mock_req("storage-hit", priority=0, max_new_tokens=1)
|
|
req.host_hit_length = 12
|
|
req.host_loaded_length = 4
|
|
req.storage_hit_length = 8
|
|
req.storage_hit_start = 4
|
|
req.materialized_host_hit_len.return_value = 4
|
|
req.fulfilled_storage_hit_len.return_value = 8
|
|
req.needs_host_load_back.return_value = True
|
|
|
|
adder._account_prefill_cache_admission(req, prefix_len=12)
|
|
|
|
self.mock_tree_cache.finish_storage_prefetch_admission.assert_called_once_with(
|
|
req.cache_request_handle,
|
|
fulfilled_tokens=8,
|
|
reason=None,
|
|
)
|
|
self.assertEqual(adder.log_device_hit_tokens, 4)
|
|
self.assertEqual(adder.log_host_hit_tokens, 0)
|
|
self.assertEqual(adder.log_storage_hit_tokens, 8)
|
|
|
|
self.mock_tree_cache.finish_storage_prefetch_admission.reset_mock()
|
|
req.host_loaded_length = 0
|
|
req.materialized_host_hit_len.return_value = 0
|
|
req.fulfilled_storage_hit_len.return_value = 0
|
|
adder._account_prefill_cache_admission(req, prefix_len=0)
|
|
self.mock_tree_cache.finish_storage_prefetch_admission.assert_called_once_with(
|
|
req.cache_request_handle, fulfilled_tokens=0, reason="device_capacity"
|
|
)
|
|
|
|
self.mock_tree_cache.finish_storage_prefetch_admission.reset_mock()
|
|
req.host_hit_length = 4
|
|
req.host_loaded_length = 4
|
|
req.storage_hit_length = 8
|
|
req.storage_hit_start = 4
|
|
req.materialized_host_hit_len.return_value = 4
|
|
req.fulfilled_storage_hit_len.return_value = 4
|
|
req.needs_host_load_back.return_value = True
|
|
adder._account_prefill_cache_admission(req, prefix_len=8)
|
|
self.mock_tree_cache.finish_storage_prefetch_admission.assert_called_once_with(
|
|
req.cache_request_handle,
|
|
fulfilled_tokens=4,
|
|
reason="cache_admission_shortfall",
|
|
)
|
|
self.assertEqual(adder.log_device_hit_tokens, 8)
|
|
self.assertEqual(adder.log_host_hit_tokens, 0)
|
|
self.assertEqual(adder.log_storage_hit_tokens, 12)
|
|
|
|
def test_retracted_storage_prefetch_accounting_is_omitted(self):
|
|
adder = self.create_adder(self.create_running_batch())
|
|
req = self.create_mock_req(
|
|
"retracted-storage-hit", priority=0, max_new_tokens=1
|
|
)
|
|
req.retracted_stain = True
|
|
|
|
adder._account_prefill_cache_admission(req, prefix_len=8)
|
|
|
|
self.mock_tree_cache.discard_storage_prefetch_accounting.assert_called_once_with(
|
|
req.cache_request_handle
|
|
)
|
|
self.mock_tree_cache.finish_storage_prefetch_admission.assert_not_called()
|
|
|
|
def test_preempt_success_high_priority_values_first(self):
|
|
params = [
|
|
("run1", 0, 50),
|
|
("run2", 1, 75),
|
|
("run3", 2, 100),
|
|
]
|
|
running_reqs = [
|
|
self.create_mock_req(rid, priority, max_new_tokens)
|
|
for rid, priority, max_new_tokens in params
|
|
]
|
|
self.scheduling_order(schedule_low_priority_values_first=False)
|
|
running_batch = self.create_running_batch(running_reqs)
|
|
adder = self.create_adder(running_batch)
|
|
|
|
self.assertEqual(adder.memory_budget.total_offset, 225)
|
|
|
|
self.mock_token_allocator.full_available_size.return_value = (
|
|
225 # full occupation of GRam
|
|
)
|
|
self.mock_token_allocator.available_size.return_value = 225
|
|
|
|
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=49)
|
|
|
|
success = adder.preempt_to_schedule(new_req)
|
|
|
|
self.assertTrue(success)
|
|
self.assertIn(running_reqs[0], adder.preempt_list)
|
|
self.assertEqual(
|
|
adder.memory_budget.total_offset, 175
|
|
) # 50 + 75 + 100 - 50 = 175
|
|
running_batch.release_req.assert_called_once()
|
|
|
|
def test_preempt_success_low_priority_values_first(self):
|
|
params = [
|
|
("run1", 0, 50),
|
|
("run2", 1, 75),
|
|
("run3", 2, 100),
|
|
]
|
|
running_reqs = [
|
|
self.create_mock_req(rid, priority, max_new_tokens)
|
|
for rid, priority, max_new_tokens in params
|
|
]
|
|
self.scheduling_order(schedule_low_priority_values_first=True)
|
|
running_batch = self.create_running_batch(running_reqs)
|
|
adder = self.create_adder(running_batch)
|
|
|
|
self.assertEqual(adder.memory_budget.total_offset, 225)
|
|
|
|
self.mock_token_allocator.full_available_size.return_value = (
|
|
225 # full occupation of GRam
|
|
)
|
|
self.mock_token_allocator.available_size.return_value = 225
|
|
|
|
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=49)
|
|
|
|
success = adder.preempt_to_schedule(new_req)
|
|
|
|
self.assertTrue(success)
|
|
self.assertIn(running_reqs[2], adder.preempt_list)
|
|
self.assertEqual(
|
|
adder.memory_budget.total_offset, 125
|
|
) # 50 + 75 + 100 - 100 = 125
|
|
running_batch.release_req.assert_called_once()
|
|
|
|
def test_preempt_fail_low_priority_values_first(self):
|
|
params = [
|
|
("run1", 0, 50),
|
|
("run2", 1, 75),
|
|
("run3", 2, 100),
|
|
]
|
|
running_reqs = [
|
|
self.create_mock_req(rid, priority, max_new_tokens)
|
|
for rid, priority, max_new_tokens in params
|
|
]
|
|
self.scheduling_order(schedule_low_priority_values_first=True)
|
|
running_batch = self.create_running_batch(running_reqs)
|
|
adder = self.create_adder(running_batch)
|
|
|
|
self.assertEqual(adder.memory_budget.total_offset, 225)
|
|
|
|
self.mock_token_allocator.full_available_size.return_value = (
|
|
225 # full occupation of GRam
|
|
)
|
|
self.mock_token_allocator.available_size.return_value = 225
|
|
|
|
new_req_fail_by_priority_check = self.create_mock_req(
|
|
"new1", priority=2, max_new_tokens=49
|
|
)
|
|
|
|
success_by_priority_check = adder.preempt_to_schedule(
|
|
new_req_fail_by_priority_check
|
|
)
|
|
self.assertFalse(success_by_priority_check)
|
|
|
|
new_req_fail_by_priority_check = self.create_mock_req(
|
|
"new2", priority=1, max_new_tokens=110
|
|
)
|
|
success_by_capacity_check = adder.preempt_to_schedule(
|
|
new_req_fail_by_priority_check
|
|
)
|
|
self.assertFalse(success_by_capacity_check)
|
|
|
|
def test_preempt_fail_high_priority_values_first(self):
|
|
params = [
|
|
("run1", 0, 50),
|
|
("run2", 1, 75),
|
|
("run3", 2, 100),
|
|
]
|
|
running_reqs = [
|
|
self.create_mock_req(rid, priority, max_new_tokens)
|
|
for rid, priority, max_new_tokens in params
|
|
]
|
|
self.scheduling_order(schedule_low_priority_values_first=False)
|
|
running_batch = self.create_running_batch(running_reqs)
|
|
adder = self.create_adder(running_batch)
|
|
|
|
self.assertEqual(adder.memory_budget.total_offset, 225)
|
|
|
|
self.mock_token_allocator.full_available_size.return_value = (
|
|
225 # full occupation of GRam
|
|
)
|
|
self.mock_token_allocator.available_size.return_value = 225
|
|
|
|
new_req_fail_by_priority_check = self.create_mock_req(
|
|
"new1", priority=0, max_new_tokens=49
|
|
)
|
|
|
|
success_by_priority_check = adder.preempt_to_schedule(
|
|
new_req_fail_by_priority_check
|
|
)
|
|
self.assertFalse(success_by_priority_check)
|
|
|
|
new_req_fail_by_priority_check = self.create_mock_req(
|
|
"new2", priority=-1, max_new_tokens=110
|
|
)
|
|
success_by_capacity_check = adder.preempt_to_schedule(
|
|
new_req_fail_by_priority_check
|
|
)
|
|
self.assertFalse(success_by_capacity_check)
|
|
|
|
def test_preempt_skip_already_preempted_request(self):
|
|
params = [
|
|
("req_prio_0", 0, 50),
|
|
("req_prio_1", 1, 75),
|
|
("req_prio_2", 2, 100),
|
|
]
|
|
running_reqs = [
|
|
self.create_mock_req(rid, priority, max_new_tokens)
|
|
for rid, priority, max_new_tokens in params
|
|
]
|
|
self.scheduling_order(schedule_low_priority_values_first=False)
|
|
running_batch = self.create_running_batch(running_reqs)
|
|
adder = self.create_adder(running_batch)
|
|
|
|
self.assertEqual(adder.memory_budget.total_offset, 225)
|
|
|
|
self.mock_token_allocator.full_available_size.return_value = 225
|
|
self.mock_token_allocator.available_size.return_value = 225
|
|
|
|
# New request preempts req_prio_0
|
|
first_req = self.create_mock_req(
|
|
"new_req_prio_1", priority=1, max_new_tokens=49
|
|
)
|
|
first_success = adder.preempt_to_schedule(first_req)
|
|
self.assertTrue(first_success)
|
|
self.assertIn(running_reqs[0], adder.preempt_list)
|
|
self.assertEqual(adder.memory_budget.total_offset, 175)
|
|
running_batch.release_req.assert_called_once()
|
|
|
|
# Second call needs more tokens than currently free, so it would need to
|
|
# preempt req_prio_0 again if already-preempted requests were not filtered out.
|
|
second_req = self.create_mock_req(
|
|
"second_new_req_prio_1", priority=1, max_new_tokens=76
|
|
)
|
|
second_success = adder.preempt_to_schedule(second_req)
|
|
|
|
self.assertFalse(second_success)
|
|
self.assertEqual(adder.memory_budget.total_offset, 175)
|
|
self.assertEqual(adder.preempt_list.count(running_reqs[0]), 1)
|
|
running_batch.release_req.assert_called_once()
|
|
|
|
def test_preempt_success_low_priority_values_first_exact_once(self):
|
|
params = [
|
|
("run1", 0, 50),
|
|
("run2", 1, 75),
|
|
("run3", 2, 100),
|
|
("run4", 2, 125),
|
|
("run4", 2, 125),
|
|
]
|
|
running_reqs = [
|
|
self.create_mock_req(rid, priority, max_new_tokens)
|
|
for rid, priority, max_new_tokens in params
|
|
]
|
|
self.scheduling_order(schedule_low_priority_values_first=True)
|
|
running_batch = self.create_running_batch(running_reqs)
|
|
adder = self.create_adder(running_batch)
|
|
|
|
self.assertEqual(adder.memory_budget.total_offset, 475)
|
|
|
|
self.mock_token_allocator.full_available_size.return_value = (
|
|
475 # full occupation of GRam
|
|
)
|
|
self.mock_token_allocator.available_size.return_value = 475
|
|
|
|
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=75)
|
|
|
|
success = adder.preempt_to_schedule(new_req)
|
|
self.assertTrue(success)
|
|
self.assertIn(running_reqs[2], adder.preempt_list)
|
|
self.assertEqual(
|
|
adder.memory_budget.total_offset, 375
|
|
) # 50 + 75 + 100 + 125 + 125 - 100 = 375
|
|
running_batch.release_req.assert_called_once()
|
|
|
|
def test_preempt_success_low_priority_values_first_exact_twice(self):
|
|
params = [
|
|
("run1", 0, 50),
|
|
("run2", 1, 75),
|
|
("run3", 2, 100),
|
|
("run4", 2, 125),
|
|
("run4", 2, 125),
|
|
]
|
|
running_reqs = [
|
|
self.create_mock_req(rid, priority, max_new_tokens)
|
|
for rid, priority, max_new_tokens in params
|
|
]
|
|
self.scheduling_order(schedule_low_priority_values_first=True)
|
|
running_batch = self.create_running_batch(running_reqs)
|
|
adder = self.create_adder(running_batch)
|
|
|
|
self.assertEqual(adder.memory_budget.total_offset, 475)
|
|
|
|
self.mock_token_allocator.full_available_size.return_value = (
|
|
475 # full occupation of GRam
|
|
)
|
|
self.mock_token_allocator.available_size.return_value = 475
|
|
|
|
new_req = self.create_mock_req("new1", priority=1, max_new_tokens=200)
|
|
|
|
success = adder.preempt_to_schedule(new_req)
|
|
self.assertTrue(success)
|
|
self.assertIn(running_reqs[2], adder.preempt_list)
|
|
self.assertIn(running_reqs[3], adder.preempt_list)
|
|
self.assertEqual(
|
|
adder.memory_budget.total_offset, 250
|
|
) # 50 + 75 + 100 + 125 + 125 - 100 - 125 = 250
|
|
self.assertEqual(running_batch.release_req.call_count, 2)
|
|
|
|
def test_mixed_chunk_prefill_budgets(self):
|
|
self.mock_token_allocator.available_size.return_value = 1000
|
|
|
|
decode_reqs = [
|
|
self.create_mock_req(f"decode_{i}", priority=0, max_new_tokens=50)
|
|
for i in range(8)
|
|
]
|
|
running_batch = self.create_running_batch(decode_reqs)
|
|
|
|
adder = self.create_adder(
|
|
running_batch,
|
|
rem_input_tokens=200,
|
|
rem_chunk_tokens=64,
|
|
num_mixed_decode_tokens=len(decode_reqs),
|
|
)
|
|
|
|
self.assertEqual(adder.rem_input_tokens, 192) # 200 - 8
|
|
self.assertEqual(adder.rem_chunk_tokens, 56) # 64 - 8
|
|
self.assertEqual(adder.memory_budget.total_offset, 408) # 8 + 8 * 50
|
|
self.assertEqual(adder.memory_budget.current_offset, 8)
|
|
self.assertEqual(adder.budget_state(), AddReqResult.CONTINUE)
|
|
|
|
# Add a prefill that exactly consumes the chunk budget
|
|
req1 = self.create_mock_req("req1", priority=0, max_new_tokens=64)
|
|
req1.host_hit_length = 0
|
|
req1.prefix_indices = []
|
|
req1.full_untruncated_fill_ids = list(range(56))
|
|
req1.last_node = MagicMock()
|
|
req1.sampling_params.ignore_eos = False
|
|
# add_one_req reads req.extend_range.length after set_extend_range;
|
|
# emulate the real Req writer (a spec=Req mock lacks the attribute).
|
|
req1.set_extend_range = MagicMock(
|
|
side_effect=lambda start, end: setattr(
|
|
req1, "extend_range", Range(start, end)
|
|
)
|
|
)
|
|
|
|
result1 = adder.add_one_req(
|
|
req1, has_chunked_req=False, truncation_align_size=None
|
|
)
|
|
|
|
self.assertEqual(len(adder.can_run_list), 1)
|
|
self.assertEqual(adder.rem_chunk_tokens, 0) # 56 - 56
|
|
self.assertEqual(adder.rem_input_tokens, 136) # 192 - 56
|
|
self.assertEqual(result1, AddReqResult.OTHER)
|
|
|
|
# 3 decode requests finished
|
|
remaining_decode_reqs = decode_reqs[3:]
|
|
running_batch2 = self.create_running_batch(remaining_decode_reqs)
|
|
|
|
adder2 = self.create_adder(
|
|
running_batch2,
|
|
rem_input_tokens=200,
|
|
rem_chunk_tokens=64,
|
|
num_mixed_decode_tokens=len(remaining_decode_reqs),
|
|
)
|
|
|
|
self.assertEqual(adder2.rem_input_tokens, 195) # 200 - 5
|
|
self.assertEqual(adder2.rem_chunk_tokens, 59) # 64 - 5
|
|
self.assertEqual(adder2.memory_budget.total_offset, 255) # 5 + 5 * 50
|
|
self.assertEqual(adder2.budget_state(), AddReqResult.CONTINUE)
|
|
|
|
# Same prefill no longer exhausts the chunk budget
|
|
req2 = self.create_mock_req("req2", priority=0, max_new_tokens=64)
|
|
req2.host_hit_length = 0
|
|
req2.prefix_indices = []
|
|
req2.full_untruncated_fill_ids = list(range(56))
|
|
req2.last_node = MagicMock()
|
|
req2.sampling_params.ignore_eos = False
|
|
req2.set_extend_range = MagicMock(
|
|
side_effect=lambda start, end: setattr(
|
|
req2, "extend_range", Range(start, end)
|
|
)
|
|
)
|
|
|
|
result2 = adder2.add_one_req(
|
|
req2, has_chunked_req=False, truncation_align_size=None
|
|
)
|
|
|
|
self.assertEqual(len(adder2.can_run_list), 1)
|
|
self.assertEqual(adder2.rem_chunk_tokens, 3) # 59 - 56 = 3 remaining
|
|
self.assertEqual(result2, AddReqResult.CONTINUE)
|
|
|
|
# Fit last small prefill request
|
|
req3 = self.create_mock_req("req3", priority=0, max_new_tokens=16)
|
|
req3.host_hit_length = 0
|
|
req3.prefix_indices = []
|
|
req3.full_untruncated_fill_ids = list(range(3))
|
|
req3.last_node = MagicMock()
|
|
req3.sampling_params.ignore_eos = False
|
|
req3.set_extend_range = MagicMock(
|
|
side_effect=lambda start, end: setattr(
|
|
req3, "extend_range", Range(start, end)
|
|
)
|
|
)
|
|
|
|
result3 = adder2.add_one_req(
|
|
req3, has_chunked_req=False, truncation_align_size=None
|
|
)
|
|
|
|
self.assertEqual(len(adder2.can_run_list), 2)
|
|
self.assertEqual(adder2.rem_chunk_tokens, 0) # 3 - 3 = 0
|
|
self.assertEqual(result3, AddReqResult.OTHER)
|
|
|
|
@patch(
|
|
"sglang.srt.managers.schedule_policy.page_interleave_shard_size",
|
|
return_value=4,
|
|
)
|
|
def test_ignore_eos_reserves_all_shard_class_pages(self, _shard_size):
|
|
"""Disabled-radix admission must charge one page per shard class."""
|
|
self.mock_tree_cache.disable = True
|
|
self.mock_token_allocator.page_size = 16
|
|
self.mock_token_allocator.shard_spec = SimpleNamespace(
|
|
max_prefix_tokens=1024, chunk_tokens=1024
|
|
)
|
|
self.mock_token_allocator.available_size.return_value = 64
|
|
adder = self.create_adder(self.create_running_batch(), page_size=16)
|
|
|
|
req = self.create_mock_req("ignore_eos", priority=0, max_new_tokens=1)
|
|
req.sampling_params.ignore_eos = True
|
|
req.origin_input_ids = list(range(16))
|
|
req.full_untruncated_fill_ids = list(range(16))
|
|
req.last_node = MagicMock()
|
|
req.set_extend_range = MagicMock(
|
|
side_effect=lambda start, end: setattr(
|
|
req, "extend_range", Range(start, end)
|
|
)
|
|
)
|
|
|
|
result = adder.add_one_req(
|
|
req, has_chunked_req=False, truncation_align_size=None
|
|
)
|
|
|
|
self.assertEqual(result, AddReqResult.NO_TOKEN)
|
|
self.assertEqual(adder.can_run_list, [])
|
|
req.set_extend_range.assert_not_called()
|
|
|
|
def create_sharded_adder(
|
|
self,
|
|
*,
|
|
chunk_tokens=16,
|
|
max_prefix_tokens=16,
|
|
shard_spec=None,
|
|
allocator_size=1024,
|
|
**kwargs,
|
|
):
|
|
self.mock_tree_cache.supports_mamba.return_value = False
|
|
self.mock_tree_cache.is_tree_cache.return_value = False
|
|
if shard_spec is None:
|
|
shard_spec = PageShardSpec(
|
|
shard_rank=0,
|
|
shard_size=2,
|
|
page_size=4,
|
|
max_prefix_tokens=max_prefix_tokens,
|
|
chunk_tokens=chunk_tokens,
|
|
)
|
|
allocator = PageInterleavePoolAllocator(
|
|
size=allocator_size,
|
|
physical_page_size=shard_spec.page_size,
|
|
shard_size=shard_spec.shard_size,
|
|
dtype=torch.int64,
|
|
device="cpu",
|
|
kvcache=None,
|
|
need_sort=False,
|
|
shard_spec=shard_spec,
|
|
)
|
|
return self.create_adder(
|
|
self.create_running_batch(),
|
|
page_size=shard_spec.page_size,
|
|
rem_chunk_tokens=shard_spec.chunk_tokens,
|
|
token_to_kv_pool_allocator=allocator,
|
|
**kwargs,
|
|
)
|
|
|
|
def create_sharded_req(self, rid, *, prefix_len=12, extend_len=4):
|
|
req = self.create_shared_req(rid, max_new_tokens=1)
|
|
req.prefix_indices = list(range(prefix_len))
|
|
req.full_untruncated_fill_ids = list(range(prefix_len + extend_len))
|
|
return req
|
|
|
|
def test_sharded_admission_defers_prefix_scratch_overflow(self):
|
|
adder = self.create_sharded_adder()
|
|
first = self.create_sharded_req("first")
|
|
second = self.create_sharded_req("second")
|
|
self.assertEqual(
|
|
adder.add_one_req(first, has_chunked_req=False, truncation_align_size=None),
|
|
AddReqResult.CONTINUE,
|
|
)
|
|
remaining = (adder.rem_total_tokens, adder.rem_chunk_tokens)
|
|
|
|
# Both requests fit KV capacity, but prefix scratch fits only one.
|
|
self.assertEqual(
|
|
adder.add_one_req(
|
|
second, has_chunked_req=False, truncation_align_size=None
|
|
),
|
|
AddReqResult.OTHER,
|
|
)
|
|
self.assertEqual(adder.can_run_list, [first])
|
|
second.set_extend_range.assert_not_called()
|
|
self.assertEqual((adder.rem_total_tokens, adder.rem_chunk_tokens), remaining)
|
|
self.assertEqual(adder.kv_shard_block_bound_pages, 2)
|
|
self.assertEqual(adder.kv_shard_chunk_pages, 1)
|
|
|
|
# Rejection leaves the remaining chunk scratch usable.
|
|
third = self.create_sharded_req("no-prefix", prefix_len=0)
|
|
self.assertEqual(
|
|
adder.add_one_req(third, has_chunked_req=False, truncation_align_size=None),
|
|
AddReqResult.CONTINUE,
|
|
)
|
|
self.assertEqual(adder.can_run_list, [first, third])
|
|
self.assertEqual(adder.kv_shard_block_bound_pages, 2)
|
|
self.assertEqual(adder.kv_shard_chunk_pages, 2)
|
|
|
|
def test_eight_long_prefixes_fit_an_eight_k_chunk(self):
|
|
context_len = 16384
|
|
extend_len = 1024
|
|
prefix_len = context_len - extend_len
|
|
kvc = SimpleNamespace(
|
|
page_size=4,
|
|
model_config=SimpleNamespace(context_len=context_len),
|
|
)
|
|
for chunk_tokens in (8192, 16384):
|
|
with self.subTest(chunk_tokens=chunk_tokens):
|
|
with (
|
|
patch(
|
|
"sglang.srt.mem_cache.page_interleave.get_kv_shard_group_info",
|
|
return_value=(0, 2),
|
|
),
|
|
patch(
|
|
"sglang.srt.mem_cache.page_interleave.get_schedule",
|
|
return_value=SimpleNamespace(chunked_prefill_size=chunk_tokens),
|
|
),
|
|
):
|
|
spec = make_page_shard_spec(kvc)
|
|
self.assertEqual(spec.max_prefix_tokens, 8 * context_len)
|
|
adder = self.create_sharded_adder(
|
|
shard_spec=spec,
|
|
allocator_size=8 * context_len,
|
|
rem_input_tokens=32768,
|
|
)
|
|
reqs = [
|
|
self.create_sharded_req(
|
|
str(i), prefix_len=prefix_len, extend_len=extend_len
|
|
)
|
|
for i in range(9)
|
|
]
|
|
for i, req in enumerate(reqs[:8]):
|
|
result = adder.add_one_req(
|
|
req, has_chunked_req=False, truncation_align_size=None
|
|
)
|
|
# The eighth request commits even if it exhausts the budget.
|
|
expected = (
|
|
AddReqResult.OTHER
|
|
if i == 7 and chunk_tokens == 8192
|
|
else AddReqResult.CONTINUE
|
|
)
|
|
self.assertEqual(result, expected)
|
|
self.assertEqual(adder.can_run_list, reqs[:8])
|
|
self.assertEqual(adder.rem_chunk_tokens, chunk_tokens - 8192)
|
|
self.assertEqual(adder.kv_shard_chunk_pages * spec.page_size, 8192)
|
|
self.assertEqual(
|
|
adder.kv_shard_block_bound_pages * spec.logical_page_size,
|
|
8 * prefix_len,
|
|
)
|
|
remaining = (
|
|
adder.rem_total_tokens,
|
|
adder.rem_chunk_tokens,
|
|
adder.kv_shard_block_bound_pages,
|
|
adder.kv_shard_chunk_pages,
|
|
)
|
|
self.assertEqual(
|
|
adder.add_one_req(
|
|
reqs[8], has_chunked_req=False, truncation_align_size=None
|
|
),
|
|
AddReqResult.OTHER,
|
|
)
|
|
self.assertEqual(adder.can_run_list, reqs[:8])
|
|
reqs[8].set_extend_range.assert_not_called()
|
|
self.assertEqual(
|
|
(
|
|
adder.rem_total_tokens,
|
|
adder.rem_chunk_tokens,
|
|
adder.kv_shard_block_bound_pages,
|
|
adder.kv_shard_chunk_pages,
|
|
),
|
|
remaining,
|
|
)
|
|
if chunk_tokens == 16384:
|
|
# Prefix scratch is full; a request without a prefix still fits.
|
|
no_prefix = self.create_sharded_req(
|
|
"no-prefix", prefix_len=0, extend_len=extend_len
|
|
)
|
|
self.assertEqual(
|
|
adder.add_one_req(
|
|
no_prefix,
|
|
has_chunked_req=False,
|
|
truncation_align_size=None,
|
|
),
|
|
AddReqResult.CONTINUE,
|
|
)
|
|
self.assertEqual(adder.can_run_list, reqs[:8] + [no_prefix])
|
|
|
|
def test_sharded_admission_reserves_only_selected_chunk(self):
|
|
# Use normal page rounding without a sharding alignment override.
|
|
for remaining, selected, verdict in (
|
|
(8, 8, AddReqResult.OTHER),
|
|
(7, 4, AddReqResult.CONTINUE),
|
|
):
|
|
with self.subTest(remaining=remaining):
|
|
adder = self.create_sharded_adder(chunk_tokens=8)
|
|
adder.exact_chunk_fill = False
|
|
adder.rem_chunk_tokens = remaining
|
|
req = self.create_sharded_req("chunked", extend_len=28)
|
|
|
|
self.assertEqual(
|
|
adder.add_one_req(
|
|
req, has_chunked_req=False, truncation_align_size=None
|
|
),
|
|
verdict,
|
|
)
|
|
self.assertEqual(adder.can_run_list, [req])
|
|
self.assertIs(adder.new_chunked_req, req)
|
|
self.assertEqual(req.extend_range, Range(12, 12 + selected))
|
|
self.assertEqual(adder.kv_shard_block_bound_pages, 2)
|
|
self.assertEqual(adder.kv_shard_chunk_pages, selected // 4)
|
|
|
|
def test_sharded_admission_reserves_nothing_when_delayed(self):
|
|
delayer = _RecordingDelayer(allow=False)
|
|
adder = self.create_sharded_adder(prefill_delayer_single_pass=delayer)
|
|
req = self.create_sharded_req("delayed")
|
|
|
|
self.assertEqual(
|
|
adder.add_one_req(req, has_chunked_req=False, truncation_align_size=None),
|
|
AddReqResult.OTHER,
|
|
)
|
|
self.assertEqual(adder.can_run_list, [])
|
|
req.set_extend_range.assert_not_called()
|
|
self.assertEqual(adder.kv_shard_block_bound_pages, 0)
|
|
self.assertEqual(adder.kv_shard_chunk_pages, 0)
|
|
|
|
delayer.allow = True
|
|
self.assertEqual(
|
|
adder.add_one_req(req, has_chunked_req=False, truncation_align_size=None),
|
|
AddReqResult.CONTINUE,
|
|
)
|
|
self.assertEqual(adder.can_run_list, [req])
|
|
self.assertEqual(adder.kv_shard_block_bound_pages, 2)
|
|
self.assertEqual(adder.kv_shard_chunk_pages, 1)
|
|
|
|
def _build_hybrid_swa_chunked_req(
|
|
self,
|
|
*,
|
|
page_size,
|
|
rem_swa,
|
|
rem_chunk=2048,
|
|
extend_input_len=500,
|
|
is_hybrid_swa=True,
|
|
full_available=100_000,
|
|
):
|
|
self.mock_token_allocator.swa_available_size.return_value = rem_swa
|
|
self.mock_token_allocator.full_available_size.return_value = full_available
|
|
self.mock_token_allocator.available_size.return_value = full_available
|
|
self.mock_tree_cache.sliding_window_size = 128
|
|
adder = self.create_adder(
|
|
self.create_running_batch(),
|
|
page_size=page_size,
|
|
rem_chunk_tokens=rem_chunk,
|
|
)
|
|
adder.is_hybrid_swa = is_hybrid_swa
|
|
if is_hybrid_swa:
|
|
adder.memory_budget = SWAPrefillBudget(
|
|
self.mock_token_allocator, self.mock_tree_cache
|
|
)
|
|
|
|
req = self.create_mock_req("chunked", priority=0, max_new_tokens=128)
|
|
req.prefix_indices = []
|
|
req.full_untruncated_fill_ids = list(range(extend_input_len))
|
|
# set_extend_range is the only writer of extend_range; the production
|
|
# path reads req.extend_range.length right after calling it, so the mock
|
|
# must actually set the attribute (a spec=Req mock has the method but
|
|
# not the instance attribute).
|
|
req.set_extend_range = MagicMock(
|
|
side_effect=lambda start, end: setattr(
|
|
req, "extend_range", Range(start, end)
|
|
)
|
|
)
|
|
return adder, req
|
|
|
|
def test_add_chunked_req_hybrid_swa_reserves_page_for_alloc_extend(self):
|
|
# alloc_extend needs extend_num_tokens + page_size per request. If the
|
|
# scheduler hands out all of rem_swa_tokens, alloc_extend cannot get its
|
|
# extra page and OOMs. With the fix, extend_input_len must cap at
|
|
# rem_swa_tokens - page_size so the page is reserved.
|
|
PAGE_SIZE = 64
|
|
REM_SWA = 100
|
|
adder, req = self._build_hybrid_swa_chunked_req(
|
|
page_size=PAGE_SIZE, rem_swa=REM_SWA
|
|
)
|
|
|
|
result = adder.add_chunked_req(req)
|
|
|
|
self.assertIs(result, req) # truncated → chunked prefill continues
|
|
req.set_extend_range.assert_called_once()
|
|
start, end = req.set_extend_range.call_args.args
|
|
new_len = end - start
|
|
self.assertLessEqual(new_len + PAGE_SIZE, REM_SWA)
|
|
self.assertEqual(new_len, REM_SWA - PAGE_SIZE)
|
|
|
|
def test_add_chunked_req_hybrid_swa_defers_when_swa_below_page(self):
|
|
# When rem_swa_tokens <= page_size there is no room to serve even the
|
|
# reservation, so the chunked req must be deferred (returned unchanged)
|
|
# instead of falling back to rem_chunk_tokens and bypassing SWA budget.
|
|
PAGE_SIZE = 64
|
|
adder, req = self._build_hybrid_swa_chunked_req(
|
|
page_size=PAGE_SIZE, rem_swa=PAGE_SIZE
|
|
)
|
|
|
|
result = adder.add_chunked_req(req)
|
|
|
|
self.assertIs(result, req)
|
|
req.set_extend_range.assert_not_called()
|
|
self.assertEqual(len(adder.can_run_list), 0)
|
|
|
|
def test_swa_budget_for_req(self):
|
|
# budget = max(alloc - window, 0) + min(extend + max_new, window) + page,
|
|
# where alloc = min(extend, rem_chunk). The decode headroom is the SWA the
|
|
# request adds to its own sliding window (extend + decode), capped at the
|
|
# window -- a cached prefix funds the rest -- rather than a constant
|
|
# window, which over-charged short requests into an admission livelock.
|
|
cases = [
|
|
# (extend, max_new, rem_chunk, window, page, expected, label)
|
|
(64, 512, None, 128, 16, 128 + 16, "long_decode_hits_window_floor"),
|
|
(64, 32, None, 128, 16, 96 + 16, "short_req_reserves_below_window"),
|
|
(10, 20, None, 512, 8, 30 + 8, "short_resume_tiny_budget"),
|
|
(300, 512, None, 128, 16, 300 + 16, "big_extend_over_window"),
|
|
(200, 512, 50, 64, 8, 64 + 8, "chunk_capped_alloc_hits_floor"),
|
|
# Multi-chunk: alloc = rem_chunk (1024) caps a huge extend, and the
|
|
# chunk itself exceeds the window, so term1 = alloc - window is driven
|
|
# by the chunk, not the full extend -> budget = chunk + page.
|
|
(2000, 256, 1024, 512, 16, 1024 + 16, "multichunk_alloc_over_window"),
|
|
(0, 512, None, 128, 16, 128 + 16, "extend_zero_long_decode"),
|
|
(0, 40, None, 128, 16, 40 + 16, "extend_zero_short_decode"),
|
|
]
|
|
for extend, max_new, rem_chunk, window, page, expected, label in cases:
|
|
with self.subTest(label=label):
|
|
self.mock_tree_cache.sliding_window_size = window
|
|
adder = self.create_adder(
|
|
self.create_running_batch(),
|
|
page_size=page,
|
|
rem_chunk_tokens=rem_chunk,
|
|
)
|
|
self.assertEqual(
|
|
estimate_swa_kv_tokens(
|
|
extend,
|
|
max_new,
|
|
sliding_window_size=window,
|
|
page_size=page,
|
|
allocation_limit=rem_chunk,
|
|
),
|
|
expected,
|
|
)
|
|
|
|
def test_swa_admission_admits_short_cached_resume_at_two_window_pool(self):
|
|
# Livelock regression (real incident). At an SWA pool ~= 2 sliding
|
|
# windows, a cached-prefix resume matches >= 1 window (locked, excluded
|
|
# from rem_swa) and has only a short uncached tail + a little decode
|
|
# left. The pre-fix constant-window reservation charged a second full
|
|
# window, so the admission gate (swa_needed >= rem_swa_tokens) rejected
|
|
# it every scheduler iteration while LPM kept it at the queue head --
|
|
# 100% scheduler CPU, idle GPU. Capping the reservation at
|
|
# min(extend + decode, window) admits it.
|
|
WINDOW, PAGE, REM_SWA = 128, 8, 100
|
|
PREFIX, EXTEND = 200, 16 # cached prefix > window; short uncached tail
|
|
self.mock_token_allocator.swa_available_size.return_value = REM_SWA
|
|
self.mock_token_allocator.full_available_size.return_value = 100_000
|
|
self.mock_token_allocator.available_size.return_value = 100_000
|
|
self.mock_tree_cache.sliding_window_size = WINDOW
|
|
self.mock_tree_cache.is_tree_cache.return_value = False
|
|
adder = self.create_adder(self.create_running_batch(), page_size=PAGE)
|
|
adder.is_hybrid_swa = True
|
|
adder.memory_budget = SWAPrefillBudget(
|
|
self.mock_token_allocator, self.mock_tree_cache
|
|
)
|
|
|
|
req = self.create_mock_req(
|
|
"resume", priority=0, max_new_tokens=40, output_len=10
|
|
)
|
|
req.prefix_indices = list(range(PREFIX))
|
|
req.full_untruncated_fill_ids = list(range(PREFIX + EXTEND))
|
|
req.host_hit_length = 0
|
|
req.swa_host_hit_length = 0
|
|
req.last_node = MagicMock()
|
|
req.set_extend_range = MagicMock(
|
|
side_effect=lambda start, end: setattr(
|
|
req, "extend_range", Range(start, end)
|
|
)
|
|
)
|
|
req.sampling_params = SimpleNamespace(max_new_tokens=40, ignore_eos=False)
|
|
|
|
# Pre-fix: a constant sliding-window reservation rejects the resume.
|
|
with patch.object(
|
|
adder.memory_budget, "swa_tokens", return_value=WINDOW + PAGE
|
|
):
|
|
self.assertIs(
|
|
adder.add_one_req(
|
|
req, has_chunked_req=False, truncation_align_size=None
|
|
),
|
|
AddReqResult.NO_TOKEN,
|
|
)
|
|
self.assertEqual(len(adder.can_run_list), 0)
|
|
|
|
# Fix: min(extend + decode, window) reservation admits it.
|
|
adder.add_one_req(req, has_chunked_req=False, truncation_align_size=None)
|
|
self.assertIn(req, adder.can_run_list)
|
|
|
|
def test_load_back_delivery_mismatch_reselects_the_prefill_shape(self):
|
|
# Two incidents: a load that delivers nothing left the SWA gate sized
|
|
# for the tail and the allocator OOMed; a cache-mode load that also
|
|
# surfaces FULL device tokens behind a host-only SWA window tripped a
|
|
# strict promised==loaded check and crashed the scheduler.
|
|
WINDOW, PAGE = 128, 8
|
|
SPAN, HOST_HIT = 1024, 1016
|
|
self.mock_token_allocator.swa_available_size.return_value = 400
|
|
self.mock_token_allocator.full_available_size.return_value = 100_000
|
|
self.mock_token_allocator.available_size.return_value = 100_000
|
|
self.mock_tree_cache.sliding_window_size = WINDOW
|
|
self.mock_tree_cache.is_tree_cache.return_value = False
|
|
|
|
def run(delivered: int, remaining_after_load: int = 100_000):
|
|
self.mock_token_allocator.full_available_size.return_value = 100_000
|
|
self.mock_token_allocator.swa_available_size.return_value = 400
|
|
adder = self.create_adder(self.create_running_batch(), page_size=PAGE)
|
|
adder.is_hybrid_swa = True
|
|
adder.memory_budget = SWAPrefillBudget(
|
|
self.mock_token_allocator, self.mock_tree_cache
|
|
)
|
|
req = self.create_mock_req("dropped-fetch", priority=0, max_new_tokens=8)
|
|
req.prefix_indices = torch.empty(0, dtype=torch.int64)
|
|
req.full_untruncated_fill_ids = list(range(SPAN))
|
|
req.host_hit_length = HOST_HIT
|
|
req.swa_host_hit_length = WINDOW
|
|
req.needs_host_load_back.return_value = True
|
|
req.last_node = MagicMock()
|
|
req.best_match_node = MagicMock()
|
|
req.kv = SimpleNamespace(cache_protected_len=0)
|
|
|
|
def set_extend_range(start, end):
|
|
req.extend_range = Range(start, end)
|
|
|
|
req.set_extend_range = MagicMock(side_effect=set_extend_range)
|
|
req.sampling_params = SimpleNamespace(max_new_tokens=8, ignore_eos=False)
|
|
|
|
def load_back(params):
|
|
self.mock_token_allocator.full_available_size.return_value = (
|
|
remaining_after_load
|
|
)
|
|
if remaining_after_load == 0:
|
|
self.mock_token_allocator.swa_available_size.return_value = 0
|
|
return torch.arange(delivered, dtype=torch.int64), req.last_node
|
|
|
|
self.mock_tree_cache.init_load_back.side_effect = load_back
|
|
verdict = adder.add_one_req(
|
|
req, has_chunked_req=False, truncation_align_size=None
|
|
)
|
|
return verdict, list(adder.can_run_list), req
|
|
|
|
# Promise kept: only the 8-token tail is prefilled, which fits.
|
|
_, admitted, _ = run(HOST_HIT)
|
|
self.assertEqual(len(admitted), 1)
|
|
# Nothing delivered: the whole span is prefilled and no longer fits, so
|
|
# admission must decline rather than OOM the pool.
|
|
verdict, admitted, _ = run(0)
|
|
self.assertIs(verdict, AddReqResult.NO_TOKEN)
|
|
self.assertEqual(admitted, [])
|
|
# Over-delivery: admitted with the loaded prefix, not the promise.
|
|
# The loaded prefix is now pinned and no longer part of the evictable
|
|
# budget. A successful load must not run admission gates again.
|
|
_, admitted, req = run(HOST_HIT + 4, remaining_after_load=0)
|
|
self.assertEqual(len(admitted), 1)
|
|
self.assertEqual(len(req.prefix_indices), HOST_HIT + 4)
|
|
self.assertEqual(req.kv.cache_protected_len, HOST_HIT + 4)
|
|
req.set_extend_range.assert_called_once_with(HOST_HIT + 4, SPAN)
|
|
# A partial FULL load stays fatal.
|
|
with self.assertRaisesRegex(RuntimeError, "promised"):
|
|
run(HOST_HIT // 2)
|
|
|
|
def _create_host_hit_req(self, *, prefix_len=0, host_hit=8192, tail=1024):
|
|
req = self._create_delayer_req(prefix_len + host_hit + tail)
|
|
req.prefix_indices = torch.arange(prefix_len)
|
|
req.host_hit_length = host_hit
|
|
req.needs_host_load_back.return_value = True
|
|
req.best_match_node = req.last_node
|
|
req.kv = SimpleNamespace(cache_protected_len=prefix_len)
|
|
return req
|
|
|
|
def test_successful_load_back_commits_the_selected_shape_once(self):
|
|
cases = (
|
|
("full", 0, 24, None, None, 8, 8),
|
|
("full_unaligned", 0, 24, None, None, 7, 8),
|
|
("retracted_unaligned", 0, 24, None, None, 7, 8),
|
|
("chunk", 0, 24, 4, None, 4, 0),
|
|
("aux_only", 24, 0, None, None, 8, 8),
|
|
("overdelivery_full", 0, 24, None, None, 8, 8),
|
|
("overdelivery_chunk", 0, 24, 4, None, 4, 0),
|
|
("overdelivery_chunk_end", 0, 24, 8, None, 8, 8),
|
|
(
|
|
"dllm",
|
|
0,
|
|
24,
|
|
None,
|
|
SimpleNamespace(
|
|
block_size=4,
|
|
max_running_requests=2,
|
|
requires_separate_context_encoding=False,
|
|
),
|
|
4,
|
|
0,
|
|
),
|
|
(
|
|
"overdelivery_dllm",
|
|
0,
|
|
24,
|
|
None,
|
|
SimpleNamespace(
|
|
block_size=4,
|
|
max_running_requests=2,
|
|
requires_separate_context_encoding=False,
|
|
),
|
|
4,
|
|
0,
|
|
),
|
|
)
|
|
for name, prefix_len, host_hit, chunk, dllm, extend, decode in cases:
|
|
with self.subTest(mode=name):
|
|
self.mock_tree_cache.reset_mock()
|
|
adder = self._create_delayer_adder(
|
|
available_tokens=100_000,
|
|
delayer=None,
|
|
page_size=2,
|
|
rem_chunk_tokens=chunk,
|
|
dllm_config=dllm,
|
|
)
|
|
req = self._create_host_hit_req(
|
|
prefix_len=prefix_len,
|
|
host_hit=host_hit,
|
|
tail=extend if chunk is None and dllm is None else 8,
|
|
)
|
|
req.retracted_stain = name == "retracted_unaligned"
|
|
if name.startswith("overdelivery"):
|
|
req.host_hit_length -= 4
|
|
old_node, restored_node = req.last_node, object()
|
|
if name == "aux_only":
|
|
req.swa_host_hit_length = 8
|
|
|
|
def load_back(params):
|
|
self.assertIs(params.req, req)
|
|
tile_gate.assert_called_once()
|
|
tile_gate.return_value = AddReqResult.OTHER
|
|
return torch.arange(host_hit), restored_node
|
|
|
|
self.mock_tree_cache.init_load_back.side_effect = load_back
|
|
with patch.object(
|
|
adder, "_check_prefill_tile_budget", return_value=None
|
|
) as tile_gate:
|
|
adder.add_one_req(req, False, None)
|
|
tile_gate.assert_called_once()
|
|
self.mock_tree_cache.init_load_back.assert_called_once()
|
|
self.assertEqual(adder.can_run_list, [req])
|
|
req.set_extend_range.assert_called_once_with(24, 24 + extend)
|
|
self.mock_tree_cache.inc_lock_ref.assert_any_call(restored_node)
|
|
self.assertIs(
|
|
self.mock_tree_cache.dec_lock_ref.call_args.args[0], old_node
|
|
)
|
|
self.assertEqual(adder.log_hit_tokens, 24)
|
|
self.assertEqual(adder.log_input_tokens, extend)
|
|
self.assertEqual(
|
|
adder.reprocessed_log_input_tokens,
|
|
extend if req.retracted_stain else 0,
|
|
)
|
|
self.assertEqual(
|
|
adder.memory_budget.total_offset,
|
|
adder.ceil_paged_tokens(extend) + decode + 2,
|
|
)
|
|
self.assertEqual(
|
|
adder.new_chunked_req is req,
|
|
name in ("chunk", "overdelivery_chunk"),
|
|
)
|
|
self.mock_tree_cache.init_load_back.side_effect = None
|
|
|
|
def test_swa_new_tokens_clamps_remaining_not_total(self):
|
|
# Remaining decode headroom must be min(max_new - generated, CLIP)
|
|
# (subtract-then-clip). The reversed order (clip-then-subtract) zeroes
|
|
# out a request that has already generated >= CLIP tokens but still has a
|
|
# long decode ahead, under-reserving its SWA window -> OOM risk on resume
|
|
# of a long-generation request.
|
|
from sglang.srt.managers.schedule_policy import CLIP_MAX_NEW_TOKENS as CLIP
|
|
|
|
adder = self.create_adder(self.create_running_batch())
|
|
cases = [
|
|
# (max_new, generated, expected, label)
|
|
(100, 10, 90, "below_clip_normal"),
|
|
(40, 100, 0, "already_finished"),
|
|
# clip-then-subtract would give 3996; subtract-then-clip caps at CLIP.
|
|
(CLIP + 6000, 100, CLIP, "long_gen_small_output_caps_at_clip"),
|
|
# clip-then-subtract would give 0; the long decode still needs CLIP.
|
|
(CLIP + 6000, CLIP + 100, CLIP, "long_gen_output_over_clip"),
|
|
]
|
|
for max_new, generated, expected, label in cases:
|
|
with self.subTest(label=label):
|
|
req = self.create_mock_req(
|
|
label, priority=0, max_new_tokens=max_new, output_len=generated
|
|
)
|
|
self.assertEqual(adder._swa_new_tokens(req), expected)
|
|
|
|
def test_delayer_not_consulted_when_kv_budget_rejects(self):
|
|
"""A rank whose first candidate fails the KV-budget gate must NOT
|
|
report local_prefillable=True: add_one_req returns NO_TOKEN before
|
|
negotiating, and finalize() later reports the rank as not
|
|
prefillable. Regression guard: the negotiate used to run at the top
|
|
of add_one_req, so under KV pressure a full rank still claimed
|
|
prefillable=True, the delayer saw "all prefillable" and allowed, the
|
|
full ranks then NO_TOKEN'ed out, and the DP-synced forward mixed
|
|
prefill and decode — the exact pattern the delayer exists to
|
|
prevent."""
|
|
delayer = _RecordingDelayer(allow=True)
|
|
adder = self._create_delayer_adder(available_tokens=10, delayer=delayer)
|
|
|
|
result = adder.add_one_req(
|
|
self._create_delayer_req(50),
|
|
has_chunked_req=False,
|
|
truncation_align_size=None,
|
|
)
|
|
|
|
self.assertEqual(result, AddReqResult.NO_TOKEN)
|
|
self.assertEqual(delayer.calls, [])
|
|
self.assertEqual(adder.can_run_list, [])
|
|
|
|
def test_delayer_not_consulted_when_post_lock_recheck_rejects(self):
|
|
"""Locking the request's own prefix converts evictable tokens into
|
|
protected ones, so a request can pass the pre-lock KV gate yet fail
|
|
the post-lock recheck — precisely the high-utilization regime the
|
|
delayer targets. The negotiate must sit after that recheck too, or
|
|
the rank claims prefillable=True and then runs decode."""
|
|
delayer = _RecordingDelayer(allow=True)
|
|
adder = self._create_delayer_adder(available_tokens=10, delayer=delayer)
|
|
# Pre-lock budget is covered by evictable tokens; inc_lock_ref pins
|
|
# them (evictable -> protected), shrinking the budget below demand.
|
|
self.mock_tree_cache.evictable_size.return_value = 1000
|
|
self.mock_tree_cache.full_evictable_size.return_value = 1000
|
|
|
|
def _pin_prefix(node):
|
|
self.mock_tree_cache.evictable_size.return_value = 0
|
|
self.mock_tree_cache.full_evictable_size.return_value = 0
|
|
return IncLockRefResult()
|
|
|
|
self.mock_tree_cache.inc_lock_ref.side_effect = _pin_prefix
|
|
|
|
result = adder.add_one_req(
|
|
self._create_delayer_req(50),
|
|
has_chunked_req=False,
|
|
truncation_align_size=None,
|
|
)
|
|
|
|
self.assertEqual(result, AddReqResult.NO_TOKEN)
|
|
self.assertEqual(delayer.calls, [])
|
|
self.assertEqual(adder.can_run_list, [])
|
|
|
|
def test_delay_verdict_blocks_admissible_request(self):
|
|
"""An admissible request must still be gated by the (relocated)
|
|
negotiate: on a delay verdict it is not admitted, and the rank
|
|
reported prefillable=True exactly once."""
|
|
delayer = _RecordingDelayer(allow=False)
|
|
adder = self._create_delayer_adder(available_tokens=100_000, delayer=delayer)
|
|
|
|
result = adder.add_one_req(
|
|
self._create_delayer_req(50),
|
|
has_chunked_req=False,
|
|
truncation_align_size=None,
|
|
)
|
|
|
|
self.assertEqual(result, AddReqResult.OTHER)
|
|
self.assertEqual(delayer.calls, [True])
|
|
self.assertEqual(adder.can_run_list, [])
|
|
|
|
def test_allow_verdict_admits_request(self):
|
|
"""On an allow verdict the request proceeds through admission — the
|
|
relocated negotiate must not block the commit path."""
|
|
delayer = _RecordingDelayer(allow=True)
|
|
adder = self._create_delayer_adder(available_tokens=100_000, delayer=delayer)
|
|
req = self._create_delayer_req(50)
|
|
|
|
result = adder.add_one_req(
|
|
req, has_chunked_req=False, truncation_align_size=None
|
|
)
|
|
|
|
self.assertEqual(result, AddReqResult.CONTINUE)
|
|
self.assertEqual(delayer.calls, [True])
|
|
self.assertIn(req, adder.can_run_list)
|
|
|
|
def test_chunked_req_negotiates_prefillable_and_proceeds(self):
|
|
"""A rank resuming a chunked prefill runs it this pass regardless of
|
|
the verdict, so add_chunked_req must report prefillable=True (else a
|
|
rank with an empty waiting queue reports False via finalize() and
|
|
peers delay while it prefills alone) and must not drop the chunk on
|
|
a delay verdict (that would leak memory)."""
|
|
delayer = _RecordingDelayer(allow=False)
|
|
adder = self._create_delayer_adder(
|
|
available_tokens=100_000, delayer=delayer, rem_chunk_tokens=500
|
|
)
|
|
req = self._create_delayer_req(200)
|
|
|
|
result = adder.add_chunked_req(req)
|
|
|
|
self.assertIsNone(result) # chunk fully admitted, not truncated
|
|
self.assertEqual(delayer.calls, [True])
|
|
self.assertIn(req, adder.can_run_list)
|
|
|
|
def _create_delayer_adder(self, *, available_tokens, delayer, **kwargs):
|
|
self.mock_token_allocator.available_size.return_value = available_tokens
|
|
self.mock_token_allocator.full_available_size.return_value = available_tokens
|
|
return self.create_adder(
|
|
self.create_running_batch(),
|
|
prefill_delayer_single_pass=delayer,
|
|
**kwargs,
|
|
)
|
|
|
|
def _create_delayer_req(self, num_tokens: int):
|
|
req = self.create_mock_req("delayer_req", priority=0, max_new_tokens=8)
|
|
req.full_untruncated_fill_ids = list(range(num_tokens))
|
|
req.host_hit_length = 0
|
|
req.last_node = MagicMock()
|
|
req.sampling_params.ignore_eos = False
|
|
req.set_extend_range = MagicMock(
|
|
side_effect=lambda start, end: setattr(
|
|
req, "extend_range", Range(start, end)
|
|
)
|
|
)
|
|
return req
|
|
|
|
def test_add_chunked_req_non_hybrid_no_swa_reservation(self):
|
|
# Non-hybrid path: the SWA-pool reservation must NOT apply, otherwise
|
|
# the fix would regress non-SWA models.
|
|
PAGE_SIZE = 16
|
|
adder, req = self._build_hybrid_swa_chunked_req(
|
|
page_size=PAGE_SIZE,
|
|
rem_swa=10,
|
|
rem_chunk=500,
|
|
extend_input_len=200,
|
|
is_hybrid_swa=False,
|
|
full_available=300,
|
|
)
|
|
|
|
result = adder.add_chunked_req(req)
|
|
self.assertIsNone(result)
|
|
req.set_extend_range.assert_called_once_with(0, 200)
|
|
self.assertIn(req, adder.can_run_list)
|
|
|
|
def _adder_with_extend_lens(self, extend_lens):
|
|
adder = PrefillAdder.__new__(PrefillAdder)
|
|
adder.can_run_list = [
|
|
SimpleNamespace(extend_input_len=length) for length in extend_lens
|
|
]
|
|
# BLOCK_M is auto-detected from the attention backend in production; the
|
|
# __new__ helper bypasses __init__, so set it explicitly. 64 matches the
|
|
# block_m the tile-count assertions below are computed against.
|
|
adder.prefill_tile_block_m = 64
|
|
return adder
|
|
|
|
def test_estimate_prefill_extend_tile_metrics(self):
|
|
metrics = estimate_prefill_extend_tile_metrics([1, 7, 13, 129], block_m=64)
|
|
|
|
self.assertEqual(metrics["q_tiles_per_request"], [1, 1, 1, 3])
|
|
self.assertEqual(metrics["legacy_q_tiles_per_head"], 12)
|
|
self.assertEqual(metrics["compact_q_tiles_per_head"], 6)
|
|
self.assertEqual(metrics["saved_q_tiles_per_head"], 6)
|
|
self.assertEqual(metrics["saved_q_tile_ratio"], 0.5)
|
|
|
|
def test_compact_prefill_tile_budget_admits_more_than_legacy(self):
|
|
adder = self._adder_with_extend_lens([1, 7, 13])
|
|
|
|
# The tile-budget admission is gated on HIP in production; force the gate
|
|
# on so this vendor-neutral admission-math check runs on any CI runner.
|
|
with (
|
|
patch.object(schedule_policy, "_IS_HIP", True),
|
|
patch.object(schedule_policy, "PREFILL_TILE_BUDGET", 6),
|
|
patch.object(schedule_policy, "PREFILL_TILE_BUDGET_MODE", "compact"),
|
|
):
|
|
self.assertIsNone(adder._check_prefill_tile_budget(129))
|
|
|
|
with (
|
|
patch.object(schedule_policy, "_IS_HIP", True),
|
|
patch.object(schedule_policy, "PREFILL_TILE_BUDGET", 6),
|
|
patch.object(schedule_policy, "PREFILL_TILE_BUDGET_MODE", "legacy"),
|
|
):
|
|
self.assertEqual(adder._check_prefill_tile_budget(129), AddReqResult.OTHER)
|
|
|
|
def test_prefill_tile_budget_always_allows_first_request(self):
|
|
adder = self._adder_with_extend_lens([])
|
|
|
|
with (
|
|
patch.object(schedule_policy, "_IS_HIP", True),
|
|
patch.object(schedule_policy, "PREFILL_TILE_BUDGET", 1),
|
|
):
|
|
self.assertIsNone(adder._check_prefill_tile_budget(4096))
|
|
|
|
def test_prefill_tile_budget_disabled_on_non_hip(self):
|
|
# AMD-only: on non-HIP vendors the tile-budget admission must be a no-op
|
|
# even when the budget env is set, so scheduler behavior is unchanged.
|
|
adder = self._adder_with_extend_lens([1, 7, 13])
|
|
|
|
with (
|
|
patch.object(schedule_policy, "_IS_HIP", False),
|
|
patch.object(schedule_policy, "PREFILL_TILE_BUDGET", 6),
|
|
patch.object(schedule_policy, "PREFILL_TILE_BUDGET_MODE", "legacy"),
|
|
):
|
|
self.assertIsNone(adder._check_prefill_tile_budget(129))
|
|
|
|
# ---- _swa_req_never_fits: the gate on the _swa_chunk_cap escape hatch ----
|
|
# The hatch shrinks a chunk and admits it instead of waiting; it must fire
|
|
# ONLY for a request whose SWA budget can never fit the drained pool (true
|
|
# head-of-line livelock). Admitting transient-pressure requests instead
|
|
# collapses the SWA evictable cushion and causes a retraction storm.
|
|
|
|
def create_swa_adder(
|
|
self, *, size_swa: int, sliding_window: int, page_size: int = 16
|
|
) -> PrefillAdder:
|
|
self.mock_tree_cache.sliding_window_size = sliding_window
|
|
self.mock_token_allocator = self.create_token_allocator(size_swa=size_swa)
|
|
self.mock_token_allocator.create_prefill_budget.side_effect = (
|
|
lambda tree_cache, **kwargs: SWAPrefillBudget(
|
|
self.mock_token_allocator, tree_cache, **kwargs
|
|
)
|
|
)
|
|
return self.create_adder(
|
|
self.create_running_batch(),
|
|
page_size=page_size,
|
|
rem_chunk_tokens=512,
|
|
)
|
|
|
|
def test_swa_never_fits_false_under_transient_pressure(self):
|
|
# Small request against an ample pool: budget << pool, so it would fit
|
|
# once running decodes drain -> must wait, not take the hatch.
|
|
adder = self.create_swa_adder(size_swa=1024, sliding_window=128)
|
|
self.assertFalse(
|
|
adder.memory_budget.swa_never_fits(extend_input_len=256, max_new_tokens=64)
|
|
)
|
|
|
|
def test_swa_never_fits_true_when_budget_exceeds_whole_pool(self):
|
|
# A large host-hit load-back charge pushes the budget past the entire
|
|
# pool: it can never fit however far the pool drains -> hatch.
|
|
adder = self.create_swa_adder(size_swa=1024, sliding_window=128)
|
|
self.assertTrue(
|
|
adder.memory_budget.swa_never_fits(
|
|
extend_input_len=256, max_new_tokens=64, swa_host_hit_length=4096
|
|
)
|
|
)
|
|
|
|
def test_swa_never_fits_is_gated_by_pool_capacity(self):
|
|
# Same request; only the pool size changes. Proves the check compares
|
|
# the budget against size_swa (guards against a wrong-accessor bug).
|
|
req = dict(extend_input_len=256, max_new_tokens=64, swa_host_hit_length=600)
|
|
self.assertTrue(
|
|
self.create_swa_adder(
|
|
size_swa=512, sliding_window=128
|
|
).memory_budget.swa_never_fits(**req)
|
|
)
|
|
self.assertFalse(
|
|
self.create_swa_adder(
|
|
size_swa=4096, sliding_window=128
|
|
).memory_budget.swa_never_fits(**req)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|