Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
924 lines
37 KiB
Python
924 lines
37 KiB
Python
import json
|
|
import sys
|
|
import threading
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import numpy as np
|
|
import torch
|
|
|
|
from sglang.srt.disaggregation.base import KVPoll # noqa: E402
|
|
from sglang.srt.disaggregation.decode import ( # noqa: E402
|
|
DecodePreallocQueue,
|
|
SchedulerDisaggregationDecodeMixin,
|
|
)
|
|
from sglang.srt.disaggregation.prefill import ( # noqa: E402
|
|
PrefillBootstrapQueue,
|
|
SchedulerDisaggregationPrefillMixin,
|
|
)
|
|
from sglang.srt.disaggregation.utils import DisaggregationMode # noqa: E402
|
|
from sglang.srt.environ import envs # noqa: E402
|
|
from sglang.srt.lora.lora_manager import LoRAManager # noqa: E402
|
|
from sglang.srt.managers.io_struct import AbortReq # noqa: E402
|
|
from sglang.srt.managers.schedule_batch import ( # noqa: E402
|
|
FINISH_ABORT,
|
|
Req,
|
|
ReqKvInfo,
|
|
)
|
|
from sglang.srt.managers.scheduler import Scheduler # noqa: E402
|
|
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestOutcome
|
|
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache # noqa: E402
|
|
from sglang.srt.runtime_context import get_context, publish, reset_context # noqa: E402
|
|
from sglang.srt.server_args import ServerArgs
|
|
from sglang.test.ci.ci_register import register_cpu_ci
|
|
from sglang.test.separate_buffer_allocator_double import (
|
|
bind_separate_buffer_capacity,
|
|
)
|
|
|
|
register_cpu_ci(est_time=12, suite="base-a-test-cpu")
|
|
|
|
|
|
class TestDisaggregationPriorityQueueing(unittest.TestCase):
|
|
def setUp(self):
|
|
# The code under test reads its config from the bags.
|
|
reset_context()
|
|
self.addCleanup(reset_context)
|
|
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
|
|
|
def _new_scheduler(self, disaggregation_mode: DisaggregationMode) -> Scheduler:
|
|
scheduler = Scheduler.__new__(Scheduler)
|
|
scheduler.disaggregation_mode = disaggregation_mode
|
|
scheduler.enable_priority_scheduling = True
|
|
scheduler.schedule_low_priority_values_first = False
|
|
scheduler.abort_on_priority_when_disabled = False
|
|
scheduler.enable_hierarchical_cache = False
|
|
scheduler.enable_hicache_storage = False
|
|
scheduler.enable_unified_cache_external_linker = False
|
|
scheduler.processed_tokens_counter = 0
|
|
scheduler.waiting_queue = []
|
|
scheduler._prefetch_kvcache = MagicMock()
|
|
scheduler.tree_cache = MagicMock()
|
|
scheduler._abort_on_queued_limit = MagicMock(return_value=False)
|
|
scheduler.model_config = SimpleNamespace(num_key_value_heads=8)
|
|
scheduler.disagg_prefill_bootstrap_queue = MagicMock()
|
|
scheduler.disagg_decode_prealloc_queue = MagicMock()
|
|
scheduler.ipc_channels = MagicMock()
|
|
return scheduler
|
|
|
|
def _new_req(self, priority=None):
|
|
req = MagicMock()
|
|
req.priority = priority
|
|
req.rid = "req"
|
|
req.output_ids = []
|
|
req.weight_version_events = []
|
|
req.time_stats = MagicMock()
|
|
req.time_stats.trace_ctx = MagicMock()
|
|
return req
|
|
|
|
def test_prefill_mode_assigns_default_priority_before_bootstrap_queue(self):
|
|
scheduler = self._new_scheduler(DisaggregationMode.PREFILL)
|
|
req = self._new_req(priority=None)
|
|
|
|
scheduler._add_request_to_queue(req)
|
|
|
|
self.assertEqual(req.priority, -sys.maxsize - 1)
|
|
scheduler.disagg_prefill_bootstrap_queue.add.assert_called_once_with(req, 8)
|
|
req.time_stats.set_prefill_bootstrap_queue_entry_time.assert_called_once()
|
|
|
|
def test_decode_mode_assigns_default_priority_before_prealloc_queue(self):
|
|
scheduler = self._new_scheduler(DisaggregationMode.DECODE)
|
|
req = self._new_req(priority=None)
|
|
|
|
scheduler._add_request_to_queue(req)
|
|
|
|
self.assertEqual(req.priority, -sys.maxsize - 1)
|
|
scheduler.disagg_decode_prealloc_queue.add.assert_called_once_with(
|
|
req, is_retracted=False
|
|
)
|
|
req.time_stats.set_decode_prealloc_queue_entry_time.assert_called_once()
|
|
|
|
def test_priority_disabled_abort_validation_applies_to_decode_mode(self):
|
|
scheduler = self._new_scheduler(DisaggregationMode.DECODE)
|
|
scheduler.enable_priority_scheduling = False
|
|
scheduler.abort_on_priority_when_disabled = True
|
|
req = self._new_req(priority=10)
|
|
|
|
with patch(
|
|
"sglang.srt.managers.scheduler.get_serving",
|
|
return_value=SimpleNamespace(weight_version="v0"),
|
|
):
|
|
scheduler._add_request_to_queue(req)
|
|
|
|
scheduler.disagg_decode_prealloc_queue.add.assert_not_called()
|
|
scheduler.ipc_channels.send_to_tokenizer.send_output.assert_called_once()
|
|
req.time_stats.trace_ctx.abort.assert_called_once()
|
|
|
|
def test_priority_rejection_releases_early_prefetch(self):
|
|
for mode in DisaggregationMode:
|
|
with self.subTest(mode=mode):
|
|
scheduler = self._new_scheduler(mode)
|
|
scheduler.enable_priority_scheduling = False
|
|
scheduler.abort_on_priority_when_disabled = True
|
|
scheduler.enable_hicache_storage = True
|
|
scheduler.tree_cache = MagicMock(spec=["finish"])
|
|
req = self._new_req(priority=10)
|
|
|
|
with patch(
|
|
"sglang.srt.managers.scheduler.get_serving",
|
|
return_value=SimpleNamespace(weight_version="v0"),
|
|
):
|
|
scheduler._add_request_to_queue(req)
|
|
|
|
scheduler.tree_cache.finish.assert_called_once_with(
|
|
req.cache_request_handle, CacheRequestOutcome.ABORT
|
|
)
|
|
scheduler._prefetch_kvcache.assert_not_called()
|
|
self.assertEqual(scheduler.waiting_queue, [])
|
|
scheduler.disagg_prefill_bootstrap_queue.add.assert_not_called()
|
|
scheduler.disagg_decode_prealloc_queue.add.assert_not_called()
|
|
|
|
def test_full_queue_releases_only_the_rejected_request(self):
|
|
for priority in (0, 2):
|
|
with self.subTest(incoming_priority=priority):
|
|
scheduler = self._new_scheduler(DisaggregationMode.NULL)
|
|
del scheduler._abort_on_queued_limit # Exercise the real queue check.
|
|
scheduler.max_queued_requests = 1
|
|
scheduler.enable_hicache_storage = True
|
|
scheduler.tree_cache = MagicMock(spec=["finish"])
|
|
scheduler.beam_coordinator = MagicMock()
|
|
queued = self._new_req(priority=1)
|
|
queued.rid = "queued"
|
|
scheduler.waiting_queue = [queued]
|
|
incoming = self._new_req(priority=priority)
|
|
|
|
with patch(
|
|
"sglang.srt.managers.scheduler.get_serving",
|
|
return_value=SimpleNamespace(weight_version="v0"),
|
|
):
|
|
scheduler._add_request_to_queue(incoming)
|
|
|
|
rejected = incoming if priority == 0 else queued
|
|
scheduler.tree_cache.finish.assert_called_once_with(
|
|
rejected.cache_request_handle, CacheRequestOutcome.ABORT
|
|
)
|
|
if priority == 0:
|
|
self.assertEqual(scheduler.waiting_queue, [queued])
|
|
scheduler._prefetch_kvcache.assert_not_called()
|
|
else:
|
|
self.assertEqual(scheduler.waiting_queue, [incoming])
|
|
scheduler._prefetch_kvcache.assert_called_once_with(incoming)
|
|
|
|
|
|
class TestOptimisticPrefillCacheOwnership(unittest.TestCase):
|
|
def _new_unpublished_swa_state(self):
|
|
cache = UnifiedRadixCache.__new__(UnifiedRadixCache)
|
|
cache.is_mamba_enabled = False
|
|
cache.tree_core = SimpleNamespace(is_eagle=False)
|
|
cache.req_to_token_pool = SimpleNamespace(
|
|
req_to_token=torch.arange(16, dtype=torch.int64).reshape(2, 8)
|
|
)
|
|
component = MagicMock()
|
|
component.prepare_for_caching_req.return_value = None
|
|
cache._components_tuple = (component,)
|
|
cache.insert = MagicMock()
|
|
req = SimpleNamespace(
|
|
rid="req",
|
|
get_fill_ids=lambda: [1, 2, 3, 4],
|
|
kv=SimpleNamespace(
|
|
req_pool_idx=1,
|
|
cache_protected_len=2,
|
|
),
|
|
prefix_indices=torch.tensor([8, 9], dtype=torch.int64),
|
|
priority=3,
|
|
extra_key=None,
|
|
cache_salt=None,
|
|
kv_rotation_base=None,
|
|
last_node=object(),
|
|
)
|
|
return cache, component, req
|
|
|
|
def _pop_bootstrapping_swa_branch(self, write_policy):
|
|
queue = PrefillBootstrapQueue.__new__(PrefillBootstrapQueue)
|
|
queue.scheduler_stage_metrics = None
|
|
queue.pp_size = 1
|
|
cache = UnifiedRadixCache.__new__(UnifiedRadixCache)
|
|
cache.cache_controller = SimpleNamespace(write_policy=write_policy)
|
|
queue.scheduler = SimpleNamespace(
|
|
attn_cp_cpu_group=None,
|
|
attn_tp_cpu_group=None,
|
|
tree_cache=cache,
|
|
processed_tokens_counter=0,
|
|
)
|
|
queue.ensure_metadata_buffer = MagicMock(return_value=True)
|
|
req = SimpleNamespace(
|
|
rid="req",
|
|
disagg_kv_sender=object(),
|
|
prefill_attempt_count=0,
|
|
is_retracted=False,
|
|
swa_branching_seqlen=4,
|
|
kv=SimpleNamespace(cache_protected_len=2),
|
|
time_stats=MagicMock(),
|
|
)
|
|
queue.queue = [req]
|
|
|
|
with (
|
|
patch(
|
|
"sglang.srt.disaggregation.prefill.poll_and_all_reduce_attn_cp_tp_group",
|
|
return_value=[KVPoll.Bootstrapping],
|
|
),
|
|
patch(
|
|
"sglang.srt.disaggregation.prefill.get_disagg",
|
|
return_value=SimpleNamespace(optimistic_prefill_attempts=2),
|
|
),
|
|
):
|
|
result = queue.pop_bootstrapped()
|
|
|
|
return queue, req, result
|
|
|
|
def test_write_through_pending_swa_branch_waits_for_bootstrap(self):
|
|
queue, req, result = self._pop_bootstrapping_swa_branch("write_through")
|
|
|
|
self.assertEqual(result, [])
|
|
self.assertEqual(queue.queue, [req])
|
|
self.assertEqual(req.prefill_attempt_count, 0)
|
|
queue.ensure_metadata_buffer.assert_not_called()
|
|
|
|
def test_other_write_policies_stay_optimistic(self):
|
|
for write_policy in ("write_back", "write_through_selective"):
|
|
with self.subTest(write_policy=write_policy):
|
|
queue, req, result = self._pop_bootstrapping_swa_branch(write_policy)
|
|
|
|
self.assertEqual(result, [req])
|
|
self.assertEqual(queue.queue, [])
|
|
self.assertEqual(req.prefill_attempt_count, 1)
|
|
queue.ensure_metadata_buffer.assert_called_once_with(req)
|
|
|
|
def test_unpublished_advance_preserves_swa_frontier_and_ownership(self):
|
|
cache, component, req = self._new_unpublished_swa_state()
|
|
last_node = req.last_node
|
|
|
|
with envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True):
|
|
cache.advance_unpublished_req(req, chunked=True)
|
|
|
|
self.assertTrue(torch.equal(req.prefix_indices, torch.tensor([8, 9, 10, 11])))
|
|
self.assertEqual(req.kv.cache_protected_len, 2)
|
|
self.assertIs(req.last_node, last_node)
|
|
prepare_params = component.prepare_for_caching_req.call_args.kwargs[
|
|
"insert_params"
|
|
]
|
|
self.assertTrue(prepare_params.chunked)
|
|
self.assertEqual(prepare_params.prev_prefix_len, 2)
|
|
component.free_out_of_window_slots.assert_called_once_with(
|
|
req, 3, prepare_params
|
|
)
|
|
component.cleanup_after_caching_req.assert_called_once_with(
|
|
req, is_finished=False, insert_params=prepare_params
|
|
)
|
|
cache.insert.assert_not_called()
|
|
|
|
def test_bootstrap_failure_after_swa_eviction_releases_once(self):
|
|
cache, component, req = self._new_unpublished_swa_state()
|
|
req.bootstrap_room = 7
|
|
req.disagg_kv_sender = MagicMock()
|
|
req.time_stats = SimpleNamespace(trace_ctx=MagicMock())
|
|
req.return_logprob = False
|
|
req.pending_bootstrap = True
|
|
req.metadata_buffer_index = -1
|
|
req.cache_request_handle = object()
|
|
req.kv.holds_kv = True
|
|
req.kv.holds_mamba = False
|
|
cache.finish = MagicMock()
|
|
scheduler = SimpleNamespace(
|
|
tree_cache=cache,
|
|
clear_pending_chunk_send=MagicMock(),
|
|
ps=SimpleNamespace(tp_rank=0),
|
|
req_to_metadata_buffer_idx_allocator=MagicMock(),
|
|
output_streamer=MagicMock(),
|
|
metrics_reporter=SimpleNamespace(enable_metrics=False),
|
|
enable_hicache_storage=False,
|
|
)
|
|
|
|
with (
|
|
envs.SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS.override(True),
|
|
patch(
|
|
"sglang.srt.disaggregation.prefill.release_kv_cache"
|
|
) as release_kv_cache,
|
|
patch("sglang.srt.disaggregation.prefill.prepare_abort"),
|
|
patch(
|
|
"sglang.srt.disaggregation.prefill.get_parallel",
|
|
return_value=SimpleNamespace(tp_rank=0),
|
|
),
|
|
):
|
|
cache.advance_unpublished_req(req, chunked=True)
|
|
SchedulerDisaggregationPrefillMixin.handle_bootstrap_failure(scheduler, req)
|
|
|
|
component.free_out_of_window_slots.assert_called_once()
|
|
cache.insert.assert_not_called()
|
|
release_kv_cache.assert_called_once_with(req, cache, is_insert=False)
|
|
cache.finish.assert_called_once_with(
|
|
req.cache_request_handle, CacheRequestOutcome.ABORT
|
|
)
|
|
self.assertFalse(req.pending_bootstrap)
|
|
|
|
def test_write_through_pending_chunk_does_not_publish(self):
|
|
scheduler = SimpleNamespace()
|
|
cache = UnifiedRadixCache.__new__(UnifiedRadixCache)
|
|
cache.cache_controller = SimpleNamespace(write_policy="write_through")
|
|
cache.advance_unpublished_req = MagicMock()
|
|
scheduler.tree_cache = cache
|
|
req = SimpleNamespace(pending_bootstrap=True)
|
|
|
|
SchedulerDisaggregationPrefillMixin.cache_unfinished_disagg_prefill(
|
|
scheduler, req, chunked=True
|
|
)
|
|
|
|
cache.advance_unpublished_req.assert_called_once_with(req, chunked=True)
|
|
|
|
def test_bootstrap_success_publishes_once(self):
|
|
scheduler = SimpleNamespace(
|
|
disagg_prefill_bootstrap_queue=MagicMock(),
|
|
cache_unfinished_disagg_prefill=MagicMock(),
|
|
)
|
|
scheduler.disagg_prefill_bootstrap_queue.finalize_bootstrap.return_value = True
|
|
req = SimpleNamespace(rid="req")
|
|
|
|
result = SchedulerDisaggregationPrefillMixin.handle_pending_bootstrap(
|
|
scheduler, req, KVPoll.WaitingForInput
|
|
)
|
|
|
|
self.assertTrue(result)
|
|
scheduler.disagg_prefill_bootstrap_queue.finalize_bootstrap.assert_called_once_with(
|
|
req
|
|
)
|
|
scheduler.cache_unfinished_disagg_prefill.assert_called_once_with(req)
|
|
|
|
def test_waiting_abort_releases_without_insert(self):
|
|
scheduler = SimpleNamespace(
|
|
tree_cache=MagicMock(),
|
|
req_to_metadata_buffer_idx_allocator=MagicMock(),
|
|
clear_pending_chunk_send=MagicMock(),
|
|
)
|
|
sender = MagicMock()
|
|
req = SimpleNamespace(
|
|
disagg_kv_sender=sender,
|
|
metadata_buffer_index=3,
|
|
kv=SimpleNamespace(holds_kv=True, holds_mamba=False),
|
|
pending_bootstrap=True,
|
|
)
|
|
|
|
with patch(
|
|
"sglang.srt.disaggregation.prefill.release_kv_cache"
|
|
) as release_kv_cache:
|
|
SchedulerDisaggregationPrefillMixin.release_aborted_prefill_waiting_req(
|
|
scheduler, req
|
|
)
|
|
|
|
release_kv_cache.assert_called_once_with(
|
|
req, scheduler.tree_cache, is_insert=False
|
|
)
|
|
sender.abort.assert_called_once()
|
|
self.assertFalse(req.pending_bootstrap)
|
|
|
|
def test_scheduler_waiting_abort_delegates_release_once(self):
|
|
scheduler = Scheduler.__new__(Scheduler)
|
|
scheduler.chunked_req = None
|
|
scheduler.mm_receiver = None
|
|
scheduler.waiting_queue = [MagicMock(rid="req")]
|
|
scheduler.beam_coordinator = MagicMock()
|
|
scheduler.enable_hicache_storage = False
|
|
scheduler.ipc_channels = MagicMock()
|
|
scheduler.disaggregation_mode = DisaggregationMode.PREFILL
|
|
scheduler.release_aborted_prefill_waiting_req = MagicMock()
|
|
scheduler.dllm_config = None
|
|
scheduler.grammar_manager = MagicMock()
|
|
scheduler.disagg_prefill_bootstrap_queue = SimpleNamespace(queue=[])
|
|
scheduler.disagg_prefill_inflight_queue = []
|
|
scheduler.collect_inflight_reqs = MagicMock(return_value=set())
|
|
scheduler.tree_cache = MagicMock()
|
|
|
|
with (
|
|
patch("sglang.srt.managers.scheduler.get_serving") as get_serving,
|
|
patch("sglang.srt.managers.scheduler.release_kv_cache") as release,
|
|
):
|
|
get_serving.return_value.weight_version = None
|
|
scheduler.abort_request(AbortReq(rid="req"))
|
|
|
|
scheduler.release_aborted_prefill_waiting_req.assert_called_once()
|
|
scheduler.tree_cache.finish.assert_called_once()
|
|
release.assert_not_called()
|
|
|
|
|
|
class TestDecodePreallocQueuePriority(unittest.TestCase):
|
|
def setUp(self):
|
|
# The code under test reads its config from the bags.
|
|
reset_context()
|
|
self.addCleanup(reset_context)
|
|
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
|
|
|
def _new_decode_req(self, rid: str, priority: int, *, failed: bool = False):
|
|
req = SimpleNamespace(
|
|
rid=rid,
|
|
priority=priority,
|
|
origin_input_ids=[1, 2, 3],
|
|
output_ids=[],
|
|
finished_reason=FINISH_ABORT("failed") if failed else None,
|
|
return_logprob=False,
|
|
sampling_params=SimpleNamespace(max_new_tokens=8),
|
|
kv=ReqKvInfo(req_pool_idx=int(priority) % 8, cache_protected_len=0),
|
|
time_stats=MagicMock(),
|
|
)
|
|
return SimpleNamespace(
|
|
req=req,
|
|
waiting_for_input=True,
|
|
kv_receiver=MagicMock(),
|
|
metadata_buffer_index=-1,
|
|
is_rebootstrap=False,
|
|
host_staged=False,
|
|
)
|
|
|
|
def _new_queue(self, decode_reqs, *, low_priority_values_first: bool = False):
|
|
queue = DecodePreallocQueue.__new__(DecodePreallocQueue)
|
|
queue.pp_size = 1
|
|
queue.queue = list(decode_reqs)
|
|
queue.pending_reqs = []
|
|
queue.retracted_queue = []
|
|
queue.num_reserved_decode_tokens = 0
|
|
# `pop_preallocated` credits this counter; `__new__` skips the __init__
|
|
# that seeds it.
|
|
queue._num_published_destinations = 0
|
|
queue._resolve_pending_reqs = MagicMock()
|
|
queue._update_handshake_waiters = MagicMock()
|
|
queue._allocatable_tokens = MagicMock(return_value=1000)
|
|
|
|
def pre_alloc_mock(req, prefix_indices=None, prefix_len=0, total_prefix_len=0):
|
|
return torch.arange(
|
|
len(req.origin_input_ids) - prefix_len, dtype=torch.int64
|
|
)
|
|
|
|
queue._pre_alloc = MagicMock(side_effect=pre_alloc_mock)
|
|
|
|
queue.req_to_token_pool = MagicMock()
|
|
queue.req_to_token_pool.available_size.return_value = 100
|
|
# Non-hybrid pools have no mamba allocator; MagicMock would otherwise
|
|
# auto-create one and break the `available_size() <= 0` comparison in
|
|
# pop_preallocated.
|
|
queue.req_to_token_pool.mamba_allocator = None
|
|
queue.req_to_token_pool.req_to_token = torch.arange(
|
|
8 * 16, dtype=torch.int64
|
|
).reshape(8, 16)
|
|
|
|
queue.req_to_metadata_buffer_idx_allocator = MagicMock()
|
|
queue.req_to_metadata_buffer_idx_allocator.available_size.return_value = 100
|
|
queue.req_to_metadata_buffer_idx_allocator.alloc.side_effect = iter(range(100))
|
|
|
|
queue.token_to_kv_pool_allocator = MagicMock()
|
|
bind_separate_buffer_capacity(queue.token_to_kv_pool_allocator)
|
|
queue.token_to_kv_pool_allocator.page_size = 1
|
|
queue.token_to_kv_pool_allocator.available_size.return_value = 1000
|
|
queue.token_to_kv_pool = MagicMock()
|
|
queue.transfer_queue = SimpleNamespace(queue=[], enable_staging=False)
|
|
queue.kv_manager = SimpleNamespace(
|
|
kv_args=SimpleNamespace(state_types=[]),
|
|
)
|
|
queue.tree_cache = MagicMock()
|
|
|
|
scheduler = MagicMock()
|
|
scheduler.enable_priority_scheduling = True
|
|
scheduler.schedule_low_priority_values_first = low_priority_values_first
|
|
scheduler.running_batch.reqs = []
|
|
scheduler.server_args.disaggregation_decode_enable_radix_cache = False
|
|
scheduler.enable_hisparse = False
|
|
scheduler.enable_lora = False
|
|
scheduler.waiting_queue = []
|
|
scheduler.last_batch = None
|
|
scheduler.output_streamer = MagicMock()
|
|
queue.scheduler = scheduler
|
|
return queue
|
|
|
|
def test_prealloc_lora_slots_cover_inflight_microbatches_and_queues(self):
|
|
"""In-flight requests retain their adapter slots."""
|
|
running, finished, transferring, waiting = [
|
|
self._new_decode_req(rid, 0)
|
|
for rid in ("running", "finished", "transferring", "waiting")
|
|
]
|
|
reqs = [
|
|
self._new_decode_req("new", 3),
|
|
self._new_decode_req("blocked", 2),
|
|
self._new_decode_req("same-adapter", 1),
|
|
]
|
|
for entry in [running, finished, transferring, waiting, *reqs]:
|
|
entry.req.lora_id = entry.req.rid
|
|
entry.req.finished = MagicMock(return_value=entry is finished)
|
|
reqs[2].req.lora_id = "running"
|
|
|
|
queue = self._new_queue(reqs)
|
|
queue.pp_size = 2
|
|
queue.transfer_queue.queue = [transferring]
|
|
scheduler = queue.scheduler
|
|
scheduler.running_batch.reqs = [finished.req]
|
|
scheduler.running_mbs = [
|
|
scheduler.running_batch,
|
|
SimpleNamespace(reqs=[running.req]),
|
|
]
|
|
scheduler.waiting_queue = [waiting.req]
|
|
scheduler.enable_decode_hicache = False
|
|
scheduler.enable_lora = True
|
|
scheduler.enable_lora_overlap_loading = False
|
|
scheduler.lora_drainer = None
|
|
scheduler.can_schedule_lora_req = Scheduler.can_schedule_lora_req.__get__(
|
|
scheduler
|
|
)
|
|
manager = LoRAManager.__new__(LoRAManager)
|
|
manager.max_loras_per_batch = 5
|
|
manager.num_pinned_loras = 0
|
|
scheduler.tp_worker.model_runner.lora_manager = manager
|
|
|
|
preallocated, failed = queue.pop_preallocated(
|
|
pp_good_rids=[entry.req.rid for entry in reqs], pp_bad_rids=[]
|
|
)
|
|
|
|
self.assertEqual(preallocated, [reqs[0], reqs[2]])
|
|
self.assertEqual(failed, [])
|
|
self.assertEqual(queue.queue, [reqs[1]])
|
|
self.assertEqual(
|
|
[call.args[0] for call in queue._pre_alloc.call_args_list],
|
|
[reqs[0].req, reqs[2].req],
|
|
)
|
|
|
|
def test_prealloc_queue_schedules_higher_priority_values_first_by_default(self):
|
|
reqs = [
|
|
self._new_decode_req("low", 1),
|
|
self._new_decode_req("high", 10),
|
|
self._new_decode_req("mid", 5),
|
|
]
|
|
queue = self._new_queue(reqs)
|
|
|
|
with patch("sglang.srt.disaggregation.decode.CLIP_MAX_NEW_TOKEN", 4096):
|
|
preallocated, failed = queue.pop_preallocated()
|
|
|
|
self.assertEqual(
|
|
[decode_req.req.rid for decode_req in preallocated],
|
|
[
|
|
"high",
|
|
"mid",
|
|
"low",
|
|
],
|
|
)
|
|
self.assertEqual(failed, [])
|
|
|
|
def test_prealloc_queue_can_schedule_lower_priority_values_first(self):
|
|
reqs = [
|
|
self._new_decode_req("mid", 5),
|
|
self._new_decode_req("high", 10),
|
|
self._new_decode_req("low", 1),
|
|
]
|
|
queue = self._new_queue(reqs, low_priority_values_first=True)
|
|
|
|
with patch("sglang.srt.disaggregation.decode.CLIP_MAX_NEW_TOKEN", 4096):
|
|
preallocated, failed = queue.pop_preallocated()
|
|
|
|
self.assertEqual(
|
|
[decode_req.req.rid for decode_req in preallocated],
|
|
[
|
|
"low",
|
|
"mid",
|
|
"high",
|
|
],
|
|
)
|
|
self.assertEqual(failed, [])
|
|
|
|
def test_failed_request_indices_stay_valid_after_priority_sort(self):
|
|
failed_low = self._new_decode_req("failed-low", 1, failed=True)
|
|
healthy_high = self._new_decode_req("healthy-high", 10)
|
|
queue = self._new_queue([failed_low, healthy_high])
|
|
|
|
with patch("sglang.srt.disaggregation.decode.CLIP_MAX_NEW_TOKEN", 4096):
|
|
preallocated, failed = queue.pop_preallocated()
|
|
|
|
self.assertEqual(
|
|
[decode_req.req.rid for decode_req in preallocated], ["healthy-high"]
|
|
)
|
|
self.assertEqual([decode_req.req.rid for decode_req in failed], ["failed-low"])
|
|
self.assertEqual(queue.queue, [])
|
|
queue.scheduler.output_streamer.stream_output.assert_called_once_with(
|
|
[failed_low.req], failed_low.req.return_logprob
|
|
)
|
|
|
|
|
|
class TestDecodePreallocQueueRebootstrapPayload(unittest.TestCase):
|
|
"""The decode scheduler builds the rebootstrap ``/generate`` payload; the
|
|
dispatch itself now lives on the kv manager (see
|
|
``TestCommonKVManagerPrefillRecompute``)."""
|
|
|
|
def setUp(self):
|
|
# The code under test reads its config from the bags.
|
|
reset_context()
|
|
self.addCleanup(reset_context)
|
|
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
|
|
|
def _sampling_params(self):
|
|
return SimpleNamespace(
|
|
temperature=0.0,
|
|
top_p=1.0,
|
|
top_k=-1,
|
|
min_p=0.0,
|
|
frequency_penalty=0.0,
|
|
presence_penalty=0.0,
|
|
repetition_penalty=1.0,
|
|
ignore_eos=False,
|
|
skip_special_tokens=True,
|
|
spaces_between_special_tokens=True,
|
|
no_stop_trim=False,
|
|
)
|
|
|
|
def _new_req(self):
|
|
return SimpleNamespace(
|
|
rid="rid-0",
|
|
origin_input_ids=np.array([1, 2], dtype=np.int32),
|
|
output_ids=[np.int32(3), np.int32(4)],
|
|
sampling_params=self._sampling_params(),
|
|
bootstrap_host="127.0.0.1",
|
|
bootstrap_port=30000,
|
|
bootstrap_room=7,
|
|
priority=10,
|
|
extra_key=None,
|
|
cache_salt=None,
|
|
routing_key=None,
|
|
disagg_prefill_dp_rank=None,
|
|
)
|
|
|
|
def test_build_rebootstrap_payload_converts_numpy_ids_to_json_lists(self):
|
|
req = self._new_req()
|
|
|
|
# build_rebootstrap_payload lives on Req; exercise it unbound with a
|
|
# namespace that carries the attributes it reads.
|
|
payload = Req.build_rebootstrap_payload(req)
|
|
|
|
# origin_input_ids + output_ids, coerced to plain python ints.
|
|
self.assertEqual(payload["input_ids"], [1, 2, 3, 4])
|
|
self.assertTrue(all(type(x) is int for x in payload["input_ids"]))
|
|
self.assertEqual(payload["sampling_params"]["max_new_tokens"], 1)
|
|
self.assertEqual(payload["bootstrap_room"], 7)
|
|
self.assertIsNone(payload["cache_salt"])
|
|
# The prefill /generate URL is derived from bootstrap info on the decode
|
|
# side, not sent in the payload; and the boundary token is replayed via
|
|
# the decode-side override, so neither belongs in the payload.
|
|
self.assertNotIn("pd_rebootstrap_prefill_url", payload)
|
|
self.assertNotIn("pd_rebootstrap_forced_output_id", payload)
|
|
# Must be JSON-serializable (numpy scalars would raise here).
|
|
json.dumps(payload)
|
|
|
|
|
|
class TestCommonKVManagerPrefillRecompute(unittest.TestCase):
|
|
"""The kv manager owns the shared executor + HTTP session and routes any
|
|
rebootstrap ``/generate`` failure through ``kv_receiver.abort()`` ->
|
|
``KVPoll.Failed`` so the scheduler's normal transfer-failure streaming runs.
|
|
"""
|
|
|
|
def setUp(self):
|
|
# The code under test reads its config from the bags.
|
|
reset_context()
|
|
self.addCleanup(reset_context)
|
|
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
|
|
|
def _new_manager(self):
|
|
from sglang.srt.disaggregation.common.conn import CommonKVManager
|
|
|
|
mgr = CommonKVManager.__new__(CommonKVManager)
|
|
mgr._prefill_recompute_executor = None
|
|
mgr._prefill_recompute_executor_lock = threading.Lock()
|
|
mgr._prefill_recompute_sessions = threading.local()
|
|
mgr.waiting_timeout = 300
|
|
mgr.failure_records = {}
|
|
mgr.failure_lock = threading.Lock()
|
|
# Only the attn-tp/attn-cp group leader on the first PP stage issues the
|
|
# single rebootstrap /generate; default the mock manager to that leader.
|
|
mgr.attn_tp_rank = 0
|
|
mgr.attn_cp_rank = 0
|
|
mgr.pp_rank = 0
|
|
# Decode-side prefill info cache; the rebootstrap /generate URL is derived
|
|
# from here (bootstrap_addr host + self-registered prefill_http_port)
|
|
# instead of a router-injected pd_rebootstrap_prefill_url.
|
|
mgr.prefill_info_table = {}
|
|
return mgr
|
|
|
|
def _register_prefill_info(self, mgr, bootstrap_addr, http_port):
|
|
from sglang.srt.disaggregation.common.conn import PrefillServerInfo
|
|
|
|
mgr.prefill_info_table[bootstrap_addr] = PrefillServerInfo(
|
|
attn_tp_size=1,
|
|
attn_cp_size=1,
|
|
dp_size=1,
|
|
pp_size=1,
|
|
page_size=1,
|
|
kv_cache_dtype=None,
|
|
follow_bootstrap_room=True,
|
|
prefill_http_port=http_port,
|
|
)
|
|
|
|
def _payload(self):
|
|
return {
|
|
"input_ids": [1, 2, 3, 4],
|
|
"rid": "rid-0",
|
|
}
|
|
|
|
def test_submit_dispatches_run_to_shared_executor(self):
|
|
mgr = self._new_manager()
|
|
mgr._prefill_recompute_executor = MagicMock()
|
|
receiver = MagicMock(bootstrap_room=7, bootstrap_addr="127.0.0.1:8998")
|
|
self._register_prefill_info(mgr, "127.0.0.1:8998", 30000)
|
|
|
|
mgr.submit_prefill_recompute(receiver, self._payload())
|
|
|
|
mgr._prefill_recompute_executor.submit.assert_called_once()
|
|
args = mgr._prefill_recompute_executor.submit.call_args[0]
|
|
self.assertEqual(args[0], mgr._run_prefill_recompute)
|
|
self.assertIs(args[1], receiver)
|
|
# URL derived from bootstrap_addr host + registered prefill_http_port.
|
|
self.assertEqual(args[2], "http://127.0.0.1:30000")
|
|
receiver.abort.assert_not_called()
|
|
|
|
def test_submit_is_noop_on_non_leader_ranks(self):
|
|
# A retracted request is replicated across every rank in its attention
|
|
# TP/CP group and every PP stage; only the group/first-stage leader must
|
|
# POST the single /generate, or the prefill recomputes it once per rank.
|
|
for attn_tp_rank, attn_cp_rank, pp_rank in (
|
|
(1, 0, 0),
|
|
(0, 1, 0),
|
|
(0, 0, 1),
|
|
):
|
|
with self.subTest(
|
|
attn_tp_rank=attn_tp_rank,
|
|
attn_cp_rank=attn_cp_rank,
|
|
pp_rank=pp_rank,
|
|
):
|
|
mgr = self._new_manager()
|
|
mgr.attn_tp_rank = attn_tp_rank
|
|
mgr.attn_cp_rank = attn_cp_rank
|
|
mgr.pp_rank = pp_rank
|
|
mgr._prefill_recompute_executor = MagicMock()
|
|
receiver = MagicMock(bootstrap_room=7, bootstrap_addr="127.0.0.1:8998")
|
|
self._register_prefill_info(mgr, "127.0.0.1:8998", 30000)
|
|
|
|
mgr.submit_prefill_recompute(receiver, self._payload())
|
|
|
|
mgr._prefill_recompute_executor.submit.assert_not_called()
|
|
receiver.abort.assert_not_called()
|
|
self.assertEqual(mgr.failure_records, {})
|
|
|
|
def test_submit_unresolved_url_fails_via_abort(self):
|
|
mgr = self._new_manager()
|
|
mgr._prefill_recompute_executor = MagicMock()
|
|
# No prefill_info registered for this bootstrap_addr -> URL unresolved.
|
|
receiver = MagicMock(bootstrap_room=7, bootstrap_addr="127.0.0.1:8998")
|
|
|
|
mgr.submit_prefill_recompute(receiver, self._payload())
|
|
|
|
receiver.abort.assert_called_once()
|
|
mgr._prefill_recompute_executor.submit.assert_not_called()
|
|
self.assertIn(7, mgr.failure_records)
|
|
|
|
def test_run_aborts_on_http_error(self):
|
|
mgr = self._new_manager()
|
|
session = MagicMock()
|
|
session.post.return_value = SimpleNamespace(status_code=500, text="boom")
|
|
mgr._prefill_recompute_sessions.session = session
|
|
receiver = MagicMock(bootstrap_room=7)
|
|
|
|
mgr._run_prefill_recompute(receiver, "http://prefill", self._payload())
|
|
|
|
session.post.assert_called_once()
|
|
receiver.abort.assert_called_once()
|
|
self.assertIn(7, mgr.failure_records)
|
|
|
|
def test_run_aborts_on_exception(self):
|
|
mgr = self._new_manager()
|
|
session = MagicMock()
|
|
session.post.side_effect = RuntimeError("network down")
|
|
mgr._prefill_recompute_sessions.session = session
|
|
receiver = MagicMock(bootstrap_room=7)
|
|
|
|
mgr._run_prefill_recompute(receiver, "http://prefill", self._payload())
|
|
|
|
receiver.abort.assert_called_once()
|
|
self.assertIn(7, mgr.failure_records)
|
|
|
|
def test_run_success_does_not_abort(self):
|
|
mgr = self._new_manager()
|
|
session = MagicMock()
|
|
session.post.return_value = SimpleNamespace(status_code=200, text="")
|
|
mgr._prefill_recompute_sessions.session = session
|
|
receiver = MagicMock(bootstrap_room=7)
|
|
|
|
mgr._run_prefill_recompute(receiver, "http://prefill", self._payload())
|
|
|
|
session.post.assert_called_once()
|
|
receiver.abort.assert_not_called()
|
|
self.assertEqual(mgr.failure_records, {})
|
|
|
|
|
|
class TestDecodePrebuilt(unittest.TestCase):
|
|
def setUp(self):
|
|
# The code under test reads its config from the bags.
|
|
reset_context()
|
|
self.addCleanup(reset_context)
|
|
publish(ServerArgs(model_path="dummy"), role="tokenizer")
|
|
|
|
def _new_scheduler(self, *, enable_overlap: bool) -> Scheduler:
|
|
scheduler = Scheduler.__new__(Scheduler)
|
|
scheduler.grammar_manager = MagicMock()
|
|
scheduler.grammar_manager.has_waiting_grammars.return_value = False
|
|
scheduler.waiting_queue = []
|
|
scheduler.enable_priority_scheduling = False
|
|
scheduler.running_batch = MagicMock()
|
|
scheduler.running_batch.batch_size.return_value = 0
|
|
scheduler.req_to_token_pool = MagicMock(size=1)
|
|
scheduler.token_to_kv_pool_allocator = MagicMock()
|
|
scheduler.tree_cache = MagicMock()
|
|
scheduler.model_config = MagicMock()
|
|
scheduler.enable_overlap = enable_overlap
|
|
scheduler.spec_algorithm = MagicMock()
|
|
scheduler.max_running_requests = 1
|
|
scheduler.future_map = MagicMock()
|
|
scheduler.policy = MagicMock()
|
|
scheduler.schedule_stream = MagicMock()
|
|
scheduler.forward_stream = MagicMock()
|
|
scheduler.ngram_embedding_manager = MagicMock()
|
|
scheduler.chunked_req = None
|
|
return scheduler
|
|
|
|
def test_waiting_queue_is_sorted_before_prebuilt_selection(self):
|
|
scheduler = self._new_scheduler(enable_overlap=False)
|
|
original_waiting_queue = [MagicMock(rid="low"), MagicMock(rid="high")]
|
|
scheduler.waiting_queue = original_waiting_queue
|
|
scheduler.waiting_queue[0].priority = 1
|
|
scheduler.waiting_queue[1].priority = 10
|
|
scheduler.enable_priority_scheduling = True
|
|
scheduler.policy.calc_priority.side_effect = lambda waiting_queue, _: (
|
|
waiting_queue.sort(key=lambda req: -req.priority)
|
|
)
|
|
|
|
new_batch = MagicMock()
|
|
new_batch.is_empty.return_value = False
|
|
# get_new_prebuilt_batch reads the published disagg config
|
|
# (disaggregation_decode_enable_radix_cache).
|
|
with (
|
|
patch(
|
|
"sglang.srt.disaggregation.decode.ScheduleBatch.init_new",
|
|
return_value=new_batch,
|
|
) as init_new,
|
|
get_context().override_server_args(
|
|
disaggregation_decode_enable_radix_cache=False
|
|
),
|
|
):
|
|
ret = SchedulerDisaggregationDecodeMixin.get_new_prebuilt_batch(
|
|
scheduler, scheduler.running_batch
|
|
)
|
|
|
|
self.assertIs(ret, new_batch)
|
|
scheduler.policy.calc_priority.assert_called_once_with(
|
|
original_waiting_queue, scheduler.running_batch
|
|
)
|
|
selected_reqs = init_new.call_args.args[0]
|
|
self.assertEqual([req.rid for req in selected_reqs], ["high"])
|
|
self.assertEqual([req.rid for req in scheduler.waiting_queue], ["low"])
|
|
|
|
def test_overlap_waits_for_forward_before_processing_prebuilt(self):
|
|
scheduler = self._new_scheduler(enable_overlap=True)
|
|
scheduler.waiting_queue = [MagicMock(rid="request")]
|
|
|
|
call_order = []
|
|
new_batch = MagicMock()
|
|
new_batch.is_empty.return_value = False
|
|
new_batch.prepare_for_prebuilt.side_effect = lambda: call_order.append(
|
|
"prepare"
|
|
)
|
|
scheduler.schedule_stream.wait_stream.side_effect = lambda _: call_order.append(
|
|
"wait"
|
|
)
|
|
new_batch.process_prebuilt.side_effect = lambda *_: call_order.append("process")
|
|
|
|
with (
|
|
patch(
|
|
"sglang.srt.disaggregation.decode.ScheduleBatch.init_new",
|
|
return_value=new_batch,
|
|
),
|
|
get_context().override_server_args(
|
|
disaggregation_decode_enable_radix_cache=False
|
|
),
|
|
):
|
|
ret = SchedulerDisaggregationDecodeMixin.get_new_prebuilt_batch(
|
|
scheduler, scheduler.running_batch
|
|
)
|
|
|
|
self.assertIs(ret, new_batch)
|
|
scheduler.schedule_stream.wait_stream.assert_called_once_with(
|
|
scheduler.forward_stream
|
|
)
|
|
self.assertEqual(call_order, ["prepare", "wait", "process"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|