From 2a9f6a25a7394f4fef43d1e18ee04c50bc51b2d8 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Wed, 14 Oct 2026 16:42:30 -0700 Subject: [PATCH] [Core] Add subclass hooks for the generation request class and optimistic prefill eligibility (#44113) Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: CuiCoco --- python/sglang/srt/disaggregation/prefill.py | 27 ++-- python/sglang/srt/managers/scheduler.py | 13 +- .../sglang/srt/managers/tokenizer_manager.py | 7 +- .../sglang/srt/session/session_controller.py | 4 +- ...test_priority_scheduling_disaggregation.py | 29 +++- .../test_request_class_override_points.py | 151 ++++++++++++++++++ 6 files changed, 209 insertions(+), 22 deletions(-) create mode 100644 test/registered/unit/managers/test_request_class_override_points.py diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 27414f266c..da0aed989f 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -458,6 +458,18 @@ class PrefillBootstrapQueue: """ req.sampling_params.max_new_tokens = 1 + def can_prefill_before_bootstrap(self, req: Req) -> bool: + """Check eligibility; pause and metadata admission stay in the shared queue.""" + return ( + req.prefill_attempt_count < get_disagg().optimistic_prefill_attempts + or get_disagg().disaggregation_decode_allocation_policy + == "prefill_complete" + ) and not ( + _uses_write_through_cache(self.scheduler.tree_cache) + and req.swa_branching_seqlen is not None + and req.swa_branching_seqlen > req.kv.cache_protected_len + ) + @scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE) def pop_bootstrapped(self) -> List[Req]: """Pop the reqs which have finished bootstrapping.""" @@ -490,20 +502,7 @@ class PrefillBootstrapQueue: self.scheduler.handle_bootstrap_failure(req) indices_to_remove.add(i) elif poll == KVPoll.Bootstrapping: - if ( - ( - req.prefill_attempt_count - < get_disagg().optimistic_prefill_attempts - or get_disagg().disaggregation_decode_allocation_policy - == "prefill_complete" - ) - and not req.is_retracted # engine paused - and not ( - _uses_write_through_cache(self.scheduler.tree_cache) - and req.swa_branching_seqlen is not None - and req.swa_branching_seqlen > req.kv.cache_protected_len - ) - ): + if not req.is_retracted and self.can_prefill_before_bootstrap(req): if not self.ensure_metadata_buffer(req): continue # no more metadata buffer req.prefill_attempt_count += 1 diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 5d9f0be785..f79649f917 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2828,6 +2828,12 @@ class Scheduler( mm_inputs.release_features() req.multimodal_inputs = None + def _create_generation_req( + self, recv_req: TokenizedGenerateReqInput, *args, **kwargs + ) -> Req: + """Extension point for request-local state, including session requests.""" + return Req(*args, **kwargs) + def handle_generate_request( self, recv_req: TokenizedGenerateReqInput, @@ -2870,7 +2876,8 @@ class Scheduler( recv_req.input_ids = array("q", [1]) * seq_length is_beam = BeamCoordinator.request_beam_width(recv_req) > 1 - req = Req( + req = self._create_generation_req( + recv_req, recv_req.rid, recv_req.input_text, recv_req.input_ids, @@ -2963,6 +2970,7 @@ class Scheduler( self.model_config.vocab_size, eos_token_ids=self.model_config.hf_eos_token_id, disagg_mode=self.disaggregation_mode, + req_factory=partial(self._create_generation_req, recv_req), ) if self.enable_session_radix_cache: req.session_generation = self.tree_cache.ensure_session_generation( @@ -2984,7 +2992,8 @@ class Scheduler( ) else: error_msg = f"Invalid request: session id {session_id} does not exist" - req = Req( + req = self._create_generation_req( + recv_req, recv_req.rid, recv_req.input_text, recv_req.input_ids, diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 4a393cd000..ae7b0015f0 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -453,6 +453,9 @@ _SCHEDULER_EXIT_TIMEOUT_SECS = 15 class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): """TokenizerManager is a process that tokenizes the text.""" + tokenized_generate_req_type = TokenizedGenerateReqInput + batch_tokenized_generate_req_type = BatchTokenizedGenerateReqInput + # Set by whoever owns the event loop, and left None for Engine and grpc, # which own no server. Class-level to leave the frozen __init__ alone. _server_stop_hook: Optional[Callable[[], None]] = None @@ -1589,7 +1592,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): bootstrap_room = self.fake_bootstrap_room_counter self.fake_bootstrap_room_counter += 1 - tokenized_obj = TokenizedGenerateReqInput( + tokenized_obj = self.tokenized_generate_req_type( input_text=input_text, input_ids=input_ids_arr, mm_inputs=mm_inputs, @@ -1843,7 +1846,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): tokenized_obj.wrap_pickle_fields() if isinstance(tokenized_objs[0], TokenizedGenerateReqInput): - batch_req = BatchTokenizedGenerateReqInput(batch=tokenized_objs) + batch_req = self.batch_tokenized_generate_req_type(batch=tokenized_objs) else: batch_req = BatchTokenizedEmbeddingReqInput(batch=tokenized_objs) diff --git a/python/sglang/srt/session/session_controller.py b/python/sglang/srt/session/session_controller.py index f291cdc26e..41f29b18ae 100644 --- a/python/sglang/srt/session/session_controller.py +++ b/python/sglang/srt/session/session_controller.py @@ -16,6 +16,7 @@ import logging import time import uuid from array import array +from collections.abc import Callable from typing import TYPE_CHECKING, Dict, Optional from sglang.srt.disaggregation.utils import DisaggregationMode @@ -208,6 +209,7 @@ class Session: vocab_size: int, eos_token_ids=None, disagg_mode: Optional[DisaggregationMode] = None, + req_factory: Callable[..., Req] = Req, ): assert req.session_params is not None session_params = req.session_params @@ -296,7 +298,7 @@ class Session: "A session request must contain input tokens after restoring history." ) - new_req = Req( + new_req = req_factory( rid=req.rid, origin_input_text=None, origin_input_ids=input_ids, diff --git a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py index 89616c7505..f77cb53bb6 100644 --- a/test/registered/unit/managers/test_priority_scheduling_disaggregation.py +++ b/test/registered/unit/managers/test_priority_scheduling_disaggregation.py @@ -217,8 +217,10 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase): ) return cache, component, req - def _pop_bootstrapping_swa_branch(self, write_policy): - queue = PrefillBootstrapQueue.__new__(PrefillBootstrapQueue) + def _pop_bootstrapping_swa_branch( + self, write_policy, *, queue_cls=PrefillBootstrapQueue, is_retracted=False + ): + queue = queue_cls.__new__(queue_cls) queue.scheduler_stage_metrics = None queue.pp_size = 1 cache = UnifiedRadixCache.__new__(UnifiedRadixCache) @@ -234,7 +236,7 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase): rid="req", disagg_kv_sender=object(), prefill_attempt_count=0, - is_retracted=False, + is_retracted=is_retracted, swa_branching_seqlen=4, kv=SimpleNamespace(cache_protected_len=2), time_stats=MagicMock(), @@ -273,6 +275,27 @@ class TestOptimisticPrefillCacheOwnership(unittest.TestCase): self.assertEqual(req.prefill_attempt_count, 1) queue.ensure_metadata_buffer.assert_called_once_with(req) + def test_subclass_eligibility_admits_prefill_but_never_a_paused_request(self): + """`can_prefill_before_bootstrap` is the override point; the pause check + stays outside it, so a permissive subclass cannot resume a retracted req.""" + + class _Eager(PrefillBootstrapQueue): + def can_prefill_before_bootstrap(self, req): + return True + + queue, req, result = self._pop_bootstrapping_swa_branch( + "write_through", queue_cls=_Eager + ) + self.assertEqual(result, [req]) + self.assertEqual(req.prefill_attempt_count, 1) + + queue, req, result = self._pop_bootstrapping_swa_branch( + "write_through", queue_cls=_Eager, is_retracted=True + ) + self.assertEqual(result, []) + self.assertEqual(queue.queue, [req]) + queue.ensure_metadata_buffer.assert_not_called() + def test_unpublished_advance_preserves_swa_frontier_and_ownership(self): cache, component, req = self._new_unpublished_swa_state() last_node = req.last_node diff --git a/test/registered/unit/managers/test_request_class_override_points.py b/test/registered/unit/managers/test_request_class_override_points.py new file mode 100644 index 0000000000..3795decfbc --- /dev/null +++ b/test/registered/unit/managers/test_request_class_override_points.py @@ -0,0 +1,151 @@ +"""Subclass-provided request classes must survive every construction site. + +A fork that keeps request-local state on a `Req` or `TokenizedGenerateReqInput` +subclass overrides one hook per class. A bare `Req(...)` or +`TokenizedGenerateReqInput(...)` added beside the hook would silently drop the +subclass on that one path, so these cases walk every path. +""" + +import asyncio +import unittest +from array import array +from types import SimpleNamespace +from unittest.mock import AsyncMock + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel + +maybe_stub_sgl_kernel() + +from sglang.srt.disaggregation.utils import DisaggregationMode +from sglang.srt.managers.io_struct import ( + BatchTokenizedGenerateReqInput, + GenerateReqInput, + SessionParams, + TokenizedGenerateReqInput, +) +from sglang.srt.managers.schedule_batch import Req +from sglang.srt.managers.scheduler import Scheduler +from sglang.srt.managers.tokenizer_manager import TokenizerManager +from sglang.srt.observability.req_time_stats import APIServerReqTimeStats +from sglang.srt.runtime_context import get_context, get_parallel +from sglang.srt.sampling.sampling_params import SamplingParams +from sglang.srt.session.session_controller import Session + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + +VOCAB = 128 + + +class _TaggedReq(Req): + def __init__(self, *args, source_rid: str, **kwargs): + super().__init__(*args, **kwargs) + self.source_rid = source_rid + + +class _TaggedScheduler(Scheduler): + def _create_generation_req(self, recv_req, *args, **kwargs) -> _TaggedReq: + return _TaggedReq(*args, source_rid=recv_req.rid, **kwargs) + + +class _TaggedTokenized(TokenizedGenerateReqInput): + pass + + +class _TaggedBatch(BatchTokenizedGenerateReqInput): + pass + + +class _TaggedTokenizerManager(TokenizerManager): + tokenized_generate_req_type = _TaggedTokenized + batch_tokenized_generate_req_type = _TaggedBatch + + +def _scheduler(sessions, queued): + scheduler = Scheduler.__new__(_TaggedScheduler) + scheduler.session_controller = sessions + scheduler.enable_session_radix_cache = False + scheduler.disaggregation_mode = DisaggregationMode.NULL + scheduler.model_config = SimpleNamespace(hf_eos_token_id=None, vocab_size=VOCAB) + scheduler.metrics_reporter = SimpleNamespace(enable_metrics=False) + scheduler.tokenizer = None + scheduler.dllm_config = None + scheduler._maybe_namespace_elastic_radix_cache = lambda req: None + scheduler.init_req_max_new_tokens = lambda req: None + scheduler._add_request_to_queue = queued.append + return scheduler + + +def _recv(rid, session_params=None): + return TokenizedGenerateReqInput( + rid=rid, + input_text="hello", + input_ids=array("q", [1, 2, 3]), + input_embeds=None, + mm_inputs=None, + token_type_ids=None, + sampling_params=SamplingParams(max_new_tokens=4), + return_logprob=False, + logprob_start_len=-1, + top_logprobs_num=0, + token_ids_logprob=None, + stream=False, + session_params=session_params, + bootstrap_port=1, + ) + + +class TestRequestClassOverridePoints(CustomTestCase): + def test_scheduler_builds_every_generation_request_through_the_factory(self): + """Plain, session, and unknown-session paths all yield the subclass.""" + live = Session(capacity_of_str_len=0, session_id="live", streaming=True) + paths = { + "plain": (None, {}), + "session": (SessionParams(id="live"), {"live": live}), + "unknown-session": (SessionParams(id="gone"), {}), + } + for name, (session_params, sessions) in paths.items(): + with self.subTest(path=name): + queued = [] + scheduler = _scheduler(sessions, queued) + recv = _recv(f"rid-{name}", session_params) + # An mm_input_error stops every path right after construction. + with get_parallel().override(tp_rank=0): + scheduler.handle_generate_request(recv, mm_input_error="stop") + (req,) = queued + self.assertIsInstance(req, _TaggedReq) + self.assertEqual(req.source_rid, recv.rid) + self.assertEqual(req.rid, recv.rid) + + def test_tokenizer_manager_emits_the_subclass_wire_types(self): + """Single and batched dispatch hand the subclass types to the scheduler.""" + manager = TokenizerManager.__new__(_TaggedTokenizerManager) + manager.preferred_sampling_params = None + manager.sampling_params_class = SamplingParams + manager.tokenizer = None + manager.model_config = SimpleNamespace(vocab_size=VOCAB) + manager.fake_bootstrap_room_counter = 0 + manager.rid_to_state = { + "r0": SimpleNamespace(time_stats=APIServerReqTimeStats()) + } + obj = GenerateReqInput(input_ids=[1, 2], rid="r0", sampling_params={}) + + with get_context().override_server_args(): + tokenized = manager._create_tokenized_object(obj, "hello", [1, 2]) + self.assertIsInstance(tokenized, _TaggedTokenized) + + sent = [] + manager.cuda_vmm_feature_transport = SimpleNamespace( + prepare_for_dispatch_async=AsyncMock(return_value=[]), + cancel_for_dispatch=lambda items: None, + ) + manager._dispatch_to_scheduler = sent.append + manager._mark_state_dispatched = lambda rid: None + asyncio.run(manager._send_batch_request([tokenized])) + (batch,) = sent + self.assertIsInstance(batch, _TaggedBatch) + self.assertEqual(batch.batch, [tokenized]) + + +if __name__ == "__main__": + unittest.main()