[Core] Add subclass hooks for the generation request class and optimistic prefill eligibility (#44113)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: CuiCoco <CuiCoco@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-14 16:42:30 -07:00
committed by GitHub
co-authored by Hanming Lu CuiCoco
parent 5f2861bf59
commit 2a9f6a25a7
6 changed files with 209 additions and 22 deletions
+13 -14
View File
@@ -458,6 +458,18 @@ class PrefillBootstrapQueue:
"""
req.sampling_params.max_new_tokens = 1
def can_prefill_before_bootstrap(self, req: Req) -> bool:
"""Check eligibility; pause and metadata admission stay in the shared queue."""
return (
req.prefill_attempt_count < get_disagg().optimistic_prefill_attempts
or get_disagg().disaggregation_decode_allocation_policy
== "prefill_complete"
) and not (
_uses_write_through_cache(self.scheduler.tree_cache)
and req.swa_branching_seqlen is not None
and req.swa_branching_seqlen > req.kv.cache_protected_len
)
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE)
def pop_bootstrapped(self) -> List[Req]:
"""Pop the reqs which have finished bootstrapping."""
@@ -490,20 +502,7 @@ class PrefillBootstrapQueue:
self.scheduler.handle_bootstrap_failure(req)
indices_to_remove.add(i)
elif poll == KVPoll.Bootstrapping:
if (
(
req.prefill_attempt_count
< get_disagg().optimistic_prefill_attempts
or get_disagg().disaggregation_decode_allocation_policy
== "prefill_complete"
)
and not req.is_retracted # engine paused
and not (
_uses_write_through_cache(self.scheduler.tree_cache)
and req.swa_branching_seqlen is not None
and req.swa_branching_seqlen > req.kv.cache_protected_len
)
):
if not req.is_retracted and self.can_prefill_before_bootstrap(req):
if not self.ensure_metadata_buffer(req):
continue # no more metadata buffer
req.prefill_attempt_count += 1
+11 -2
View File
@@ -2828,6 +2828,12 @@ class Scheduler(
mm_inputs.release_features()
req.multimodal_inputs = None
def _create_generation_req(
self, recv_req: TokenizedGenerateReqInput, *args, **kwargs
) -> Req:
"""Extension point for request-local state, including session requests."""
return Req(*args, **kwargs)
def handle_generate_request(
self,
recv_req: TokenizedGenerateReqInput,
@@ -2870,7 +2876,8 @@ class Scheduler(
recv_req.input_ids = array("q", [1]) * seq_length
is_beam = BeamCoordinator.request_beam_width(recv_req) > 1
req = Req(
req = self._create_generation_req(
recv_req,
recv_req.rid,
recv_req.input_text,
recv_req.input_ids,
@@ -2963,6 +2970,7 @@ class Scheduler(
self.model_config.vocab_size,
eos_token_ids=self.model_config.hf_eos_token_id,
disagg_mode=self.disaggregation_mode,
req_factory=partial(self._create_generation_req, recv_req),
)
if self.enable_session_radix_cache:
req.session_generation = self.tree_cache.ensure_session_generation(
@@ -2984,7 +2992,8 @@ class Scheduler(
)
else:
error_msg = f"Invalid request: session id {session_id} does not exist"
req = Req(
req = self._create_generation_req(
recv_req,
recv_req.rid,
recv_req.input_text,
recv_req.input_ids,
@@ -453,6 +453,9 @@ _SCHEDULER_EXIT_TIMEOUT_SECS = 15
class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
"""TokenizerManager is a process that tokenizes the text."""
tokenized_generate_req_type = TokenizedGenerateReqInput
batch_tokenized_generate_req_type = BatchTokenizedGenerateReqInput
# Set by whoever owns the event loop, and left None for Engine and grpc,
# which own no server. Class-level to leave the frozen __init__ alone.
_server_stop_hook: Optional[Callable[[], None]] = None
@@ -1589,7 +1592,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
bootstrap_room = self.fake_bootstrap_room_counter
self.fake_bootstrap_room_counter += 1
tokenized_obj = TokenizedGenerateReqInput(
tokenized_obj = self.tokenized_generate_req_type(
input_text=input_text,
input_ids=input_ids_arr,
mm_inputs=mm_inputs,
@@ -1843,7 +1846,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
tokenized_obj.wrap_pickle_fields()
if isinstance(tokenized_objs[0], TokenizedGenerateReqInput):
batch_req = BatchTokenizedGenerateReqInput(batch=tokenized_objs)
batch_req = self.batch_tokenized_generate_req_type(batch=tokenized_objs)
else:
batch_req = BatchTokenizedEmbeddingReqInput(batch=tokenized_objs)
@@ -16,6 +16,7 @@ import logging
import time
import uuid
from array import array
from collections.abc import Callable
from typing import TYPE_CHECKING, Dict, Optional
from sglang.srt.disaggregation.utils import DisaggregationMode
@@ -208,6 +209,7 @@ class Session:
vocab_size: int,
eos_token_ids=None,
disagg_mode: Optional[DisaggregationMode] = None,
req_factory: Callable[..., Req] = Req,
):
assert req.session_params is not None
session_params = req.session_params
@@ -296,7 +298,7 @@ class Session:
"A session request must contain input tokens after restoring history."
)
new_req = Req(
new_req = req_factory(
rid=req.rid,
origin_input_text=None,
origin_input_ids=input_ids,