[Core] Add subclass hooks for the generation request class and optimistic prefill eligibility (#44113)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: CuiCoco <CuiCoco@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
CuiCoco
parent
5f2861bf59
commit
2a9f6a25a7
@@ -458,6 +458,18 @@ class PrefillBootstrapQueue:
|
||||
"""
|
||||
req.sampling_params.max_new_tokens = 1
|
||||
|
||||
def can_prefill_before_bootstrap(self, req: Req) -> bool:
|
||||
"""Check eligibility; pause and metadata admission stay in the shared queue."""
|
||||
return (
|
||||
req.prefill_attempt_count < get_disagg().optimistic_prefill_attempts
|
||||
or get_disagg().disaggregation_decode_allocation_policy
|
||||
== "prefill_complete"
|
||||
) and not (
|
||||
_uses_write_through_cache(self.scheduler.tree_cache)
|
||||
and req.swa_branching_seqlen is not None
|
||||
and req.swa_branching_seqlen > req.kv.cache_protected_len
|
||||
)
|
||||
|
||||
@scheduler_stage_method(SCHEDULER_STAGE_PROCESS_QUEUE)
|
||||
def pop_bootstrapped(self) -> List[Req]:
|
||||
"""Pop the reqs which have finished bootstrapping."""
|
||||
@@ -490,20 +502,7 @@ class PrefillBootstrapQueue:
|
||||
self.scheduler.handle_bootstrap_failure(req)
|
||||
indices_to_remove.add(i)
|
||||
elif poll == KVPoll.Bootstrapping:
|
||||
if (
|
||||
(
|
||||
req.prefill_attempt_count
|
||||
< get_disagg().optimistic_prefill_attempts
|
||||
or get_disagg().disaggregation_decode_allocation_policy
|
||||
== "prefill_complete"
|
||||
)
|
||||
and not req.is_retracted # engine paused
|
||||
and not (
|
||||
_uses_write_through_cache(self.scheduler.tree_cache)
|
||||
and req.swa_branching_seqlen is not None
|
||||
and req.swa_branching_seqlen > req.kv.cache_protected_len
|
||||
)
|
||||
):
|
||||
if not req.is_retracted and self.can_prefill_before_bootstrap(req):
|
||||
if not self.ensure_metadata_buffer(req):
|
||||
continue # no more metadata buffer
|
||||
req.prefill_attempt_count += 1
|
||||
|
||||
@@ -2828,6 +2828,12 @@ class Scheduler(
|
||||
mm_inputs.release_features()
|
||||
req.multimodal_inputs = None
|
||||
|
||||
def _create_generation_req(
|
||||
self, recv_req: TokenizedGenerateReqInput, *args, **kwargs
|
||||
) -> Req:
|
||||
"""Extension point for request-local state, including session requests."""
|
||||
return Req(*args, **kwargs)
|
||||
|
||||
def handle_generate_request(
|
||||
self,
|
||||
recv_req: TokenizedGenerateReqInput,
|
||||
@@ -2870,7 +2876,8 @@ class Scheduler(
|
||||
recv_req.input_ids = array("q", [1]) * seq_length
|
||||
|
||||
is_beam = BeamCoordinator.request_beam_width(recv_req) > 1
|
||||
req = Req(
|
||||
req = self._create_generation_req(
|
||||
recv_req,
|
||||
recv_req.rid,
|
||||
recv_req.input_text,
|
||||
recv_req.input_ids,
|
||||
@@ -2963,6 +2970,7 @@ class Scheduler(
|
||||
self.model_config.vocab_size,
|
||||
eos_token_ids=self.model_config.hf_eos_token_id,
|
||||
disagg_mode=self.disaggregation_mode,
|
||||
req_factory=partial(self._create_generation_req, recv_req),
|
||||
)
|
||||
if self.enable_session_radix_cache:
|
||||
req.session_generation = self.tree_cache.ensure_session_generation(
|
||||
@@ -2984,7 +2992,8 @@ class Scheduler(
|
||||
)
|
||||
else:
|
||||
error_msg = f"Invalid request: session id {session_id} does not exist"
|
||||
req = Req(
|
||||
req = self._create_generation_req(
|
||||
recv_req,
|
||||
recv_req.rid,
|
||||
recv_req.input_text,
|
||||
recv_req.input_ids,
|
||||
|
||||
@@ -453,6 +453,9 @@ _SCHEDULER_EXIT_TIMEOUT_SECS = 15
|
||||
class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
"""TokenizerManager is a process that tokenizes the text."""
|
||||
|
||||
tokenized_generate_req_type = TokenizedGenerateReqInput
|
||||
batch_tokenized_generate_req_type = BatchTokenizedGenerateReqInput
|
||||
|
||||
# Set by whoever owns the event loop, and left None for Engine and grpc,
|
||||
# which own no server. Class-level to leave the frozen __init__ alone.
|
||||
_server_stop_hook: Optional[Callable[[], None]] = None
|
||||
@@ -1589,7 +1592,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
bootstrap_room = self.fake_bootstrap_room_counter
|
||||
self.fake_bootstrap_room_counter += 1
|
||||
|
||||
tokenized_obj = TokenizedGenerateReqInput(
|
||||
tokenized_obj = self.tokenized_generate_req_type(
|
||||
input_text=input_text,
|
||||
input_ids=input_ids_arr,
|
||||
mm_inputs=mm_inputs,
|
||||
@@ -1843,7 +1846,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
tokenized_obj.wrap_pickle_fields()
|
||||
|
||||
if isinstance(tokenized_objs[0], TokenizedGenerateReqInput):
|
||||
batch_req = BatchTokenizedGenerateReqInput(batch=tokenized_objs)
|
||||
batch_req = self.batch_tokenized_generate_req_type(batch=tokenized_objs)
|
||||
else:
|
||||
batch_req = BatchTokenizedEmbeddingReqInput(batch=tokenized_objs)
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ import logging
|
||||
import time
|
||||
import uuid
|
||||
from array import array
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Dict, Optional
|
||||
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
@@ -208,6 +209,7 @@ class Session:
|
||||
vocab_size: int,
|
||||
eos_token_ids=None,
|
||||
disagg_mode: Optional[DisaggregationMode] = None,
|
||||
req_factory: Callable[..., Req] = Req,
|
||||
):
|
||||
assert req.session_params is not None
|
||||
session_params = req.session_params
|
||||
@@ -296,7 +298,7 @@ class Session:
|
||||
"A session request must contain input tokens after restoring history."
|
||||
)
|
||||
|
||||
new_req = Req(
|
||||
new_req = req_factory(
|
||||
rid=req.rid,
|
||||
origin_input_text=None,
|
||||
origin_input_ids=input_ids,
|
||||
|
||||
Reference in New Issue
Block a user