Add overridable hooks for custom chat serving implementations (#25807)
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -216,6 +216,16 @@ class InputFormat(Enum):
|
||||
class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
"""TokenizerManager is a process that tokenizes the text."""
|
||||
|
||||
@property
|
||||
def serving_chat_class(self):
|
||||
"""Return the serving chat class for OpenAI API.
|
||||
|
||||
Override in subclass to provide custom serving behavior.
|
||||
"""
|
||||
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
|
||||
|
||||
return OpenAIServingChat
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
server_args: ServerArgs,
|
||||
|
||||
Reference in New Issue
Block a user