[Fix] srt: Apply --gc-threshold in every server process and freeze every tokenizer worker (#44785)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: joshuadeng <joshuadeng@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
joshuadeng
parent
d230f2c35b
commit
d8dd49d21d
@@ -27,6 +27,7 @@ import threading
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from functools import partial
|
||||
from http import HTTPStatus
|
||||
from typing import (
|
||||
Annotated,
|
||||
@@ -183,6 +184,7 @@ from sglang.srt.utils import (
|
||||
add_prometheus_track_response_middleware,
|
||||
build_server_info,
|
||||
delete_directory,
|
||||
freeze_gc,
|
||||
get_bool_env_var,
|
||||
is_mps,
|
||||
kill_process_tree,
|
||||
@@ -295,8 +297,9 @@ async def lifespan(fast_api_app: FastAPI):
|
||||
else:
|
||||
# Initialize multi-tokenizer support for worker processes
|
||||
server_args = await init_multi_tokenizer()
|
||||
warmup_thread_kwargs = dict(server_args=server_args)
|
||||
thread_label = f"MultiTokenizer-{_global_state.tokenizer_manager.worker_id}"
|
||||
worker_id = _global_state.tokenizer_manager.worker_id
|
||||
warmup_thread_kwargs = _multi_tokenizer_warmup_kwargs(server_args, worker_id)
|
||||
thread_label = f"MultiTokenizer-{worker_id}"
|
||||
|
||||
# Add prometheus middleware
|
||||
if get_observability().enable_metrics:
|
||||
@@ -2459,6 +2462,19 @@ def _execute_server_warmup(server_args: ServerArgs):
|
||||
return success
|
||||
|
||||
|
||||
def _tokenizer_worker_gc_callback(worker_id: int, gc_threshold) -> Callable[[], None]:
|
||||
return partial(freeze_gc, f"Tokenizer worker {worker_id}", gc_threshold)
|
||||
|
||||
|
||||
def _multi_tokenizer_warmup_kwargs(server_args: ServerArgs, worker_id: int):
|
||||
return dict(
|
||||
server_args=server_args,
|
||||
post_warmup_callback=_tokenizer_worker_gc_callback(
|
||||
worker_id, server_args.gc_threshold
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _freeze_gc_after_server_warmup(server_args: ServerArgs):
|
||||
# Freeze GC after server warmup so static objects skip future GC gen2 collection.
|
||||
# Use /freeze_gc to freeze scheduler and detokenizer as well.
|
||||
@@ -2482,6 +2498,7 @@ def _wait_and_warmup(
|
||||
server_args: ServerArgs,
|
||||
launch_callback: Optional[Callable[[], None]] = None,
|
||||
execute_warmup_func: Callable = _execute_server_warmup,
|
||||
post_warmup_callback: Optional[Callable[[], None]] = None,
|
||||
):
|
||||
if get_model().checkpoint_engine_wait_weights_before_ready:
|
||||
_wait_weights_ready()
|
||||
@@ -2500,6 +2517,8 @@ def _wait_and_warmup(
|
||||
else:
|
||||
_global_state.tokenizer_manager.server_status = ServerStatus.Up
|
||||
|
||||
if post_warmup_callback is not None:
|
||||
post_warmup_callback()
|
||||
_freeze_gc_after_server_warmup(server_args)
|
||||
|
||||
# The server is ready for requests
|
||||
|
||||
@@ -511,7 +511,7 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
|
||||
)
|
||||
|
||||
def handle_freeze_gc_req(self, recv_req: FreezeGCReq):
|
||||
freeze_gc("Detokenizer Manager")
|
||||
freeze_gc("Detokenizer Manager", recv_req.gc_threshold)
|
||||
return None
|
||||
|
||||
def handle_configure_logging_req(self, recv_req: ConfigureLoggingReq):
|
||||
|
||||
@@ -2392,7 +2392,7 @@ class ProfileReqOutput(BaseReq, kw_only=True):
|
||||
|
||||
|
||||
class FreezeGCReq(BaseReq, kw_only=True):
|
||||
pass
|
||||
gc_threshold: Optional[List[int]] = None
|
||||
|
||||
|
||||
class ShutdownReq(BaseReq, kw_only=True):
|
||||
|
||||
@@ -6015,7 +6015,7 @@ class Scheduler(
|
||||
|
||||
def handle_freeze_gc(self, recv_req: FreezeGCReq):
|
||||
"""Handle freeze_gc request: freeze scheduler's GC and forward to detokenizer."""
|
||||
freeze_gc("Scheduler")
|
||||
freeze_gc("Scheduler", recv_req.gc_threshold)
|
||||
self.ipc_channels.send_to_detokenizer.send_output(recv_req, recv_req)
|
||||
return None
|
||||
|
||||
|
||||
@@ -2434,8 +2434,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
|
||||
async def freeze_gc(self):
|
||||
"""Send a freeze_gc message to the scheduler first, then freeze locally."""
|
||||
self._dispatch_to_scheduler(FreezeGCReq())
|
||||
freeze_gc("Tokenizer Manager")
|
||||
gc_threshold = get_device().gc_threshold
|
||||
self._dispatch_to_scheduler(FreezeGCReq(gc_threshold=gc_threshold))
|
||||
freeze_gc("Tokenizer Manager", gc_threshold)
|
||||
return None
|
||||
|
||||
def create_abort_task(self, obj: GenerateReqInput):
|
||||
|
||||
@@ -4455,7 +4455,9 @@ def configure_gc_warning(warn_threshold_secs):
|
||||
gc.callbacks.append(gc_callback)
|
||||
|
||||
|
||||
def freeze_gc(context: str):
|
||||
def freeze_gc(context: str, gc_threshold: Optional[Sequence[int]] = None):
|
||||
if gc_threshold is not None:
|
||||
gc.set_threshold(*gc_threshold)
|
||||
g0_before, g1_before, g2_before = gc_object_counts()
|
||||
gc.freeze()
|
||||
g0_after, g1_after, g2_after = gc_object_counts()
|
||||
|
||||
Reference in New Issue
Block a user