[Fix] srt: Apply --gc-threshold in every server process and freeze every tokenizer worker (#44785)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: joshuadeng <joshuadeng@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-16 16:55:54 -07:00
committed by GitHub
co-authored by metamergebot joshuadeng
parent d230f2c35b
commit d8dd49d21d
8 changed files with 88 additions and 8 deletions
+21 -2
View File
@@ -27,6 +27,7 @@ import threading
import time
import uuid
from contextlib import asynccontextmanager
from functools import partial
from http import HTTPStatus
from typing import (
Annotated,
@@ -183,6 +184,7 @@ from sglang.srt.utils import (
add_prometheus_track_response_middleware,
build_server_info,
delete_directory,
freeze_gc,
get_bool_env_var,
is_mps,
kill_process_tree,
@@ -295,8 +297,9 @@ async def lifespan(fast_api_app: FastAPI):
else:
# Initialize multi-tokenizer support for worker processes
server_args = await init_multi_tokenizer()
warmup_thread_kwargs = dict(server_args=server_args)
thread_label = f"MultiTokenizer-{_global_state.tokenizer_manager.worker_id}"
worker_id = _global_state.tokenizer_manager.worker_id
warmup_thread_kwargs = _multi_tokenizer_warmup_kwargs(server_args, worker_id)
thread_label = f"MultiTokenizer-{worker_id}"
# Add prometheus middleware
if get_observability().enable_metrics:
@@ -2459,6 +2462,19 @@ def _execute_server_warmup(server_args: ServerArgs):
return success
def _tokenizer_worker_gc_callback(worker_id: int, gc_threshold) -> Callable[[], None]:
return partial(freeze_gc, f"Tokenizer worker {worker_id}", gc_threshold)
def _multi_tokenizer_warmup_kwargs(server_args: ServerArgs, worker_id: int):
return dict(
server_args=server_args,
post_warmup_callback=_tokenizer_worker_gc_callback(
worker_id, server_args.gc_threshold
),
)
def _freeze_gc_after_server_warmup(server_args: ServerArgs):
# Freeze GC after server warmup so static objects skip future GC gen2 collection.
# Use /freeze_gc to freeze scheduler and detokenizer as well.
@@ -2482,6 +2498,7 @@ def _wait_and_warmup(
server_args: ServerArgs,
launch_callback: Optional[Callable[[], None]] = None,
execute_warmup_func: Callable = _execute_server_warmup,
post_warmup_callback: Optional[Callable[[], None]] = None,
):
if get_model().checkpoint_engine_wait_weights_before_ready:
_wait_weights_ready()
@@ -2500,6 +2517,8 @@ def _wait_and_warmup(
else:
_global_state.tokenizer_manager.server_status = ServerStatus.Up
if post_warmup_callback is not None:
post_warmup_callback()
_freeze_gc_after_server_warmup(server_args)
# The server is ready for requests
@@ -511,7 +511,7 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
)
def handle_freeze_gc_req(self, recv_req: FreezeGCReq):
freeze_gc("Detokenizer Manager")
freeze_gc("Detokenizer Manager", recv_req.gc_threshold)
return None
def handle_configure_logging_req(self, recv_req: ConfigureLoggingReq):
+1 -1
View File
@@ -2392,7 +2392,7 @@ class ProfileReqOutput(BaseReq, kw_only=True):
class FreezeGCReq(BaseReq, kw_only=True):
pass
gc_threshold: Optional[List[int]] = None
class ShutdownReq(BaseReq, kw_only=True):
+1 -1
View File
@@ -6015,7 +6015,7 @@ class Scheduler(
def handle_freeze_gc(self, recv_req: FreezeGCReq):
"""Handle freeze_gc request: freeze scheduler's GC and forward to detokenizer."""
freeze_gc("Scheduler")
freeze_gc("Scheduler", recv_req.gc_threshold)
self.ipc_channels.send_to_detokenizer.send_output(recv_req, recv_req)
return None
@@ -2434,8 +2434,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
async def freeze_gc(self):
"""Send a freeze_gc message to the scheduler first, then freeze locally."""
self._dispatch_to_scheduler(FreezeGCReq())
freeze_gc("Tokenizer Manager")
gc_threshold = get_device().gc_threshold
self._dispatch_to_scheduler(FreezeGCReq(gc_threshold=gc_threshold))
freeze_gc("Tokenizer Manager", gc_threshold)
return None
def create_abort_task(self, obj: GenerateReqInput):
+3 -1
View File
@@ -4455,7 +4455,9 @@ def configure_gc_warning(warn_threshold_secs):
gc.callbacks.append(gc_callback)
def freeze_gc(context: str):
def freeze_gc(context: str, gc_threshold: Optional[Sequence[int]] = None):
if gc_threshold is not None:
gc.set_threshold(*gc_threshold)
g0_before, g1_before, g2_before = gc_object_counts()
gc.freeze()
g0_after, g1_after, g2_after = gc_object_counts()