[Fix] srt: Apply --gc-threshold in every server process and freeze every tokenizer worker (#44785)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: joshuadeng <joshuadeng@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-16 16:55:54 -07:00
committed by GitHub
co-authored by metamergebot joshuadeng
parent d230f2c35b
commit d8dd49d21d
8 changed files with 88 additions and 8 deletions
+21 -2
View File
@@ -27,6 +27,7 @@ import threading
import time
import uuid
from contextlib import asynccontextmanager
from functools import partial
from http import HTTPStatus
from typing import (
Annotated,
@@ -183,6 +184,7 @@ from sglang.srt.utils import (
add_prometheus_track_response_middleware,
build_server_info,
delete_directory,
freeze_gc,
get_bool_env_var,
is_mps,
kill_process_tree,
@@ -295,8 +297,9 @@ async def lifespan(fast_api_app: FastAPI):
else:
# Initialize multi-tokenizer support for worker processes
server_args = await init_multi_tokenizer()
warmup_thread_kwargs = dict(server_args=server_args)
thread_label = f"MultiTokenizer-{_global_state.tokenizer_manager.worker_id}"
worker_id = _global_state.tokenizer_manager.worker_id
warmup_thread_kwargs = _multi_tokenizer_warmup_kwargs(server_args, worker_id)
thread_label = f"MultiTokenizer-{worker_id}"
# Add prometheus middleware
if get_observability().enable_metrics:
@@ -2459,6 +2462,19 @@ def _execute_server_warmup(server_args: ServerArgs):
return success
def _tokenizer_worker_gc_callback(worker_id: int, gc_threshold) -> Callable[[], None]:
return partial(freeze_gc, f"Tokenizer worker {worker_id}", gc_threshold)
def _multi_tokenizer_warmup_kwargs(server_args: ServerArgs, worker_id: int):
return dict(
server_args=server_args,
post_warmup_callback=_tokenizer_worker_gc_callback(
worker_id, server_args.gc_threshold
),
)
def _freeze_gc_after_server_warmup(server_args: ServerArgs):
# Freeze GC after server warmup so static objects skip future GC gen2 collection.
# Use /freeze_gc to freeze scheduler and detokenizer as well.
@@ -2482,6 +2498,7 @@ def _wait_and_warmup(
server_args: ServerArgs,
launch_callback: Optional[Callable[[], None]] = None,
execute_warmup_func: Callable = _execute_server_warmup,
post_warmup_callback: Optional[Callable[[], None]] = None,
):
if get_model().checkpoint_engine_wait_weights_before_ready:
_wait_weights_ready()
@@ -2500,6 +2517,8 @@ def _wait_and_warmup(
else:
_global_state.tokenizer_manager.server_status = ServerStatus.Up
if post_warmup_callback is not None:
post_warmup_callback()
_freeze_gc_after_server_warmup(server_args)
# The server is ready for requests
@@ -511,7 +511,7 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
)
def handle_freeze_gc_req(self, recv_req: FreezeGCReq):
freeze_gc("Detokenizer Manager")
freeze_gc("Detokenizer Manager", recv_req.gc_threshold)
return None
def handle_configure_logging_req(self, recv_req: ConfigureLoggingReq):
+1 -1
View File
@@ -2392,7 +2392,7 @@ class ProfileReqOutput(BaseReq, kw_only=True):
class FreezeGCReq(BaseReq, kw_only=True):
pass
gc_threshold: Optional[List[int]] = None
class ShutdownReq(BaseReq, kw_only=True):
+1 -1
View File
@@ -6015,7 +6015,7 @@ class Scheduler(
def handle_freeze_gc(self, recv_req: FreezeGCReq):
"""Handle freeze_gc request: freeze scheduler's GC and forward to detokenizer."""
freeze_gc("Scheduler")
freeze_gc("Scheduler", recv_req.gc_threshold)
self.ipc_channels.send_to_detokenizer.send_output(recv_req, recv_req)
return None
@@ -2434,8 +2434,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
async def freeze_gc(self):
"""Send a freeze_gc message to the scheduler first, then freeze locally."""
self._dispatch_to_scheduler(FreezeGCReq())
freeze_gc("Tokenizer Manager")
gc_threshold = get_device().gc_threshold
self._dispatch_to_scheduler(FreezeGCReq(gc_threshold=gc_threshold))
freeze_gc("Tokenizer Manager", gc_threshold)
return None
def create_abort_task(self, obj: GenerateReqInput):
+3 -1
View File
@@ -4455,7 +4455,9 @@ def configure_gc_warning(warn_threshold_secs):
gc.callbacks.append(gc_callback)
def freeze_gc(context: str):
def freeze_gc(context: str, gc_threshold: Optional[Sequence[int]] = None):
if gc_threshold is not None:
gc.set_threshold(*gc_threshold)
g0_before, g1_before, g2_before = gc_object_counts()
gc.freeze()
g0_after, g1_after, g2_after = gc_object_counts()
@@ -1,6 +1,7 @@
import pickle
import unittest
from types import SimpleNamespace
from unittest import mock
from unittest.mock import DEFAULT, MagicMock, patch
import numpy as np
@@ -14,6 +15,7 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
maybe_stub_sgl_kernel()
from sglang.srt.beam_search.types import BeamSearchSequence
from sglang.srt.entrypoints import http_server
from sglang.srt.managers import multi_tokenizer_mixin
from sglang.srt.managers.io_struct import (
AbortReq,
@@ -35,6 +37,7 @@ from sglang.srt.managers.multi_tokenizer_mixin import (
_send_output_to_tokenizer_workers,
get_tokenizer_worker_class,
)
from sglang.srt.runtime_context import get_context
register_cpu_ci(est_time=12, suite="base-a-test-cpu")
@@ -200,6 +203,44 @@ def _make_router() -> MultiDetokenizerRouter:
class TestMultiTokenizerMixin(CustomTestCase):
def test_each_tokenizer_worker_freezes_locally(self):
threshold = [700, 10, 1000]
server_args = mock.Mock(gc_threshold=threshold)
timeline = mock.Mock()
execution = mock.Mock(moe=mock.Mock(is_ep_scale_joiner=False))
with (
get_context().override_server_args(
checkpoint_engine_wait_weights_before_ready=False,
delete_ckpt_after_loading=False,
skip_server_warmup=False,
debug_tensor_dump_input_file=None,
),
mock.patch.object(http_server, "get_exec", return_value=execution),
mock.patch.object(http_server, "freeze_gc", side_effect=timeline.worker),
mock.patch.object(
http_server,
"_freeze_gc_after_server_warmup",
side_effect=timeline.routed,
),
):
for worker_id in range(4):
timeline.reset_mock()
kwargs = http_server._multi_tokenizer_warmup_kwargs(
server_args, worker_id
)
http_server._wait_and_warmup(
**kwargs,
execute_warmup_func=lambda _: timeline.warmup() or True,
)
self.assertEqual(
timeline.mock_calls,
[
mock.call.warmup(),
mock.call.worker(f"Tokenizer worker {worker_id}", threshold),
mock.call.routed(server_args),
],
)
def test_router_cpu_metric_started_only_when_enabled(self):
for enabled in (False, True):
with self.subTest(enable_metrics=enabled):
+17
View File
@@ -1,3 +1,4 @@
import gc
import io
import sys
import unittest
@@ -11,6 +12,7 @@ from sglang.srt.utils.common import (
_get_device_sm_via_nvml,
_load_image,
flatten_arrays_to_int64_tensor,
freeze_gc,
get_device_sm_nvidia_smi,
get_nvidia_driver_version_str,
)
@@ -251,5 +253,20 @@ class TestGetDeviceSmViaNvml(CustomTestCase):
self.assertFalse(fake.initialized, "must not query NVML without the mapping")
class TestFreezeGc(CustomTestCase):
def setUp(self):
self.addCleanup(gc.set_threshold, *gc.get_threshold())
self.addCleanup(gc.unfreeze)
def test_preserves_default_threshold(self):
gc.set_threshold(700, 10, 10)
freeze_gc("test")
self.assertEqual(gc.get_threshold(), (700, 10, 10))
def test_applies_configured_threshold(self):
freeze_gc("test", [700, 10, 1000])
self.assertEqual(gc.get_threshold(), (700, 10, 1000))
if __name__ == "__main__":
unittest.main()