From d8dd49d21d56ea4cbe9b1ae98f4be37fdc412de7 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Fri, 16 Oct 2026 16:55:54 -0700 Subject: [PATCH] [Fix] srt: Apply --gc-threshold in every server process and freeze every tokenizer worker (#44785) Co-authored-by: metamergebot Co-authored-by: joshuadeng --- python/sglang/srt/entrypoints/http_server.py | 23 ++++++++++- .../srt/managers/detokenizer_manager.py | 2 +- python/sglang/srt/managers/io_struct.py | 2 +- python/sglang/srt/managers/scheduler.py | 2 +- .../sglang/srt/managers/tokenizer_manager.py | 5 ++- python/sglang/srt/utils/common.py | 4 +- .../managers/test_multi_tokenizer_mixin.py | 41 +++++++++++++++++++ test/registered/unit/utils/test_common.py | 17 ++++++++ 8 files changed, 88 insertions(+), 8 deletions(-) diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index a8ccfe74d2..5bb8ca4289 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -27,6 +27,7 @@ import threading import time import uuid from contextlib import asynccontextmanager +from functools import partial from http import HTTPStatus from typing import ( Annotated, @@ -183,6 +184,7 @@ from sglang.srt.utils import ( add_prometheus_track_response_middleware, build_server_info, delete_directory, + freeze_gc, get_bool_env_var, is_mps, kill_process_tree, @@ -295,8 +297,9 @@ async def lifespan(fast_api_app: FastAPI): else: # Initialize multi-tokenizer support for worker processes server_args = await init_multi_tokenizer() - warmup_thread_kwargs = dict(server_args=server_args) - thread_label = f"MultiTokenizer-{_global_state.tokenizer_manager.worker_id}" + worker_id = _global_state.tokenizer_manager.worker_id + warmup_thread_kwargs = _multi_tokenizer_warmup_kwargs(server_args, worker_id) + thread_label = f"MultiTokenizer-{worker_id}" # Add prometheus middleware if get_observability().enable_metrics: @@ -2459,6 +2462,19 @@ def _execute_server_warmup(server_args: ServerArgs): return success +def _tokenizer_worker_gc_callback(worker_id: int, gc_threshold) -> Callable[[], None]: + return partial(freeze_gc, f"Tokenizer worker {worker_id}", gc_threshold) + + +def _multi_tokenizer_warmup_kwargs(server_args: ServerArgs, worker_id: int): + return dict( + server_args=server_args, + post_warmup_callback=_tokenizer_worker_gc_callback( + worker_id, server_args.gc_threshold + ), + ) + + def _freeze_gc_after_server_warmup(server_args: ServerArgs): # Freeze GC after server warmup so static objects skip future GC gen2 collection. # Use /freeze_gc to freeze scheduler and detokenizer as well. @@ -2482,6 +2498,7 @@ def _wait_and_warmup( server_args: ServerArgs, launch_callback: Optional[Callable[[], None]] = None, execute_warmup_func: Callable = _execute_server_warmup, + post_warmup_callback: Optional[Callable[[], None]] = None, ): if get_model().checkpoint_engine_wait_weights_before_ready: _wait_weights_ready() @@ -2500,6 +2517,8 @@ def _wait_and_warmup( else: _global_state.tokenizer_manager.server_status = ServerStatus.Up + if post_warmup_callback is not None: + post_warmup_callback() _freeze_gc_after_server_warmup(server_args) # The server is ready for requests diff --git a/python/sglang/srt/managers/detokenizer_manager.py b/python/sglang/srt/managers/detokenizer_manager.py index 7ef28668fc..0e5f9a6d08 100644 --- a/python/sglang/srt/managers/detokenizer_manager.py +++ b/python/sglang/srt/managers/detokenizer_manager.py @@ -511,7 +511,7 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin): ) def handle_freeze_gc_req(self, recv_req: FreezeGCReq): - freeze_gc("Detokenizer Manager") + freeze_gc("Detokenizer Manager", recv_req.gc_threshold) return None def handle_configure_logging_req(self, recv_req: ConfigureLoggingReq): diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index e645e0230f..3148deb920 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -2392,7 +2392,7 @@ class ProfileReqOutput(BaseReq, kw_only=True): class FreezeGCReq(BaseReq, kw_only=True): - pass + gc_threshold: Optional[List[int]] = None class ShutdownReq(BaseReq, kw_only=True): diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 3fb96254ec..88b1a1d9cd 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -6015,7 +6015,7 @@ class Scheduler( def handle_freeze_gc(self, recv_req: FreezeGCReq): """Handle freeze_gc request: freeze scheduler's GC and forward to detokenizer.""" - freeze_gc("Scheduler") + freeze_gc("Scheduler", recv_req.gc_threshold) self.ipc_channels.send_to_detokenizer.send_output(recv_req, recv_req) return None diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index e8e4b450f9..79b82f4f8f 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -2434,8 +2434,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): async def freeze_gc(self): """Send a freeze_gc message to the scheduler first, then freeze locally.""" - self._dispatch_to_scheduler(FreezeGCReq()) - freeze_gc("Tokenizer Manager") + gc_threshold = get_device().gc_threshold + self._dispatch_to_scheduler(FreezeGCReq(gc_threshold=gc_threshold)) + freeze_gc("Tokenizer Manager", gc_threshold) return None def create_abort_task(self, obj: GenerateReqInput): diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index a66496264a..0e60c57001 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -4455,7 +4455,9 @@ def configure_gc_warning(warn_threshold_secs): gc.callbacks.append(gc_callback) -def freeze_gc(context: str): +def freeze_gc(context: str, gc_threshold: Optional[Sequence[int]] = None): + if gc_threshold is not None: + gc.set_threshold(*gc_threshold) g0_before, g1_before, g2_before = gc_object_counts() gc.freeze() g0_after, g1_after, g2_after = gc_object_counts() diff --git a/test/registered/unit/managers/test_multi_tokenizer_mixin.py b/test/registered/unit/managers/test_multi_tokenizer_mixin.py index b881cca211..23f1d99ba1 100644 --- a/test/registered/unit/managers/test_multi_tokenizer_mixin.py +++ b/test/registered/unit/managers/test_multi_tokenizer_mixin.py @@ -1,6 +1,7 @@ import pickle import unittest from types import SimpleNamespace +from unittest import mock from unittest.mock import DEFAULT, MagicMock, patch import numpy as np @@ -14,6 +15,7 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel maybe_stub_sgl_kernel() from sglang.srt.beam_search.types import BeamSearchSequence +from sglang.srt.entrypoints import http_server from sglang.srt.managers import multi_tokenizer_mixin from sglang.srt.managers.io_struct import ( AbortReq, @@ -35,6 +37,7 @@ from sglang.srt.managers.multi_tokenizer_mixin import ( _send_output_to_tokenizer_workers, get_tokenizer_worker_class, ) +from sglang.srt.runtime_context import get_context register_cpu_ci(est_time=12, suite="base-a-test-cpu") @@ -200,6 +203,44 @@ def _make_router() -> MultiDetokenizerRouter: class TestMultiTokenizerMixin(CustomTestCase): + def test_each_tokenizer_worker_freezes_locally(self): + threshold = [700, 10, 1000] + server_args = mock.Mock(gc_threshold=threshold) + timeline = mock.Mock() + execution = mock.Mock(moe=mock.Mock(is_ep_scale_joiner=False)) + with ( + get_context().override_server_args( + checkpoint_engine_wait_weights_before_ready=False, + delete_ckpt_after_loading=False, + skip_server_warmup=False, + debug_tensor_dump_input_file=None, + ), + mock.patch.object(http_server, "get_exec", return_value=execution), + mock.patch.object(http_server, "freeze_gc", side_effect=timeline.worker), + mock.patch.object( + http_server, + "_freeze_gc_after_server_warmup", + side_effect=timeline.routed, + ), + ): + for worker_id in range(4): + timeline.reset_mock() + kwargs = http_server._multi_tokenizer_warmup_kwargs( + server_args, worker_id + ) + http_server._wait_and_warmup( + **kwargs, + execute_warmup_func=lambda _: timeline.warmup() or True, + ) + self.assertEqual( + timeline.mock_calls, + [ + mock.call.warmup(), + mock.call.worker(f"Tokenizer worker {worker_id}", threshold), + mock.call.routed(server_args), + ], + ) + def test_router_cpu_metric_started_only_when_enabled(self): for enabled in (False, True): with self.subTest(enable_metrics=enabled): diff --git a/test/registered/unit/utils/test_common.py b/test/registered/unit/utils/test_common.py index 8b335a6511..ec5909678c 100644 --- a/test/registered/unit/utils/test_common.py +++ b/test/registered/unit/utils/test_common.py @@ -1,3 +1,4 @@ +import gc import io import sys import unittest @@ -11,6 +12,7 @@ from sglang.srt.utils.common import ( _get_device_sm_via_nvml, _load_image, flatten_arrays_to_int64_tensor, + freeze_gc, get_device_sm_nvidia_smi, get_nvidia_driver_version_str, ) @@ -251,5 +253,20 @@ class TestGetDeviceSmViaNvml(CustomTestCase): self.assertFalse(fake.initialized, "must not query NVML without the mapping") +class TestFreezeGc(CustomTestCase): + def setUp(self): + self.addCleanup(gc.set_threshold, *gc.get_threshold()) + self.addCleanup(gc.unfreeze) + + def test_preserves_default_threshold(self): + gc.set_threshold(700, 10, 10) + freeze_gc("test") + self.assertEqual(gc.get_threshold(), (700, 10, 10)) + + def test_applies_configured_threshold(self): + freeze_gc("test", [700, 10, 1000]) + self.assertEqual(gc.get_threshold(), (700, 10, 1000)) + + if __name__ == "__main__": unittest.main()