[Fix] srt: Apply --gc-threshold in every server process and freeze every tokenizer worker (#44785)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: joshuadeng <joshuadeng@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
joshuadeng
parent
d230f2c35b
commit
d8dd49d21d
@@ -27,6 +27,7 @@ import threading
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from functools import partial
|
||||
from http import HTTPStatus
|
||||
from typing import (
|
||||
Annotated,
|
||||
@@ -183,6 +184,7 @@ from sglang.srt.utils import (
|
||||
add_prometheus_track_response_middleware,
|
||||
build_server_info,
|
||||
delete_directory,
|
||||
freeze_gc,
|
||||
get_bool_env_var,
|
||||
is_mps,
|
||||
kill_process_tree,
|
||||
@@ -295,8 +297,9 @@ async def lifespan(fast_api_app: FastAPI):
|
||||
else:
|
||||
# Initialize multi-tokenizer support for worker processes
|
||||
server_args = await init_multi_tokenizer()
|
||||
warmup_thread_kwargs = dict(server_args=server_args)
|
||||
thread_label = f"MultiTokenizer-{_global_state.tokenizer_manager.worker_id}"
|
||||
worker_id = _global_state.tokenizer_manager.worker_id
|
||||
warmup_thread_kwargs = _multi_tokenizer_warmup_kwargs(server_args, worker_id)
|
||||
thread_label = f"MultiTokenizer-{worker_id}"
|
||||
|
||||
# Add prometheus middleware
|
||||
if get_observability().enable_metrics:
|
||||
@@ -2459,6 +2462,19 @@ def _execute_server_warmup(server_args: ServerArgs):
|
||||
return success
|
||||
|
||||
|
||||
def _tokenizer_worker_gc_callback(worker_id: int, gc_threshold) -> Callable[[], None]:
|
||||
return partial(freeze_gc, f"Tokenizer worker {worker_id}", gc_threshold)
|
||||
|
||||
|
||||
def _multi_tokenizer_warmup_kwargs(server_args: ServerArgs, worker_id: int):
|
||||
return dict(
|
||||
server_args=server_args,
|
||||
post_warmup_callback=_tokenizer_worker_gc_callback(
|
||||
worker_id, server_args.gc_threshold
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _freeze_gc_after_server_warmup(server_args: ServerArgs):
|
||||
# Freeze GC after server warmup so static objects skip future GC gen2 collection.
|
||||
# Use /freeze_gc to freeze scheduler and detokenizer as well.
|
||||
@@ -2482,6 +2498,7 @@ def _wait_and_warmup(
|
||||
server_args: ServerArgs,
|
||||
launch_callback: Optional[Callable[[], None]] = None,
|
||||
execute_warmup_func: Callable = _execute_server_warmup,
|
||||
post_warmup_callback: Optional[Callable[[], None]] = None,
|
||||
):
|
||||
if get_model().checkpoint_engine_wait_weights_before_ready:
|
||||
_wait_weights_ready()
|
||||
@@ -2500,6 +2517,8 @@ def _wait_and_warmup(
|
||||
else:
|
||||
_global_state.tokenizer_manager.server_status = ServerStatus.Up
|
||||
|
||||
if post_warmup_callback is not None:
|
||||
post_warmup_callback()
|
||||
_freeze_gc_after_server_warmup(server_args)
|
||||
|
||||
# The server is ready for requests
|
||||
|
||||
@@ -511,7 +511,7 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
|
||||
)
|
||||
|
||||
def handle_freeze_gc_req(self, recv_req: FreezeGCReq):
|
||||
freeze_gc("Detokenizer Manager")
|
||||
freeze_gc("Detokenizer Manager", recv_req.gc_threshold)
|
||||
return None
|
||||
|
||||
def handle_configure_logging_req(self, recv_req: ConfigureLoggingReq):
|
||||
|
||||
@@ -2392,7 +2392,7 @@ class ProfileReqOutput(BaseReq, kw_only=True):
|
||||
|
||||
|
||||
class FreezeGCReq(BaseReq, kw_only=True):
|
||||
pass
|
||||
gc_threshold: Optional[List[int]] = None
|
||||
|
||||
|
||||
class ShutdownReq(BaseReq, kw_only=True):
|
||||
|
||||
@@ -6015,7 +6015,7 @@ class Scheduler(
|
||||
|
||||
def handle_freeze_gc(self, recv_req: FreezeGCReq):
|
||||
"""Handle freeze_gc request: freeze scheduler's GC and forward to detokenizer."""
|
||||
freeze_gc("Scheduler")
|
||||
freeze_gc("Scheduler", recv_req.gc_threshold)
|
||||
self.ipc_channels.send_to_detokenizer.send_output(recv_req, recv_req)
|
||||
return None
|
||||
|
||||
|
||||
@@ -2434,8 +2434,9 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
|
||||
async def freeze_gc(self):
|
||||
"""Send a freeze_gc message to the scheduler first, then freeze locally."""
|
||||
self._dispatch_to_scheduler(FreezeGCReq())
|
||||
freeze_gc("Tokenizer Manager")
|
||||
gc_threshold = get_device().gc_threshold
|
||||
self._dispatch_to_scheduler(FreezeGCReq(gc_threshold=gc_threshold))
|
||||
freeze_gc("Tokenizer Manager", gc_threshold)
|
||||
return None
|
||||
|
||||
def create_abort_task(self, obj: GenerateReqInput):
|
||||
|
||||
@@ -4455,7 +4455,9 @@ def configure_gc_warning(warn_threshold_secs):
|
||||
gc.callbacks.append(gc_callback)
|
||||
|
||||
|
||||
def freeze_gc(context: str):
|
||||
def freeze_gc(context: str, gc_threshold: Optional[Sequence[int]] = None):
|
||||
if gc_threshold is not None:
|
||||
gc.set_threshold(*gc_threshold)
|
||||
g0_before, g1_before, g2_before = gc_object_counts()
|
||||
gc.freeze()
|
||||
g0_after, g1_after, g2_after = gc_object_counts()
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import pickle
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
from unittest.mock import DEFAULT, MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
@@ -14,6 +15,7 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.beam_search.types import BeamSearchSequence
|
||||
from sglang.srt.entrypoints import http_server
|
||||
from sglang.srt.managers import multi_tokenizer_mixin
|
||||
from sglang.srt.managers.io_struct import (
|
||||
AbortReq,
|
||||
@@ -35,6 +37,7 @@ from sglang.srt.managers.multi_tokenizer_mixin import (
|
||||
_send_output_to_tokenizer_workers,
|
||||
get_tokenizer_worker_class,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context
|
||||
|
||||
register_cpu_ci(est_time=12, suite="base-a-test-cpu")
|
||||
|
||||
@@ -200,6 +203,44 @@ def _make_router() -> MultiDetokenizerRouter:
|
||||
|
||||
|
||||
class TestMultiTokenizerMixin(CustomTestCase):
|
||||
def test_each_tokenizer_worker_freezes_locally(self):
|
||||
threshold = [700, 10, 1000]
|
||||
server_args = mock.Mock(gc_threshold=threshold)
|
||||
timeline = mock.Mock()
|
||||
execution = mock.Mock(moe=mock.Mock(is_ep_scale_joiner=False))
|
||||
with (
|
||||
get_context().override_server_args(
|
||||
checkpoint_engine_wait_weights_before_ready=False,
|
||||
delete_ckpt_after_loading=False,
|
||||
skip_server_warmup=False,
|
||||
debug_tensor_dump_input_file=None,
|
||||
),
|
||||
mock.patch.object(http_server, "get_exec", return_value=execution),
|
||||
mock.patch.object(http_server, "freeze_gc", side_effect=timeline.worker),
|
||||
mock.patch.object(
|
||||
http_server,
|
||||
"_freeze_gc_after_server_warmup",
|
||||
side_effect=timeline.routed,
|
||||
),
|
||||
):
|
||||
for worker_id in range(4):
|
||||
timeline.reset_mock()
|
||||
kwargs = http_server._multi_tokenizer_warmup_kwargs(
|
||||
server_args, worker_id
|
||||
)
|
||||
http_server._wait_and_warmup(
|
||||
**kwargs,
|
||||
execute_warmup_func=lambda _: timeline.warmup() or True,
|
||||
)
|
||||
self.assertEqual(
|
||||
timeline.mock_calls,
|
||||
[
|
||||
mock.call.warmup(),
|
||||
mock.call.worker(f"Tokenizer worker {worker_id}", threshold),
|
||||
mock.call.routed(server_args),
|
||||
],
|
||||
)
|
||||
|
||||
def test_router_cpu_metric_started_only_when_enabled(self):
|
||||
for enabled in (False, True):
|
||||
with self.subTest(enable_metrics=enabled):
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import gc
|
||||
import io
|
||||
import sys
|
||||
import unittest
|
||||
@@ -11,6 +12,7 @@ from sglang.srt.utils.common import (
|
||||
_get_device_sm_via_nvml,
|
||||
_load_image,
|
||||
flatten_arrays_to_int64_tensor,
|
||||
freeze_gc,
|
||||
get_device_sm_nvidia_smi,
|
||||
get_nvidia_driver_version_str,
|
||||
)
|
||||
@@ -251,5 +253,20 @@ class TestGetDeviceSmViaNvml(CustomTestCase):
|
||||
self.assertFalse(fake.initialized, "must not query NVML without the mapping")
|
||||
|
||||
|
||||
class TestFreezeGc(CustomTestCase):
|
||||
def setUp(self):
|
||||
self.addCleanup(gc.set_threshold, *gc.get_threshold())
|
||||
self.addCleanup(gc.unfreeze)
|
||||
|
||||
def test_preserves_default_threshold(self):
|
||||
gc.set_threshold(700, 10, 10)
|
||||
freeze_gc("test")
|
||||
self.assertEqual(gc.get_threshold(), (700, 10, 10))
|
||||
|
||||
def test_applies_configured_threshold(self):
|
||||
freeze_gc("test", [700, 10, 1000])
|
||||
self.assertEqual(gc.get_threshold(), (700, 10, 1000))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user