[gRPC] Stream engine state changes (#39915)

Signed-off-by: William Arnold <7565007+Aphoh@users.noreply.github.com>
Co-authored-by: ishandhanani <82981111+ishandhanani@users.noreply.github.com>
This commit is contained in:
William Arnold
2026-09-17 19:17:25 -07:00
committed by GitHub
co-authored by ishandhanani
parent a407915c17
commit 0214954f26
6 changed files with 273 additions and 38 deletions
@@ -4,6 +4,7 @@ import unittest
from types import SimpleNamespace
from sglang.srt.entrypoints.grpc_bridge import RuntimeHandle
from sglang.srt.managers.tokenizer_manager import ServerStatus, TokenizerManager
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -43,18 +44,6 @@ def _make_runtime_handle(responses):
return handle
class TestNativeGrpcReadiness(CustomTestCase):
def test_readiness_comes_from_tokenizer_manager(self):
tokenizer_manager = SimpleNamespace(is_ready=lambda: True)
handle = RuntimeHandle.__new__(RuntimeHandle)
handle.tokenizer_manager = tokenizer_manager
self.assertTrue(handle.get_is_ready())
tokenizer_manager.is_ready = lambda: False
self.assertFalse(handle.get_is_ready())
class TestNativeGrpcParallelResponses(CustomTestCase):
def test_non_streaming_returns_every_choice_before_finishing(self):
callback = _RecordingCallback()
@@ -123,5 +112,51 @@ class TestNativeGrpcParallelResponses(CustomTestCase):
self.assertEqual([call[1] for call in callback.calls], [False, False, True])
class TestEngineStateNotifications(CustomTestCase):
def setUp(self):
self.manager = TokenizerManager.__new__(TokenizerManager)
self.manager._engine_state_changed_callback = None
self.manager._server_status = ServerStatus.Starting
self.manager._gracefully_exit = False
self.manager._is_pause = False
self.notifications = 0
self.manager.set_engine_state_changed_callback(self._notify)
def _notify(self):
self.notifications += 1
def test_observable_state_notifies_only_on_changes(self):
self.manager.is_pause = False
self.manager.server_status = ServerStatus.Starting
self.manager.gracefully_exit = False
self.assertEqual(self.notifications, 0)
self.manager.is_pause = True
self.manager.server_status = ServerStatus.Up
self.manager.gracefully_exit = True
self.assertEqual(self.notifications, 3)
def test_runtime_handle_registers_callback_with_manager(self):
handle = RuntimeHandle.__new__(RuntimeHandle)
handle.tokenizer_manager = self.manager
callback = object()
handle.set_engine_state_changed_callback(callback)
self.assertIs(self.manager._engine_state_changed_callback, callback)
def test_graceful_exit_notifies_and_changes_computed_health(self):
handle = RuntimeHandle.__new__(RuntimeHandle)
handle.tokenizer_manager = self.manager
self.manager.server_status = ServerStatus.Up
self.notifications = 0
self.assertTrue(handle.health_check())
self.manager.gracefully_exit = True
self.assertEqual(self.notifications, 1)
self.assertFalse(handle.health_check())
if __name__ == "__main__":
unittest.main(verbosity=2)