[Feat] sglang-server: Serve /flush_cache from the Rust frontend (#44766)

Co-authored-by: fei-xx <fei-xx@users.noreply.github.com>
Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-16 17:24:01 -07:00
committed by GitHub
co-authored by fei-xx shodoco
parent d8dd49d21d
commit b62a48ee15
6 changed files with 259 additions and 29 deletions
+13 -8
View File
@@ -1310,7 +1310,7 @@ class Scheduler(
self.flush_wrapper = SchedulerFlushWrapper(
flush_cache=self.flush_cache,
is_fully_idle=self.is_fully_idle,
ipc_channels=self.ipc_channels,
send_output=self.send_control_output,
)
self._last_logged_elastic_radix_namespace: Optional[str] = None
self.session_controller = SessionController(self.tree_cache)
@@ -2189,16 +2189,11 @@ class Scheduler(
output = self._request_dispatcher(recv_req)
if output is not None:
if self.rust_server is not None:
# Embedded Rust server: every control-request response goes
# back through the egress ring (the zmq tokenizer socket is
# not consumed); the Rust api_server shapes it per-endpoint.
self.rust_server.push_control_output(recv_req, output)
elif isinstance(output, RpcReqOutput):
if self.rust_server is None and isinstance(output, RpcReqOutput):
if self.ipc_channels.recv_from_rpc is not None:
sock_send(self.ipc_channels.recv_from_rpc, output)
else:
self.ipc_channels.send_to_tokenizer.send_output(output, recv_req)
self.send_control_output(output, recv_req)
self.flush_wrapper.check_pending()
if self.external_corpus_manager is not None:
@@ -2211,6 +2206,16 @@ class Scheduler(
self._event_loop_should_restart = False
raise role_switch.PdRoleSwitchRestart()
def send_control_output(self, output, recv_req) -> None:
"""Answer a control request, at dispatch or later (a deferred flush)."""
if self.rust_server is not None:
# Embedded Rust server: every control-request response goes
# back through the egress ring (the zmq tokenizer socket is
# not consumed); the Rust api_server shapes it per-endpoint.
self.rust_server.push_control_output(recv_req, output)
else:
self.ipc_channels.send_to_tokenizer.send_output(output, recv_req)
@staticmethod
def _tokenized_requests(recv_req):
if isinstance(
@@ -3,9 +3,6 @@ import time
from typing import Callable, Optional, Tuple
from sglang.srt.managers.io_struct import FlushCacheReqInput, FlushCacheReqOutput
from sglang.srt.managers.scheduler_components.ipc_channels import (
SchedulerIpcChannels,
)
class SchedulerFlushWrapper:
@@ -14,11 +11,12 @@ class SchedulerFlushWrapper:
*,
flush_cache: Callable[[], bool],
is_fully_idle: Callable[[], bool],
ipc_channels: SchedulerIpcChannels,
send_output: Callable[[FlushCacheReqOutput, FlushCacheReqInput], None],
) -> None:
self._flush_cache = flush_cache
self._is_fully_idle = is_fully_idle
self._ipc_channels = ipc_channels
# Answers a deferred flush through the scheduler's control-reply path.
self._send_output = send_output
self._pending: Optional[Tuple[FlushCacheReqInput, float]] = None
def handle(self, recv_req: FlushCacheReqInput) -> Optional[FlushCacheReqOutput]:
@@ -47,9 +45,7 @@ class SchedulerFlushWrapper:
if self._is_fully_idle():
success = self._flush_cache()
self._pending = None
self._ipc_channels.send_to_tokenizer.send_output(
FlushCacheReqOutput(success=success), pending_req
)
self._send_output(FlushCacheReqOutput(success=success), pending_req)
return
if time.monotonic() >= deadline:
@@ -57,7 +53,7 @@ class SchedulerFlushWrapper:
"Deferred flush_cache timed out while waiting for idle state."
)
self._pending = None
self._ipc_channels.send_to_tokenizer.send_output(
self._send_output(
FlushCacheReqOutput(
success=False, message="Timed out waiting for idle state."
),