[Feat] sglang-server: Serve /flush_cache from the Rust frontend (#44766)
Co-authored-by: fei-xx <fei-xx@users.noreply.github.com> Co-authored-by: shodoco <shodoco@users.noreply.github.com>
This commit is contained in:
co-authored by
fei-xx
shodoco
parent
d8dd49d21d
commit
b62a48ee15
@@ -1310,7 +1310,7 @@ class Scheduler(
|
||||
self.flush_wrapper = SchedulerFlushWrapper(
|
||||
flush_cache=self.flush_cache,
|
||||
is_fully_idle=self.is_fully_idle,
|
||||
ipc_channels=self.ipc_channels,
|
||||
send_output=self.send_control_output,
|
||||
)
|
||||
self._last_logged_elastic_radix_namespace: Optional[str] = None
|
||||
self.session_controller = SessionController(self.tree_cache)
|
||||
@@ -2189,16 +2189,11 @@ class Scheduler(
|
||||
|
||||
output = self._request_dispatcher(recv_req)
|
||||
if output is not None:
|
||||
if self.rust_server is not None:
|
||||
# Embedded Rust server: every control-request response goes
|
||||
# back through the egress ring (the zmq tokenizer socket is
|
||||
# not consumed); the Rust api_server shapes it per-endpoint.
|
||||
self.rust_server.push_control_output(recv_req, output)
|
||||
elif isinstance(output, RpcReqOutput):
|
||||
if self.rust_server is None and isinstance(output, RpcReqOutput):
|
||||
if self.ipc_channels.recv_from_rpc is not None:
|
||||
sock_send(self.ipc_channels.recv_from_rpc, output)
|
||||
else:
|
||||
self.ipc_channels.send_to_tokenizer.send_output(output, recv_req)
|
||||
self.send_control_output(output, recv_req)
|
||||
|
||||
self.flush_wrapper.check_pending()
|
||||
if self.external_corpus_manager is not None:
|
||||
@@ -2211,6 +2206,16 @@ class Scheduler(
|
||||
self._event_loop_should_restart = False
|
||||
raise role_switch.PdRoleSwitchRestart()
|
||||
|
||||
def send_control_output(self, output, recv_req) -> None:
|
||||
"""Answer a control request, at dispatch or later (a deferred flush)."""
|
||||
if self.rust_server is not None:
|
||||
# Embedded Rust server: every control-request response goes
|
||||
# back through the egress ring (the zmq tokenizer socket is
|
||||
# not consumed); the Rust api_server shapes it per-endpoint.
|
||||
self.rust_server.push_control_output(recv_req, output)
|
||||
else:
|
||||
self.ipc_channels.send_to_tokenizer.send_output(output, recv_req)
|
||||
|
||||
@staticmethod
|
||||
def _tokenized_requests(recv_req):
|
||||
if isinstance(
|
||||
|
||||
@@ -3,9 +3,6 @@ import time
|
||||
from typing import Callable, Optional, Tuple
|
||||
|
||||
from sglang.srt.managers.io_struct import FlushCacheReqInput, FlushCacheReqOutput
|
||||
from sglang.srt.managers.scheduler_components.ipc_channels import (
|
||||
SchedulerIpcChannels,
|
||||
)
|
||||
|
||||
|
||||
class SchedulerFlushWrapper:
|
||||
@@ -14,11 +11,12 @@ class SchedulerFlushWrapper:
|
||||
*,
|
||||
flush_cache: Callable[[], bool],
|
||||
is_fully_idle: Callable[[], bool],
|
||||
ipc_channels: SchedulerIpcChannels,
|
||||
send_output: Callable[[FlushCacheReqOutput, FlushCacheReqInput], None],
|
||||
) -> None:
|
||||
self._flush_cache = flush_cache
|
||||
self._is_fully_idle = is_fully_idle
|
||||
self._ipc_channels = ipc_channels
|
||||
# Answers a deferred flush through the scheduler's control-reply path.
|
||||
self._send_output = send_output
|
||||
self._pending: Optional[Tuple[FlushCacheReqInput, float]] = None
|
||||
|
||||
def handle(self, recv_req: FlushCacheReqInput) -> Optional[FlushCacheReqOutput]:
|
||||
@@ -47,9 +45,7 @@ class SchedulerFlushWrapper:
|
||||
if self._is_fully_idle():
|
||||
success = self._flush_cache()
|
||||
self._pending = None
|
||||
self._ipc_channels.send_to_tokenizer.send_output(
|
||||
FlushCacheReqOutput(success=success), pending_req
|
||||
)
|
||||
self._send_output(FlushCacheReqOutput(success=success), pending_req)
|
||||
return
|
||||
|
||||
if time.monotonic() >= deadline:
|
||||
@@ -57,7 +53,7 @@ class SchedulerFlushWrapper:
|
||||
"Deferred flush_cache timed out while waiting for idle state."
|
||||
)
|
||||
self._pending = None
|
||||
self._ipc_channels.send_to_tokenizer.send_output(
|
||||
self._send_output(
|
||||
FlushCacheReqOutput(
|
||||
success=False, message="Timed out waiting for idle state."
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user