Files
sglang/sgl-router/src/server/grpc.rs
T

336 lines
13 KiB
Rust

// SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
// SPDX-License-Identifier: Apache-2.0
//! The engine's native gRPC interface (`sglang.runtime.v1.SglangService`),
//! served so a gRPC client can point at the router or at an engine. Each served
//! RPC is routed like its HTTP route and forwarded to the same RPC on an engine.
use crate::proxy::grpc::{
embed_to_text, generate_to_text, text_to_embed, text_to_generate, GrpcResponse, OpenAiRpc,
OpenAiUnaryRpc, Replies,
};
use crate::server::app::pod_id;
use crate::server::app_context::AppContext;
use crate::server::error::ApiError;
use crate::server::metrics::{outcome_from_status, RequestLogContext};
use crate::server::routes::chat::{
prompt_ids, route_grpc, route_typed, Endpoint, MAX_CHAT_BODY_BYTES,
};
use crate::workers::GRPC_MAX_MESSAGE_BYTES;
use axum::http::{HeaderMap, HeaderName, HeaderValue};
use bytes::Bytes;
use futures::StreamExt;
use sglang_api_types::runtime::v1 as proto;
use sglang_api_types::runtime::v1::sglang_service_server::{SglangService, SglangServiceServer};
use std::collections::HashMap;
use std::future::Future;
use std::sync::Arc;
use std::time::Instant;
use tokio::net::TcpListener;
use tonic::{Request, Response, Status};
type RpcResult<T> = Result<Response<T>, Status>;
fn unserved<T>(rpc: &str) -> RpcResult<T> {
Err(Status::unimplemented(format!(
"{rpc} is not served by sgl-router"
)))
}
/// Serve until `shutdown` resolves, then let open calls finish.
pub async fn serve(
listener: TcpListener,
ctx: Arc<AppContext>,
shutdown: impl Future<Output = ()> + Send + 'static,
) -> anyhow::Result<()> {
let service = SglangServiceServer::new(RouterGrpc { ctx })
.max_decoding_message_size(MAX_CHAT_BODY_BYTES)
.max_encoding_message_size(GRPC_MAX_MESSAGE_BYTES);
tonic::transport::Server::builder()
.add_service(service)
.serve_with_incoming_shutdown(
tokio_stream::wrappers::TcpListenerStream::new(listener),
shutdown,
)
.await?;
Ok(())
}
struct RouterGrpc {
ctx: Arc<AppContext>,
}
type Routed<T> = (Result<GrpcResponse<T>, ApiError>, Option<RequestLogContext>);
/// Client metadata plus `trace_headers`, which the engine reads as request
/// headers, so header-driven routing (sticky keys, sessions, SLOs) works as over HTTP.
fn request_headers<T>(
request: Request<T>,
trace_headers: impl Fn(&T) -> &HashMap<String, String>,
) -> (HeaderMap, T) {
let (metadata, _, request) = request.into_parts();
let mut headers = metadata.into_headers();
for (name, value) in trace_headers(&request) {
if let (Ok(name), Ok(value)) = (
HeaderName::try_from(name.as_str()),
HeaderValue::try_from(value.as_str()),
) {
headers.insert(name, value);
}
}
(headers, request)
}
/// The access-log line the HTTP edge middleware writes, for a gRPC call.
fn log_call(
route: &str,
headers: &HeaderMap,
status: u16,
log_context: Option<&RequestLogContext>,
started: Instant,
) {
tracing::info!(
pod_id = %pod_id(),
request_id = headers
.get("x-request-id")
.and_then(|v| v.to_str().ok())
.unwrap_or("-"),
path = route,
status,
outcome = log_context
.map(|c| c.outcome)
.unwrap_or_else(|| outcome_from_status(status))
.as_str(),
worker = log_context.map(|c| c.worker_url.as_str()).unwrap_or(""),
engine_rid = log_context
.and_then(|c| c.engine_rid.as_deref())
.unwrap_or(""),
model = log_context.map(|c| c.model_id.as_str()).unwrap_or(""),
stream = log_context.is_some_and(|c| c.streaming),
latency_ms = started.elapsed().as_millis() as u64,
"grpc_request",
);
}
impl RouterGrpc {
/// Run a routed call with the HTTP edge's metrics, in-flight count and access log.
async fn finish<T: Send + 'static>(
&self,
rpc: &str,
headers: HeaderMap,
call: impl Future<Output = Routed<T>>,
) -> Result<Replies<T>, Status> {
let started = Instant::now();
let route = format!("/sglang.runtime.v1.SglangService/{rpc}");
let metrics = &self.ctx.metrics;
metrics.record_ingress(&route, "POST");
let inflight = self.ctx.inflight_http.enter();
let (result, log_context) = call.await;
let status = match &result {
Ok(response) => response.status.as_u16(),
Err(error) => error.status_code().as_u16(),
};
metrics.record_response(&route, "POST", status);
log_call(&route, &headers, status, log_context.as_ref(), started);
let replies = result.map_err(ApiError::into_grpc_status)?.replies;
// Counted until the client's stream ends, as an HTTP body is.
Ok(replies
.map(move |reply| {
let _ = &inflight;
reply
})
.boxed())
}
/// [`Self::finish`] for a unary RPC, whose one reply the engine sent.
async fn finish_unary<T: Send + 'static>(
&self,
rpc: &str,
headers: HeaderMap,
call: impl Future<Output = Routed<T>>,
) -> RpcResult<T> {
let mut replies = self.finish(rpc, headers, call).await?;
let reply = replies.next().await;
reply
.unwrap_or_else(|| Err(Status::internal("the engine sent no reply")))
.map(Response::new)
}
async fn openai(
&self,
rpc: OpenAiRpc,
endpoint: Endpoint,
request: Request<proto::OpenAiRequest>,
) -> RpcResult<Replies<proto::OpenAiStreamChunk>> {
let (headers, request) = request_headers(request, |r| &r.trace_headers);
let body = Bytes::from(request.json_body);
let call = route_grpc(&self.ctx, endpoint, rpc, headers.clone(), body);
let name = match rpc {
OpenAiRpc::ChatComplete => "ChatComplete",
OpenAiRpc::Complete => "Complete",
};
self.finish(name, headers, call).await.map(Response::new)
}
/// Tokenized here, as `/generate` tokenizes `text`, then sent as `Generate` for its text.
async fn text_generate(
&self,
request: Request<proto::TextGenerateRequest>,
) -> Result<Replies<proto::TextGenerateResponse>, Status> {
let (headers, request) = request_headers(request, |r| &r.trace_headers);
let ids = (request.return_text_in_logprobs != Some(true))
.then(|| prompt_ids(&self.ctx, Endpoint::Generate, &request.text))
.flatten();
let Some(ids) = ids else {
let call = route_typed(&self.ctx, Endpoint::Generate, request, headers.clone());
return self.finish("TextGenerate", headers, call).await;
};
let request = text_to_generate(request, ids);
let call = route_typed(&self.ctx, Endpoint::Generate, request, headers.clone());
let replies = self.finish("TextGenerate", headers, call).await?;
Ok(replies.map(|reply| reply.map(generate_to_text)).boxed())
}
/// Tokenized here, as `/v1/embeddings` tokenizes text, then sent as `Embed`.
async fn text_embed(
&self,
request: Request<proto::TextEmbedRequest>,
) -> RpcResult<proto::TextEmbedResponse> {
let (headers, request) = request_headers(request, |r| &r.trace_headers);
let Some(ids) = prompt_ids(&self.ctx, Endpoint::Embeddings, &request.text) else {
let call = route_typed(&self.ctx, Endpoint::Embeddings, request, headers.clone());
return self.finish_unary("TextEmbed", headers, call).await;
};
let call = route_typed(
&self.ctx,
Endpoint::Embeddings,
text_to_embed(request, ids),
headers.clone(),
);
let reply = self.finish_unary("TextEmbed", headers, call).await?;
Ok(reply.map(embed_to_text))
}
async fn openai_unary(
&self,
rpc: OpenAiUnaryRpc,
endpoint: Endpoint,
request: Request<proto::OpenAiRequest>,
) -> RpcResult<proto::OpenAiResponse> {
let (headers, request) = request_headers(request, |r| &r.trace_headers);
let body = Bytes::from(request.json_body);
let call = route_grpc(&self.ctx, endpoint, rpc, headers.clone(), body);
let name = match rpc {
OpenAiUnaryRpc::Embed => "OpenAIEmbed",
OpenAiUnaryRpc::Classify => "OpenAIClassify",
OpenAiUnaryRpc::Rerank => "Rerank",
};
self.finish_unary(name, headers, call).await
}
}
/// Implements `SglangService` from the served items; every listed RPC answers
/// `Unimplemented`. A macro because `async_trait` must see the expanded methods.
macro_rules! sglang_service {
(
served { $($served:tt)* }
unserved { $($rpc:ident($req:ident) -> $resp:ident;)* }
unserved_streams { $($srpc:ident($sreq:ident) -> $stream:ident<$sresp:ident>;)* }
) => {
#[tonic::async_trait]
impl SglangService for RouterGrpc {
$($served)*
$(
async fn $rpc(&self, _: Request<proto::$req>) -> RpcResult<proto::$resp> {
unserved(stringify!($rpc))
}
)*
$(
async fn $srpc(&self, _: Request<proto::$sreq>) -> RpcResult<tonic::codegen::BoxStream<proto::$sresp>> {
unserved(stringify!($srpc))
}
)*
}
};
}
/// A typed RPC: prepared from its fields like its HTTP route, then sent as typed.
macro_rules! typed {
($self:ident, $request:ident, $endpoint:ident, $rpc:literal, $finish:ident) => {{
let (headers, request) = request_headers($request, |r| &r.trace_headers);
let call = route_typed(&$self.ctx, Endpoint::$endpoint, request, headers.clone());
$self.$finish($rpc, headers, call).await
}};
}
sglang_service! {
served {
async fn chat_complete(&self, request: Request<proto::OpenAiRequest>) -> RpcResult<Replies<proto::OpenAiStreamChunk>> {
self.openai(OpenAiRpc::ChatComplete, Endpoint::Chat, request).await
}
async fn complete(&self, request: Request<proto::OpenAiRequest>) -> RpcResult<Replies<proto::OpenAiStreamChunk>> {
self.openai(OpenAiRpc::Complete, Endpoint::Completions, request).await
}
async fn generate(&self, request: Request<proto::GenerateRequest>) -> RpcResult<Replies<proto::GenerateResponse>> {
typed!(self, request, Generate, "Generate", finish).map(Response::new)
}
async fn text_generate(&self, request: Request<proto::TextGenerateRequest>) -> RpcResult<Replies<proto::TextGenerateResponse>> {
self.text_generate(request).await.map(Response::new)
}
async fn embed(&self, request: Request<proto::EmbedRequest>) -> RpcResult<proto::EmbedResponse> {
typed!(self, request, Embeddings, "Embed", finish_unary)
}
async fn text_embed(&self, request: Request<proto::TextEmbedRequest>) -> RpcResult<proto::TextEmbedResponse> {
self.text_embed(request).await
}
async fn classify(&self, mut request: Request<proto::ClassifyRequest>) -> RpcResult<proto::ClassifyResponse> {
// A text prompt is sent as ids, as `/v1/classify` forwards it.
let classify = request.get_mut();
if classify.input_ids.is_empty() {
if let Some(ids) = prompt_ids(&self.ctx, Endpoint::Classify, &classify.text) {
(classify.input_ids, classify.text) = (ids, String::new());
}
}
typed!(self, request, Classify, "Classify", finish_unary)
}
async fn open_ai_embed(&self, request: Request<proto::OpenAiRequest>) -> RpcResult<proto::OpenAiResponse> {
self.openai_unary(OpenAiUnaryRpc::Embed, Endpoint::Embeddings, request).await
}
async fn open_ai_classify(&self, request: Request<proto::OpenAiRequest>) -> RpcResult<proto::OpenAiResponse> {
self.openai_unary(OpenAiUnaryRpc::Classify, Endpoint::Classify, request).await
}
async fn rerank(&self, request: Request<proto::OpenAiRequest>) -> RpcResult<proto::OpenAiResponse> {
self.openai_unary(OpenAiUnaryRpc::Rerank, Endpoint::Rerank, request).await
}
}
unserved {
tokenize(TokenizeRequest) -> TokenizeResponse;
detokenize(DetokenizeRequest) -> DetokenizeResponse;
health_check(HealthCheckRequest) -> HealthCheckResponse;
get_model_info(GetModelInfoRequest) -> GetModelInfoResponse;
get_server_info(GetServerInfoRequest) -> GetServerInfoResponse;
list_models(ListModelsRequest) -> ListModelsResponse;
get_load(GetLoadRequest) -> GetLoadResponse;
abort(AbortRequest) -> AbortResponse;
flush_cache(FlushCacheRequest) -> FlushCacheResponse;
pause_generation(PauseGenerationRequest) -> PauseGenerationResponse;
continue_generation(ContinueGenerationRequest) -> ContinueGenerationResponse;
score(OpenAiRequest) -> OpenAiResponse;
start_profile(StartProfileRequest) -> StartProfileResponse;
stop_profile(StopProfileRequest) -> StopProfileResponse;
update_weights_from_disk(UpdateWeightsRequest) -> UpdateWeightsResponse;
}
unserved_streams {
watch_engine_state(WatchEngineStateRequest) -> WatchEngineStateStream<EngineStateSnapshot>;
}
}