Files
sglang/rust/sglang-bench/tests/grpc.rs
T
2026-10-23 17:10:23 +08:00

208 lines
7.8 KiB
Rust

//! End to end over the api.v1 gRPC transport: a tonic mock of the server's
//! service answers model info, server info and a token-id generate stream,
//! and the client is driven through its public entry point with ids on both
//! ends and no tokenizer. This is the one path the unit tests cannot reach,
//! since they stop at the frame parser.
use std::net::TcpListener;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use futures::Stream;
use sglang_api_types::api::v1 as api;
use sglang_api_types::api::v1::sglang_service_server::{SglangService, SglangServiceServer};
use tokio_stream::wrappers::TcpListenerStream;
use tonic::{Request, Response, Status};
/// Serves `tokens_per_request` ids per generate call, two per frame, with an
/// empty `text` as a `--skip-tokenizer-init` server would.
struct Mock {
tokens_per_request: usize,
/// Requests that arrived with `input_ids` rather than `text`.
id_requests: Arc<AtomicUsize>,
}
type ItemStream = Pin<Box<dyn Stream<Item = Result<api::GenerateStreamItem, Status>> + Send>>;
#[tonic::async_trait]
impl SglangService for Mock {
type GenerateStream = ItemStream;
async fn generate(
&self,
request: Request<api::GenerateRequest>,
) -> Result<Response<Self::GenerateStream>, Status> {
let request = request.into_inner();
if request.input_ids.is_some() && request.text.is_none() {
self.id_requests.fetch_add(1, Ordering::Relaxed);
}
let want = self.tokens_per_request;
let mut frames = Vec::new();
let mut ids: Vec<i64> = Vec::new();
while ids.len() < want {
ids.push(ids.len() as i64 + 100);
if ids.len() < want {
ids.push(ids.len() as i64 + 100);
}
let done = ids.len() == want;
frames.push(Ok(api::GenerateStreamItem {
item: Some(api::generate_stream_item::Item::Frame(
api::GenerateResponse {
text: String::new(),
meta_info: Some(api::GenerateMetaInfo {
id: "mock".into(),
prompt_tokens: 9,
completion_tokens: ids.len() as u64,
finish_reason: done.then_some(api::FinishReason {
kind: Some(api::finish_reason::Kind::Length(api::FinishLength {
length: Some(want as u64),
})),
}),
..Default::default()
}),
output_ids: Some(api::TokenIds { ids: ids.clone() }),
index: None,
prompt_token_ids: None,
},
)),
}));
}
Ok(Response::new(Box::pin(futures::stream::iter(frames))))
}
async fn health_check(
&self,
_: Request<api::HealthCheckRequest>,
) -> Result<Response<api::HealthCheckResponse>, Status> {
Ok(Response::new(api::HealthCheckResponse { healthy: true }))
}
async fn get_model_info(
&self,
_: Request<api::GetModelInfoRequest>,
) -> Result<Response<api::GetModelInfoResponse>, Status> {
Ok(Response::new(api::GetModelInfoResponse {
model_path: "/models/mock".into(),
served_model_name: "mock-model".into(),
tokenizer_path: "/models/mock".into(),
is_generation: true,
..Default::default()
}))
}
async fn get_server_info(
&self,
_: Request<api::GetServerInfoRequest>,
) -> Result<Response<api::GetServerInfoResponse>, Status> {
Ok(Response::new(api::GetServerInfoResponse {
model_path: "/models/mock".into(),
served_model_name: "mock-model".into(),
tokenizer_path: "/models/mock".into(),
max_context_length: 4096,
max_total_num_tokens: Some(8192),
version: "test".into(),
internal_states: vec![r#"{"avg_spec_accept_length": 2.5}"#.into()],
..Default::default()
}))
}
async fn list_models(
&self,
_: Request<api::ListModelsRequest>,
) -> Result<Response<api::ListModelsResponse>, Status> {
Err(Status::unimplemented("list_models"))
}
async fn detokenize(
&self,
_: Request<api::DetokenizeRequest>,
) -> Result<Response<api::DetokenizeResponse>, Status> {
Err(Status::unimplemented("detokenize"))
}
async fn abort(
&self,
_: Request<api::AbortRequest>,
) -> Result<Response<api::AbortResponse>, Status> {
Err(Status::unimplemented("abort"))
}
}
/// Bind on a free port and serve the mock on its own runtime thread, so the
/// client's own Tokio runtime is what the test exercises.
fn serve_mock(mock: Mock) -> u16 {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
listener.set_nonblocking(true).unwrap();
std::thread::spawn(move || {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(async {
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
tonic::transport::Server::builder()
.add_service(SglangServiceServer::new(mock))
.serve_with_incoming(TcpListenerStream::new(listener))
.await
.unwrap();
});
});
port
}
/// The whole client, ids in and ids out, with no tokenizer loaded: every
/// request completes, tokens are counted from the ids the frames carry, the
/// accept length is read from the gRPC server info, and the per-run result
/// lands in the file with the Python client's keys.
#[test]
fn grpc_backend_runs_end_to_end_on_token_ids() {
let id_requests = Arc::new(AtomicUsize::new(0));
let port = serve_mock(Mock {
tokens_per_request: 6,
id_requests: id_requests.clone(),
});
let output_file =
std::env::temp_dir().join(format!("sglang-bench-grpc-{}.jsonl", std::process::id()));
let args: sglang_bench::Args = serde_json::from_value(serde_json::json!({
"backend": "sglang",
"protocol": "grpc",
"base_url": format!("http://127.0.0.1:{port}"),
"dataset_name": "random-ids",
"tokenize_prompt": true,
"vocab_size": 1000,
"num_prompts": 12,
"random_input_len": 8,
"random_output_len": 6,
"random_range_ratio": 1.0,
"max_concurrency": 4,
"warmup_requests": 1,
"ready_check_timeout_sec": 10,
"disable_tqdm": true,
"output_file": output_file.to_str().unwrap(),
}))
.unwrap();
let result = sglang_bench::run_blocking(args).expect("benchmark runs");
assert_eq!(result["completed"], 12);
assert_eq!(result["backend"], "sglang");
assert_eq!(result["protocol"], "grpc");
// Every prompt is 8 ids and every reply 6 ids; no text crossed the wire,
// so the retokenized total is 0 by design.
assert_eq!(result["total_input_tokens"], 12 * 8);
assert_eq!(result["total_output_tokens"], 12 * 6);
assert_eq!(result["total_output_tokens_retokenized"], 0);
assert_eq!(result["accept_length"], 2.5);
assert!(result["mean_ttft_ms"].as_f64().unwrap() > 0.0);
// 12 measured requests plus the warmup, all as input_ids.
assert_eq!(id_requests.load(Ordering::Relaxed), 13);
let written = std::fs::read_to_string(&output_file).unwrap();
let line: serde_json::Value = serde_json::from_str(written.trim()).unwrap();
assert_eq!(line["completed"], 12);
std::fs::remove_file(&output_file).unwrap();
}