minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[gateway] Align /v1/loads and /model_info with sglang server; drop dead /rerank (#24167)

Browse Source 
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Kangyan-Zhou
2026-05-02 11:43:30 -07:00
committed by GitHub
co-authored by Claude Opus 4.7
parent e0474fdd9b
commit d74d9bda49
6 changed files with 10 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sgl-model-gateway/tests/api/api_endpoints_test.rs
-257
View File
@@ -1570,182 +1570,6 @@ mod rerank_tests {
use super::*;
// Note: RerankRequest and RerankResult are available for future use
#[tokio::test]
async fn test_rerank_success() {
let ctx = AppTestContext::new(vec![MockWorkerConfig {
port: 18105,
worker_type: WorkerType::Regular,
health_status: HealthStatus::Healthy,
response_delay_ms: 0,
fail_rate: 0.0,
}])
.await;
let app = ctx.create_app().await;
let payload = json!({
"query": "machine learning algorithms",
"documents": [
"Introduction to machine learning concepts",
"Deep learning neural networks tutorial"
],
"model": "test-rerank-model",
"top_k": 2,
"return_documents": true,
"rid": "test-request-123"
});
let req = Request::builder()
.method("POST")
.uri("/rerank")
.header(CONTENT_TYPE, "application/json")
.body(Body::from(serde_json::to_string(&payload).unwrap()))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert!(body_json.get("results").is_some());
assert!(body_json.get("model").is_some());
assert_eq!(body_json["model"], "test-rerank-model");
let results = body_json["results"].as_array().unwrap();
assert_eq!(results.len(), 2);
assert!(results[0]["score"].as_f64().unwrap() >= results[1]["score"].as_f64().unwrap());
ctx.shutdown().await;
}
#[tokio::test]
async fn test_rerank_with_top_k() {
let ctx = AppTestContext::new(vec![MockWorkerConfig {
port: 18106,
worker_type: WorkerType::Regular,
health_status: HealthStatus::Healthy,
response_delay_ms: 0,
fail_rate: 0.0,
}])
.await;
let app = ctx.create_app().await;
let payload = json!({
"query": "test query",
"documents": [
"Document 1",
"Document 2",
"Document 3"
],
"model": "test-model",
"top_k": 1,
"return_documents": true
});
let req = Request::builder()
.method("POST")
.uri("/rerank")
.header(CONTENT_TYPE, "application/json")
.body(Body::from(serde_json::to_string(&payload).unwrap()))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap();
// Should only return top_k results
let results = body_json["results"].as_array().unwrap();
assert_eq!(results.len(), 1);
ctx.shutdown().await;
}
#[tokio::test]
async fn test_rerank_without_documents() {
let ctx = AppTestContext::new(vec![MockWorkerConfig {
port: 18107,
worker_type: WorkerType::Regular,
health_status: HealthStatus::Healthy,
response_delay_ms: 0,
fail_rate: 0.0,
}])
.await;
let app = ctx.create_app().await;
let payload = json!({
"query": "test query",
"documents": ["Document 1", "Document 2"],
"model": "test-model",
"return_documents": false
});
let req = Request::builder()
.method("POST")
.uri("/rerank")
.header(CONTENT_TYPE, "application/json")
.body(Body::from(serde_json::to_string(&payload).unwrap()))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap();
// Documents should be null when return_documents is false
let results = body_json["results"].as_array().unwrap();
for result in results {
assert!(result.get("document").is_none());
}
ctx.shutdown().await;
}
#[tokio::test]
async fn test_rerank_worker_failure() {
let ctx = AppTestContext::new(vec![MockWorkerConfig {
port: 18108,
worker_type: WorkerType::Regular,
health_status: HealthStatus::Healthy,
response_delay_ms: 0,
fail_rate: 1.0, // Always fail
}])
.await;
let app = ctx.create_app().await;
let payload = json!({
"query": "test query",
"documents": ["Document 1"],
"model": "test-model"
});
let req = Request::builder()
.method("POST")
.uri("/rerank")
.header(CONTENT_TYPE, "application/json")
.body(Body::from(serde_json::to_string(&payload).unwrap()))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
// Should return the worker's error response
assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR);
ctx.shutdown().await;
}
#[tokio::test]
async fn test_v1_rerank_compatibility() {
let ctx = AppTestContext::new(vec![MockWorkerConfig {
@@ -1802,85 +1626,4 @@ mod rerank_tests {
ctx.shutdown().await;
}
#[tokio::test]
async fn test_rerank_invalid_request() {
let ctx = AppTestContext::new(vec![MockWorkerConfig {
port: 18111,
worker_type: WorkerType::Regular,
health_status: HealthStatus::Healthy,
response_delay_ms: 0,
fail_rate: 0.0,
}])
.await;
let app = ctx.create_app().await;
let payload = json!({
"query": "",
"documents": ["Document 1", "Document 2"],
"model": "test-model"
});
let req = Request::builder()
.method("POST")
.uri("/rerank")
.header(CONTENT_TYPE, "application/json")
.body(Body::from(serde_json::to_string(&payload).unwrap()))
.unwrap();
let resp = app.clone().oneshot(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
let payload = json!({
"query": " ",
"documents": ["Document 1", "Document 2"],
"model": "test-model"
});
let req = Request::builder()
.method("POST")
.uri("/rerank")
.header(CONTENT_TYPE, "application/json")
.body(Body::from(serde_json::to_string(&payload).unwrap()))
.unwrap();
let resp = app.clone().oneshot(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
let payload = json!({
"query": "test query",
"documents": [],
"model": "test-model"
});
let req = Request::builder()
.method("POST")
.uri("/rerank")
.header(CONTENT_TYPE, "application/json")
.body(Body::from(serde_json::to_string(&payload).unwrap()))
.unwrap();
let resp = app.clone().oneshot(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
let payload = json!({
"query": "test query",
"documents": ["Document 1", "Document 2"],
"model": "test-model",
"top_k": 0
});
let req = Request::builder()
.method("POST")
.uri("/rerank")
.header(CONTENT_TYPE, "application/json")
.body(Body::from(serde_json::to_string(&payload).unwrap()))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
ctx.shutdown().await;
}
}
sgl-model-gateway/tests/common/mock_worker.rs
+1 -1
View File
@@ -83,7 +83,7 @@ impl MockWorker {
.route("/health", get(health_handler))
.route("/health_generate", get(health_generate_handler))
.route("/server_info", get(server_info_handler))
.route("/get_model_info", get(model_info_handler))
.route("/model_info", get(model_info_handler))
.route("/generate", post(generate_handler))
.route("/v1/chat/completions", post(chat_completions_handler))
.route("/v1/completions", post(completions_handler))
sgl-model-gateway/tests/routing/test_pd_routing.rs
+2 -2
View File
@@ -767,12 +767,12 @@ mod pd_routing_unit_tests {
("/health_generate", "GET", true), // Note: Python uses POST, we use GET
("/server_info", "GET", true),
("/v1/models", "GET", true),
("/get_model_info", "GET", true),
("/model_info", "GET", true),
("/generate", "POST", true),
("/v1/chat/completions", "POST", true),
("/v1/completions", "POST", true),
("/flush_cache", "POST", true),
("/get_loads", "GET", true),
("/v1/loads", "GET", true),
("/register", "POST", false), // NOT IMPLEMENTED - needs dynamic worker management
];