minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[model-gateway] Fix IGW routing for external OpenAI workers (#16633)

Browse Source
This commit is contained in:
Ziwen Zhao
2026-01-07 08:15:23 -08:00
committed by GitHub
parent c356ed03dd
commit b5a94f8a8e
1 changed files with 8 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sgl-model-gateway/src/routers/router_manager.rs
+8 -2
View File
@@ -21,7 +21,7 @@ use tracing::{debug, info, warn};
use crate::{ use crate::{
app_context::AppContext, app_context::AppContext,
config::RoutingMode, config::RoutingMode,
core::{ConnectionMode, WorkerRegistry, WorkerType}, core::{ConnectionMode, RuntimeType, WorkerRegistry, WorkerType},
protocols::{ protocols::{
chat::ChatCompletionRequest, chat::ChatCompletionRequest,
classify::ClassifyRequest, classify::ClassifyRequest,
@@ -281,7 +281,7 @@ impl RouterManager {
let workers = self.worker_registry.get_by_model(model_id); let workers = self.worker_registry.get_by_model(model_id);
// Find the best router ID based on worker capabilities // Find the best router ID based on worker capabilities
// Priority: grpc-pd > http-pd > grpc-regular > http-regular // Priority: external (OpenAI) > grpc-pd > http-pd > grpc-regular > http-regular
let best_router_id = workers let best_router_id = workers
.iter() .iter()
.map(|w| { .map(|w| {
@@ -290,6 +290,12 @@ impl RouterManager {
WorkerType::Prefill { .. } | WorkerType::Decode WorkerType::Prefill { .. } | WorkerType::Decode
); );
let is_grpc = matches!(w.connection_mode(), ConnectionMode::Grpc { .. }); let is_grpc = matches!(w.connection_mode(), ConnectionMode::Grpc { .. });
let is_external = matches!(w.metadata().runtime_type, RuntimeType::External);
if is_external {
// External workers should be routed via OpenAI-compatible router
return (4, &router_ids::HTTP_OPENAI);
}
match (is_grpc, is_pd) { match (is_grpc, is_pd) {
(true, true) => (3, &router_ids::GRPC_PD), (true, true) => (3, &router_ids::GRPC_PD),