[model-gateway] Replace tokenizer with tokenizer registry for dynamic tokenizer loading in gRPC router (#12968)

sgl-model-gateway/tests/common/mod.rs

@@ -27,6 +27,7 @@ use sgl_model_gateway::{
     policies::PolicyRegistry,
     protocols::common::{Function, Tool},
     reasoning_parser::ParserFactory as ReasoningParserFactory,
+    tokenizer::registry::TokenizerRegistry,
     tool_parser::ParserFactory as ToolParserFactory,
 };
 
@@ -76,7 +77,7 @@ pub async fn create_test_context(config: RouterConfig) -> Arc<AppContext> {
             .router_config(config.clone())
             .client(client)
             .rate_limiter(rate_limiter)
-            .tokenizer(None) // tokenizer
+            .tokenizer_registry(Arc::new(TokenizerRegistry::new())) // tokenizer
             .reasoning_parser_factory(None) // reasoning_parser_factory
             .tool_parser_factory(None) // tool_parser_factory
             .worker_registry(worker_registry)
@@ -181,6 +182,7 @@ pub async fn create_test_context_with_parsers(config: RouterConfig) -> Arc<AppCo
     };
 
     // Initialize registries
+    let tokenizer_registry = Arc::new(TokenizerRegistry::new());
     let worker_registry = Arc::new(WorkerRegistry::new());
     let policy_registry = Arc::new(PolicyRegistry::new(config.policy.clone()));
 
@@ -211,7 +213,7 @@ pub async fn create_test_context_with_parsers(config: RouterConfig) -> Arc<AppCo
             .router_config(config.clone())
             .client(client)
             .rate_limiter(rate_limiter)
-            .tokenizer(None) // tokenizer
+            .tokenizer_registry(tokenizer_registry)
             .reasoning_parser_factory(reasoning_parser_factory)
             .tool_parser_factory(tool_parser_factory)
             .worker_registry(worker_registry)
@@ -347,7 +349,7 @@ pub async fn create_test_context_with_mcp_config(
             .router_config(config.clone())
             .client(client)
             .rate_limiter(rate_limiter)
-            .tokenizer(None) // tokenizer
+            .tokenizer_registry(Arc::new(TokenizerRegistry::new())) // tokenizer
             .reasoning_parser_factory(None) // reasoning_parser_factory
             .tool_parser_factory(None) // tool_parser_factory
             .worker_registry(worker_registry)

sgl-model-gateway/tests/common/test_app.rs

@@ -16,6 +16,7 @@ use sgl_model_gateway::{
     policies::PolicyRegistry,
     routers::RouterTrait,
     server::{build_app, AppState},
+    tokenizer::registry::TokenizerRegistry,
 };
 
 /// Create a test Axum application using the actual server's build_app function
@@ -67,7 +68,7 @@ pub fn create_test_app(
             .router_config(router_config.clone())
             .client(client)
             .rate_limiter(rate_limiter)
-            .tokenizer(None) // tokenizer
+            .tokenizer_registry(Arc::new(TokenizerRegistry::new())) // tokenizer
             .reasoning_parser_factory(None) // reasoning_parser_factory
             .tool_parser_factory(None) // tool_parser_factory
             .worker_registry(worker_registry)
@@ -195,7 +196,7 @@ pub async fn create_test_app_context() -> Arc<AppContext> {
             .router_config(router_config)
             .client(client)
             .rate_limiter(None)
-            .tokenizer(None)
+            .tokenizer_registry(Arc::new(TokenizerRegistry::new()))
             .reasoning_parser_factory(None)
             .tool_parser_factory(None)
             .worker_registry(worker_registry)

sgl-model-gateway/tests/test_pd_routing.rs

@@ -6,6 +6,7 @@ mod test_pd_routing {
         config::{PolicyConfig, RouterConfig, RoutingMode},
         core::{BasicWorkerBuilder, Worker, WorkerType},
         routers::{http::pd_types::PDSelectionPolicy, RouterFactory},
+        tokenizer::registry::TokenizerRegistry,
     };
 
     #[derive(Debug)]
@@ -256,7 +257,7 @@ mod test_pd_routing {
                         .router_config(config)
                         .client(client)
                         .rate_limiter(rate_limiter)
-                        .tokenizer(None) // tokenizer
+                        .tokenizer_registry(Arc::new(TokenizerRegistry::new())) // tokenizer
                         .reasoning_parser_factory(None) // reasoning_parser_factory
                         .tool_parser_factory(None) // tool_parser_factory
                         .worker_registry(worker_registry)

sgl-model-gateway/tests/wasm_test.rs

@@ -28,6 +28,7 @@ use sgl_model_gateway::{
     policies::PolicyRegistry,
     routers::RouterFactory,
     server::{build_app, AppState},
+    tokenizer::TokenizerRegistry,
     wasm::{
         module::{
             WasmModuleAddRequest, WasmModuleAddResponse, WasmModuleAttachPoint,
@@ -53,6 +54,7 @@ async fn create_test_context_with_wasm() -> Arc<AppContext> {
     // Create AppContext with wasm_manager from the start
     let client = reqwest::Client::new();
 
+    let tokenizer_registry = Arc::new(TokenizerRegistry::new());
     let worker_registry = Arc::new(WorkerRegistry::new());
     let policy_registry = Arc::new(PolicyRegistry::new(config.policy.clone()));
 
@@ -80,7 +82,7 @@ async fn create_test_context_with_wasm() -> Arc<AppContext> {
             .router_config(config.clone())
             .client(client)
             .rate_limiter(None)
-            .tokenizer(None)
+            .tokenizer_registry(tokenizer_registry)
             .reasoning_parser_factory(None)
             .tool_parser_factory(None)
             .worker_registry(worker_registry)