[rust-renderer] Standalone preprocessing (#36718)

Signed-off-by: Sage Ahrac <sagiahrak@gmail.com> Co-authored-by: Shangming Cai <csmthu@gmail.com> Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com> Co-authored-by: Rain Jiang <96632942+rainj-me@users.noreply.github.com>
2026-09-20 22:03:12 +08:00
parent 6880a47955
commit 7b1c2ed0a4
51 changed files with 17729 additions and 0 deletions
@@ -0,0 +1,104 @@
+use std::sync::{Arc, Mutex};
+
+use dynamo_protocols::types::ChatCompletionRequestMessage;
+use sglang_renderer::{
+    ChatRequest, GenerateRequestMetadata, GenerationOptions, RendererConfig, RendererError,
+    RendererLimits, RendererService, SamplingDefaults, SamplingParams, TextRequest, TextTokenizer,
+};
+
+#[derive(Clone, Default)]
+struct RecordingTokenizer {
+    prompts: Arc<Mutex<Vec<(String, bool)>>>,
+}
+
+impl TextTokenizer for RecordingTokenizer {
+    fn encode(&self, text: &str, add_special_tokens: bool) -> Result<Vec<i32>, RendererError> {
+        self.prompts
+            .lock()
+            .unwrap()
+            .push((text.to_owned(), add_special_tokens));
+        Ok(vec![7])
+    }
+}
+
+fn config() -> RendererConfig {
+    RendererConfig {
+        served_model_name: "model".into(),
+        tokenizer_path: ".".into(),
+        revision: None,
+        model_path: String::new(),
+        chat_template: Some("chatml".into()),
+        tool_call_parser: None,
+        reasoning_parser: None,
+        default_chat_template_kwargs: Default::default(),
+        stream_response_default_include_usage: false,
+        default_sampling_params: SamplingDefaults::default(),
+        limits: RendererLimits {
+            vocab_size: 128,
+            context_len: 128,
+            num_reserved_tokens: 0,
+            allow_auto_truncate: false,
+            enable_return_hidden_states: false,
+        },
+    }
+}
+
+#[test]
+fn completion_and_chat_share_the_public_text_preparation_boundary() {
+    let tokenizer = RecordingTokenizer::default();
+    let prompts = tokenizer.prompts.clone();
+    let renderer = RendererService::with_tokenizer(config(), Arc::new(tokenizer), 1, 8);
+
+    let completion = TextRequest::text(
+        "completion-0",
+        "plain completion",
+        true,
+        GenerationOptions {
+            sampling_params: SamplingParams {
+                max_new_tokens: Some(1),
+                ..Default::default()
+            },
+            ..Default::default()
+        },
+    );
+    futures::executor::block_on(renderer.prepare_text_requests(vec![completion])).unwrap();
+
+    let messages: Vec<ChatCompletionRequestMessage> = serde_json::from_value(serde_json::json!([
+        {"role": "user", "content": "hello"}
+    ]))
+    .unwrap();
+    let chat = ChatRequest {
+        rid: "chat".into(),
+        model: "model".into(),
+        messages,
+        tools: None,
+        tool_choice: None,
+        response_format: None,
+        reasoning_effort: None,
+        continue_final_message: false,
+        chat_template_args: None,
+        sampling_params: SamplingParams {
+            max_new_tokens: Some(1),
+            ..Default::default()
+        },
+        choice_count: 1,
+        stream: false,
+        return_logprob: false,
+        top_logprobs_num: 0,
+        parallel_tool_calls: true,
+        metadata: GenerateRequestMetadata::default(),
+    };
+    futures::executor::block_on(renderer.prepare_chat(chat)).unwrap();
+
+    let prompts = prompts.lock().unwrap();
+    assert!(
+        prompts
+            .iter()
+            .any(|(text, add_special_tokens)| text == "plain completion" && *add_special_tokens)
+    );
+    assert!(
+        prompts
+            .iter()
+            .any(|(text, add_special_tokens)| text.contains("hello") && !add_special_tokens)
+    );
+}
@@ -0,0 +1,122 @@
+// Keep the inner attribute off the first line so shebang lint does not misclassify it.
+#![cfg(feature = "http")]
+
+use std::process::{Child, Command, Stdio};
+use std::time::{Duration, Instant};
+
+use reqwest::StatusCode;
+use serde_json::{Value, json};
+
+struct ChildGuard(Child);
+
+struct TestDirectory(std::path::PathBuf);
+
+impl Drop for ChildGuard {
+    fn drop(&mut self) {
+        let _ = self.0.kill();
+        let _ = self.0.wait();
+    }
+}
+
+impl Drop for TestDirectory {
+    fn drop(&mut self) {
+        let _ = std::fs::remove_dir_all(&self.0);
+    }
+}
+
+fn available_port() -> u16 {
+    let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+    listener.local_addr().unwrap().port()
+}
+
+async fn wait_until_ready(child: &mut Child, client: &reqwest::Client, health_url: &str) {
+    let deadline = Instant::now() + Duration::from_secs(10);
+    loop {
+        if let Some(status) = child.try_wait().unwrap() {
+            panic!("render-only process exited during startup with {status}");
+        }
+        if client
+            .get(health_url)
+            .send()
+            .await
+            .is_ok_and(|response| response.status() == StatusCode::OK)
+        {
+            return;
+        }
+        assert!(
+            Instant::now() < deadline,
+            "render-only process did not start"
+        );
+        tokio::time::sleep(Duration::from_millis(25)).await;
+    }
+}
+
+#[tokio::test]
+async fn binary_starts_without_an_engine_and_serves_only_preprocessing() {
+    let source_tokenizer = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("../../experimental/sgl-router/tests/fixtures/tiny_tokenizer.json");
+    let model = TestDirectory(
+        std::env::temp_dir().join(format!("sglang-render-only-{}", uuid::Uuid::new_v4())),
+    );
+    std::fs::create_dir(&model.0).unwrap();
+    std::fs::copy(source_tokenizer, model.0.join("tokenizer.json")).unwrap();
+    let port = available_port();
+    let mut child = ChildGuard(
+        Command::new(env!("CARGO_BIN_EXE_sglang-renderer"))
+            .arg(&model.0)
+            .arg("--tokenizer-path")
+            .arg(&model.0)
+            .arg("--served-model-name")
+            .arg("model")
+            .arg("--resolved-sampling-params")
+            .arg("{}")
+            .arg("--context-length")
+            .arg("64")
+            .arg("--vocab-size")
+            .arg("512")
+            .arg("--host")
+            .arg("127.0.0.1")
+            .arg("--port")
+            .arg(port.to_string())
+            .stdout(Stdio::null())
+            .stderr(Stdio::inherit())
+            .spawn()
+            .unwrap(),
+    );
+    let client = reqwest::Client::builder().no_proxy().build().unwrap();
+    let origin = format!("http://127.0.0.1:{port}");
+    wait_until_ready(&mut child.0, &client, &format!("{origin}/health")).await;
+
+    let tokenized = client
+        .post(format!("{origin}/v1/tokenize"))
+        .json(&json!({"prompt": "hello"}))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(tokenized.status(), StatusCode::OK);
+    let tokenized: Value = tokenized.json().await.unwrap();
+    assert!(tokenized["count"].as_u64().is_some_and(|count| count > 0));
+
+    let completion = json!({"model": "model", "prompt": "hello"});
+    let rendered = client
+        .post(format!("{origin}/v1/completions/render"))
+        .json(&completion)
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(rendered.status(), StatusCode::OK);
+    let rendered: Value = rendered.json().await.unwrap();
+    assert!(
+        rendered
+            .as_array()
+            .is_some_and(|requests| requests.len() == 1)
+    );
+
+    let inference = client
+        .post(format!("{origin}/v1/completions"))
+        .json(&completion)
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(inference.status(), StatusCode::NOT_FOUND);
+}