minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Build Rust extensions on demand in source checkouts (#34994)

Browse Source
This commit is contained in:
Lianmin Zheng
2026-08-16 14:58:06 -07:00
committed by GitHub
parent 0e231d365a
commit 67e12131df
39 changed files with 880 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/Cargo.lock
Generated
+20 -17
View File
@@ -982,9 +982,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]]
name = "dynamo-parsers"
version = "7.0.1"
version = "7.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97cce1ec70f4c4896ff9a4e0bb29cd603f465e599c94bf2085b71872a3a58c70"
checksum = "7a7f84812e7ce1d3dace1b8d69853cc6c49759cbd07dfbb59452aa928645b031"
dependencies = [
"aho-corasick",
"anyhow",
@@ -1005,9 +1005,9 @@ dependencies = [
[[package]]
name = "dynamo-protocols"
version = "5.1.0"
version = "5.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "696b6df4c817c16dc0a619d43f6af5ecc4f045a6b6be7ef516bfe502f977764b"
checksum = "97fd951c32c033f4f220b6087db91d7f7d475f1bf1f886c8e60798ac1cf4cc9a"
dependencies = [
"async-openai",
"derive_builder",
@@ -1022,9 +1022,9 @@ dependencies = [
[[package]]
name = "dynamo-renderer"
version = "5.0.0"
version = "5.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0423da7bec83734e7567b1c8895fd2e7a1a98256253a29fe128032ac01cefac1"
checksum = "dfe04753d666e3462e3eed4abcf2a4d05997cf2dea45858733f2bdea5b61f733"
dependencies = [
"anyhow",
"chrono",
@@ -1040,9 +1040,9 @@ dependencies = [
[[package]]
name = "dynamo-tokenizers"
version = "1.7.0"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a81f7b7693282f4693a3b4c60174e14260cf011ab493a7ca9b4deb2fa98ecee"
checksum = "821c767e896f1f225b6411a5ce41dba7f114c2c97db862a13fd962195665075e"
dependencies = [
"aho-corasick",
"anyhow",
@@ -1162,21 +1162,24 @@ dependencies = [
[[package]]
name = "fastokens"
version = "0.2.1"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8728655e193e0d08d7a95d63cf1fdb9b768d282cab0a112ecb006615bae9f067"
checksum = "35ff8cf975d8f772e7d4c3be439659b0b23b1ae1086ebee7204573807053ff7f"
dependencies = [
"daachorse",
"fancy-regex 0.17.0",
"hf-hub",
"icu_normalizer",
"libc",
"memchr",
"pcre2",
"rayon",
"regex-syntax",
"serde",
"serde_json",
"strum",
"thiserror",
"ureq",
]
[[package]]
@@ -1633,7 +1636,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.6.5",
"socket2 0.5.10",
"tokio",
"tower-service",
"tracing",
@@ -2123,9 +2126,9 @@ dependencies = [
[[package]]
name = "minijinja"
version = "2.21.0"
version = "2.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb3d648e68cea56d9858d535ee28f9538404e2dd8cb08ed0bd05dca379477f39"
checksum = "86886cf6dbf4e614b19c9a1eec9775f021869d7eadde0fc73921a81b90c9b4c9"
dependencies = [
"indexmap 2.14.0",
"memo-map",
@@ -2135,9 +2138,9 @@ dependencies = [
[[package]]
name = "minijinja-contrib"
version = "2.21.0"
version = "2.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85342f6fac0be8ccd5bd00d9066be538f34f393f577b75d81b17c8398a6b43bb"
checksum = "bd3e5f077bc2379f0f7d911e7cfdd921114ed99fc884533dca502944cb355b11"
dependencies = [
"minijinja",
"serde",
@@ -2940,7 +2943,7 @@ dependencies = [
"quinn-udp",
"rustc-hash 2.1.3",
"rustls",
"socket2 0.6.5",
"socket2 0.5.10",
"thiserror",
"tokio",
"tracing",
@@ -2978,7 +2981,7 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.6.5",
"socket2 0.5.10",
"tracing",
"windows-sys 0.61.2",
]
rust/sglang-grpc/Cargo.toml
+1 -1
View File
@@ -8,7 +8,7 @@ license.workspace = true
# Consumed by python/setup.py: registers this crate as a PyO3 extension module
# of the main sglang wheel at the given import path.
[package.metadata.sglang]
python-module = "sglang.srt.grpc._core"
python-module = "sglang.srt.rust_extensions._grpc"
# Always build optimized, even for an editable install.
debug = false
rust/sglang-grpc/pyproject.toml
+3 -3
View File
@@ -1,7 +1,7 @@
# Standalone dev builds only (`maturin build` / `maturin develop`); the
# extension ships to users inside the main sglang wheel via setuptools-rust
# (python/pyproject.toml, target sglang.srt.grpc._core). Requires a repo
# checkout (build.rs reads ../../proto), so this is not publishable as an
# (python/pyproject.toml, target sglang.srt.rust_extensions._grpc). Requires a
# repo checkout (build.rs reads ../../proto), so this is not publishable as an
# sdist. protoc is not required: build.rs falls back to a vendored binary.
[build-system]
requires = ["maturin>=1.5,<2"]
@@ -14,4 +14,4 @@ description = "In-process Rust gRPC server for SGLang"
requires-python = ">=3.10"
[tool.maturin]
module-name = "_core"
module-name = "_grpc"
rust/sglang-grpc/src/lib.rs
+1 -1
View File
@@ -257,7 +257,7 @@ fn start_server(
}
#[pymodule]
fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
fn _grpc(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_function(wrap_pyfunction!(start_server, m)?)?;
m.add_class::<GrpcServerHandle>()?;
m.add_class::<ChunkSendStatus>()?;
rust/sglang-mm/Cargo.toml
+5 -5
View File
@@ -11,17 +11,17 @@ license.workspace = true
# features: the wheel build needs the PyO3 bindings, which are NOT default (see
# [features] below).
[package.metadata.sglang]
python-module = "sglang.srt.multimodal._core"
python-module = "sglang.srt.rust_extensions._multimodal"
debug = false
features = ["python", "parallel"]
[lib]
# Unique per-crate artifact name (the shared workspace target/ dir cannot hold
# two `lib_core` cdylibs). The importable Python module is still `_core`: the
# name comes from the `#[pymodule]` entry point and the setuptools-rust
# `target` in python/pyproject.toml, which renames the built artifact.
# multiple `lib_core` cdylibs). The importable Python module is `_multimodal`: the
# name comes from the `#[pymodule]` entry point and the setuptools-rust target,
# which renames the built artifact.
name = "sglang_mm_core"
# cdylib: the PyO3 module (`sglang.srt.multimodal._core`).
# cdylib: the PyO3 module (`sglang.srt.rust_extensions._multimodal`).
# rlib: pure-Rust core linked by sglang-server's native MM path.
crate-type = ["cdylib", "rlib"]
rust/sglang-mm/README.md
+7 -6
View File
@@ -6,9 +6,9 @@ GIL-released.
Built two ways:
- **PyO3 extension** `sglang.srt.multimodal._core` (features `python,parallel`,
requested by the wheel build) via setuptools-rust when installing sglang —
used by Python processors and parity tests.
- **PyO3 extension** `sglang.srt.rust_extensions._multimodal` (features
`python,parallel`, requested by the wheel build) via setuptools-rust when
installing sglang — used by Python processors and parity tests.
- **Pure-Rust `rlib`** (default features, i.e. neither) linked by
`sglang-server`'s MM worker path — that copy needs no pyo3, no libpython, and
no rayon: it spawns no threads and runs inline on the calling thread, because
@@ -19,7 +19,7 @@ Built two ways:
```
src/
├── lib.rs # module root; PyO3 module (_core) feature-gated
├── lib.rs # module root; PyO3 module (_multimodal) feature-gated
├── pipeline.rs # the server-pipeline contract: MmFamilyProcessor
│ # trait + the carriers (Tensor, TokenLayout, ...)
├── driver.rs # model-independent request driver (fetch →
@@ -115,7 +115,7 @@ un-stripped URL to `open()`).
## Python API
```python
from sglang.srt.multimodal._core import common, inkling
from sglang.srt.rust_extensions._multimodal import common, inkling
# Common (model-agnostic)
common.resize_rgb(arr, out_w, out_h)
@@ -174,7 +174,8 @@ impl ImageProcessorSpec for MyModelProcessor {
4. Wire up in `src/lib.rs`: `mod my_model;` and `my_model::register(m)?;`.
5. Add Python processor class that calls `from sglang.srt.multimodal._core import my_model`.
5. Add Python processor class that calls
`from sglang.srt.rust_extensions._multimodal import my_model`.
## Available transform primitives (`common::transforms`)
rust/sglang-mm/bench/bench_parity.py
+1 -1
View File
@@ -5,7 +5,6 @@ import numpy as np
import torch
from PIL import Image
from sglang.srt.multimodal._core import inkling as _rs_inkling
from sglang.srt.multimodal.inkling.image_processing import (
IMAGE_MEAN,
IMAGE_STD,
@@ -13,6 +12,7 @@ from sglang.srt.multimodal.inkling.image_processing import (
_encode_image_bytes,
_fill_patches_numba,
)
from sglang.srt.rust_extensions._multimodal import inkling as _rs_inkling
PS = 40
rust/sglang-mm/pyproject.toml
+1 -1
View File
@@ -9,4 +9,4 @@ description = "Rust-accelerated multimodal preprocessing for SGLang"
requires-python = ">=3.10"
[tool.maturin]
module-name = "_core"
module-name = "_multimodal"
rust/sglang-mm/src/lib.rs
+2 -2
View File
@@ -1,7 +1,7 @@
//! sglang-mm: Rust-accelerated multimodal preprocessing for SGLang.
//!
//! Built two ways:
//! * PyO3 extension `sglang.srt.multimodal._core` (feature `python`, default),
//! * PyO3 extension `sglang.srt.rust_extensions._multimodal` (feature `python`),
//! used by Python processors (e.g. Inkling) and by parity tests.
//! * Pure-Rust `rlib` (`default-features = false`), linked by `sglang-server`'s
//! MM worker path — no pyo3 in that dependency graph.
@@ -18,7 +18,7 @@ use pyo3::prelude::*;
#[cfg(feature = "python")]
#[pymodule]
fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
fn _multimodal(m: &Bound<'_, PyModule>) -> PyResult<()> {
common::register(m)?;
inkling::register(m)?;
qwen_vl::register(m)?;
rust/sglang-mm/tests/test_golden.py
+1 -1
View File
@@ -4,7 +4,7 @@ import os
import numpy as np
import pytest
from sglang.srt.multimodal._core import inkling as _rs_inkling
from sglang.srt.rust_extensions._multimodal import inkling as _rs_inkling
GOLDEN_DIR = os.environ.get(
"INKLING_MM_GOLDEN_DIR",
rust/sglang-mm/tests/test_hash_fetch.py
+1 -1
View File
@@ -15,12 +15,12 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "bench"))
from bench_parity import make_photo_like
from sglang.srt.managers.mm_utils import data_hash, hash_feature
from sglang.srt.multimodal._core import common as _rs_common
from sglang.srt.multimodal.inkling import InklingProcessor
from sglang.srt.multimodal.inkling.image_processing_rust import (
InklingRustImageProcessor,
)
from sglang.srt.multimodal.processors import inkling as prc
from sglang.srt.rust_extensions._multimodal import common as _rs_common
def png_bytes(arr):
rust/sglang-mm/tests/test_resize_parity.py
+2 -2
View File
@@ -6,8 +6,8 @@ import numpy as np
import pytest
from PIL import Image
from sglang.srt.multimodal._core import common as _rs_common
from sglang.srt.multimodal._core import inkling as _rs_inkling
from sglang.srt.rust_extensions._multimodal import common as _rs_common
from sglang.srt.rust_extensions._multimodal import inkling as _rs_inkling
def py_scaled_dims(
rust/sglang-server/Cargo.toml
+1 -1
View File
@@ -7,7 +7,7 @@ license.workspace = true
# Consumed by python/setup.py: registers this crate as a PyO3 extension module.
[package.metadata.sglang]
python-module = "sglang.srt.server._core"
python-module = "sglang.srt.rust_extensions._server"
# Always build optimized, even for an editable install.
debug = false
rust/sglang-server/pyproject.toml
+1 -1
View File
@@ -13,4 +13,4 @@ classifiers = [
dynamic = ["version"]
[tool.maturin]
module-name = "_core"
module-name = "_server"
rust/sglang-server/src/api_server/openai/chat.rs
+2
View File
@@ -836,6 +836,7 @@ pub(super) fn chat_logprobs(extras: Option<&ChunkExtras>) -> ChatChoiceLogprobs
bytes: Some(token.as_bytes().to_vec()),
token,
logprob,
token_id: u32::try_from(token_id).ok(),
top_logprobs,
});
}
@@ -1020,6 +1021,7 @@ mod tests {
let logprobs = chat_logprobs(Some(&extras));
let token = &logprobs.content.unwrap()[0];
assert_eq!(token.token, "x");
assert_eq!(token.token_id, Some(7));
assert_eq!(token.top_logprobs.len(), 2);
assert_eq!(token.top_logprobs[1].token, "y");
}
rust/sglang-server/src/lib.rs
+1 -1
View File
@@ -292,7 +292,7 @@ static LOG_GUARD: std::sync::OnceLock<tracing_appender::non_blocking::WorkerGuar
std::sync::OnceLock::new();
#[pymodule]
fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
fn _server(m: &Bound<'_, PyModule>) -> PyResult<()> {
// Initialize tracing once; ignore if already set by the host process.
// Non-blocking writer: emitting threads (axum workers, egress, detok) only
// enqueue; a dedicated thread does the stdout formatting-flush + syscall.