[mm] sglang-mm: server vision pipeline core (fetch/driver/pipeline) + Qwen VL (#32364)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

rust/sglang-mm/tests/generate_golden.py

@@ -6,7 +6,7 @@ import numpy as np
 import torch
 from PIL import Image
 
-sys.path.insert(0, os.path.dirname(__file__))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "bench"))
 from bench_parity import PS, make_photo_like, ref_patchify
 
 OUT = (

rust/sglang-mm/tests/rlib_is_single_threaded.rs

@@ -0,0 +1,68 @@
+//! The pure-Rust `rlib` that `sglang-server` links must not own worker
+//! threads: the server supplies concurrency across requests and pins its own
+//! cores, so a library spawning pools behind its back would fight it.
+//!
+//! Guarding this from the outside (thread count of the process) rather than by
+//! inspecting the code, so it stays true no matter how the fan-out seam in
+//! `common::par` is refactored. Runs only in the default (rayon-less) build;
+//! under `--features parallel` the pools are expected.
+
+#![cfg(not(feature = "parallel"))]
+
+use sglang_mm_core::driver::{ImageSource, MmInput, process};
+use sglang_mm_core::registry::pipeline_from_spec;
+
+const SPEC: &str = r#"{"family":"qwen_vl","image_token_id":1,"patch_size":14,
+    "merge_size":2,"temporal_patch_size":2,"min_pixels":3136,
+    "max_pixels":12845056,"image_mean":[0.0,0.0,0.0],"image_std":[1.0,1.0,1.0]}"#;
+
+fn thread_names() -> Vec<String> {
+    std::fs::read_dir("/proc/self/task")
+        .expect("procfs")
+        .filter_map(|entry| {
+            let comm = entry.ok()?.path().join("comm");
+            Some(std::fs::read_to_string(comm).ok()?.trim().to_string())
+        })
+        .collect()
+}
+
+fn png(w: u32, h: u32) -> Vec<u8> {
+    let img = image::RgbImage::from_fn(w, h, |x, y| image::Rgb([x as u8, y as u8, 7]));
+    let mut buf = std::io::Cursor::new(Vec::new());
+    img.write_to(&mut buf, image::ImageFormat::Png).unwrap();
+    buf.into_inner()
+}
+
+#[test]
+fn processing_a_request_spawns_no_worker_threads() {
+    let before = thread_names().len();
+    let family = pipeline_from_spec(SPEC).unwrap();
+
+    // Two images, so the per-item fan-out seam is exercised, not bypassed.
+    let out = process(
+        family.as_ref(),
+        MmInput {
+            text: None,
+            input_ids: Some(vec![7, 1, 8, 1, 9]),
+            images: vec![
+                ImageSource::Bytes(png(112, 112)),
+                ImageSource::Bytes(png(84, 140)),
+            ],
+        },
+        |_| Err("no tokenizer".into()),
+    )
+    .expect("request should succeed");
+    assert_eq!(out.items.len(), 2);
+
+    let after = thread_names();
+    let spawned: Vec<&String> = after.iter().filter(|t| t.starts_with("sgl-mm")).collect();
+    assert!(
+        spawned.is_empty(),
+        "rlib build spawned crate-owned worker threads: {spawned:?}"
+    );
+    assert_eq!(
+        after.len(),
+        before,
+        "rlib build changed the process thread count: {after:?}"
+    );
+}

rust/sglang-mm/tests/test_golden.py

@@ -4,7 +4,7 @@ import os
 import numpy as np
 import pytest
 
-import sglang.srt.multimodal._core.inkling
+from sglang.srt.multimodal._core import inkling as _rs_inkling
 
 GOLDEN_DIR = os.environ.get(
     "INKLING_MM_GOLDEN_DIR",
@@ -20,9 +20,7 @@ def bf16_bits_to_f32(bits: np.ndarray) -> np.ndarray:
 @pytest.mark.parametrize("path", GOLDENS, ids=[os.path.basename(p) for p in GOLDENS])
 def test_patchify_rgb_bit_exact(path):
     g = np.load(path)
-    got = sglang.srt.multimodal._core.inkling.patchify_rgb(
-        g["arr"], int(g["patch_size"])
-    )
+    got = _rs_inkling.patchify_rgb(g["arr"], int(g["patch_size"]))
     np.testing.assert_array_equal(got, g["bits"].reshape(-1))
 
 
@@ -30,9 +28,7 @@ def test_patchify_rgb_bit_exact(path):
 def test_decode_patchify_png_bit_exact(path):
     g = np.load(path)
     h_ref, w_ref = g["arr"].shape[:2]
-    h, w, got = sglang.srt.multimodal._core.inkling.decode_patchify(
-        g["png"].tobytes(), int(g["patch_size"])
-    )
+    h, w, got = _rs_inkling.decode_patchify(g["png"].tobytes(), int(g["patch_size"]))
     assert (h, w) == (h_ref, w_ref)
     np.testing.assert_array_equal(got, g["bits"].reshape(-1))
 
@@ -41,9 +37,7 @@ def test_batch_matches_single():
     gs = [np.load(p) for p in GOLDENS]
     data = [g["png"].tobytes() for g in gs]
     ps = int(gs[0]["patch_size"])
-    for (h, w, bits), g in zip(
-        sglang.srt.multimodal._core.inkling.decode_patchify_batch(data, ps), gs
-    ):
+    for (h, w, bits), g in zip(_rs_inkling.decode_patchify_batch(data, ps), gs):
         np.testing.assert_array_equal(bits, g["bits"].reshape(-1))
 
 

rust/sglang-mm/tests/test_hash_fetch.py

@@ -11,11 +11,15 @@ import soundfile as sf
 import torch
 from PIL import Image
 
-sys.path.insert(0, os.path.dirname(__file__))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "bench"))
 from bench_parity import make_photo_like
 
 from sglang.srt.managers.mm_utils import data_hash, hash_feature
+from sglang.srt.multimodal._core import common as _rs_common
 from sglang.srt.multimodal.inkling import InklingProcessor
+from sglang.srt.multimodal.inkling.image_processing_rust import (
+    InklingRustImageProcessor,
+)
 from sglang.srt.multimodal.processors import inkling as prc
 
 
@@ -39,7 +43,9 @@ def make_proc():
     proc.IMAGE_TOKEN_ID = 100
     proc.AUDIO_TOKEN_ID = 101
     proc.AUDIO_END_TOKEN_ID = 102
-    proc.inkling_processor = InklingProcessor()
+    proc.inkling_processor = InklingProcessor(
+        image_processor=InklingRustImageProcessor()
+    )
     return proc
 
 
@@ -50,14 +56,19 @@ aud = wav_bytes()
 out = proc.assemble([1, 100, 2, 101, 3], [img], [aud])
 img_item = next(i for i in out.mm_items if i.modality.name == "IMAGE")
 aud_item = next(i for i in out.mm_items if i.modality.name == "AUDIO")
-assert img_item.hash == data_hash(img), "image hash != data_hash(raw bytes)"
-assert aud_item.hash == data_hash(aud), "audio hash != data_hash(raw bytes)"
-print(f"  assemble: image hash={img_item.hash:#x} audio hash={aud_item.hash:#x} OK")
+# The rust image path stores its raw-bytes content hash (blake3) eagerly;
+# non-rust items get hashed lazily from the feature at set_pad_value time.
+assert img_item.hash == _rs_common.content_hash(img), "image hash != rust content_hash"
+assert aud_item.hash is None, "audio hash expected to be lazy (feature-based)"
+print(f"  assemble: image hash={img_item.hash:#x} OK")
 
 h0 = img_item.hash
 img_item.set_pad_value()
 assert img_item.hash == h0 and img_item.pad_value is not None
 print(f"  set_pad_value: hash preserved, pad_value={img_item.pad_value} OK")
+aud_item.set_pad_value()
+assert aud_item.hash is not None and aud_item.pad_value is not None
+print("  set_pad_value: audio feature-hash filled OK")
 
 out2 = proc.assemble([1, 100, 2], [img], [])
 assert out2.mm_items[0].hash == h0
@@ -73,14 +84,8 @@ out3 = asyncio.run(
 assert all(i.hash == h0 for i in out3.mm_items), "data: URL roundtrip hash mismatch"
 print("  process_mm_data_async: concurrent resolve + hash OK")
 
-orig = prc._resolve_media_item
-prc._resolve_media_item = lambda it: (time.sleep(0.3), orig(it))[1]
-t0 = time.perf_counter()
-asyncio.run(prc._resolve_media_items([data_url] * 8))
-elapsed = time.perf_counter() - t0
-prc._resolve_media_item = orig
-assert elapsed < 1.2, f"8x 0.3s resolves took {elapsed:.2f}s; expected ~0.3s"
-print(f"  concurrency: 8 x 0.3s resolves in {elapsed:.2f}s OK")
+# (The old `_resolve_media_items` concurrency helper is gone; resolution now
+# happens inline in `process_mm_data_async`, covered by the roundtrip above.)
 
 imgs_5 = [png_bytes(make_photo_like(1080, 1920, seed=s)) for s in range(5)]
 feats = [

rust/sglang-mm/tests/test_integration.py

@@ -6,10 +6,13 @@ import numpy as np
 import torch
 from PIL import Image
 
-sys.path.insert(0, os.path.dirname(__file__))
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "bench"))
 from bench_parity import make_photo_like
 
-import sglang.srt.multimodal.inkling.image_processing as ip
+from sglang.srt.multimodal.inkling.image_processing import InklingImageProcessor
+from sglang.srt.multimodal.inkling.image_processing_rust import (
+    InklingRustImageProcessor,
+)
 
 
 def encode(arr, fmt):
@@ -21,17 +24,14 @@ def encode(arr, fmt):
 
 
 def run(images, use_rs: bool, rescale: bool):
-    ip._rs_module = None
-    os.environ["SGLANG_RS_MM_PREPROCESS"] = "1" if use_rs else "0"
     kwargs = (
         {}
         if rescale
         else {"rescale_image_frac": None, "rescale_image_max_upscaled_long_edge": None}
     )
-    proc = ip.InklingImageProcessor(patch_size=40, **kwargs)
-    out = proc.preprocess(images)
-    assert (ip._rs_module is not False) == use_rs, "rust module gating mismatch"
-    return out
+    cls = InklingRustImageProcessor if use_rs else InklingImageProcessor
+    proc = cls(patch_size=40, **kwargs)
+    return proc.preprocess(images)
 
 
 def compare(tag, images, expect_exact, rescale=False):
@@ -59,7 +59,7 @@ arr1 = make_photo_like(1080, 1920, seed=1)
 arr2 = make_photo_like(720, 1280, seed=2)
 arr3 = make_photo_like(480, 640, seed=3)
 
-print("=== integration: InklingImageProcessor env-gated rust path ===")
+print("=== integration: Inkling rust vs python processor parity ===")
 compare("single PNG", [encode(arr1, "PNG")], expect_exact=True)
 compare("single JPEG", [encode(arr1, "JPEG")], expect_exact=False)
 compare(

rust/sglang-mm/tests/test_resize_parity.py

@@ -6,7 +6,8 @@ import numpy as np
 import pytest
 from PIL import Image
 
-import sglang.srt.multimodal._core.inkling
+from sglang.srt.multimodal._core import common as _rs_common
+from sglang.srt.multimodal._core import inkling as _rs_inkling
 
 
 def py_scaled_dims(
@@ -41,9 +42,7 @@ def pil_resize(arr: np.ndarray, tw: int, th: int) -> np.ndarray:
 
 
 def rs_resize(arr: np.ndarray, tw: int, th: int) -> np.ndarray:
-    return sglang.srt.multimodal._core.inkling.resize_rgb(arr, tw, th).reshape(
-        th, tw, 3
-    )
+    return _rs_common.resize_rgb(arr, tw, th).reshape(th, tw, 3)
 
 
 CASES = [
@@ -74,9 +73,9 @@ def test_scaled_dims_sweep():
     sizes += [(2048, 1024), (2049, 100), (1024, 2048), (1, 1), (4096, 4096)]
     for frac, cap in [(2.0, 2048), (1.5, 2048), (3.0, None), (None, None), (2.0, 1)]:
         for w, h in sizes:
-            assert sglang.srt.multimodal._core.inkling.scaled_dims(
+            assert _rs_common.scaled_dims(w, h, frac, cap) == py_scaled_dims(
                 w, h, frac, cap
-            ) == py_scaled_dims(w, h, frac, cap), (
+            ), (
                 w,
                 h,
                 frac,
@@ -93,12 +92,10 @@ def test_decode_patchify_rescaled_matches_pil_pipeline():
     arr = rng.integers(0, 256, (1080, 1920, 3), dtype=np.uint8)
     buf = io.BytesIO()
     Image.fromarray(arr).save(buf, format="PNG")
-    h, w, bits = sglang.srt.multimodal._core.inkling.decode_patchify(
-        buf.getvalue(), 40, 2.0, 2048
-    )
+    h, w, bits = _rs_inkling.decode_patchify(buf.getvalue(), 40, 2.0, 2048)
     assert (w, h) == py_scaled_dims(1920, 1080, 2.0, 2048)
     ref_arr = pil_resize(arr, w, h)
-    ref_bits = sglang.srt.multimodal._core.inkling.patchify_rgb(ref_arr, 40)
+    ref_bits = _rs_inkling.patchify_rgb(ref_arr, 40)
     np.testing.assert_array_equal(bits, ref_bits)
     assert torch.from_numpy(bits).view(torch.bfloat16).shape[0] > 0