[mm] rust-server: native multimodal processing for Qwen VL (integrate sglang-mm, e2e) (#32365)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Claude Fable 5
Cursor
parent
dea07b348b
commit
32e5d788bd
@@ -15,49 +15,111 @@ use base64::Engine;
|
||||
/// payloads reject the request here).
|
||||
pub const MAX_FETCH_BYTES: u64 = 64 << 20;
|
||||
|
||||
/// Charge granularity of a streaming read: the most an in-flight source can
|
||||
/// over-charge a shared [`ByteBudget`] by.
|
||||
const CHUNK_BYTES: u64 = 256 << 10;
|
||||
|
||||
/// A byte allowance shared by every source of one request, charged *as they
|
||||
/// stream*, so concurrent fetches stop at their combined size rather than each
|
||||
/// stopping at [`MAX_FETCH_BYTES`].
|
||||
#[derive(Debug)]
|
||||
pub struct ByteBudget(std::sync::atomic::AtomicU64);
|
||||
|
||||
impl ByteBudget {
|
||||
pub fn new(total: u64) -> Self {
|
||||
Self(std::sync::atomic::AtomicU64::new(total))
|
||||
}
|
||||
|
||||
/// Claim `n` bytes, or `Err` once the allowance is spent.
|
||||
fn claim(&self, n: u64) -> Result<(), ()> {
|
||||
use std::sync::atomic::Ordering::{AcqRel, Acquire};
|
||||
self.0
|
||||
.fetch_update(AcqRel, Acquire, |left| left.checked_sub(n))
|
||||
.map(|_| ())
|
||||
.map_err(|_| ())
|
||||
}
|
||||
|
||||
/// Give back bytes claimed for a chunk but not filled by the read.
|
||||
fn release(&self, n: u64) {
|
||||
self.0.fetch_add(n, std::sync::atomic::Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve one string-typed image source into raw encoded-image bytes.
|
||||
/// An `Err` rejects the request, matching the Python per-request
|
||||
/// exception → 400.
|
||||
pub fn fetch_bytes(src: &str) -> Result<Vec<u8>, String> {
|
||||
fetch_bytes_budgeted(src, &ByteBudget::new(MAX_FETCH_BYTES))
|
||||
}
|
||||
|
||||
/// [`fetch_bytes`] against a caller-owned allowance, for resolving several
|
||||
/// sources under one whole-request bound. [`MAX_FETCH_BYTES`] still caps each.
|
||||
pub fn fetch_bytes_budgeted(src: &str, budget: &ByteBudget) -> Result<Vec<u8>, String> {
|
||||
if src.starts_with("http://") || src.starts_with("https://") {
|
||||
return http_get(src);
|
||||
return http_get(src, budget);
|
||||
}
|
||||
if let Some(path) = src.strip_prefix("file://") {
|
||||
return read_file(path);
|
||||
return read_file(path, budget);
|
||||
}
|
||||
if src.starts_with('/') {
|
||||
return read_file(src);
|
||||
return read_file(src, budget);
|
||||
}
|
||||
if let Some(rest) = src.strip_prefix("data:") {
|
||||
let encoded = rest
|
||||
.split_once(',')
|
||||
.ok_or_else(|| "media fetch: malformed data: URL".to_string())?
|
||||
.1;
|
||||
return b64(encoded);
|
||||
return charge_decoded(b64(encoded)?, budget);
|
||||
}
|
||||
// Python treats any other string as bare base64.
|
||||
b64(src)
|
||||
charge_decoded(b64(src)?, budget)
|
||||
}
|
||||
|
||||
/// Base64 payloads are already resident in the request body — they cannot
|
||||
/// amplify the way a download can, so they charge once decoded, not per chunk.
|
||||
fn charge_decoded(decoded: Vec<u8>, budget: &ByteBudget) -> Result<Vec<u8>, String> {
|
||||
budget
|
||||
.claim(decoded.len() as u64)
|
||||
.map_err(|()| over_budget("base64 payload"))?;
|
||||
Ok(decoded)
|
||||
}
|
||||
|
||||
fn over_budget(what: &str) -> String {
|
||||
format!("media fetch: {what}: exceeds the request media byte budget")
|
||||
}
|
||||
|
||||
/// Bounded read: never trusts metadata, so huge and non-regular files
|
||||
/// (`/dev/zero`) hit the cap instead of exhausting memory.
|
||||
fn read_file(path: &str) -> Result<Vec<u8>, String> {
|
||||
fn read_file(path: &str, budget: &ByteBudget) -> Result<Vec<u8>, String> {
|
||||
let file = std::fs::File::open(path).map_err(|e| format!("media fetch: {path}: {e}"))?;
|
||||
read_capped(file, path)
|
||||
read_capped(file, path, budget)
|
||||
}
|
||||
|
||||
fn read_capped(reader: impl Read, what: &str) -> Result<Vec<u8>, String> {
|
||||
/// Read to EOF, charging `budget` per chunk, so an oversized source stops
|
||||
/// mid-stream instead of going fully resident first.
|
||||
fn read_capped(mut reader: impl Read, what: &str, budget: &ByteBudget) -> Result<Vec<u8>, String> {
|
||||
let too_big = || format!("media fetch: {what}: exceeds {MAX_FETCH_BYTES} bytes");
|
||||
let mut buf = Vec::new();
|
||||
reader
|
||||
.take(MAX_FETCH_BYTES + 1)
|
||||
.read_to_end(&mut buf)
|
||||
.map_err(|e| format!("media fetch: read {what}: {e}"))?;
|
||||
if buf.len() as u64 > MAX_FETCH_BYTES {
|
||||
return Err(format!(
|
||||
"media fetch: {what}: exceeds {MAX_FETCH_BYTES} bytes"
|
||||
));
|
||||
loop {
|
||||
// `+ 1`: read one byte past the cap, so oversized is detected, not truncated.
|
||||
let want = CHUNK_BYTES.min(MAX_FETCH_BYTES + 1 - buf.len() as u64);
|
||||
if want == 0 {
|
||||
return Err(too_big());
|
||||
}
|
||||
budget.claim(want).map_err(|()| over_budget(what))?;
|
||||
let read = reader
|
||||
.by_ref()
|
||||
.take(want)
|
||||
.read_to_end(&mut buf)
|
||||
.map_err(|e| format!("media fetch: read {what}: {e}"))? as u64;
|
||||
budget.release(want - read);
|
||||
if buf.len() as u64 > MAX_FETCH_BYTES {
|
||||
return Err(too_big());
|
||||
}
|
||||
if read < want {
|
||||
return Ok(buf); // short read == EOF
|
||||
}
|
||||
}
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
fn b64(encoded: &str) -> Result<Vec<u8>, String> {
|
||||
@@ -163,7 +225,7 @@ fn in_ipv4_network(ip: std::net::Ipv4Addr, net: std::net::Ipv4Addr, bits: u32) -
|
||||
u32::from(ip) & mask == u32::from(net) & mask
|
||||
}
|
||||
|
||||
fn http_get(url: &str) -> Result<Vec<u8>, String> {
|
||||
fn http_get(url: &str, budget: &ByteBudget) -> Result<Vec<u8>, String> {
|
||||
// Python: `int(os.getenv("REQUEST_TIMEOUT", "3"))` seconds per image GET.
|
||||
let timeout = std::env::var("REQUEST_TIMEOUT")
|
||||
.ok()
|
||||
@@ -178,7 +240,7 @@ fn http_get(url: &str) -> Result<Vec<u8>, String> {
|
||||
.timeout(std::time::Duration::from_secs(timeout))
|
||||
.call()
|
||||
.map_err(|e| format!("media fetch: GET {url}: {e}"))?;
|
||||
read_capped(resp.into_reader(), url)
|
||||
read_capped(resp.into_reader(), url, budget)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -221,6 +283,38 @@ mod tests {
|
||||
assert!(err.contains("exceeds"), "{err}");
|
||||
}
|
||||
|
||||
/// One budget spans sources: each fits alone, the set does not.
|
||||
#[test]
|
||||
fn shared_budget_spans_sources() {
|
||||
let payload = base64::engine::general_purpose::STANDARD.encode([7u8; 4096]);
|
||||
let budget = ByteBudget::new(6144);
|
||||
assert_eq!(fetch_bytes_budgeted(&payload, &budget).unwrap().len(), 4096);
|
||||
let err = fetch_bytes_budgeted(&payload, &budget).err().unwrap();
|
||||
assert!(err.contains("request media byte budget"), "{err}");
|
||||
}
|
||||
|
||||
/// Unused claims come back, so small sources fit in a budget their
|
||||
/// worst-case sizes would have exhausted.
|
||||
#[test]
|
||||
fn short_reads_release_their_claim() {
|
||||
let path = std::env::temp_dir().join(format!("sglang-budget-{}", std::process::id()));
|
||||
std::fs::write(&path, [0u8; 1024]).unwrap();
|
||||
let src = path.display().to_string();
|
||||
let budget = ByteBudget::new(CHUNK_BYTES + 4096);
|
||||
for _ in 0..4 {
|
||||
assert_eq!(fetch_bytes_budgeted(&src, &budget).unwrap().len(), 1024);
|
||||
}
|
||||
std::fs::remove_file(&path).ok();
|
||||
}
|
||||
|
||||
/// The per-source cap holds even under a larger shared budget.
|
||||
#[test]
|
||||
fn per_source_cap_survives_a_large_budget() {
|
||||
let budget = ByteBudget::new(MAX_FETCH_BYTES * 4);
|
||||
let err = fetch_bytes_budgeted("/dev/zero", &budget).err().unwrap();
|
||||
assert!(err.contains(&format!("exceeds {MAX_FETCH_BYTES}")), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_parsing_strips_userinfo_and_path() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -80,16 +80,32 @@ mod python {
|
||||
|
||||
use super::{decode_rgb, resize};
|
||||
|
||||
/// `resample` names the implementation to reproduce: `"pil_lanczos"` (the
|
||||
/// inkling default), `"pil_bicubic"`, or `"aten_u8"` (torchvision's uint8
|
||||
/// antialias bicubic). Exposed so the bit-exactness tests can cover each.
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (arr, out_w, out_h, resample="pil_lanczos"))]
|
||||
pub fn resize_rgb<'py>(
|
||||
py: Python<'py>,
|
||||
arr: PyReadonlyArray3<'py, u8>,
|
||||
out_w: usize,
|
||||
out_h: usize,
|
||||
resample: &str,
|
||||
) -> PyResult<Bound<'py, PyArray1<u8>>> {
|
||||
if out_w == 0 || out_h == 0 {
|
||||
return Err(PyValueError::new_err("output size must be positive"));
|
||||
}
|
||||
let resample = match resample {
|
||||
"pil_lanczos" => resize::Resample::Pil(resize::Filter::Lanczos),
|
||||
"pil_bicubic" => resize::Resample::Pil(resize::Filter::Bicubic),
|
||||
"aten_u8" => resize::Resample::AtenU8,
|
||||
other => {
|
||||
return Err(PyValueError::new_err(format!(
|
||||
"unknown resample {other:?}; expected \"pil_lanczos\", \
|
||||
\"pil_bicubic\" or \"aten_u8\""
|
||||
)));
|
||||
}
|
||||
};
|
||||
let shape = arr.shape();
|
||||
let (h, w, c) = (shape[0], shape[1], shape[2]);
|
||||
if c != 3 {
|
||||
@@ -101,7 +117,7 @@ mod python {
|
||||
.as_slice()
|
||||
.map_err(|_| PyValueError::new_err("array must be C-contiguous"))?
|
||||
.to_vec();
|
||||
let out = py.detach(move || resize::resize_lanczos_rgb(&data, h, w, out_h, out_w));
|
||||
let out = py.detach(move || resize::resize_rgb(&data, h, w, out_h, out_w, resample));
|
||||
Ok(out.into_pyarray(py))
|
||||
}
|
||||
|
||||
|
||||
@@ -1,17 +1,52 @@
|
||||
use super::par;
|
||||
|
||||
const PRECISION_BITS: i32 = 32 - 8 - 2;
|
||||
/// PIL's `PRECISION_BITS` for 8-bit images: weights quantized to i32.
|
||||
const PIL_PRECISION_BITS: u32 = 32 - 8 - 2;
|
||||
|
||||
/// Resampling filters, bit-exact clones of PIL's kernels.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Filter {
|
||||
/// support 3.0 — PIL `LANCZOS`.
|
||||
Lanczos,
|
||||
/// support 2.0, a = -0.5 — PIL `BICUBIC` (≈ torchvision antialiased
|
||||
/// bicubic, which the HF "fast" image processors use).
|
||||
/// support 2.0, a = -0.5 — PIL `BICUBIC`.
|
||||
Bicubic,
|
||||
}
|
||||
|
||||
/// A resampler reproduced bit-exactly. Both share PIL's geometry, kernels and
|
||||
/// per-pass u8 rounding, and differ only in how the weights are quantized.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Resample {
|
||||
/// PIL `Image.resize`, i32 weights.
|
||||
Pil(Filter),
|
||||
/// ATen's uint8 antialias bicubic — torchvision `resize(antialias=True)` on
|
||||
/// a uint8 tensor. i16 weights, so it rounds unlike `Pil(Bicubic)`.
|
||||
AtenU8,
|
||||
}
|
||||
|
||||
impl Resample {
|
||||
fn filter(self) -> Filter {
|
||||
match self {
|
||||
Resample::Pil(filter) => filter,
|
||||
Resample::AtenU8 => Filter::Bicubic,
|
||||
}
|
||||
}
|
||||
|
||||
/// Fixed-point precision for one axis's already-normalized weights. ATen
|
||||
/// (`_compute_weights_precision`) takes the widest that stays inside i16.
|
||||
fn precision(self, weights: &[f64]) -> u32 {
|
||||
match self {
|
||||
Resample::Pil(_) => PIL_PRECISION_BITS,
|
||||
Resample::AtenU8 => {
|
||||
let wmax = weights.iter().fold(0.0f64, |m, w| m.max(w.abs()));
|
||||
(1..PIL_PRECISION_BITS)
|
||||
.take_while(|&p| (0.5 + wmax * (1u64 << p) as f64) < (1 << 15) as f64)
|
||||
.last()
|
||||
.unwrap_or(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Filter {
|
||||
fn support(self) -> f64 {
|
||||
match self {
|
||||
@@ -60,9 +95,11 @@ struct Coeffs {
|
||||
bounds: Vec<(usize, usize)>,
|
||||
kk: Vec<i32>,
|
||||
ksize: usize,
|
||||
prec: u32,
|
||||
}
|
||||
|
||||
fn precompute_coeffs(in_size: usize, out_size: usize, filter: Filter) -> Coeffs {
|
||||
fn precompute_coeffs(in_size: usize, out_size: usize, resample: Resample) -> Coeffs {
|
||||
let filter = resample.filter();
|
||||
let scale = in_size as f64 / out_size as f64;
|
||||
let filterscale = if scale < 1.0 { 1.0 } else { scale };
|
||||
let support = filter.support() * filterscale;
|
||||
@@ -97,7 +134,8 @@ fn precompute_coeffs(in_size: usize, out_size: usize, filter: Filter) -> Coeffs
|
||||
bounds[xx] = (xmin as usize, count);
|
||||
}
|
||||
|
||||
let factor = (1i64 << PRECISION_BITS) as f64;
|
||||
let prec = resample.precision(&kkf);
|
||||
let factor = (1i64 << prec) as f64;
|
||||
let kk = kkf
|
||||
.iter()
|
||||
.map(|&v| {
|
||||
@@ -108,17 +146,22 @@ fn precompute_coeffs(in_size: usize, out_size: usize, filter: Filter) -> Coeffs
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
Coeffs { bounds, kk, ksize }
|
||||
Coeffs {
|
||||
bounds,
|
||||
kk,
|
||||
ksize,
|
||||
prec,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn clip8(v: i32) -> u8 {
|
||||
if v >= 1 << (PRECISION_BITS + 8) {
|
||||
fn clip8(v: i32, prec: u32) -> u8 {
|
||||
if v >= 1 << (prec + 8) {
|
||||
255
|
||||
} else if v <= 0 {
|
||||
0
|
||||
} else {
|
||||
(v >> PRECISION_BITS) as u8
|
||||
(v >> prec) as u8
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,7 +172,7 @@ fn resample_horizontal(src: &[u8], h: usize, w: usize, out_w: usize, c: &Coeffs)
|
||||
for xx in 0..out_w {
|
||||
let (xmin, count) = c.bounds[xx];
|
||||
let k = &c.kk[xx * c.ksize..xx * c.ksize + count];
|
||||
let mut s = [1i32 << (PRECISION_BITS - 1); 3];
|
||||
let mut s = [1i32 << (c.prec - 1); 3];
|
||||
for (x, &coef) in k.iter().enumerate() {
|
||||
let p = (xmin + x) * 3;
|
||||
s[0] += src_row[p] as i32 * coef;
|
||||
@@ -137,9 +180,9 @@ fn resample_horizontal(src: &[u8], h: usize, w: usize, out_w: usize, c: &Coeffs)
|
||||
s[2] += src_row[p + 2] as i32 * coef;
|
||||
}
|
||||
let o = xx * 3;
|
||||
row[o] = clip8(s[0]);
|
||||
row[o + 1] = clip8(s[1]);
|
||||
row[o + 2] = clip8(s[2]);
|
||||
row[o] = clip8(s[0], c.prec);
|
||||
row[o + 1] = clip8(s[1], c.prec);
|
||||
row[o + 2] = clip8(s[2], c.prec);
|
||||
}
|
||||
});
|
||||
out
|
||||
@@ -151,7 +194,7 @@ fn resample_vertical(src: &[u8], w: usize, out_h: usize, c: &Coeffs) -> Vec<u8>
|
||||
let (ymin, count) = c.bounds[yy];
|
||||
let k = &c.kk[yy * c.ksize..yy * c.ksize + count];
|
||||
for x in 0..w {
|
||||
let mut s = [1i32 << (PRECISION_BITS - 1); 3];
|
||||
let mut s = [1i32 << (c.prec - 1); 3];
|
||||
for (y, &coef) in k.iter().enumerate() {
|
||||
let p = ((ymin + y) * w + x) * 3;
|
||||
s[0] += src[p] as i32 * coef;
|
||||
@@ -159,27 +202,27 @@ fn resample_vertical(src: &[u8], w: usize, out_h: usize, c: &Coeffs) -> Vec<u8>
|
||||
s[2] += src[p + 2] as i32 * coef;
|
||||
}
|
||||
let o = x * 3;
|
||||
row[o] = clip8(s[0]);
|
||||
row[o + 1] = clip8(s[1]);
|
||||
row[o + 2] = clip8(s[2]);
|
||||
row[o] = clip8(s[0], c.prec);
|
||||
row[o + 1] = clip8(s[1], c.prec);
|
||||
row[o + 2] = clip8(s[2], c.prec);
|
||||
}
|
||||
});
|
||||
out
|
||||
}
|
||||
|
||||
/// PIL-exact separable resize of a flat HWC RGB buffer with the given filter.
|
||||
/// Separable resize of a flat HWC RGB buffer, bit-exact against `resample`.
|
||||
///
|
||||
/// Enters the fan-out pool once for both passes; the per-row `for_chunks_mut`
|
||||
/// calls inside then reuse that entry rather than injecting a job per pass.
|
||||
pub fn resize_rgb_filter(
|
||||
pub fn resize_rgb(
|
||||
src: &[u8],
|
||||
h: usize,
|
||||
w: usize,
|
||||
out_h: usize,
|
||||
out_w: usize,
|
||||
filter: Filter,
|
||||
resample: Resample,
|
||||
) -> Vec<u8> {
|
||||
par::in_pool(move || resize_passes(src, h, w, out_h, out_w, filter))
|
||||
par::in_pool(move || resize_passes(src, h, w, out_h, out_w, resample))
|
||||
}
|
||||
|
||||
fn resize_passes(
|
||||
@@ -188,28 +231,23 @@ fn resize_passes(
|
||||
w: usize,
|
||||
out_h: usize,
|
||||
out_w: usize,
|
||||
filter: Filter,
|
||||
resample: Resample,
|
||||
) -> Vec<u8> {
|
||||
let need_h = out_w != w;
|
||||
let need_v = out_h != h;
|
||||
if need_h && need_v {
|
||||
let ch = precompute_coeffs(w, out_w, filter);
|
||||
let tmp = resample_horizontal(src, h, w, out_w, &ch);
|
||||
let cv = precompute_coeffs(h, out_h, filter);
|
||||
resample_vertical(&tmp, out_w, out_h, &cv)
|
||||
} else if need_h {
|
||||
let ch = precompute_coeffs(w, out_w, filter);
|
||||
resample_horizontal(src, h, w, out_w, &ch)
|
||||
} else if need_v {
|
||||
let cv = precompute_coeffs(h, out_h, filter);
|
||||
resample_vertical(src, w, out_h, &cv)
|
||||
} else {
|
||||
src.to_vec()
|
||||
// Per-axis coefficients — and, under `AtenU8`, a per-axis precision.
|
||||
let coeffs = |in_size, out_size| precompute_coeffs(in_size, out_size, resample);
|
||||
match (out_w != w, out_h != h) {
|
||||
(true, true) => {
|
||||
let tmp = resample_horizontal(src, h, w, out_w, &coeffs(w, out_w));
|
||||
resample_vertical(&tmp, out_w, out_h, &coeffs(h, out_h))
|
||||
}
|
||||
(true, false) => resample_horizontal(src, h, w, out_w, &coeffs(w, out_w)),
|
||||
(false, true) => resample_vertical(src, w, out_h, &coeffs(h, out_h)),
|
||||
(false, false) => src.to_vec(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn resize_lanczos_rgb(src: &[u8], h: usize, w: usize, out_h: usize, out_w: usize) -> Vec<u8> {
|
||||
resize_rgb_filter(src, h, w, out_h, out_w, Filter::Lanczos)
|
||||
resize_rgb(src, h, w, out_h, out_w, Resample::Pil(Filter::Lanczos))
|
||||
}
|
||||
|
||||
pub fn scaled_dims(w: usize, h: usize, frac: Option<f64>, cap: Option<i64>) -> (usize, usize) {
|
||||
|
||||
@@ -34,21 +34,60 @@ pub struct QwenVlSpec {
|
||||
pub max_pixels: usize,
|
||||
pub image_mean: [f32; 3],
|
||||
pub image_std: [f32; 3],
|
||||
#[serde(default)]
|
||||
pub resample: Resampler,
|
||||
}
|
||||
|
||||
/// The HF image processor the pipeline must match bit-exactly. Defaults to the
|
||||
/// one a default server runs.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Resampler {
|
||||
/// `Qwen2VLImageProcessor` / `…Fast` — torchvision on a uint8 tensor.
|
||||
#[default]
|
||||
AtenU8,
|
||||
/// `Qwen2VLImageProcessorPil`, behind `--disable-fast-image-processor`.
|
||||
Pil,
|
||||
}
|
||||
|
||||
impl From<Resampler> for resize::Resample {
|
||||
fn from(r: Resampler) -> Self {
|
||||
match r {
|
||||
Resampler::AtenU8 => resize::Resample::AtenU8,
|
||||
Resampler::Pil => resize::Resample::Pil(resize::Filter::Bicubic),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct QwenVlProcessor {
|
||||
spec: QwenVlSpec,
|
||||
/// Per-channel u8 → normalized-f32 lookup: `(v/255 - mean) / std`.
|
||||
/// Per-channel u8 → normalized-f32 lookup; see [`normalize_lut`].
|
||||
lut: [[f32; 256]; 3],
|
||||
}
|
||||
|
||||
/// `1 / rescale_factor`; `resolve_native_spec` rejects any other factor.
|
||||
const INV_RESCALE: f32 = 255.0;
|
||||
|
||||
/// u8 → normalized f32, rounded as the mirrored processor rounds. The slow one
|
||||
/// rescales then normalizes; the fast one folds the rescale into mean/std first
|
||||
/// (`_fuse_mean_std_and_rescale_factor`), which differs on 128 of the 256 inputs.
|
||||
fn normalize_lut(resample: Resampler, mean: f32, std: f32) -> [f32; 256] {
|
||||
match resample {
|
||||
Resampler::Pil => core::array::from_fn(|v| (v as f32 / INV_RESCALE - mean) / std),
|
||||
Resampler::AtenU8 => {
|
||||
let (mean, std) = (mean * INV_RESCALE, std * INV_RESCALE);
|
||||
core::array::from_fn(|v| (v as f32 - mean) / std)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl QwenVlProcessor {
|
||||
pub fn new(spec: QwenVlSpec) -> Result<Self, String> {
|
||||
if spec.patch_size == 0 || spec.merge_size == 0 || spec.temporal_patch_size == 0 {
|
||||
return Err("qwen_vl spec: sizes must be positive".into());
|
||||
}
|
||||
let lut = core::array::from_fn(|c| {
|
||||
core::array::from_fn(|v| (v as f32 / 255.0 - spec.image_mean[c]) / spec.image_std[c])
|
||||
normalize_lut(spec.resample, spec.image_mean[c], spec.image_std[c])
|
||||
});
|
||||
Ok(Self { spec, lut })
|
||||
}
|
||||
@@ -127,7 +166,7 @@ impl MmFamilyProcessor for QwenVlProcessor {
|
||||
)?;
|
||||
let resized;
|
||||
let data = if (th, tw) != (h, w) {
|
||||
resized = resize::resize_rgb_filter(rgb, h, w, th, tw, resize::Filter::Bicubic);
|
||||
resized = resize::resize_rgb(rgb, h, w, th, tw, self.spec.resample.into());
|
||||
&resized
|
||||
} else {
|
||||
rgb.as_slice()
|
||||
@@ -513,6 +552,22 @@ mod tests {
|
||||
max_pixels: 1 << 30,
|
||||
image_mean: [0.0; 3],
|
||||
image_std: [1.0; 3],
|
||||
resample: Resampler::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The fused and unfused normalize forms are not interchangeable: with
|
||||
/// mean = std = 0.5 they disagree on 128 of the 256 u8 inputs, so picking
|
||||
/// the wrong one silently costs bit-exactness with the HF processor.
|
||||
#[test]
|
||||
fn normalize_lut_differs_per_resampler() {
|
||||
let pil = normalize_lut(Resampler::Pil, 0.5, 0.5);
|
||||
let aten = normalize_lut(Resampler::AtenU8, 0.5, 0.5);
|
||||
assert_eq!(pil.iter().zip(aten).filter(|(p, a)| *p != a).count(), 128);
|
||||
// Both still span [-1, 1] — this is rounding, not a scale error.
|
||||
for lut in [pil, aten] {
|
||||
assert_eq!(lut[0], -1.0);
|
||||
assert_eq!(lut[255], 1.0);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -34,15 +34,34 @@ def py_scaled_dims(
|
||||
return scale(width), scale(height)
|
||||
|
||||
|
||||
def pil_resize(arr: np.ndarray, tw: int, th: int) -> np.ndarray:
|
||||
return np.array(
|
||||
Image.fromarray(arr).resize((tw, th), resample=Image.Resampling.LANCZOS),
|
||||
dtype=np.uint8,
|
||||
def pil_resize(arr: np.ndarray, tw: int, th: int, filter=Image.Resampling.LANCZOS):
|
||||
return np.array(Image.fromarray(arr).resize((tw, th), resample=filter), np.uint8)
|
||||
|
||||
|
||||
def tv_resize(arr: np.ndarray, tw: int, th: int) -> np.ndarray:
|
||||
"""torchvision's uint8 antialias bicubic — ATen's fixed-point kernel."""
|
||||
import torch
|
||||
from torchvision.transforms.v2 import functional as F
|
||||
|
||||
tensor = torch.from_numpy(arr).permute(2, 0, 1).unsqueeze(0)
|
||||
out = F.resize(
|
||||
tensor, [th, tw], interpolation=F.InterpolationMode.BICUBIC, antialias=True
|
||||
)
|
||||
return out[0].permute(1, 2, 0).numpy()
|
||||
|
||||
|
||||
def rs_resize(arr: np.ndarray, tw: int, th: int) -> np.ndarray:
|
||||
return _rs_common.resize_rgb(arr, tw, th).reshape(th, tw, 3)
|
||||
# Every resampler the Rust resize claims, and its reference: `aten_u8` for a
|
||||
# default server, `pil_bicubic` for --disable-fast-image-processor, `pil_lanczos`
|
||||
# for inkling.
|
||||
REFERENCES = {
|
||||
"pil_lanczos": lambda a, tw, th: pil_resize(a, tw, th, Image.Resampling.LANCZOS),
|
||||
"pil_bicubic": lambda a, tw, th: pil_resize(a, tw, th, Image.Resampling.BICUBIC),
|
||||
"aten_u8": tv_resize,
|
||||
}
|
||||
|
||||
|
||||
def rs_resize(arr, tw: int, th: int, resample: str = "pil_lanczos") -> np.ndarray:
|
||||
return _rs_common.resize_rgb(arr, tw, th, resample).reshape(th, tw, 3)
|
||||
|
||||
|
||||
CASES = [
|
||||
@@ -58,13 +77,36 @@ CASES = [
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("resample", sorted(REFERENCES))
|
||||
@pytest.mark.parametrize(
|
||||
"h,w,th,tw", CASES, ids=[f"{h}x{w}->{th}x{tw}" for h, w, th, tw in CASES]
|
||||
)
|
||||
def test_resize_bit_exact(h, w, th, tw):
|
||||
def test_resize_bit_exact(h, w, th, tw, resample):
|
||||
rng = np.random.default_rng(h * 10000 + w)
|
||||
arr = rng.integers(0, 256, (h, w, 3), dtype=np.uint8)
|
||||
np.testing.assert_array_equal(rs_resize(arr, tw, th), pil_resize(arr, tw, th))
|
||||
np.testing.assert_array_equal(
|
||||
rs_resize(arr, tw, th, resample), REFERENCES[resample](arr, tw, th)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("resample", sorted(REFERENCES))
|
||||
def test_resize_bit_exact_random_sweep(resample):
|
||||
"""`aten_u8`'s weight precision varies with the scale factor, so the fixed
|
||||
cases above are not enough coverage on their own."""
|
||||
rng = np.random.default_rng(7)
|
||||
for h, w, th, tw in rng.integers(1, 200, (40, 4)):
|
||||
arr = rng.integers(0, 256, (h, w, 3), dtype=np.uint8)
|
||||
np.testing.assert_array_equal(
|
||||
rs_resize(arr, tw, th, resample),
|
||||
REFERENCES[resample](arr, tw, th),
|
||||
err_msg=f"{h}x{w}->{th}x{tw} under {resample}",
|
||||
)
|
||||
|
||||
|
||||
def test_unknown_resample_rejected():
|
||||
arr = np.zeros((4, 4, 3), dtype=np.uint8)
|
||||
with pytest.raises(ValueError, match="unknown resample"):
|
||||
_rs_common.resize_rgb(arr, 2, 2, "nearest")
|
||||
|
||||
|
||||
def test_scaled_dims_sweep():
|
||||
|
||||
Reference in New Issue
Block a user