[mm] rust-server: native multimodal processing for Qwen VL (integrate sglang-mm, e2e) (#32365)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Kan Wu
co-authored by
Claude Fable 5
Cursor
Claude Fable 5
Cursor
parent
dea07b348b
commit
32e5d788bd
@@ -88,5 +88,12 @@ def image_bytes(width, height, seed=0):
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def spec_json(config, image_token_id=IMAGE_TOKEN_ID):
|
||||
return json.dumps({"family": "qwen_vl", "image_token_id": image_token_id, **config})
|
||||
def spec_json(config, image_token_id=IMAGE_TOKEN_ID, resample="aten_u8"):
|
||||
return json.dumps(
|
||||
{
|
||||
"family": "qwen_vl",
|
||||
"image_token_id": image_token_id,
|
||||
"resample": resample,
|
||||
**config,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
from tokenizers import Tokenizer, decoders
|
||||
from tokenizers.models import WordLevel
|
||||
from tokenizers.pre_tokenizers import WhitespaceSplit
|
||||
from transformers import (
|
||||
PreTrainedTokenizerFast,
|
||||
Qwen2VLProcessor,
|
||||
Qwen2VLVideoProcessor,
|
||||
)
|
||||
from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
|
||||
Qwen2VLImageProcessor as HfQwenImageProcessor,
|
||||
)
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.multimodal.processors.qwen_vl import ( # noqa: E402
|
||||
QwenVLImageProcessor,
|
||||
)
|
||||
|
||||
register_cpu_ci(est_time=0, suite="base-a-test-cpu", disabled="Qwen test fixtures")
|
||||
|
||||
|
||||
def make_processor(config, image_processor_cls=None):
|
||||
"""A ``QwenVLImageProcessor`` over a tiny hand-built tokenizer.
|
||||
``image_processor_cls`` picks the HF backend; they resample differently."""
|
||||
image_processor_cls = image_processor_cls or HfQwenImageProcessor
|
||||
vocab = [
|
||||
"<unk>",
|
||||
"<|vision_start|>",
|
||||
"<|image_pad|>",
|
||||
"<|vision_end|>",
|
||||
"hello",
|
||||
"<|video_pad|>",
|
||||
"<pad>",
|
||||
]
|
||||
backend = Tokenizer(
|
||||
WordLevel(
|
||||
{token: index for index, token in enumerate(vocab)}, unk_token=vocab[0]
|
||||
)
|
||||
)
|
||||
backend.pre_tokenizer, backend.decoder = WhitespaceSplit(), decoders.Fuse()
|
||||
tokenizer = PreTrainedTokenizerFast(
|
||||
tokenizer_object=backend,
|
||||
unk_token=vocab[0],
|
||||
pad_token=vocab[-1],
|
||||
additional_special_tokens=vocab[1:4] + [vocab[5]],
|
||||
)
|
||||
processor = Qwen2VLProcessor(
|
||||
image_processor=image_processor_cls(**config),
|
||||
video_processor=Qwen2VLVideoProcessor(),
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
hf_config = SimpleNamespace(
|
||||
model_type="qwen2_5_vl",
|
||||
architectures=["Qwen2_5_VLForConditionalGeneration"],
|
||||
vision_start_token_id=1,
|
||||
image_token_id=2,
|
||||
vision_end_token_id=3,
|
||||
video_token_id=5,
|
||||
vision_config=SimpleNamespace(spatial_merge_size=2, tokens_per_second=2),
|
||||
)
|
||||
server_args = SimpleNamespace(
|
||||
# Non-auto: get_resolved_model_impl would choke on a SimpleNamespace.
|
||||
model_impl="sglang",
|
||||
keep_mm_feature_on_device=False,
|
||||
mm_feature_transport="cpu",
|
||||
disable_fast_image_processor=True,
|
||||
skip_tokenizer_init=False,
|
||||
# Read by NativeMmHost._use_feature_shm (single-rank fixture → the
|
||||
# inline zero-copy transport, like the 1-GPU e2e).
|
||||
tp_size=1,
|
||||
dist_init_addr=None,
|
||||
mm_process_config={},
|
||||
mm_io_worker_num=1,
|
||||
mm_processor_worker_num=1,
|
||||
tokenizer_worker_num=1,
|
||||
base_gpu_id=0,
|
||||
)
|
||||
return QwenVLImageProcessor(
|
||||
hf_config, server_args, processor, None, skip_mm_pool=True
|
||||
)
|
||||
|
||||
|
||||
def snapshot(input_ids, output):
|
||||
return {
|
||||
"input_ids": tuple(input_ids),
|
||||
"grids": tuple(
|
||||
tuple(item.image_grid_thw.flatten().tolist()) for item in output.mm_items
|
||||
),
|
||||
"offsets": tuple(item.offsets[0] for item in output.mm_items),
|
||||
"features": np.concatenate(
|
||||
[item.feature.detach().cpu().numpy() for item in output.mm_items]
|
||||
),
|
||||
"mrope": output.mrope_positions.detach().cpu().numpy(),
|
||||
"delta": int(output.mrope_position_delta.item()),
|
||||
"tokens": (output.im_start_id, output.im_token_id, output.im_end_id),
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
"""End-to-end parity at the scheduler-input boundary.
|
||||
|
||||
`test_preprocess.py` pins the `preprocess` binding; this drives the whole native
|
||||
path — the `process_native_mm` driver, then `NativeMmHost.build_native_mm` — and
|
||||
compares every field the scheduler reads against the Python `mm_processor`.
|
||||
Bitwise, for both HF backends: the Rust resize clones PIL's fixed-point bicubic
|
||||
and ATen's uint8 antialias kernel, so whichever one a server is configured with
|
||||
is reproduced exactly.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.managers.rust_server import NativeMmHost # noqa: E402
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from _fixtures import make_processor, snapshot # noqa: E402
|
||||
from _mm_rust_utils import PROCESSOR_CONFIGS, image_bytes, load_core # noqa: E402
|
||||
|
||||
register_cpu_ci(est_time=40, suite="base-a-test-cpu")
|
||||
|
||||
CORE = load_core()
|
||||
DRIVER = getattr(getattr(CORE, "qwen_vl", None), "process_native_mm", None)
|
||||
|
||||
# The fixture tokenizer's vocab (see `_fixtures.make_processor`):
|
||||
# 1 = <|vision_start|>, 2 = <|image_pad|>, 3 = <|vision_end|>, 4 = "hello".
|
||||
PROMPT_PER_IMAGE = [1, 2, 3, 4]
|
||||
|
||||
|
||||
@unittest.skipUnless(DRIVER, "sglang-mm native Qwen driver not built")
|
||||
class TestQwenE2eParity(CustomTestCase):
|
||||
"""The torchvision backend a default server runs."""
|
||||
|
||||
image_processor = "Qwen2VLImageProcessor"
|
||||
|
||||
def setUp(self):
|
||||
import transformers.models.qwen2_vl as qwen2_vl
|
||||
|
||||
self.processor = make_processor(
|
||||
PROCESSOR_CONFIGS["qwen2_5_vl"], getattr(qwen2_vl, self.image_processor)
|
||||
)
|
||||
|
||||
def tearDown(self):
|
||||
self.processor.io_executor.shutdown()
|
||||
self.processor.cpu_executor.shutdown()
|
||||
|
||||
# --- the two paths under comparison ---
|
||||
|
||||
def native_spec(self):
|
||||
"""Resolve the spec through the production gate, so a gate that stops
|
||||
recognizing this image processor fails here too."""
|
||||
from sglang.srt.managers.multimodal_processor import import_processors
|
||||
|
||||
import_processors("sglang.srt.multimodal.processors")
|
||||
# Skip __init__: it would build a processor; reuse the fixture's.
|
||||
host = NativeMmHost.__new__(NativeMmHost)
|
||||
host.model_config = SimpleNamespace(hf_config=self.processor.hf_config)
|
||||
host._processor = self.processor._processor
|
||||
host.server_args = self.processor.server_args
|
||||
spec = host.resolve_native_spec()
|
||||
self.assertIsNotNone(spec, f"gate rejected {self.image_processor}")
|
||||
return spec
|
||||
|
||||
def run_native(self, spec, sources):
|
||||
"""The Rust path: the `process_native_mm` driver, then the drain
|
||||
adapter — the same two steps `RustServer.drain` performs."""
|
||||
ids, features, grids, hashes, offsets, mrope, delta = DRIVER(
|
||||
PROMPT_PER_IMAGE * len(sources), sources, spec.rust_json()
|
||||
)
|
||||
# The shape of Rust's MmHandoff, inline transport (test_build_native_mm
|
||||
# pins the shm shape).
|
||||
handoff = SimpleNamespace(
|
||||
features=features,
|
||||
shm_names=None,
|
||||
grids=grids,
|
||||
hashes=hashes,
|
||||
offsets=offsets,
|
||||
mrope=mrope,
|
||||
mrope_delta=delta,
|
||||
)
|
||||
return snapshot(ids, NativeMmHost.build_native_mm(spec, handoff))
|
||||
|
||||
def run_python(self, sources):
|
||||
"""The reference path: the Python `mm_processor` the scheduler would use."""
|
||||
output = asyncio.run(
|
||||
self.processor.process_mm_data_async(
|
||||
image_data=sources,
|
||||
input_text=PROMPT_PER_IMAGE * len(sources),
|
||||
request_obj=SimpleNamespace(
|
||||
video_data=None, audio_data=None, rid="parity"
|
||||
),
|
||||
)
|
||||
)
|
||||
return snapshot(output.input_ids, output)
|
||||
|
||||
def assert_parity(self, spec, sources):
|
||||
rust, python = self.run_native(spec, sources), self.run_python(sources)
|
||||
for field in ("input_ids", "grids", "offsets", "delta", "tokens"):
|
||||
with self.subTest(field=field):
|
||||
self.assertEqual(rust[field], python[field])
|
||||
with self.subTest(field="mrope"):
|
||||
np.testing.assert_array_equal(rust["mrope"], python["mrope"])
|
||||
with self.subTest(field="features"):
|
||||
# Bytes, not allclose: the scheduler gets these float32 buffers verbatim.
|
||||
self.assertEqual(rust["features"].tobytes(), python["features"].tobytes())
|
||||
|
||||
# --- inputs ---
|
||||
|
||||
def source_forms(self, directory):
|
||||
"""One image in each accepted transport form, plus a two-image batch."""
|
||||
first, second = image_bytes(96, 80), image_bytes(112, 88, 1)
|
||||
path = Path(directory) / "image.png"
|
||||
path.write_bytes(first)
|
||||
return {
|
||||
"raw_bytes": [first],
|
||||
"data_url": ["data:image/png;base64," + base64.b64encode(first).decode()],
|
||||
"file_uri": [path.as_uri()],
|
||||
"two_image_batch": [first, second],
|
||||
}
|
||||
|
||||
def test_parity_across_source_forms(self):
|
||||
spec = self.native_spec()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
for form, sources in self.source_forms(directory).items():
|
||||
with self.subTest(form=form):
|
||||
self.assert_parity(spec, sources)
|
||||
|
||||
|
||||
class TestQwenE2eParityPil(TestQwenE2eParity):
|
||||
"""The PIL backend (`--disable-fast-image-processor`): the same parity
|
||||
suite, plus the transport-invariance check — a property of the native path
|
||||
alone, so running it under one backend is enough."""
|
||||
|
||||
image_processor = "Qwen2VLImageProcessorPil"
|
||||
|
||||
def test_source_form_is_transport_only(self):
|
||||
"""Bytes, a data: URL and a file:// path must yield one identical result."""
|
||||
spec = self.native_spec()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
forms = self.source_forms(directory)
|
||||
features = {
|
||||
form: self.run_native(spec, forms[form])["features"].tobytes()
|
||||
for form in ("raw_bytes", "data_url", "file_uri")
|
||||
}
|
||||
self.assertEqual(len(set(features.values())), 1, sorted(features))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,105 @@
|
||||
"""The mm-item hash contract of the native Qwen path.
|
||||
|
||||
The two paths hash different things, deliberately. Native hashes are `content_hash`
|
||||
over the raw encoded source bytes, computed on an MM worker; the Python path hashes
|
||||
the decoded feature tensor. Both feed `set_pad_value`, so the native drain can skip
|
||||
`hash_feature` on the scheduler loop — the point of precomputing them.
|
||||
|
||||
Field-by-field parity of everything else is `test_e2e_parity.py`.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.managers.mm_utils import hash_feature # noqa: E402
|
||||
from sglang.srt.managers.rust_server import NativeMmHost # noqa: E402
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from _fixtures import make_processor # noqa: E402
|
||||
from _mm_rust_utils import PROCESSOR_CONFIGS, image_bytes, load_core # noqa: E402
|
||||
|
||||
register_cpu_ci(est_time=15, suite="base-a-test-cpu")
|
||||
|
||||
CORE = load_core()
|
||||
DRIVER = getattr(getattr(CORE, "qwen_vl", None), "process_native_mm", None)
|
||||
|
||||
|
||||
def raw_bytes(source):
|
||||
"""The encoded bytes behind any accepted source form."""
|
||||
if isinstance(source, bytes):
|
||||
return source
|
||||
if source.startswith("data:"):
|
||||
return base64.b64decode(source.split(",", 1)[1])
|
||||
return Path(source.removeprefix("file://")).read_bytes()
|
||||
|
||||
|
||||
@unittest.skipUnless(DRIVER, "sglang-mm native Qwen driver not built")
|
||||
class TestQwenNativeMmHashes(CustomTestCase):
|
||||
def setUp(self):
|
||||
from sglang.srt.managers.multimodal_processor import import_processors
|
||||
|
||||
import_processors("sglang.srt.multimodal.processors")
|
||||
self.processor = make_processor(PROCESSOR_CONFIGS["qwen2_5_vl"])
|
||||
|
||||
def tearDown(self):
|
||||
self.processor.io_executor.shutdown()
|
||||
self.processor.cpu_executor.shutdown()
|
||||
|
||||
def native_hashes(self, sources):
|
||||
"""Per-item hashes the Rust driver returns, via the production gate."""
|
||||
host = NativeMmHost.__new__(NativeMmHost)
|
||||
host.model_config = SimpleNamespace(hf_config=self.processor.hf_config)
|
||||
host._processor = self.processor._processor
|
||||
host.server_args = self.processor.server_args
|
||||
spec = host.resolve_native_spec()
|
||||
self.assertIsNotNone(spec, "gate rejected the fixture processor")
|
||||
input_ids = [t for _ in sources for t in (1, 2, 3, 4)]
|
||||
return DRIVER(input_ids, sources, spec.rust_json())[3]
|
||||
|
||||
def test_native_hashes_the_raw_source_bytes(self):
|
||||
"""Same image in any source form hashes identically, because the hash is
|
||||
over the bytes and not over anything the transport changes."""
|
||||
first, second = image_bytes(96, 80), image_bytes(112, 88, 1)
|
||||
data_url = "data:image/png;base64," + base64.b64encode(first).decode()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "image.png"
|
||||
path.write_bytes(first)
|
||||
for sources in ([first], [data_url], [path.as_uri()], [first, second]):
|
||||
with self.subTest(n=len(sources), form=type(sources[0]).__name__):
|
||||
hashes = self.native_hashes(sources)
|
||||
self.assertEqual(
|
||||
list(hashes),
|
||||
[CORE.common.content_hash(raw_bytes(s)) for s in sources],
|
||||
)
|
||||
|
||||
def test_python_hashes_the_feature(self):
|
||||
"""The contrast: `set_pad_value` on the Python path derives its hash from
|
||||
the feature tensor, which is why the native path must precompute one."""
|
||||
output = asyncio.run(
|
||||
self.processor.process_mm_data_async(
|
||||
image_data=[image_bytes(96, 80)],
|
||||
input_text=[1, 2, 3, 4],
|
||||
request_obj=SimpleNamespace(
|
||||
video_data=None, audio_data=None, rid="hash"
|
||||
),
|
||||
)
|
||||
)
|
||||
for item in output.mm_items:
|
||||
expected = hash_feature(item.feature)
|
||||
item.set_pad_value()
|
||||
self.assertEqual(item.hash, expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -5,14 +5,13 @@ Covers ``QwenVlProcessor::process_item`` and ``smart_resize`` in
|
||||
and ``smart_resize_py`` bindings), against the HF Qwen2-VL image processors
|
||||
and the Python ``smart_resize``.
|
||||
|
||||
Both HF processors are pinned, because they resample differently and only one
|
||||
of them is what a server actually runs. On transformers 5.x
|
||||
``Qwen2VLImageProcessor`` is the torchvision path (the ``Fast`` suffix was
|
||||
dropped) and is what ``AutoImageProcessor`` hands SGLang by default;
|
||||
``Qwen2VLImageProcessorPil`` is the PIL path, reachable via
|
||||
``--disable-fast-image-processor``. The Rust resize is a bit-exact clone of
|
||||
PIL's fixed-point kernel, so the PIL processor is asserted exactly and the
|
||||
torchvision one carries the cross-implementation envelope.
|
||||
Both HF processors are pinned, because they resample differently and either can
|
||||
be what a server runs. On transformers 5.x ``Qwen2VLImageProcessor`` is the
|
||||
torchvision path (the ``Fast`` suffix was dropped) and is what
|
||||
``AutoImageProcessor`` hands SGLang by default; ``Qwen2VLImageProcessorPil`` is
|
||||
the PIL path, reachable via ``--disable-fast-image-processor``. The Rust resize
|
||||
clones both kernels, so each is asserted with zero tolerance under the
|
||||
``resample`` its spec selects.
|
||||
"""
|
||||
|
||||
import sys
|
||||
@@ -43,48 +42,39 @@ SIZES = ((640, 480), (1024, 683), (50, 40), (300, 301))
|
||||
|
||||
@unittest.skipUnless(QWEN_CORE, "sglang-mm Qwen binding not built")
|
||||
class TestQwenImagePreprocess(CustomTestCase):
|
||||
def _assert_matches(self, processor, max_diff, mean_diff):
|
||||
def _assert_bit_exact(self, processor, resample):
|
||||
"""Zero tolerance, so any drift in any stage — smart_resize geometry, the
|
||||
fixed-point kernel, rescale/normalize, HF patch order — shows up here
|
||||
instead of being absorbed."""
|
||||
for family, config in PROCESSOR_CONFIGS.items():
|
||||
hf = processor(**config)
|
||||
for index, size in enumerate(SIZES):
|
||||
with self.subTest(family=family, size=size):
|
||||
image = make_image(*size, seed=index)
|
||||
actual, grid = QWEN_CORE.preprocess(
|
||||
image_bytes(*size, seed=index), spec_json(config)
|
||||
image_bytes(*size, seed=index),
|
||||
spec_json(config, resample=resample),
|
||||
)
|
||||
expected = hf(images=[image], return_tensors="pt")
|
||||
self.assertEqual(grid, tuple(expected.image_grid_thw[0].tolist()))
|
||||
diff = np.abs(
|
||||
np.asarray(actual).reshape(expected.pixel_values.shape)
|
||||
- expected.pixel_values.numpy()
|
||||
np.testing.assert_array_equal(
|
||||
np.asarray(actual).reshape(expected.pixel_values.shape),
|
||||
expected.pixel_values.numpy(),
|
||||
)
|
||||
# LessEqual, not Less: max_diff=0.0 is a real bound here.
|
||||
self.assertLessEqual(diff.max(), max_diff)
|
||||
self.assertLessEqual(diff.mean(), mean_diff)
|
||||
|
||||
def test_features_match_pil_processor_exactly(self):
|
||||
"""Against the PIL processor the native path is bit-exact, so this is
|
||||
asserted with zero tolerance: every stage (smart_resize geometry,
|
||||
the fixed-point bicubic kernel, rescale/normalize, HF patch order) is
|
||||
pinned, and any drift in any of them shows up here rather than being
|
||||
absorbed by a tolerance."""
|
||||
from transformers.models.qwen2_vl.image_processing_pil_qwen2_vl import (
|
||||
Qwen2VLImageProcessorPil,
|
||||
)
|
||||
|
||||
self._assert_matches(Qwen2VLImageProcessorPil, max_diff=0.0, mean_diff=0.0)
|
||||
self._assert_bit_exact(Qwen2VLImageProcessorPil, "pil")
|
||||
|
||||
def test_features_match_torchvision_processor_within_envelope(self):
|
||||
"""The torchvision processor is what a default server runs, so its
|
||||
divergence is bounded separately — it is a different antialiased-bicubic
|
||||
implementation, which the bit-exact PIL assertion above says nothing
|
||||
about. Measured worst case is max 0.030 / mean 6.7e-5 (≈2 u8 levels
|
||||
after normalize with the qwen2_vl std)."""
|
||||
def test_features_match_torchvision_processor_exactly(self):
|
||||
from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
|
||||
Qwen2VLImageProcessor,
|
||||
)
|
||||
|
||||
self._assert_matches(Qwen2VLImageProcessor, max_diff=0.035, mean_diff=1e-3)
|
||||
self._assert_bit_exact(Qwen2VLImageProcessor, "aten_u8")
|
||||
|
||||
def test_smart_resize_matches_python(self):
|
||||
from sglang.srt.multimodal.processors.qwen_vl import smart_resize
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
"""``NativeMmHost.build_native_mm`` (managers/rust_server.py): the drain-time
|
||||
wrapping contracts — tensors are zero-copy views over the Rust-owned buffers, and
|
||||
pad values come from worker-precomputed hashes, since the scheduler loop must
|
||||
never hash features. Synthetic buffers, so this needs no Rust extension."""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.managers.rust_server import NativeMmHost, NativeMmSpec # noqa: E402
|
||||
|
||||
register_cpu_ci(est_time=3, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestBuildNativeMm(CustomTestCase):
|
||||
def setUp(self):
|
||||
# feature_dim == 3 * temporal_patch_size * patch_size**2 == 6.
|
||||
self.spec = NativeMmSpec(
|
||||
family="qwen_vl",
|
||||
feature_shm=False,
|
||||
image_token_id=10,
|
||||
patch_size=1,
|
||||
merge_size=1,
|
||||
temporal_patch_size=2,
|
||||
min_pixels=1,
|
||||
max_pixels=1 << 30,
|
||||
image_mean=(0.0, 0.0, 0.0),
|
||||
image_std=(1.0, 1.0, 1.0),
|
||||
resample="aten_u8",
|
||||
vision_start_token_id=11,
|
||||
vision_end_token_id=12,
|
||||
video_token_id=13,
|
||||
)
|
||||
|
||||
GRIDS = [(1, 2, 2), (1, 1, 1)]
|
||||
HASHES = [101, 202]
|
||||
OFFSETS = [(2, 5), (8, 8)]
|
||||
|
||||
def transport(self, features):
|
||||
"""Inline: the features ride the numpy array itself."""
|
||||
return dict(features=features, shm_names=None)
|
||||
|
||||
def build(self):
|
||||
features = np.arange(30, dtype=np.float32)
|
||||
output = NativeMmHost.build_native_mm(
|
||||
self.spec,
|
||||
SimpleNamespace( # the shape of Rust's MmHandoff
|
||||
grids=self.GRIDS,
|
||||
hashes=self.HASHES,
|
||||
offsets=self.OFFSETS,
|
||||
mrope=np.arange(30, dtype=np.int64),
|
||||
mrope_delta=-3,
|
||||
**self.transport(features),
|
||||
),
|
||||
)
|
||||
return output, features
|
||||
|
||||
def test_wraps_and_slices_native_buffers(self):
|
||||
output, features = self.build()
|
||||
self.assertEqual(
|
||||
[tuple(item.feature.shape) for item in output.mm_items], [(4, 6), (1, 6)]
|
||||
)
|
||||
self.assertEqual([item.hash for item in output.mm_items], [101, 202])
|
||||
self.assertEqual(
|
||||
[item.offsets for item in output.mm_items], [[(2, 5)], [(8, 8)]]
|
||||
)
|
||||
self.assertEqual(tuple(output.mrope_positions.shape), (3, 10))
|
||||
self.assertEqual(output.mrope_position_delta.item(), -3)
|
||||
self.assertEqual(
|
||||
(output.im_start_id, output.im_token_id, output.im_end_id), (11, 10, 12)
|
||||
)
|
||||
features[0] = 99
|
||||
self.assertEqual(output.mm_items[0].feature[0, 0].item(), 99)
|
||||
|
||||
def test_optional_pad_values_use_precomputed_hashes(self):
|
||||
from sglang.srt.managers.schedule_batch import _compute_pad_value
|
||||
|
||||
# The whole point of worker-precomputed hashes is that the scheduler
|
||||
# loop never runs hash_feature — make any call a hard failure.
|
||||
with (
|
||||
patch.dict(os.environ, {"SGLANG_MM_PRECOMPUTE_HASH": "1"}),
|
||||
patch(
|
||||
"sglang.srt.managers.mm_utils.hash_feature",
|
||||
side_effect=AssertionError("scheduler loop must not hash features"),
|
||||
),
|
||||
):
|
||||
output, _ = self.build()
|
||||
self.assertEqual(
|
||||
[item.pad_value for item in output.mm_items],
|
||||
[_compute_pad_value(101), _compute_pad_value(202)],
|
||||
)
|
||||
|
||||
|
||||
class TestBuildNativeMmShm(TestBuildNativeMm):
|
||||
"""The shm entry shape (TP>1): features arrive as named POSIX segments, and
|
||||
each item becomes a ``ShmPointerMMData`` stub whose ``materialize()`` yields
|
||||
that item's slice — and unlinks, taking the cleanup duty exactly once."""
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self._segments = []
|
||||
|
||||
def tearDown(self):
|
||||
# Defensive: unlink anything a failing test left behind.
|
||||
for shm in self._segments:
|
||||
try:
|
||||
shm.close()
|
||||
shm.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
def _park(self, features):
|
||||
from multiprocessing import shared_memory
|
||||
|
||||
names, row = [], 0
|
||||
for t, h, w in self.GRIDS:
|
||||
n = t * h * w
|
||||
payload = features[row * 6 : (row + n) * 6].tobytes()
|
||||
shm = shared_memory.SharedMemory(create=True, size=len(payload))
|
||||
shm.buf[:] = payload
|
||||
self._segments.append(shm)
|
||||
names.append(shm.name)
|
||||
row += n
|
||||
return names
|
||||
|
||||
def transport(self, features):
|
||||
"""Shm: the worker parked each item's slice in its own segment."""
|
||||
return dict(features=None, shm_names=self._park(features))
|
||||
|
||||
def test_wraps_and_slices_native_buffers(self):
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.mm_utils import ShmPointerMMData
|
||||
|
||||
output, features = self.build()
|
||||
for item in output.mm_items:
|
||||
self.assertIsInstance(item.feature, ShmPointerMMData)
|
||||
# The stub is a zero-copy view over the segment until materialized.
|
||||
self.assertEqual(
|
||||
[tuple(item.feature.shape) for item in output.mm_items], [(4, 6), (1, 6)]
|
||||
)
|
||||
self.assertEqual(
|
||||
[item.feature.precomputed_hash for item in output.mm_items], self.HASHES
|
||||
)
|
||||
tensors = [item.feature.materialize() for item in output.mm_items]
|
||||
expected = torch.from_numpy(features).reshape(-1, 6)
|
||||
self.assertTrue(torch.equal(tensors[0], expected[:4]))
|
||||
self.assertTrue(torch.equal(tensors[1], expected[4:]))
|
||||
# materialize() unlinked: the names must be gone.
|
||||
from multiprocessing import shared_memory
|
||||
|
||||
for item in output.mm_items:
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
shared_memory.SharedMemory(name=item.feature.shm_name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,61 @@
|
||||
"""The native-MM launch gate's family selection (managers/rust_server.py).
|
||||
|
||||
``NATIVE_MM_FAMILIES`` decides which models the Rust pipeline serves natively;
|
||||
for everything else ``native_mm_family_for`` must return ``None``, which
|
||||
``RustServer.launch`` turns into a hard launch error. Pins that non-Qwen
|
||||
multimodal models — Inkling being the in-tree case — keep their Python
|
||||
processor and never match a native family, so growing the registry cannot
|
||||
silently reroute them.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.managers.multimodal_processor import ( # noqa: E402
|
||||
get_mm_processor_cls,
|
||||
import_processors,
|
||||
)
|
||||
from sglang.srt.managers.rust_server import native_mm_family_for # noqa: E402
|
||||
|
||||
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
def processor_cls_for(architecture, model_type):
|
||||
"""Through the production selection, as `resolve_native_spec` calls it."""
|
||||
hf_config = SimpleNamespace(architectures=[architecture], model_type=model_type)
|
||||
return get_mm_processor_cls(hf_config, SimpleNamespace(model_impl="sglang"))
|
||||
|
||||
|
||||
class TestNativeMmGate(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
import_processors("sglang.srt.multimodal.processors")
|
||||
|
||||
def test_qwen_vl_resolves_its_family(self):
|
||||
cls = processor_cls_for("Qwen2_5_VLForConditionalGeneration", "qwen2_5_vl")
|
||||
family = native_mm_family_for(cls, "qwen2_5_vl")
|
||||
self.assertEqual(family and family.name, "qwen_vl")
|
||||
|
||||
def test_inkling_keeps_its_python_processor(self):
|
||||
from sglang.srt.multimodal.processors.inkling import InklingMultimodalProcessor
|
||||
|
||||
cls = processor_cls_for("InklingForConditionalGeneration", "inkling_model")
|
||||
self.assertIs(cls, InklingMultimodalProcessor)
|
||||
self.assertIsNone(native_mm_family_for(cls, "inkling_model"))
|
||||
|
||||
def test_family_requires_both_processor_and_model_type(self):
|
||||
qwen = processor_cls_for("Qwen2_5_VLForConditionalGeneration", "qwen2_5_vl")
|
||||
self.assertIsNone(native_mm_family_for(qwen, "inkling_model"))
|
||||
# Identity, not name: an override class must not match (the
|
||||
# SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE contract).
|
||||
impostor = type("QwenVLImageProcessor", (), {})
|
||||
self.assertIsNone(native_mm_family_for(impostor, "qwen2_5_vl"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,54 @@
|
||||
"""``RustServer._partition_cores`` (managers/rust_server.py): the pool cores must
|
||||
be a *bounded* slice of this rank's allowed cores, not the whole remainder —
|
||||
sibling TP ranks share the NUMA node, so an unbounded mask lets MM preprocessing
|
||||
bursts preempt a sibling's CUDA-launch thread (measured: ~20 ms of ViT wall time
|
||||
per image request on TP4). Pure computation, so no Rust extension needed."""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.managers.rust_server import RustServer # noqa: E402
|
||||
|
||||
register_cpu_ci(est_time=1, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
def partition(node_cores, **kwargs):
|
||||
with patch("os.sched_getaffinity", return_value=set(node_cores), create=True):
|
||||
return RustServer._partition_cores(**kwargs)
|
||||
|
||||
|
||||
class TestPartitionCores(CustomTestCase):
|
||||
def test_pool_is_bounded_not_the_node_remainder(self):
|
||||
# A 120-core NUMA node shared with sibling TP ranks: the pools must
|
||||
# NOT get cores 2..119.
|
||||
launch, pool = partition(range(120), mm_workers=8)
|
||||
self.assertEqual(launch, [0, 1])
|
||||
self.assertEqual(pool, list(range(2, 14))) # max(8, 8 + 4) after reserve
|
||||
|
||||
def test_budget_scales_with_mm_workers_with_a_floor(self):
|
||||
_, pool_text = partition(range(120), mm_workers=0)
|
||||
_, pool_mm = partition(range(120), mm_workers=16)
|
||||
self.assertEqual(len(pool_text), 8) # the floor covers the I/O threads
|
||||
self.assertEqual(len(pool_mm), 20)
|
||||
|
||||
def test_small_allowance_degrades_gracefully(self):
|
||||
# Fewer allowed cores than the budget: take what exists after the
|
||||
# launch reserve, never raise.
|
||||
launch, pool = partition(range(6), mm_workers=8)
|
||||
self.assertEqual(launch, [0]) # min(2, 6 // 4) == 1
|
||||
self.assertEqual(pool, list(range(1, 6)))
|
||||
# Below the split threshold: unpinned.
|
||||
self.assertEqual(partition(range(3), mm_workers=8), (None, None))
|
||||
|
||||
def test_launch_and_pool_cores_are_disjoint(self):
|
||||
launch, pool = partition(range(32), mm_workers=8)
|
||||
self.assertFalse(set(launch) & set(pool))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,129 @@
|
||||
"""E2E: Rust tokenizer-manager native multimodal path (``SGLANG_RUST_SERVER=1``).
|
||||
|
||||
Covers what the CPU parity units structurally cannot: the sidecar handoff, the
|
||||
drain ordering, Rust-side tokenization of multimodal prompts, and the rejection
|
||||
of inputs outside the native pipeline's scope (there is no Python fallback).
|
||||
"""
|
||||
|
||||
import base64
|
||||
import importlib.util
|
||||
import io
|
||||
import os
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=600, stage="base-b", runner_config="1-gpu-large")
|
||||
|
||||
IMAGE_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/man_ironing_on_back_of_suv.png"
|
||||
VISION_BLOCK = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||
|
||||
|
||||
def chat_prompt(question, image_count=1):
|
||||
return (
|
||||
f"<|im_start|>user\n{VISION_BLOCK * image_count}{question}<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
|
||||
def solid_image_data_url(fmt):
|
||||
buffer = io.BytesIO()
|
||||
Image.fromarray(np.full((64, 64, 3), (255, 0, 0), dtype=np.uint8)).save(
|
||||
buffer, format=fmt
|
||||
)
|
||||
encoded = base64.b64encode(buffer.getvalue()).decode()
|
||||
return f"data:image/{fmt.lower()};base64,{encoded}"
|
||||
|
||||
|
||||
@unittest.skipIf(
|
||||
importlib.util.find_spec("sglang.srt.server._core") is None,
|
||||
"sglang-server rust extension not installed (e.g. AMD suite)",
|
||||
)
|
||||
class TestRustServerNativeMm(CustomTestCase):
|
||||
env = {"SGLANG_RUST_SERVER": "1"}
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.process = popen_launch_server(
|
||||
DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=["--enable-multimodal", "--mem-fraction-static", "0.8"],
|
||||
env={**os.environ, **cls.env},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
if hasattr(cls, "process") and cls.process:
|
||||
kill_process_tree(cls.process.pid)
|
||||
|
||||
def generate(self, prompt, image_data):
|
||||
response = requests.post(
|
||||
DEFAULT_URL_FOR_TEST + "/generate",
|
||||
json={
|
||||
"text": prompt,
|
||||
"image_data": image_data,
|
||||
"sampling_params": {"temperature": 0, "max_new_tokens": 48},
|
||||
},
|
||||
)
|
||||
self.assertEqual(response.status_code, 200, response.text)
|
||||
return response.json()["text"].lower()
|
||||
|
||||
def test_single_image_url(self):
|
||||
text = self.generate(
|
||||
chat_prompt("Describe this image in one sentence."), [IMAGE_URL]
|
||||
)
|
||||
keywords = ("iron", "man", "taxi", "cab", "car", "suv", "street")
|
||||
self.assertTrue(any(w in text for w in keywords), text)
|
||||
|
||||
def test_two_images(self):
|
||||
red = solid_image_data_url("PNG")
|
||||
text = self.generate(
|
||||
chat_prompt("What color is the second image?", image_count=2),
|
||||
[IMAGE_URL, red],
|
||||
)
|
||||
self.assertIn("red", text)
|
||||
|
||||
def test_unsupported_format_is_rejected(self):
|
||||
# An undecodable format must be rejected, never silently answered, and
|
||||
# must not crash the server. PCX is the probe because feature unification
|
||||
# in the server binary (dynamo-parsers → openai-harmony) widens
|
||||
# sglang-mm's jpeg/png/webp/gif/bmp set to every image-crate default, so
|
||||
# the probe has to be a format the image crate does not know at all.
|
||||
response = requests.post(
|
||||
DEFAULT_URL_FOR_TEST + "/generate",
|
||||
json={
|
||||
"text": chat_prompt("What color is this image?"),
|
||||
"image_data": [solid_image_data_url("PCX")],
|
||||
"sampling_params": {"max_new_tokens": 8},
|
||||
},
|
||||
)
|
||||
self.assertIn(response.status_code, (400, 500), response.text)
|
||||
|
||||
def test_corrupt_image_is_rejected(self):
|
||||
response = requests.post(
|
||||
DEFAULT_URL_FOR_TEST + "/generate",
|
||||
json={
|
||||
"text": chat_prompt("Describe this image."),
|
||||
"image_data": ["data:image/png;base64,aW52YWxpZA=="],
|
||||
"sampling_params": {"max_new_tokens": 8},
|
||||
},
|
||||
)
|
||||
# Surfaced as Error::Encode (500); rejected without killing the server.
|
||||
self.assertIn(response.status_code, (400, 500), response.text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=3)
|
||||
@@ -0,0 +1,168 @@
|
||||
"""MMMU accuracy gate for the Rust tokenizer manager's native multimodal path.
|
||||
|
||||
``test_rust_native_mm_e2e.py`` checks that the output is *valid*; this checks that
|
||||
native Rust preprocessing yields *equally good* model inputs. A systematic skew
|
||||
(wrong resample filter, channel order, normalization, patch layout) still reads as
|
||||
fluent text and passes a keyword smoke check, but drops MMMU below the gate.
|
||||
|
||||
The rust server has no ``/v1/chat/completions`` route yet, so the eval drives
|
||||
``/generate`` with hand-rendered Qwen chat prompts instead of lmms-eval's OpenAI
|
||||
client (``MMMUMixin``).
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.simple_eval_common import MessageList, SamplerBase
|
||||
from sglang.test.simple_eval_mmmu_vlm import MMMUVLMEval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
CustomTestCase,
|
||||
dump_metric,
|
||||
popen_launch_server,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=900, stage="base-b", runner_config="1-gpu-large")
|
||||
|
||||
MODEL = "Qwen/Qwen3.5-0.8B"
|
||||
VISION_BLOCK = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||
|
||||
NUM_EXAMPLES = 100
|
||||
# Calibrated 2026-07-24 on H200: the native path scores 0.37 on this fixed subset
|
||||
# at temperature 0 (two runs), matching the Python reference (0.37, same sampler
|
||||
# and samples). The gate leaves headroom for batching nondeterminism.
|
||||
MMMU_ACCURACY_THRESHOLD = 0.30
|
||||
|
||||
|
||||
class QwenGenerateVisionSampler(SamplerBase):
|
||||
"""Drive ``/generate`` with Qwen chat prompts and ``image_data``.
|
||||
|
||||
``MMMUVLMEval`` emits OpenAI-style messages mixing ``text`` and ``image_url``
|
||||
parts. This sampler renders the Qwen chat format by hand — each image part
|
||||
becomes a ``VISION_BLOCK`` at its original position — and ships the images
|
||||
through ``image_data``.
|
||||
"""
|
||||
|
||||
def __init__(self, base_url: str, max_tokens: int = 1024):
|
||||
self.generate_url = base_url + "/generate"
|
||||
self.max_tokens = max_tokens
|
||||
|
||||
def __call__(self, message_list: MessageList) -> str:
|
||||
segments = []
|
||||
images = []
|
||||
for message in message_list:
|
||||
content = message["content"]
|
||||
parts = (
|
||||
[{"type": "text", "text": content}]
|
||||
if isinstance(content, str)
|
||||
else content
|
||||
)
|
||||
for part in parts:
|
||||
if part["type"] == "image_url":
|
||||
images.append(part["image_url"]["url"])
|
||||
segments.append(VISION_BLOCK)
|
||||
else:
|
||||
segments.append(part["text"])
|
||||
prompt = (
|
||||
"<|im_start|>user\n"
|
||||
+ "".join(segments)
|
||||
+ "<|im_end|>\n<|im_start|>assistant\n"
|
||||
)
|
||||
payload = {
|
||||
"text": prompt,
|
||||
"image_data": images,
|
||||
"sampling_params": {
|
||||
"temperature": 0,
|
||||
"max_new_tokens": self.max_tokens,
|
||||
},
|
||||
}
|
||||
# Retry transient failures but fail loudly when they persist: returning ""
|
||||
# would silently degrade the score and blur the gate.
|
||||
for attempt in range(3):
|
||||
try:
|
||||
response = requests.post(self.generate_url, json=payload, timeout=600)
|
||||
response.raise_for_status()
|
||||
return response.json()["text"]
|
||||
except requests.RequestException:
|
||||
if attempt == 2:
|
||||
raise
|
||||
time.sleep(2**attempt)
|
||||
|
||||
|
||||
@unittest.skipIf(
|
||||
importlib.util.find_spec("sglang.srt.server._core") is None,
|
||||
"sglang-server rust extension not installed (e.g. AMD suite)",
|
||||
)
|
||||
class TestRustNativeMmMMMU(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# Capture the server log so the test can pin that the native MM
|
||||
# pipeline is active.
|
||||
cls.log_dir = tempfile.TemporaryDirectory()
|
||||
cls.server_logs = tuple(
|
||||
open(os.path.join(cls.log_dir.name, name), "w")
|
||||
for name in ("stdout.log", "stderr.log")
|
||||
)
|
||||
cls.process = popen_launch_server(
|
||||
MODEL,
|
||||
DEFAULT_URL_FOR_TEST,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=["--enable-multimodal", "--mem-fraction-static", "0.8"],
|
||||
env={**os.environ, "SGLANG_RUST_SERVER": "1"},
|
||||
return_stdout_stderr=cls.server_logs,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
if hasattr(cls, "process") and cls.process:
|
||||
kill_process_tree(cls.process.pid)
|
||||
if hasattr(cls, "server_logs"):
|
||||
for f in cls.server_logs:
|
||||
f.close()
|
||||
if hasattr(cls, "log_dir"):
|
||||
cls.log_dir.cleanup()
|
||||
|
||||
def _read_server_log(self):
|
||||
text = []
|
||||
for f in self.server_logs:
|
||||
with open(f.name) as reader:
|
||||
text.append(reader.read())
|
||||
return "\n".join(text)
|
||||
|
||||
def test_mmmu_accuracy(self):
|
||||
# Guard the path under test: if the model ever drops off
|
||||
# NATIVE_MM_FAMILIES, launch fails and this names why.
|
||||
self.assertIn(
|
||||
"native MM pipeline enabled",
|
||||
self._read_server_log(),
|
||||
"rust server did not enable the native MM pipeline for "
|
||||
f"{MODEL}; this test must exercise the native path",
|
||||
)
|
||||
|
||||
eval_obj = MMMUVLMEval(num_examples=NUM_EXAMPLES, num_threads=32)
|
||||
sampler = QwenGenerateVisionSampler(base_url=DEFAULT_URL_FOR_TEST)
|
||||
result = eval_obj(sampler)
|
||||
print(f"MMMU metrics: {result.metrics}")
|
||||
dump_metric(
|
||||
"mmmu_score",
|
||||
result.score,
|
||||
labels={"model": MODEL, "eval": "mmmu", "api": "generate-rust-native-mm"},
|
||||
)
|
||||
self.assertGreaterEqual(
|
||||
result.score,
|
||||
MMMU_ACCURACY_THRESHOLD,
|
||||
f"Rust native MM path scored {result.score:.4f} on MMMU, below the "
|
||||
f"{MMMU_ACCURACY_THRESHOLD:.2f} gate",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=3)
|
||||
Reference in New Issue
Block a user