[mm] refactor mm code for rust tokenizer manager (#34660)

Co-authored-by: Rain Jiang <rain-jiang@outlook.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

test/registered/unit/multimodal/rust/qwen/test_e2e_parity.py

@@ -1,7 +1,7 @@
 """End-to-end parity at the scheduler-input boundary.
 
 `test_preprocess.py` pins the `preprocess` binding; this drives the whole native
-path — the `process_mm` driver, then `RustMmProcessor.build_output` — and
+path — the `process_mm` driver, then `RustMmProcessor.wrap_encoded` — and
 compares every field the scheduler reads against the Python `mm_processor`.
 Bitwise, for both HF backends: the Rust resize clones PIL's fixed-point bicubic
 and ATen's uint8 antialias kernel, so whichever one a server is configured with
@@ -82,9 +82,9 @@ class TestQwenE2eParity(CustomTestCase):
         ids, features, grids, hashes, offsets, mrope, delta = DRIVER(
             PROMPT_PER_IMAGE * len(sources), sources, spec.rust_json()
         )
-        # The shape of Rust's MmEncodeResult, inline transport (test_build_output
-        # pins the shm shape).
-        handoff = SimpleNamespace(
+        # The shape of Rust's MmEncodedResult, inline transport
+        # (test_wrap_encoded pins the shm shape).
+        encoded = SimpleNamespace(
             features=features,
             shm_names=None,
             grids=grids,
@@ -93,7 +93,7 @@ class TestQwenE2eParity(CustomTestCase):
             mrope=mrope,
             mrope_delta=delta,
         )
-        return snapshot(ids, RustMmProcessor.build_output(spec, handoff))
+        return snapshot(ids, RustMmProcessor.wrap_encoded(spec, encoded))
 
     def run_python(self, sources):
         """The reference path: the Python `mm_processor` the scheduler would use."""

test/registered/unit/multimodal/rust/qwen/test_rust_mm_processor.py

@@ -49,6 +49,8 @@ class TestQwenRustMmHashes(CustomTestCase):
     def setUp(self):
         from sglang.srt.managers.multimodal_processor import import_processors
 
+        # The hash helper builds RustMmProcessor via __new__ (no __init__),
+        # so processors must be registered here for resolve_spec's lookup.
         import_processors("sglang.srt.multimodal.processors")
         self.processor = make_processor(self, PROCESSOR_CONFIGS["qwen2_5_vl"])
 

test/registered/unit/multimodal/rust/shared/test_wrap_encoded.py

@@ -1,4 +1,4 @@
-"""``RustMmProcessor.build_output``: the drain-time
+"""``RustMmProcessor.wrap_encoded``: the drain-time
 wrapping contracts — tensors are zero-copy views over the Rust-owned buffers, and
 pad values come from worker-precomputed hashes, since the scheduler loop must
 never hash features. Synthetic buffers, so this needs no Rust extension."""
@@ -23,7 +23,7 @@ from sglang.srt.rust_server.multimodal import (  # noqa: E402
 register_cpu_ci(est_time=3, suite="base-a-test-cpu")
 
 
-class TestBuildRustMmOutput(CustomTestCase):
+class TestWrapEncoded(CustomTestCase):
     def setUp(self):
         # feature_dim == 3 * temporal_patch_size * patch_size**2 == 6.
         self.spec = RustMmSpec(
@@ -53,9 +53,9 @@ class TestBuildRustMmOutput(CustomTestCase):
 
     def build(self):
         features = np.arange(30, dtype=np.float32)
-        output = RustMmProcessor.build_output(
+        output = RustMmProcessor.wrap_encoded(
             self.spec,
-            SimpleNamespace(  # the shape of Rust's MmEncodeResult
+            SimpleNamespace(  # the shape of Rust's MmEncodedResult
                 grids=self.GRIDS,
                 hashes=self.HASHES,
                 offsets=self.OFFSETS,
@@ -102,7 +102,7 @@ class TestBuildRustMmOutput(CustomTestCase):
         )
 
 
-class TestBuildRustMmOutputShm(TestBuildRustMmOutput):
+class TestWrapEncodedShm(TestWrapEncoded):
     """The shm entry shape (TP>1): features arrive as named POSIX segments, and
     each item becomes a ``ShmPointerMMData`` stub whose ``materialize()`` yields
     that item's slice — and unlinks, taking the cleanup duty exactly once."""