From b77833c5046238f4aae8074e006c4fdd19016c9d Mon Sep 17 00:00:00 2001 From: Jialin Ouyang Date: Tue, 22 Sep 2026 15:12:07 -0700 Subject: [PATCH] [MM] Keep scheduler padding in packed token arrays (#40357) --- benchmark/io/bench_mm_token_padding.py | 198 ++++++++++++++++++ python/sglang/srt/managers/mm_utils.py | 52 +++-- python/sglang/srt/managers/scheduler.py | 11 +- .../srt/mem_cache/rust_tree_core/adapter.py | 5 +- python/sglang/srt/models/bailing_mm.py | 3 +- python/sglang/srt/models/bailing_mm_v3.py | 3 +- python/sglang/srt/models/clip.py | 3 +- python/sglang/srt/models/cohere2_vision.py | 5 +- python/sglang/srt/models/cosmos3_edge.py | 5 +- .../sglang/srt/models/deepseek_janus_pro.py | 3 +- python/sglang/srt/models/deepseek_ocr.py | 3 +- python/sglang/srt/models/deepseek_v4.py | 3 +- python/sglang/srt/models/deepseek_vl2.py | 3 +- .../srt/models/dots3_common/modeling.py | 7 +- python/sglang/srt/models/dots_ocr.py | 3 +- python/sglang/srt/models/dots_vlm.py | 3 +- python/sglang/srt/models/ernie45_vl.py | 3 +- python/sglang/srt/models/gemma3_mm.py | 5 +- python/sglang/srt/models/gemma3n_mm.py | 5 +- python/sglang/srt/models/gemma4_mm.py | 5 +- python/sglang/srt/models/glm4v.py | 3 +- python/sglang/srt/models/glm5_next.py | 3 +- python/sglang/srt/models/glm_image_vl.py | 3 +- python/sglang/srt/models/glmasr.py | 3 +- python/sglang/srt/models/inkling.py | 3 +- python/sglang/srt/models/interns1.py | 3 +- python/sglang/srt/models/internvl.py | 3 +- python/sglang/srt/models/jet_vlm.py | 5 +- python/sglang/srt/models/kimi_k25.py | 3 +- python/sglang/srt/models/kimi_k3.py | 3 +- python/sglang/srt/models/kimi_vl.py | 3 +- python/sglang/srt/models/lfm2_vl.py | 5 +- python/sglang/srt/models/lightonocr.py | 3 +- python/sglang/srt/models/llava.py | 2 +- python/sglang/srt/models/locate_anything.py | 3 +- python/sglang/srt/models/midashenglm.py | 3 +- python/sglang/srt/models/mimo_v2.py | 3 +- python/sglang/srt/models/mimo_v2_asr.py | 3 +- python/sglang/srt/models/minicpmo.py | 3 +- python/sglang/srt/models/minicpmv.py | 9 +- python/sglang/srt/models/minimax_m3_vl.py | 3 +- python/sglang/srt/models/mllama4.py | 3 +- python/sglang/srt/models/muse_glimmer.py | 3 +- python/sglang/srt/models/nano_nemotron_vl.py | 3 +- python/sglang/srt/models/nvila.py | 5 +- python/sglang/srt/models/nvila_lite.py | 5 +- python/sglang/srt/models/paddleocr_vl.py | 3 +- python/sglang/srt/models/phi4mm.py | 3 +- python/sglang/srt/models/pixtral.py | 5 +- python/sglang/srt/models/points_v15_chat.py | 3 +- python/sglang/srt/models/qwen2_5_vl.py | 3 +- python/sglang/srt/models/qwen2_audio.py | 3 +- python/sglang/srt/models/qwen2_vl.py | 3 +- python/sglang/srt/models/qwen3_asr.py | 3 +- python/sglang/srt/models/qwen3_vl.py | 3 +- python/sglang/srt/models/sarashina2_vision.py | 3 +- python/sglang/srt/models/step3_vl.py | 3 +- python/sglang/srt/models/step3_vl_10b.py | 3 +- python/sglang/srt/models/step3p7.py | 3 +- python/sglang/srt/models/transformers.py | 3 +- python/sglang/srt/models/unlimited_ocr.py | 3 +- python/sglang/srt/models/voxtral.py | 3 +- python/sglang/srt/multimodal/evs/evs_core.py | 10 +- python/sglang/srt/utils/common.py | 7 + 64 files changed, 376 insertions(+), 110 deletions(-) create mode 100644 benchmark/io/bench_mm_token_padding.py diff --git a/benchmark/io/bench_mm_token_padding.py b/benchmark/io/bench_mm_token_padding.py new file mode 100644 index 0000000000..23b5991fe3 --- /dev/null +++ b/benchmark/io/bench_mm_token_padding.py @@ -0,0 +1,198 @@ +"""CPU comparison of baseline and optimized offset-based scheduler padding. + +Run from the repository root: + PYTHONPATH=python python benchmark/io/bench_mm_token_padding.py --baseline-ref BASE_SHA + +Inputs are synthetic. Both paths include the scheduler's final ownership copy. +This measures padding CPU time, not model forward time or end-to-end TTFT. +""" + +import argparse +import ast +import gc +import json +import platform +import statistics +import subprocess +import time +from array import array +from pathlib import Path + +import torch + +from sglang.srt.managers import mm_utils +from sglang.srt.managers.mm_utils import ( + MultiModalityDataPaddingPatternMultimodalTokens, + duplicate_pad_mm_input_ids, +) +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalInputs, +) + + +def load_baseline(ref): + """Load the exact baseline helper classes without replacing installed SGLang.""" + root = Path(__file__).resolve().parents[2] + commit = subprocess.check_output( + ["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], cwd=root, text=True + ).strip() + path = "python/sglang/srt/managers/mm_utils.py" + source = subprocess.check_output( + ["git", "show", f"{commit}:{path}"], cwd=root, text=True + ) + names = { + "MultiModalityDataPaddingPattern", + "MultiModalityDataPaddingPatternMultimodalTokens", + } + classes = [ + node + for node in ast.parse(source).body + if isinstance(node, ast.ClassDef) and node.name in names + ] + if {node.name for node in classes} != names: + raise ValueError("Baseline does not contain the expected padding helpers") + namespace = dict(vars(mm_utils)) + # Reuse dependency imports, but compile the original method bodies verbatim. + exec( + compile(ast.Module(body=classes, type_ignores=[]), f"{commit}:{path}", "exec"), + namespace, + ) + return commit, namespace["MultiModalityDataPaddingPatternMultimodalTokens"]() + + +def make_case(num_tokens, num_items, span_tokens): + if num_items and num_tokens < 3 * num_items: + raise ValueError( + "Each item needs three tokens for its span and surrounding text" + ) + input_ids = array("q", range(257, 257 + num_tokens)) + items = [] + stride = num_tokens // max(num_items, 1) + for index in range(num_items): + start = index * stride + 1 + end = start + min(span_tokens, stride - 2) - 1 + items.append( + MultimodalDataItem( + modality=Modality.IMAGE, + pad_value=2**40 + index, + offsets=[(start, end)], + ) + ) + return input_ids, MultimodalInputs(mm_items=items, im_token_id=1) + + +def benchmark_case(num_tokens, num_items, args, baseline): + pattern = MultiModalityDataPaddingPatternMultimodalTokens() + input_ids, mm_inputs = make_case(num_tokens, num_items, args.span_tokens) + original = input_ids.tobytes() + + def baseline_padding(): + return array("q", baseline.pad_input_tokens(input_ids, mm_inputs)) + + def optimized_padding(): + return duplicate_pad_mm_input_ids( + input_ids, mm_inputs, pattern.pad_input_tokens + ) + + expected = baseline_padding() + actual = optimized_padding() + assert actual == expected, "Token values differ" + assert input_ids.tobytes() == original, "Original input was mutated" + assert actual is not input_ids, "Scheduler result aliases the original input" + + functions = {"baseline": baseline_padding, "optimized": optimized_padding} + timings = {name: [] for name in functions} + for _ in range(args.warmups): + for function in functions.values(): + function() + gc_was_enabled = gc.isenabled() + gc.disable() + try: + for trial in range(args.trials): + order = ( + ("baseline", "optimized") if trial % 2 else ("optimized", "baseline") + ) + for name in order: + start = time.perf_counter_ns() + result = functions[name]() + elapsed = time.perf_counter_ns() - start + timings[name].append(elapsed / 1e6) + del result + finally: + if gc_was_enabled: + gc.enable() + assert input_ids.tobytes() == original + medians = {name: statistics.median(samples) for name, samples in timings.items()} + return { + "tokens": num_tokens, + "items": num_items, + "baseline_ms": medians["baseline"], + "optimized_ms": medians["optimized"], + "speedup": medians["baseline"] / medians["optimized"], + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--baseline-ref", required=True, help="Local pre-change git ref" + ) + parser.add_argument( + "--tokens", nargs="+", type=int, default=[128, 4096, 16384, 131072] + ) + parser.add_argument("--items", nargs="+", type=int, default=[0, 1, 10]) + parser.add_argument("--span-tokens", type=int, default=32) + parser.add_argument("--trials", type=int, default=41) + parser.add_argument("--warmups", type=int, default=5) + parser.add_argument("--threads", type=int, default=1) + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + if ( + min(args.tokens + args.items) < 0 + or min(args.span_tokens, args.trials, args.threads) < 1 + or args.warmups < 0 + ): + parser.error("Lengths/items/warmups must be nonnegative; other counts positive") + if any(n < 3 * items for n in args.tokens for items in args.items): + parser.error("Each token length must accommodate at least 3 tokens per item") + torch.set_num_threads(args.threads) + baseline_commit, baseline = load_baseline(args.baseline_ref) + rows = [ + benchmark_case(tokens, items, args, baseline) + for tokens in args.tokens + for items in args.items + ] + if args.json: + print( + json.dumps( + { + "baseline_commit": baseline_commit, + "python": platform.python_version(), + "torch": torch.__version__, + "architecture": platform.machine(), + "threads": args.threads, + "trials": args.trials, + "warmups": args.warmups, + "span_tokens": args.span_tokens, + "results": rows, + }, + indent=2, + ) + ) + return + print("Offset-based CPU padding; median milliseconds; synthetic inputs") + print( + f"{'Tokens':>8} {'Items':>6} {'Baseline (ms)':>14} {'Optimized (ms)':>14} {'Speedup':>9}" + ) + for row in rows: + print( + f"{row['tokens']:>8} {row['items']:>6} " + f"{row['baseline_ms']:>14.4f} {row['optimized_ms']:>14.4f} " + f"{row['speedup']:>8.2f}x" + ) + + +if __name__ == "__main__": + main() diff --git a/python/sglang/srt/managers/mm_utils.py b/python/sglang/srt/managers/mm_utils.py index e07bfb8c4d..41805b258c 100644 --- a/python/sglang/srt/managers/mm_utils.py +++ b/python/sglang/srt/managers/mm_utils.py @@ -9,9 +9,10 @@ import os import pickle import sys from abc import abstractmethod +from array import array from collections import defaultdict from multiprocessing import shared_memory -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple import numpy as np import torch @@ -49,7 +50,11 @@ from sglang.srt.runtime_context import ( get_server_args, get_serving, ) -from sglang.srt.utils import flatten_nested_list, print_warning_once +from sglang.srt.utils import ( + assert_int64_array, + flatten_nested_list, + print_warning_once, +) from sglang.srt.utils.stale_shm_cleanup import make_shm_name from sglang.utils import logger @@ -232,6 +237,19 @@ class TransportProxyTensor(torch.Tensor): return self._metadata.get("transport_mode", "default") +def duplicate_pad_mm_input_ids( + input_ids: array, + mm_inputs: MultimodalInputs, + pad_input_ids_func: Callable[[array, MultimodalInputs], array], +) -> array: + """Validate and duplicate model-padded tokens for scheduler ownership.""" + padded_ids = pad_input_ids_func(input_ids, mm_inputs) + assert_int64_array(padded_ids, "pad_input_ids result") + # Preserve scheduler ownership: no-op models return the unpadded input, + # and other models can retain the result in multimodal metadata. + return padded_ids[:] + + class MultiModalityDataPaddingPattern: """ Data tokens (like image tokens) often need special handling during padding @@ -240,11 +258,10 @@ class MultiModalityDataPaddingPattern: """ @abstractmethod - def pad_input_tokens( - self, input_ids: List[int], mm_inputs: MultimodalInputs - ) -> List[int]: + def pad_input_tokens(self, input_ids: array, mm_inputs: MultimodalInputs) -> array: """ - Pad the input ids sequence containing data tokens, and replace them with pad_values + Replace data tokens with pad_values using native int64 arrays. + Implementations must not mutate the original input. """ pass @@ -273,12 +290,11 @@ class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern) s for s, _e in data_token_pairs ] - def pad_input_tokens( - self, input_ids: List[int], mm_inputs: MultimodalInputs - ) -> List[int]: + def pad_input_tokens(self, input_ids: array, mm_inputs: MultimodalInputs) -> array: """ This function will replace the data-tokens in between with pad_values accordingly """ + assert_int64_array(input_ids, "input_ids") pad_values = [item.pad_value for item in mm_inputs.mm_items] data_token_pairs = self.data_token_id_pairs mm_inputs.data_offsets = [] @@ -292,7 +308,7 @@ class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern) start_token_ids = {s for s, _e in data_token_pairs} end_tokens_ids = {e for _s, e in data_token_pairs} - padded_ids = [] + padded_ids = array("q") last_idx = 0 data_idx = -1 @@ -314,7 +330,8 @@ class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern) num_tokens = end_idx - start_idx - 1 pad_value = pad_values[data_idx] - padded_ids.extend([pad_value] * num_tokens) + if num_tokens > 0: + padded_ids.extend(array("q", [pad_value]) * num_tokens) last_idx = end_idx @@ -329,17 +346,19 @@ class MultiModalityDataPaddingPatternMultimodalTokens(MultiModalityDataPaddingPa e.g. ...., or