From 77fc5c128e170db43d0abb3498062f4b1027af34 Mon Sep 17 00:00:00 2001 From: Mick Date: Wed, 19 Aug 2026 08:20:55 +0800 Subject: [PATCH] [perf] overlap page preprocessing, pack the vit, enable prefill CUDA graph for paddle-ocr (#35318) Co-authored-by: Claude Opus 5 --- .../autoregressive/Baidu/PaddleOCR-VL.mdx | 275 ++++++++++ .../autoregressive/Baidu/Unlimited-OCR.mdx | 1 - docs/cookbook/autoregressive/intro.mdx | 2 +- docs/docs.json | 1 + .../multimodal_language_models.mdx | 6 + .../configs/PaddlePaddle/paddleocr-vl.jsx | 295 +++++++++++ python/sglang/srt/configs/model_config.py | 1 + python/sglang/srt/models/ernie4.py | 3 + python/sglang/srt/models/paddleocr_vl.py | 490 ++++++++---------- .../multimodal/processors/paddleocr_vlm.py | 15 + .../test_paddleocr_vl_serving_defaults.py | 69 +++ .../unit/models/test_paddleocr_vl_vision.py | 268 ++++++++++ .../vlm/test_paddleocr_vl_server.py | 117 +++++ 13 files changed, 1278 insertions(+), 265 deletions(-) create mode 100644 docs/cookbook/autoregressive/Baidu/PaddleOCR-VL.mdx create mode 100644 docs/src/snippets/configs/PaddlePaddle/paddleocr-vl.jsx create mode 100644 test/registered/unit/models/test_paddleocr_vl_serving_defaults.py create mode 100644 test/registered/unit/models/test_paddleocr_vl_vision.py create mode 100644 test/registered/vlm/test_paddleocr_vl_server.py diff --git a/docs/cookbook/autoregressive/Baidu/PaddleOCR-VL.mdx b/docs/cookbook/autoregressive/Baidu/PaddleOCR-VL.mdx new file mode 100644 index 000000000..b29ee0774 --- /dev/null +++ b/docs/cookbook/autoregressive/Baidu/PaddleOCR-VL.mdx @@ -0,0 +1,275 @@ +--- +title: PaddleOCR-VL +description: "Deploy PaddleOCR-VL 1.6 / 1.5 / 0.9B with SGLang — Baidu's 0.9B NaViT + ERNIE-4.5 document-parsing VLM for OCR, tables, formulas and charts in 109 languages, on a single H100, H200 or B200." +tag: NEW +--- + +## Deployment + + + + + +For all methods and hardware platforms, see the [official SGLang installation guide](../../../docs/get-started/install). The two paths below match the **Python / Docker** toggle in the command panel. + + + + + +```bash Command +pip install --upgrade pip +pip install uv +uv pip install sglang +``` + +Then run the **Python** output of the command panel below in that environment. + + + + + +```bash Command +docker pull lmsysorg/sglang:dev +``` + +For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces. + + + + + + + +Pick a release and your hardware to generate the launch command. The model is 0.9B and single-GPU, so there is one serving recipe per platform; the axis that actually moves cost is **Page Resolution**, which caps how many image tokens one page is worth. + +import { Deployment } from "/src/snippets/_deployment.jsx"; +import { config } from "/src/snippets/configs/PaddlePaddle/paddleocr-vl.jsx"; + + + +## Playground + +Use the Playground to layer tensor parallelism on top of the selected deployment cell. At this size TP is a latency knob, not a capacity one — the weights fit on one GPU. + +import { Playground } from "/src/snippets/_playground.jsx"; + + + +## 1. Model Introduction + +**PaddleOCR-VL** is Baidu's compact document-parsing vision-language model: a NaViT-style dynamic-resolution SigLIP vision encoder feeding an **ERNIE-4.5-0.3B** language backbone, 0.9B parameters in total, released under **Apache 2.0**. It targets end-to-end page parsing — text, tables, formulas, charts, seals and reading order — across **109 languages**, and is small enough that a single GPU serves it comfortably. + +All three releases share an identical `config.json` (same `PaddleOCRVLForConditionalGeneration` architecture, same tower and backbone dimensions), so one SGLang recipe serves every variant and only the model path changes. + + + + + + + + + + + + + + + + + + + + + + + + + + +
VariantTotal paramsUse
PaddleOCR-VL-1.60.9BLatest. Best tables, Chinese characters and seals; drop-in for 1.5.
PaddleOCR-VL-1.50.9BPrevious generation; pin it if you have calibrated against its output.
PaddleOCR-VL0.9BThe original 0.9B release.
+ +**Recommended generation:** greedy decoding (`temperature=0`) with a per-page `max_tokens` budget — the model card uses 512 for a single region and the reference server allows more for a full page. These are informational; do not hardcode them in library code. + +**Resources:** [Hugging Face](https://huggingface.co/PaddlePaddle/PaddleOCR-VL-1.6) · [PaddleOCR on GitHub](https://github.com/PaddlePaddle/PaddleOCR) + +## 2. Configuration Tips + +- **Page resolution is the main cost knob.** The vision tower and the prefill both scale with the patch count of a page. `max_pixels` is expressed in 28x28 units (patch size 14 with a 2x2 merge), so `max_pixels / 784` is the image-token budget per page. The checkpoint's own default is 1280 tokens; the **Page Resolution** selector in the Deploy panel emits the corresponding `--mm-process-config` value. Lower it for clean born-digital PDFs, raise it for dense scans and small print. +- **Prompt selects the task.** PaddleOCR-VL is prompt-conditioned rather than instruction-following — use the exact task strings in §3.1. A free-form question will not behave like a chat model. +- **Leave `--trust-remote-code` off.** The checkpoints ship their own `configuration_paddleocr_vl.py` / `processing_paddleocr_vl.py`, but `transformers` 5.12 supports `paddleocr_vl` natively — and the bundled remote image processor is the slower of the two implementations (measured 87.4 ms vs 39.1 ms per 1080p page). Passing the flag pins SGLang to the remote copy. Serving without it produced byte-identical OCR output on every page we checked and about 5% more requests per second at 32-way concurrency. +- **Preprocessing is parallelized for you.** A full-resolution page costs tens of milliseconds of CPU to resize, normalize and patchify, which caps throughput long before the GPU saturates, so this model runs the image processor across several workers by default. `--mm-processor-worker-num` overrides the count; raising it past the default did not help in our measurements. +- **Keep the radix cache on for repeated pages.** Unlike whole-document batch OCR over unique scans, a workload that re-asks about the same page (different task prompts on one image) reuses the image prefix. Add `--disable-radix-cache` only if every request carries a different page. +- **The saturated-throughput flags earn their place.** A page is ~2700 tokens, so the default 8192-token prefill budget packs only three of them into a forward. Raising it to 16384 and letting decode ride along in the same batch (`--enable-mixed-chunk`, `--num-continuous-decode-steps 2`) measured +11% requests per second at 32-way concurrency and cut queued TTFT by 23%, with single-stream latency unchanged. Measured on an H200; on a smaller card lower `--chunked-prefill-size` until it fits. +- **Prefill CUDA graph is on for this model.** SGLang normally switches the breakable prefill graph off for every multimodal architecture; PaddleOCR-VL is allowlisted back in, which is worth 16.1 ms → 11.5 ms of single-stream TTFT on text-only prompts. Image-carrying batches are rejected at graph replay and run eager, so this helps mixed and text traffic, not pure page parsing. No flag needed. +- **Tensor parallelism is optional.** The weights are under 2 GB in BF16; TP>1 only shortens the vision-encoder and prefill critical path, at the cost of a collective per layer. Measure before adopting it. +- **Context length.** The backbone advertises 131072 positions, but a parsed page rarely needs more than a few thousand tokens. The recipe pins `--context-length 16384` so the KV pool stays small and concurrency stays high; raise it only if you batch many pages into one request. + +### Measured on one H200 + +One 1080p page (~2700 image tokens) in, 128 tokens out, prefix cache disabled, median TTFT: + + + + + + + + + + + + + + + + + + + + + +
ConfigurationTTFT, 1 streamreq/s at 32 concurrent
With --trust-remote-code (remote image processor)219 ms10.9
Recipe above (native image processor)114 ms11.3
+ +Throughput at saturation is bound by the vision tower, which runs full attention over +every patch of the page — so the **Page Resolution** selector is the lever that moves it, +not tensor parallelism. + +## 3. Advanced Usage + +### 3.1 Task prompts + +PaddleOCR-VL exposes its capabilities through a small set of fixed prompts. Send the prompt as the text part and the page as the image part of the same user turn. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PromptTask
OCR:Plain text recognition.
Table Recognition:Table structure and cell contents.
Formula Recognition:Mathematical expressions.
Chart Recognition:Chart contents.
Spotting:Text with locations. Benefits from the high-detail resolution setting.
Seal Recognition:Seals and stamps (1.6).
+ + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") + +response = client.chat.completions.create( + model="PaddlePaddle/PaddleOCR-VL-1.6", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "OCR:"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/your_page.png"}, + }, + ], + } + ], + max_tokens=2048, +) + +print(response.choices[0].message.content) +``` + + + + + +```text Output +Pending update — paste the server's verbatim output for your page here. +``` + + + +### 3.2 Parsing a multi-page document + +The model parses one page per request. Render each page to an image, then fan the pages out concurrently — SGLang batches the vision encoders of in-flight requests into a single forward, so concurrency is what keeps the GPU busy on a model this small. + + + +```python Example +import base64 +from concurrent.futures import ThreadPoolExecutor + +import pymupdf +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") + + +def render(page, dpi=200): + pixmap = page.get_pixmap(dpi=dpi) + return base64.b64encode(pixmap.tobytes("png")).decode("ascii") + + +def parse(page_png_b64): + response = client.chat.completions.create( + model="PaddlePaddle/PaddleOCR-VL-1.6", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "OCR:"}, + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{page_png_b64}" + }, + }, + ], + } + ], + max_tokens=2048, + ) + return response.choices[0].message.content + + +document = pymupdf.open("your_document.pdf") +pages = [render(page) for page in document] + +with ThreadPoolExecutor(max_workers=16) as pool: + for index, text in enumerate(pool.map(parse, pages)): + print(f"--- page {index + 1} ---") + print(text) +``` + + + + + +```text Output +Pending update — paste the server's verbatim output for your document here. +``` + + diff --git a/docs/cookbook/autoregressive/Baidu/Unlimited-OCR.mdx b/docs/cookbook/autoregressive/Baidu/Unlimited-OCR.mdx index 9e7b79780..f276f9cbf 100644 --- a/docs/cookbook/autoregressive/Baidu/Unlimited-OCR.mdx +++ b/docs/cookbook/autoregressive/Baidu/Unlimited-OCR.mdx @@ -1,7 +1,6 @@ --- title: Unlimited-OCR description: "Deploy Baidu Unlimited-OCR with SGLang for long document OCR using prefill-aware sliding-window attention." -tag: NEW --- ## Deployment diff --git a/docs/cookbook/autoregressive/intro.mdx b/docs/cookbook/autoregressive/intro.mdx index e3f94335c..dca685146 100644 --- a/docs/cookbook/autoregressive/intro.mdx +++ b/docs/cookbook/autoregressive/intro.mdx @@ -88,7 +88,7 @@ metatags: GLM-4.5V and GLM-4.1V-Thinking: Towards Versatile Multimodal Reasoning with Scalable Reinforcement Learning Use --chat-template glm-4v + + PaddleOCR-VL (0.9B, 1.5, 1.6) + PaddlePaddle/PaddleOCR-VL-1.6 + Baidu's 0.9B document-parsing VLM: a NaViT-style dynamic-resolution SigLIP encoder on an ERNIE-4.5-0.3B backbone, covering text, tables, formulas, charts and seals in 109 languages. See the cookbook page. + Task is selected by the prompt (OCR:, Table Recognition:, ...). Leave --trust-remote-code off — transformers supports this architecture natively and its image processor is the faster one. + GLM-OCR zai-org/GLM-OCR diff --git a/docs/src/snippets/configs/PaddlePaddle/paddleocr-vl.jsx b/docs/src/snippets/configs/PaddlePaddle/paddleocr-vl.jsx new file mode 100644 index 000000000..1b79dc3bd --- /dev/null +++ b/docs/src/snippets/configs/PaddlePaddle/paddleocr-vl.jsx @@ -0,0 +1,295 @@ +// PaddleOCR-VL cookbook config. Consumed by _deployment.jsx + _playground.jsx. +// +// All three releases (0.9B / 1.5 / 1.6) ship an identical `config.json` — same +// PaddleOCRVLForConditionalGeneration architecture, same SigLIP tower and +// ERNIE-4.5-0.3B backbone — so one recipe serves every variant and only the HF +// slug changes. + +export const config = { + modelName: "PaddleOCR-VL", + + supportedHardware: ["h100", "h200", "b200"], + + variants: [ + { id: "v16", label: "1.6", subtitle: "Latest" }, + { id: "v15", label: "1.5" }, + { id: "v09", label: "0.9B", subtitle: "Original" }, + ], + quantizations: [{ id: "bf16", label: "BF16" }], + strategies: [{ id: "balanced", label: "Balanced" }], + nodesOptions: [{ id: "single", label: "Single Node" }], + + modelNames: { + "v16|bf16": "PaddlePaddle/PaddleOCR-VL-1.6", + "v15|bf16": "PaddlePaddle/PaddleOCR-VL-1.5", + "v09|bf16": "PaddlePaddle/PaddleOCR-VL", + }, + + // Page resolution is the dominant cost knob: the ViT and the prefill both + // scale with the patch count, and `max_pixels` is expressed in 28x28 units + // (patch 14 x 2x2 merge), so the value divided by 784 is the image-token + // budget per page. 1280 is the checkpoint's own preprocessor default. + overlayDims: [ + { + id: "pageRes", + title: "Page Resolution", + default: "default", + options: [ + { + id: "fast", + label: "Fast (768 tok)", + flags: [ + "--mm-process-config '{\"image\": {\"max_pixels\": 602112}}'", + ], + }, + { id: "default", label: "Default (1280 tok)", flags: [] }, + { + id: "detail", + label: "High detail (2048 tok)", + flags: [ + "--mm-process-config '{\"image\": {\"max_pixels\": 1605632}}'", + ], + }, + ], + }, + ], + + placeholders: { + HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" }, + PORT: { target: "command", label: "Bind port", default: "30000" }, + HF_TOKEN: { + target: "command", + label: "HF token (Docker)", + default: "", + }, + CURL_HOST: { target: "curl", label: "Server host", default: "localhost" }, + CURL_PORT: { target: "curl", label: "Server port", default: "30000" }, + }, + + curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\ +-H 'Content-Type: application/json' \\ +-d '{ + "model": "{{MODEL_NAME}}", + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "OCR:"}, + {"type": "image_url", "image_url": {"url": "https://example.com/your_document.png"}} + ] + }], + "temperature": 0, + "max_tokens": 2048 +}'`, + + dockerImages: { + h100: "lmsysorg/sglang:dev", + h200: "lmsysorg/sglang:dev", + b200: "lmsysorg/sglang:dev", + }, + + github: { + cookbookModel: "PaddlePaddle/PaddleOCR-VL", + }, + + playgroundFeatures: { + attention: { + knobs: [{ id: "tp", label: "TP", values: [null, 1, 2, 4] }], + }, + }, + + cells: [ + // ==== 1.6 ==== + { + match: { + hw: "h100", + variant: "v16", + quant: "bf16", + strategy: "balanced", + nodes: "single", + }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--context-length 16384", + "--mem-fraction-static 0.8", + "--chunked-prefill-size 16384", + "--max-prefill-tokens 32768", + "--enable-mixed-chunk", + "--num-continuous-decode-steps 2", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { + hw: "h200", + variant: "v16", + quant: "bf16", + strategy: "balanced", + nodes: "single", + }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--context-length 16384", + "--mem-fraction-static 0.8", + "--chunked-prefill-size 16384", + "--max-prefill-tokens 32768", + "--enable-mixed-chunk", + "--num-continuous-decode-steps 2", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { + hw: "b200", + variant: "v16", + quant: "bf16", + strategy: "balanced", + nodes: "single", + }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--context-length 16384", + "--mem-fraction-static 0.8", + "--chunked-prefill-size 16384", + "--max-prefill-tokens 32768", + "--enable-mixed-chunk", + "--num-continuous-decode-steps 2", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // ==== 1.5 ==== + { + match: { + hw: "h100", + variant: "v15", + quant: "bf16", + strategy: "balanced", + nodes: "single", + }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--context-length 16384", + "--mem-fraction-static 0.8", + "--chunked-prefill-size 16384", + "--max-prefill-tokens 32768", + "--enable-mixed-chunk", + "--num-continuous-decode-steps 2", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { + hw: "h200", + variant: "v15", + quant: "bf16", + strategy: "balanced", + nodes: "single", + }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--context-length 16384", + "--mem-fraction-static 0.8", + "--chunked-prefill-size 16384", + "--max-prefill-tokens 32768", + "--enable-mixed-chunk", + "--num-continuous-decode-steps 2", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { + hw: "b200", + variant: "v15", + quant: "bf16", + strategy: "balanced", + nodes: "single", + }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--context-length 16384", + "--mem-fraction-static 0.8", + "--chunked-prefill-size 16384", + "--max-prefill-tokens 32768", + "--enable-mixed-chunk", + "--num-continuous-decode-steps 2", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + // ==== 0.9B ==== + { + match: { + hw: "h100", + variant: "v09", + quant: "bf16", + strategy: "balanced", + nodes: "single", + }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--context-length 16384", + "--mem-fraction-static 0.8", + "--chunked-prefill-size 16384", + "--max-prefill-tokens 32768", + "--enable-mixed-chunk", + "--num-continuous-decode-steps 2", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { + hw: "h200", + variant: "v09", + quant: "bf16", + strategy: "balanced", + nodes: "single", + }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--context-length 16384", + "--mem-fraction-static 0.8", + "--chunked-prefill-size 16384", + "--max-prefill-tokens 32768", + "--enable-mixed-chunk", + "--num-continuous-decode-steps 2", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { + hw: "b200", + variant: "v09", + quant: "bf16", + strategy: "balanced", + nodes: "single", + }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--context-length 16384", + "--mem-fraction-static 0.8", + "--chunked-prefill-size 16384", + "--max-prefill-tokens 32768", + "--enable-mixed-chunk", + "--num-continuous-decode-steps 2", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + ], +}; diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index b2c1a6da9..0b55e81ba 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -1920,6 +1920,7 @@ multimodal_piecewise_cuda_graph_supported_model_archs = [ # capturing cleanly. multimodal_breakable_cuda_graph_supported_model_archs = [ "InternS2MobiusForConditionalGeneration", + "PaddleOCRVLForConditionalGeneration", "Qwen3_5ForConditionalGeneration", "Qwen3_5MoeForConditionalGeneration", "MuseGlimmerForConditionalGeneration", diff --git a/python/sglang/srt/models/ernie4.py b/python/sglang/srt/models/ernie4.py index 6ed8a15f3..c3be1dde1 100644 --- a/python/sglang/srt/models/ernie4.py +++ b/python/sglang/srt/models/ernie4.py @@ -267,6 +267,9 @@ class Ernie4Model(nn.Module): self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + def get_input_embeddings(self) -> nn.Embedding: + return self.embed_tokens + @torch.no_grad() def forward( self, diff --git a/python/sglang/srt/models/paddleocr_vl.py b/python/sglang/srt/models/paddleocr_vl.py index c163a86ea..1e375ad16 100644 --- a/python/sglang/srt/models/paddleocr_vl.py +++ b/python/sglang/srt/models/paddleocr_vl.py @@ -13,14 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""PaddleOCR-VL: a NaViT-style SigLIP vision encoder on an ERNIE-4.5 backbone. +The vision tower runs on a *packed* layout: every image of a (possibly +cross-request) batch is concatenated into one ``[total_patches, dim]`` tensor and +the per-image boundaries live on the host as ``grid_thws`` plus ``cu_seqlens``. +Keeping the boundaries host-side is what lets the whole ViT forward run without a +single device-to-host synchronization, and it lets the shape-independent +projections run once for the batch instead of once per image. +""" + +import itertools from collections.abc import Iterable -from typing import List, Optional, Set, Tuple, Union +from typing import List, Optional, Set, Tuple -import numpy as np import torch import torch.nn as nn -from einops import rearrange from transformers.activations import GELUActivation from transformers.utils import torch_int @@ -43,8 +51,80 @@ from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.ernie4 import Ernie4_5_ForCausalLM from sglang.srt.utils import add_prefix, is_npu +_is_npu = is_npu() + +# Patch counts (t, h, w) of one image, always materialized on the host. +GridTHW = Tuple[int, int, int] + + +def build_packed_2d_position_ids( + grid_thws: List[GridTHW], device: torch.device +) -> Tuple[torch.Tensor, int]: + """Row/column patch indices of a packed batch, plus the rope table size. + + Returns ``([total_patches, 2], max_grid_size)``. ``max_grid_size`` is derived + from the host grids rather than from a device-side ``max()``, so building the + rope table never synchronizes. + """ + split_hids = list() + split_wids = list() + for t, h, w in grid_thws: + frame_ids = torch.arange(h * w, device=device) + hids = frame_ids // w + wids = frame_ids - hids * w + if t > 1: + hids = hids.repeat(t) + wids = wids.repeat(t) + split_hids.append(hids) + split_wids.append(wids) + + if len(grid_thws) == 1: + height_position_ids, width_position_ids = split_hids[0], split_wids[0] + else: + height_position_ids = torch.cat(split_hids, dim=0) + width_position_ids = torch.cat(split_wids, dim=0) + + pids = torch.stack([height_position_ids, width_position_ids], dim=-1) + max_grid_size = max(max(h, w) for _, h, w in grid_thws) + return pids, max_grid_size + + +def merge_patch_neighbourhoods( + hidden_states: torch.Tensor, + grid_thws: List[GridTHW], + merge_kernel_size: Tuple[int, int], +) -> torch.Tensor: + """Group each image's ``m1 x m2`` patch neighbourhoods into single tokens. + + Takes the packed ``[total_patches, dim]`` batch and returns + ``[total_patches / (m1 * m2), m1 * m2 * dim]``. Only this step depends on an + image's ``h``/``w``, which is why it is separated from the projections that + follow: those are row-wise and run once for the whole batch. + """ + m1, m2 = merge_kernel_size + dim = hidden_states.shape[-1] + merged = hidden_states.new_empty(hidden_states.shape[0] // (m1 * m2), m1 * m2 * dim) + + in_offset = out_offset = 0 + for t, h, w in grid_thws: + num_patches = t * h * w + num_merged = num_patches // (m1 * m2) + # Row-major patches (t, h, w) regroup as (t, h/m1, m1, w/m2, m2, d); the + # merged token concatenates the m1*m2 neighbours along `d`. + merged[out_offset : out_offset + num_merged].view( + t, h // m1, w // m2, m1, m2, dim + ).copy_( + hidden_states[in_offset : in_offset + num_patches] + .view(t, h // m1, m1, w // m2, m2, dim) + .permute(0, 1, 3, 2, 4, 5) + ) + in_offset += num_patches + out_offset += num_merged + return merged + class Projector(nn.Module): + """Merge 2x2 patch neighbourhoods, then project into the language space.""" def __init__( self, @@ -73,40 +153,22 @@ class Projector(nn.Module): def forward( self, image_features: torch.Tensor, - image_grid_thw: List[Tuple[int, int, int]], + grid_thws: List[GridTHW], ) -> torch.Tensor: - m1, m2 = self.merge_kernel_size - if isinstance(image_features, (list, tuple)): - processed_features = list() - for image_feature, image_grid in zip(image_features, image_grid_thw): - image_feature = self.pre_norm(image_feature) - t, h, w = image_grid + """Project packed ViT features ``[total_patches, dim]`` for the batch. - image_feature = rearrange( - image_feature, - "(t h p1 w p2) d -> (t h w) (p1 p2 d)", - t=t, - h=h // m1, - p1=m1, - w=w // m2, - p2=m2, - ) - hidden_states = self.linear_1(image_feature) - hidden_states = self.act(hidden_states) - hidden_states = self.linear_2(hidden_states) - processed_features.append(hidden_states) - - return processed_features - - dims = image_features.shape[:-1] - dim = image_features.shape[-1] - image_features = image_features.view(np.prod(dims), dim) - hidden_states = self.pre_norm(image_features).view(-1, self.hidden_size) - hidden_states = self.linear_1(hidden_states) + Only the 2x2 merge depends on an image's ``h``/``w``; the norm and both + projections are row-wise, so they run once over the packed batch. Each + image contributes a single strided copy into the merged buffer, so an + N-image batch costs N copies plus 3 kernels rather than 4N kernels. + """ + hidden_states = self.pre_norm(image_features) + merged = merge_patch_neighbourhoods( + hidden_states, grid_thws, self.merge_kernel_size + ) + hidden_states = self.linear_1(merged) hidden_states = self.act(hidden_states) - hidden_states = self.linear_2(hidden_states) - - return hidden_states.view(*dims, -1) + return self.linear_2(hidden_states) class SiglipVisionEmbeddings(nn.Module): @@ -118,12 +180,16 @@ class SiglipVisionEmbeddings(nn.Module): self.image_size = config.image_size self.patch_size = config.patch_size + # kernel_size == stride and padding == 0, so this convolution is exactly + # an unfold plus a matmul. Taking that path avoids a cuDNN convolution + # launch over a [total_patches, 3, p, p] input on every ViT forward. self.patch_embedding = Conv2dLayer( in_channels=config.num_channels, out_channels=self.embed_dim, kernel_size=self.patch_size, stride=self.patch_size, padding="valid", + disable_linear=False, ) self.num_patches = (self.image_size // self.patch_size) ** 2 @@ -139,44 +205,43 @@ class SiglipVisionEmbeddings(nn.Module): persistent=False, ) - def interpolate_pos_encoding( - self, - embeddings: torch.Tensor, - height: int, - width: int, - is_after_patchify: bool = False, - ) -> torch.Tensor: - + def interpolate_pos_encoding(self, height: int, width: int) -> torch.Tensor: + """Resample the square learned position grid onto a ``height x width`` grid.""" num_positions = self.position_embedding.weight.shape[0] - patch_pos_embed = self.position_embedding.weight.unsqueeze(0) - dim = embeddings.shape[-1] - - if is_after_patchify: - new_height = height - new_width = width - else: - new_height = height // self.patch_size - new_width = width // self.patch_size - sqrt_num_positions = torch_int(num_positions**0.5) patch_pos_embed = patch_pos_embed.reshape( - 1, sqrt_num_positions, sqrt_num_positions, dim + 1, sqrt_num_positions, sqrt_num_positions, self.embed_dim ) patch_pos_embed = patch_pos_embed.permute(0, 3, 1, 2) patch_pos_embed = nn.functional.interpolate( patch_pos_embed, - size=(new_height, new_width), + size=(height, width), mode="bilinear", align_corners=False, ) - patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim) - return patch_pos_embed + patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view( + 1, -1, self.embed_dim + ) + # Materialize contiguously. The permute leaves the channel dim strided, + # and this tensor is cached and broadcast-added to the packed activations + # on every forward, so a strided read would be paid over and over. + return patch_pos_embed.contiguous() - def fetch_position_embedding_lfu_cache(self, embeddings, h, w, max_cache: int = 20): + def fetch_position_embedding_lfu_cache( + self, h: int, w: int, max_cache: int = 20 + ) -> torch.Tensor: + """Return the interpolated position grid for ``(h, w)``, LFU-cached. + + The interpolation depends only on the grid, so document batches that + repeat a resolution reuse the tensor instead of re-running the bilinear + resample once per image per forward. The cache holds at most `max_cache` + grids of `h * w * hidden_size` each (~12 MiB at the checkpoint's default + 1280-token page budget), evicting the least frequently used. + """ grid = (h, w) if grid in self.cache_position_embedding: self.cache_position_count[grid] += 1 @@ -190,7 +255,7 @@ class SiglipVisionEmbeddings(nn.Module): self.cache_position_count.pop(min_hit_grid) self.cache_position_embedding.pop(min_hit_grid) - position_embedding = self.interpolate_pos_encoding(embeddings, h, w, True) + position_embedding = self.interpolate_pos_encoding(h, w) self.cache_position_count[grid] = 1 self.cache_position_embedding[grid] = position_embedding return position_embedding @@ -198,61 +263,39 @@ class SiglipVisionEmbeddings(nn.Module): def forward( self, pixel_values: torch.FloatTensor, + grid_thws: List[GridTHW], position_ids: Optional[torch.Tensor] = None, - image_grid_thw: Optional[ - List[ - Union[ - Tuple[int, int, int], - List[Tuple[int, int, int]], - ] - ] - ] = None, - interpolate_pos_encoding=False, ) -> torch.Tensor: - if pixel_values.dim() == 4: - pixel_values = pixel_values.unsqueeze(0) if pixel_values.dim() == 5: - if position_ids is None: - raise ValueError( - "position_ids cannot be None when pixel_values.dim() is 5." - ) - ( - batch_size, - squence_len, - channel, - height, - width, - ) = pixel_values.shape - target_dtype = self.patch_embedding.weight.dtype - pixel_values = rearrange(pixel_values, "b l c h w -> (b l) c h w") - patch_embeds = self.patch_embedding(pixel_values.to(dtype=target_dtype)) - embeddings = patch_embeds.flatten(-2).squeeze(-1) - - if interpolate_pos_encoding and image_grid_thw is not None: - start = 0 - tmp_embeddings = list() - for image_grid in image_grid_thw: - t, h, w = image_grid - end = start + t * h * w - image_embeddings = embeddings[start:end, :] - position_embedding = ( - self.interpolate_pos_encoding(image_embeddings, h, w, True) - .squeeze(0) - .repeat(t, 1) - ) - image_embeddings = image_embeddings + position_embedding - tmp_embeddings.append(image_embeddings) - start = end - embeddings = torch.concat(tmp_embeddings, dim=0).unsqueeze(0) - else: - embeddings = embeddings + self.packing_position_embedding(position_ids) - return embeddings - else: + # [batch, patches, c, ph, pw] -> [batch * patches, c, ph, pw] + pixel_values = pixel_values.flatten(0, 1) + if pixel_values.dim() != 4: raise ValueError( "Unsupported pixel_values dimension:" f" {pixel_values.dim()}. Expected 4 or 5." ) + patch_embeds = self.patch_embedding( + pixel_values.to(dtype=self.patch_embedding.weight.dtype) + ) + # Each patch convolves to a 1x1 map, so this is a reshape to [patches, dim]. + embeddings = patch_embeds.flatten(-2).squeeze(-1) + + if position_ids is None: + # Interpolated per-image position grids, added in place so the packed + # activation is never copied into a second buffer. + offset = 0 + for t, h, w in grid_thws: + num_patches = t * h * w + embeddings[offset : offset + num_patches].view( + t, h * w, self.embed_dim + ).add_(self.fetch_position_embedding_lfu_cache(h, w)) + offset += num_patches + else: + embeddings += self.packing_position_embedding(position_ids) + + return embeddings.unsqueeze(0) + class SigLIPRotaryEmbedding(nn.Module): @@ -347,10 +390,10 @@ class SiglipEncoderLayer(nn.Module): def forward( self, hidden_states: torch.Tensor, - cu_seqlens: Optional[List[torch.Tensor]] = None, - rope_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, - forward_metadata: Optional[VisionAttentionMetadata] = None, - ) -> Tuple[torch.FloatTensor]: + cu_seqlens: torch.Tensor, + rope_emb: Tuple[torch.Tensor, torch.Tensor], + forward_metadata: VisionAttentionMetadata, + ) -> torch.Tensor: residual = hidden_states @@ -399,69 +442,38 @@ class SiglipEncoder(nn.Module): ) self.rotary_pos_emb = SigLIPRotaryEmbedding(head_dim // 2) - @staticmethod - def flatten_list(image_grid_thw): - tmp_image_grid_thw = list() - for image_grid in image_grid_thw: - if isinstance(image_grid, list): - tmp_image_grid_thw.extend(image_grid) - else: - tmp_image_grid_thw.append(image_grid) - return tmp_image_grid_thw + def _build_rope_emb( + self, grid_thws: List[GridTHW], device: torch.device + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Build the packed 2D rope cos/sin table for the batch.""" + pids, max_grid_size = build_packed_2d_position_ids(grid_thws, device) + rope_emb = self.rotary_pos_emb(max_grid_size)[pids].flatten(1) + rope_emb = rope_emb.repeat(1, 2) + return rope_emb.cos(), rope_emb.sin() def forward( self, - inputs_embeds, - cu_seqlens: Optional[List[torch.Tensor]] = None, - image_grid_thw: Optional[ - List[ - Union[ - Tuple[int, int, int], - List[Tuple[int, int, int]], - ] - ] - ] = None, - height_position_ids: Optional[torch.Tensor] = None, - width_position_ids: Optional[torch.Tensor] = None, + inputs_embeds: torch.Tensor, + cu_seqlens: torch.Tensor, + grid_thws: List[GridTHW], + max_seqlen: int, ) -> torch.Tensor: - device = inputs_embeds.device - hidden_states = inputs_embeds - flatten_image_grid_thw = self.flatten_list(image_grid_thw) + rope_emb = self._build_rope_emb(grid_thws, inputs_embeds.device) - if width_position_ids is None or height_position_ids is None: - split_hids = list() - split_wids = list() - for t, h, w in flatten_image_grid_thw: - image_pids = torch.arange(t * h * w, device=device) % (h * w) - sample_hids = image_pids // w - sample_wids = image_pids % w - split_hids.append(sample_hids) - split_wids.append(sample_wids) - width_position_ids = torch.concat(split_wids, dim=0) - height_position_ids = torch.concat(split_hids, dim=0) - - pids = torch.stack( - [height_position_ids, width_position_ids], - dim=-1, - ) - max_grid_size = pids.max() + 1 - rope_emb_max_grid = self.rotary_pos_emb(max_grid_size) - rope_emb = rope_emb_max_grid[pids].flatten(1) - rope_emb = rope_emb.repeat(1, 2) - rope_emb = (rope_emb.cos(), rope_emb.sin()) # cu_seqlens must be on cpu because of npu_flash_attention_unpad operator restriction - if is_npu() and isinstance(cu_seqlens, torch.Tensor): + if _is_npu: cu_seqlens = cu_seqlens.to("cpu") - attn_cu_seqlens = cu_seqlens + # `max_seqlen` comes from the host grids, so the metadata is built once + # for every layer without reading a device tensor back. forward_metadata = prepare_vision_attention_metadata( - attn_cu_seqlens, device=hidden_states.device + cu_seqlens, device=inputs_embeds.device, max_seqlen=max_seqlen ) - hidden_states = inputs_embeds + hidden_states = inputs_embeds for encoder_layer in self.layers: hidden_states = encoder_layer( hidden_states, - cu_seqlens=attn_cu_seqlens, + cu_seqlens=cu_seqlens, rope_emb=rope_emb, forward_metadata=forward_metadata, ) @@ -490,52 +502,28 @@ class SiglipVisionTransformer(nn.Module): def forward( self, - pixel_values, - interpolate_pos_encoding: Optional[bool] = False, + pixel_values: torch.Tensor, + grid_thws: List[GridTHW], + cu_seqlens: torch.Tensor, + max_seqlen: int, position_ids: Optional[torch.Tensor] = None, - height_position_ids: Optional[torch.Tensor] = None, - width_position_ids: Optional[torch.Tensor] = None, - cu_seqlens: Optional[List[torch.Tensor]] = None, - image_grid_thw: Optional[ - List[ - Union[ - Tuple[int, int, int], - List[Tuple[int, int, int]], - ] - ] - ] = None, - ) -> list[torch.Tensor]: - + ) -> torch.Tensor: hidden_states = self.embeddings( pixel_values, - interpolate_pos_encoding=interpolate_pos_encoding, + grid_thws=grid_thws, position_ids=position_ids, - image_grid_thw=image_grid_thw, ) - last_hidden_state = self.encoder( + hidden_states = self.encoder( inputs_embeds=hidden_states, cu_seqlens=cu_seqlens, - image_grid_thw=image_grid_thw, - height_position_ids=height_position_ids, - width_position_ids=width_position_ids, + grid_thws=grid_thws, + max_seqlen=max_seqlen, ) - last_hidden_state = self.post_layernorm(last_hidden_state) - - sample_hidden_state = list() - if cu_seqlens is None: - raise ValueError( - "cu_seqlens cannot be None for " - "SiglipVisionTransformer output processing." - ) - for i in range(cu_seqlens.shape[0] - 1): - start = cu_seqlens[i] - end = cu_seqlens[i + 1] - tensor = last_hidden_state[:, start:end, :].squeeze(0) - sample_hidden_state.append(tensor) - - return sample_hidden_state + # Stay packed: the projector slices per image on the host, so splitting + # here would index `cu_seqlens` on the device and stall once per image. + return self.post_layernorm(hidden_states).squeeze(0) class SiglipVisionModel(nn.Module): @@ -570,48 +558,38 @@ class SiglipVisionModel(nn.Module): def forward( self, - pixel_values, - interpolate_pos_encoding: bool = False, + pixel_values: torch.Tensor, + grid_thws: List[GridTHW], + cu_seqlens: torch.Tensor, + max_seqlen: int, position_ids: Optional[torch.Tensor] = None, - image_grid_thw: Optional[ - List[ - Union[ - Tuple[int, int, int], - List[Tuple[int, int, int]], - ] - ] - ] = None, - cu_seqlens: Optional[List[torch.Tensor]] = None, - ) -> list[torch.Tensor]: - + ) -> torch.Tensor: return self.vision_model( pixel_values=pixel_values, - interpolate_pos_encoding=interpolate_pos_encoding, - position_ids=position_ids, - image_grid_thw=image_grid_thw, + grid_thws=grid_thws, cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + position_ids=position_ids, ) class PaddleOCRVLForConditionalGeneration(Ernie4_5_ForCausalLM): def __init__(self, *, config, quant_config=None, prefix: str = ""): - super().__init__(config=config, prefix=prefix) + super().__init__(config=config, quant_config=quant_config, prefix=prefix) config = self.config self.mlp_AR = Projector( config, config.vision_config, prefix=add_prefix("mlp_AR", prefix) ) + # NOTE: only BitsAndBytes 4-bit quantization is exercised for the SigLIP + # tower; other methods fall back to bf16 through SiglipMLP's own gate. self.visual = SiglipVisionModel( - config=config.vision_config, prefix=add_prefix("visual", prefix) + config=config.vision_config, + quant_config=quant_config, + prefix=add_prefix("visual", prefix), ) - if not hasattr(self.model, "get_input_embeddings"): - import types - - self.model.get_input_embeddings = types.MethodType( - get_input_embeddings, self.model - ) - self.is_mrope_enabled = "mrope_section" in self.config.rope_scaling + self.is_mrope_enabled = "mrope_section" in (self.config.rope_scaling or {}) def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs): pattern = MultiModalityDataPaddingPatternMultimodalTokens() @@ -620,46 +598,38 @@ class PaddleOCRVLForConditionalGeneration(Ernie4_5_ForCausalLM): def get_input_embeddings(self): return self.model.embed_tokens - def encode_image(self, pixel_values, image_grid_thw): - pixel_values = pixel_values.type(self.visual.dtype) - siglip_position_ids = list() - image_grid_hws = list() - cu_seqlens = [0] - - for idx, grid_thw in enumerate(image_grid_thw): - thw_tuple = tuple(grid_thw.detach().cpu().numpy().tolist()) - numel = np.prod(thw_tuple) - image_grid_hws.append(thw_tuple) - image_position_ids = torch.arange(numel) % np.prod(thw_tuple[1:]) - siglip_position_ids.append(image_position_ids) - cu_seqlens.append(cu_seqlens[-1] + numel) - - siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to( - pixel_values.device + def encode_image( + self, pixel_values: torch.Tensor, image_grid_thw: torch.Tensor + ) -> torch.Tensor: + # One host transfer for the whole batch. Every consumer of the grids + # (rope table, patch merge, cu_seqlens) needs them on the host, so + # reading them per image would cost one synchronization per image. + grid_thws: List[GridTHW] = [(t, h, w) for t, h, w in image_grid_thw.tolist()] + seq_lens = [t * h * w for t, h, w in grid_thws] + cu_seqlens = torch.tensor( + [0, *itertools.accumulate(seq_lens)], + dtype=torch.int32, + device=pixel_values.device, ) - cu_seqlens = torch.tensor(cu_seqlens, dtype=torch.int32).to(pixel_values.device) + vision_outputs = self.visual( pixel_values=pixel_values, - image_grid_thw=image_grid_hws, - position_ids=siglip_position_ids, - interpolate_pos_encoding=True, + grid_thws=grid_thws, cu_seqlens=cu_seqlens, + max_seqlen=max(seq_lens), ) - image_embeds = self.mlp_AR(vision_outputs, image_grid_thw) - - # image_embeds = torch.stack(image_embeds, dim=0) - image_embeds = torch.cat(image_embeds, dim=0) - - return image_embeds + return self.mlp_AR(vision_outputs, grid_thws) def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: - pixel_values = torch.cat([item.feature for item in items], dim=0).type( - self.visual.dtype - ) - image_grid_thw = torch.concat([item.image_grid_thw for item in items], dim=0) - image_embeds = self.encode_image(pixel_values, image_grid_thw) - - return image_embeds + if len(items) == 1: + # torch.cat allocates even for a single input; a document batch is + # usually one image, and its pixel buffer is the largest tensor here. + pixel_values = items[0].feature + image_grid_thw = items[0].image_grid_thw + else: + pixel_values = torch.cat([item.feature for item in items], dim=0) + image_grid_thw = torch.cat([item.image_grid_thw for item in items], dim=0) + return self.encode_image(pixel_values, image_grid_thw) def forward( self, @@ -670,11 +640,10 @@ class PaddleOCRVLForConditionalGeneration(Ernie4_5_ForCausalLM): ): if self.is_mrope_enabled: positions = forward_batch.mrope_positions - if not ( - forward_batch.forward_mode.is_decode() - or not forward_batch.contains_image_inputs() - ): - if self.is_mrope_enabled: + if ( + not forward_batch.forward_mode.is_decode() + and forward_batch.contains_image_inputs() + ): assert positions.ndim == 2 and positions.size(0) == 3, ( "multimodal section rotary embedding requires " f"(3, seq_len) positions, but got {positions.size()}" @@ -732,9 +701,4 @@ class PaddleOCRVLForConditionalGeneration(Ernie4_5_ForCausalLM): raise KeyError(f"Parameter '{name}' not found in model.") -# monkey patch -def get_input_embeddings(self) -> nn.Embedding: - return self.embed_tokens - - EntryClass = [PaddleOCRVLForConditionalGeneration] diff --git a/python/sglang/srt/multimodal/processors/paddleocr_vlm.py b/python/sglang/srt/multimodal/processors/paddleocr_vlm.py index 05dd21fba..e58749603 100644 --- a/python/sglang/srt/multimodal/processors/paddleocr_vlm.py +++ b/python/sglang/srt/multimodal/processors/paddleocr_vlm.py @@ -20,6 +20,21 @@ from sglang.srt.multimodal.processors.qwen_vl import QwenVLImageProcessor class PaddleOCRVLImageProcessor(QwenVLImageProcessor): models = [PaddleOCRVLForConditionalGeneration] + # A document page is a far heavier preprocessing unit than a chat image: + # resize + normalize + patchify of a full-resolution scan costs tens of + # milliseconds, so a single worker caps request throughput at + # 1 / preprocess_time regardless of how much GPU is left idle. Overlap it + # across workers; the work itself is unchanged. + # + # Two, not more: measured on an H200 at 32-way concurrency, two workers beat + # both one and four on every shape tried (1080p pages 6.72 -> 9.55 req/s at + # two, 8.92 at four; 360p pages with 512-token outputs 22.38 -> 25.20 at + # two, 25.06 at four). Past two, spreading request arrivals fragments the + # GPU prefill batches faster than the extra overlap pays for itself. + auto_mm_processor_worker_num = 2 + auto_mm_io_worker_num = 16 + supports_mm_processor_concurrency = True + def __init__(self, hf_config, server_args, _processor, *args, **kwargs): super().__init__(hf_config, server_args, _processor, *args, **kwargs) diff --git a/test/registered/unit/models/test_paddleocr_vl_serving_defaults.py b/test/registered/unit/models/test_paddleocr_vl_serving_defaults.py new file mode 100644 index 000000000..3d4c5b1ec --- /dev/null +++ b/test/registered/unit/models/test_paddleocr_vl_serving_defaults.py @@ -0,0 +1,69 @@ +"""Guard the PaddleOCR-VL serving defaults that a refactor could silently drop. + +Both settings here live in allowlists keyed by model type / architecture, so +nothing in PaddleOCR-VL's own code path breaks if an entry disappears — the +model just quietly serves slower. + +A document page costs tens of milliseconds to resize + normalize + patchify, so +a single synchronous processor worker caps request throughput at +1 / preprocess_time no matter how much GPU is idle. Measured on an H200 with +1080p pages, opting into concurrent workers moved 32-way concurrent throughput +from 6.6 to 8.9 req/s and made single-stream TTFT stable (the single-worker +path alternated between ~282 ms and ~790 ms). + +The opt-in lives on the class, and `QwenVLImageProcessor` grants it only to an +explicit `model_type` allowlist that PaddleOCR-VL is not on — so it is exactly +the kind of setting a refactor can silently drop. +""" + +import pytest + +from sglang.srt.configs.model_config import ( + multimodal_breakable_cuda_graph_supported_model_archs, +) +from sglang.srt.multimodal.processors.base_processor import BaseMultimodalProcessor +from sglang.srt.multimodal.processors.paddleocr_vlm import PaddleOCRVLImageProcessor +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + + +def test_processor_opts_into_concurrency(): + assert PaddleOCRVLImageProcessor.supports_mm_processor_concurrency is True + assert PaddleOCRVLImageProcessor.auto_mm_processor_worker_num > 1 + assert PaddleOCRVLImageProcessor.auto_mm_io_worker_num > 1 + + +def test_worker_count_stays_at_the_measured_optimum(): + """Two beat both one and four at 32-way concurrency on an H200, on document + pages and on small images with long outputs alike. Past two, spreading + request arrivals fragments GPU prefill batches faster than the extra overlap + pays for itself.""" + assert PaddleOCRVLImageProcessor.auto_mm_processor_worker_num == 2 + + +def test_concurrency_opt_in_is_not_inherited_by_accident(): + """The base class must stay conservative; this model opts in explicitly.""" + assert BaseMultimodalProcessor.supports_mm_processor_concurrency is False + assert BaseMultimodalProcessor.auto_mm_processor_worker_num == 1 + assert ( + PaddleOCRVLImageProcessor.__dict__["supports_mm_processor_concurrency"] is True + ), "the opt-in must be declared on PaddleOCRVLImageProcessor itself" + + +def test_prefill_breakable_cuda_graph_is_allowlisted(): + """Breakable CG is the CUDA default but is switched off for every multimodal + arch; PaddleOCR-VL opts back in so its text-only prefill keeps the graph. + + Measured on an H200 (2704-token text prompts): single-stream TTFT 16.1 ms + without the graph, 11.5 ms with it. Image-carrying batches are rejected at + replay and run eager either way, so this is a text/mixed-traffic win only. + """ + assert ( + "PaddleOCRVLForConditionalGeneration" + in multimodal_breakable_cuda_graph_supported_model_archs + ) + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-v"])) diff --git a/test/registered/unit/models/test_paddleocr_vl_vision.py b/test/registered/unit/models/test_paddleocr_vl_vision.py new file mode 100644 index 000000000..7f43b6a80 --- /dev/null +++ b/test/registered/unit/models/test_paddleocr_vl_vision.py @@ -0,0 +1,268 @@ +"""CPU coverage for the PaddleOCR-VL packed vision-tower fast paths. + +The tower encodes a whole (possibly cross-request) batch as one packed +``[total_patches, dim]`` tensor. These tests pin the packed results to the +straightforward per-image reference so the packing stays a pure optimization. +""" + +import pytest +import torch +import torch.nn as nn +from einops import rearrange + +from sglang.srt.models.paddleocr_vl import ( + Projector, + SiglipVisionEmbeddings, + build_packed_2d_position_ids, + merge_patch_neighbourhoods, +) +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=20, suite="base-a-test-cpu") + +# Mixes repeated grids (LFU cache hits), an odd aspect ratio, and t > 1. +GRIDS = [(1, 4, 6), (1, 8, 10), (2, 2, 4), (1, 8, 10)] + + +class _VisionConfig: + """Minimal stand-in for PaddleOCR-VL's `vision_config`.""" + + hidden_size = 32 + image_size = 56 + patch_size = 14 + num_channels = 3 + + +class _TextConfig: + hidden_size = 48 + + +def _grid_offsets(): + """Start row of each image inside the packed batch.""" + offset = 0 + for t, h, w in GRIDS: + yield offset + offset += t * h * w + + +def _packed_features(dtype=torch.float64) -> torch.Tensor: + total = sum(t * h * w for t, h, w in GRIDS) + return torch.randn(total, _VisionConfig.hidden_size, dtype=dtype) + + +def _reference_projector_output( + projector: Projector, packed: torch.Tensor +) -> torch.Tensor: + """Per-image merge + projection, i.e. the pre-packing formulation.""" + m1, m2 = projector.merge_kernel_size + outputs = [] + offset = 0 + for t, h, w in GRIDS: + num_patches = t * h * w + feature = projector.pre_norm(packed[offset : offset + num_patches]) + feature = rearrange( + feature, + "(t h p1 w p2) d -> (t h w) (p1 p2 d)", + t=t, + h=h // m1, + p1=m1, + w=w // m2, + p2=m2, + ) + outputs.append(projector.linear_2(projector.act(projector.linear_1(feature)))) + offset += num_patches + return torch.cat(outputs, dim=0) + + +def _build_projector() -> Projector: + torch.manual_seed(0) + projector = Projector(_TextConfig(), _VisionConfig()).to(torch.float64) + return projector + + +# The merge is pure data movement, so it must be bit-exact. The projections that +# follow are not: batching N per-image GEMMs into one changes the blocking, and +# with it the summation order, so the results differ in the last bits (observed +# up to 3e-14 relative in fp64, and it is BLAS-implementation dependent -- equal +# on Apple silicon, unequal on x86). A permutation bug would move values by +# order 1, so this tolerance still catches one decisively. +_GEMM_REORDER_RTOL = 1e-12 +_GEMM_REORDER_ATOL = 1e-12 + + +def test_projector_merge_permutation_is_exact(): + """The 2x2 regroup moves data without arithmetic, so it must be bit-exact.""" + torch.manual_seed(1) + projector = _build_projector() + packed = _packed_features() + normed = projector.pre_norm(packed) + + actual = merge_patch_neighbourhoods(normed, GRIDS, projector.merge_kernel_size) + + m1, m2 = projector.merge_kernel_size + expected = torch.cat( + [ + rearrange( + normed[offset : offset + t * h * w], + "(t h p1 w p2) d -> (t h w) (p1 p2 d)", + t=t, + h=h // m1, + p1=m1, + w=w // m2, + p2=m2, + ) + for offset, (t, h, w) in zip(_grid_offsets(), GRIDS) + ], + dim=0, + ) + + assert actual.shape == expected.shape + assert torch.equal(actual, expected) + + +def test_projector_packed_merge_matches_per_image_reference(): + torch.manual_seed(1) + projector = _build_projector() + packed = _packed_features() + + actual = projector(packed, GRIDS) + expected = _reference_projector_output(projector, packed) + + assert actual.shape == expected.shape + assert actual.shape[0] == sum(t * h * w for t, h, w in GRIDS) // 4 + assert actual.shape[1] == _TextConfig.hidden_size + torch.testing.assert_close( + actual, expected, rtol=_GEMM_REORDER_RTOL, atol=_GEMM_REORDER_ATOL + ) + + +def test_projector_is_batch_invariant(): + """Encoding images together must equal encoding them one at a time.""" + torch.manual_seed(2) + projector = _build_projector() + packed = _packed_features() + + together = projector(packed, GRIDS) + + apart = [] + offset = 0 + for grid in GRIDS: + num_patches = grid[0] * grid[1] * grid[2] + apart.append(projector(packed[offset : offset + num_patches], [grid])) + offset += num_patches + apart = torch.cat(apart, dim=0) + + torch.testing.assert_close( + together, apart, rtol=_GEMM_REORDER_RTOL, atol=_GEMM_REORDER_ATOL + ) + + +def _build_embeddings() -> SiglipVisionEmbeddings: + torch.manual_seed(3) + embeddings = SiglipVisionEmbeddings(_VisionConfig()).to(torch.float64) + nn.init.normal_(embeddings.position_embedding.weight) + return embeddings + + +def _reference_position_embedding_add( + embeddings: SiglipVisionEmbeddings, patch_embeds: torch.Tensor +) -> torch.Tensor: + """Uncached interpolation per image, concatenated — the pre-cache formulation.""" + outputs = [] + offset = 0 + for t, h, w in GRIDS: + num_patches = t * h * w + image = patch_embeds[offset : offset + num_patches] + position = embeddings.interpolate_pos_encoding(h, w).squeeze(0).repeat(t, 1) + outputs.append(image + position) + offset += num_patches + return torch.cat(outputs, dim=0) + + +def test_position_embedding_cache_matches_uncached_interpolation(): + embeddings = _build_embeddings() + torch.manual_seed(4) + patch_embeds = torch.randn( + sum(t * h * w for t, h, w in GRIDS), + _VisionConfig.hidden_size, + dtype=torch.float64, + ) + expected = _reference_position_embedding_add(embeddings, patch_embeds) + + actual = patch_embeds.clone() + offset = 0 + for t, h, w in GRIDS: + num_patches = t * h * w + actual[offset : offset + num_patches].view( + t, h * w, _VisionConfig.hidden_size + ).add_(embeddings.fetch_position_embedding_lfu_cache(h, w)) + offset += num_patches + + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + # (8, 10) appears twice in GRIDS, so it must have been served from the cache. + assert embeddings.cache_position_count[(8, 10)] == 2 + assert len(embeddings.cache_position_embedding) == 3 + + +def test_position_embedding_cache_evicts_least_frequently_used(): + embeddings = _build_embeddings() + + embeddings.fetch_position_embedding_lfu_cache(4, 4, max_cache=2) + embeddings.fetch_position_embedding_lfu_cache(4, 4, max_cache=2) + embeddings.fetch_position_embedding_lfu_cache(6, 6, max_cache=2) + embeddings.fetch_position_embedding_lfu_cache(8, 8, max_cache=2) + + assert set(embeddings.cache_position_embedding) == {(4, 4), (8, 8)} + + +def test_patch_embedding_takes_the_matmul_path(): + """kernel == stride and zero padding, so the conv must lower to a matmul.""" + embeddings = _build_embeddings() + assert embeddings.patch_embedding.enable_linear + + torch.manual_seed(5) + patch_size = _VisionConfig.patch_size + pixel_values = torch.randn(7, 3, patch_size, patch_size, dtype=torch.float64) + + actual = embeddings.patch_embedding(pixel_values) + expected = nn.functional.conv2d( + pixel_values, + embeddings.patch_embedding.weight, + embeddings.patch_embedding.bias, + stride=(patch_size, patch_size), + ) + + torch.testing.assert_close(actual, expected, rtol=0, atol=1e-12) + # The tower adds position embeddings in place on this view. + assert actual.flatten(-2).squeeze(-1).is_contiguous() + + +def test_packed_2d_position_ids_match_per_image_reference(): + pids, max_grid_size = build_packed_2d_position_ids(GRIDS, torch.device("cpu")) + + expected_hids = [] + expected_wids = [] + for t, h, w in GRIDS: + image_pids = torch.arange(t * h * w) % (h * w) + expected_hids.append(image_pids // w) + expected_wids.append(image_pids % w) + expected = torch.stack([torch.cat(expected_hids), torch.cat(expected_wids)], dim=-1) + + assert torch.equal(pids, expected) + # Must match the device-side `pids.max() + 1` it replaces. + assert max_grid_size == int(expected.max()) + 1 + + +def test_packed_2d_position_ids_single_image_avoids_cat(): + grid = (1, 3, 5) + pids, max_grid_size = build_packed_2d_position_ids([grid], torch.device("cpu")) + + image_pids = torch.arange(15) + expected = torch.stack([image_pids // 5, image_pids % 5], dim=-1) + + assert torch.equal(pids, expected) + assert max_grid_size == 5 + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-v"])) diff --git a/test/registered/vlm/test_paddleocr_vl_server.py b/test/registered/vlm/test_paddleocr_vl_server.py new file mode 100644 index 000000000..83bfa14fd --- /dev/null +++ b/test/registered/vlm/test_paddleocr_vl_server.py @@ -0,0 +1,117 @@ +"""End-to-end OpenAI-API coverage for PaddleOCR-VL. + +The vision tower encodes a whole batch as one packed tensor, so the test drives +a single-image request plus several concurrent requests with differently sized +images — the shape that makes the scheduler hand several images to one ViT +forward. Bit-exactness of the packing itself is pinned on CPU by +`test/registered/unit/models/test_paddleocr_vl_vision.py`. +""" + +import base64 +import io +import unittest +from concurrent.futures import ThreadPoolExecutor + +import openai +from PIL import Image, ImageDraw, ImageFont + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.vlm_utils import TestOpenAIMLLMServerBase + +register_cuda_ci(est_time=240, stage="base-b", runner_config="1-gpu-large") + + +class TestPaddleOCRVLServer(TestOpenAIMLLMServerBase): + model = "PaddlePaddle/PaddleOCR-VL" + extra_args = [ + "--context-length=8192", + "--mem-fraction-static=0.7", + "--cuda-graph-max-bs-decode=4", + ] + + @staticmethod + def _font(size: int): + for path in ( + "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", + "/usr/share/fonts/truetype/liberation2/LiberationSans-Regular.ttf", + ): + try: + return ImageFont.truetype(path, size=size) + except OSError: + pass + return ImageFont.load_default() + + @classmethod + def _make_ocr_image_url(cls, text: str, size=(640, 360)) -> str: + width, height = size + img = Image.new("RGB", size, "white") + draw = ImageDraw.Draw(img) + draw.rectangle((16, 16, width - 16, height - 16), outline="black", width=4) + font_size = height // 6 + font = cls._font(font_size) + text_width = draw.textbbox((0, 0), text, font=font)[2] + if text_width > width - 96: + font = cls._font((width - 96) * font_size // text_width) + draw.text((48, height // 3), text, fill="black", font=font) + + buffer = io.BytesIO() + img.save(buffer, format="PNG") + encoded = base64.b64encode(buffer.getvalue()).decode("ascii") + return f"data:image/png;base64,{encoded}" + + def _ocr(self, client, image_url: str, max_tokens: int = 64) -> str: + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "OCR:"}, + {"type": "image_url", "image_url": {"url": image_url}}, + ], + }, + ], + temperature=0, + max_tokens=max_tokens, + ) + self.assertEqual(response.choices[0].message.role, "assistant") + self.assertGreater(response.usage.prompt_tokens, 0) + self.assertGreater(response.usage.completion_tokens, 0) + return response.choices[0].message.content + + def test_single_image_ocr(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + text = self._ocr(client, self._make_ocr_image_url("SGLANG 12345")) + + self.assertIsInstance(text, str) + self.assertIn("12345", text) + self.assertIn("sglang", text.lower()) + + def test_concurrent_requests_batch_the_vision_tower(self): + """Different image sizes in flight at once must not bleed across images.""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + cases = [ + ("ALPHA 111", (640, 360)), + ("BRAVO 222", (800, 320)), + ("CHARLIE 333", (512, 512)), + ("DELTA 444", (960, 288)), + ] + urls = [self._make_ocr_image_url(text, size) for text, size in cases] + + with ThreadPoolExecutor(max_workers=len(urls)) as pool: + results = list(pool.map(lambda url: self._ocr(client, url), urls)) + + for (expected_text, _), actual in zip(cases, results): + word, digits = expected_text.split() + self.assertIn(digits, actual, f"{expected_text!r} -> {actual!r}") + self.assertIn( + word.lower(), actual.lower(), f"{expected_text!r} -> {actual!r}" + ) + + +del TestOpenAIMLLMServerBase + + +if __name__ == "__main__": + unittest.main()